270 files changed, 34005 insertions, 12678 deletions

diff --git a/bitbake/.b4-config b/bitbake/.b4-config
new file mode 100644
index 0000000000..047f0b94a4
--- /dev/null
+++ b/bitbake/.b4-config
@@ -0,0 +1,4 @@
+[b4]
+  send-series-to = bitbake-devel@lists.openembedded.org
+  send-auto-cc-cmd = ./contrib/b4-wrapper-bitbake.py send-auto-cc-cmd
+  prep-pre-flight-checks = disable-needs-checking
diff --git a/bitbake/README b/bitbake/README
index 479c376588..e9f4c858ee 100644
--- a/bitbake/README
+++ b/bitbake/README
@@ -7,29 +7,57 @@ One of BitBake's main users, OpenEmbedded, takes this core and builds embedded L
 stacks using a task-oriented approach.
 
 For information about Bitbake, see the OpenEmbedded website:
-    http://www.openembedded.org/
+    https://www.openembedded.org/
 
 Bitbake plain documentation can be found under the doc directory or its integrated
 html version at the Yocto Project website:
-    http://yoctoproject.org/documentation
+    https://docs.yoctoproject.org
+
+Bitbake requires Python version 3.8 or newer.
 
 Contributing
 ------------
 
-Please refer to
-http://www.openembedded.org/wiki/How_to_submit_a_patch_to_OpenEmbedded
-for guidelines on how to submit patches, just note that the latter documentation is intended
-for OpenEmbedded (and its core) not bitbake patches (bitbake-devel@lists.openembedded.org)
-but in general main guidelines apply. Once the commit(s) have been created, the way to send
-the patch is through git-send-email. For example, to send the last commit (HEAD) on current
-branch, type:
+Please refer to our contributor guide here: https://docs.yoctoproject.org/contributor-guide/
+for full details on how to submit changes.
+
+As a quick guide, patches should be sent to bitbake-devel@lists.openembedded.org
+The git command to do that would be:
 
     git send-email -M -1 --to bitbake-devel@lists.openembedded.org
 
+If you're sending a patch related to the BitBake manual, make sure you copy
+the Yocto Project documentation mailing list:
+
+    git send-email -M -1 --to bitbake-devel@lists.openembedded.org --cc docs@lists.yoctoproject.org
+
 Mailing list:
 
-    http://lists.openembedded.org/mailman/listinfo/bitbake-devel
+    https://lists.openembedded.org/g/bitbake-devel
 
 Source code:
 
-    http://git.openembedded.org/bitbake/
+    https://git.openembedded.org/bitbake/
+
+Testing
+-------
+
+Bitbake has a testsuite located in lib/bb/tests/ whichs aim to try and prevent regressions.
+You can run this with "bitbake-selftest". In particular the fetcher is well covered since
+it has so many corner cases. The datastore has many tests too. Testing with the testsuite is
+recommended before submitting patches, particularly to the fetcher and datastore. We also
+appreciate new test cases and may require them for more obscure issues.
+
+To run the tests "zstd" and "git" must be installed.
+
+The assumption is made that this testsuite is run from an initialized OpenEmbedded build
+environment (i.e. `source oe-init-build-env` is used). If this is not the case, run the
+testsuite as follows:
+
+    export PATH=$(pwd)/bin:$PATH
+    bin/bitbake-selftest
+
+The testsuite can alternatively be executed using pytest, e.g. obtained from PyPI (in this
+case, the PATH is configured automatically):
+
+    pytest
diff --git a/bitbake/SECURITY.md b/bitbake/SECURITY.md
new file mode 100644
index 0000000000..7d2ce1f631
--- /dev/null
+++ b/bitbake/SECURITY.md
@@ -0,0 +1,24 @@
+How to Report a Potential Vulnerability?
+========================================
+
+If you would like to report a public issue (for example, one with a released
+CVE number), please report it using the
+[https://bugzilla.yoctoproject.org/enter_bug.cgi?product=Security Security Bugzilla].
+If you have a patch ready, submit it following the same procedure as any other
+patch as described in README.md.
+
+If you are dealing with a not-yet released or urgent issue, please send a
+message to security AT yoctoproject DOT org, including as many details as
+possible: the layer or software module affected, the recipe and its version,
+and any example code, if available.
+
+Branches maintained with security fixes
+---------------------------------------
+
+See [https://wiki.yoctoproject.org/wiki/Stable_Release_and_LTS Stable release and LTS]
+for detailed info regarding the policies and maintenance of Stable branches.
+
+The [https://wiki.yoctoproject.org/wiki/Releases Release page] contains a list of all
+releases of the Yocto Project. Versions in grey are no longer actively maintained with
+security patches, but well-tested patches may still be accepted for them for
+significant issues.
diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake
index bc762bfc15..40b5d895c1 100755
--- a/bitbake/bin/bitbake
+++ b/bitbake/bin/bitbake
@@ -12,6 +12,8 @@
 
 import os
 import sys
+import warnings
+warnings.simplefilter("default")
 
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),
                                 'lib'))
@@ -23,10 +25,9 @@ except RuntimeError as exc:
 from bb import cookerdata
 from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+bb.utils.check_system_locale()
 
-__version__ = "1.49.2"
+__version__ = "2.15.1"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:
diff --git a/bitbake/bin/bitbake-config-build b/bitbake/bin/bitbake-config-build
new file mode 120000
index 0000000000..11e6df80c4
--- /dev/null
+++ b/bitbake/bin/bitbake-config-build
@@ -0,0 +1 @@
+bitbake-layers
\ No newline at end of file
diff --git a/bitbake/bin/bitbake-diffsigs b/bitbake/bin/bitbake-diffsigs
index 19420a2df6..9d6cb8c944 100755
--- a/bitbake/bin/bitbake-diffsigs
+++ b/bitbake/bin/bitbake-diffsigs
@@ -11,6 +11,8 @@
 import os
 import sys
 import warnings
+
+warnings.simplefilter("default")
 import argparse
 import logging
 import pickle
@@ -26,6 +28,7 @@ logger = bb.msg.logger_create(myname)
 
 is_dump = myname == 'bitbake-dumpsig'
 
+
 def find_siginfo(tinfoil, pn, taskname, sigs=None):
     result = None
     tinfoil.set_event_mask(['bb.event.FindSigInfoResult',
@@ -51,6 +54,7 @@ def find_siginfo(tinfoil, pn, taskname, sigs=None):
         sys.exit(2)
     return result
 
+
 def find_siginfo_task(bbhandler, pn, taskname, sig1=None, sig2=None):
     """ Find the most recent signature files for the specified PN/task """
 
@@ -59,22 +63,26 @@ def find_siginfo_task(bbhandler, pn, taskname, sig1=None, sig2=None):
 
     if sig1 and sig2:
         sigfiles = find_siginfo(bbhandler, pn, taskname, [sig1, sig2])
-        if len(sigfiles) == 0:
+        if not sigfiles:
             logger.error('No sigdata files found matching %s %s matching either %s or %s' % (pn, taskname, sig1, sig2))
             sys.exit(1)
-        elif not sig1 in sigfiles:
+        elif sig1 not in sigfiles:
             logger.error('No sigdata files found matching %s %s with signature %s' % (pn, taskname, sig1))
             sys.exit(1)
-        elif not sig2 in sigfiles:
+        elif sig2 not in sigfiles:
             logger.error('No sigdata files found matching %s %s with signature %s' % (pn, taskname, sig2))
             sys.exit(1)
-        latestfiles = [sigfiles[sig1], sigfiles[sig2]]
+
+        latestfiles = [sigfiles[sig1]['path'], sigfiles[sig2]['path']]
     else:
-        filedates = find_siginfo(bbhandler, pn, taskname)
-        latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-2:]
-        if not latestfiles:
+        sigfiles = find_siginfo(bbhandler, pn, taskname)
+        latestsigs = sorted(sigfiles.keys(), key=lambda h: sigfiles[h]['time'])[-2:]
+        if not latestsigs:
             logger.error('No sigdata files found matching %s %s' % (pn, taskname))
             sys.exit(1)
+        latestfiles = [sigfiles[latestsigs[0]]['path']]
+        if len(latestsigs) > 1:
+            latestfiles.append(sigfiles[latestsigs[1]]['path'])
 
     return latestfiles
 
@@ -85,14 +93,14 @@ def recursecb(key, hash1, hash2):
     hashfiles = find_siginfo(tinfoil, key, None, hashes)
 
     recout = []
-    if len(hashfiles) == 0:
+    if not hashfiles:
         recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
-    elif not hash1 in hashfiles:
+    elif hash1 not in hashfiles:
         recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash1))
-    elif not hash2 in hashfiles:
+    elif hash2 not in hashfiles:
         recout.append("Unable to find matching sigdata for %s with hash %s" % (key, hash2))
     else:
-        out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb, color=color)
+        out2 = bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb, color=color)
         for change in out2:
             for line in change.splitlines():
                 recout.append('    ' + line)
@@ -109,36 +117,36 @@ parser.add_argument('-D', '--debug',
 
 if is_dump:
     parser.add_argument("-t", "--task",
-            help="find the signature data file for the last run of the specified task",
-            action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
+                        help="find the signature data file for the last run of the specified task",
+                        action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
 
     parser.add_argument("sigdatafile1",
-            help="Signature file to dump. Not used when using -t/--task.",
-            action="store", nargs='?', metavar="sigdatafile")
+                        help="Signature file to dump. Not used when using -t/--task.",
+                        action="store", nargs='?', metavar="sigdatafile")
 else:
     parser.add_argument('-c', '--color',
-            help='Colorize the output (where %(metavar)s is %(choices)s)',
-            choices=['auto', 'always', 'never'], default='auto', metavar='color')
+                        help='Colorize the output (where %(metavar)s is %(choices)s)',
+                        choices=['auto', 'always', 'never'], default='auto', metavar='color')
 
     parser.add_argument('-d', '--dump',
-            help='Dump the last signature data instead of comparing (equivalent to using bitbake-dumpsig)',
-            action='store_true')
+                        help='Dump the last signature data instead of comparing (equivalent to using bitbake-dumpsig)',
+                        action='store_true')
 
     parser.add_argument("-t", "--task",
-            help="find the signature data files for the last two runs of the specified task and compare them",
-            action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
+                        help="find the signature data files for the last two runs of the specified task and compare them",
+                        action="store", dest="taskargs", nargs=2, metavar=('recipename', 'taskname'))
 
     parser.add_argument("-s", "--signature",
-            help="With -t/--task, specify the signatures to look for instead of taking the last two",
-            action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
+                        help="With -t/--task, specify the signatures to look for instead of taking the last two",
+                        action="store", dest="sigargs", nargs=2, metavar=('fromsig', 'tosig'))
 
     parser.add_argument("sigdatafile1",
-            help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
-            action="store", nargs='?')
+                        help="First signature file to compare (or signature file to dump, if second not specified). Not used when using -t/--task.",
+                        action="store", nargs='?')
 
     parser.add_argument("sigdatafile2",
-            help="Second signature file to compare",
-            action="store", nargs='?')
+                        help="Second signature file to compare",
+                        action="store", nargs='?')
 
 options = parser.parse_args()
 if is_dump:
@@ -156,7 +164,8 @@ if options.taskargs:
     with bb.tinfoil.Tinfoil() as tinfoil:
         tinfoil.prepare(config_only=True)
         if not options.dump and options.sigargs:
-            files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0], options.sigargs[1])
+            files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1], options.sigargs[0],
+                                      options.sigargs[1])
         else:
             files = find_siginfo_task(tinfoil, options.taskargs[0], options.taskargs[1])
 
@@ -165,7 +174,8 @@ if options.taskargs:
             output = bb.siggen.dump_sigfile(files[-1])
         else:
             if len(files) < 2:
-                logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (options.taskargs[0], options.taskargs[1]))
+                logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (
+                    options.taskargs[0], options.taskargs[1]))
                 sys.exit(1)
 
             # Recurse into signature comparison
diff --git a/bitbake/bin/bitbake-getvar b/bitbake/bin/bitbake-getvar
new file mode 100755
index 0000000000..378fb13572
--- /dev/null
+++ b/bitbake/bin/bitbake-getvar
@@ -0,0 +1,71 @@
+#! /usr/bin/env python3
+#
+# Copyright (C) 2021 Richard Purdie
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import argparse
+import io
+import os
+import sys
+import warnings
+import logging
+warnings.simplefilter("default")
+
+bindir = os.path.dirname(__file__)
+topdir = os.path.dirname(bindir)
+sys.path[0:0] = [os.path.join(topdir, 'lib')]
+
+import bb.providers
+import bb.tinfoil
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Bitbake Query Variable")
+    parser.add_argument("variable", help="variable name to query")
+    parser.add_argument("-r", "--recipe", help="Recipe name to query", default=None, required=False)
+    parser.add_argument('-u', '--unexpand', help='Do not expand the value (with --value)', action="store_true")
+    parser.add_argument('-f', '--flag', help='Specify a variable flag to query (with --value)', default=None)
+    parser.add_argument('--value', help='Only report the value, no history and no variable name', action="store_true")
+    parser.add_argument('-q', '--quiet', help='Silence bitbake server logging', action="store_true")
+    parser.add_argument('--ignore-undefined', help='Suppress any errors related to undefined variables', action="store_true")
+    args = parser.parse_args()
+
+    if not args.value:
+        if args.unexpand:
+            sys.exit("--unexpand only makes sense with --value")
+
+        if args.flag:
+            sys.exit("--flag only makes sense with --value")
+
+    quiet = args.quiet or args.value
+    if quiet:
+        logger = logging.getLogger("BitBake")
+        logger.setLevel(logging.WARNING)
+
+    with bb.tinfoil.Tinfoil(tracking=True, setup_logging=not quiet) as tinfoil:
+        if args.recipe:
+            tinfoil.prepare(quiet=3 if quiet else 2)
+            try:
+                d = tinfoil.parse_recipe(args.recipe)
+            except bb.providers.NoProvider as e:
+                sys.exit(str(e))
+        else:
+            tinfoil.prepare(quiet=2, config_only=True)
+            # Expand keys and run anonymous functions to get identical result to
+            # "bitbake -e"
+            d = tinfoil.finalizeData()
+
+        value = None
+        if args.flag:
+            value = d.getVarFlag(args.variable, args.flag, expand=not args.unexpand)
+            if value is None and not args.ignore_undefined:
+                sys.exit(f"The flag '{args.flag}' is not defined for variable '{args.variable}'")
+        else:
+            value = d.getVar(args.variable, expand=not args.unexpand)
+            if value is None and not args.ignore_undefined:
+                sys.exit(f"The variable '{args.variable}' is not defined")
+        if args.value:
+            print(str(value if value is not None else ""))
+        else:
+            bb.data.emit_var(args.variable, d=d, all=True)
diff --git a/bitbake/bin/bitbake-hashclient b/bitbake/bin/bitbake-hashclient
index a89290217b..b8755c5797 100755
--- a/bitbake/bin/bitbake-hashclient
+++ b/bitbake/bin/bitbake-hashclient
@@ -13,6 +13,12 @@ import pprint
 import sys
 import threading
 import time
+import warnings
+import netrc
+import json
+import statistics
+import textwrap
+warnings.simplefilter("default")
 
 try:
     import tqdm
@@ -34,18 +40,42 @@ except ImportError:
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib'))
 
 import hashserv
+import bb.asyncrpc
 
 DEFAULT_ADDRESS = 'unix://./hashserve.sock'
 METHOD = 'stress.test.method'
 
+def print_user(u):
+    print(f"Username:    {u['username']}")
+    if "permissions" in u:
+        print("Permissions: " + " ".join(u["permissions"]))
+    if "token" in u:
+        print(f"Token:       {u['token']}")
+
 
 def main():
+    def handle_get(args, client):
+        result = client.get_taskhash(args.method, args.taskhash, all_properties=True)
+        if not result:
+            return 0
+
+        print(json.dumps(result, sort_keys=True, indent=4))
+        return 0
+
+    def handle_get_outhash(args, client):
+        result = client.get_outhash(args.method, args.outhash, args.taskhash)
+        if not result:
+            return 0
+
+        print(json.dumps(result, sort_keys=True, indent=4))
+        return 0
+
     def handle_stats(args, client):
         if args.reset:
             s = client.reset_stats()
         else:
             s = client.get_stats()
-        pprint.pprint(s)
+        print(json.dumps(s, sort_keys=True, indent=4))
         return 0
 
     def handle_stress(args, client):
@@ -53,47 +83,59 @@ def main():
             nonlocal found_hashes
             nonlocal missed_hashes
             nonlocal max_time
+            nonlocal times
 
-            client = hashserv.create_client(args.address)
-
-            for i in range(args.requests):
-                taskhash = hashlib.sha256()
-                taskhash.update(args.taskhash_seed.encode('utf-8'))
-                taskhash.update(str(i).encode('utf-8'))
+            with hashserv.create_client(args.address) as client:
+                for i in range(args.requests):
+                    taskhash = hashlib.sha256()
+                    taskhash.update(args.taskhash_seed.encode('utf-8'))
+                    taskhash.update(str(i).encode('utf-8'))
 
-                start_time = time.perf_counter()
-                l = client.get_unihash(METHOD, taskhash.hexdigest())
-                elapsed = time.perf_counter() - start_time
+                    start_time = time.perf_counter()
+                    l = client.get_unihash(METHOD, taskhash.hexdigest())
+                    elapsed = time.perf_counter() - start_time
 
-                with lock:
-                    if l:
-                        found_hashes += 1
-                    else:
-                        missed_hashes += 1
+                    with lock:
+                        if l:
+                            found_hashes += 1
+                        else:
+                            missed_hashes += 1
 
-                    max_time = max(elapsed, max_time)
-                    pbar.update()
+                        times.append(elapsed)
+                        pbar.update()
 
         max_time = 0
         found_hashes = 0
         missed_hashes = 0
         lock = threading.Lock()
-        total_requests = args.clients * args.requests
+        times = []
         start_time = time.perf_counter()
-        with ProgressBar(total=total_requests) as pbar:
+        with ProgressBar(total=args.clients * args.requests) as pbar:
             threads = [threading.Thread(target=thread_main, args=(pbar, lock), daemon=False) for _ in range(args.clients)]
             for t in threads:
                 t.start()
 
             for t in threads:
                 t.join()
+        total_elapsed = time.perf_counter() - start_time
 
-        elapsed = time.perf_counter() - start_time
         with lock:
-            print("%d requests in %.1fs. %.1f requests per second" % (total_requests, elapsed, total_requests / elapsed))
-            print("Average request time %.8fs" % (elapsed / total_requests))
-            print("Max request time was %.8fs" % max_time)
-            print("Found %d hashes, missed %d" % (found_hashes, missed_hashes))
+            mean = statistics.mean(times)
+            median = statistics.median(times)
+            stddev = statistics.pstdev(times)
+
+            print(f"Number of clients:    {args.clients}")
+            print(f"Requests per client:  {args.requests}")
+            print(f"Number of requests:   {len(times)}")
+            print(f"Total elapsed time:   {total_elapsed:.3f}s")
+            print(f"Total request rate:   {len(times)/total_elapsed:.3f} req/s")
+            print(f"Average request time: {mean:.3f}s")
+            print(f"Median request time:  {median:.3f}s")
+            print(f"Request time std dev: {stddev:.3f}s")
+            print(f"Maximum request time: {max(times):.3f}s")
+            print(f"Minimum request time: {min(times):.3f}s")
+            print(f"Hashes found:         {found_hashes}")
+            print(f"Hashes missed:        {missed_hashes}")
 
         if args.report:
             with ProgressBar(total=args.requests) as pbar:
@@ -111,12 +153,173 @@ def main():
                     with lock:
                         pbar.update()
 
-    parser = argparse.ArgumentParser(description='Hash Equivalence Client')
+    def handle_remove(args, client):
+        where = {k: v for k, v in args.where}
+        if where:
+            result = client.remove(where)
+            print("Removed %d row(s)" % (result["count"]))
+        else:
+            print("No query specified")
+
+    def handle_clean_unused(args, client):
+        result = client.clean_unused(args.max_age)
+        print("Removed %d rows" % (result["count"]))
+        return 0
+
+    def handle_refresh_token(args, client):
+        r = client.refresh_token(args.username)
+        print_user(r)
+
+    def handle_set_user_permissions(args, client):
+        r = client.set_user_perms(args.username, args.permissions)
+        print_user(r)
+
+    def handle_get_user(args, client):
+        r = client.get_user(args.username)
+        print_user(r)
+
+    def handle_get_all_users(args, client):
+        users = client.get_all_users()
+        print("{username:20}| {permissions}".format(username="Username", permissions="Permissions"))
+        print(("-" * 20) + "+" + ("-" * 20))
+        for u in users:
+            print("{username:20}| {permissions}".format(username=u["username"], permissions=" ".join(u["permissions"])))
+
+    def handle_new_user(args, client):
+        r = client.new_user(args.username, args.permissions)
+        print_user(r)
+
+    def handle_delete_user(args, client):
+        r = client.delete_user(args.username)
+        print_user(r)
+
+    def handle_get_db_usage(args, client):
+        usage = client.get_db_usage()
+        print(usage)
+        tables = sorted(usage.keys())
+        print("{name:20}| {rows:20}".format(name="Table name", rows="Rows"))
+        print(("-" * 20) + "+" + ("-" * 20))
+        for t in tables:
+            print("{name:20}| {rows:<20}".format(name=t, rows=usage[t]["rows"]))
+        print()
+
+        total_rows = sum(t["rows"] for t in usage.values())
+        print(f"Total rows: {total_rows}")
+
+    def handle_get_db_query_columns(args, client):
+        columns = client.get_db_query_columns()
+        print("\n".join(sorted(columns)))
+
+    def handle_gc_status(args, client):
+        result = client.gc_status()
+        if not result["mark"]:
+            print("No Garbage collection in progress")
+            return 0
+
+        print("Current Mark: %s" % result["mark"])
+        print("Total hashes to keep: %d" % result["keep"])
+        print("Total hashes to remove: %s" % result["remove"])
+        return 0
+
+    def handle_gc_mark(args, client):
+        where = {k: v for k, v in args.where}
+        result = client.gc_mark(args.mark, where)
+        print("New hashes marked: %d" % result["count"])
+        return 0
+
+    def handle_gc_mark_stream(args, client):
+        stdin = (l.strip() for l in sys.stdin)
+        marked_hashes = 0
+
+        try:
+            result = client.gc_mark_stream(args.mark, stdin)
+            marked_hashes = result["count"]
+        except ConnectionError:
+            logger.warning(
+                "Server doesn't seem to support `gc-mark-stream`. Sending "
+                "hashes sequentially using `gc-mark` API."
+            )
+            for line in stdin:
+                pairs = line.split()
+                condition = dict(zip(pairs[::2], pairs[1::2]))
+                result = client.gc_mark(args.mark, condition)
+                marked_hashes += result["count"]
+
+        print("New hashes marked: %d" % marked_hashes)
+        return 0
+
+    def handle_gc_sweep(args, client):
+        result = client.gc_sweep(args.mark)
+        print("Removed %d rows" % result["count"])
+        return 0
+
+    def handle_unihash_exists(args, client):
+        result = client.unihash_exists(args.unihash)
+        if args.quiet:
+            return 0 if result else 1
+
+        print("true" if result else "false")
+        return 0
+
+    def handle_ping(args, client):
+        times = []
+        for i in range(1, args.count + 1):
+            if not args.quiet:
+                print(f"Ping {i} of {args.count}... ", end="")
+            start_time = time.perf_counter()
+            client.ping()
+            elapsed = time.perf_counter() - start_time
+            times.append(elapsed)
+            if not args.quiet:
+                print(f"{elapsed:.3f}s")
+
+        mean = statistics.mean(times)
+        median = statistics.median(times)
+        std_dev = statistics.pstdev(times)
+
+        if not args.quiet:
+            print("------------------------")
+        print(f"Number of pings:         {len(times)}")
+        print(f"Average round trip time: {mean:.3f}s")
+        print(f"Median round trip time:  {median:.3f}s")
+        print(f"Round trip time std dev: {std_dev:.3f}s")
+        print(f"Min time is:             {min(times):.3f}s")
+        print(f"Max time is:             {max(times):.3f}s")
+        return 0
+
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='Hash Equivalence Client',
+        epilog=textwrap.dedent(
+            """
+            Possible ADDRESS options are:
+                unix://PATH         Connect to UNIX domain socket at PATH
+                ws://HOST[:PORT]    Connect to websocket at HOST:PORT (default port is 80)
+                wss://HOST[:PORT]   Connect to secure websocket at HOST:PORT (default port is 443)
+                HOST:PORT           Connect to TCP server at HOST:PORT
+            """
+        ),
+    )
     parser.add_argument('--address', default=DEFAULT_ADDRESS, help='Server address (default "%(default)s")')
     parser.add_argument('--log', default='WARNING', help='Set logging level')
+    parser.add_argument('--login', '-l', metavar="USERNAME", help="Authenticate as USERNAME")
+    parser.add_argument('--password', '-p', metavar="TOKEN", help="Authenticate using token TOKEN")
+    parser.add_argument('--become', '-b', metavar="USERNAME", help="Impersonate user USERNAME (if allowed) when performing actions")
+    parser.add_argument('--no-netrc', '-n', action="store_false", dest="netrc", help="Do not use .netrc")
 
     subparsers = parser.add_subparsers()
 
+    get_parser = subparsers.add_parser('get', help="Get the unihash for a taskhash")
+    get_parser.add_argument("method", help="Method to query")
+    get_parser.add_argument("taskhash", help="Task hash to query")
+    get_parser.set_defaults(func=handle_get)
+
+    get_outhash_parser = subparsers.add_parser('get-outhash', help="Get output hash information")
+    get_outhash_parser.add_argument("method", help="Method to query")
+    get_outhash_parser.add_argument("outhash", help="Output hash to query")
+    get_outhash_parser.add_argument("taskhash", help="Task hash to query")
+    get_outhash_parser.set_defaults(func=handle_get_outhash)
+
     stats_parser = subparsers.add_parser('stats', help='Show server stats')
     stats_parser.add_argument('--reset', action='store_true',
                               help='Reset server stats')
@@ -135,6 +338,79 @@ def main():
                                help='Include string in outhash')
     stress_parser.set_defaults(func=handle_stress)
 
+    remove_parser = subparsers.add_parser('remove', help="Remove hash entries")
+    remove_parser.add_argument("--where", "-w", metavar="KEY VALUE", nargs=2, action="append", default=[],
+                               help="Remove entries from table where KEY == VALUE")
+    remove_parser.set_defaults(func=handle_remove)
+
+    clean_unused_parser = subparsers.add_parser('clean-unused', help="Remove unused database entries")
+    clean_unused_parser.add_argument("max_age", metavar="SECONDS", type=int, help="Remove unused entries older than SECONDS old")
+    clean_unused_parser.set_defaults(func=handle_clean_unused)
+
+    refresh_token_parser = subparsers.add_parser('refresh-token', help="Refresh auth token")
+    refresh_token_parser.add_argument("--username", "-u", help="Refresh the token for another user (if authorized)")
+    refresh_token_parser.set_defaults(func=handle_refresh_token)
+
+    set_user_perms_parser = subparsers.add_parser('set-user-perms', help="Set new permissions for user")
+    set_user_perms_parser.add_argument("--username", "-u", help="Username", required=True)
+    set_user_perms_parser.add_argument("permissions", metavar="PERM", nargs="*", default=[], help="New permissions")
+    set_user_perms_parser.set_defaults(func=handle_set_user_permissions)
+
+    get_user_parser = subparsers.add_parser('get-user', help="Get user")
+    get_user_parser.add_argument("--username", "-u", help="Username")
+    get_user_parser.set_defaults(func=handle_get_user)
+
+    get_all_users_parser = subparsers.add_parser('get-all-users', help="List all users")
+    get_all_users_parser.set_defaults(func=handle_get_all_users)
+
+    new_user_parser = subparsers.add_parser('new-user', help="Create new user")
+    new_user_parser.add_argument("--username", "-u", help="Username", required=True)
+    new_user_parser.add_argument("permissions", metavar="PERM", nargs="*", default=[], help="New permissions")
+    new_user_parser.set_defaults(func=handle_new_user)
+
+    delete_user_parser = subparsers.add_parser('delete-user', help="Delete user")
+    delete_user_parser.add_argument("--username", "-u", help="Username", required=True)
+    delete_user_parser.set_defaults(func=handle_delete_user)
+
+    db_usage_parser = subparsers.add_parser('get-db-usage', help="Database Usage")
+    db_usage_parser.set_defaults(func=handle_get_db_usage)
+
+    db_query_columns_parser = subparsers.add_parser('get-db-query-columns', help="Show columns that can be used in database queries")
+    db_query_columns_parser.set_defaults(func=handle_get_db_query_columns)
+
+    gc_status_parser = subparsers.add_parser("gc-status", help="Show garbage collection status")
+    gc_status_parser.set_defaults(func=handle_gc_status)
+
+    gc_mark_parser = subparsers.add_parser('gc-mark', help="Mark hashes to be kept for garbage collection")
+    gc_mark_parser.add_argument("mark", help="Mark for this garbage collection operation")
+    gc_mark_parser.add_argument("--where", "-w", metavar="KEY VALUE", nargs=2, action="append", default=[],
+                             help="Keep entries in table where KEY == VALUE")
+    gc_mark_parser.set_defaults(func=handle_gc_mark)
+
+    gc_mark_parser_stream = subparsers.add_parser(
+        'gc-mark-stream',
+        help=(
+            "Mark multiple hashes to be retained for garbage collection. Input should be provided via stdin, "
+            "with each line formatted as key-value pairs separated by spaces, for example 'column1 foo column2 bar'."
+        )
+    )
+    gc_mark_parser_stream.add_argument("mark", help="Mark for this garbage collection operation")
+    gc_mark_parser_stream.set_defaults(func=handle_gc_mark_stream)
+
+    gc_sweep_parser = subparsers.add_parser('gc-sweep', help="Perform garbage collection and delete any entries that are not marked")
+    gc_sweep_parser.add_argument("mark", help="Mark for this garbage collection operation")
+    gc_sweep_parser.set_defaults(func=handle_gc_sweep)
+
+    unihash_exists_parser = subparsers.add_parser('unihash-exists', help="Check if a unihash is known to the server")
+    unihash_exists_parser.add_argument("--quiet", action="store_true", help="Don't print status. Instead, exit with 0 if unihash exists and 1 if it does not")
+    unihash_exists_parser.add_argument("unihash", help="Unihash to check")
+    unihash_exists_parser.set_defaults(func=handle_unihash_exists)
+
+    ping_parser = subparsers.add_parser('ping', help="Ping server")
+    ping_parser.add_argument("-n", "--count", type=int, help="Number of pings. Default is %(default)s", default=10)
+    ping_parser.add_argument("-q", "--quiet", action="store_true", help="Don't print each ping; only print results")
+    ping_parser.set_defaults(func=handle_ping)
+
     args = parser.parse_args()
 
     logger = logging.getLogger('hashserv')
@@ -148,11 +424,30 @@ def main():
     console.setLevel(level)
     logger.addHandler(console)
 
+    login = args.login
+    password = args.password
+
+    if login is None and args.netrc:
+        try:
+            n = netrc.netrc()
+            auth = n.authenticators(args.address)
+            if auth is not None:
+                login, _, password = auth
+        except FileNotFoundError:
+            pass
+        except netrc.NetrcParseError as e:
+            sys.stderr.write(f"Error parsing {e.filename}:{e.lineno}: {e.msg}\n")
+
     func = getattr(args, 'func', None)
     if func:
-        client = hashserv.create_client(args.address)
-
-        return func(args, client)
+        try:
+            with hashserv.create_client(args.address, login, password) as client:
+                if args.become:
+                    client.become_user(args.become)
+                return func(args, client)
+        except bb.asyncrpc.InvokeError as e:
+            print(f"ERROR: {e}")
+            return 1
 
     return 0
 
diff --git a/bitbake/bin/bitbake-hashserv b/bitbake/bin/bitbake-hashserv
index 153f65a378..01503736b9 100755
--- a/bitbake/bin/bitbake-hashserv
+++ b/bitbake/bin/bitbake-hashserv
@@ -10,55 +10,170 @@ import sys
 import logging
 import argparse
 import sqlite3
+import warnings
 
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib'))
+warnings.simplefilter("default")
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib"))
 
 import hashserv
+from hashserv.server import DEFAULT_ANON_PERMS
 
 VERSION = "1.0.0"
 
-DEFAULT_BIND = 'unix://./hashserve.sock'
+DEFAULT_BIND = "unix://./hashserve.sock"
 
 
 def main():
-    parser = argparse.ArgumentParser(description='Hash Equivalence Reference Server. Version=%s' % VERSION,
-                                     epilog='''The bind address is the path to a unix domain socket if it is
-                                               prefixed with "unix://". Otherwise, it is an IP address
-                                               and port in form ADDRESS:PORT. To bind to all addresses, leave
-                                               the ADDRESS empty, e.g. "--bind :8686". To bind to a specific
-                                               IPv6 address, enclose the address in "[]", e.g.
-                                               "--bind [::1]:8686"'''
-                                     )
-
-    parser.add_argument('-b', '--bind', default=DEFAULT_BIND, help='Bind address (default "%(default)s")')
-    parser.add_argument('-d', '--database', default='./hashserv.db', help='Database file (default "%(default)s")')
-    parser.add_argument('-l', '--log', default='WARNING', help='Set logging level')
-    parser.add_argument('-u', '--upstream', help='Upstream hashserv to pull hashes from')
-    parser.add_argument('-r', '--read-only', action='store_true', help='Disallow write operations from clients')
+    parser = argparse.ArgumentParser(
+        description="Hash Equivalence Reference Server. Version=%s" % VERSION,
+        formatter_class=argparse.RawTextHelpFormatter,
+        epilog="""
+The bind address may take one of the following formats:
+    unix://PATH         - Bind to unix domain socket at PATH
+    ws://ADDRESS:PORT   - Bind to websocket on ADDRESS:PORT
+    ADDRESS:PORT        - Bind to raw TCP socket on ADDRESS:PORT
+
+To bind to all addresses, leave the ADDRESS empty, e.g. "--bind :8686" or
+"--bind ws://:8686". To bind to a specific IPv6 address, enclose the address in
+"[]", e.g. "--bind [::1]:8686" or "--bind ws://[::1]:8686"
+
+Note that the default Anonymous permissions are designed to not break existing
+server instances when upgrading, but are not particularly secure defaults. If
+you want to use authentication, it is recommended that you use "--anon-perms
+@read" to only give anonymous users read access, or "--anon-perms @none" to
+give un-authenticated users no access at all.
+
+Setting "--anon-perms @all" or "--anon-perms @user-admin" is not allowed, since
+this would allow anonymous users to manage all users accounts, which is a bad
+idea.
+
+If you are using user authentication, you should run your server in websockets
+mode with an SSL terminating load balancer in front of it (as this server does
+not implement SSL). Otherwise all usernames and passwords will be transmitted
+in the clear. When configured this way, clients can connect using a secure
+websocket, as in "wss://SERVER:PORT"
+
+The following permissions are supported by the server:
+
+    @none       - No permissions
+    @read       - The ability to read equivalent hashes from the server
+    @report     - The ability to report equivalent hashes to the server
+    @db-admin   - Manage the hash database(s). This includes cleaning the
+                  database, removing hashes, etc.
+    @user-admin - The ability to manage user accounts. This includes, creating
+                  users, deleting users, resetting login tokens, and assigning
+                  permissions.
+    @all        - All possible permissions, including any that may be added
+                  in the future
+        """,
+    )
+
+    parser.add_argument(
+        "-b",
+        "--bind",
+        default=os.environ.get("HASHSERVER_BIND", DEFAULT_BIND),
+        help='Bind address (default $HASHSERVER_BIND, "%(default)s")',
+    )
+    parser.add_argument(
+        "-d",
+        "--database",
+        default=os.environ.get("HASHSERVER_DB", "./hashserv.db"),
+        help='Database file (default $HASHSERVER_DB, "%(default)s")',
+    )
+    parser.add_argument(
+        "-l",
+        "--log",
+        default=os.environ.get("HASHSERVER_LOG_LEVEL", "WARNING"),
+        help='Set logging level (default $HASHSERVER_LOG_LEVEL, "%(default)s")',
+    )
+    parser.add_argument(
+        "-u",
+        "--upstream",
+        default=os.environ.get("HASHSERVER_UPSTREAM", None),
+        help="Upstream hashserv to pull hashes from ($HASHSERVER_UPSTREAM)",
+    )
+    parser.add_argument(
+        "-r",
+        "--read-only",
+        action="store_true",
+        help="Disallow write operations from clients ($HASHSERVER_READ_ONLY)",
+    )
+    parser.add_argument(
+        "--db-username",
+        default=os.environ.get("HASHSERVER_DB_USERNAME", None),
+        help="Database username ($HASHSERVER_DB_USERNAME)",
+    )
+    parser.add_argument(
+        "--db-password",
+        default=os.environ.get("HASHSERVER_DB_PASSWORD", None),
+        help="Database password ($HASHSERVER_DB_PASSWORD)",
+    )
+    parser.add_argument(
+        "--anon-perms",
+        metavar="PERM[,PERM[,...]]",
+        default=os.environ.get("HASHSERVER_ANON_PERMS", ",".join(DEFAULT_ANON_PERMS)),
+        help='Permissions to give anonymous users (default $HASHSERVER_ANON_PERMS, "%(default)s")',
+    )
+    parser.add_argument(
+        "--admin-user",
+        default=os.environ.get("HASHSERVER_ADMIN_USER", None),
+        help="Create default admin user with name ADMIN_USER ($HASHSERVER_ADMIN_USER)",
+    )
+    parser.add_argument(
+        "--admin-password",
+        default=os.environ.get("HASHSERVER_ADMIN_PASSWORD", None),
+        help="Create default admin user with password ADMIN_PASSWORD ($HASHSERVER_ADMIN_PASSWORD)",
+    )
+    parser.add_argument(
+        "--reuseport",
+        action="store_true",
+        help="Enable SO_REUSEPORT, allowing multiple servers to bind to the same port for load balancing",
+    )
 
     args = parser.parse_args()
 
-    logger = logging.getLogger('hashserv')
+    logger = logging.getLogger("hashserv")
 
     level = getattr(logging, args.log.upper(), None)
     if not isinstance(level, int):
-        raise ValueError('Invalid log level: %s' % args.log)
+        raise ValueError(
+            "Invalid log level: %s (Try ERROR/WARNING/INFO/DEBUG)" % args.log
+        )
 
     logger.setLevel(level)
     console = logging.StreamHandler()
     console.setLevel(level)
     logger.addHandler(console)
 
-    server = hashserv.create_server(args.bind, args.database, upstream=args.upstream, read_only=args.read_only)
+    read_only = (os.environ.get("HASHSERVER_READ_ONLY", "0") == "1") or args.read_only
+    if "," in args.anon_perms:
+        anon_perms = args.anon_perms.split(",")
+    else:
+        anon_perms = args.anon_perms.split()
+
+    server = hashserv.create_server(
+        args.bind,
+        args.database,
+        upstream=args.upstream,
+        read_only=read_only,
+        db_username=args.db_username,
+        db_password=args.db_password,
+        anon_perms=anon_perms,
+        admin_username=args.admin_user,
+        admin_password=args.admin_password,
+        reuseport=args.reuseport,
+    )
     server.serve_forever()
     return 0
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     try:
         ret = main()
     except Exception:
         ret = 1
         import traceback
+
         traceback.print_exc()
     sys.exit(ret)
diff --git a/bitbake/bin/bitbake-layers b/bitbake/bin/bitbake-layers
index ff085d6744..341ecbcd97 100755
--- a/bitbake/bin/bitbake-layers
+++ b/bitbake/bin/bitbake-layers
@@ -14,15 +14,18 @@ import logging
 import os
 import sys
 import argparse
+import warnings
+warnings.simplefilter("default")
 
 bindir = os.path.dirname(__file__)
+toolname = os.path.basename(__file__).split(".")[0]
 topdir = os.path.dirname(bindir)
 sys.path[0:0] = [os.path.join(topdir, 'lib')]
 
 import bb.tinfoil
 import bb.msg
 
-logger = bb.msg.logger_create('bitbake-layers', sys.stdout)
+logger = bb.msg.logger_create(toolname, sys.stdout)
 
 def main():
     parser = argparse.ArgumentParser(
@@ -31,7 +34,7 @@ def main():
         add_help=False)
     parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
     parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
-    parser.add_argument('-F', '--force', help='Force add without recipe parse verification', action='store_true')
+    parser.add_argument('-F', '--force', help='Forced execution: can be specified multiple times. -F will force add without recipe parse verification and -FF will additionally force the run withput layer parsing.', action='count', default=0)
     parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR')
 
     global_args, unparsed_args = parser.parse_known_args()
@@ -55,22 +58,27 @@ def main():
                          level=logger.getEffectiveLevel())
 
     plugins = []
-    tinfoil = bb.tinfoil.Tinfoil(tracking=True)
-    tinfoil.logger.setLevel(logger.getEffectiveLevel())
-    try:
-        tinfoil.prepare(True)
-        for path in ([topdir] +
-                    tinfoil.config_data.getVar('BBPATH').split(':')):
-            pluginpath = os.path.join(path, 'lib', 'bblayers')
+    with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
+        tinfoil.logger.setLevel(logger.getEffectiveLevel())
+
+        if global_args.force > 1:
+            bbpaths = []
+        else:
+            tinfoil.prepare(True)
+            bbpaths = tinfoil.config_data.getVar('BBPATH').split(':')
+
+        for path in ([topdir] + bbpaths):
+            pluginbasepath = {"bitbake-layers":'bblayers', 'bitbake-config-build':'bbconfigbuild'}[toolname]
+            pluginpath = os.path.join(path, 'lib', pluginbasepath)
             bb.utils.load_plugins(logger, plugins, pluginpath)
 
         registered = False
         for plugin in plugins:
+            if hasattr(plugin, 'tinfoil_init') and global_args.force <= 1:
+                plugin.tinfoil_init(tinfoil)
             if hasattr(plugin, 'register_commands'):
                 registered = True
                 plugin.register_commands(subparsers)
-            if hasattr(plugin, 'tinfoil_init'):
-                plugin.tinfoil_init(tinfoil)
 
         if not registered:
             logger.error("No commands registered - missing plugins?")
@@ -84,8 +92,6 @@ def main():
             tinfoil.config_data.enableTracking()
 
         return args.func(args)
-    finally:
-        tinfoil.shutdown()
 
 
 if __name__ == "__main__":
diff --git a/bitbake/bin/bitbake-prserv b/bitbake/bin/bitbake-prserv
index 1e9b6cbc1b..3992e84eab 100755
--- a/bitbake/bin/bitbake-prserv
+++ b/bitbake/bin/bitbake-prserv
@@ -1,49 +1,103 @@
 #!/usr/bin/env python3
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
 import os
 import sys,logging
-import optparse
+import argparse
+import warnings
+warnings.simplefilter("default")
 
-sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib'))
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib"))
 
 import prserv
 import prserv.serv
 
-__version__="1.0.0"
+VERSION = "2.0.0"
 
-PRHOST_DEFAULT='0.0.0.0'
+PRHOST_DEFAULT="0.0.0.0"
 PRPORT_DEFAULT=8585
 
+def init_logger(logfile, loglevel):
+    numeric_level = getattr(logging, loglevel.upper(), None)
+    if not isinstance(numeric_level, int):
+        raise ValueError("Invalid log level: %s" % loglevel)
+    FORMAT = "%(asctime)-15s %(message)s"
+    logging.basicConfig(level=numeric_level, filename=logfile, format=FORMAT)
+
 def main():
-    parser = optparse.OptionParser(
-        version="Bitbake PR Service Core version %s, %%prog version %s" % (prserv.__version__, __version__),
-        usage = "%prog < --start | --stop > [options]")
-
-    parser.add_option("-f", "--file", help="database filename(default: prserv.sqlite3)", action="store",
-                      dest="dbfile", type="string", default="prserv.sqlite3")
-    parser.add_option("-l", "--log", help="log filename(default: prserv.log)", action="store",
-                      dest="logfile", type="string", default="prserv.log")
-    parser.add_option("--loglevel", help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG",
-                      action = "store", type="string", dest="loglevel", default = "INFO")
-    parser.add_option("--start", help="start daemon",
-                      action="store_true", dest="start")
-    parser.add_option("--stop", help="stop daemon",
-                      action="store_true", dest="stop")
-    parser.add_option("--host", help="ip address to bind", action="store",
-                      dest="host", type="string", default=PRHOST_DEFAULT)
-    parser.add_option("--port", help="port number(default: 8585)", action="store",
-                      dest="port", type="int", default=PRPORT_DEFAULT)
-
-    options, args = parser.parse_args(sys.argv)
-    prserv.init_logger(os.path.abspath(options.logfile),options.loglevel)
-
-    if options.start:
-        ret=prserv.serv.start_daemon(options.dbfile, options.host, options.port,os.path.abspath(options.logfile))
-    elif options.stop:
-        ret=prserv.serv.stop_daemon(options.host, options.port)
+    parser = argparse.ArgumentParser(
+        description="BitBake PR Server. Version=%s" % VERSION,
+        formatter_class=argparse.RawTextHelpFormatter)
+
+    parser.add_argument(
+        "-f",
+        "--file",
+        default="prserv.sqlite3",
+        help="database filename (default: prserv.sqlite3)",
+    )
+    parser.add_argument(
+        "-l",
+        "--log",
+        default="prserv.log",
+        help="log filename(default: prserv.log)",
+    )
+    parser.add_argument(
+        "--loglevel",
+        default="INFO",
+        help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG",
+    )
+    parser.add_argument(
+        "--start",
+        action="store_true",
+        help="start daemon",
+    )
+    parser.add_argument(
+        "--stop",
+        action="store_true",
+        help="stop daemon",
+    )
+    parser.add_argument(
+        "--host",
+        help="ip address to bind",
+        default=PRHOST_DEFAULT,
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=PRPORT_DEFAULT,
+        help="port number (default: 8585)",
+    )
+    parser.add_argument(
+        "-r",
+        "--read-only",
+        action="store_true",
+        help="open database in read-only mode",
+    )
+    parser.add_argument(
+        "-u",
+        "--upstream",
+        default=os.environ.get("PRSERV_UPSTREAM", None),
+        help="Upstream PR service (host:port)",
+    )
+
+    args = parser.parse_args()
+    init_logger(os.path.abspath(args.log), args.loglevel)
+
+    if args.start:
+        ret=prserv.serv.start_daemon(
+            args.file,
+            args.host,
+            args.port,
+            os.path.abspath(args.log),
+            args.read_only,
+            args.upstream
+        )
+    elif args.stop:
+        ret=prserv.serv.stop_daemon(args.host, args.port)
     else:
         ret=parser.print_help()
     return ret
diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest
index 6c0737416b..1b7a783fdc 100755
--- a/bitbake/bin/bitbake-selftest
+++ b/bitbake/bin/bitbake-selftest
@@ -7,12 +7,15 @@
 
 import os
 import sys, logging
+import warnings
+warnings.simplefilter("default")
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib'))
 
 import unittest
 try:
     import bb
     import hashserv
+    import prserv
     import layerindexlib
 except RuntimeError as exc:
     sys.exit(str(exc))
@@ -25,11 +28,12 @@ tests = ["bb.tests.codeparser",
          "bb.tests.event",
          "bb.tests.fetch",
          "bb.tests.parse",
-         "bb.tests.persist_data",
          "bb.tests.runqueue",
          "bb.tests.siggen",
          "bb.tests.utils",
+         "bb.tests.compression",
          "hashserv.tests",
+         "prserv.tests",
          "layerindexlib.tests.layerindexobj",
          "layerindexlib.tests.restapi",
          "layerindexlib.tests.cooker"]
diff --git a/bitbake/bin/bitbake-server b/bitbake/bin/bitbake-server
index ffbc7894ef..a559109e3f 100755
--- a/bitbake/bin/bitbake-server
+++ b/bitbake/bin/bitbake-server
@@ -8,14 +8,17 @@
 import os
 import sys
 import warnings
+warnings.simplefilter("default")
+warnings.filterwarnings("ignore", category=DeprecationWarning, message=".*use.of.fork.*may.lead.to.deadlocks.in.the.child.*")
 import logging
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+import bb
+
+bb.utils.check_system_locale()
 
 # Users shouldn't be running this code directly
-if len(sys.argv) != 10 or not sys.argv[1].startswith("decafbad"):
+if len(sys.argv) != 11 or not sys.argv[1].startswith("decafbad"):
     print("bitbake-server is meant for internal execution by bitbake itself, please don't use it standalone.")
     sys.exit(1)
 
@@ -26,20 +29,19 @@ readypipeinfd = int(sys.argv[3])
 logfile = sys.argv[4]
 lockname = sys.argv[5]
 sockname = sys.argv[6]
-timeout = sys.argv[7]
-xmlrpcinterface = (sys.argv[8], int(sys.argv[9]))
+timeout = float(sys.argv[7])
+profile = bool(int(sys.argv[8]))
+xmlrpcinterface = (sys.argv[9], int(sys.argv[10]))
 if xmlrpcinterface[0] == "None":
     xmlrpcinterface = (None, xmlrpcinterface[1])
-if timeout == "None":
-    timeout = None
 
 # Replace standard fds with our own
 with open('/dev/null', 'r') as si:
     os.dup2(si.fileno(), sys.stdin.fileno())
 
-so = open(logfile, 'a+')
-os.dup2(so.fileno(), sys.stdout.fileno())
-os.dup2(so.fileno(), sys.stderr.fileno())
+with open(logfile, 'a+') as so:
+    os.dup2(so.fileno(), sys.stdout.fileno())
+    os.dup2(so.fileno(), sys.stderr.fileno())
 
 # Have stdout and stderr be the same so log output matches chronologically
 # and there aren't two seperate buffers
@@ -50,5 +52,5 @@ logger = logging.getLogger("BitBake")
 handler = bb.event.LogHandler()
 logger.addHandler(handler)
 
-bb.server.process.execServer(lockfd, readypipeinfd, lockname, sockname, timeout, xmlrpcinterface)
+bb.server.process.execServer(lockfd, readypipeinfd, lockname, sockname, timeout, xmlrpcinterface, profile)
 
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker
index 7765b9368b..d2b146a6a9 100755
--- a/bitbake/bin/bitbake-worker
+++ b/bitbake/bin/bitbake-worker
@@ -1,11 +1,15 @@
 #!/usr/bin/env python3
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
 import os
 import sys
 import warnings
+warnings.simplefilter("default")
+warnings.filterwarnings("ignore", category=DeprecationWarning, message=".*use.of.fork.*may.lead.to.deadlocks.in.the.child.*")
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 from bb import fetch2
 import logging
@@ -16,11 +20,17 @@ import signal
 import pickle
 import traceback
 import queue
+import shlex
+import subprocess
+import fcntl
 from multiprocessing import Lock
 from threading import Thread
 
-if sys.getfilesystemencoding() != "utf-8":
-    sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\nPython can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+# Remove when we have a minimum of python 3.10
+if not hasattr(fcntl, 'F_SETPIPE_SZ'):
+    fcntl.F_SETPIPE_SZ = 1031
+
+bb.utils.check_system_locale()
 
 # Users shouldn't be running this code directly
 if len(sys.argv) != 2 or not sys.argv[1].startswith("decafbad"):
@@ -40,7 +50,6 @@ if sys.argv[1].startswith("decafbadbad"):
 # updates to log files for use with tail
 try:
     if sys.stdout.name == '<stdout>':
-        import fcntl
         fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
         fl |= os.O_SYNC 
         fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
@@ -52,6 +61,12 @@ logger = logging.getLogger("BitBake")
 
 worker_pipe = sys.stdout.fileno()
 bb.utils.nonblockingfd(worker_pipe)
+# Try to make the pipe buffers larger as it is much more efficient. If we can't
+# e.g. out of buffer space (/proc/sys/fs/pipe-user-pages-soft) then just pass over.
+try:
+    fcntl.fcntl(worker_pipe, fcntl.F_SETPIPE_SZ, 512 * 1024)
+except:
+    pass
 # Need to guard against multiprocessing being used in child processes
 # and multiple processes trying to write to the parent at the same time
 worker_pipe_lock = None
@@ -87,21 +102,21 @@ def worker_fire_prepickled(event):
 worker_thread_exit = False
 
 def worker_flush(worker_queue):
-    worker_queue_int = b""
+    worker_queue_int = bytearray()
     global worker_pipe, worker_thread_exit
 
     while True:
         try:
-            worker_queue_int = worker_queue_int + worker_queue.get(True, 1)
+            worker_queue_int.extend(worker_queue.get(True, 1))
         except queue.Empty:
             pass
         while (worker_queue_int or not worker_queue.empty()):
             try:
                 (_, ready, _) = select.select([], [worker_pipe], [], 1)
                 if not worker_queue.empty():
-                    worker_queue_int = worker_queue_int + worker_queue.get()
+                    worker_queue_int.extend(worker_queue.get())
                 written = os.write(worker_pipe, worker_queue_int)
-                worker_queue_int = worker_queue_int[written:]
+                del worker_queue_int[0:written]
             except (IOError, OSError) as e:
                 if e.errno != errno.EAGAIN and e.errno != errno.EPIPE:
                     raise
@@ -117,11 +132,10 @@ def worker_child_fire(event, d):
 
     data = b"<event>" + pickle.dumps(event) + b"</event>"
     try:
-        worker_pipe_lock.acquire()
-        while(len(data)):
-            written = worker_pipe.write(data)
-            data = data[written:]
-        worker_pipe_lock.release()
+        with bb.utils.lock_timeout(worker_pipe_lock):
+            while(len(data)):
+                written = worker_pipe.write(data)
+                data = data[written:]
     except IOError:
         sigterm_handler(None, None)
         raise
@@ -140,44 +154,56 @@ def sigterm_handler(signum, frame):
     os.killpg(0, signal.SIGTERM)
     sys.exit()
 
-def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, extraconfigdata, quieterrors=False, dry_run_exec=False):
+def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
+
+    fn = runtask['fn']
+    task = runtask['task']
+    taskname = runtask['taskname']
+    taskhash = runtask['taskhash']
+    unihash = runtask['unihash']
+    appends = runtask['appends']
+    layername = runtask['layername']
+    taskdepdata = runtask['taskdepdata']
+    quieterrors = runtask['quieterrors']
     # We need to setup the environment BEFORE the fork, since
     # a fork() or exec*() activates PSEUDO...
 
     envbackup = {}
+    fakeroot = False
     fakeenv = {}
     umask = None
 
-    taskdep = workerdata["taskdeps"][fn]
+    uid = os.getuid()
+    gid = os.getgid()
+
+    taskdep = runtask['taskdep']
     if 'umask' in taskdep and taskname in taskdep['umask']:
         umask = taskdep['umask'][taskname]
     elif workerdata["umask"]:
         umask = workerdata["umask"]
     if umask:
-        # umask might come in as a number or text string..
-        try:
-             umask = int(umask, 8)
-        except TypeError:
-             pass
+        # Convert to a python numeric value as it could be a string
+        umask = bb.utils.to_filemode(umask)
 
-    dry_run = cfg.dry_run or dry_run_exec
+    dry_run = cfg.dry_run or runtask['dry_run']
 
     # We can't use the fakeroot environment in a dry run as it possibly hasn't been built
     if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not dry_run:
-        envvars = (workerdata["fakerootenv"][fn] or "").split()
-        for key, value in (var.split('=') for var in envvars):
+        fakeroot = True
+        envvars = (runtask['fakerootenv'] or "").split()
+        for key, value in (var.split('=',1) for var in envvars):
             envbackup[key] = os.environ.get(key)
             os.environ[key] = value
             fakeenv[key] = value
 
-        fakedirs = (workerdata["fakerootdirs"][fn] or "").split()
+        fakedirs = (runtask['fakerootdirs'] or "").split()
         for p in fakedirs:
             bb.utils.mkdirhier(p)
         logger.debug2('Running %s:%s under fakeroot, fakedirs: %s' %
                         (fn, taskname, ', '.join(fakedirs)))
     else:
-        envvars = (workerdata["fakerootnoenv"][fn] or "").split()
-        for key, value in (var.split('=') for var in envvars):
+        envvars = (runtask['fakerootnoenv'] or "").split()
+        for key, value in (var.split('=',1) for var in envvars):
             envbackup[key] = os.environ.get(key)
             os.environ[key] = value
             fakeenv[key] = value
@@ -219,19 +245,21 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
             # Let SIGHUP exit as SIGTERM
             signal.signal(signal.SIGHUP, sigterm_handler)
 
-            # No stdin
-            newsi = os.open(os.devnull, os.O_RDWR)
-            os.dup2(newsi, sys.stdin.fileno())
+            # No stdin & stdout
+            # stdout is used as a status report channel and must not be used by child processes.
+            dumbio = os.open(os.devnull, os.O_RDWR)
+            os.dup2(dumbio, sys.stdin.fileno())
+            os.dup2(dumbio, sys.stdout.fileno())
 
-            if umask:
+            if umask is not None:
                 os.umask(umask)
 
             try:
-                bb_cache = bb.cache.NoCache(databuilder)
                 (realfn, virtual, mc) = bb.cache.virtualfn2realfn(fn)
                 the_data = databuilder.mcdata[mc]
                 the_data.setVar("BB_WORKERCONTEXT", "1")
                 the_data.setVar("BB_TASKDEPDATA", taskdepdata)
+                the_data.setVar('BB_CURRENTTASK', taskname.replace("do_", ""))
                 if cfg.limited_deps:
                     the_data.setVar("BB_LIMITEDDEPS", "1")
                 the_data.setVar("BUILDNAME", workerdata["buildname"])
@@ -245,12 +273,20 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
                     bb.parse.siggen.set_taskhashes(workerdata["newhashes"])
                 ret = 0
 
-                the_data = bb_cache.loadDataFull(fn, appends)
+                the_data = databuilder.parseRecipe(fn, appends, layername)
                 the_data.setVar('BB_TASKHASH', taskhash)
                 the_data.setVar('BB_UNIHASH', unihash)
+                bb.parse.siggen.setup_datacache_from_datastore(fn, the_data)
 
                 bb.utils.set_process_name("%s:%s" % (the_data.getVar("PN"), taskname.replace("do_", "")))
 
+                if not bb.utils.to_boolean(the_data.getVarFlag(taskname, 'network')):
+                    if bb.utils.is_local_uid(uid):
+                        logger.debug("Attempting to disable network for %s" % taskname)
+                        bb.utils.disable_network(uid, gid)
+                    else:
+                        logger.debug("Skipping disable network for %s since %s is not a local uid." % (taskname, uid))
+
                 # exported_vars() returns a generator which *cannot* be passed to os.environ.update() 
                 # successfully. We also need to unset anything from the environment which shouldn't be there 
                 exports = bb.data.exported_vars(the_data)
@@ -279,10 +315,20 @@ def fork_off_task(cfg, data, databuilder, workerdata, fn, task, taskname, taskha
                 if not quieterrors:
                     logger.critical(traceback.format_exc())
                 os._exit(1)
+
+            sys.stdout.flush()
+            sys.stderr.flush()
+
             try:
                 if dry_run:
                     return 0
-                return bb.build.exec_task(fn, taskname, the_data, cfg.profile)
+                try:
+                    ret = bb.build.exec_task(fn, taskname, the_data, cfg.profile)
+                finally:
+                    if fakeroot:
+                        fakerootcmd = shlex.split(the_data.getVar("FAKEROOTCMD"))
+                        subprocess.run(fakerootcmd + ['-S'], check=True, stdout=subprocess.PIPE)
+                return ret
             except:
                 os._exit(1)
         if not profiling:
@@ -314,12 +360,12 @@ class runQueueWorkerPipe():
         if pipeout:
             pipeout.close()
         bb.utils.nonblockingfd(self.input)
-        self.queue = b""
+        self.queue = bytearray()
 
     def read(self):
         start = len(self.queue)
         try:
-            self.queue = self.queue + (self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(512*1024) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise
@@ -347,7 +393,7 @@ class BitbakeWorker(object):
     def __init__(self, din):
         self.input = din
         bb.utils.nonblockingfd(self.input)
-        self.queue = b""
+        self.queue = bytearray()
         self.cookercfg = None
         self.databuilder = None
         self.data = None
@@ -381,7 +427,7 @@ class BitbakeWorker(object):
                     if len(r) == 0:
                         # EOF on pipe, server must have terminated
                         self.sigterm_exception(signal.SIGTERM, None)
-                    self.queue = self.queue + r
+                    self.queue.extend(r)
                 except (OSError, IOError):
                     pass
             if len(self.queue):
@@ -401,19 +447,35 @@ class BitbakeWorker(object):
                 while self.process_waitpid():
                     continue
 
-
     def handle_item(self, item, func):
-        if self.queue.startswith(b"<" + item + b">"):
-            index = self.queue.find(b"</" + item + b">")
-            while index != -1:
-                func(self.queue[(len(item) + 2):index])
-                self.queue = self.queue[(index + len(item) + 3):]
-                index = self.queue.find(b"</" + item + b">")
+        opening_tag = b"<" + item + b">"
+        if not self.queue.startswith(opening_tag):
+            return
+
+        tag_len = len(opening_tag)
+        if len(self.queue) < tag_len + 4:
+            # we need to receive more data
+            return
+        header = self.queue[tag_len:tag_len + 4]
+        payload_len = int.from_bytes(header, 'big')
+        # closing tag has length (tag_len + 1)
+        if len(self.queue) < tag_len * 2 + 1 + payload_len:
+            # we need to receive more data
+            return
+
+        index = self.queue.find(b"</" + item + b">")
+        if index != -1:
+            try:
+                func(self.queue[(tag_len + 4):index])
+            except pickle.UnpicklingError:
+                workerlog_write("Unable to unpickle data: %s\n" % ":".join("{:02x}".format(c) for c in self.queue))
+                raise
+            self.queue = self.queue[(index + len(b"</") + len(item) + len(b">")):]
 
     def handle_cookercfg(self, data):
         self.cookercfg = pickle.loads(data)
         self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True)
-        self.databuilder.parseBaseConfiguration()
+        self.databuilder.parseBaseConfiguration(worker=True)
         self.data = self.databuilder.data
 
     def handle_extraconfigdata(self, data):
@@ -428,6 +490,7 @@ class BitbakeWorker(object):
         for mc in self.databuilder.mcdata:
             self.databuilder.mcdata[mc].setVar("PRSERV_HOST", self.workerdata["prhost"])
             self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.workerdata["hashservaddr"])
+            self.databuilder.mcdata[mc].setVar("__bbclasstype", "recipe")
 
     def handle_newtaskhashes(self, data):
         self.workerdata["newhashes"] = pickle.loads(data)
@@ -445,11 +508,15 @@ class BitbakeWorker(object):
         sys.exit(0)
 
     def handle_runtask(self, data):
-        fn, task, taskname, taskhash, unihash, quieterrors, appends, taskdepdata, dry_run_exec = pickle.loads(data)
-        workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
+        runtask = pickle.loads(data)
+
+        fn = runtask['fn']
+        task = runtask['task']
+        taskname = runtask['taskname']
 
-        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, fn, task, taskname, taskhash, unihash, appends, taskdepdata, self.extraconfigdata, quieterrors, dry_run_exec)
+        workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname))
 
+        pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.databuilder, self.workerdata, self.extraconfigdata, runtask)
         self.build_pids[pid] = task
         self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout)
 
@@ -513,9 +580,11 @@ except BaseException as e:
         import traceback
         sys.stderr.write(traceback.format_exc())
         sys.stderr.write(str(e))
+finally:
+    worker_thread_exit = True
+    worker_thread.join()
 
-worker_thread_exit = True
-worker_thread.join()
-
-workerlog_write("exitting")
+workerlog_write("exiting")
+if not normalexit:
+    sys.exit(1)
 sys.exit(0)
diff --git a/bitbake/bin/git-make-shallow b/bitbake/bin/git-make-shallow
index 57069f7edf..e6c180b4d6 100755
--- a/bitbake/bin/git-make-shallow
+++ b/bitbake/bin/git-make-shallow
@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -16,19 +18,23 @@ import itertools
 import os
 import subprocess
 import sys
+import warnings
+warnings.simplefilter("default")
 
 version = 1.0
 
 
+git_cmd = ['git', '-c', 'safe.bareRepository=all']
+
 def main():
     if sys.version_info < (3, 4, 0):
         sys.exit('Python 3.4 or greater is required')
 
-    git_dir = check_output(['git', 'rev-parse', '--git-dir']).rstrip()
+    git_dir = check_output(git_cmd + ['rev-parse', '--git-dir']).rstrip()
     shallow_file = os.path.join(git_dir, 'shallow')
     if os.path.exists(shallow_file):
         try:
-            check_output(['git', 'fetch', '--unshallow'])
+            check_output(git_cmd + ['fetch', '--unshallow'])
         except subprocess.CalledProcessError:
             try:
                 os.unlink(shallow_file)
@@ -37,21 +43,21 @@ def main():
                     raise
 
     args = process_args()
-    revs = check_output(['git', 'rev-list'] + args.revisions).splitlines()
+    revs = check_output(git_cmd + ['rev-list'] + args.revisions).splitlines()
 
     make_shallow(shallow_file, args.revisions, args.refs)
 
-    ref_revs = check_output(['git', 'rev-list'] + args.refs).splitlines()
+    ref_revs = check_output(git_cmd + ['rev-list'] + args.refs).splitlines()
     remaining_history = set(revs) & set(ref_revs)
     for rev in remaining_history:
-        if check_output(['git', 'rev-parse', '{}^@'.format(rev)]):
+        if check_output(git_cmd + ['rev-parse', '{}^@'.format(rev)]):
             sys.exit('Error: %s was not made shallow' % rev)
 
     filter_refs(args.refs)
 
     if args.shrink:
         shrink_repo(git_dir)
-        subprocess.check_call(['git', 'fsck', '--unreachable'])
+        subprocess.check_call(git_cmd + ['fsck', '--unreachable'])
 
 
 def process_args():
@@ -68,12 +74,12 @@ def process_args():
     args = parser.parse_args()
 
     if args.refs:
-        args.refs = check_output(['git', 'rev-parse', '--symbolic-full-name'] + args.refs).splitlines()
+        args.refs = check_output(git_cmd + ['rev-parse', '--symbolic-full-name'] + args.refs).splitlines()
     else:
         args.refs = get_all_refs(lambda r, t, tt: t == 'commit' or tt == 'commit')
 
     args.refs = list(filter(lambda r: not r.endswith('/HEAD'), args.refs))
-    args.revisions = check_output(['git', 'rev-parse'] + ['%s^{}' % i for i in args.revisions]).splitlines()
+    args.revisions = check_output(git_cmd + ['rev-parse'] + ['%s^{}' % i for i in args.revisions]).splitlines()
     return args
 
 
@@ -91,7 +97,7 @@ def make_shallow(shallow_file, revisions, refs):
 
 def get_all_refs(ref_filter=None):
     """Return all the existing refs in this repository, optionally filtering the refs."""
-    ref_output = check_output(['git', 'for-each-ref', '--format=%(refname)\t%(objecttype)\t%(*objecttype)'])
+    ref_output = check_output(git_cmd + ['for-each-ref', '--format=%(refname)\t%(objecttype)\t%(*objecttype)'])
     ref_split = [tuple(iter_extend(l.rsplit('\t'), 3)) for l in ref_output.splitlines()]
     if ref_filter:
         ref_split = (e for e in ref_split if ref_filter(*e))
@@ -109,8 +115,8 @@ def filter_refs(refs):
     all_refs = get_all_refs()
     to_remove = set(all_refs) - set(refs)
     if to_remove:
-        check_output(['xargs', '-0', '-n', '1', 'git', 'update-ref', '-d', '--no-deref'],
-                     input=''.join(l + '\0' for l in to_remove))
+        check_output(git_cmd + ['update-ref', '--no-deref', '--stdin', '-z'],
+                     input=''.join('delete ' + l + '\0\0' for l in to_remove))
 
 
 def follow_history_intersections(revisions, refs):
@@ -122,7 +128,7 @@ def follow_history_intersections(revisions, refs):
         if rev in seen:
             continue
 
-        parents = check_output(['git', 'rev-parse', '%s^@' % rev]).splitlines()
+        parents = check_output(git_cmd + ['rev-parse', '%s^@' % rev]).splitlines()
 
         yield rev
         seen.add(rev)
@@ -130,12 +136,12 @@ def follow_history_intersections(revisions, refs):
         if not parents:
             continue
 
-        check_refs = check_output(['git', 'merge-base', '--independent'] + sorted(refs)).splitlines()
+        check_refs = check_output(git_cmd + ['merge-base', '--independent'] + sorted(refs)).splitlines()
         for parent in parents:
             for ref in check_refs:
                 print("Checking %s vs %s" % (parent, ref))
                 try:
-                    merge_base = check_output(['git', 'merge-base', parent, ref]).rstrip()
+                    merge_base = check_output(git_cmd + ['merge-base', parent, ref]).rstrip()
                 except subprocess.CalledProcessError:
                     continue
                 else:
@@ -155,14 +161,14 @@ def iter_except(func, exception, start=None):
 
 def shrink_repo(git_dir):
     """Shrink the newly shallow repository, removing the unreachable objects."""
-    subprocess.check_call(['git', 'reflog', 'expire', '--expire-unreachable=now', '--all'])
-    subprocess.check_call(['git', 'repack', '-ad'])
+    subprocess.check_call(git_cmd + ['reflog', 'expire', '--expire-unreachable=now', '--all'])
+    subprocess.check_call(git_cmd + ['repack', '-ad'])
     try:
         os.unlink(os.path.join(git_dir, 'objects', 'info', 'alternates'))
     except OSError as exc:
         if exc.errno != errno.ENOENT:
             raise
-    subprocess.check_call(['git', 'prune', '--expire', 'now'])
+    subprocess.check_call(git_cmd + ['prune', '--expire', 'now'])
 
 
 if __name__ == '__main__':
diff --git a/bitbake/bin/toaster b/bitbake/bin/toaster
index 6b90ee187e..f002c8c159 100755
--- a/bitbake/bin/toaster
+++ b/bitbake/bin/toaster
@@ -33,7 +33,7 @@ databaseCheck()
     $MANAGE migrate --noinput || retval=1
 
     if [ $retval -eq 1 ]; then
-        echo "Failed migrations, aborting system start" 1>&2
+        echo "Failed migrations, halting system start" 1>&2
         return $retval
     fi
     # Make sure that checksettings can pick up any value for TEMPLATECONF
@@ -41,7 +41,7 @@ databaseCheck()
     $MANAGE checksettings --traceback || retval=1
 
     if [ $retval -eq 1 ]; then
-        printf "\nError while checking settings; aborting\n"
+        printf "\nError while checking settings; exiting\n"
         return $retval
     fi
 
@@ -84,7 +84,7 @@ webserverStartAll()
     echo "Starting webserver..."
 
     $MANAGE runserver --noreload "$ADDR_PORT" \
-           </dev/null >>${BUILDDIR}/toaster_web.log 2>&1 \
+           </dev/null >>${TOASTER_LOGS_DIR}/web.log 2>&1 \
            & echo $! >${BUILDDIR}/.toastermain.pid
 
     sleep 1
@@ -181,6 +181,14 @@ WEBSERVER=1
 export TOASTER_BUILDSERVER=1
 ADDR_PORT="localhost:8000"
 TOASTERDIR=`dirname $BUILDDIR`
+# ${BUILDDIR}/toaster_logs/ became the default location for toaster logs
+# This is needed for implemented django-log-viewer: https://pypi.org/project/django-log-viewer/
+# If the directory does not exist, create it.
+TOASTER_LOGS_DIR="${BUILDDIR}/toaster_logs/"
+if [ ! -d $TOASTER_LOGS_DIR ]
+then
+    mkdir $TOASTER_LOGS_DIR
+fi
 unset CMD
 for param in $*; do
     case $param in
@@ -248,7 +256,7 @@ fi
 # 3) the sqlite db if that is being used.
 # 4) pid's we need to clean up on exit/shutdown
 export TOASTER_DIR=$TOASTERDIR
-export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE TOASTER_DIR"
+export BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS TOASTER_DIR"
 
 # Determine the action. If specified by arguments, fine, if not, toggle it
 if [ "$CMD" = "start" ] ; then
@@ -299,7 +307,7 @@ case $CMD in
         export BITBAKE_UI='toasterui'
         if [ $TOASTER_BUILDSERVER -eq 1 ] ; then
             $MANAGE runbuilds \
-               </dev/null >>${BUILDDIR}/toaster_runbuilds.log 2>&1 \
+               </dev/null >>${TOASTER_LOGS_DIR}/toaster_runbuilds.log 2>&1 \
                & echo $! >${BUILDDIR}/.runbuilds.pid
         else
             echo "Toaster build server not started."
diff --git a/bitbake/bin/toaster-eventreplay b/bitbake/bin/toaster-eventreplay
index 8fa4ab7116..74a319320e 100755
--- a/bitbake/bin/toaster-eventreplay
+++ b/bitbake/bin/toaster-eventreplay
@@ -19,6 +19,8 @@ import sys
 import json
 import pickle
 import codecs
+import warnings
+warnings.simplefilter("default")
 
 from collections import namedtuple
 
@@ -28,79 +30,23 @@ sys.path.insert(0, join(dirname(dirname(abspath(__file__))), 'lib'))
 
 import bb.cooker
 from bb.ui import toasterui
-
-class EventPlayer:
-    """Emulate a connection to a bitbake server."""
-
-    def __init__(self, eventfile, variables):
-        self.eventfile = eventfile
-        self.variables = variables
-        self.eventmask = []
-
-    def waitEvent(self, _timeout):
-        """Read event from the file."""
-        line = self.eventfile.readline().strip()
-        if not line:
-            return
-        try:
-            event_str = json.loads(line)['vars'].encode('utf-8')
-            event = pickle.loads(codecs.decode(event_str, 'base64'))
-            event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
-            if event_name not in self.eventmask:
-                return
-            return event
-        except ValueError as err:
-            print("Failed loading ", line)
-            raise err
-
-    def runCommand(self, command_line):
-        """Emulate running a command on the server."""
-        name = command_line[0]
-
-        if name == "getVariable":
-            var_name = command_line[1]
-            variable = self.variables.get(var_name)
-            if variable:
-                return variable['v'], None
-            return None, "Missing variable %s" % var_name
-
-        elif name == "getAllKeysWithFlags":
-            dump = {}
-            flaglist = command_line[1]
-            for key, val in self.variables.items():
-                try:
-                    if not key.startswith("__"):
-                        dump[key] = {
-                            'v': val['v'],
-                            'history' : val['history'],
-                        }
-                        for flag in flaglist:
-                            dump[key][flag] = val[flag]
-                except Exception as err:
-                    print(err)
-            return (dump, None)
-
-        elif name == 'setEventMask':
-            self.eventmask = command_line[-1]
-            return True, None
-
-        else:
-            raise Exception("Command %s not implemented" % command_line[0])
-
-    def getEventHandle(self):
-        """
-        This method is called by toasterui.
-        The return value is passed to self.runCommand but not used there.
-        """
-        pass
+from bb.ui import eventreplay
 
 def main(argv):
     with open(argv[-1]) as eventfile:
         # load variables from the first line
-        variables = json.loads(eventfile.readline().strip())['allvariables']
-
+        variables = None
+        while line := eventfile.readline().strip():
+            try:
+                variables = json.loads(line)['allvariables']
+                break
+            except (KeyError, json.JSONDecodeError):
+                continue
+        if not variables:
+            sys.exit("Cannot find allvariables entry in event log file %s" % argv[-1])
+        eventfile.seek(0)
         params = namedtuple('ConfigParams', ['observe_only'])(True)
-        player = EventPlayer(eventfile, variables)
+        player = eventreplay.EventPlayer(eventfile, variables)
 
         return toasterui.main(player, player, params)
 
diff --git a/bitbake/contrib/b4-wrapper-bitbake.py b/bitbake/contrib/b4-wrapper-bitbake.py
new file mode 100755
index 0000000000..87dff2c3a7
--- /dev/null
+++ b/bitbake/contrib/b4-wrapper-bitbake.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: MIT
+#
+# This script is to be called by b4:
+# - through b4.send-auto-cc-cmd with "send-auto-cc-cmd" as first argument,
+#
+# When send-auto-cc-cmd is passed:
+#
+#  This returns the list of Cc recipients for a patch.
+#
+# This script takes as stdin a patch.
+
+import subprocess
+import sys
+
+cmd = sys.argv[1]
+if cmd != "send-auto-cc-cmd":
+    sys.exit(-1)
+
+patch = sys.stdin.read()
+
+if subprocess.call(["which", "lsdiff"], stdout=subprocess.DEVNULL) != 0:
+    print("lsdiff missing from host, please install patchutils")
+    sys.exit(-1)
+
+files = subprocess.check_output(["lsdiff", "--strip-match=1", "--strip=1", "--include=doc/*"],
+                                input=patch, text=True)
+if len(files):
+    print("docs@lists.yoctoproject.org")
+else:
+# Handle patches made with --no-prefix
+    files = subprocess.check_output(["lsdiff", "--include=doc/*"],
+                                    input=patch, text=True)
+    if len(files):
+        print("docs@lists.yoctoproject.org")
+
+sys.exit(0)
diff --git a/bitbake/contrib/hashserv/Dockerfile b/bitbake/contrib/hashserv/Dockerfile
index d6fc728f37..aec1f86fc9 100644
--- a/bitbake/contrib/hashserv/Dockerfile
+++ b/bitbake/contrib/hashserv/Dockerfile
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: MIT
 #
 # Copyright (c) 2021 Joshua Watt <JPEWhacker@gmail.com>
-# 
+#
 # Dockerfile to build a bitbake hash equivalence server container
 #
 # From the root of the bitbake repository, run:
@@ -11,9 +11,13 @@
 
 FROM alpine:3.13.1
 
-RUN apk add --no-cache python3
+RUN apk add --no-cache python3 libgcc
 
 COPY bin/bitbake-hashserv /opt/bbhashserv/bin/
 COPY lib/hashserv /opt/bbhashserv/lib/hashserv/
+COPY lib/bb /opt/bbhashserv/lib/bb/
+COPY lib/codegen.py /opt/bbhashserv/lib/codegen.py
+COPY lib/ply /opt/bbhashserv/lib/ply/
+COPY lib/bs4 /opt/bbhashserv/lib/bs4/
 
 ENTRYPOINT ["/opt/bbhashserv/bin/bitbake-hashserv"]
diff --git a/bitbake/contrib/prserv/Dockerfile b/bitbake/contrib/prserv/Dockerfile
new file mode 100644
index 0000000000..9585fe3f07
--- /dev/null
+++ b/bitbake/contrib/prserv/Dockerfile
@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: MIT
+#
+# Copyright (c) 2022 Daniel Gomez <daniel@qtec.com>
+#
+# Dockerfile to build a bitbake PR service container
+#
+# From the root of the bitbake repository, run:
+#
+#   docker build -f contrib/prserv/Dockerfile . -t prserv
+#
+# Running examples:
+#
+# 1. PR Service in RW mode, port 18585:
+#
+# docker run --detach --tty \
+# --env PORT=18585 \
+# --publish 18585:18585 \
+# --volume $PWD:/var/lib/bbprserv \
+# prserv
+#
+# 2. PR Service in RO mode, default port (8585) and custom LOGFILE:
+#
+# docker run --detach --tty \
+# --env DBMODE="--read-only" \
+# --env LOGFILE=/var/lib/bbprserv/prservro.log \
+# --publish 8585:8585 \
+# --volume $PWD:/var/lib/bbprserv \
+# prserv
+#
+
+FROM alpine:3.14.4
+
+RUN apk add --no-cache python3
+
+COPY bin/bitbake-prserv /opt/bbprserv/bin/
+COPY lib/prserv /opt/bbprserv/lib/prserv/
+COPY lib/bb /opt/bbprserv/lib/bb/
+COPY lib/codegen.py /opt/bbprserv/lib/codegen.py
+COPY lib/ply /opt/bbprserv/lib/ply/
+COPY lib/bs4 /opt/bbprserv/lib/bs4/
+
+ENV PATH=$PATH:/opt/bbprserv/bin
+
+RUN mkdir -p /var/lib/bbprserv
+
+ENV DBFILE=/var/lib/bbprserv/prserv.sqlite3 \
+    LOGFILE=/var/lib/bbprserv/prserv.log \
+    LOGLEVEL=debug \
+    HOST=0.0.0.0 \
+    PORT=8585 \
+    DBMODE=""
+
+ENTRYPOINT [ "/bin/sh", "-c", \
+"bitbake-prserv \
+--file=$DBFILE \
+--log=$LOGFILE \
+--loglevel=$LOGLEVEL \
+--start \
+--host=$HOST \
+--port=$PORT \
+$DBMODE \
+&& tail -f $LOGFILE"]
diff --git a/bitbake/contrib/vim/ftdetect/bitbake.vim b/bitbake/contrib/vim/ftdetect/bitbake.vim
index 09fc4dc74c..427ab5b987 100644
--- a/bitbake/contrib/vim/ftdetect/bitbake.vim
+++ b/bitbake/contrib/vim/ftdetect/bitbake.vim
@@ -11,10 +11,18 @@ if &compatible || version < 600 || exists("b:loaded_bitbake_plugin")
 endif
 
 " .bb, .bbappend and .bbclass
-au BufNewFile,BufRead *.{bb,bbappend,bbclass}  set filetype=bitbake
+au BufNewFile,BufRead *.{bb,bbappend,bbclass}  setfiletype bitbake
 
-" .inc
-au BufNewFile,BufRead *.inc             set filetype=bitbake
+" .inc -- meanwhile included upstream
+if !has("patch-9.0.0055")
+    au BufNewFile,BufRead *.inc                call s:BBIncDetect()
+    def s:BBIncDetect()
+        l:lines = getline(1) .. getline(2) .. getline(3)
+        if l:lines =~# '\<\%(require\|inherit\)\>' || lines =~# '[A-Z][A-Za-z0-9_:${}]*\s\+\%(??\|[?:+]\)\?= '
+            set filetype bitbake
+        endif
+    enddef
+endif
 
 " .conf
 au BufNewFile,BufRead *.conf
diff --git a/bitbake/contrib/vim/indent/bitbake.vim b/bitbake/contrib/vim/indent/bitbake.vim
index 1381034098..7ee9d69938 100644
--- a/bitbake/contrib/vim/indent/bitbake.vim
+++ b/bitbake/contrib/vim/indent/bitbake.vim
@@ -40,7 +40,7 @@ set cpo&vim
 
 let s:maxoff = 50       " maximum number of lines to look backwards for ()
 
-function GetPythonIndent(lnum)
+function! GetBBPythonIndent(lnum)
 
   " If this line is explicitly joined: If the previous line was also joined,
   " line it up with that one, otherwise add two 'shiftwidth'
@@ -257,7 +257,7 @@ let b:did_indent = 1
 setlocal indentkeys+=0\"
 
 
-function BitbakeIndent(lnum)
+function! BitbakeIndent(lnum)
     if !has('syntax_items')
         return -1
     endif
@@ -315,7 +315,7 @@ function BitbakeIndent(lnum)
     endif
 
     if index(["bbPyDefRegion", "bbPyFuncRegion"], name) != -1
-        let ret = GetPythonIndent(a:lnum)
+        let ret = GetBBPythonIndent(a:lnum)
         " Should normally always be indented by at least one shiftwidth; but allow
         " return of -1 (defer to autoindent) or -2 (force indent to 0)
         if ret == 0
diff --git a/bitbake/contrib/vim/plugin/newbbappend.vim b/bitbake/contrib/vim/plugin/newbbappend.vim
index e04174cf62..3f65f79cdc 100644
--- a/bitbake/contrib/vim/plugin/newbbappend.vim
+++ b/bitbake/contrib/vim/plugin/newbbappend.vim
@@ -20,7 +20,7 @@ fun! NewBBAppendTemplate()
     set nopaste
 
     " New bbappend template
-    0 put ='FILESEXTRAPATHS_prepend := \"${THISDIR}/${PN}:\"'
+    0 put ='FILESEXTRAPATHS:prepend := \"${THISDIR}/${PN}:\"'
     2
 
     if paste == 1
diff --git a/bitbake/contrib/vim/syntax/bitbake.vim b/bitbake/contrib/vim/syntax/bitbake.vim
index f964621ae5..8f39b8f951 100644
--- a/bitbake/contrib/vim/syntax/bitbake.vim
+++ b/bitbake/contrib/vim/syntax/bitbake.vim
@@ -51,9 +51,9 @@ syn region bbString             matchgroup=bbQuote start=+'+ skip=+\\$+ end=+'+
 syn match bbExport            "^export" nextgroup=bbIdentifier skipwhite
 syn keyword bbExportFlag        export contained nextgroup=bbIdentifier skipwhite
 syn match bbIdentifier          "[a-zA-Z0-9\-_\.\/\+]\+" display contained
-syn match bbVarDeref            "${[a-zA-Z0-9\-_\.\/\+]\+}" contained
+syn match bbVarDeref            "${[a-zA-Z0-9\-_:\.\/\+]\+}" contained
 syn match bbVarEq               "\(:=\|+=\|=+\|\.=\|=\.\|?=\|??=\|=\)" contained nextgroup=bbVarValue
-syn match bbVarDef              "^\(export\s*\)\?\([a-zA-Z0-9\-_\.\/\+]\+\(_[${}a-zA-Z0-9\-_\.\/\+]\+\)\?\)\s*\(:=\|+=\|=+\|\.=\|=\.\|?=\|??=\|=\)\@=" contains=bbExportFlag,bbIdentifier,bbVarDeref nextgroup=bbVarEq
+syn match bbVarDef              "^\(export\s*\)\?\([a-zA-Z0-9\-_\.\/\+][${}a-zA-Z0-9\-_:\.\/\+]*\)\s*\(:=\|+=\|=+\|\.=\|=\.\|?=\|??=\|=\)\@=" contains=bbExportFlag,bbIdentifier,bbOverrideOperator,bbVarDeref nextgroup=bbVarEq
 syn match bbVarValue            ".*$" contained contains=bbString,bbVarDeref,bbVarPyValue
 syn region bbVarPyValue         start=+${@+ skip=+\\$+ end=+}+ contained contains=@python
 
@@ -63,13 +63,14 @@ syn region bbVarFlagFlag        matchgroup=bbArrayBrackets start="\[" end="\]\s*
 
 " Includes and requires
 syn keyword bbInclude           inherit include require contained 
-syn match bbIncludeRest         ".*$" contained contains=bbString,bbVarDeref
+syn match bbIncludeRest         ".*$" contained contains=bbString,bbVarDeref,bbVarPyValue
 syn match bbIncludeLine         "^\(inherit\|include\|require\)\s\+" contains=bbInclude nextgroup=bbIncludeRest
 
 " Add taks and similar
 syn keyword bbStatement         addtask deltask addhandler after before EXPORT_FUNCTIONS contained
-syn match bbStatementRest       ".*$" skipwhite contained contains=bbStatement
-syn match bbStatementLine       "^\(addtask\|deltask\|addhandler\|after\|before\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest
+syn match bbStatementRest       /[^\\]*$/ skipwhite contained contains=bbStatement,bbVarDeref,bbVarPyValue
+syn region bbStatementRestCont  start=/.*\\$/ end=/^[^\\]*$/ contained contains=bbStatement,bbVarDeref,bbVarPyValue,bbContinue keepend
+syn match bbStatementLine       "^\(addtask\|deltask\|addhandler\|after\|before\|EXPORT_FUNCTIONS\)\s\+" contains=bbStatement nextgroup=bbStatementRest,bbStatementRestCont
 
 " OE Important Functions
 syn keyword bbOEFunctions       do_fetch do_unpack do_patch do_configure do_compile do_stage do_install do_package contained
@@ -77,13 +78,15 @@ syn keyword bbOEFunctions       do_fetch do_unpack do_patch do_configure do_comp
 " Generic Functions
 syn match bbFunction            "\h[0-9A-Za-z_\-\.]*" display contained contains=bbOEFunctions
 
+syn keyword bbOverrideOperator  append prepend remove contained
+
 " BitBake shell metadata
 syn include @shell syntax/sh.vim
 if exists("b:current_syntax")
   unlet b:current_syntax
 endif
 syn keyword bbShFakeRootFlag    fakeroot contained
-syn match bbShFuncDef           "^\(fakeroot\s*\)\?\([\.0-9A-Za-z_${}\-\.]\+\)\(python\)\@<!\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbShFuncRegion skipwhite
+syn match bbShFuncDef           "^\(fakeroot\s*\)\?\([\.0-9A-Za-z_:${}\-\.]\+\)\(python\)\@<!\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbFunction,bbOverrideOperator,bbVarDeref,bbDelimiter nextgroup=bbShFuncRegion skipwhite
 syn region bbShFuncRegion       matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" contained contains=@shell
 
 " Python value inside shell functions
@@ -91,7 +94,7 @@ syn region shDeref         start=+${@+ skip=+\\$+ excludenl end=+}+ contained co
 
 " BitBake python metadata
 syn keyword bbPyFlag            python contained
-syn match bbPyFuncDef           "^\(fakeroot\s*\)\?\(python\)\(\s\+[0-9A-Za-z_${}\-\.]\+\)\?\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbPyFlag,bbFunction,bbVarDeref,bbDelimiter nextgroup=bbPyFuncRegion skipwhite
+syn match bbPyFuncDef           "^\(fakeroot\s*\)\?\(python\)\(\s\+[0-9A-Za-z_:${}\-\.]\+\)\?\(\s*()\s*\)\({\)\@=" contains=bbShFakeRootFlag,bbPyFlag,bbFunction,bbOverrideOperator,bbVarDeref,bbDelimiter nextgroup=bbPyFuncRegion skipwhite
 syn region bbPyFuncRegion       matchgroup=bbDelimiter start="{\s*$" end="^}\s*$" contained contains=@python
 
 " BitBake 'def'd python functions
@@ -120,7 +123,9 @@ hi def link bbPyFlag            Type
 hi def link bbPyDef             Statement
 hi def link bbStatement         Statement
 hi def link bbStatementRest     Identifier
+hi def link bbStatementRestCont Identifier
 hi def link bbOEFunctions       Special
 hi def link bbVarPyValue        PreProc
+hi def link bbOverrideOperator  Operator
 
 let b:current_syntax = "bb"
diff --git a/bitbake/doc/Makefile b/bitbake/doc/Makefile
index d40f390e2b..996f01b7d5 100644
--- a/bitbake/doc/Makefile
+++ b/bitbake/doc/Makefile
@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line, and also
 # from the environment for the first two.
-SPHINXOPTS    ?= -j auto
+SPHINXOPTS    ?= -W --keep-going -j auto
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build
diff --git a/bitbake/doc/README b/bitbake/doc/README
index 62595820bd..d4f56afa37 100644
--- a/bitbake/doc/README
+++ b/bitbake/doc/README
@@ -8,12 +8,12 @@ Manual Organization
 
 Folders exist for individual manuals as follows:
 
-* bitbake-user-manual      - The BitBake User Manual 
+* bitbake-user-manual      --- The BitBake User Manual 
 
 Each folder is self-contained regarding content and figures.
 
 If you want to find HTML versions of the BitBake manuals on the web, 
-go to http://www.openembedded.org/wiki/Documentation. 
+go to https://www.openembedded.org/wiki/Documentation.
 
 Sphinx
 ======
@@ -47,8 +47,8 @@ To install all required packages run:
 
 To build the documentation locally, run:
 
- $ cd documentation
- $ make -f Makefile.sphinx html
+ $ cd doc
+ $ make html
 
 The resulting HTML index page will be _build/html/index.html, and you
 can browse your own copy of the locally generated documentation with
diff --git a/bitbake/doc/_templates/footer.html b/bitbake/doc/_templates/footer.html
new file mode 100644
index 0000000000..1398f20d7e
--- /dev/null
+++ b/bitbake/doc/_templates/footer.html
@@ -0,0 +1,9 @@
+<footer>
+  <hr/>
+  <div role="contentinfo">
+    <p>&copy; Copyright {{ copyright }}
+       <br>Last updated on {{ last_updated }} from the <a href="https://git.openembedded.org/bitbake/">bitbake</a> git repository.
+    </p>
+  </div>
+</footer>
+
diff --git a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst
index d74e768f69..d58fbb32ea 100644
--- a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst
+++ b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-execution.rst
@@ -16,7 +16,7 @@ data, or simply return information about the execution environment.
 
 This chapter describes BitBake's execution process from start to finish
 when you use it to create an image. The execution process is launched
-using the following command form: ::
+using the following command form::
 
   $ bitbake target
 
@@ -32,7 +32,7 @@ the BitBake command and its options, see ":ref:`The BitBake Command
    your project's ``local.conf`` configuration file.
 
    A common method to determine this value for your build host is to run
-   the following: ::
+   the following::
 
      $ grep processor /proc/cpuinfo
 
@@ -40,7 +40,7 @@ the BitBake command and its options, see ":ref:`The BitBake Command
    the number of processors, which takes into account hyper-threading.
    Thus, a quad-core build host with hyper-threading most likely shows
    eight processors, which is the value you would then assign to
-   ``BB_NUMBER_THREADS``.
+   :term:`BB_NUMBER_THREADS`.
 
    A possibly simpler solution is that some Linux distributions (e.g.
    Debian and Ubuntu) provide the ``ncpus`` command.
@@ -65,13 +65,13 @@ data itself is of various types:
 
 The ``layer.conf`` files are used to construct key variables such as
 :term:`BBPATH` and :term:`BBFILES`.
-``BBPATH`` is used to search for configuration and class files under the
-``conf`` and ``classes`` directories, respectively. ``BBFILES`` is used
+:term:`BBPATH` is used to search for configuration and class files under the
+``conf`` and ``classes`` directories, respectively. :term:`BBFILES` is used
 to locate both recipe and recipe append files (``.bb`` and
 ``.bbappend``). If there is no ``bblayers.conf`` file, it is assumed the
-user has set the ``BBPATH`` and ``BBFILES`` directly in the environment.
+user has set the :term:`BBPATH` and :term:`BBFILES` directly in the environment.
 
-Next, the ``bitbake.conf`` file is located using the ``BBPATH`` variable
+Next, the ``bitbake.conf`` file is located using the :term:`BBPATH` variable
 that was just constructed. The ``bitbake.conf`` file may also include
 other configuration files using the ``include`` or ``require``
 directives.
@@ -79,8 +79,8 @@ directives.
 Prior to parsing configuration files, BitBake looks at certain
 variables, including:
 
--  :term:`BB_ENV_WHITELIST`
--  :term:`BB_ENV_EXTRAWHITE`
+-  :term:`BB_ENV_PASSTHROUGH`
+-  :term:`BB_ENV_PASSTHROUGH_ADDITIONS`
 -  :term:`BB_PRESERVE_ENV`
 -  :term:`BB_ORIGENV`
 -  :term:`BITBAKE_UI`
@@ -104,7 +104,7 @@ BitBake first searches the current working directory for an optional
 contain a :term:`BBLAYERS` variable that is a
 space-delimited list of 'layer' directories. Recall that if BitBake
 cannot find a ``bblayers.conf`` file, then it is assumed the user has
-set the ``BBPATH`` and ``BBFILES`` variables directly in the
+set the :term:`BBPATH` and :term:`BBFILES` variables directly in the
 environment.
 
 For each directory (layer) in this list, a ``conf/layer.conf`` file is
@@ -114,7 +114,7 @@ files automatically set up :term:`BBPATH` and other
 variables correctly for a given build directory.
 
 BitBake then expects to find the ``conf/bitbake.conf`` file somewhere in
-the user-specified ``BBPATH``. That configuration file generally has
+the user-specified :term:`BBPATH`. That configuration file generally has
 include directives to pull in any other metadata such as files specific
 to the architecture, the machine, the local environment, and so forth.
 
@@ -135,11 +135,11 @@ The ``base.bbclass`` file is always included. Other classes that are
 specified in the configuration using the
 :term:`INHERIT` variable are also included. BitBake
 searches for class files in a ``classes`` subdirectory under the paths
-in ``BBPATH`` in the same way as configuration files.
+in :term:`BBPATH` in the same way as configuration files.
 
 A good way to get an idea of the configuration files and the class files
 used in your execution environment is to run the following BitBake
-command: ::
+command::
 
   $ bitbake -e > mybb.log
 
@@ -155,7 +155,7 @@ execution environment.
    pair of curly braces in a shell function, the closing curly brace
    must not be located at the start of the line without leading spaces.
 
-   Here is an example that causes BitBake to produce a parsing error: ::
+   Here is an example that causes BitBake to produce a parsing error::
 
       fakeroot create_shar() {
          cat << "EOF" > ${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.sh
@@ -184,13 +184,13 @@ Locating and Parsing Recipes
 During the configuration phase, BitBake will have set
 :term:`BBFILES`. BitBake now uses it to construct a
 list of recipes to parse, along with any append files (``.bbappend``) to
-apply. ``BBFILES`` is a space-separated list of available files and
-supports wildcards. An example would be: ::
+apply. :term:`BBFILES` is a space-separated list of available files and
+supports wildcards. An example would be::
 
   BBFILES = "/path/to/bbfiles/*.bb /path/to/appends/*.bbappend"
 
 BitBake parses each
-recipe and append file located with ``BBFILES`` and stores the values of
+recipe and append file located with :term:`BBFILES` and stores the values of
 various variables into the datastore.
 
 .. note::
@@ -201,18 +201,18 @@ For each file, a fresh copy of the base configuration is made, then the
 recipe is parsed line by line. Any inherit statements cause BitBake to
 find and then parse class files (``.bbclass``) using
 :term:`BBPATH` as the search path. Finally, BitBake
-parses in order any append files found in ``BBFILES``.
+parses in order any append files found in :term:`BBFILES`.
 
 One common convention is to use the recipe filename to define pieces of
 metadata. For example, in ``bitbake.conf`` the recipe name and version
 are used to set the variables :term:`PN` and
-:term:`PV`: ::
+:term:`PV`::
 
-   PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
-   PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
+   PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
+   PV = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
 
 In this example, a recipe called "something_1.2.3.bb" would set
-``PN`` to "something" and ``PV`` to "1.2.3".
+:term:`PN` to "something" and :term:`PV` to "1.2.3".
 
 By the time parsing is complete for a recipe, BitBake has a list of
 tasks that the recipe defines and a set of data consisting of keys and
@@ -228,7 +228,7 @@ and then reload it.
 Where possible, subsequent BitBake commands reuse this cache of recipe
 information. The validity of this cache is determined by first computing
 a checksum of the base configuration data (see
-:term:`BB_HASHCONFIG_WHITELIST`) and
+:term:`BB_HASHCONFIG_IGNORE_VARS`) and
 then checking if the checksum matches. If that checksum matches what is
 in the cache and the recipe and class files have not changed, BitBake is
 able to use the cache. BitBake then reloads the cached information about
@@ -238,13 +238,14 @@ Recipe file collections exist to allow the user to have multiple
 repositories of ``.bb`` files that contain the same exact package. For
 example, one could easily use them to make one's own local copy of an
 upstream repository, but with custom modifications that one does not
-want upstream. Here is an example: ::
+want upstream. Here is an example::
 
   BBFILES = "/stuff/openembedded/*/*.bb /stuff/openembedded.modified/*/*.bb"
   BBFILE_COLLECTIONS = "upstream local"
   BBFILE_PATTERN_upstream = "^/stuff/openembedded/"
   BBFILE_PATTERN_local = "^/stuff/openembedded.modified/"
-  BBFILE_PRIORITY_upstream = "5" BBFILE_PRIORITY_local = "10"
+  BBFILE_PRIORITY_upstream = "5"
+  BBFILE_PRIORITY_local = "10"
 
 .. note::
 
@@ -259,21 +260,21 @@ Providers
 
 Assuming BitBake has been instructed to execute a target and that all
 the recipe files have been parsed, BitBake starts to figure out how to
-build the target. BitBake looks through the ``PROVIDES`` list for each
-of the recipes. A ``PROVIDES`` list is the list of names by which the
-recipe can be known. Each recipe's ``PROVIDES`` list is created
+build the target. BitBake looks through the :term:`PROVIDES` list for each
+of the recipes. A :term:`PROVIDES` list is the list of names by which the
+recipe can be known. Each recipe's :term:`PROVIDES` list is created
 implicitly through the recipe's :term:`PN` variable and
 explicitly through the recipe's :term:`PROVIDES`
 variable, which is optional.
 
-When a recipe uses ``PROVIDES``, that recipe's functionality can be
-found under an alternative name or names other than the implicit ``PN``
+When a recipe uses :term:`PROVIDES`, that recipe's functionality can be
+found under an alternative name or names other than the implicit :term:`PN`
 name. As an example, suppose a recipe named ``keyboard_1.0.bb``
-contained the following: ::
+contained the following::
 
   PROVIDES += "fullkeyboard"
 
-The ``PROVIDES``
+The :term:`PROVIDES`
 list for this recipe becomes "keyboard", which is implicit, and
 "fullkeyboard", which is explicit. Consequently, the functionality found
 in ``keyboard_1.0.bb`` can be found under two different names.
@@ -283,14 +284,14 @@ in ``keyboard_1.0.bb`` can be found under two different names.
 Preferences
 ===========
 
-The ``PROVIDES`` list is only part of the solution for figuring out a
+The :term:`PROVIDES` list is only part of the solution for figuring out a
 target's recipes. Because targets might have multiple providers, BitBake
 needs to prioritize providers by determining provider preferences.
 
 A common example in which a target has multiple providers is
-"virtual/kernel", which is on the ``PROVIDES`` list for each kernel
+"virtual/kernel", which is on the :term:`PROVIDES` list for each kernel
 recipe. Each machine often selects the best kernel provider by using a
-line similar to the following in the machine configuration file: ::
+line similar to the following in the machine configuration file::
 
   PREFERRED_PROVIDER_virtual/kernel = "linux-yocto"
 
@@ -308,10 +309,10 @@ specify a particular version. You can influence the order by using the
 :term:`DEFAULT_PREFERENCE` variable.
 
 By default, files have a preference of "0". Setting
-``DEFAULT_PREFERENCE`` to "-1" makes the recipe unlikely to be used
-unless it is explicitly referenced. Setting ``DEFAULT_PREFERENCE`` to
-"1" makes it likely the recipe is used. ``PREFERRED_VERSION`` overrides
-any ``DEFAULT_PREFERENCE`` setting. ``DEFAULT_PREFERENCE`` is often used
+:term:`DEFAULT_PREFERENCE` to "-1" makes the recipe unlikely to be used
+unless it is explicitly referenced. Setting :term:`DEFAULT_PREFERENCE` to
+"1" makes it likely the recipe is used. :term:`PREFERRED_VERSION` overrides
+any :term:`DEFAULT_PREFERENCE` setting. :term:`DEFAULT_PREFERENCE` is often used
 to mark newer and more experimental recipe versions until they have
 undergone sufficient testing to be considered stable.
 
@@ -330,7 +331,7 @@ If the first recipe is named ``a_1.1.bb``, then the
 
 Thus, if a recipe named ``a_1.2.bb`` exists, BitBake will choose 1.2 by
 default. However, if you define the following variable in a ``.conf``
-file that BitBake parses, you can change that preference: ::
+file that BitBake parses, you can change that preference::
 
   PREFERRED_VERSION_a = "1.1"
 
@@ -393,7 +394,7 @@ ready to run, those tasks have all their dependencies met, and the
 thread threshold has not been exceeded.
 
 It is worth noting that you can greatly speed up the build time by
-properly setting the ``BB_NUMBER_THREADS`` variable.
+properly setting the :term:`BB_NUMBER_THREADS` variable.
 
 As each task completes, a timestamp is written to the directory
 specified by the :term:`STAMP` variable. On subsequent
@@ -434,7 +435,7 @@ BitBake writes a shell script to
 executes the script. The generated shell script contains all the
 exported variables, and the shell functions with all variables expanded.
 Output from the shell script goes to the file
-``${T}/log.do_taskname.pid``. Looking at the expanded shell functions in
+``${``\ :term:`T`\ ``}/log.do_taskname.pid``. Looking at the expanded shell functions in
 the run file and the output in the log files is a useful debugging
 technique.
 
@@ -476,7 +477,7 @@ changes because it should not affect the output for target packages. The
 simplistic approach for excluding the working directory is to set it to
 some fixed value and create the checksum for the "run" script. BitBake
 goes one step better and uses the
-:term:`BB_HASHBASE_WHITELIST` variable
+:term:`BB_BASEHASH_IGNORE_VARS` variable
 to define a list of variables that should never be included when
 generating the signatures.
 
@@ -497,7 +498,7 @@ to the task.
 
 Like the working directory case, situations exist where dependencies
 should be ignored. For these cases, you can instruct the build process
-to ignore a dependency by using a line like the following: ::
+to ignore a dependency by using a line like the following::
 
   PACKAGE_ARCHS[vardepsexclude] = "MACHINE"
 
@@ -507,7 +508,7 @@ even if it does reference it.
 
 Equally, there are cases where we need to add dependencies BitBake is
 not able to find. You can accomplish this by using a line like the
-following: ::
+following::
 
   PACKAGE_ARCHS[vardeps] = "MACHINE"
 
@@ -522,7 +523,7 @@ it cannot figure out dependencies.
 Thus far, this section has limited discussion to the direct inputs into
 a task. Information based on direct inputs is referred to as the
 "basehash" in the code. However, there is still the question of a task's
-indirect inputs - the things that were already built and present in the
+indirect inputs --- the things that were already built and present in the
 build directory. The checksum (or signature) for a particular task needs
 to add the hashes of all the tasks on which the particular task depends.
 Choosing which dependencies to add is a policy decision. However, the
@@ -533,11 +534,11 @@ At the code level, there are a variety of ways both the basehash and the
 dependent task hashes can be influenced. Within the BitBake
 configuration file, we can give BitBake some extra information to help
 it construct the basehash. The following statement effectively results
-in a list of global variable dependency excludes - variables never
+in a list of global variable dependency excludes --- variables never
 included in any checksum. This example uses variables from OpenEmbedded
-to help illustrate the concept: ::
+to help illustrate the concept::
 
-   BB_HASHBASE_WHITELIST ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \
+   BB_BASEHASH_IGNORE_VARS ?= "TMPDIR FILE PATH PWD BB_TASKHASH BBPATH DL_DIR \
        SSTATE_DIR THISDIR FILESEXTRAPATHS FILE_DIRNAME HOME LOGNAME SHELL \
        USER FILESPATH STAGING_DIR_HOST STAGING_DIR_TARGET COREBASE PRSERV_HOST \
        PRSERV_DUMPDIR PRSERV_DUMPFILE PRSERV_LOCKDOWN PARALLEL_MAKE \
@@ -551,23 +552,22 @@ through dependency chains are more complex and are generally
 accomplished with a Python function. The code in
 ``meta/lib/oe/sstatesig.py`` shows two examples of this and also
 illustrates how you can insert your own policy into the system if so
-desired. This file defines the two basic signature generators
-OpenEmbedded-Core uses: "OEBasic" and "OEBasicHash". By default, there
+desired. This file defines the basic signature generator
+OpenEmbedded-Core uses: "OEBasicHash". By default, there
 is a dummy "noop" signature handler enabled in BitBake. This means that
 behavior is unchanged from previous versions. ``OE-Core`` uses the
 "OEBasicHash" signature handler by default through this setting in the
-``bitbake.conf`` file: ::
+``bitbake.conf`` file::
 
   BB_SIGNATURE_HANDLER ?= "OEBasicHash"
 
-The "OEBasicHash" ``BB_SIGNATURE_HANDLER`` is the same as the "OEBasic"
-version but adds the task hash to the stamp files. This results in any
-metadata change that changes the task hash, automatically causing the
-task to be run again. This removes the need to bump
-:term:`PR` values, and changes to metadata automatically
-ripple across the build.
+The main feature of the "OEBasicHash" :term:`BB_SIGNATURE_HANDLER` is that
+it adds the task hash to the stamp files. Thanks to this, any metadata
+change will change the task hash, automatically causing the task to be run
+again. This removes the need to bump :term:`PR` values, and changes to
+metadata automatically ripple across the build.
 
-It is also worth noting that the end result of these signature
+It is also worth noting that the end result of signature
 generators is to make some dependency and hash information available to
 the build. This information includes:
 
@@ -577,10 +577,7 @@ the build. This information includes:
 -  ``BB_BASEHASH_``\ *filename:taskname*: The base hashes for each
    dependent task.
 
--  ``BBHASHDEPS_``\ *filename:taskname*: The task dependencies for
-   each task.
-
--  ``BB_TASKHASH``: The hash of the currently running task.
+-  :term:`BB_TASKHASH`: The hash of the currently running task.
 
 It is worth noting that BitBake's "-S" option lets you debug BitBake's
 processing of signatures. The options passed to -S allow different
@@ -589,10 +586,11 @@ or possibly those defined in the metadata/signature handler itself. The
 simplest parameter to pass is "none", which causes a set of signature
 information to be written out into ``STAMPS_DIR`` corresponding to the
 targets specified. The other currently available parameter is
-"printdiff", which causes BitBake to try to establish the closest
+"printdiff", which causes BitBake to try to establish the most recent
 signature match it can (e.g. in the sstate cache) and then run
-``bitbake-diffsigs`` over the matches to determine the stamps and delta
-where these two stamp trees diverge.
+compare the matched signatures to determine the stamps and delta
+where these two stamp trees diverge. This can be used to determine why
+tasks need to be re-run in situations where that is not expected.
 
 .. note::
 
@@ -647,13 +645,6 @@ compiled binary. To handle this, BitBake calls the
 each successful setscene task to know whether or not it needs to obtain
 the dependencies of that task.
 
-Finally, after all the setscene tasks have executed, BitBake calls the
-function listed in
-:term:`BB_SETSCENE_VERIFY_FUNCTION2`
-with the list of tasks BitBake thinks has been "covered". The metadata
-can then ensure that this list is correct and can inform BitBake that it
-wants specific tasks to be run regardless of the setscene result.
-
 You can find more information on setscene metadata in the
 :ref:`bitbake-user-manual/bitbake-user-manual-metadata:task checksums and setscene`
 section.
@@ -666,7 +657,7 @@ builds are when execute, bitbake also supports user defined
 configuration of the `Python
 logging <https://docs.python.org/3/library/logging.html>`__ facilities
 through the :term:`BB_LOGCONFIG` variable. This
-variable defines a json or yaml `logging
+variable defines a JSON or YAML `logging
 configuration <https://docs.python.org/3/library/logging.config.html>`__
 that will be intelligently merged into the default configuration. The
 logging configuration is merged using the following rules:
@@ -700,9 +691,9 @@ logging configuration is merged using the following rules:
    adds a filter called ``BitBake.defaultFilter``, both filters will be
    applied to the logger
 
-As an example, consider the following user logging configuration file
-which logs all Hash Equivalence related messages of VERBOSE or higher to
-a file called ``hashequiv.log`` ::
+As a first example, you can create a ``hashequiv.json`` user logging
+configuration file to log all Hash Equivalence related messages of ``VERBOSE``
+or higher priority to a file called ``hashequiv.log``::
 
    {
        "version": 1,
@@ -731,3 +722,40 @@ a file called ``hashequiv.log`` ::
            }
        }
    }
+
+Then set the :term:`BB_LOGCONFIG` variable in ``conf/local.conf``::
+
+   BB_LOGCONFIG = "hashequiv.json"
+
+Another example is this ``warn.json`` file to log all ``WARNING`` and
+higher priority messages to a ``warn.log`` file::
+
+  {
+      "version": 1,
+      "formatters": {
+          "warnlogFormatter": {
+              "()": "bb.msg.BBLogFormatter",
+              "format": "%(levelname)s: %(message)s"
+          }
+      },
+
+      "handlers": {
+          "warnlog": {
+              "class": "logging.FileHandler",
+              "formatter": "warnlogFormatter",
+              "level": "WARNING",
+              "filename": "warn.log"
+          }
+      },
+
+      "loggers": {
+          "BitBake": {
+              "handlers": ["warnlog"]
+          }
+      },
+
+      "@disable_existing_loggers": false
+  }
+
+Note that BitBake's helper classes for structured logging are implemented in
+``lib/bb/msg.py``.
diff --git a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst
index 6760b10828..a2c2432db1 100644
--- a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst
+++ b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-fetching.rst
@@ -27,7 +27,7 @@ and unpacking the files is often optionally followed by patching.
 Patching, however, is not covered by this module.
 
 The code to execute the first part of this process, a fetch, looks
-something like the following: ::
+something like the following::
 
    src_uri = (d.getVar('SRC_URI') or "").split()
    fetcher = bb.fetch2.Fetch(src_uri, d)
@@ -37,12 +37,12 @@ This code sets up an instance of the fetch class. The instance uses a
 space-separated list of URLs from the :term:`SRC_URI`
 variable and then calls the ``download`` method to download the files.
 
-The instantiation of the fetch class is usually followed by: ::
+The instantiation of the fetch class is usually followed by::
 
-   rootdir = l.getVar('WORKDIR')
+   rootdir = l.getVar('UNPACKDIR')
    fetcher.unpack(rootdir)
 
-This code unpacks the downloaded files to the specified by ``WORKDIR``.
+This code unpacks the downloaded files to the specified by ``UNPACKDIR``.
 
 .. note::
 
@@ -51,7 +51,7 @@ This code unpacks the downloaded files to the specified by ``WORKDIR``.
    examine the OpenEmbedded class file ``base.bbclass``
    .
 
-The ``SRC_URI`` and ``WORKDIR`` variables are not hardcoded into the
+The :term:`SRC_URI` and ``UNPACKDIR`` variables are not hardcoded into the
 fetcher, since those fetcher methods can be (and are) called with
 different variable names. In OpenEmbedded for example, the shared state
 (sstate) code uses the fetch module to fetch the sstate files.
@@ -64,38 +64,38 @@ URLs by looking for source files in a specific search order:
    :term:`PREMIRRORS` variable.
 
 -  *Source URI:* If pre-mirrors fail, BitBake uses the original URL (e.g
-   from ``SRC_URI``).
+   from :term:`SRC_URI`).
 
 -  *Mirror Sites:* If fetch failures occur, BitBake next uses mirror
    locations as defined by the :term:`MIRRORS` variable.
 
 For each URL passed to the fetcher, the fetcher calls the submodule that
 handles that particular URL type. This behavior can be the source of
-some confusion when you are providing URLs for the ``SRC_URI`` variable.
-Consider the following two URLs: ::
+some confusion when you are providing URLs for the :term:`SRC_URI` variable.
+Consider the following two URLs::
 
-   http://git.yoctoproject.org/git/poky;protocol=git
+   https://git.yoctoproject.org/git/poky;protocol=git
    git://git.yoctoproject.org/git/poky;protocol=http
 
 In the former case, the URL is passed to the ``wget`` fetcher, which does not
 understand "git". Therefore, the latter case is the correct form since the Git
 fetcher does know how to use HTTP as a transport.
 
-Here are some examples that show commonly used mirror definitions: ::
+Here are some examples that show commonly used mirror definitions::
 
    PREMIRRORS ?= "\
-      bzr://.*/.\*  http://somemirror.org/sources/ \\n \
-      cvs://.*/.\*  http://somemirror.org/sources/ \\n \
-      git://.*/.\*  http://somemirror.org/sources/ \\n \
-      hg://.*/.\*   http://somemirror.org/sources/ \\n \
-      osc://.*/.\*  http://somemirror.org/sources/ \\n \
-      p4://.*/.\*   http://somemirror.org/sources/ \\n \
-     svn://.*/.\*   http://somemirror.org/sources/ \\n"
+      bzr://.*/.\*  http://somemirror.org/sources/ \
+      cvs://.*/.\*  http://somemirror.org/sources/ \
+      git://.*/.\*  http://somemirror.org/sources/ \
+      hg://.*/.\*   http://somemirror.org/sources/ \
+      osc://.*/.\*  http://somemirror.org/sources/ \
+      p4://.*/.\*   http://somemirror.org/sources/ \
+     svn://.*/.\*   http://somemirror.org/sources/"
 
    MIRRORS =+ "\
-      ftp://.*/.\*   http://somemirror.org/sources/ \\n \
-      http://.*/.\*  http://somemirror.org/sources/ \\n \
-      https://.*/.\* http://somemirror.org/sources/ \\n"
+      ftp://.*/.\*   http://somemirror.org/sources/ \
+      http://.*/.\*  http://somemirror.org/sources/ \
+      https://.*/.\* http://somemirror.org/sources/"
 
 It is useful to note that BitBake
 supports cross-URLs. It is possible to mirror a Git repository on an
@@ -110,26 +110,26 @@ which is specified by the :term:`DL_DIR` variable.
 File integrity is of key importance for reproducing builds. For
 non-local archive downloads, the fetcher code can verify SHA-256 and MD5
 checksums to ensure the archives have been downloaded correctly. You can
-specify these checksums by using the ``SRC_URI`` variable with the
-appropriate varflags as follows: ::
+specify these checksums by using the :term:`SRC_URI` variable with the
+appropriate varflags as follows::
 
    SRC_URI[md5sum] = "value"
    SRC_URI[sha256sum] = "value"
 
 You can also specify the checksums as
-parameters on the ``SRC_URI`` as shown below: ::
+parameters on the :term:`SRC_URI` as shown below::
 
   SRC_URI = "http://example.com/foobar.tar.bz2;md5sum=4a8e0f237e961fd7785d19d07fdb994d"
 
 If multiple URIs exist, you can specify the checksums either directly as
 in the previous example, or you can name the URLs. The following syntax
-shows how you name the URIs: ::
+shows how you name the URIs::
 
    SRC_URI = "http://example.com/foobar.tar.bz2;name=foo"
    SRC_URI[foo.md5sum] = 4a8e0f237e961fd7785d19d07fdb994d
 
 After a file has been downloaded and
-has had its checksum checked, a ".done" stamp is placed in ``DL_DIR``.
+has had its checksum checked, a ".done" stamp is placed in :term:`DL_DIR`.
 BitBake uses this stamp during subsequent builds to avoid downloading or
 comparing a checksum for the file again.
 
@@ -144,6 +144,10 @@ download without a checksum triggers an error message. The
 make any attempted network access a fatal error, which is useful for
 checking that mirrors are complete as well as other things.
 
+If :term:`BB_CHECK_SSL_CERTS` is set to ``0`` then SSL certificate checking will
+be disabled. This variable defaults to ``1`` so SSL certificates are normally
+checked.
+
 .. _bb-the-unpack:
 
 The Unpack
@@ -163,8 +167,8 @@ govern the behavior of the unpack stage:
 -  *dos:* Applies to ``.zip`` and ``.jar`` files and specifies whether
    to use DOS line ending conversion on text files.
 
--  *basepath:* Instructs the unpack stage to strip the specified
-   directories from the source path when unpacking.
+-  *striplevel:* Strip specified number of leading components (levels)
+   from file names on extraction
 
 -  *subdir:* Unpacks the specific URL to the specified subdirectory
    within the root directory.
@@ -204,7 +208,7 @@ time the ``download()`` method is called.
 If you specify a directory, the entire directory is unpacked.
 
 Here are a couple of example URLs, the first relative and the second
-absolute: ::
+absolute::
 
    SRC_URI = "file://relativefile.patch"
    SRC_URI = "file:///Users/ich/very_important_software"
@@ -225,7 +229,12 @@ downloaded file is useful for avoiding collisions in
 :term:`DL_DIR` when dealing with multiple files that
 have the same name.
 
-Some example URLs are as follows: ::
+If a username and password are specified in the ``SRC_URI``, a Basic
+Authorization header will be added to each request, including across redirects.
+To instead limit the Authorization header to the first request, add
+"redirectauth=0" to the list of parameters.
+
+Some example URLs are as follows::
 
    SRC_URI = "http://oe.handhelds.org/not_there.aac"
    SRC_URI = "ftp://oe.handhelds.org/not_there_as_well.aac"
@@ -235,15 +244,13 @@ Some example URLs are as follows: ::
 
    Because URL parameters are delimited by semi-colons, this can
    introduce ambiguity when parsing URLs that also contain semi-colons,
-   for example:
-   ::
+   for example::
 
            SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git;a=snapshot;h=a5dd47"
 
 
    Such URLs should should be modified by replacing semi-colons with '&'
-   characters:
-   ::
+   characters::
 
            SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&a=snapshot&h=a5dd47"
 
@@ -251,8 +258,7 @@ Some example URLs are as follows: ::
    In most cases this should work. Treating semi-colons and '&' in
    queries identically is recommended by the World Wide Web Consortium
    (W3C). Note that due to the nature of the URL, you may have to
-   specify the name of the downloaded file as well:
-   ::
+   specify the name of the downloaded file as well::
 
            SRC_URI = "http://abc123.org/git/?p=gcc/gcc.git&a=snapshot&h=a5dd47;downloadfilename=myfile.bz2"
 
@@ -321,7 +327,7 @@ The supported parameters are as follows:
 
 -  *"port":* The port to which the CVS server connects.
 
-Some example URLs are as follows: ::
+Some example URLs are as follows::
 
    SRC_URI = "cvs://CVSROOT;module=mymodule;tag=some-version;method=ext"
    SRC_URI = "cvs://CVSROOT;module=mymodule;date=20060126;localdir=usethat"
@@ -363,7 +369,7 @@ The supported parameters are as follows:
    username is different than the username used in the main URL, which
    is passed to the subversion command.
 
-Following are three examples using svn: ::
+Following are three examples using svn::
 
    SRC_URI = "svn://myrepos/proj1;module=vip;protocol=http;rev=667"
    SRC_URI = "svn://myrepos/proj1;module=opie;protocol=svn+ssh"
@@ -390,6 +396,19 @@ This fetcher supports the following parameters:
    protocol is "file". You can also use "http", "https", "ssh" and
    "rsync".
 
+   .. note::
+
+     When ``protocol`` is "ssh", the URL expected in :term:`SRC_URI` differs
+     from the one that is typically passed to ``git clone`` command and provided
+     by the Git server to fetch from. For example, the URL returned by GitLab
+     server for ``mesa`` when cloning over SSH is
+     ``git@gitlab.freedesktop.org:mesa/mesa.git``, however the expected URL in
+     :term:`SRC_URI` is the following::
+
+       SRC_URI = "git://git@gitlab.freedesktop.org/mesa/mesa.git;branch=main;protocol=ssh;..."
+
+     Note the ``:`` character changed for a ``/`` before the path to the project.
+
 -  *"nocheckout":* Tells the fetcher to not checkout source code when
    unpacking when set to "1". Set this option for the URL where there is
    a custom routine to checkout code. The default is "0".
@@ -405,17 +424,17 @@ This fetcher supports the following parameters:
 
 -  *"nobranch":* Tells the fetcher to not check the SHA validation for
    the branch when set to "1". The default is "0". Set this option for
-   the recipe that refers to the commit that is valid for a tag instead
-   of the branch.
+   the recipe that refers to the commit that is valid for any namespace
+   (branch, tag, ...) instead of the branch.
 
 -  *"bareclone":* Tells the fetcher to clone a bare clone into the
    destination directory without checking out a working tree. Only the
    raw Git metadata is provided. This parameter implies the "nocheckout"
    parameter as well.
 
--  *"branch":* The branch(es) of the Git tree to clone. If unset, this
-   is assumed to be "master". The number of branch parameters much match
-   the number of name parameters.
+-  *"branch":* The branch(es) of the Git tree to clone. Unless
+   "nobranch" is set to "1", this is a mandatory parameter. The number of
+   branch parameters must match the number of name parameters.
 
 -  *"rev":* The revision to use for the checkout. The default is
    "master".
@@ -436,10 +455,28 @@ This fetcher supports the following parameters:
    parameter implies no branch and only works when the transfer protocol
    is ``file://``.
 
-Here are some example URLs: ::
+Here are some example URLs::
+
+   SRC_URI = "git://github.com/fronteed/icheck.git;protocol=https;branch=${PV};tag=${PV}"
+   SRC_URI = "git://github.com/asciidoc/asciidoc-py;protocol=https;branch=main"
+   SRC_URI = "git://git@gitlab.freedesktop.org/mesa/mesa.git;branch=main;protocol=ssh;..."
+
+.. note::
 
-   SRC_URI = "git://git.oe.handhelds.org/git/vip.git;tag=version-1"
-   SRC_URI = "git://git.oe.handhelds.org/git/vip.git;protocol=http"
+   Specifying passwords directly in ``git://`` urls is not supported.
+   There are several reasons: :term:`SRC_URI` is often written out to logs and
+   other places, and that could easily leak passwords; it is also all too
+   easy to share metadata without removing passwords. SSH keys, ``~/.netrc``
+   and ``~/.ssh/config`` files can be used as alternatives.
+
+Using tags with the git fetcher may cause surprising behaviour. Bitbake needs to
+resolve the tag to a specific revision and to do that, it has to connect to and use
+the upstream repository. This is because the revision the tags point at can change and
+we've seen cases of this happening in well known public repositories. This can mean
+many more network connections than expected and recipes may be reparsed at every build.
+Source mirrors will also be bypassed as the upstream repository is the only source
+of truth to resolve the revision accurately. For these reasons, whilst the fetcher
+can support tags, we recommend being specific about revisions in recipes.
 
 .. _gitsm-fetcher:
 
@@ -475,7 +512,7 @@ repository.
 
 To use this fetcher, make sure your recipe has proper
 :term:`SRC_URI`, :term:`SRCREV`, and
-:term:`PV` settings. Here is an example: ::
+:term:`PV` settings. Here is an example::
 
    SRC_URI = "ccrc://cc.example.org/ccrc;vob=/example_vob;module=/example_module"
    SRCREV = "EXAMPLE_CLEARCASE_TAG"
@@ -484,7 +521,7 @@ To use this fetcher, make sure your recipe has proper
 The fetcher uses the ``rcleartool`` or
 ``cleartool`` remote client, depending on which one is available.
 
-Following are options for the ``SRC_URI`` statement:
+Following are options for the :term:`SRC_URI` statement:
 
 -  *vob*: The name, which must include the prepending "/" character,
    of the ClearCase VOB. This option is required.
@@ -497,7 +534,7 @@ Following are options for the ``SRC_URI`` statement:
       The module and vob options are combined to create the load rule in the
       view config spec. As an example, consider the vob and module values from
       the SRC_URI statement at the start of this section. Combining those values
-      results in the following: ::
+      results in the following::
 
          load /example_vob/example_module
 
@@ -546,32 +583,32 @@ password if you do not wish to keep those values in a recipe itself. If
 you choose not to use ``P4CONFIG``, or to explicitly set variables that
 ``P4CONFIG`` can contain, you can specify the ``P4PORT`` value, which is
 the server's URL and port number, and you can specify a username and
-password directly in your recipe within ``SRC_URI``.
+password directly in your recipe within :term:`SRC_URI`.
 
 Here is an example that relies on ``P4CONFIG`` to specify the server URL
-and port, username, and password, and fetches the Head Revision: ::
+and port, username, and password, and fetches the Head Revision::
 
    SRC_URI = "p4://example-depot/main/source/..."
    SRCREV = "${AUTOREV}"
    PV = "p4-${SRCPV}"
-   S = "${WORKDIR}/p4"
+   S = "${UNPACKDIR}/p4"
 
 Here is an example that specifies the server URL and port, username, and
-password, and fetches a Revision based on a Label: ::
+password, and fetches a Revision based on a Label::
 
    P4PORT = "tcp:p4server.example.net:1666"
    SRC_URI = "p4://user:passwd@example-depot/main/source/..."
    SRCREV = "release-1.0"
    PV = "p4-${SRCPV}"
-   S = "${WORKDIR}/p4"
+   S = "${UNPACKDIR}/p4"
 
 .. note::
 
-   You should always set S to "${WORKDIR}/p4" in your recipe.
+   You should always set S to "${UNPACKDIR}/p4" in your recipe.
 
 By default, the fetcher strips the depot location from the local file paths. In
 the above example, the content of ``example-depot/main/source/`` will be placed
-in ``${WORKDIR}/p4``.  For situations where preserving parts of the remote depot
+in ``${UNPACKDIR}/p4``.  For situations where preserving parts of the remote depot
 paths locally is desirable, the fetcher supports two parameters:
 
 - *"module":*
@@ -583,7 +620,7 @@ paths locally is desirable, the fetcher supports two parameters:
     paths locally for the specified location, even in combination with the
     ``module`` parameter.
 
-Here is an example use of the the ``module`` parameter: ::
+Here is an example use of the the ``module`` parameter::
 
    SRC_URI = "p4://user:passwd@example-depot/main;module=source/..."
 
@@ -591,7 +628,7 @@ In this case, the content of the top-level directory ``source/`` will be fetched
 to ``${P4DIR}``, including the directory itself.  The top-level directory will
 be accesible at ``${P4DIR}/source/``.
 
-Here is an example use of the the ``remotepath`` parameter: ::
+Here is an example use of the the ``remotepath`` parameter::
 
    SRC_URI = "p4://user:passwd@example-depot/main;module=source/...;remotepath=keep"
 
@@ -619,11 +656,166 @@ This fetcher supports the following parameters:
 
 -  *"manifest":* Name of the manifest file (default: ``default.xml``).
 
-Here are some example URLs: ::
+Here are some example URLs::
 
    SRC_URI = "repo://REPOROOT;protocol=git;branch=some_branch;manifest=my_manifest.xml"
    SRC_URI = "repo://REPOROOT;protocol=file;branch=some_branch;manifest=my_manifest.xml"
 
+.. _az-fetcher:
+
+Az Fetcher (``az://``)
+--------------------------
+
+This submodule fetches data from an
+`Azure Storage account <https://docs.microsoft.com/en-us/azure/storage/>`__ ,
+it inherits its functionality from the HTTP wget fetcher, but modifies its
+behavior to accomodate the usage of a
+`Shared Access Signature (SAS) <https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview>`__
+for non-public data.
+
+Such functionality is set by the variable:
+
+-  :term:`AZ_SAS`: The Azure Storage Shared Access Signature provides secure
+   delegate access to resources, if this variable is set, the Az Fetcher will
+   use it when fetching artifacts from the cloud.
+
+You can specify the AZ_SAS variable prefixed with a ? as shown below::
+
+   AZ_SAS = "?se=2021-01-01&sp=r&sv=2018-11-09&sr=c&skoid=<skoid>&sig=<signature>"
+
+Here is an example URL::
+
+   SRC_URI = "az://<azure-storage-account>.blob.core.windows.net/<foo_container>/<bar_file>"
+
+It can also be used when setting mirrors definitions using the :term:`PREMIRRORS` variable.
+
+.. _gcp-fetcher:
+
+GCP Fetcher (``gs://``)
+--------------------------
+
+This submodule fetches data from a
+`Google Cloud Storage Bucket <https://cloud.google.com/storage/docs/buckets>`__.
+It uses the `Google Cloud Storage Python Client <https://cloud.google.com/python/docs/reference/storage/latest>`__
+to check the status of objects in the bucket and download them.
+The use of the Python client makes it substantially faster than using command
+line tools such as gsutil.
+
+The fetcher requires the Google Cloud Storage Python Client to be installed, along
+with the gsutil tool.
+
+The fetcher requires that the machine has valid credentials for accessing the
+chosen bucket. Instructions for authentication can be found in the
+`Google Cloud documentation <https://cloud.google.com/docs/authentication/provide-credentials-adc#local-dev>`__.
+
+If it used from the OpenEmbedded build system, the fetcher can be used for
+fetching sstate artifacts from a GCS bucket by specifying the
+``SSTATE_MIRRORS`` variable as shown below::
+
+   SSTATE_MIRRORS ?= "\
+       file://.* gs://<bucket name>/PATH \
+   "
+
+The fetcher can also be used in recipes::
+
+   SRC_URI = "gs://<bucket name>/<foo_container>/<bar_file>"
+
+However, the checksum of the file should be also be provided::
+
+   SRC_URI[sha256sum] = "<sha256 string>"
+
+.. _crate-fetcher:
+
+Crate Fetcher (``crate://``)
+----------------------------
+
+This submodule fetches code for
+`Rust language "crates" <https://doc.rust-lang.org/reference/glossary.html?highlight=crate#crate>`__
+corresponding to Rust libraries and programs to compile. Such crates are typically shared
+on https://crates.io/ but this fetcher supports other crate registries too.
+
+The format for the :term:`SRC_URI` setting must be::
+
+   SRC_URI = "crate://REGISTRY/NAME/VERSION"
+
+Here is an example URL::
+
+   SRC_URI = "crate://crates.io/glob/0.2.11"
+
+.. _npm-fetcher:
+
+NPM Fetcher (``npm://``)
+------------------------
+
+This submodule fetches source code from an
+`NPM <https://en.wikipedia.org/wiki/Npm_(software)>`__
+Javascript package registry.
+
+The format for the :term:`SRC_URI` setting must be::
+
+   SRC_URI = "npm://some.registry.url;ParameterA=xxx;ParameterB=xxx;..."
+
+This fetcher supports the following parameters:
+
+-  *"package":* The NPM package name. This is a mandatory parameter.
+
+-  *"version":* The NPM package version. This is a mandatory parameter.
+
+-  *"downloadfilename":* Specifies the filename used when storing the downloaded file.
+
+-  *"destsuffix":* Specifies the directory to use to unpack the package (default: ``npm``).
+
+Note that NPM fetcher only fetches the package source itself. The dependencies
+can be fetched through the `npmsw-fetcher`_.
+
+Here is an example URL with both fetchers::
+
+   SRC_URI = " \
+       npm://registry.npmjs.org/;package=cute-files;version=${PV} \
+       npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json \
+       "
+
+See :yocto_docs:`Creating Node Package Manager (NPM) Packages
+</dev-manual/packages.html#creating-node-package-manager-npm-packages>`
+in the Yocto Project manual for details about using
+:yocto_docs:`devtool <https://docs.yoctoproject.org/ref-manual/devtool-reference.html>`
+to automatically create a recipe from an NPM URL.
+
+.. _npmsw-fetcher:
+
+NPM shrinkwrap Fetcher (``npmsw://``)
+-------------------------------------
+
+This submodule fetches source code from an
+`NPM shrinkwrap <https://docs.npmjs.com/cli/v8/commands/npm-shrinkwrap>`__
+description file, which lists the dependencies
+of an NPM package while locking their versions.
+
+The format for the :term:`SRC_URI` setting must be::
+
+   SRC_URI = "npmsw://some.registry.url;ParameterA=xxx;ParameterB=xxx;..."
+
+This fetcher supports the following parameters:
+
+-  *"dev":* Set this parameter to ``1`` to install "devDependencies".
+
+-  *"destsuffix":* Specifies the directory to use to unpack the dependencies
+   (``${S}`` by default).
+
+Note that the shrinkwrap file can also be provided by the recipe for
+the package which has such dependencies, for example::
+
+   SRC_URI = " \
+       npm://registry.npmjs.org/;package=cute-files;version=${PV} \
+       npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json \
+       "
+
+Such a file can automatically be generated using
+:yocto_docs:`devtool <https://docs.yoctoproject.org/ref-manual/devtool-reference.html>`
+as described in the :yocto_docs:`Creating Node Package Manager (NPM) Packages
+</dev-manual/packages.html#creating-node-package-manager-npm-packages>`
+section of the Yocto Project.
+
 Other Fetchers
 --------------
 
@@ -633,10 +825,10 @@ Fetch submodules also exist for the following:
 
 -  Mercurial (``hg://``)
 
--  npm (``npm://``)
-
 -  OSC (``osc://``)
 
+-  S3 (``s3://``)
+
 -  Secure FTP (``sftp://``)
 
 -  Secure Shell (``ssh://``)
@@ -649,4 +841,4 @@ submodules. However, you might find the code helpful and readable.
 Auto Revisions
 ==============
 
-We need to document ``AUTOREV`` and ``SRCREV_FORMAT`` here.
+We need to document ``AUTOREV`` and :term:`SRCREV_FORMAT` here.
diff --git a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst
index e3fd321588..654196ca24 100644
--- a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst
+++ b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-hello.rst
@@ -18,28 +18,32 @@ it.
 Obtaining BitBake
 =================
 
-See the :ref:`bitbake-user-manual/bitbake-user-manual-hello:obtaining bitbake` section for
+See the :ref:`bitbake-user-manual/bitbake-user-manual-intro:obtaining bitbake` section for
 information on how to obtain BitBake. Once you have the source code on
-your machine, the BitBake directory appears as follows: ::
+your machine, the BitBake directory appears as follows::
 
    $ ls -al
-   total 100
-   drwxrwxr-x. 9 wmat wmat  4096 Jan 31 13:44 .
-   drwxrwxr-x. 3 wmat wmat  4096 Feb  4 10:45 ..
-   -rw-rw-r--. 1 wmat wmat   365 Nov 26 04:55 AUTHORS
-   drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 bin
-   drwxrwxr-x. 4 wmat wmat  4096 Jan 31 13:44 build
-   -rw-rw-r--. 1 wmat wmat 16501 Nov 26 04:55 ChangeLog
-   drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 classes
-   drwxrwxr-x. 2 wmat wmat  4096 Nov 26 04:55 conf
-   drwxrwxr-x. 3 wmat wmat  4096 Nov 26 04:55 contrib
-   -rw-rw-r--. 1 wmat wmat 17987 Nov 26 04:55 COPYING
-   drwxrwxr-x. 3 wmat wmat  4096 Nov 26 04:55 doc
-   -rw-rw-r--. 1 wmat wmat    69 Nov 26 04:55 .gitignore
-   -rw-rw-r--. 1 wmat wmat   849 Nov 26 04:55 HEADER
-   drwxrwxr-x. 5 wmat wmat  4096 Jan 31 13:44 lib
-   -rw-rw-r--. 1 wmat wmat   195 Nov 26 04:55 MANIFEST.in
-   -rw-rw-r--. 1 wmat wmat  2887 Nov 26 04:55 TODO
+   total 108
+   drwxr-xr-x  9 fawkh 10000  4096 feb 24 12:10 .
+   drwx------ 36 fawkh 10000  4096 mar  2 17:00 ..
+   -rw-r--r--  1 fawkh 10000   365 feb 24 12:10 AUTHORS
+   drwxr-xr-x  2 fawkh 10000  4096 feb 24 12:10 bin
+   -rw-r--r--  1 fawkh 10000 16501 feb 24 12:10 ChangeLog
+   drwxr-xr-x  2 fawkh 10000  4096 feb 24 12:10 classes
+   drwxr-xr-x  2 fawkh 10000  4096 feb 24 12:10 conf
+   drwxr-xr-x  5 fawkh 10000  4096 feb 24 12:10 contrib
+   drwxr-xr-x  6 fawkh 10000  4096 feb 24 12:10 doc
+   drwxr-xr-x  8 fawkh 10000  4096 mar  2 16:26 .git
+   -rw-r--r--  1 fawkh 10000    31 feb 24 12:10 .gitattributes
+   -rw-r--r--  1 fawkh 10000   392 feb 24 12:10 .gitignore
+   drwxr-xr-x 13 fawkh 10000  4096 feb 24 12:11 lib
+   -rw-r--r--  1 fawkh 10000  1224 feb 24 12:10 LICENSE
+   -rw-r--r--  1 fawkh 10000 15394 feb 24 12:10 LICENSE.GPL-2.0-only
+   -rw-r--r--  1 fawkh 10000  1286 feb 24 12:10 LICENSE.MIT
+   -rw-r--r--  1 fawkh 10000   229 feb 24 12:10 MANIFEST.in
+   -rw-r--r--  1 fawkh 10000  2413 feb 24 12:10 README
+   -rw-r--r--  1 fawkh 10000    43 feb 24 12:10 toaster-requirements.txt
+   -rw-r--r--  1 fawkh 10000  2887 feb 24 12:10 TODO
 
 At this point, you should have BitBake cloned to a directory that
 matches the previous listing except for dates and user names.
@@ -49,10 +53,10 @@ Setting Up the BitBake Environment
 
 First, you need to be sure that you can run BitBake. Set your working
 directory to where your local BitBake files are and run the following
-command: ::
+command::
 
   $ ./bin/bitbake --version
-  BitBake Build Tool Core version 1.23.0, bitbake version 1.23.0
+  BitBake Build Tool Core version 2.3.1
 
 The console output tells you what version
 you are running.
@@ -61,14 +65,14 @@ The recommended method to run BitBake is from a directory of your
 choice. To be able to run BitBake from any directory, you need to add
 the executable binary to your binary to your shell's environment
 ``PATH`` variable. First, look at your current ``PATH`` variable by
-entering the following: ::
+entering the following::
 
   $ echo $PATH
 
 Next, add the directory location
 for the BitBake binary to the ``PATH``. Here is an example that adds the
 ``/home/scott-lenovo/bitbake/bin`` directory to the front of the
-``PATH`` variable: ::
+``PATH`` variable::
 
   $ export PATH=/home/scott-lenovo/bitbake/bin:$PATH
 
@@ -99,7 +103,7 @@ discussion mailing list about the BitBake build tool.
 
    This example was inspired by and drew heavily from
    `Mailing List post - The BitBake equivalent of "Hello, World!"
-   <http://www.mail-archive.com/yocto@yoctoproject.org/msg09379.html>`_.
+   <https://www.mail-archive.com/yocto@yoctoproject.org/msg09379.html>`_.
 
 As stated earlier, the goal of this example is to eventually compile
 "Hello World". However, it is unknown what BitBake needs and what you
@@ -116,7 +120,7 @@ Following is the complete "Hello World" example.
 
 #.  **Create a Project Directory:** First, set up a directory for the
     "Hello World" project. Here is how you can do so in your home
-    directory: ::
+    directory::
 
       $ mkdir ~/hello
       $ cd ~/hello
@@ -127,41 +131,26 @@ Following is the complete "Hello World" example.
     directory is a good way to isolate your project.
 
 #.  **Run BitBake:** At this point, you have nothing but a project
-    directory. Run the ``bitbake`` command and see what it does: ::
+    directory. Run the ``bitbake`` command and see what it does::
 
        $ bitbake
-       The BBPATH variable is not set and bitbake did not
-       find a conf/bblayers.conf file in the expected location.
+       ERROR: The BBPATH variable is not set and bitbake did not find a conf/bblayers.conf file in the expected location.
        Maybe you accidentally invoked bitbake from the wrong directory?
-       DEBUG: Removed the following variables from the environment:
-       GNOME_DESKTOP_SESSION_ID, XDG_CURRENT_DESKTOP,
-       GNOME_KEYRING_CONTROL, DISPLAY, SSH_AGENT_PID, LANG, no_proxy,
-       XDG_SESSION_PATH, XAUTHORITY, SESSION_MANAGER, SHLVL,
-       MANDATORY_PATH, COMPIZ_CONFIG_PROFILE, WINDOWID, EDITOR,
-       GPG_AGENT_INFO, SSH_AUTH_SOCK, GDMSESSION, GNOME_KEYRING_PID,
-       XDG_SEAT_PATH, XDG_CONFIG_DIRS, LESSOPEN, DBUS_SESSION_BUS_ADDRESS,
-       _, XDG_SESSION_COOKIE, DESKTOP_SESSION, LESSCLOSE, DEFAULTS_PATH,
-       UBUNTU_MENUPROXY, OLDPWD, XDG_DATA_DIRS, COLORTERM, LS_COLORS
-
-    The majority of this output is specific to environment variables that
-    are not directly relevant to BitBake. However, the very first
-    message regarding the ``BBPATH`` variable and the
-    ``conf/bblayers.conf`` file is relevant.
 
     When you run BitBake, it begins looking for metadata files. The
     :term:`BBPATH` variable is what tells BitBake where
-    to look for those files. ``BBPATH`` is not set and you need to set
-    it. Without ``BBPATH``, BitBake cannot find any configuration files
+    to look for those files. :term:`BBPATH` is not set and you need to set
+    it. Without :term:`BBPATH`, BitBake cannot find any configuration files
     (``.conf``) or recipe files (``.bb``) at all. BitBake also cannot
     find the ``bitbake.conf`` file.
 
-#.  **Setting BBPATH:** For this example, you can set ``BBPATH`` in
+#.  **Setting BBPATH:** For this example, you can set :term:`BBPATH` in
     the same manner that you set ``PATH`` earlier in the appendix. You
     should realize, though, that it is much more flexible to set the
-    ``BBPATH`` variable up in a configuration file for each project.
+    :term:`BBPATH` variable up in a configuration file for each project.
 
     From your shell, enter the following commands to set and export the
-    ``BBPATH`` variable: ::
+    :term:`BBPATH` variable::
 
       $ BBPATH="projectdirectory"
       $ export BBPATH
@@ -175,24 +164,18 @@ Following is the complete "Hello World" example.
        ("~") character as BitBake does not expand that character as the
        shell would.
 
-#.  **Run BitBake:** Now that you have ``BBPATH`` defined, run the
-    ``bitbake`` command again: ::
+#.  **Run BitBake:** Now that you have :term:`BBPATH` defined, run the
+    ``bitbake`` command again::
 
        $ bitbake
-       ERROR: Traceback (most recent call last):
-         File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped
-           return func(fn, *args)
-         File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 173, in parse_config_file
-           return bb.parse.handle(fn, data, include)
-         File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 99, in handle
-           return h['handle'](fn, data, include)
-         File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 120, in handle
-           abs_fn = resolve_file(fn, data)
-         File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 117, in resolve_file
-           raise IOError("file %s not found in %s" % (fn, bbpath))
-       IOError: file conf/bitbake.conf not found in /home/scott-lenovo/hello
-
-       ERROR: Unable to parse conf/bitbake.conf: file conf/bitbake.conf not found in /home/scott-lenovo/hello
+       ERROR: Unable to parse /home/scott-lenovo/bitbake/lib/bb/parse/__init__.py
+       Traceback (most recent call last):
+       File "/home/scott-lenovo/bitbake/lib/bb/parse/__init__.py", line 127, in resolve_file(fn='conf/bitbake.conf', d=<bb.data_smart.DataSmart object at 0x7f22919a3df0>):
+             if not newfn:
+       >            raise IOError(errno.ENOENT, "file %s not found in %s" % (fn, bbpath))
+             fn = newfn
+       FileNotFoundError: [Errno 2] file conf/bitbake.conf not found in <projectdirectory>
+
 
     This sample output shows that BitBake could not find the
     ``conf/bitbake.conf`` file in the project directory. This file is
@@ -205,18 +188,18 @@ Following is the complete "Hello World" example.
     recipe files. For this example, you need to create the file in your
     project directory and define some key BitBake variables. For more
     information on the ``bitbake.conf`` file, see
-    http://git.openembedded.org/bitbake/tree/conf/bitbake.conf.
+    https://git.openembedded.org/bitbake/tree/conf/bitbake.conf.
 
     Use the following commands to create the ``conf`` directory in the
-    project directory: ::
+    project directory::
 
       $ mkdir conf
 
     From within the ``conf`` directory,
     use some editor to create the ``bitbake.conf`` so that it contains
-    the following: ::
+    the following::
 
-       PN  = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
+       PN  = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
 
        TMPDIR  = "${TOPDIR}/tmp"
        CACHE   = "${TMPDIR}/cache"
@@ -226,12 +209,12 @@ Following is the complete "Hello World" example.
 
     .. note::
 
-       Without a value for PN , the variables STAMP , T , and B , prevent more
-       than one recipe from working. You can fix this by either setting PN to
+       Without a value for :term:`PN`, the variables :term:`STAMP`, :term:`T`, and :term:`B`, prevent more
+       than one recipe from working. You can fix this by either setting :term:`PN` to
        have a value similar to what OpenEmbedded and BitBake use in the default
-       bitbake.conf file (see previous example). Or, by manually updating each
-       recipe to set PN . You will also need to include PN as part of the STAMP
-       , T , and B variable definitions in the local.conf file.
+       ``bitbake.conf`` file (see previous example). Or, by manually updating each
+       recipe to set :term:`PN`. You will also need to include :term:`PN` as part of the :term:`STAMP`,
+       :term:`T`, and :term:`B` variable definitions in the ``local.conf`` file.
 
     The ``TMPDIR`` variable establishes a directory that BitBake uses
     for build output and intermediate files other than the cached
@@ -251,21 +234,17 @@ Following is the complete "Hello World" example.
     glossary.
 
 #.  **Run BitBake:** After making sure that the ``conf/bitbake.conf`` file
-    exists, you can run the ``bitbake`` command again: ::
+    exists, you can run the ``bitbake`` command again::
 
        $ bitbake
-       ERROR: Traceback (most recent call last):
-         File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 163, in wrapped
-           return func(fn, *args)
-         File "/home/scott-lenovo/bitbake/lib/bb/cookerdata.py", line 177, in _inherit
-           bb.parse.BBHandler.inherit(bbclass, "configuration INHERITs", 0, data)
-         File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 92, in inherit
-           include(fn, file, lineno, d, "inherit")
-         File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/ConfHandler.py", line 100, in include
-           raise ParseError("Could not %(error_out)s file %(fn)s" % vars(), oldfn, lineno)
-       ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
-
-       ERROR: Unable to parse base: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
+       ERROR: Unable to parse /home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py
+       Traceback (most recent call last):
+       File "/home/scott-lenovo/bitbake/lib/bb/parse/parse_py/BBHandler.py", line 67, in inherit(files=['base'], fn='configuration INHERITs', lineno=0, d=<bb.data_smart.DataSmart object at 0x7fab6815edf0>):
+             if not os.path.exists(file):
+       >            raise ParseError("Could not inherit file %s" % (file), fn, lineno)
+
+       bb.parse.ParseError: ParseError in configuration INHERITs: Could not inherit file classes/base.bbclass
+
 
     In the sample output,
     BitBake could not find the ``classes/base.bbclass`` file. You need
@@ -278,20 +257,23 @@ Following is the complete "Hello World" example.
     in the ``classes`` directory of the project (i.e ``hello/classes``
     in this example).
 
-    Create the ``classes`` directory as follows: ::
+    Create the ``classes`` directory as follows::
 
       $ cd $HOME/hello
       $ mkdir classes
 
     Move to the ``classes`` directory and then create the
-    ``base.bbclass`` file by inserting this single line: addtask build
+    ``base.bbclass`` file by inserting this single line::
+
+      addtask build
+
     The minimal task that BitBake runs is the ``do_build`` task. This is
     all the example needs in order to build the project. Of course, the
     ``base.bbclass`` can have much more depending on which build
     environments BitBake is supporting.
 
 #.  **Run BitBake:** After making sure that the ``classes/base.bbclass``
-    file exists, you can run the ``bitbake`` command again: ::
+    file exists, you can run the ``bitbake`` command again::
 
        $ bitbake
        Nothing to do. Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.
@@ -314,7 +296,7 @@ Following is the complete "Hello World" example.
     Minimally, you need a recipe file and a layer configuration file in
     your layer. The configuration file needs to be in the ``conf``
     directory inside the layer. Use these commands to set up the layer
-    and the ``conf`` directory: ::
+    and the ``conf`` directory::
 
        $ cd $HOME
        $ mkdir mylayer
@@ -322,20 +304,29 @@ Following is the complete "Hello World" example.
        $ mkdir conf
 
     Move to the ``conf`` directory and create a ``layer.conf`` file that has the
-    following: ::
+    following::
 
       BBPATH .= ":${LAYERDIR}"
-      BBFILES += "${LAYERDIR}/\*.bb"
+      BBFILES += "${LAYERDIR}/*.bb"
       BBFILE_COLLECTIONS += "mylayer"
-     `BBFILE_PATTERN_mylayer := "^${LAYERDIR_RE}/"
+      BBFILE_PATTERN_mylayer := "^${LAYERDIR_RE}/"
+      LAYERSERIES_CORENAMES = "hello_world_example"
+      LAYERSERIES_COMPAT_mylayer = "hello_world_example"
 
     For information on these variables, click on :term:`BBFILES`,
-    :term:`LAYERDIR`, :term:`BBFILE_COLLECTIONS` or :term:`BBFILE_PATTERN_mylayer <BBFILE_PATTERN>`
-    to go to the definitions in the glossary.
+    :term:`LAYERDIR`, :term:`BBFILE_COLLECTIONS`, :term:`BBFILE_PATTERN_mylayer <BBFILE_PATTERN>`
+    or :term:`LAYERSERIES_COMPAT` to go to the definitions in the glossary.
+
+    .. note::
+
+       We are setting both ``LAYERSERIES_CORENAMES`` and :term:`LAYERSERIES_COMPAT` in this particular case, because we
+       are using bitbake without OpenEmbedded.
+       You should usually just use :term:`LAYERSERIES_COMPAT` to specify the OE-Core versions for which your layer
+       is compatible, and add the meta-openembedded layer to your project.
 
     You need to create the recipe file next. Inside your layer at the
     top-level, use an editor and create a recipe file named
-    ``printhello.bb`` that has the following: ::
+    ``printhello.bb`` that has the following::
 
        DESCRIPTION = "Prints Hello World"
        PN = 'printhello'
@@ -356,7 +347,7 @@ Following is the complete "Hello World" example.
     follow the links to the glossary.
 
 #. **Run BitBake With a Target:** Now that a BitBake target exists, run
-    the command and provide that target: ::
+    the command and provide that target::
 
       $ cd $HOME/hello
       $ bitbake printhello
@@ -376,7 +367,7 @@ Following is the complete "Hello World" example.
     ``hello/conf`` for this example).
 
     Set your working directory to the ``hello/conf`` directory and then
-    create the ``bblayers.conf`` file so that it contains the following: ::
+    create the ``bblayers.conf`` file so that it contains the following::
 
        BBLAYERS ?= " \
            /home/<you>/mylayer \
@@ -386,15 +377,17 @@ Following is the complete "Hello World" example.
 
 #. **Run BitBake With a Target:** Now that you have supplied the
     ``bblayers.conf`` file, run the ``bitbake`` command and provide the
-    target: ::
+    target::
 
        $ bitbake printhello
+       Loading cache: 100% |
+       Loaded 0 entries from dependency cache.
        Parsing recipes: 100% |##################################################################################|
-       Time: 00:00:00
        Parsing of 1 .bb files complete (0 cached, 1 parsed). 1 targets, 0 skipped, 0 masked, 0 errors.
        NOTE: Resolving any missing task queue dependencies
-       NOTE: Preparing RunQueue
-       NOTE: Executing RunQueue Tasks
+       Initialising tasks: 100% |###############################################################################|
+       NOTE: No setscene tasks
+       NOTE: Executing Tasks
        ********************
        *                  *
        *  Hello, World!   *
diff --git a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst
index 6f9d392935..539bb62d81 100644
--- a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst
+++ b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-intro.rst
@@ -27,7 +27,7 @@ Linux software stacks using a task-oriented approach.
 Conceptually, BitBake is similar to GNU Make in some regards but has
 significant differences:
 
--  BitBake executes tasks according to provided metadata that builds up
+-  BitBake executes tasks according to the provided metadata that builds up
    the tasks. Metadata is stored in recipe (``.bb``) and related recipe
    "append" (``.bbappend``) files, configuration (``.conf``) and
    underlying include (``.inc``) files, and in class (``.bbclass``)
@@ -60,11 +60,10 @@ member Chris Larson split the project into two distinct pieces:
 -  OpenEmbedded, a metadata set utilized by BitBake
 
 Today, BitBake is the primary basis of the
-`OpenEmbedded <http://www.openembedded.org/>`__ project, which is being
-used to build and maintain Linux distributions such as the `Angstrom
-Distribution <http://www.angstrom-distribution.org/>`__, and which is
-also being used as the build tool for Linux projects such as the `Yocto
-Project <http://www.yoctoproject.org>`__.
+`OpenEmbedded <https://www.openembedded.org/>`__ project, which is being
+used to build and maintain Linux distributions such as the `Poky
+Reference Distribution <https://www.yoctoproject.org/software-item/poky/>`__,
+developed under the umbrella of the `Yocto Project <https://www.yoctoproject.org>`__.
 
 Prior to BitBake, no other build tool adequately met the needs of an
 aspiring embedded Linux distribution. All of the build systems used by
@@ -248,13 +247,13 @@ underlying, similarly-named recipe files.
 
 When you name an append file, you can use the "``%``" wildcard character
 to allow for matching recipe names. For example, suppose you have an
-append file named as follows: ::
+append file named as follows::
 
   busybox_1.21.%.bbappend
 
 That append file
 would match any ``busybox_1.21.``\ x\ ``.bb`` version of the recipe. So,
-the append file would match the following recipe names: ::
+the append file would match the following recipe names::
 
   busybox_1.21.1.bb
   busybox_1.21.2.bb
@@ -290,7 +289,7 @@ You can obtain BitBake several different ways:
    are using. The metadata is generally backwards compatible but not
    forward compatible.
 
-   Here is an example that clones the BitBake repository: ::
+   Here is an example that clones the BitBake repository::
 
      $ git clone git://git.openembedded.org/bitbake
 
@@ -298,7 +297,7 @@ You can obtain BitBake several different ways:
    Git repository into a directory called ``bitbake``. Alternatively,
    you can designate a directory after the ``git clone`` command if you
    want to call the new directory something other than ``bitbake``. Here
-   is an example that names the directory ``bbdev``: ::
+   is an example that names the directory ``bbdev``::
 
      $ git clone git://git.openembedded.org/bitbake bbdev
 
@@ -317,9 +316,9 @@ You can obtain BitBake several different ways:
          method for getting BitBake. Cloning the repository makes it easier
          to update as patches are added to the stable branches.
 
-   The following example downloads a snapshot of BitBake version 1.17.0: ::
+   The following example downloads a snapshot of BitBake version 1.17.0::
 
-     $ wget http://git.openembedded.org/bitbake/snapshot/bitbake-1.17.0.tar.gz
+     $ wget https://git.openembedded.org/bitbake/snapshot/bitbake-1.17.0.tar.gz
      $ tar zxpvf bitbake-1.17.0.tar.gz
 
    After extraction of the tarball using
@@ -347,43 +346,87 @@ execution examples.
 Usage and syntax
 ----------------
 
-Following is the usage and syntax for BitBake: ::
+Following is the usage and syntax for BitBake::
 
    $ bitbake -h
-   Usage: bitbake [options] [recipename/target recipe:do_task ...]
-
-       Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
-       It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
-       will provide the layer, BBFILES and other configuration information.
+   usage: bitbake [-s] [-e] [-g] [-u UI] [--version] [-h] [-f] [-c CMD]
+                  [-C INVALIDATE_STAMP] [--runall RUNALL] [--runonly RUNONLY]
+                  [--no-setscene] [--skip-setscene] [--setscene-only] [-n] [-p]
+                  [-k] [-P] [-S SIGNATURE_HANDLER] [--revisions-changed]
+                  [-b BUILDFILE] [-D] [-l DEBUG_DOMAINS] [-v] [-q]
+                  [-w WRITEEVENTLOG] [-B BIND] [-T SERVER_TIMEOUT]
+                  [--remote-server REMOTE_SERVER] [-m] [--token XMLRPCTOKEN]
+                  [--observe-only] [--status-only] [--server-only] [-r PREFILE]
+                  [-R POSTFILE] [-I EXTRA_ASSUME_PROVIDED]
+                  [recipename/target ...]
+
+   It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH
+   which will provide the layer, BBFILES and other configuration information.
+
+   General options:
+     recipename/target     Execute the specified task (default is 'build') for
+                           these target recipes (.bb files).
+     -s, --show-versions   Show current and preferred versions of all recipes.
+     -e, --environment     Show the global or per-recipe environment complete
+                           with information about where variables were
+                           set/changed.
+     -g, --graphviz        Save dependency tree information for the specified
+                           targets in the dot syntax.
+     -u UI, --ui UI        The user interface to use (knotty, ncurses, taskexp,
+                           taskexp_ncurses or teamcity - default knotty).
+     --version             Show programs version and exit.
+     -h, --help            Show this help message and exit.
 
-   Options:
-     --version             show program's version number and exit
-     -h, --help            show this help message and exit
-     -b BUILDFILE, --buildfile=BUILDFILE
-                           Execute tasks from a specific .bb recipe directly.
-                           WARNING: Does not handle any dependencies from other
-                           recipes.
-     -k, --continue        Continue as much as possible after an error. While the
-                           target that failed and anything depending on it cannot
-                           be built, as much as possible will be built before
-                           stopping.
+   Task control options:
      -f, --force           Force the specified targets/task to run (invalidating
                            any existing stamp file).
-     -c CMD, --cmd=CMD     Specify the task to execute. The exact options
+     -c CMD, --cmd CMD     Specify the task to execute. The exact options
                            available depend on the metadata. Some examples might
                            be 'compile' or 'populate_sysroot' or 'listtasks' may
                            give a list of the tasks available.
-     -C INVALIDATE_STAMP, --clear-stamp=INVALIDATE_STAMP
+     -C INVALIDATE_STAMP, --clear-stamp INVALIDATE_STAMP
                            Invalidate the stamp for the specified task such as
                            'compile' and then run the default task for the
                            specified target(s).
-     -r PREFILE, --read=PREFILE
-                           Read the specified file before bitbake.conf.
-     -R POSTFILE, --postread=POSTFILE
-                           Read the specified file after bitbake.conf.
-     -v, --verbose         Enable tracing of shell tasks (with 'set -x'). Also
-                           print bb.note(...) messages to stdout (in addition to
-                           writing them to ${T}/log.do_&lt;task&gt;).
+     --runall RUNALL       Run the specified task for any recipe in the taskgraph
+                           of the specified target (even if it wouldn't otherwise
+                           have run).
+     --runonly RUNONLY     Run only the specified task within the taskgraph of
+                           the specified targets (and any task dependencies those
+                           tasks may have).
+     --no-setscene         Do not run any setscene tasks. sstate will be ignored
+                           and everything needed, built.
+     --skip-setscene       Skip setscene tasks if they would be executed. Tasks
+                           previously restored from sstate will be kept, unlike
+                           --no-setscene.
+     --setscene-only       Only run setscene tasks, don't run any real tasks.
+
+   Execution control options:
+     -n, --dry-run         Don't execute, just go through the motions.
+     -p, --parse-only      Quit after parsing the BB recipes.
+     -k, --continue        Continue as much as possible after an error. While the
+                           target that failed and anything depending on it cannot
+                           be built, as much as possible will be built before
+                           stopping.
+     -P, --profile         Profile the command and save reports.
+     -S SIGNATURE_HANDLER, --dump-signatures SIGNATURE_HANDLER
+                           Dump out the signature construction information, with
+                           no task execution. The SIGNATURE_HANDLER parameter is
+                           passed to the handler. Two common values are none and
+                           printdiff but the handler may define more/less. none
+                           means only dump the signature, printdiff means
+                           recursively compare the dumped signature with the most
+                           recent one in a local build or sstate cache (can be
+                           used to find out why tasks re-run when that is not
+                           expected)
+     --revisions-changed   Set the exit code depending on whether upstream
+                           floating revisions have changed or not.
+     -b BUILDFILE, --buildfile BUILDFILE
+                           Execute tasks from a specific .bb recipe directly.
+                           WARNING: Does not handle any dependencies from other
+                           recipes.
+
+   Logging/output control options:
      -D, --debug           Increase the debug level. You can specify this more
                            than once. -D sets the debug level to 1, where only
                            bb.debug(1, ...) messages are printed to stdout; -DD
@@ -393,62 +436,47 @@ Following is the usage and syntax for BitBake: ::
                            -D only affects output to stdout. All debug messages
                            are written to ${T}/log.do_taskname, regardless of the
                            debug level.
+     -l DEBUG_DOMAINS, --log-domains DEBUG_DOMAINS
+                           Show debug logging for the specified logging domains.
+     -v, --verbose         Enable tracing of shell tasks (with 'set -x'). Also
+                           print bb.note(...) messages to stdout (in addition to
+                           writing them to ${T}/log.do_<task>).
      -q, --quiet           Output less log message data to the terminal. You can
                            specify this more than once.
-     -n, --dry-run         Don't execute, just go through the motions.
-     -S SIGNATURE_HANDLER, --dump-signatures=SIGNATURE_HANDLER
-                           Dump out the signature construction information, with
-                           no task execution. The SIGNATURE_HANDLER parameter is
-                           passed to the handler. Two common values are none and
-                           printdiff but the handler may define more/less. none
-                           means only dump the signature, printdiff means compare
-                           the dumped signature with the cached one.
-     -p, --parse-only      Quit after parsing the BB recipes.
-     -s, --show-versions   Show current and preferred versions of all recipes.
-     -e, --environment     Show the global or per-recipe environment complete
-                           with information about where variables were
-                           set/changed.
-     -g, --graphviz        Save dependency tree information for the specified
-                           targets in the dot syntax.
-     -I EXTRA_ASSUME_PROVIDED, --ignore-deps=EXTRA_ASSUME_PROVIDED
-                           Assume these dependencies don't exist and are already
-                           provided (equivalent to ASSUME_PROVIDED). Useful to
-                           make dependency graphs more appealing
-     -l DEBUG_DOMAINS, --log-domains=DEBUG_DOMAINS
-                           Show debug logging for the specified logging domains
-     -P, --profile         Profile the command and save reports.
-     -u UI, --ui=UI        The user interface to use (knotty, ncurses or taskexp
-                           - default knotty).
-     --token=XMLRPCTOKEN   Specify the connection token to be used when
-                           connecting to a remote server.
-     --revisions-changed   Set the exit code depending on whether upstream
-                           floating revisions have changed or not.
-     --server-only         Run bitbake without a UI, only starting a server
-                           (cooker) process.
-     -B BIND, --bind=BIND  The name/address for the bitbake xmlrpc server to bind
+     -w WRITEEVENTLOG, --write-log WRITEEVENTLOG
+                           Writes the event log of the build to a bitbake event
+                           json file. Use '' (empty string) to assign the name
+                           automatically.
+
+   Server options:
+     -B BIND, --bind BIND  The name/address for the bitbake xmlrpc server to bind
                            to.
-     -T SERVER_TIMEOUT, --idle-timeout=SERVER_TIMEOUT
+     -T SERVER_TIMEOUT, --idle-timeout SERVER_TIMEOUT
                            Set timeout to unload bitbake server due to
                            inactivity, set to -1 means no unload, default:
                            Environment variable BB_SERVER_TIMEOUT.
-     --no-setscene         Do not run any setscene tasks. sstate will be ignored
-                           and everything needed, built.
-     --setscene-only       Only run setscene tasks, don't run any real tasks.
-     --remote-server=REMOTE_SERVER
+     --remote-server REMOTE_SERVER
                            Connect to the specified server.
      -m, --kill-server     Terminate any running bitbake server.
+     --token XMLRPCTOKEN   Specify the connection token to be used when
+                           connecting to a remote server.
      --observe-only        Connect to a server as an observing-only client.
      --status-only         Check the status of the remote bitbake server.
-     -w WRITEEVENTLOG, --write-log=WRITEEVENTLOG
-                           Writes the event log of the build to a bitbake event
-                           json file. Use '' (empty string) to assign the name
-                           automatically.
-     --runall=RUNALL       Run the specified task for any recipe in the taskgraph
-                           of the specified target (even if it wouldn't otherwise
-                           have run).
-     --runonly=RUNONLY     Run only the specified task within the taskgraph of
-                           the specified targets (and any task dependencies those
-                           tasks may have).
+     --server-only         Run bitbake without a UI, only starting a server
+                           (cooker) process.
+
+   Configuration options:
+     -r PREFILE, --read PREFILE
+                           Read the specified file before bitbake.conf.
+     -R POSTFILE, --postread POSTFILE
+                           Read the specified file after bitbake.conf.
+     -I EXTRA_ASSUME_PROVIDED, --ignore-deps EXTRA_ASSUME_PROVIDED
+                           Assume these dependencies don't exist and are already
+                           provided (equivalent to ASSUME_PROVIDED). Useful to
+                           make dependency graphs more appealing.
+
+..
+    Bitbake help output generated with "stty columns 80; bin/bitbake -h"
 
 .. _bitbake-examples:
 
@@ -469,11 +497,11 @@ default task, which is "build". BitBake obeys inter-task dependencies
 when doing so.
 
 The following command runs the build task, which is the default task, on
-the ``foo_1.0.bb`` recipe file: ::
+the ``foo_1.0.bb`` recipe file::
 
   $ bitbake -b foo_1.0.bb
 
-The following command runs the clean task on the ``foo.bb`` recipe file: ::
+The following command runs the clean task on the ``foo.bb`` recipe file::
 
   $ bitbake -b foo.bb -c clean
 
@@ -497,13 +525,13 @@ functionality, or when there are multiple versions of a recipe.
 The ``bitbake`` command, when not using "--buildfile" or "-b" only
 accepts a "PROVIDES". You cannot provide anything else. By default, a
 recipe file generally "PROVIDES" its "packagename" as shown in the
-following example: ::
+following example::
 
   $ bitbake foo
 
 This next example "PROVIDES" the
 package name and also uses the "-c" option to tell BitBake to just
-execute the ``do_clean`` task: ::
+execute the ``do_clean`` task::
 
   $ bitbake -c clean foo
 
@@ -514,7 +542,7 @@ The BitBake command line supports specifying different tasks for
 individual targets when you specify multiple targets. For example,
 suppose you had two targets (or recipes) ``myfirstrecipe`` and
 ``mysecondrecipe`` and you needed BitBake to run ``taskA`` for the first
-recipe and ``taskB`` for the second recipe: ::
+recipe and ``taskB`` for the second recipe::
 
   $ bitbake myfirstrecipe:do_taskA mysecondrecipe:do_taskB
 
@@ -534,13 +562,13 @@ current working directory:
 -  ``pn-buildlist``: Shows a simple list of targets that are to be
    built.
 
-To stop depending on common depends, use the "-I" depend option and
+To stop depending on common depends, use the ``-I`` depend option and
 BitBake omits them from the graph. Leaving this information out can
 produce more readable graphs. This way, you can remove from the graph
-``DEPENDS`` from inherited classes such as ``base.bbclass``.
+:term:`DEPENDS` from inherited classes such as ``base.bbclass``.
 
 Here are two examples that create dependency graphs. The second example
-omits depends common in OpenEmbedded from the graph: ::
+omits depends common in OpenEmbedded from the graph::
 
   $ bitbake -g foo
 
@@ -564,7 +592,7 @@ for two separate targets:
 .. image:: figures/bb_multiconfig_files.png
    :align: center
 
-The reason for this required file hierarchy is because the ``BBPATH``
+The reason for this required file hierarchy is because the :term:`BBPATH`
 variable is not constructed until the layers are parsed. Consequently,
 using the configuration file as a pre-configuration file is not possible
 unless it is located in the current working directory.
@@ -582,17 +610,17 @@ accomplished by setting the
 configuration files for ``target1`` and ``target2`` defined in the build
 directory. The following statement in the ``local.conf`` file both
 enables BitBake to perform multiple configuration builds and specifies
-the two extra multiconfigs: ::
+the two extra multiconfigs::
 
   BBMULTICONFIG = "target1 target2"
 
 Once the target configuration files are in place and BitBake has been
 enabled to perform multiple configuration builds, use the following
-command form to start the builds: ::
+command form to start the builds::
 
   $ bitbake [mc:multiconfigname:]target [[[mc:multiconfigname:]target] ... ]
 
-Here is an example for two extra multiconfigs: ``target1`` and ``target2``: ::
+Here is an example for two extra multiconfigs: ``target1`` and ``target2``::
 
   $ bitbake mc::target mc:target1:target mc:target2:target
 
@@ -613,12 +641,12 @@ multiconfig.
 
 To enable dependencies in a multiple configuration build, you must
 declare the dependencies in the recipe using the following statement
-form: ::
+form::
 
   task_or_package[mcdepends] = "mc:from_multiconfig:to_multiconfig:recipe_name:task_on_which_to_depend"
 
 To better show how to use this statement, consider an example with two
-multiconfigs: ``target1`` and ``target2``: ::
+multiconfigs: ``target1`` and ``target2``::
 
   image_task[mcdepends] = "mc:target1:target2:image2:rootfs_task"
 
@@ -629,7 +657,7 @@ completion of the rootfs_task used to build out image2, which is
 associated with the "target2" multiconfig.
 
 Once you set up this dependency, you can build the "target1" multiconfig
-using a BitBake command as follows: ::
+using a BitBake command as follows::
 
   $ bitbake mc:target1:image1
 
@@ -639,7 +667,7 @@ the ``rootfs_task`` for the "target2" multiconfig build.
 
 Having a recipe depend on the root filesystem of another build might not
 seem that useful. Consider this change to the statement in the image1
-recipe: ::
+recipe::
 
   image_task[mcdepends] = "mc:target1:target2:image2:image_task"
 
diff --git a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-library-functions.rst b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-library-functions.rst
new file mode 100644
index 0000000000..09e353945b
--- /dev/null
+++ b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-library-functions.rst
@@ -0,0 +1,59 @@
+.. SPDX-License-Identifier: CC-BY-2.5
+
+=================
+Library Functions
+=================
+
+|
+
+This chapter lists common library functions available under the ``lib/``
+directory in BitBake.
+
+These functions can be used in recipes or configuration files with
+:ref:`inline-Python <bitbake-user-manual/bitbake-user-manual-metadata:Inline
+Python Variable Expansion>` or :ref:`Python
+<bitbake-user-manual/bitbake-user-manual-metadata:BitBake-Style Python
+Functions>` functions.
+
+Logging utilities
+=================
+
+Different logging utilities can be used from Python code in recipes or
+configuration files.
+
+The strings passed below can be formatted with ``str.format()``, for example::
+
+   bb.warn("Houston, we have a %s", "bit of a problem")
+
+Formatted string can also be used directly::
+
+   bb.error("%s, we have a %s" % ("Houston", "big problem"))
+
+Python f-strings may also be used::
+
+   h = "Houston"
+   bb.fatal(f"{h}, we have a critical problem")
+
+.. automodule:: bb
+   :members:
+      debug,
+      error,
+      erroronce,
+      fatal,
+      note,
+      plain,
+      verbnote,
+      warn,
+      warnonce,
+
+``bb.utils``
+============
+
+.. automodule:: bb.utils
+   :members:
+   :exclude-members:
+      LogCatcher,
+      PrCtlError,
+      VersionStringException,
+      better_compile,
+      better_exec,
diff --git a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst
index 7ea68ade72..f60a9d8312 100644
--- a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst
+++ b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-metadata.rst
@@ -26,7 +26,7 @@ assignment. ::
    VARIABLE = "value"
 
 As expected, if you include leading or
-trailing spaces as part of an assignment, the spaces are retained: ::
+trailing spaces as part of an assignment, the spaces are retained::
 
    VARIABLE = " value"
    VARIABLE = "value "
@@ -40,7 +40,7 @@ blank space (i.e. these are not the same values). ::
 
 You can use single quotes instead of double quotes when setting a
 variable's value. Doing so allows you to use values that contain the
-double quote character: ::
+double quote character::
 
    VARIABLE = 'I have a " in my value'
 
@@ -77,7 +77,7 @@ occurs, you can use BitBake to check the actual value of the suspect
 variable. You can make these checks for both configuration and recipe
 level changes:
 
--  For configuration changes, use the following: ::
+-  For configuration changes, use the following::
 
       $ bitbake -e
 
@@ -91,9 +91,10 @@ level changes:
       Variables that are exported to the environment are preceded by the
       string "export" in the command's output.
 
--  For recipe changes, use the following: ::
+-  To find changes to a given variable in a specific recipe, use the
+   following::
 
-      $ bitbake recipe -e \| grep VARIABLE="
+      $ bitbake recipename -e | grep VARIABLENAME=\"
 
    This command checks to see if the variable actually makes
    it into a specific recipe.
@@ -103,20 +104,20 @@ Line Joining
 
 Outside of :ref:`functions <bitbake-user-manual/bitbake-user-manual-metadata:functions>`,
 BitBake joins any line ending in
-a backslash character ("\") with the following line before parsing
-statements. The most common use for the "\" character is to split
-variable assignments over multiple lines, as in the following example: ::
+a backslash character ("\\") with the following line before parsing
+statements. The most common use for the "\\" character is to split
+variable assignments over multiple lines, as in the following example::
 
    FOO = "bar \
           baz \
           qaz"
 
-Both the "\" character and the newline
+Both the "\\" character and the newline
 character that follow it are removed when joining lines. Thus, no
 newline characters end up in the value of ``FOO``.
 
 Consider this additional example where the two assignments both assign
-"barbaz" to ``FOO``: ::
+"barbaz" to ``FOO``::
 
    FOO = "barbaz"
    FOO = "bar\
@@ -124,7 +125,7 @@ Consider this additional example where the two assignments both assign
 
 .. note::
 
-   BitBake does not interpret escape sequences like "\n" in variable
+   BitBake does not interpret escape sequences like "\\n" in variable
    values. For these to have an effect, the value must be passed to some
    utility that interprets escape sequences, such as
    ``printf`` or ``echo -n``.
@@ -149,7 +150,7 @@ The "=" operator does not immediately expand variable references in the
 right-hand side. Instead, expansion is deferred until the variable
 assigned to is actually used. The result depends on the current values
 of the referenced variables. The following example should clarify this
-behavior: ::
+behavior::
 
    A = "${B} baz"
    B = "${C} bar"
@@ -158,7 +159,7 @@ behavior: ::
    C = "qux"
    *At this point, ${A} equals "qux bar baz"*
    B = "norf"
-   *At this point, ${A} equals "norf baz"\*
+   *At this point, ${A} equals "norf baz"*
 
 Contrast this behavior with the
 :ref:`bitbake-user-manual/bitbake-user-manual-metadata:immediate variable
@@ -177,7 +178,7 @@ Setting a default value (?=)
 You can use the "?=" operator to achieve a "softer" assignment for a
 variable. This type of assignment allows you to define a variable if it
 is undefined when the statement is parsed, but to leave the value alone
-if the variable has a value. Here is an example: ::
+if the variable has a value. Here is an example::
 
    A ?= "aval"
 
@@ -194,28 +195,51 @@ value. However, if ``A`` is not set, the variable is set to "aval".
 Setting a weak default value (??=)
 ----------------------------------
 
-It is possible to use a "weaker" assignment than in the previous section
-by using the "??=" operator. This assignment behaves identical to "?="
-except that the assignment is made at the end of the parsing process
-rather than immediately. Consequently, when multiple "??=" assignments
-exist, the last one is used. Also, any "=" or "?=" assignment will
-override the value set with "??=". Here is an example: ::
+The weak default value of a variable is the value which that variable
+will expand to if no value has been assigned to it via any of the other
+assignment operators. The "??=" operator takes effect immediately, replacing
+any previously defined weak default value. Here is an example::
 
-   A ??= "somevalue"
-   A ??= "someothervalue"
+   W ??= "x"
+   A := "${W}" # Immediate variable expansion
+   W ??= "y"
+   B := "${W}" # Immediate variable expansion
+   W ??= "z"
+   C = "${W}"
+   W ?= "i"
 
-If ``A`` is set before the above statements are
-parsed, the variable retains its value. If ``A`` is not set, the
-variable is set to "someothervalue".
+After parsing we will have::
 
-Again, this assignment is a "lazy" or "weak" assignment because it does
-not occur until the end of the parsing process.
+   A = "x"
+   B = "y"
+   C = "i"
+   W = "i"
+
+Appending and prepending non-override style will not substitute the weak
+default value, which means that after parsing::
+
+   W ??= "x"
+   W += "y"
+
+we will have::
+
+   W = " y"
+
+On the other hand, override-style appends/prepends/removes are applied after
+any active weak default value has been substituted::
+
+   W ??= "x"
+   W:append = "y"
+
+After parsing we will have::
+
+   W = "xy"
 
 Immediate variable expansion (:=)
 ---------------------------------
 
 The ":=" operator results in a variable's contents being expanded
-immediately, rather than when the variable is actually used: ::
+immediately, rather than when the variable is actually used::
 
    T = "123"
    A := "test ${T}"
@@ -225,7 +249,7 @@ immediately, rather than when the variable is actually used: ::
    C := "${C}append"
 
 In this example, ``A`` contains "test 123", even though the final value
-of ``T`` is "456". The variable ``B`` will end up containing "456
+of :term:`T` is "456". The variable :term:`B` will end up containing "456
 cvalappend". This is because references to undefined variables are
 preserved as is during (immediate)expansion. This is in contrast to GNU
 Make, where undefined variables expand to nothing. The variable ``C``
@@ -241,14 +265,14 @@ the "+=" and "=+" operators. These operators insert a space between the
 current value and prepended or appended value.
 
 These operators take immediate effect during parsing. Here are some
-examples: ::
+examples::
 
    B = "bval"
    B += "additionaldata"
    C = "cval"
    C =+ "test"
 
-The variable ``B`` contains "bval additionaldata" and ``C`` contains "test
+The variable :term:`B` contains "bval additionaldata" and ``C`` contains "test
 cval".
 
 .. _appending-and-prepending-without-spaces:
@@ -260,14 +284,14 @@ If you want to append or prepend values without an inserted space, use
 the ".=" and "=." operators.
 
 These operators take immediate effect during parsing. Here are some
-examples: ::
+examples::
 
    B = "bval"
    B .= "additionaldata"
    C = "cval"
    C =. "test"
 
-The variable ``B`` contains "bvaladditionaldata" and ``C`` contains
+The variable :term:`B` contains "bvaladditionaldata" and ``C`` contains
 "testcval".
 
 Appending and Prepending (Override Style Syntax)
@@ -278,16 +302,16 @@ style syntax. When you use this syntax, no spaces are inserted.
 
 These operators differ from the ":=", ".=", "=.", "+=", and "=+"
 operators in that their effects are applied at variable expansion time
-rather than being immediately applied. Here are some examples: ::
+rather than being immediately applied. Here are some examples::
 
    B = "bval"
-   B_append = " additional data"
+   B:append = " additional data"
    C = "cval"
-   C_prepend = "additional data "
+   C:prepend = "additional data "
    D = "dval"
-   D_append = "additional data"
+   D:append = "additional data"
 
-The variable ``B``
+The variable :term:`B`
 becomes "bval additional data" and ``C`` becomes "additional data cval".
 The variable ``D`` becomes "dvaladditional data".
 
@@ -295,6 +319,10 @@ The variable ``D`` becomes "dvaladditional data".
 
    You must control all spacing when you use the override syntax.
 
+.. note::
+
+   The overrides are applied in this order, ":append", ":prepend", ":remove".
+
 It is also possible to append and prepend to shell functions and
 BitBake-style Python functions. See the ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:shell functions`" and ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:bitbake-style python functions`"
 sections for examples.
@@ -306,16 +334,17 @@ Removal (Override Style Syntax)
 
 You can remove values from lists using the removal override style
 syntax. Specifying a value for removal causes all occurrences of that
-value to be removed from the variable.
+value to be removed from the variable. Unlike ":append" and ":prepend",
+there is no need to add a leading or trailing space to the value.
 
 When you use this syntax, BitBake expects one or more strings.
-Surrounding spaces and spacing are preserved. Here is an example: ::
+Surrounding spaces and spacing are preserved. Here is an example::
 
    FOO = "123 456 789 123456 123 456 123 456"
-   FOO_remove = "123"
-   FOO_remove = "456"
+   FOO:remove = "123"
+   FOO:remove = "456"
    FOO2 = " abc def ghi abcdef abc def abc def def"
-   FOO2_remove = "\
+   FOO2:remove = "\
        def \
        abc \
        ghi \
@@ -324,40 +353,62 @@ Surrounding spaces and spacing are preserved. Here is an example: ::
 The variable ``FOO`` becomes
 "  789 123456    " and ``FOO2`` becomes "    abcdef     ".
 
-Like "_append" and "_prepend", "_remove" is applied at variable
+Like ":append" and ":prepend", ":remove" is applied at variable
 expansion time.
 
+.. note::
+
+   The overrides are applied in this order, ":append", ":prepend", ":remove".
+   This implies it is not possible to re-append previously removed strings.
+   However, one can undo a ":remove" by using an intermediate variable whose
+   content is passed to the ":remove" so that modifying the intermediate
+   variable equals to keeping the string in::
+
+     FOOREMOVE = "123 456 789"
+     FOO:remove = "${FOOREMOVE}"
+     ...
+     FOOREMOVE = "123 789"
+
+   This expands to ``FOO:remove = "123 789"``.
+
+.. note::
+
+   Override application order may not match variable parse history, i.e.
+   the output of ``bitbake -e`` may contain ":remove" before ":append",
+   but the result will be removed string, because ":remove" is handled
+   last.
+
 Override Style Operation Advantages
 -----------------------------------
 
-An advantage of the override style operations "_append", "_prepend", and
-"_remove" as compared to the "+=" and "=+" operators is that the
+An advantage of the override style operations ":append", ":prepend", and
+":remove" as compared to the "+=" and "=+" operators is that the
 override style operators provide guaranteed operations. For example,
 consider a class ``foo.bbclass`` that needs to add the value "val" to
-the variable ``FOO``, and a recipe that uses ``foo.bbclass`` as follows: ::
+the variable ``FOO``, and a recipe that uses ``foo.bbclass`` as follows::
 
    inherit foo
    FOO = "initial"
 
 If ``foo.bbclass`` uses the "+=" operator,
 as follows, then the final value of ``FOO`` will be "initial", which is
-not what is desired: ::
+not what is desired::
 
    FOO += "val"
 
 If, on the other hand, ``foo.bbclass``
-uses the "_append" operator, then the final value of ``FOO`` will be
-"initial val", as intended: ::
+uses the ":append" operator, then the final value of ``FOO`` will be
+"initial val", as intended::
 
-   FOO_append = " val"
+   FOO:append = " val"
 
 .. note::
 
-   It is never necessary to use "+=" together with "_append". The following
-   sequence of assignments appends "barbaz" to FOO: ::
+   It is never necessary to use "+=" together with ":append". The following
+   sequence of assignments appends "barbaz" to FOO::
 
-       FOO_append = "bar"
-       FOO_append = "baz"
+       FOO:append = "bar"
+       FOO:append = "baz"
 
 
    The only effect of changing the second assignment in the previous
@@ -378,10 +429,10 @@ You can find more out about variable flags in general in the
 
 You can define, append, and prepend values to variable flags. All the
 standard syntax operations previously mentioned work for variable flags
-except for override style syntax (i.e. "_prepend", "_append", and
-"_remove").
+except for override style syntax (i.e. ":prepend", ":append", and
+":remove").
 
-Here are some examples showing how to set variable flags: ::
+Here are some examples showing how to set variable flags::
 
    FOO[a] = "abc"
    FOO[b] = "123"
@@ -393,15 +444,21 @@ respectively. The ``[a]`` flag becomes "abc 456".
 
 No need exists to pre-define variable flags. You can simply start using
 them. One extremely common application is to attach some brief
-documentation to a BitBake variable as follows: ::
+documentation to a BitBake variable as follows::
 
    CACHE[doc] = "The directory holding the cache of the metadata."
 
+.. note::
+
+   Variable flag names starting with an underscore (``_``) character
+   are allowed but are ignored by ``d.getVarFlags("VAR")``
+   in Python code. Such flag names are used internally by BitBake.
+
 Inline Python Variable Expansion
 --------------------------------
 
 You can use inline Python variable expansion to set variables. Here is
-an example: ::
+an example::
 
    DATE = "${@time.strftime('%Y%m%d',time.gmtime())}"
 
@@ -410,21 +467,21 @@ This example results in the ``DATE`` variable being set to the current date.
 Probably the most common use of this feature is to extract the value of
 variables from BitBake's internal data dictionary, ``d``. The following
 lines select the values of a package name and its version number,
-respectively: ::
+respectively::
 
-   PN = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
-   PV = "${@bb.parse.BBHandler.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
+   PN = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[0] or 'defaultpkgname'}"
+   PV = "${@bb.parse.vars_from_file(d.getVar('FILE', False),d)[1] or '1.0'}"
 
 .. note::
 
    Inline Python expressions work just like variable expansions insofar as the
    "=" and ":=" operators are concerned. Given the following assignment, foo()
-   is called each time FOO is expanded: ::
+   is called each time FOO is expanded::
 
       FOO = "${@foo()}"
 
    Contrast this with the following immediate assignment, where foo() is only
-   called once, while the assignment is parsed: ::
+   called once, while the assignment is parsed::
 
       FOO := "${@foo()}"
 
@@ -437,7 +494,7 @@ Unsetting variables
 
 It is possible to completely remove a variable or a variable flag from
 BitBake's internal data dictionary by using the "unset" keyword. Here is
-an example: ::
+an example::
 
    unset DATE
    unset do_fetch[noexec]
@@ -452,7 +509,7 @@ When specifying pathnames for use with BitBake, do not use the tilde
 cause BitBake to not recognize the path since BitBake does not expand
 this character in the same way a shell would.
 
-Instead, provide a fuller path as the following example illustrates: ::
+Instead, provide a fuller path as the following example illustrates::
 
    BBLAYERS ?= " \
        /home/scott-lenovo/LayerA \
@@ -463,7 +520,7 @@ Exporting Variables to the Environment
 
 You can export variables to the environment of running tasks by using
 the ``export`` keyword. For example, in the following example, the
-``do_foo`` task prints "value from the environment" when run: ::
+``do_foo`` task prints "value from the environment" when run::
 
    export ENV_VARIABLE
    ENV_VARIABLE = "value from the environment"
@@ -481,7 +538,7 @@ It does not matter whether ``export ENV_VARIABLE`` appears before or
 after assignments to ``ENV_VARIABLE``.
 
 It is also possible to combine ``export`` with setting a value for the
-variable. Here is an example: ::
+variable. Here is an example::
 
    export ENV_VARIABLE = "variable-value"
 
@@ -496,78 +553,78 @@ Conditional Syntax (Overrides)
 
 BitBake uses :term:`OVERRIDES` to control what
 variables are overridden after BitBake parses recipes and configuration
-files. This section describes how you can use ``OVERRIDES`` as
+files. This section describes how you can use :term:`OVERRIDES` as
 conditional metadata, talks about key expansion in relationship to
-``OVERRIDES``, and provides some examples to help with understanding.
+:term:`OVERRIDES`, and provides some examples to help with understanding.
 
 Conditional Metadata
 --------------------
 
-You can use ``OVERRIDES`` to conditionally select a specific version of
+You can use :term:`OVERRIDES` to conditionally select a specific version of
 a variable and to conditionally append or prepend the value of a
 variable.
 
 .. note::
 
-   Overrides can only use lower-case characters. Additionally,
-   underscores are not permitted in override names as they are used to
+   Overrides can only use lower-case characters, digits and dashes.
+   In particular, colons are not permitted in override names as they are used to
    separate overrides from each other and from the variable name.
 
--  *Selecting a Variable:* The ``OVERRIDES`` variable is a
+-  *Selecting a Variable:* The :term:`OVERRIDES` variable is a
    colon-character-separated list that contains items for which you want
    to satisfy conditions. Thus, if you have a variable that is
-   conditional on "arm", and "arm" is in ``OVERRIDES``, then the
+   conditional on "arm", and "arm" is in :term:`OVERRIDES`, then the
    "arm"-specific version of the variable is used rather than the
-   non-conditional version. Here is an example: ::
+   non-conditional version. Here is an example::
 
       OVERRIDES = "architecture:os:machine"
       TEST = "default"
-      TEST_os = "osspecific"
-      TEST_nooverride = "othercondvalue"
+      TEST:os = "osspecific"
+      TEST:nooverride = "othercondvalue"
 
-   In this example, the ``OVERRIDES``
+   In this example, the :term:`OVERRIDES`
    variable lists three overrides: "architecture", "os", and "machine".
    The variable ``TEST`` by itself has a default value of "default". You
    select the os-specific version of the ``TEST`` variable by appending
-   the "os" override to the variable (i.e. ``TEST_os``).
+   the "os" override to the variable (i.e. ``TEST:os``).
 
    To better understand this, consider a practical example that assumes
    an OpenEmbedded metadata-based Linux kernel recipe file. The
    following lines from the recipe file first set the kernel branch
    variable ``KBRANCH`` to a default value, then conditionally override
-   that value based on the architecture of the build: ::
+   that value based on the architecture of the build::
 
       KBRANCH = "standard/base"
-      KBRANCH_qemuarm = "standard/arm-versatile-926ejs"
-      KBRANCH_qemumips = "standard/mti-malta32"
-      KBRANCH_qemuppc = "standard/qemuppc"
-      KBRANCH_qemux86 = "standard/common-pc/base"
-      KBRANCH_qemux86-64 = "standard/common-pc-64/base"
-      KBRANCH_qemumips64 = "standard/mti-malta64"
+      KBRANCH:qemuarm = "standard/arm-versatile-926ejs"
+      KBRANCH:qemumips = "standard/mti-malta32"
+      KBRANCH:qemuppc = "standard/qemuppc"
+      KBRANCH:qemux86 = "standard/common-pc/base"
+      KBRANCH:qemux86-64 = "standard/common-pc-64/base"
+      KBRANCH:qemumips64 = "standard/mti-malta64"
 
 -  *Appending and Prepending:* BitBake also supports append and prepend
    operations to variable values based on whether a specific item is
-   listed in ``OVERRIDES``. Here is an example: ::
+   listed in :term:`OVERRIDES`. Here is an example::
 
       DEPENDS = "glibc ncurses"
       OVERRIDES = "machine:local"
-      DEPENDS_append_machine = "libmad"
+      DEPENDS:append:machine = "libmad"
 
-   In this example, ``DEPENDS`` becomes "glibc ncurses libmad".
+   In this example, :term:`DEPENDS` becomes "glibc ncurses libmad".
 
    Again, using an OpenEmbedded metadata-based kernel recipe file as an
    example, the following lines will conditionally append to the
-   ``KERNEL_FEATURES`` variable based on the architecture: ::
+   ``KERNEL_FEATURES`` variable based on the architecture::
 
-      KERNEL_FEATURES_append = " ${KERNEL_EXTRA_FEATURES}"
-      KERNEL_FEATURES_append_qemux86=" cfg/sound.scc cfg/paravirt_kvm.scc"
-      KERNEL_FEATURES_append_qemux86-64=" cfg/sound.scc cfg/paravirt_kvm.scc"
+      KERNEL_FEATURES:append = " ${KERNEL_EXTRA_FEATURES}"
+      KERNEL_FEATURES:append:qemux86=" cfg/sound.scc cfg/paravirt_kvm.scc"
+      KERNEL_FEATURES:append:qemux86-64=" cfg/sound.scc cfg/paravirt_kvm.scc"
 
 -  *Setting a Variable for a Single Task:* BitBake supports setting a
-   variable just for the duration of a single task. Here is an example: ::
+   variable just for the duration of a single task. Here is an example::
 
-      FOO_task-configure = "val 1"
-      FOO_task-compile = "val 2"
+      FOO:task-configure = "val 1"
+      FOO:task-compile = "val 2"
 
    In the
    previous example, ``FOO`` has the value "val 1" while the
@@ -580,15 +637,25 @@ variable.
    ``do_compile`` task.
 
    You can also use this syntax with other combinations (e.g.
-   "``_prepend``") as shown in the following example: ::
+   "``:prepend``") as shown in the following example::
+
+      EXTRA_OEMAKE:prepend:task-compile = "${PARALLEL_MAKE} "
+
+.. note::
 
-      EXTRA_OEMAKE_prepend_task-compile = "${PARALLEL_MAKE} "
+   Before BitBake 1.52 (Honister 3.4), the syntax for :term:`OVERRIDES`
+   used ``_`` instead of ``:``, so you will still find a lot of documentation
+   using ``_append``, ``_prepend``, and ``_remove``, for example.
+
+   For details, see the
+   :yocto_docs:`Overrides Syntax Changes </migration-guides/migration-3.4.html#override-syntax-changes>`
+   section in the Yocto Project manual migration notes.
 
 Key Expansion
 -------------
 
 Key expansion happens when the BitBake datastore is finalized. To better
-understand this, consider the following example: ::
+understand this, consider the following example::
 
    A${B} = "X"
    B = "2"
@@ -612,57 +679,57 @@ users.
 
 There is often confusion concerning the order in which overrides and
 various "append" operators take effect. Recall that an append or prepend
-operation using "_append" and "_prepend" does not result in an immediate
+operation using ":append" and ":prepend" does not result in an immediate
 assignment as would "+=", ".=", "=+", or "=.". Consider the following
-example: ::
+example::
 
    OVERRIDES = "foo"
    A = "Z"
-   A_foo_append = "X"
+   A:foo:append = "X"
 
 For this case,
 ``A`` is unconditionally set to "Z" and "X" is unconditionally and
-immediately appended to the variable ``A_foo``. Because overrides have
-not been applied yet, ``A_foo`` is set to "X" due to the append and
+immediately appended to the variable ``A:foo``. Because overrides have
+not been applied yet, ``A:foo`` is set to "X" due to the append and
 ``A`` simply equals "Z".
 
 Applying overrides, however, changes things. Since "foo" is listed in
-``OVERRIDES``, the conditional variable ``A`` is replaced with the "foo"
-version, which is equal to "X". So effectively, ``A_foo`` replaces
+:term:`OVERRIDES`, the conditional variable ``A`` is replaced with the "foo"
+version, which is equal to "X". So effectively, ``A:foo`` replaces
 ``A``.
 
-This next example changes the order of the override and the append: ::
+This next example changes the order of the override and the append::
 
    OVERRIDES = "foo"
    A = "Z"
-   A_append_foo = "X"
+   A:append:foo = "X"
 
 For this case, before
-overrides are handled, ``A`` is set to "Z" and ``A_append_foo`` is set
+overrides are handled, ``A`` is set to "Z" and ``A:append:foo`` is set
 to "X". Once the override for "foo" is applied, however, ``A`` gets
 appended with "X". Consequently, ``A`` becomes "ZX". Notice that spaces
 are not appended.
 
 This next example has the order of the appends and overrides reversed
-back as in the first example: ::
+back as in the first example::
 
    OVERRIDES = "foo"
    A = "Y"
-   A_foo_append = "Z"
-   A_foo_append = "X"
+   A:foo:append = "Z"
+   A:foo:append = "X"
 
 For this case, before any overrides are resolved,
 ``A`` is set to "Y" using an immediate assignment. After this immediate
-assignment, ``A_foo`` is set to "Z", and then further appended with "X"
+assignment, ``A:foo`` is set to "Z", and then further appended with "X"
 leaving the variable set to "ZX". Finally, applying the override for
 "foo" results in the conditional variable ``A`` becoming "ZX" (i.e.
-``A`` is replaced with ``A_foo``).
+``A`` is replaced with ``A:foo``).
 
-This final example mixes in some varying operators: ::
+This final example mixes in some varying operators::
 
    A = "1"
-   A_append = "2"
-   A_append = "3"
+   A:append = "2"
+   A:append = "3"
    A += "4"
    A .= "5"
 
@@ -670,7 +737,7 @@ For this case, the type of append
 operators are affecting the order of assignments as BitBake passes
 through the code multiple times. Initially, ``A`` is set to "1 45"
 because of the three statements that use immediate operators. After
-these assignments are made, BitBake applies the "_append" operations.
+these assignments are made, BitBake applies the ":append" operations.
 Those operations result in ``A`` becoming "1 4523".
 
 Sharing Functionality
@@ -686,8 +753,10 @@ share the task.
 
 This section presents the mechanisms BitBake provides to allow you to
 share functionality between recipes. Specifically, the mechanisms
-include ``include``, ``inherit``, ``INHERIT``, and ``require``
-directives.
+include ``include``, ``inherit``, :term:`INHERIT`, and ``require``
+directives. There is also a higher-level abstraction called
+``configuration fragments`` that is enabled with ``addfragments``
+directive.
 
 Locating Include and Class Files
 --------------------------------
@@ -702,7 +771,9 @@ current directory for ``include`` and ``require`` directives.
 
 In order for include and class files to be found by BitBake, they need
 to be located in a "classes" subdirectory that can be found in
-``BBPATH``.
+:term:`BBPATH`.
+
+.. _ref-bitbake-user-manual-metadata-inherit:
 
 ``inherit`` Directive
 ---------------------
@@ -720,12 +791,12 @@ file and then have your recipe inherit that class file.
 
 As an example, your recipes could use the following directive to inherit
 an ``autotools.bbclass`` file. The class file would contain common
-functionality for using Autotools that could be shared across recipes: ::
+functionality for using Autotools that could be shared across recipes::
 
    inherit autotools
 
 In this case, BitBake would search for the directory
-``classes/autotools.bbclass`` in ``BBPATH``.
+``classes/autotools.bbclass`` in :term:`BBPATH`.
 
 .. note::
 
@@ -734,7 +805,7 @@ In this case, BitBake would search for the directory
 
 If you want to use the directive to inherit multiple classes, separate
 them with spaces. The following example shows how to inherit both the
-``buildhistory`` and ``rm_work`` classes: ::
+``buildhistory`` and ``rm_work`` classes::
 
    inherit buildhistory rm_work
 
@@ -742,19 +813,43 @@ An advantage with the inherit directive as compared to both the
 :ref:`include <bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>` and :ref:`require <bitbake-user-manual/bitbake-user-manual-metadata:\`\`require\`\` directive>`
 directives is that you can inherit class files conditionally. You can
 accomplish this by using a variable expression after the ``inherit``
-statement. Here is an example: ::
+statement.
+
+For inheriting classes conditionally, using the :ref:`inherit_defer
+<ref-bitbake-user-manual-metadata-inherit-defer>` directive is advised as
+:ref:`inherit_defer <ref-bitbake-user-manual-metadata-inherit-defer>` is
+evaluated at the end of parsing.
+
+.. _ref-bitbake-user-manual-metadata-inherit-defer:
+
+``inherit_defer`` Directive
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :ref:`inherit_defer <ref-bitbake-user-manual-metadata-inherit-defer>`
+directive works like the :ref:`inherit
+<ref-bitbake-user-manual-metadata-inherit>` directive, except that it is only
+evaluated at the end of parsing. Its usage is recommended when a conditional
+expression is used.
 
-   inherit ${VARNAME}
+This allows conditional expressions to be evaluated "late", meaning changes to
+the variable after the line is parsed will take effect. With the :ref:`inherit
+<ref-bitbake-user-manual-metadata-inherit>` directive this is not the case.
+
+Here is an example::
+
+   inherit_defer ${VARNAME}
 
 If ``VARNAME`` is
-going to be set, it needs to be set before the ``inherit`` statement is
+going to be set, it needs to be set before the ``inherit_defer`` statement is
 parsed. One way to achieve a conditional inherit in this case is to use
-overrides: ::
+overrides::
 
    VARIABLE = ""
-   VARIABLE_someoverride = "myclass"
+   VARIABLE:someoverride = "myclass"
 
-Another method is by using anonymous Python. Here is an example: ::
+Another method is by using :ref:`anonymous Python
+<bitbake-user-manual/bitbake-user-manual-metadata:Anonymous Python Functions>`.
+Here is an example::
 
    python () {
        if condition == value:
@@ -763,11 +858,14 @@ Another method is by using anonymous Python. Here is an example: ::
            d.setVar('VARIABLE', '')
    }
 
-Alternatively, you could use an in-line Python expression in the
-following form: ::
+Alternatively, you could use an inline Python expression in the
+following form::
+
+   inherit_defer ${@'classname' if condition else ''}
 
-   inherit ${@'classname' if condition else ''}
-   inherit ${@functionname(params)}
+Or::
+
+   inherit_defer ${@bb.utils.contains('VARIABLE', 'something', 'classname', '', d)}
 
 In all cases, if the expression evaluates to an
 empty string, the statement does not trigger a syntax error because it
@@ -780,7 +878,7 @@ BitBake understands the ``include`` directive. This directive causes
 BitBake to parse whatever file you specify, and to insert that file at
 that location. The directive is much like its equivalent in Make except
 that if the path specified on the include line is a relative path,
-BitBake locates the first file it can find within ``BBPATH``.
+BitBake locates the first file it can find within :term:`BBPATH`.
 
 The include directive is a more generic method of including
 functionality as compared to the :ref:`inherit <bitbake-user-manual/bitbake-user-manual-metadata:\`\`inherit\`\` directive>`
@@ -790,7 +888,7 @@ encapsulated functionality or configuration that does not suit a
 ``.bbclass`` file.
 
 As an example, suppose you needed a recipe to include some self-test
-definitions: ::
+definitions::
 
    include test_defs.inc
 
@@ -802,6 +900,33 @@ definitions: ::
    of include . Doing so makes sure that an error is produced if the file cannot
    be found.
 
+``include_all`` Directive
+-------------------------
+
+The ``include_all`` directive works like the :ref:`include
+<bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>`
+directive but will include all of the files that match the specified path in
+the enabled layers (layers part of :term:`BBLAYERS`).
+
+For example, let's say a ``maintainers.inc`` file is present in different layers
+and is conventionally placed in the ``conf/distro/include`` directory of each
+layer. In that case the ``include_all`` directive can be used to include
+the ``maintainers.inc`` file for all of these layers::
+
+   include_all conf/distro/include/maintainers.inc
+
+In other words, the ``maintainers.inc`` file for each layer is included through
+the :ref:`include <bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>`
+directive.
+
+BitBake will iterate through the colon-separated :term:`BBPATH` list to look for
+matching files to include, from left to right. As a consequence, matching files
+are included in that order.
+
+As the ``include_all`` directive uses the :ref:`include
+<bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>`
+directive in the background, no error is produced if no files are matched.
+
 .. _require-inclusion:
 
 ``require`` Directive
@@ -822,7 +947,7 @@ does not suit a ``.bbclass`` file.
 
 Similar to how BitBake handles :ref:`include <bitbake-user-manual/bitbake-user-manual-metadata:\`\`include\`\` directive>`, if
 the path specified on the require line is a relative path, BitBake
-locates the first file it can find within ``BBPATH``.
+locates the first file it can find within :term:`BBPATH`.
 
 As an example, suppose you have two versions of a recipe (e.g.
 ``foo_1.2.2.bb`` and ``foo_2.0.0.bb``) where each version contains some
@@ -831,7 +956,7 @@ include file named ``foo.inc`` that contains the common definitions
 needed to build "foo". You need to be sure ``foo.inc`` is located in the
 same directory as your two recipe files as well. Once these conditions
 are set up, you can share the functionality using a ``require``
-directive from within each recipe: ::
+directive from within each recipe::
 
    require foo.inc
 
@@ -844,14 +969,14 @@ class. BitBake only supports this directive when used within a
 configuration file.
 
 As an example, suppose you needed to inherit a class file called
-``abc.bbclass`` from a configuration file as follows: ::
+``abc.bbclass`` from a configuration file as follows::
 
    INHERIT += "abc"
 
 This configuration directive causes the named class to be inherited at
 the point of the directive during parsing. As with the ``inherit``
 directive, the ``.bbclass`` file must be located in a "classes"
-subdirectory in one of the directories specified in ``BBPATH``.
+subdirectory in one of the directories specified in :term:`BBPATH`.
 
 .. note::
 
@@ -862,10 +987,69 @@ subdirectory in one of the directories specified in ``BBPATH``.
 If you want to use the directive to inherit multiple classes, you can
 provide them on the same line in the ``local.conf`` file. Use spaces to
 separate the classes. The following example shows how to inherit both
-the ``autotools`` and ``pkgconfig`` classes: ::
+the ``autotools`` and ``pkgconfig`` classes::
 
    INHERIT += "autotools pkgconfig"
 
+``addfragments`` Directive
+--------------------------
+
+This directive allows fine-tuning local configurations with configuration
+snippets contained in layers in a structured, controlled way. Typically it would
+go into ``bitbake.conf``, for example::
+
+   addfragments conf/fragments OE_FRAGMENTS OE_FRAGMENTS_METADATA_VARS OE_BUILTIN_FRAGMENTS
+
+``addfragments`` takes four parameters:
+
+-  path prefix for fragment files inside the layer file tree that bitbake
+   uses to construct full paths to the fragment files
+
+-  name of variable that holds the list of enabled fragments in an
+   active build
+
+-  name of variable that contains a list of variable names containing
+   fragment-specific metadata (such as descriptions)
+
+-  name of variable that contains definitions for built-in fragments
+
+This allows listing enabled configuration fragments in ``OE_FRAGMENTS``
+variable like this::
+
+   OE_FRAGMENTS = "core/domain/somefragment core/someotherfragment anotherlayer/anotherdomain/anotherfragment"
+
+Fragment names listed in this variable must be prefixed by the layer name
+where a fragment file is located, defined by :term:`BBFILE_COLLECTIONS` in ``layer.conf``.
+
+The implementation then expands this list into
+:ref:`require <bitbake-user-manual/bitbake-user-manual-metadata:\`\`require\`\` directive>`
+directives with full paths to respective layers::
+
+   require /path/to/core-layer/conf/fragments/domain/somefragment.conf
+   require /path/to/core-layer/conf/fragments/someotherfragment.conf
+   require /path/to/another-layer/conf/fragments/anotherdomain/anotherfragment.conf
+
+The variable containing a list of fragment metadata variables could look like this::
+
+   OE_FRAGMENTS_METADATA_VARS = "BB_CONF_FRAGMENT_SUMMARY BB_CONF_FRAGMENT_DESCRIPTION"
+
+The implementation will add a flag containing the fragment name to each of those variables
+when parsing fragments, so that the variables are namespaced by fragment name, and do not override
+each other when several fragments are enabled.
+
+The variable containing a built-in fragment definitions could look like this::
+
+   OE_BUILTIN_FRAGMENTS = "someprefix:SOMEVARIABLE anotherprefix:ANOTHERVARIABLE"
+
+and then if 'someprefix/somevalue' is added to the variable that holds the list
+of enabled fragments:
+
+  OE_FRAGMENTS = "... someprefix/somevalue"
+
+bitbake will treat that as direct value assignment in its configuration::
+
+  SOMEVARIABLE = "somevalue"
+
 Functions
 =========
 
@@ -893,9 +1077,9 @@ Regardless of the type of function, you can only define them in class
 Shell Functions
 ---------------
 
-Functions written in shell script and executed either directly as
+Functions written in shell script are executed either directly as
 functions, tasks, or both. They can also be called by other shell
-functions. Here is an example shell function definition: ::
+functions. Here is an example shell function definition::
 
    some_function () {
        echo "Hello World"
@@ -907,19 +1091,19 @@ rules. The scripts are executed by ``/bin/sh``, which may not be a bash
 shell but might be something such as ``dash``. You should not use
 Bash-specific script (bashisms).
 
-Overrides and override-style operators like ``_append`` and ``_prepend``
+Overrides and override-style operators like ``:append`` and ``:prepend``
 can also be applied to shell functions. Most commonly, this application
 would be used in a ``.bbappend`` file to modify functions in the main
 recipe. It can also be used to modify functions inherited from classes.
 
-As an example, consider the following: ::
+As an example, consider the following::
 
    do_foo() {
        bbplain first
        fn
    }
 
-   fn_prepend() {
+   fn:prepend() {
        bbplain second
    }
 
@@ -927,11 +1111,11 @@ As an example, consider the following: ::
        bbplain third
    }
 
-   do_foo_append() {
+   do_foo:append() {
        bbplain fourth
    }
 
-Running ``do_foo`` prints the following: ::
+Running ``do_foo`` prints the following::
 
    recipename do_foo: first
    recipename do_foo: second
@@ -943,7 +1127,7 @@ Running ``do_foo`` prints the following: ::
    Overrides and override-style operators can be applied to any shell
    function, not just :ref:`tasks <bitbake-user-manual/bitbake-user-manual-metadata:tasks>`.
 
-You can use the ``bitbake -e`` recipename command to view the final
+You can use the ``bitbake -e recipename`` command to view the final
 assembled function after all overrides have been applied.
 
 BitBake-Style Python Functions
@@ -952,7 +1136,7 @@ BitBake-Style Python Functions
 These functions are written in Python and executed by BitBake or other
 Python functions using ``bb.build.exec_func()``.
 
-An example BitBake function is: ::
+An example BitBake function is::
 
    python some_python_function () {
        d.setVar("TEXT", "Hello World")
@@ -975,9 +1159,9 @@ import these modules. Also in these types of functions, the datastore
 Similar to shell functions, you can also apply overrides and
 override-style operators to BitBake-style Python functions.
 
-As an example, consider the following: ::
+As an example, consider the following::
 
-   python do_foo_prepend() {
+   python do_foo:prepend() {
        bb.plain("first")
    }
 
@@ -985,17 +1169,17 @@ As an example, consider the following: ::
        bb.plain("second")
    }
 
-   python do_foo_append() {
+   python do_foo:append() {
        bb.plain("third")
    }
 
-Running ``do_foo`` prints the following: ::
+Running ``do_foo`` prints the following::
 
    recipename do_foo: first
    recipename do_foo: second
    recipename do_foo: third
 
-You can use the ``bitbake -e`` recipename command to view
+You can use the ``bitbake -e recipename`` command to view
 the final assembled function after all overrides have been applied.
 
 Python Functions
@@ -1004,7 +1188,7 @@ Python Functions
 These functions are written in Python and are executed by other Python
 code. Examples of Python functions are utility functions that you intend
 to call from in-line Python or from within other Python functions. Here
-is an example: ::
+is an example::
 
    def get_depends(d):
        if d.getVar('SOMECONDITION'):
@@ -1015,7 +1199,7 @@ is an example: ::
    SOMECONDITION = "1"
    DEPENDS = "${@get_depends(d)}"
 
-This would result in ``DEPENDS`` containing ``dependencywithcond``.
+This would result in :term:`DEPENDS` containing ``dependencywithcond``.
 
 Here are some things to know about Python functions:
 
@@ -1056,7 +1240,7 @@ functions and regular Python functions defined with "def":
 -  Regular Python functions are called with the usual Python syntax.
    BitBake-style Python functions are usually tasks and are called
    directly by BitBake, but can also be called manually from Python code
-   by using the ``bb.build.exec_func()`` function. Here is an example: ::
+   by using the ``bb.build.exec_func()`` function. Here is an example::
 
       bb.build.exec_func("my_bitbake_style_function", d)
 
@@ -1094,7 +1278,7 @@ Sometimes it is useful to set variables or perform other operations
 programmatically during parsing. To do this, you can define special
 Python functions, called anonymous Python functions, that run at the end
 of parsing. For example, the following conditionally sets a variable
-based on the value of another variable: ::
+based on the value of another variable::
 
    python () {
        if d.getVar('SOMEVAR') == 'value':
@@ -1107,7 +1291,7 @@ the name "__anonymous", rather than no name.
 Anonymous Python functions always run at the end of parsing, regardless
 of where they are defined. If a recipe contains many anonymous
 functions, they run in the same order as they are defined within the
-recipe. As an example, consider the following snippet: ::
+recipe. As an example, consider the following snippet::
 
    python () {
        d.setVar('FOO', 'foo 2')
@@ -1122,7 +1306,7 @@ recipe. As an example, consider the following snippet: ::
    BAR = "bar 1"
 
 The previous example is conceptually
-equivalent to the following snippet: ::
+equivalent to the following snippet::
 
    FOO = "foo 1"
    BAR = "bar 1"
@@ -1134,12 +1318,12 @@ equivalent to the following snippet: ::
 values set for the variables within the anonymous functions become
 available to tasks, which always run after parsing.
 
-Overrides and override-style operators such as "``_append``" are applied
+Overrides and override-style operators such as "``:append``" are applied
 before anonymous functions run. In the following example, ``FOO`` ends
-up with the value "foo from anonymous": ::
+up with the value "foo from anonymous"::
 
    FOO = "foo"
-   FOO_append = " from outside"
+   FOO:append = " from outside"
 
    python () {
        d.setVar("FOO", "foo from anonymous")
@@ -1164,7 +1348,7 @@ To understand the benefits of this feature, consider the basic scenario
 where a class defines a task function and your recipe inherits the
 class. In this basic scenario, your recipe inherits the task function as
 defined in the class. If desired, your recipe can add to the start and
-end of the function by using the "_prepend" or "_append" operations
+end of the function by using the ":prepend" or ":append" operations
 respectively, or it can redefine the function completely. However, if it
 redefines the function, there is no means for it to call the class
 version of the function. ``EXPORT_FUNCTIONS`` provides a mechanism that
@@ -1173,24 +1357,24 @@ version of the function.
 
 To make use of this technique, you need the following things in place:
 
--  The class needs to define the function as follows: ::
+-  The class needs to define the function as follows::
 
       classname_functionname
 
    For example, if you have a class file
    ``bar.bbclass`` and a function named ``do_foo``, the class must
-   define the function as follows: ::
+   define the function as follows::
 
       bar_do_foo
 
 -  The class needs to contain the ``EXPORT_FUNCTIONS`` statement as
-   follows: ::
+   follows::
 
       EXPORT_FUNCTIONS functionname
 
    For example, continuing with
    the same example, the statement in the ``bar.bbclass`` would be as
-   follows: ::
+   follows::
 
       EXPORT_FUNCTIONS do_foo
 
@@ -1199,7 +1383,7 @@ To make use of this technique, you need the following things in place:
    class version of the function, it should call ``bar_do_foo``.
    Assuming ``do_foo`` was a shell function and ``EXPORT_FUNCTIONS`` was
    used as above, the recipe's function could conditionally call the
-   class version of the function as follows: ::
+   class version of the function as follows::
 
       do_foo() {
           if [ somecondition ] ; then
@@ -1233,11 +1417,11 @@ Tasks are either :ref:`shell functions <bitbake-user-manual/bitbake-user-manual-
 that have been promoted to tasks by using the ``addtask`` command. The
 ``addtask`` command can also optionally describe dependencies between
 the task and other tasks. Here is an example that shows how to define a
-task and declare some dependencies: ::
+task and declare some dependencies::
 
    python do_printdate () {
-       import time
-       print time.strftime('%Y%m%d', time.gmtime())
+       import datetime
+       bb.plain('Date: %s' % (datetime.date.today()))
    }
    addtask printdate after do_fetch before do_build
 
@@ -1264,12 +1448,12 @@ Additionally, the ``do_printdate`` task becomes dependent upon the
    rerun for experimentation purposes, you can make BitBake always
    consider the task "out-of-date" by using the
    :ref:`[nostamp] <bitbake-user-manual/bitbake-user-manual-metadata:Variable Flags>`
-   variable flag, as follows: ::
+   variable flag, as follows::
 
       do_printdate[nostamp] = "1"
 
    You can also explicitly run the task and provide the
-   -f option as follows: ::
+   -f option as follows::
 
       $ bitbake recipe -c printdate -f
 
@@ -1278,7 +1462,7 @@ Additionally, the ``do_printdate`` task becomes dependent upon the
    name.
 
 You might wonder about the practical effects of using ``addtask``
-without specifying any dependencies as is done in the following example: ::
+without specifying any dependencies as is done in the following example::
 
    addtask printdate
 
@@ -1286,7 +1470,7 @@ In this example, assuming dependencies have not been
 added through some other means, the only way to run the task is by
 explicitly selecting it with ``bitbake`` recipe ``-c printdate``. You
 can use the ``do_listtasks`` task to list all tasks defined in a recipe
-as shown in the following example: ::
+as shown in the following example::
 
    $ bitbake recipe -c listtasks
 
@@ -1296,12 +1480,23 @@ For more information on task dependencies, see the
 See the ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:variable flags`" section for information
 on variable flags you can use with tasks.
 
+.. note::
+
+   While it's infrequent, it's possible to define multiple tasks as
+   dependencies when calling ``addtask``. For example, here's a snippet
+   from the OpenEmbedded class file ``package_tar.bbclass``::
+
+     addtask package_write_tar before do_build after do_packagedata do_package
+
+   Note how the ``package_write_tar`` task has to wait until both of
+   ``do_packagedata`` and ``do_package`` complete.
+
 Deleting a Task
 ---------------
 
 As well as being able to add tasks, you can delete them. Simply use the
 ``deltask`` command to delete a task. For example, to delete the example
-task used in the previous sections, you would use: ::
+task used in the previous sections, you would use::
 
    deltask printdate
 
@@ -1317,7 +1512,7 @@ to run before ``do_a``.
 
 If you want dependencies such as these to remain intact, use the
 ``[noexec]`` varflag to disable the task instead of using the
-``deltask`` command to delete it: ::
+``deltask`` command to delete it::
 
    do_b[noexec] = "1"
 
@@ -1331,8 +1526,8 @@ the build machine cannot influence the build.
 .. note::
 
    By default, BitBake cleans the environment to include only those
-   things exported or listed in its whitelist to ensure that the build
-   environment is reproducible and consistent. You can prevent this
+   things exported or listed in its passthrough list to ensure that the
+   build environment is reproducible and consistent. You can prevent this
    "cleaning" by setting the :term:`BB_PRESERVE_ENV` variable.
 
 Consequently, if you do want something to get passed into the build task
@@ -1340,14 +1535,14 @@ environment, you must take these two steps:
 
 #. Tell BitBake to load what you want from the environment into the
    datastore. You can do so through the
-   :term:`BB_ENV_WHITELIST` and
-   :term:`BB_ENV_EXTRAWHITE` variables. For
+   :term:`BB_ENV_PASSTHROUGH` and
+   :term:`BB_ENV_PASSTHROUGH_ADDITIONS` variables. For
    example, assume you want to prevent the build system from accessing
-   your ``$HOME/.ccache`` directory. The following command "whitelists"
-   the environment variable ``CCACHE_DIR`` causing BitBake to allow that
-   variable into the datastore: ::
+   your ``$HOME/.ccache`` directory. The following command adds the
+   the environment variable ``CCACHE_DIR`` to BitBake's passthrough
+   list to allow that variable into the datastore::
 
-      export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE CCACHE_DIR"
+      export BB_ENV_PASSTHROUGH_ADDITIONS="$BB_ENV_PASSTHROUGH_ADDITIONS CCACHE_DIR"
 
 #. Tell BitBake to export what you have loaded into the datastore to the
    task environment of every running task. Loading something from the
@@ -1355,7 +1550,7 @@ environment, you must take these two steps:
    available in the datastore. To export it to the task environment of
    every running task, use a command similar to the following in your
    local configuration file ``local.conf`` or your distribution
-   configuration file: ::
+   configuration file::
 
       export CCACHE_DIR
 
@@ -1364,17 +1559,17 @@ environment, you must take these two steps:
       A side effect of the previous steps is that BitBake records the
       variable as a dependency of the build process in things like the
       setscene checksums. If doing so results in unnecessary rebuilds of
-      tasks, you can whitelist the variable so that the setscene code
+      tasks, you can also flag the variable so that the setscene code
       ignores the dependency when it creates checksums.
 
 Sometimes, it is useful to be able to obtain information from the
 original execution environment. BitBake saves a copy of the original
 environment into a special variable named :term:`BB_ORIGENV`.
 
-The ``BB_ORIGENV`` variable returns a datastore object that can be
+The :term:`BB_ORIGENV` variable returns a datastore object that can be
 queried using the standard datastore operators such as
 ``getVar(, False)``. The datastore object is useful, for example, to
-find the original ``DISPLAY`` variable. Here is an example: ::
+find the original ``DISPLAY`` variable. Here is an example::
 
    origenv = d.getVar("BB_ORIGENV", False)
    bar = origenv.getVar("BAR", False)
@@ -1387,7 +1582,7 @@ Variable Flags
 
 Variable flags (varflags) help control a task's functionality and
 dependencies. BitBake reads and writes varflags to the datastore using
-the following command forms: ::
+the following command forms::
 
    variable = d.getVarFlags("variable")
    self.d.setVarFlags("FOO", {"func": True})
@@ -1418,12 +1613,35 @@ functionality of the task:
    directory listed is used as the current working directory for the
    task.
 
+- ``[file-checksums]``: Controls the file dependencies for a task. The
+  baseline file list is the set of files associated with
+  :term:`SRC_URI`. May be used to set additional dependencies on
+  files not associated with :term:`SRC_URI`.
+
+  The value set to the list is a file-boolean pair where the first
+  value is the file name and the second is whether or not it
+  physically exists on the filesystem. ::
+
+    do_configure[file-checksums] += "${MY_DIRPATH}/my-file.txt:True"
+
+  It is important to record any paths which the task looked at and
+  which didn't exist. This means that if these do exist at a later
+  time, the task can be rerun with the new additional files. The
+  "exists" True or False value after the path allows this to be
+  handled.
+
 -  ``[lockfiles]``: Specifies one or more lockfiles to lock while the
    task executes. Only one task may hold a lockfile, and any task that
    attempts to lock an already locked file will block until the lock is
    released. You can use this variable flag to accomplish mutual
    exclusion.
 
+-  ``[network]``: When set to "1", allows a task to access the network. By
+   default, only the ``do_fetch`` task is granted network access. Recipes
+   shouldn't access the network outside of ``do_fetch`` as it usually
+   undermines fetcher source mirroring, image and licence manifests, software
+   auditing and supply chain security.
+
 -  ``[noexec]``: When set to "1", marks the task as being empty, with
    no execution required. You can use the ``[noexec]`` flag to set up
    tasks as dependency placeholders, or to disable tasks defined
@@ -1456,7 +1674,7 @@ functionality of the task:
          can result in unpredictable behavior.
 
       -  Setting the varflag to a value greater than the value used in
-         the ``BB_NUMBER_THREADS`` variable causes ``number_threads`` to
+         the :term:`BB_NUMBER_THREADS` variable causes ``number_threads`` to
          have no effect.
 
 -  ``[postfuncs]``: List of functions to call after the completion of
@@ -1526,7 +1744,7 @@ intent is to make it easy to do things like email notification on build
 failures.
 
 Following is an example event handler that prints the name of the event
-and the content of the ``FILE`` variable: ::
+and the content of the :term:`FILE` variable::
 
    addhandler myclass_eventhandler
    python myclass_eventhandler() {
@@ -1565,11 +1783,11 @@ might have an interest in viewing:
 
 -  ``bb.event.ConfigParsed()``: Fired when the base configuration; which
    consists of ``bitbake.conf``, ``base.bbclass`` and any global
-   ``INHERIT`` statements; has been parsed. You can see multiple such
+   :term:`INHERIT` statements; has been parsed. You can see multiple such
    events when each of the workers parse the base configuration or if
    the server changes configuration and reparses. Any given datastore
    only has one such event executed against it, however. If
-   ```BB_INVALIDCONF`` <#>`__ is set in the datastore by the event
+   :term:`BB_INVALIDCONF` is set in the datastore by the event
    handler, the configuration is reparsed and a new event triggered,
    allowing the metadata to update configuration.
 
@@ -1636,13 +1854,18 @@ user interfaces:
 
 .. _variants-class-extension-mechanism:
 
-Variants - Class Extension Mechanism
-====================================
+Variants --- Class Extension Mechanism
+======================================
+
+BitBake supports multiple incarnations of a recipe file via the
+:term:`BBCLASSEXTEND` variable.
+
+The :term:`BBCLASSEXTEND` variable is a space separated list of classes used
+to "extend" the recipe for each variant. Here is an example that results in a
+second incarnation of the current recipe being available. This second
+incarnation will have the "native" class inherited. ::
 
-BitBake supports two features that facilitate creating from a single
-recipe file multiple incarnations of that recipe file where all
-incarnations are buildable. These features are enabled through the
-:term:`BBCLASSEXTEND` and :term:`BBVERSIONS` variables.
+      BBCLASSEXTEND = "native"
 
 .. note::
 
@@ -1652,34 +1875,6 @@ incarnations are buildable. These features are enabled through the
    class. For specific examples, see the OE-Core native , nativesdk , and
    multilib classes.
 
--  ``BBCLASSEXTEND``: This variable is a space separated list of
-   classes used to "extend" the recipe for each variant. Here is an
-   example that results in a second incarnation of the current recipe
-   being available. This second incarnation will have the "native" class
-   inherited. ::
-
-      BBCLASSEXTEND = "native"
-
--  ``BBVERSIONS``: This variable allows a single recipe to build
-   multiple versions of a project from a single recipe file. You can
-   also specify conditional metadata (using the
-   :term:`OVERRIDES` mechanism) for a single
-   version, or an optionally named range of versions. Here is an
-   example: ::
-
-      BBVERSIONS = "1.0 2.0 git"
-      SRC_URI_git = "git://someurl/somepath.git"
-
-      BBVERSIONS = "1.0.[0-6]:1.0.0+ 1.0.[7-9]:1.0.7+"
-      SRC_URI_append_1.0.7+ = "file://some_patch_which_the_new_versions_need.patch;patch=1"
-
-   The name of the range defaults to the original version of the recipe. For
-   example, in OpenEmbedded, the recipe file ``foo_1.0.0+.bb`` creates a default
-   name range of ``1.0.0+``. This is useful because the range name is not only
-   placed into overrides, but it is also made available for the metadata to use
-   in the variable that defines the base recipe versions for use in ``file://``
-   search paths (:term:`FILESPATH`).
-
 Dependencies
 ============
 
@@ -1708,7 +1903,7 @@ Dependencies Internal to the ``.bb`` File
 BitBake uses the ``addtask`` directive to manage dependencies that are
 internal to a given recipe file. You can use the ``addtask`` directive
 to indicate when a task is dependent on other tasks or when other tasks
-depend on that recipe. Here is an example: ::
+depend on that recipe. Here is an example::
 
    addtask printdate after do_fetch before do_build
 
@@ -1732,7 +1927,7 @@ task depends on the completion of the ``do_printdate`` task.
 
    -  The directive ``addtask mytask after do_configure`` by itself
       never causes ``do_mytask`` to run. ``do_mytask`` can still be run
-      manually as follows: ::
+      manually as follows::
 
          $ bitbake recipe -c mytask
 
@@ -1745,13 +1940,13 @@ Build Dependencies
 
 BitBake uses the :term:`DEPENDS` variable to manage
 build time dependencies. The ``[deptask]`` varflag for tasks signifies
-the task of each item listed in ``DEPENDS`` that must complete before
-that task can be executed. Here is an example: ::
+the task of each item listed in :term:`DEPENDS` that must complete before
+that task can be executed. Here is an example::
 
    do_configure[deptask] = "do_populate_sysroot"
 
 In this example, the ``do_populate_sysroot`` task
-of each item in ``DEPENDS`` must complete before ``do_configure`` can
+of each item in :term:`DEPENDS` must complete before ``do_configure`` can
 execute.
 
 Runtime Dependencies
@@ -1760,8 +1955,8 @@ Runtime Dependencies
 BitBake uses the :term:`PACKAGES`, :term:`RDEPENDS`, and :term:`RRECOMMENDS`
 variables to manage runtime dependencies.
 
-The ``PACKAGES`` variable lists runtime packages. Each of those packages
-can have ``RDEPENDS`` and ``RRECOMMENDS`` runtime dependencies. The
+The :term:`PACKAGES` variable lists runtime packages. Each of those packages
+can have :term:`RDEPENDS` and :term:`RRECOMMENDS` runtime dependencies. The
 ``[rdeptask]`` flag for tasks is used to signify the task of each item
 runtime dependency which must have completed before that task can be
 executed. ::
@@ -1769,9 +1964,9 @@ executed. ::
    do_package_qa[rdeptask] = "do_packagedata"
 
 In the previous
-example, the ``do_packagedata`` task of each item in ``RDEPENDS`` must
+example, the ``do_packagedata`` task of each item in :term:`RDEPENDS` must
 have completed before ``do_package_qa`` can execute.
-Although ``RDEPENDS`` contains entries from the
+Although :term:`RDEPENDS` contains entries from the
 runtime dependency namespace, BitBake knows how to map them back
 to the build-time dependency namespace, in which the tasks are defined.
 
@@ -1788,7 +1983,7 @@ dependencies are discovered and added.
 
 The ``[recrdeptask]`` flag is most commonly used in high-level recipes
 that need to wait for some task to finish "globally". For example,
-``image.bbclass`` has the following: ::
+``image.bbclass`` has the following::
 
    do_rootfs[recrdeptask] += "do_packagedata"
 
@@ -1797,7 +1992,7 @@ the current recipe and all recipes reachable (by way of dependencies)
 from the image recipe must run before the ``do_rootfs`` task can run.
 
 BitBake allows a task to recursively depend on itself by
-referencing itself in the task list: ::
+referencing itself in the task list::
 
    do_a[recrdeptask] = "do_a do_b"
 
@@ -1814,7 +2009,7 @@ Inter-Task Dependencies
 BitBake uses the ``[depends]`` flag in a more generic form to manage
 inter-task dependencies. This more generic form allows for
 inter-dependency checks for specific tasks rather than checks for the
-data in ``DEPENDS``. Here is an example: ::
+data in :term:`DEPENDS`. Here is an example::
 
    do_patch[depends] = "quilt-native:do_populate_sysroot"
 
@@ -1894,11 +2089,35 @@ access. Here is a list of available operations:
 Other Functions
 ---------------
 
-You can find many other functions that can be called from Python by
-looking at the source code of the ``bb`` module, which is in
-``bitbake/lib/bb``. For example, ``bitbake/lib/bb/utils.py`` includes
-the commonly used functions ``bb.utils.contains()`` and
-``bb.utils.mkdirhier()``, which come with docstrings.
+Other functions are documented in the
+:doc:`/bitbake-user-manual/bitbake-user-manual-library-functions` document.
+
+Extending Python Library Code
+-----------------------------
+
+If you wish to add your own Python library code (e.g. to provide
+functions/classes you can use from Python functions in the metadata)
+you can do so from any layer using the ``addpylib`` directive.
+This directive is typically added to your layer configuration (
+``conf/layer.conf``) although it will be handled in any ``.conf`` file.
+
+Usage is of the form::
+
+   addpylib <directory> <namespace>
+
+Where <directory> specifies the directory to add to the library path.
+The specified <namespace> is imported automatically, and if the imported
+module specifies an attribute named ``BBIMPORTS``, that list of
+sub-modules is iterated and imported too.
+
+Testing and Debugging BitBake Python code
+-----------------------------------------
+
+The OpenEmbedded build system implements a convenient ``pydevshell`` target which
+you can use to access the BitBake datastore and experiment with your own Python
+code. See :yocto_docs:`Using a Python Development Shell
+</dev-manual/python-development-shell.html#using-a-python-development-shell>` in the Yocto
+Project manual for details.
 
 Task Checksums and Setscene
 ===========================
@@ -1909,7 +2128,7 @@ To help understand how BitBake does this, the section assumes an
 OpenEmbedded metadata-based example.
 
 These checksums are stored in :term:`STAMP`. You can
-examine the checksums using the following BitBake command: ::
+examine the checksums using the following BitBake command::
 
    $ bitbake-dumpsigs
 
@@ -1932,16 +2151,6 @@ The following list describes related variables:
    Specifies a function BitBake calls that determines whether BitBake
    requires a setscene dependency to be met.
 
--  :term:`BB_SETSCENE_VERIFY_FUNCTION2`:
-   Specifies a function to call that verifies the list of planned task
-   execution before the main task execution happens.
-
--  :term:`BB_STAMP_POLICY`: Defines the mode
-   for comparing timestamps of stamp files.
-
--  :term:`BB_STAMP_WHITELIST`: Lists stamp
-   files that are looked at when the stamp policy is "whitelist".
-
 -  :term:`BB_TASKHASH`: Within an executing task,
    this variable holds the hash of the task as returned by the currently
    enabled signature generator.
@@ -1956,7 +2165,7 @@ Wildcard Support in Variables
 =============================
 
 Support for wildcard use in variables varies depending on the context in
-which it is used. For example, some variables and file names allow
+which it is used. For example, some variables and filenames allow
 limited use of wildcards through the "``%``" and "``*``" characters.
 Other variables or names support Python's
 `glob <https://docs.python.org/3/library/glob.html>`_ syntax,
diff --git a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables-context.rst b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables-context.rst
new file mode 100644
index 0000000000..e9c454ba11
--- /dev/null
+++ b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables-context.rst
@@ -0,0 +1,91 @@
+.. SPDX-License-Identifier: CC-BY-2.5
+
+================
+Variable Context
+================
+
+|
+
+Variables might only have an impact or can be used in certain contexts. Some
+should only be used in global files like ``.conf``, while others are intended only
+for local files like ``.bb``. This chapter aims to describe some important variable
+contexts.
+
+.. _ref-varcontext-configuration:
+
+BitBake's own configuration
+===========================
+
+Variables starting with ``BB_`` usually configure the behaviour of BitBake itself.
+For example, one could configure:
+
+- System resources, like disk space to be used (:term:`BB_DISKMON_DIRS`),
+  or the number of tasks to be run in parallel by BitBake (:term:`BB_NUMBER_THREADS`).
+
+- How the fetchers shall behave, e.g., :term:`BB_FETCH_PREMIRRORONLY` is used
+  by BitBake to determine if BitBake's fetcher shall search only
+  :term:`PREMIRRORS` for files.
+
+Those variables are usually configured globally.
+
+BitBake configuration
+=====================
+
+There are variables:
+
+- Like :term:`B` or :term:`T`, that are used to specify directories used by
+  BitBake during the build of a particular recipe. Those variables are
+  specified in ``bitbake.conf``. Some, like :term:`B`, are quite often
+  overwritten in recipes.
+
+- Starting with ``FAKEROOT``, to configure how the ``fakeroot`` command is
+  handled. Those are usually set by ``bitbake.conf`` and might get adapted in a
+  ``bbclass``.
+
+- Detailing where BitBake will store and fetch information from, for
+  data reuse between build runs like :term:`CACHE`, :term:`DL_DIR` or
+  :term:`PERSISTENT_DIR`. Those are usually global.
+
+
+Layers and files
+================
+
+Variables starting with ``LAYER`` configure how BitBake handles layers.
+Additionally, variables starting with ``BB`` configure how layers and files are
+handled. For example:
+
+- :term:`LAYERDEPENDS` is used to configure on which layers a given layer
+  depends.
+
+- The configured layers are contained in :term:`BBLAYERS` and files in
+  :term:`BBFILES`.
+
+Those variables are often used in the files ``layer.conf`` and ``bblayers.conf``.
+
+Recipes and packages
+====================
+
+Variables handling recipes and packages can be split into:
+
+- :term:`PN`, :term:`PV` or :term:`PF` for example, contain information about
+  the name or revision of a recipe or package. Usually, the default set in
+  ``bitbake.conf`` is used, but those are from time to time overwritten in
+  recipes.
+
+- :term:`SUMMARY`, :term:`DESCRIPTION`, :term:`LICENSE` or :term:`HOMEPAGE`
+  contain the expected information and should be set specifically for every
+  recipe.
+
+- In recipes, variables are also used to control build and runtime
+  dependencies between recipes/packages with other recipes/packages. The
+  most common should be: :term:`PROVIDES`, :term:`RPROVIDES`, :term:`DEPENDS`,
+  and :term:`RDEPENDS`.
+
+- There are further variables starting with ``SRC`` that specify the sources in
+  a recipe like :term:`SRC_URI` or :term:`SRCDATE`. Those are also usually set
+  in recipes.
+
+- Which version or provider of a recipe should be given preference when
+  multiple recipes would provide the same item, is controlled by variables
+  starting with ``PREFERRED_``. Those are normally set in the configuration
+  files of a ``MACHINE`` or ``DISTRO``.
diff --git a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst
index 6469f9d1a4..6be8dbbf63 100644
--- a/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst
+++ b/bitbake/doc/bitbake-user-manual/bitbake-user-manual-ref-variables.rst
@@ -23,22 +23,31 @@ overview of their function and contents.
       systems extend the functionality of the variable as it is
       described here in this glossary.
 
-   -  Finally, there are variables mentioned in this glossary that do
-      not appear in the BitBake glossary. These other variables are
-      variables used in systems that use BitBake.
-
 .. glossary::
+   :sorted:
 
    :term:`ASSUME_PROVIDED`
       Lists recipe names (:term:`PN` values) BitBake does not
       attempt to build. Instead, BitBake assumes these recipes have already
       been built.
 
-      In OpenEmbedded-Core, ``ASSUME_PROVIDED`` mostly specifies native
+      In OpenEmbedded-Core, :term:`ASSUME_PROVIDED` mostly specifies native
       tools that should not be built. An example is ``git-native``, which
       when specified allows for the Git binary from the host to be used
       rather than building ``git-native``.
 
+   :term:`AZ_SAS`
+      Azure Storage Shared Access Signature, when using the
+      :ref:`Azure Storage fetcher <bitbake-user-manual/bitbake-user-manual-fetching:fetchers>`
+      This variable can be defined to be used by the fetcher to authenticate
+      and gain access to non-public artifacts::
+
+         AZ_SAS = ""se=2021-01-01&sp=r&sv=2018-11-09&sr=c&skoid=<skoid>&sig=<signature>""
+
+      For more information see Microsoft's Azure Storage documentation at
+      https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview
+
+
    :term:`B`
       The directory in which BitBake executes functions during a recipe's
       build process.
@@ -74,14 +83,41 @@ overview of their function and contents.
 
       -  Attempts to access networks not in the host list cause a failure.
 
-      Using ``BB_ALLOWED_NETWORKS`` in conjunction with
+      Using :term:`BB_ALLOWED_NETWORKS` in conjunction with
       :term:`PREMIRRORS` is very useful. Adding the
-      host you want to use to ``PREMIRRORS`` results in the source code
+      host you want to use to :term:`PREMIRRORS` results in the source code
       being fetched from an allowed location and avoids raising an error
       when a host that is not allowed is in a
       :term:`SRC_URI` statement. This is because the
-      fetcher does not attempt to use the host listed in ``SRC_URI`` after
-      a successful fetch from the ``PREMIRRORS`` occurs.
+      fetcher does not attempt to use the host listed in :term:`SRC_URI` after
+      a successful fetch from the :term:`PREMIRRORS` occurs.
+
+   :term:`BB_BASEHASH_IGNORE_VARS`
+      Lists variables that are excluded from checksum and dependency data.
+      Variables that are excluded can therefore change without affecting
+      the checksum mechanism. A common example would be the variable for
+      the path of the build. BitBake's output should not (and usually does
+      not) depend on the directory in which it was built.
+
+   :term:`BB_CACHEDIR`
+      Specifies the code parser cache directory (distinct from :term:`CACHE`
+      and :term:`PERSISTENT_DIR` although they can be set to the same value
+      if desired). The default value is "${TOPDIR}/cache".
+
+   :term:`BB_CHECK_SSL_CERTS`
+      Specifies if SSL certificates should be checked when fetching. The default
+      value is ``1`` and certificates are not checked if the value is set to ``0``.
+
+   :term:`BB_HASH_CODEPARSER_VALS`
+      Specifies values for variables to use when populating the codeparser cache.
+      This can be used selectively to set dummy values for variables to avoid
+      the codeparser cache growing on every parse. Variables that would typically
+      be included are those where the value is not significant for where the
+      codeparser cache is used (i.e. when calculating variable dependencies for
+      code fragments.) The value is space-separated without quoting values, for
+      example::
+
+         BB_HASH_CODEPARSER_VALS = "T=/ WORKDIR=/ DATE=1234 TIME=1234"
 
    :term:`BB_CONSOLELOG`
       Specifies the path to a log file into which BitBake's user interface
@@ -91,17 +127,10 @@ overview of their function and contents.
       Contains the name of the currently running task. The name does not
       include the ``do_`` prefix.
 
-   :term:`BB_DANGLINGAPPENDS_WARNONLY`
-      Defines how BitBake handles situations where an append file
-      (``.bbappend``) has no corresponding recipe file (``.bb``). This
-      condition often occurs when layers get out of sync (e.g. ``oe-core``
-      bumps a recipe version and the old recipe no longer exists and the
-      other layer has not been updated to the new version of the recipe
-      yet).
-
-      The default fatal behavior is safest because it is the sane reaction
-      given something is out of sync. It is important to realize when your
-      changes are no longer being applied.
+   :term:`BB_CURRENT_MC`
+      Contains the name of the current multiconfig a task is being run under.
+      The name is taken from the multiconfig configuration file (a file
+      ``mc1.conf`` would make this variable equal to ``mc1``).
 
    :term:`BB_DEFAULT_TASK`
       The default task to use when none is specified (e.g. with the ``-c``
@@ -117,14 +146,14 @@ overview of their function and contents.
       you to control the build based on these parameters.
 
       Disk space monitoring is disabled by default. When setting this
-      variable, use the following form: ::
+      variable, use the following form::
 
          BB_DISKMON_DIRS = "<action>,<dir>,<threshold> [...]"
 
          where:
 
             <action> is:
-               ABORT:     Immediately abort the build when
+               HALT:      Immediately halt the build when
                           a threshold is broken.
                STOPTASKS: Stop the build after the currently
                           executing tasks have finished when
@@ -153,48 +182,48 @@ overview of their function and contents.
                not specify G, M, or K, Kbytes is assumed by
                default.  Do not use GB, MB, or KB.
 
-      Here are some examples: ::
+      Here are some examples::
 
-         BB_DISKMON_DIRS = "ABORT,${TMPDIR},1G,100K WARN,${SSTATE_DIR},1G,100K"
+         BB_DISKMON_DIRS = "HALT,${TMPDIR},1G,100K WARN,${SSTATE_DIR},1G,100K"
          BB_DISKMON_DIRS = "STOPTASKS,${TMPDIR},1G"
-         BB_DISKMON_DIRS = "ABORT,${TMPDIR},,100K"
+         BB_DISKMON_DIRS = "HALT,${TMPDIR},,100K"
 
       The first example works only if you also set the
       :term:`BB_DISKMON_WARNINTERVAL`
-      variable. This example causes the build system to immediately abort
+      variable. This example causes the build system to immediately halt
       when either the disk space in ``${TMPDIR}`` drops below 1 Gbyte or
       the available free inodes drops below 100 Kbytes. Because two
       directories are provided with the variable, the build system also
       issues a warning when the disk space in the ``${SSTATE_DIR}``
       directory drops below 1 Gbyte or the number of free inodes drops
       below 100 Kbytes. Subsequent warnings are issued during intervals as
-      defined by the ``BB_DISKMON_WARNINTERVAL`` variable.
+      defined by the :term:`BB_DISKMON_WARNINTERVAL` variable.
 
       The second example stops the build after all currently executing
       tasks complete when the minimum disk space in the ``${TMPDIR}``
       directory drops below 1 Gbyte. No disk monitoring occurs for the free
       inodes in this case.
 
-      The final example immediately aborts the build when the number of
+      The final example immediately halts the build when the number of
       free inodes in the ``${TMPDIR}`` directory drops below 100 Kbytes. No
       disk space monitoring for the directory itself occurs in this case.
 
    :term:`BB_DISKMON_WARNINTERVAL`
       Defines the disk space and free inode warning intervals.
 
-      If you are going to use the ``BB_DISKMON_WARNINTERVAL`` variable, you
+      If you are going to use the :term:`BB_DISKMON_WARNINTERVAL` variable, you
       must also use the :term:`BB_DISKMON_DIRS`
       variable and define its action as "WARN". During the build,
       subsequent warnings are issued each time disk space or number of free
       inodes further reduces by the respective interval.
 
-      If you do not provide a ``BB_DISKMON_WARNINTERVAL`` variable and you
-      do use ``BB_DISKMON_DIRS`` with the "WARN" action, the disk
+      If you do not provide a :term:`BB_DISKMON_WARNINTERVAL` variable and you
+      do use :term:`BB_DISKMON_DIRS` with the "WARN" action, the disk
       monitoring interval defaults to the following:
       BB_DISKMON_WARNINTERVAL = "50M,5K"
 
       When specifying the variable in your configuration file, use the
-      following form: ::
+      following form::
 
          BB_DISKMON_WARNINTERVAL = "<disk_space_interval>,<disk_inode_interval>"
 
@@ -210,7 +239,7 @@ overview of their function and contents.
                G, M, or K for Gbytes, Mbytes, or Kbytes,
                respectively. You cannot use GB, MB, or KB.
 
-      Here is an example: ::
+      Here is an example::
 
          BB_DISKMON_DIRS = "WARN,${SSTATE_DIR},1G,100K"
          BB_DISKMON_WARNINTERVAL = "50M,5K"
@@ -222,23 +251,23 @@ overview of their function and contents.
       based on the interval occur each time a respective interval is
       reached beyond the initial warning (i.e. 1 Gbytes and 100 Kbytes).
 
-   :term:`BB_ENV_WHITELIST`
-      Specifies the internal whitelist of variables to allow through from
+   :term:`BB_ENV_PASSTHROUGH`
+      Specifies the internal list of variables to allow through from
       the external environment into BitBake's datastore. If the value of
       this variable is not specified (which is the default), the following
       list is used: :term:`BBPATH`, :term:`BB_PRESERVE_ENV`,
-      :term:`BB_ENV_WHITELIST`, and :term:`BB_ENV_EXTRAWHITE`.
+      :term:`BB_ENV_PASSTHROUGH`, and :term:`BB_ENV_PASSTHROUGH_ADDITIONS`.
 
       .. note::
 
          You must set this variable in the external environment in order
          for it to work.
 
-   :term:`BB_ENV_EXTRAWHITE`
-      Specifies an additional set of variables to allow through (whitelist)
-      from the external environment into BitBake's datastore. This list of
-      variables are on top of the internal list set in
-      :term:`BB_ENV_WHITELIST`.
+   :term:`BB_ENV_PASSTHROUGH_ADDITIONS`
+      Specifies an additional set of variables to allow through from the
+      external environment into BitBake's datastore. This list of variables
+      are on top of the internal list set in
+      :term:`BB_ENV_PASSTHROUGH`.
 
       .. note::
 
@@ -254,76 +283,102 @@ overview of their function and contents.
    :term:`BB_FILENAME`
       Contains the filename of the recipe that owns the currently running
       task. For example, if the ``do_fetch`` task that resides in the
-      ``my-recipe.bb`` is executing, the ``BB_FILENAME`` variable contains
+      ``my-recipe.bb`` is executing, the :term:`BB_FILENAME` variable contains
       "/foo/path/my-recipe.bb".
 
-   :term:`BBFILES_DYNAMIC`
-      Activates content depending on presence of identified layers.  You
-      identify the layers by the collections that the layers define.
+   :term:`BB_GENERATE_MIRROR_TARBALLS`
+      Causes tarballs of the Git repositories, including the Git metadata,
+      to be placed in the :term:`DL_DIR` directory. Anyone
+      wishing to create a source mirror would want to enable this variable.
 
-      Use the ``BBFILES_DYNAMIC`` variable to avoid ``.bbappend`` files whose
-      corresponding ``.bb`` file is in a layer that attempts to modify other
-      layers through ``.bbappend`` but does not want to introduce a hard
-      dependency on those other layers.
+      For performance reasons, creating and placing tarballs of the Git
+      repositories is not the default action by BitBake. ::
 
-      Additionally you can prefix the rule with "!" to add ``.bbappend`` and
-      ``.bb`` files in case a layer is not present.  Use this avoid hard
-      dependency on those other layers.
+         BB_GENERATE_MIRROR_TARBALLS = "1"
 
-      Use the following form for ``BBFILES_DYNAMIC``: ::
+   :term:`BB_GENERATE_SHALLOW_TARBALLS`
+      Setting this variable to "1" when :term:`BB_GIT_SHALLOW` is also set to
+      "1" causes bitbake to generate shallow mirror tarballs when fetching git
+      repositories. The number of commits included in the shallow mirror
+      tarballs is controlled by :term:`BB_GIT_SHALLOW_DEPTH`.
 
-         collection_name:filename_pattern
+      If both :term:`BB_GIT_SHALLOW` and :term:`BB_GENERATE_MIRROR_TARBALLS` are
+      enabled, bitbake will generate shallow mirror tarballs by default for git
+      repositories. This separate variable exists so that shallow tarball
+      generation can be enabled without needing to also enable normal mirror
+      generation if it is not desired.
 
-      The following example identifies two collection names and two filename
-      patterns: ::
+      For example usage, see :term:`BB_GIT_SHALLOW`.
 
-         BBFILES_DYNAMIC += "\
-             clang-layer:${LAYERDIR}/bbappends/meta-clang/*/*/*.bbappend \
-             core:${LAYERDIR}/bbappends/openembedded-core/meta/*/*/*.bbappend \
-         "
+   :term:`BB_GIT_DEFAULT_DESTSUFFIX`
+      The default destination directory where the Git fetcher unpacks the
+      source code. If this variable is not set, the source code is unpacked in a
+      directory named "git".
 
-      When the collection name is prefixed with "!" it will add the file pattern in case
-      the layer is absent: ::
+   :term:`BB_GIT_SHALLOW`
+      Setting this variable to "1" enables the support for fetching, using and
+      generating mirror tarballs of `shallow git repositories <https://riptutorial.com/git/example/4584/shallow-clone>`_.
+      The external `git-make-shallow <https://git.openembedded.org/bitbake/tree/bin/git-make-shallow>`_
+      script is used for shallow mirror tarball creation.
 
-         BBFILES_DYNAMIC += "\
-             !clang-layer:${LAYERDIR}/backfill/meta-clang/*/*/*.bb \
-         "
+      When :term:`BB_GIT_SHALLOW` is enabled, bitbake will attempt to fetch a shallow
+      mirror tarball. If the shallow mirror tarball cannot be fetched, it will
+      try to fetch the full mirror tarball and use that.
 
-      This next example shows an error message that occurs because invalid
-      entries are found, which cause parsing to abort: ::
+      This setting causes an initial shallow clone instead of an initial full bare clone.
+      The amount of data transferred during the initial clone will be significantly reduced.
 
-         ERROR: BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:
-         /work/my-layer/bbappends/meta-security-isafw/*/*/*.bbappend
-         /work/my-layer/bbappends/openembedded-core/meta/*/*/*.bbappend
+      However, every time the source revision (referenced in :term:`SRCREV`)
+      changes, regardless of whether the cache within the download directory
+      (defined by :term:`DL_DIR`) has been cleaned up or not,
+      the data transfer may be significantly higher because entirely
+      new shallow clones are required for each source revision change.
 
-   :term:`BB_GENERATE_MIRROR_TARBALLS`
-      Causes tarballs of the Git repositories, including the Git metadata,
-      to be placed in the :term:`DL_DIR` directory. Anyone
-      wishing to create a source mirror would want to enable this variable.
+      Over time, numerous shallow clones may cumulatively transfer
+      the same amount of data as an initial full bare clone.
+      This is especially the case with very large repositories.
 
-      For performance reasons, creating and placing tarballs of the Git
-      repositories is not the default action by BitBake. ::
+      Existing initial full bare clones, created without this setting,
+      will still be utilized.
 
-         BB_GENERATE_MIRROR_TARBALLS = "1"
+      If the Git error "Server does not allow request for unadvertised object"
+      occurs, an initial full bare clone is fetched automatically.
+      This may happen if the Git server does not allow the request
+      or if the Git client has issues with this functionality.
 
-   :term:`BB_HASHCONFIG_WHITELIST`
-      Lists variables that are excluded from base configuration checksum,
-      which is used to determine if the cache can be reused.
+      See also :term:`BB_GIT_SHALLOW_DEPTH` and
+      :term:`BB_GENERATE_SHALLOW_TARBALLS`.
 
-      One of the ways BitBake determines whether to re-parse the main
-      metadata is through checksums of the variables in the datastore of
-      the base configuration data. There are variables that you typically
-      want to exclude when checking whether or not to re-parse and thus
-      rebuild the cache. As an example, you would usually exclude ``TIME``
-      and ``DATE`` because these variables are always changing. If you did
-      not exclude them, BitBake would never reuse the cache.
+      Example usage::
 
-   :term:`BB_HASHBASE_WHITELIST`
-      Lists variables that are excluded from checksum and dependency data.
-      Variables that are excluded can therefore change without affecting
-      the checksum mechanism. A common example would be the variable for
-      the path of the build. BitBake's output should not (and usually does
-      not) depend on the directory in which it was built.
+         BB_GIT_SHALLOW ?= "1"
+
+         # Keep only the top commit
+         BB_GIT_SHALLOW_DEPTH ?= "1"
+
+         # This defaults to enabled if both BB_GIT_SHALLOW and
+         # BB_GENERATE_MIRROR_TARBALLS are enabled
+         BB_GENERATE_SHALLOW_TARBALLS ?= "1"
+
+   :term:`BB_GIT_SHALLOW_DEPTH`
+      When used with :term:`BB_GENERATE_SHALLOW_TARBALLS`, this variable sets
+      the number of commits to include in generated shallow mirror tarballs.
+      With a depth of 1, only the commit referenced in :term:`SRCREV` is
+      included in the shallow mirror tarball. Increasing the depth includes
+      additional parent commits, working back through the commit history.
+
+      If this variable is unset, bitbake will default to a depth of 1 when
+      generating shallow mirror tarballs.
+
+      For example usage, see :term:`BB_GIT_SHALLOW`.
+
+   :term:`BB_GLOBAL_PYMODULES`
+      Specifies the list of Python modules to place in the global namespace.
+      It is intended that only the core layer should set this and it is meant
+      to be a very small list, typically just ``os`` and ``sys``.
+      :term:`BB_GLOBAL_PYMODULES` is expected to be set before the first
+      ``addpylib`` directive.
+      See also ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:extending python library code`".
 
    :term:`BB_HASHCHECK_FUNCTION`
       Specifies the name of the function to call during the "setscene" part
@@ -339,12 +394,66 @@ overview of their function and contents.
       However, the more accurate the data returned, the more efficient the
       build will be.
 
+   :term:`BB_HASHCONFIG_IGNORE_VARS`
+      Lists variables that are excluded from base configuration checksum,
+      which is used to determine if the cache can be reused.
+
+      One of the ways BitBake determines whether to re-parse the main
+      metadata is through checksums of the variables in the datastore of
+      the base configuration data. There are variables that you typically
+      want to exclude when checking whether or not to re-parse and thus
+      rebuild the cache. As an example, you would usually exclude ``TIME``
+      and ``DATE`` because these variables are always changing. If you did
+      not exclude them, BitBake would never reuse the cache.
+
+   :term:`BB_HASHSERVE`
+      Specifies the Hash Equivalence server to use.
+
+      If set to ``auto``, BitBake automatically starts its own server
+      over a UNIX domain socket. An option is to connect this server
+      to an upstream one, by setting :term:`BB_HASHSERVE_UPSTREAM`.
+
+      If set to ``unix://path``, BitBake will connect to an existing
+      hash server available over a UNIX domain socket.
+
+      If set to ``host:port``, BitBake will connect to a remote server on the
+      specified host. This allows multiple clients to share the same
+      hash equivalence data.
+
+      The remote server can be started manually through
+      the ``bin/bitbake-hashserv`` script provided by BitBake,
+      which supports UNIX domain sockets too. This script also allows
+      to start the server in read-only mode, to avoid accepting
+      equivalences that correspond to Share State caches that are
+      only available on specific clients.
+
+   :term:`BB_HASHSERVE_UPSTREAM`
+      Specifies an upstream Hash Equivalence server.
+
+      This optional setting is only useful when a local Hash Equivalence
+      server is started (setting :term:`BB_HASHSERVE` to ``auto``),
+      and you wish the local server to query an upstream server for
+      Hash Equivalence data.
+
+      Example usage::
+
+         BB_HASHSERVE_UPSTREAM = "hashserv.yoctoproject.org:8686"
+
    :term:`BB_INVALIDCONF`
       Used in combination with the ``ConfigParsed`` event to trigger
       re-parsing the base metadata (i.e. all the recipes). The
       ``ConfigParsed`` event can set the variable to trigger the re-parse.
       You must be careful to avoid recursive loops with this functionality.
 
+   :term:`BB_LOADFACTOR_MAX`
+      Setting this to a value will cause BitBake to check the system load
+      average before executing new tasks. If the load average is above the
+      the number of CPUs multipled by this factor, no new task will be started
+      unless there is no task executing. A value of "1.5" has been found to
+      work reasonably. This is helpful for systems which don't have pressure
+      regulation enabled, which is more granular. Pressure values take
+      precedence over loadfactor.
+
    :term:`BB_LOGCONFIG`
       Specifies the name of a config file that contains the user logging
       configuration. See
@@ -353,15 +462,28 @@ overview of their function and contents.
 
    :term:`BB_LOGFMT`
       Specifies the name of the log files saved into
-      ``${``\ :term:`T`\ ``}``. By default, the ``BB_LOGFMT``
-      variable is undefined and the log file names get created using the
-      following form: ::
+      ``${``\ :term:`T`\ ``}``. By default, the :term:`BB_LOGFMT`
+      variable is undefined and the log filenames get created using the
+      following form::
 
          log.{task}.{pid}
 
       If you want to force log files to take a specific name, you can set this
       variable in a configuration file.
 
+   :term:`BB_MULTI_PROVIDER_ALLOWED`
+      Allows you to suppress BitBake warnings caused when building two
+      separate recipes that provide the same output.
+
+      BitBake normally issues a warning when building two different recipes
+      where each provides the same output. This scenario is usually
+      something the user does not want. However, cases do exist where it
+      makes sense, particularly in the ``virtual/*`` namespace. You can use
+      this variable to suppress BitBake's warnings.
+
+      To use the variable, list provider names (e.g. recipe names,
+      ``virtual/kernel``, and so forth).
+
    :term:`BB_NICE_LEVEL`
       Allows BitBake to run at a specific priority (i.e. nice level).
       System permissions usually mean that BitBake can reduce its priority
@@ -377,19 +499,20 @@ overview of their function and contents.
       running builds when not connected to the Internet, and when operating
       in certain kinds of firewall environments.
 
+   :term:`BB_NUMBER_PARSE_THREADS`
+      Sets the number of threads BitBake uses when parsing. By default, the
+      number of threads is equal to the number of cores on the system.
+
    :term:`BB_NUMBER_THREADS`
       The maximum number of tasks BitBake should run in parallel at any one
       time. If your host development system supports multiple cores, a good
       rule of thumb is to set this variable to twice the number of cores.
 
-   :term:`BB_NUMBER_PARSE_THREADS`
-      Sets the number of threads BitBake uses when parsing. By default, the
-      number of threads is equal to the number of cores on the system.
-
    :term:`BB_ORIGENV`
       Contains a copy of the original external environment in which BitBake
-      was run. The copy is taken before any whitelisted variable values are
-      filtered into BitBake's datastore.
+      was run. The copy is taken before any variable values configured to
+      pass through from the external environment are filtered into BitBake's
+      datastore.
 
       .. note::
 
@@ -397,21 +520,130 @@ overview of their function and contents.
          queried using the normal datastore operations.
 
    :term:`BB_PRESERVE_ENV`
-      Disables whitelisting and instead allows all variables through from
-      the external environment into BitBake's datastore.
+      Disables environment filtering and instead allows all variables through
+      from the external environment into BitBake's datastore.
 
       .. note::
 
          You must set this variable in the external environment in order
          for it to work.
 
+   :term:`BB_PRESSURE_MAX_CPU`
+      Specifies a maximum CPU pressure threshold, above which BitBake's
+      scheduler will not start new tasks (providing there is at least
+      one active task). If no value is set, CPU pressure is not
+      monitored when starting tasks.
+
+      The pressure data is calculated based upon what Linux kernels since
+      version 4.20 expose under ``/proc/pressure``. The threshold represents
+      the difference in "total" pressure from the previous second. The
+      minimum value is 1.0 (extremely slow builds) and the maximum is
+      1000000 (a pressure value unlikely to ever be reached). See
+      https://docs.kernel.org/accounting/psi.html for more information.
+
+      A default value to limit the CPU pressure to be set in ``conf/local.conf``
+      could be::
+
+         BB_PRESSURE_MAX_CPU = "15000"
+
+      Multiple values should be tested on the build host to determine what suits
+      best, depending on the need for performances versus load average during
+      the build.
+
+      .. note::
+
+         You may see numerous messages printed by BitBake in the case the
+         :term:`BB_PRESSURE_MAX_CPU` is too low:
+
+            Pressure status changed to CPU: True, IO: False, Mem: False (CPU: 1105.9/2.0, IO: 0.0/2.0, Mem: 0.0/2.0) - using 1/64 bitbake threads
+
+         This means that the :term:`BB_PRESSURE_MAX_CPU` should be increased to
+         a reasonable value for limiting the CPU pressure on the system.
+         Monitor the varying value after ``IO:`` above to set a sensible value.
+
+   :term:`BB_PRESSURE_MAX_IO`
+      Specifies a maximum I/O pressure threshold, above which BitBake's
+      scheduler will not start new tasks (providing there is at least
+      one active task). If no value is set, I/O pressure is not
+      monitored when starting tasks.
+
+      The pressure data is calculated based upon what Linux kernels since
+      version 4.20 expose under ``/proc/pressure``. The threshold represents
+      the difference in "total" pressure from the previous second. The
+      minimum value is 1.0 (extremely slow builds) and the maximum is
+      1000000 (a pressure value unlikely to ever be reached). See
+      https://docs.kernel.org/accounting/psi.html for more information.
+
+      At this point in time, experiments show that IO pressure tends to
+      be short-lived and regulating just the CPU with
+      :term:`BB_PRESSURE_MAX_CPU` can help to reduce it.
+
+      A default value to limit the IO pressure to be set in ``conf/local.conf``
+      could be::
+
+         BB_PRESSURE_MAX_IO = "15000"
+
+      Multiple values should be tested on the build host to determine what suits
+      best, depending on the need for performances versus I/O usage during the
+      build.
+
+      .. note::
+
+         You may see numerous messages printed by BitBake in the case the
+         :term:`BB_PRESSURE_MAX_IO` is too low::
+
+            Pressure status changed to CPU: None, IO: True, Mem: False (CPU: 2236.0/None, IO: 153.6/2.0, Mem: 0.0/2.0) - using 19/64 bitbake threads
+
+         This means that the :term:`BB_PRESSURE_MAX_IO` should be increased to
+         a reasonable value for limiting the I/O pressure on the system.
+         Monitor the varying value after ``IO:`` above to set a sensible value.
+
+   :term:`BB_PRESSURE_MAX_MEMORY`
+      Specifies a maximum memory pressure threshold, above which BitBake's
+      scheduler will not start new tasks (providing there is at least
+      one active task). If no value is set, memory pressure is not
+      monitored when starting tasks.
+
+      The pressure data is calculated based upon what Linux kernels since
+      version 4.20 expose under ``/proc/pressure``. The threshold represents
+      the difference in "total" pressure from the previous second. The
+      minimum value is 1.0 (extremely slow builds) and the maximum is
+      1000000 (a pressure value unlikely to ever be reached). See
+      https://docs.kernel.org/accounting/psi.html for more information.
+
+      Memory pressure is experienced when time is spent swapping,
+      refaulting pages from the page cache or performing direct reclaim.
+      This is why memory pressure is rarely seen, but setting this variable
+      might be useful as a last resort to prevent OOM errors if they are
+      occurring during builds.
+
+      A default value to limit the memory pressure to be set in
+      ``conf/local.conf`` could be::
+
+         BB_PRESSURE_MAX_MEMORY = "15000"
+
+      Multiple values should be tested on the build host to determine what suits
+      best, depending on the need for performances versus memory consumption
+      during the build.
+
+      .. note::
+
+         You may see numerous messages printed by BitBake in the case the
+         :term:`BB_PRESSURE_MAX_MEMORY` is too low::
+
+            Pressure status changed to CPU: None, IO: False, Mem: True (CPU: 29.5/None, IO: 0.0/2.0, Mem: 2553.3/2.0) - using 17/64 bitbake threads
+
+         This means that the :term:`BB_PRESSURE_MAX_MEMORY` should be increased to
+         a reasonable value for limiting the memory pressure on the system.
+         Monitor the varying value after ``Mem:`` above to set a sensible value.
+
    :term:`BB_RUNFMT`
       Specifies the name of the executable script files (i.e. run files)
       saved into ``${``\ :term:`T`\ ``}``. By default, the
-      ``BB_RUNFMT`` variable is undefined and the run file names get
-      created using the following form: ::
+      :term:`BB_RUNFMT` variable is undefined and the run filenames get
+      created using the following form::
 
-         run.{task}.{pid}
+         run.{func}.{pid}
 
       If you want to force run files to take a specific name, you can set this
       variable in a configuration file.
@@ -425,14 +657,14 @@ overview of their function and contents.
       Selects the name of the scheduler to use for the scheduling of
       BitBake tasks. Three options exist:
 
-      -  *basic* - The basic framework from which everything derives. Using
+      -  *basic* --- the basic framework from which everything derives. Using
          this option causes tasks to be ordered numerically as they are
          parsed.
 
-      -  *speed* - Executes tasks first that have more tasks depending on
+      -  *speed* --- executes tasks first that have more tasks depending on
          them. The "speed" option is the default.
 
-      -  *completion* - Causes the scheduler to try to complete a given
+      -  *completion* --- causes the scheduler to try to complete a given
          recipe once its build has started.
 
    :term:`BB_SCHEDULERS`
@@ -452,17 +684,6 @@ overview of their function and contents.
       The function specified by this variable returns a "True" or "False"
       depending on whether the dependency needs to be met.
 
-   :term:`BB_SETSCENE_VERIFY_FUNCTION2`
-      Specifies a function to call that verifies the list of planned task
-      execution before the main task execution happens. The function is
-      called once BitBake has a list of setscene tasks that have run and
-      either succeeded or failed.
-
-      The function allows for a task list check to see if they make sense.
-      Even if BitBake was planning to skip a task, the returned value of
-      the function can force BitBake to run the task, which is necessary
-      under certain metadata defined circumstances.
-
    :term:`BB_SIGNATURE_EXCLUDE_FLAGS`
       Lists variable flags (varflags) that can be safely excluded from
       checksum and dependency data for keys in the datastore. When
@@ -485,40 +706,17 @@ overview of their function and contents.
    :term:`BB_SRCREV_POLICY`
       Defines the behavior of the fetcher when it interacts with source
       control systems and dynamic source revisions. The
-      ``BB_SRCREV_POLICY`` variable is useful when working without a
+      :term:`BB_SRCREV_POLICY` variable is useful when working without a
       network.
 
       The variable can be set using one of two policies:
 
-      -  *cache* - Retains the value the system obtained previously rather
+      -  *cache* --- retains the value the system obtained previously rather
          than querying the source control system each time.
 
-      -  *clear* - Queries the source controls system every time. With this
+      -  *clear* --- queries the source controls system every time. With this
          policy, there is no cache. The "clear" policy is the default.
 
-   :term:`BB_STAMP_POLICY`
-      Defines the mode used for how timestamps of stamp files are compared.
-      You can set the variable to one of the following modes:
-
-      -  *perfile* - Timestamp comparisons are only made between timestamps
-         of a specific recipe. This is the default mode.
-
-      -  *full* - Timestamp comparisons are made for all dependencies.
-
-      -  *whitelist* - Identical to "full" mode except timestamp
-         comparisons are made for recipes listed in the
-         :term:`BB_STAMP_WHITELIST` variable.
-
-      .. note::
-
-         Stamp policies are largely obsolete with the introduction of
-         setscene tasks.
-
-   :term:`BB_STAMP_WHITELIST`
-      Lists files whose stamp file timestamps are compared when the stamp
-      policy mode is set to "whitelist". For information on stamp policies,
-      see the :term:`BB_STAMP_POLICY` variable.
-
    :term:`BB_STRICT_CHECKSUM`
       Sets a more strict checksum mechanism for non-local URLs. Setting
       this variable to a value causes BitBake to report an error if it
@@ -529,7 +727,7 @@ overview of their function and contents.
       Allows adjustment of a task's Input/Output priority. During
       Autobuilder testing, random failures can occur for tasks due to I/O
       starvation. These failures occur during various QEMU runtime
-      timeouts. You can use the ``BB_TASK_IONICE_LEVEL`` variable to adjust
+      timeouts. You can use the :term:`BB_TASK_IONICE_LEVEL` variable to adjust
       the I/O priority of these tasks.
 
       .. note::
@@ -537,7 +735,7 @@ overview of their function and contents.
          This variable works similarly to the :term:`BB_TASK_NICE_LEVEL`
          variable except with a task's I/O priorities.
 
-      Set the variable as follows: ::
+      Set the variable as follows::
 
          BB_TASK_IONICE_LEVEL = "class.prio"
 
@@ -555,7 +753,7 @@ overview of their function and contents.
          In order for your I/O priority settings to take effect, you need the
          Completely Fair Queuing (CFQ) Scheduler selected for the backing block
          device. To select the scheduler, use the following command form where
-         device is the device (e.g. sda, sdb, and so forth): ::
+         device is the device (e.g. sda, sdb, and so forth)::
 
             $ sudo sh -c "echo cfq > /sys/block/device/queu/scheduler"
 
@@ -564,7 +762,7 @@ overview of their function and contents.
 
       You can use this variable in combination with task overrides to raise
       or lower priorities of specific tasks. For example, on the `Yocto
-      Project <http://www.yoctoproject.org>`__ autobuilder, QEMU emulation
+      Project <https://www.yoctoproject.org>`__ autobuilder, QEMU emulation
       in images is given a higher priority as compared to build tasks to
       ensure that images do not suffer timeouts on loaded systems.
 
@@ -572,6 +770,12 @@ overview of their function and contents.
       Within an executing task, this variable holds the hash of the task as
       returned by the currently enabled signature generator.
 
+   :term:`BB_USE_HOME_NPMRC`
+      Controls whether or not BitBake uses the user's .npmrc file within their
+      home directory within the npm fetcher. This can be used for authentication
+      of private NPM registries, among other uses. This is turned off by default
+      and requires the user to explicitly set it to "1" to enable.
+
    :term:`BB_VERBOSE_LOGS`
       Controls how verbose BitBake is during builds. If set, shell scripts
       echo commands and shell script output appears on standard out
@@ -596,20 +800,20 @@ overview of their function and contents.
       To build a different variant of the recipe with a minimal amount of
       code, it usually is as simple as adding the variable to your recipe.
       Here are two examples. The "native" variants are from the
-      OpenEmbedded-Core metadata: ::
+      OpenEmbedded-Core metadata::
 
          BBCLASSEXTEND =+ "native nativesdk"
          BBCLASSEXTEND =+ "multilib:multilib_name"
 
       .. note::
 
-         Internally, the ``BBCLASSEXTEND`` mechanism generates recipe
+         Internally, the :term:`BBCLASSEXTEND` mechanism generates recipe
          variants by rewriting variable values and applying overrides such
          as ``_class-native``. For example, to generate a native version of
          a recipe, a :term:`DEPENDS` on "foo" is
-         rewritten to a ``DEPENDS`` on "foo-native".
+         rewritten to a :term:`DEPENDS` on "foo-native".
 
-         Even when using ``BBCLASSEXTEND``, the recipe is only parsed once.
+         Even when using :term:`BBCLASSEXTEND`, the recipe is only parsed once.
          Parsing once adds some limitations. For example, it is not
          possible to include a different file depending on the variant,
          since ``include`` statements are processed when the recipe is
@@ -639,21 +843,25 @@ overview of their function and contents.
    :term:`BBFILE_PRIORITY`
       Assigns the priority for recipe files in each layer.
 
+      This variable is used in the ``conf/layer.conf`` file and must be
+      suffixed with a `_` followed by the name of the specific layer (e.g.
+      ``BBFILE_PRIORITY_emenlow``). Colon as separator is not supported.
+
       This variable is useful in situations where the same recipe appears
       in more than one layer. Setting this variable allows you to
       prioritize a layer against other layers that contain the same recipe
-      - effectively letting you control the precedence for the multiple
+      --- effectively letting you control the precedence for the multiple
       layers. The precedence established through this variable stands
       regardless of a recipe's version (:term:`PV` variable).
-      For example, a layer that has a recipe with a higher ``PV`` value but
-      for which the ``BBFILE_PRIORITY`` is set to have a lower precedence
+      For example, a layer that has a recipe with a higher :term:`PV` value but
+      for which the :term:`BBFILE_PRIORITY` is set to have a lower precedence
       still has a lower precedence.
 
-      A larger value for the ``BBFILE_PRIORITY`` variable results in a
+      A larger value for the :term:`BBFILE_PRIORITY` variable results in a
       higher precedence. For example, the value 6 has a higher precedence
-      than the value 5. If not specified, the ``BBFILE_PRIORITY`` variable
-      is set based on layer dependencies (see the ``LAYERDEPENDS`` variable
-      for more information. The default priority, if unspecified for a
+      than the value 5. If not specified, the :term:`BBFILE_PRIORITY` variable
+      is set based on layer dependencies (see the :term:`LAYERDEPENDS` variable
+      for more information). The default priority, if unspecified for a
       layer with no dependencies, is the lowest defined priority + 1 (or 1
       if no priorities are defined).
 
@@ -671,6 +879,45 @@ overview of their function and contents.
       For details on the syntax, see the documentation by following the
       previous link.
 
+   :term:`BBFILES_DYNAMIC`
+      Activates content depending on presence of identified layers.  You
+      identify the layers by the collections that the layers define.
+
+      Use the :term:`BBFILES_DYNAMIC` variable to avoid ``.bbappend`` files whose
+      corresponding ``.bb`` file is in a layer that attempts to modify other
+      layers through ``.bbappend`` but does not want to introduce a hard
+      dependency on those other layers.
+
+      Additionally you can prefix the rule with "!" to add ``.bbappend`` and
+      ``.bb`` files in case a layer is not present.  Use this avoid hard
+      dependency on those other layers.
+
+      Use the following form for :term:`BBFILES_DYNAMIC`::
+
+         collection_name:filename_pattern
+
+      The following example identifies two collection names and two filename
+      patterns::
+
+         BBFILES_DYNAMIC += "\
+             clang-layer:${LAYERDIR}/bbappends/meta-clang/*/*/*.bbappend \
+             core:${LAYERDIR}/bbappends/openembedded-core/meta/*/*/*.bbappend \
+         "
+
+      When the collection name is prefixed with "!" it will add the file pattern in case
+      the layer is absent::
+
+         BBFILES_DYNAMIC += "\
+             !clang-layer:${LAYERDIR}/backfill/meta-clang/*/*/*.bb \
+         "
+
+      This next example shows an error message that occurs because invalid
+      entries are found, which cause parsing to fail::
+
+         ERROR: BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:
+         /work/my-layer/bbappends/meta-security-isafw/*/*/*.bbappend
+         /work/my-layer/bbappends/openembedded-core/meta/*/*/*.bbappend
+
    :term:`BBINCLUDED`
       Contains a space-separated list of all of all files that BitBake's
       parser included during parsing of the current file.
@@ -682,13 +929,13 @@ overview of their function and contents.
    :term:`BBINCLUDELOGS_LINES`
       If :term:`BBINCLUDELOGS` is set, specifies
       the maximum number of lines from the task log file to print when
-      reporting a failed task. If you do not set ``BBINCLUDELOGS_LINES``,
+      reporting a failed task. If you do not set :term:`BBINCLUDELOGS_LINES`,
       the entire log is printed.
 
    :term:`BBLAYERS`
       Lists the layers to enable during the build. This variable is defined
       in the ``bblayers.conf`` configuration file in the build directory.
-      Here is an example: ::
+      Here is an example::
 
          BBLAYERS = " \
              /home/scottrif/poky/meta \
@@ -708,7 +955,7 @@ overview of their function and contents.
    :term:`BBMASK`
       Prevents BitBake from processing recipes and recipe append files.
 
-      You can use the ``BBMASK`` variable to "hide" these ``.bb`` and
+      You can use the :term:`BBMASK` variable to "hide" these ``.bb`` and
       ``.bbappend`` files. BitBake ignores any recipe or recipe append
       files that match any of the expressions. It is as if BitBake does not
       see them at all. Consequently, matching files are not parsed or
@@ -722,13 +969,13 @@ overview of their function and contents.
 
       The following example uses a complete regular expression to tell
       BitBake to ignore all recipe and recipe append files in the
-      ``meta-ti/recipes-misc/`` directory: ::
+      ``meta-ti/recipes-misc/`` directory::
 
          BBMASK = "meta-ti/recipes-misc/"
 
       If you want to mask out multiple directories or recipes, you can
       specify multiple regular expression fragments. This next example
-      masks out multiple directories and individual recipes: ::
+      masks out multiple directories and individual recipes::
 
          BBMASK += "/meta-ti/recipes-misc/ meta-ti/recipes-ti/packagegroup/"
          BBMASK += "/meta-oe/recipes-support/"
@@ -745,11 +992,11 @@ overview of their function and contents.
       Enables BitBake to perform multiple configuration builds and lists
       each separate configuration (multiconfig). You can use this variable
       to cause BitBake to build multiple targets where each target has a
-      separate configuration. Define ``BBMULTICONFIG`` in your
+      separate configuration. Define :term:`BBMULTICONFIG` in your
       ``conf/local.conf`` configuration file.
 
       As an example, the following line specifies three multiconfigs, each
-      having a separate configuration file: ::
+      having a separate configuration file::
 
          BBMULTIFONFIG = "configA configB configC"
 
@@ -757,20 +1004,20 @@ overview of their function and contents.
       build directory within a directory named ``conf/multiconfig`` (e.g.
       build_directory\ ``/conf/multiconfig/configA.conf``).
 
-      For information on how to use ``BBMULTICONFIG`` in an environment
+      For information on how to use :term:`BBMULTICONFIG` in an environment
       that supports building targets with multiple configurations, see the
       ":ref:`bitbake-user-manual/bitbake-user-manual-intro:executing a multiple configuration build`"
       section.
 
    :term:`BBPATH`
-      Used by BitBake to locate class (``.bbclass``) and configuration
-      (``.conf``) files. This variable is analogous to the ``PATH``
-      variable.
+      A colon-separated list used by BitBake to locate class (``.bbclass``)
+      and configuration (``.conf``) files. This variable is analogous to the
+      ``PATH`` variable.
 
       If you run BitBake from a directory outside of the build directory,
-      you must be sure to set ``BBPATH`` to point to the build directory.
+      you must be sure to set :term:`BBPATH` to point to the build directory.
       Set the variable as you would any environment variable and then run
-      BitBake: ::
+      BitBake::
 
          $ BBPATH="build_directory"
          $ export BBPATH
@@ -784,16 +1031,6 @@ overview of their function and contents.
       Allows you to use a configuration file to add to the list of
       command-line target recipes you want to build.
 
-   :term:`BBVERSIONS`
-      Allows a single recipe to build multiple versions of a project from a
-      single recipe file. You also able to specify conditional metadata
-      using the :term:`OVERRIDES` mechanism for a
-      single version or for an optionally named range of versions.
-
-      For more information on ``BBVERSIONS``, see the
-      ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:variants - class extension mechanism`"
-      section.
-
    :term:`BITBAKE_UI`
       Used to specify the UI module to use when running BitBake. Using this
       variable is equivalent to using the ``-u`` command-line option.
@@ -825,7 +1062,7 @@ overview of their function and contents.
       The most common usage of this is variable is to set it to "-1" within
       a recipe for a development version of a piece of software. Using the
       variable in this way causes the stable version of the recipe to build
-      by default in the absence of ``PREFERRED_VERSION`` being used to
+      by default in the absence of :term:`PREFERRED_VERSION` being used to
       build the development version.
 
       .. note::
@@ -838,8 +1075,8 @@ overview of their function and contents.
       Lists a recipe's build-time dependencies (i.e. other recipe files).
 
       Consider this simple example for two recipes named "a" and "b" that
-      produce similarly named packages. In this example, the ``DEPENDS``
-      statement appears in the "a" recipe: ::
+      produce similarly named packages. In this example, the :term:`DEPENDS`
+      statement appears in the "a" recipe::
 
          DEPENDS = "b"
 
@@ -856,7 +1093,7 @@ overview of their function and contents.
 
    :term:`DL_DIR`
       The central download directory used by the build process to store
-      downloads. By default, ``DL_DIR`` gets files suitable for mirroring for
+      downloads. By default, :term:`DL_DIR` gets files suitable for mirroring for
       everything except Git repositories. If you want tarballs of Git
       repositories, use the :term:`BB_GENERATE_MIRROR_TARBALLS` variable.
 
@@ -867,18 +1104,18 @@ overview of their function and contents.
       ``bblayers.conf`` configuration file.
 
       To exclude a recipe from a world build using this variable, set the
-      variable to "1" in the recipe.
+      variable to "1" in the recipe. Set it to "0" to add it back to world build.
 
       .. note::
 
-         Recipes added to ``EXCLUDE_FROM_WORLD`` may still be built during a world
+         Recipes added to :term:`EXCLUDE_FROM_WORLD` may still be built during a world
          build in order to satisfy dependencies of other recipes. Adding a
-         recipe to ``EXCLUDE_FROM_WORLD`` only ensures that the recipe is not
+         recipe to :term:`EXCLUDE_FROM_WORLD` only ensures that the recipe is not
          explicitly added to the list of build targets in a world build.
 
    :term:`FAKEROOT`
       Contains the command to use when running a shell script in a fakeroot
-      environment. The ``FAKEROOT`` variable is obsolete and has been
+      environment. The :term:`FAKEROOT` variable is obsolete and has been
       replaced by the other ``FAKEROOT*`` variables. See these entries in
       the glossary for more information.
 
@@ -925,6 +1162,11 @@ overview of their function and contents.
       environment variable. The value is a colon-separated list of
       directories that are searched left-to-right in order.
 
+   :term:`FILE_LAYERNAME`
+      During parsing and task execution, this is set to the name of the
+      layer containing the recipe file. Code can use this to identify which
+      layer a recipe is from.
+
    :term:`GITDIR`
       The directory in which a local copy of a Git repository is stored
       when it is cloned.
@@ -941,9 +1183,9 @@ overview of their function and contents.
       Causes the named class or classes to be inherited globally. Anonymous
       functions in the class or classes are not executed for the base
       configuration and in each individual recipe. The OpenEmbedded build
-      system ignores changes to ``INHERIT`` in individual recipes.
+      system ignores changes to :term:`INHERIT` in individual recipes.
 
-      For more information on ``INHERIT``, see the
+      For more information on :term:`INHERIT`, see the
       ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:\`\`inherit\`\` configuration directive`"
       section.
 
@@ -973,6 +1215,29 @@ overview of their function and contents.
       variable is not available outside of ``layer.conf`` and references
       are expanded immediately when parsing of the file completes.
 
+   :term:`LAYERSERIES_COMPAT`
+      Lists the versions of the OpenEmbedded-Core (OE-Core) for which
+      a layer is compatible. Using the :term:`LAYERSERIES_COMPAT` variable
+      allows the layer maintainer to indicate which combinations of the
+      layer and OE-Core can be expected to work. The variable gives the
+      system a way to detect when a layer has not been tested with new
+      releases of OE-Core (e.g. the layer is not maintained).
+
+      To specify the OE-Core versions for which a layer is compatible, use
+      this variable in your layer's ``conf/layer.conf`` configuration file.
+      For the list, use the Yocto Project release name (e.g. "kirkstone",
+      "mickledore"). To specify multiple OE-Core versions for the layer, use
+      a space-separated list::
+
+         LAYERSERIES_COMPAT_layer_root_name = "kirkstone mickledore"
+
+      .. note::
+
+         Setting :term:`LAYERSERIES_COMPAT` is required by the Yocto Project
+         Compatible version 2 standard.
+         The OpenEmbedded build system produces a warning if the variable
+         is not set for any given layer.
+
    :term:`LAYERVERSION`
       Optionally specifies the version of a layer as a single number. You
       can use this variable within
@@ -991,29 +1256,16 @@ overview of their function and contents.
       the build system searches for source code, it first tries the local
       download directory. If that location fails, the build system tries
       locations defined by :term:`PREMIRRORS`, the
-      upstream source, and then locations specified by ``MIRRORS`` in that
+      upstream source, and then locations specified by :term:`MIRRORS` in that
       order.
 
-   :term:`MULTI_PROVIDER_WHITELIST`
-      Allows you to suppress BitBake warnings caused when building two
-      separate recipes that provide the same output.
-
-      BitBake normally issues a warning when building two different recipes
-      where each provides the same output. This scenario is usually
-      something the user does not want. However, cases do exist where it
-      makes sense, particularly in the ``virtual/*`` namespace. You can use
-      this variable to suppress BitBake's warnings.
-
-      To use the variable, list provider names (e.g. recipe names,
-      ``virtual/kernel``, and so forth).
-
    :term:`OVERRIDES`
-      BitBake uses ``OVERRIDES`` to control what variables are overridden
-      after BitBake parses recipes and configuration files.
+      A colon-separated list that BitBake uses to control what variables are
+      overridden after BitBake parses recipes and configuration files.
 
       Following is a simple example that uses an overrides list based on
       machine architectures: OVERRIDES = "arm:x86:mips:powerpc" You can
-      find information on how to use ``OVERRIDES`` in the
+      find information on how to use :term:`OVERRIDES` in the
       ":ref:`bitbake-user-manual/bitbake-user-manual-metadata:conditional syntax
       (overrides)`" section.
 
@@ -1027,11 +1279,11 @@ overview of their function and contents.
    :term:`PACKAGES_DYNAMIC`
       A promise that your recipe satisfies runtime dependencies for
       optional modules that are found in other recipes.
-      ``PACKAGES_DYNAMIC`` does not actually satisfy the dependencies, it
+      :term:`PACKAGES_DYNAMIC` does not actually satisfy the dependencies, it
       only states that they should be satisfied. For example, if a hard,
       runtime dependency (:term:`RDEPENDS`) of another
       package is satisfied during the build through the
-      ``PACKAGES_DYNAMIC`` variable, but a package with the module name is
+      :term:`PACKAGES_DYNAMIC` variable, but a package with the module name is
       never actually produced, then the other package will be broken.
 
    :term:`PE`
@@ -1061,7 +1313,7 @@ overview of their function and contents.
       recipes provide the same item. You should always suffix the variable
       with the name of the provided item, and you should set it to the
       :term:`PN` of the recipe to which you want to give
-      precedence. Some examples: ::
+      precedence. Some examples::
 
          PREFERRED_PROVIDER_virtual/kernel ?= "linux-yocto"
          PREFERRED_PROVIDER_virtual/xserver = "xserver-xf86"
@@ -1070,30 +1322,30 @@ overview of their function and contents.
    :term:`PREFERRED_PROVIDERS`
       Determines which recipe should be given preference for cases where
       multiple recipes provide the same item. Functionally,
-      ``PREFERRED_PROVIDERS`` is identical to
-      :term:`PREFERRED_PROVIDER`. However, the ``PREFERRED_PROVIDERS`` variable
+      :term:`PREFERRED_PROVIDERS` is identical to
+      :term:`PREFERRED_PROVIDER`. However, the :term:`PREFERRED_PROVIDERS` variable
       lets you define preferences for multiple situations using the following
-      form: ::
+      form::
 
          PREFERRED_PROVIDERS = "xxx:yyy aaa:bbb ..."
 
-      This form is a convenient replacement for the following: ::
+      This form is a convenient replacement for the following::
 
          PREFERRED_PROVIDER_xxx = "yyy"
          PREFERRED_PROVIDER_aaa = "bbb"
 
    :term:`PREFERRED_VERSION`
-      If there are multiple versions of recipes available, this variable
-      determines which recipe should be given preference. You must always
+      If there are multiple versions of a recipe available, this variable
+      determines which version should be given preference. You must always
       suffix the variable with the :term:`PN` you want to
       select, and you should set :term:`PV` accordingly for
       precedence.
 
-      The ``PREFERRED_VERSION`` variable supports limited wildcard use
+      The :term:`PREFERRED_VERSION` variable supports limited wildcard use
       through the "``%``" character. You can use the character to match any
       number of characters, which can be useful when specifying versions
       that contain long revision numbers that potentially change. Here are
-      two examples: ::
+      two examples::
 
          PREFERRED_VERSION_python = "2.7.3"
          PREFERRED_VERSION_linux-yocto = "4.12%"
@@ -1104,22 +1356,26 @@ overview of their function and contents.
          end of the string. You cannot use the wildcard character in any other
          location of the string.
 
+      If a recipe with the specified version is not available, a warning
+      message will be shown. See :term:`REQUIRED_VERSION` if you want this
+      to be an error instead.
+
    :term:`PREMIRRORS`
       Specifies additional paths from which BitBake gets source code. When
       the build system searches for source code, it first tries the local
       download directory. If that location fails, the build system tries
-      locations defined by ``PREMIRRORS``, the upstream source, and then
+      locations defined by :term:`PREMIRRORS`, the upstream source, and then
       locations specified by :term:`MIRRORS` in that order.
 
       Typically, you would add a specific server for the build system to
       attempt before any others by adding something like the following to
-      your configuration: ::
+      your configuration::
 
-         PREMIRRORS_prepend = "\
-         git://.*/.* http://www.yoctoproject.org/sources/ \n \
-         ftp://.*/.* http://www.yoctoproject.org/sources/ \n \
-         http://.*/.* http://www.yoctoproject.org/sources/ \n \
-         https://.*/.* http://www.yoctoproject.org/sources/ \n"
+         PREMIRRORS:prepend = "\
+         git://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \
+         ftp://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \
+         http://.*/.* http://downloads.yoctoproject.org/mirror/sources/ \
+         https://.*/.* http://downloads.yoctoproject.org/mirror/sources/"
 
       These changes cause the build system to intercept Git, FTP, HTTP, and
       HTTPS requests and direct them to the ``http://`` sources mirror. You can
@@ -1128,25 +1384,25 @@ overview of their function and contents.
 
    :term:`PROVIDES`
       A list of aliases by which a particular recipe can be known. By
-      default, a recipe's own ``PN`` is implicitly already in its
-      ``PROVIDES`` list. If a recipe uses ``PROVIDES``, the additional
+      default, a recipe's own :term:`PN` is implicitly already in its
+      :term:`PROVIDES` list. If a recipe uses :term:`PROVIDES`, the additional
       aliases are synonyms for the recipe and can be useful satisfying
       dependencies of other recipes during the build as specified by
-      ``DEPENDS``.
+      :term:`DEPENDS`.
 
-      Consider the following example ``PROVIDES`` statement from a recipe
-      file ``libav_0.8.11.bb``: ::
+      Consider the following example :term:`PROVIDES` statement from a recipe
+      file ``libav_0.8.11.bb``::
 
          PROVIDES += "libpostproc"
 
-      The ``PROVIDES`` statement results in the "libav" recipe also being known
+      The :term:`PROVIDES` statement results in the "libav" recipe also being known
       as "libpostproc".
 
       In addition to providing recipes under alternate names, the
-      ``PROVIDES`` mechanism is also used to implement virtual targets. A
+      :term:`PROVIDES` mechanism is also used to implement virtual targets. A
       virtual target is a name that corresponds to some particular
       functionality (e.g. a Linux kernel). Recipes that provide the
-      functionality in question list the virtual target in ``PROVIDES``.
+      functionality in question list the virtual target in :term:`PROVIDES`.
       Recipes that depend on the functionality in question can include the
       virtual target in :term:`DEPENDS` to leave the
       choice of provider open.
@@ -1158,12 +1414,12 @@ overview of their function and contents.
    :term:`PRSERV_HOST`
       The network based :term:`PR` service host and port.
 
-      Following is an example of how the ``PRSERV_HOST`` variable is set: ::
+      Following is an example of how the :term:`PRSERV_HOST` variable is set::
 
          PRSERV_HOST = "localhost:0"
 
       You must set the variable if you want to automatically start a local PR
-      service. You can set ``PRSERV_HOST`` to other values to use a remote PR
+      service. You can set :term:`PRSERV_HOST` to other values to use a remote PR
       service.
 
    :term:`PV`
@@ -1175,26 +1431,26 @@ overview of their function and contents.
       a package in this list cannot be found during the build, you will get
       a build error.
 
-      Because the ``RDEPENDS`` variable applies to packages being built,
+      Because the :term:`RDEPENDS` variable applies to packages being built,
       you should always use the variable in a form with an attached package
       name. For example, suppose you are building a development package
       that depends on the ``perl`` package. In this case, you would use the
-      following ``RDEPENDS`` statement: ::
+      following :term:`RDEPENDS` statement::
 
-         RDEPENDS_${PN}-dev += "perl"
+         RDEPENDS:${PN}-dev += "perl"
 
       In the example, the development package depends on the ``perl`` package.
-      Thus, the ``RDEPENDS`` variable has the ``${PN}-dev`` package name as part
+      Thus, the :term:`RDEPENDS` variable has the ``${PN}-dev`` package name as part
       of the variable.
 
       BitBake supports specifying versioned dependencies. Although the
       syntax varies depending on the packaging format, BitBake hides these
       differences from you. Here is the general syntax to specify versions
-      with the ``RDEPENDS`` variable: ::
+      with the :term:`RDEPENDS` variable::
 
-         RDEPENDS_${PN} = "package (operator version)"
+         RDEPENDS:${PN} = "package (operator version)"
 
-      For ``operator``, you can specify the following: ::
+      For ``operator``, you can specify the following::
 
          =
          <
@@ -1203,9 +1459,9 @@ overview of their function and contents.
          >=
 
       For example, the following sets up a dependency on version 1.2 or
-      greater of the package ``foo``: ::
+      greater of the package ``foo``::
 
-         RDEPENDS_${PN} = "foo (>= 1.2)"
+         RDEPENDS:${PN} = "foo (>= 1.2)"
 
       For information on build-time dependencies, see the :term:`DEPENDS`
       variable.
@@ -1214,33 +1470,43 @@ overview of their function and contents.
       The directory in which a local copy of a ``google-repo`` directory is
       stored when it is synced.
 
+   :term:`REQUIRED_VERSION`
+      If there are multiple versions of a recipe available, this variable
+      determines which version should be given preference. :term:`REQUIRED_VERSION`
+      works in exactly the same manner as :term:`PREFERRED_VERSION`, except
+      that if the specified version is not available then an error message
+      is shown and the build fails immediately.
+
+      If both :term:`REQUIRED_VERSION` and :term:`PREFERRED_VERSION` are set for
+      the same recipe, the :term:`REQUIRED_VERSION` value applies.
+
    :term:`RPROVIDES`
       A list of package name aliases that a package also provides. These
       aliases are useful for satisfying runtime dependencies of other
       packages both during the build and on the target (as specified by
-      ``RDEPENDS``).
+      :term:`RDEPENDS`).
 
       As with all package-controlling variables, you must always use the
       variable in conjunction with a package name override. Here is an
-      example: ::
+      example::
 
-         RPROVIDES_${PN} = "widget-abi-2"
+         RPROVIDES:${PN} = "widget-abi-2"
 
    :term:`RRECOMMENDS`
       A list of packages that extends the usability of a package being
       built. The package being built does not depend on this list of
       packages in order to successfully build, but needs them for the
       extended usability. To specify runtime dependencies for packages, see
-      the ``RDEPENDS`` variable.
+      the :term:`RDEPENDS` variable.
 
       BitBake supports specifying versioned recommends. Although the syntax
       varies depending on the packaging format, BitBake hides these
       differences from you. Here is the general syntax to specify versions
-      with the ``RRECOMMENDS`` variable: ::
+      with the :term:`RRECOMMENDS` variable::
 
-         RRECOMMENDS_${PN} = "package (operator version)"
+         RRECOMMENDS:${PN} = "package (operator version)"
 
-      For ``operator``, you can specify the following: ::
+      For ``operator``, you can specify the following::
 
          =
          <
@@ -1249,76 +1515,114 @@ overview of their function and contents.
          >=
 
       For example, the following sets up a recommend on version
-      1.2 or greater of the package ``foo``: ::
+      1.2 or greater of the package ``foo``::
 
-         RRECOMMENDS_${PN} = "foo (>= 1.2)"
+         RRECOMMENDS:${PN} = "foo (>= 1.2)"
 
    :term:`SECTION`
       The section in which packages should be categorized.
 
    :term:`SRC_URI`
-      The list of source files - local or remote. This variable tells
+      The list of source files --- local or remote. This variable tells
       BitBake which bits to pull for the build and how to pull them. For
       example, if the recipe or append file needs to fetch a single tarball
-      from the Internet, the recipe or append file uses a ``SRC_URI`` entry
-      that specifies that tarball. On the other hand, if the recipe or
-      append file needs to fetch a tarball and include a custom file, the
-      recipe or append file needs an ``SRC_URI`` variable that specifies
-      all those sources.
-
-      The following list explains the available URI protocols:
+      from the Internet, the recipe or append file uses a :term:`SRC_URI`
+      entry that specifies that tarball. On the other hand, if the recipe or
+      append file needs to fetch a tarball, apply two patches, and include
+      a custom file, the recipe or append file needs an :term:`SRC_URI`
+      variable that specifies all those sources.
+
+      The following list explains the available URI protocols. URI
+      protocols are highly dependent on particular BitBake Fetcher
+      submodules. Depending on the fetcher BitBake uses, various URL
+      parameters are employed. For specifics on the supported Fetchers, see
+      the :ref:`bitbake-user-manual/bitbake-user-manual-fetching:fetchers`
+      section.
 
-      -  ``file://`` : Fetches files, which are usually files shipped
-         with the metadata, from the local machine. The path is relative to
-         the :term:`FILESPATH` variable.
+      -  ``az://``: Fetches files from an Azure Storage account using HTTPS.
 
-      -  ``bzr://`` : Fetches files from a Bazaar revision control
+      -  ``bzr://``: Fetches files from a Bazaar revision control
          repository.
 
-      -  ``git://`` : Fetches files from a Git revision control
+      -  ``ccrc://``: Fetches files from a ClearCase repository.
+
+      -  ``cvs://``: Fetches files from a CVS revision control
          repository.
 
-      -  ``osc://`` : Fetches files from an OSC (OpenSUSE Build service)
-         revision control repository.
+      -  ``file://``: Fetches files, which are usually files shipped
+         with the Metadata, from the local machine.
+         The path is relative to the :term:`FILESPATH`
+         variable. Thus, the build system searches, in order, from the
+         following directories, which are assumed to be a subdirectories of
+         the directory in which the recipe file (``.bb``) or append file
+         (``.bbappend``) resides:
 
-      -  ``repo://`` : Fetches files from a repo (Git) repository.
+         -  ``${BPN}``: the base recipe name without any special suffix
+            or version numbers.
 
-      -  ``http://`` : Fetches files from the Internet using HTTP.
+         -  ``${BP}`` - ``${BPN}-${PV}``: the base recipe name and
+            version but without any special package name suffix.
 
-      -  ``https://`` : Fetches files from the Internet using HTTPS.
+         -  ``files``: files within a directory, which is named ``files``
+            and is also alongside the recipe or append file.
 
-      -  ``ftp://`` : Fetches files from the Internet using FTP.
+      -  ``ftp://``: Fetches files from the Internet using FTP.
 
-      -  ``cvs://`` : Fetches files from a CVS revision control
+      -  ``git://``: Fetches files from a Git revision control
          repository.
 
-      -  ``hg://`` : Fetches files from a Mercurial (``hg``) revision
+      -  ``gitsm://``: Fetches submodules from a Git revision control
+         repository.
+
+      -  ``hg://``: Fetches files from a Mercurial (``hg``) revision
          control repository.
 
-      -  ``p4://`` : Fetches files from a Perforce (``p4``) revision
+      -  ``http://``: Fetches files from the Internet using HTTP.
+
+      -  ``https://``: Fetches files from the Internet using HTTPS.
+
+      -  ``npm://``: Fetches JavaScript modules from a registry.
+
+      -  ``osc://``: Fetches files from an OSC (OpenSUSE Build service)
+         revision control repository.
+
+      -  ``p4://``: Fetches files from a Perforce (``p4``) revision
          control repository.
 
-      -  ``ssh://`` : Fetches files from a secure shell.
+      -  ``repo://``: Fetches files from a repo (Git) repository.
+
+      -  ``ssh://``: Fetches files from a secure shell.
 
-      -  ``svn://`` : Fetches files from a Subversion (``svn``) revision
+      -  ``svn://``: Fetches files from a Subversion (``svn``) revision
          control repository.
 
       Here are some additional options worth mentioning:
 
-      -  ``unpack`` : Controls whether or not to unpack the file if it is
-         an archive. The default action is to unpack the file.
+      -  ``downloadfilename``: Specifies the filename used when storing
+         the downloaded file.
+
+      -  ``name``: Specifies a name to be used for association with
+         :term:`SRC_URI` checksums or :term:`SRCREV` when you have more than one
+         file or git repository specified in :term:`SRC_URI`. For example::
+
+            SRC_URI = "git://example.com/foo.git;branch=main;name=first \
+                       git://example.com/bar.git;branch=main;name=second \
+                       http://example.com/file.tar.gz;name=third"
 
-      -  ``subdir`` : Places the file (or extracts its contents) into the
+            SRCREV_first = "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15"
+            SRCREV_second = "e242ed3bffccdf271b7fbaf34ed72d089537b42f"
+            SRC_URI[third.sha256sum] = "13550350a8681c84c861aac2e5b440161c2b33a3e4f302ac680ca5b686de48de"
+
+      -  ``subdir``: Places the file (or extracts its contents) into the
          specified subdirectory. This option is useful for unusual tarballs
          or other archives that do not have their files already in a
          subdirectory within the archive.
 
-      -  ``name`` : Specifies a name to be used for association with
-         ``SRC_URI`` checksums when you have more than one file specified
-         in ``SRC_URI``.
+      -  ``subpath``: Limits the checkout to a specific subpath of the
+         tree when using the Git fetcher is used.
 
-      -  ``downloadfilename`` : Specifies the filename used when storing
-         the downloaded file.
+      -  ``unpack``: Controls whether or not to unpack the file if it is
+         an archive. The default action is to unpack the file.
 
    :term:`SRCDATE`
       The date of the source code used to build the package. This variable
@@ -1330,7 +1634,7 @@ overview of their function and contents.
       variable applies only when using Subversion, Git, Mercurial and
       Bazaar. If you want to build a fixed revision and you want to avoid
       performing a query on the remote repository every time BitBake parses
-      your recipe, you should specify a ``SRCREV`` that is a full revision
+      your recipe, you should specify a :term:`SRCREV` that is a full revision
       identifier and not just a tag.
 
    :term:`SRCREV_FORMAT`
@@ -1339,10 +1643,10 @@ overview of their function and contents.
       :term:`SRC_URI`.
 
       The system needs help constructing these values under these
-      circumstances. Each component in the ``SRC_URI`` is assigned a name
-      and these are referenced in the ``SRCREV_FORMAT`` variable. Consider
+      circumstances. Each component in the :term:`SRC_URI` is assigned a name
+      and these are referenced in the :term:`SRCREV_FORMAT` variable. Consider
       an example with URLs named "machine" and "meta". In this case,
-      ``SRCREV_FORMAT`` could look like "machine_meta" and those names
+      :term:`SRCREV_FORMAT` could look like "machine_meta" and those names
       would have the SCM versions substituted into each position. Only one
       ``AUTOINC`` placeholder is added and if needed. And, this placeholder
       is placed at the start of the returned string.
@@ -1354,7 +1658,7 @@ overview of their function and contents.
 
    :term:`STAMPCLEAN`
       Specifies the base path used to create recipe stamp files. Unlike the
-      :term:`STAMP` variable, ``STAMPCLEAN`` can contain
+      :term:`STAMP` variable, :term:`STAMPCLEAN` can contain
       wildcards to match the range of files a clean operation should
       remove. BitBake uses a clean operation to remove any other stamps it
       should be removing when creating a new stamp.
diff --git a/bitbake/doc/conf.py b/bitbake/doc/conf.py
index fc2ee08111..f61241e28b 100644
--- a/bitbake/doc/conf.py
+++ b/bitbake/doc/conf.py
@@ -17,6 +17,8 @@
 import sys
 import datetime
 
+from pathlib import Path
+
 current_version = "dev"
 
 # String used in sidebar
@@ -47,6 +49,7 @@ extlinks = {
 extensions = [
     'sphinx.ext.autosectionlabel',
     'sphinx.ext.extlinks',
+    'sphinx.ext.autodoc',
 ]
 autosectionlabel_prefix_document = True
 
@@ -99,3 +102,7 @@ html_last_updated_fmt = '%b %d, %Y'
 
 # Remove the trailing 'dot' in section numbers
 html_secnumber_suffix = " "
+
+# autoconf needs the modules available to auto-generate documentation from the
+# code
+sys.path.insert(0, str(Path('..', 'lib').resolve()))
diff --git a/bitbake/doc/index.rst b/bitbake/doc/index.rst
index 3ff8b1580f..546ef36c16 100644
--- a/bitbake/doc/index.rst
+++ b/bitbake/doc/index.rst
@@ -13,8 +13,10 @@ BitBake User Manual
    bitbake-user-manual/bitbake-user-manual-intro
    bitbake-user-manual/bitbake-user-manual-execution
    bitbake-user-manual/bitbake-user-manual-metadata
+   bitbake-user-manual/bitbake-user-manual-ref-variables-context
    bitbake-user-manual/bitbake-user-manual-fetching
    bitbake-user-manual/bitbake-user-manual-ref-variables
+   bitbake-user-manual/bitbake-user-manual-library-functions
    bitbake-user-manual/bitbake-user-manual-hello
 
 .. toctree::
diff --git a/bitbake/doc/releases.rst b/bitbake/doc/releases.rst
index d68d71599c..676db66ec5 100644
--- a/bitbake/doc/releases.rst
+++ b/bitbake/doc/releases.rst
@@ -1,32 +1,95 @@
 .. SPDX-License-Identifier: CC-BY-2.5
 
-=========================
- Current Release Manuals
-=========================
+=================================
+BitBake Supported Release Manuals
+=================================
+
+******************************
+Release Series 5.2 (walnascar)
+******************************
+
+- :yocto_docs:`BitBake 2.12 User Manual </bitbake/2.12/>`
+
+*******************************
+Release Series 5.0 (scarthgap)
+*******************************
+
+- :yocto_docs:`BitBake 2.8 User Manual </bitbake/2.8/>`
+
+******************************
+Release Series 4.0 (kirkstone)
+******************************
+
+- :yocto_docs:`BitBake 2.0 User Manual </bitbake/2.0/>`
+
+================================
+BitBake Outdated Release Manuals
+================================
+
+****************************
+Release Series 5.1 (styhead)
+****************************
+
+- :yocto_docs:`BitBake 2.10 User Manual </bitbake/2.10/>`
+
+*******************************
+Release Series 4.3 (nanbield)
+*******************************
+
+- :yocto_docs:`BitBake 2.6 User Manual </bitbake/2.6/>`
+
+*******************************
+Release Series 4.2 (mickledore)
+*******************************
+
+- :yocto_docs:`BitBake 2.4 User Manual </bitbake/2.4/>`
+
+*****************************
+Release Series 4.1 (langdale)
+*****************************
+
+- :yocto_docs:`BitBake 2.2 User Manual </bitbake/2.2/>`
+
+******************************
+Release Series 3.4 (honister)
+******************************
+
+- :yocto_docs:`BitBake 1.52 User Manual </bitbake/1.52/>`
+
+******************************
+Release Series 3.3 (hardknott)
+******************************
+
+- :yocto_docs:`BitBake 1.50 User Manual </bitbake/1.50/>`
+
+*******************************
+Release Series 3.2 (gatesgarth)
+*******************************
+
+- :yocto_docs:`BitBake 1.48 User Manual </bitbake/1.48/>`
 
 ****************************
-3.1 'dunfell' Release Series
+Release Series 3.1 (dunfell)
 ****************************
 
+- :yocto_docs:`BitBake 1.46 User Manual </bitbake/1.46/>`
 - :yocto_docs:`3.1 BitBake User Manual </3.1/bitbake-user-manual/bitbake-user-manual.html>`
 - :yocto_docs:`3.1.1 BitBake User Manual </3.1.1/bitbake-user-manual/bitbake-user-manual.html>`
 - :yocto_docs:`3.1.2 BitBake User Manual </3.1.2/bitbake-user-manual/bitbake-user-manual.html>`
-
-==========================
- Previous Release Manuals
-==========================
+- :yocto_docs:`3.1.3 BitBake User Manual </3.1.3/bitbake-user-manual/bitbake-user-manual.html>`
 
 *************************
-3.0 'zeus' Release Series
+Release Series 3.0 (zeus)
 *************************
 
 - :yocto_docs:`3.0 BitBake User Manual </3.0/bitbake-user-manual/bitbake-user-manual.html>`
 - :yocto_docs:`3.0.1 BitBake User Manual </3.0.1/bitbake-user-manual/bitbake-user-manual.html>`
 - :yocto_docs:`3.0.2 BitBake User Manual </3.0.2/bitbake-user-manual/bitbake-user-manual.html>`
 - :yocto_docs:`3.0.3 BitBake User Manual </3.0.3/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`3.0.4 BitBake User Manual </3.0.4/bitbake-user-manual/bitbake-user-manual.html>`
 
 ****************************
-2.7 'warrior' Release Series
+Release Series 2.7 (warrior)
 ****************************
 
 - :yocto_docs:`2.7 BitBake User Manual </2.7/bitbake-user-manual/bitbake-user-manual.html>`
@@ -36,7 +99,7 @@
 - :yocto_docs:`2.7.4 BitBake User Manual </2.7.4/bitbake-user-manual/bitbake-user-manual.html>`
 
 *************************
-2.6 'thud' Release Series
+Release Series 2.6 (thud)
 *************************
 
 - :yocto_docs:`2.6 BitBake User Manual </2.6/bitbake-user-manual/bitbake-user-manual.html>`
@@ -46,16 +109,16 @@
 - :yocto_docs:`2.6.4 BitBake User Manual </2.6.4/bitbake-user-manual/bitbake-user-manual.html>`
 
 *************************
-2.5 'sumo' Release Series
+Release Series 2.5 (sumo)
 *************************
 
-- :yocto_docs:`2.5 BitBake User Manual </2.5/bitbake-user-manual/bitbake-user-manual.html>`
-- :yocto_docs:`2.5.1 BitBake User Manual </2.5.1/bitbake-user-manual/bitbake-user-manual.html>`
-- :yocto_docs:`2.5.2 BitBake User Manual </2.5.2/bitbake-user-manual/bitbake-user-manual.html>`
-- :yocto_docs:`2.5.3 BitBake User Manual </2.5.3/bitbake-user-manual/bitbake-user-manual.html>`
+- :yocto_docs:`2.5 Documentation </2.5>`
+- :yocto_docs:`2.5.1 Documentation </2.5.1>`
+- :yocto_docs:`2.5.2 Documentation </2.5.2>`
+- :yocto_docs:`2.5.3 Documentation </2.5.3>`
 
 **************************
-2.4 'rocko' Release Series
+Release Series 2.4 (rocko)
 **************************
 
 - :yocto_docs:`2.4 BitBake User Manual </2.4/bitbake-user-manual/bitbake-user-manual.html>`
@@ -65,7 +128,7 @@
 - :yocto_docs:`2.4.4 BitBake User Manual </2.4.4/bitbake-user-manual/bitbake-user-manual.html>`
 
 *************************
-2.3 'pyro' Release Series
+Release Series 2.3 (pyro)
 *************************
 
 - :yocto_docs:`2.3 BitBake User Manual </2.3/bitbake-user-manual/bitbake-user-manual.html>`
@@ -75,7 +138,7 @@
 - :yocto_docs:`2.3.4 BitBake User Manual </2.3.4/bitbake-user-manual/bitbake-user-manual.html>`
 
 **************************
-2.2 'morty' Release Series
+Release Series 2.2 (morty)
 **************************
 
 - :yocto_docs:`2.2 BitBake User Manual </2.2/bitbake-user-manual/bitbake-user-manual.html>`
@@ -84,7 +147,7 @@
 - :yocto_docs:`2.2.3 BitBake User Manual </2.2.3/bitbake-user-manual/bitbake-user-manual.html>`
 
 ****************************
-2.1 'krogoth' Release Series
+Release Series 2.1 (krogoth)
 ****************************
 
 - :yocto_docs:`2.1 BitBake User Manual </2.1/bitbake-user-manual/bitbake-user-manual.html>`
@@ -93,7 +156,7 @@
 - :yocto_docs:`2.1.3 BitBake User Manual </2.1.3/bitbake-user-manual/bitbake-user-manual.html>`
 
 ***************************
-2.0 'jethro' Release Series
+Release Series 2.0 (jethro)
 ***************************
 
 - :yocto_docs:`1.9 BitBake User Manual </1.9/bitbake-user-manual/bitbake-user-manual.html>`
@@ -103,7 +166,7 @@
 - :yocto_docs:`2.0.3 BitBake User Manual </2.0.3/bitbake-user-manual/bitbake-user-manual.html>`
 
 *************************
-1.8 'fido' Release Series
+Release Series 1.8 (fido)
 *************************
 
 - :yocto_docs:`1.8 BitBake User Manual </1.8/bitbake-user-manual/bitbake-user-manual.html>`
@@ -111,7 +174,7 @@
 - :yocto_docs:`1.8.2 BitBake User Manual </1.8.2/bitbake-user-manual/bitbake-user-manual.html>`
 
 **************************
-1.7 'dizzy' Release Series
+Release Series 1.7 (dizzy)
 **************************
 
 - :yocto_docs:`1.7 BitBake User Manual </1.7/bitbake-user-manual/bitbake-user-manual.html>`
@@ -120,7 +183,7 @@
 - :yocto_docs:`1.7.3 BitBake User Manual </1.7.3/bitbake-user-manual/bitbake-user-manual.html>`
 
 **************************
-1.6 'daisy' Release Series
+Release Series 1.6 (daisy)
 **************************
 
 - :yocto_docs:`1.6 BitBake User Manual </1.6/bitbake-user-manual/bitbake-user-manual.html>`
diff --git a/bitbake/lib/bb/COW.py b/bitbake/lib/bb/COW.py
index 23c22b65ef..4af03c54ad 100644
--- a/bitbake/lib/bb/COW.py
+++ b/bitbake/lib/bb/COW.py
@@ -3,6 +3,8 @@
 #
 # Copyright (C) 2006 Tim Ansell
 #
+# SPDX-License-Identifier: GPL-2.0-only
+#
 # Please Note:
 # Be careful when using mutable types (ie Dict and Lists) - operations involving these are SLOW.
 # Assign a file to __warn__ to get warnings about slow operations.
@@ -34,8 +36,9 @@ class COWDictMeta(COWMeta):
     __marker__ = tuple()
 
     def __str__(cls):
-        # FIXME: I have magic numbers!
-        return "<COWDict Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) - 3)
+        ignored_keys = set(["__count__", "__doc__", "__module__", "__firstlineno__", "__static_attributes__"])
+        keys = set(cls.__dict__.keys()) - ignored_keys
+        return "<COWDict Level: %i Current Keys: %i>" % (cls.__count__, len(keys))
 
     __repr__ = __str__
 
@@ -159,8 +162,9 @@ class COWDictMeta(COWMeta):
 
 class COWSetMeta(COWDictMeta):
     def __str__(cls):
-        # FIXME: I have magic numbers!
-        return "<COWSet Level: %i Current Keys: %i>" % (cls.__count__, len(cls.__dict__) - 3)
+        ignored_keys = set(["__count__", "__doc__", "__module__", "__firstlineno__", "__static_attributes__"])
+        keys = set(cls.__dict__.keys()) - ignored_keys
+        return "<COWSet Level: %i Current Keys: %i>" % (cls.__count__, len(keys))
 
     __repr__ = __str__
 
diff --git a/bitbake/lib/bb/__init__.py b/bitbake/lib/bb/__init__.py
index 84a9051c13..bf4c54d829 100644
--- a/bitbake/lib/bb/__init__.py
+++ b/bitbake/lib/bb/__init__.py
@@ -9,12 +9,19 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-__version__ = "1.49.2"
+__version__ = "2.15.1"
 
 import sys
-if sys.version_info < (3, 5, 0):
-    raise RuntimeError("Sorry, python 3.5.0 or later is required for this version of bitbake")
+if sys.version_info < (3, 9, 0):
+    raise RuntimeError("Sorry, python 3.9.0 or later is required for this version of bitbake")
 
+if sys.version_info < (3, 10, 0):
+    # With python 3.8 and 3.9, we see errors of "libgcc_s.so.1 must be installed for pthread_cancel to work"
+    # https://stackoverflow.com/questions/64797838/libgcc-s-so-1-must-be-installed-for-pthread-cancel-to-work
+    # https://bugs.ams1.psf.io/issue42888
+    # so ensure libgcc_s is loaded early on
+    import ctypes
+    libgcc_s = ctypes.CDLL('libgcc_s.so.1')
 
 class BBHandledException(Exception):
     """
@@ -29,6 +36,7 @@ class BBHandledException(Exception):
 
 import os
 import logging
+from collections import namedtuple
 
 
 class NullHandler(logging.Handler):
@@ -58,8 +66,12 @@ class BBLoggerMixin(object):
         if not bb.event.worker_pid:
             if self.name in bb.msg.loggerDefaultDomains and loglevel > (bb.msg.loggerDefaultDomains[self.name]):
                 return
-            if loglevel > bb.msg.loggerDefaultLogLevel:
+            if loglevel < bb.msg.loggerDefaultLogLevel:
                 return
+
+        if not isinstance(level, int) or not isinstance(msg, str):
+            mainlogger.warning("Invalid arguments in bbdebug: %s" % repr((level, msg,) + args))
+
         return self.log(loglevel, msg, *args, **kwargs)
 
     def plain(self, msg, *args, **kwargs):
@@ -71,6 +83,13 @@ class BBLoggerMixin(object):
     def verbnote(self, msg, *args, **kwargs):
         return self.log(logging.INFO + 2, msg, *args, **kwargs)
 
+    def warnonce(self, msg, *args, **kwargs):
+        return self.log(logging.WARNING - 1, msg, *args, **kwargs)
+
+    def erroronce(self, msg, *args, **kwargs):
+        return self.log(logging.ERROR - 1, msg, *args, **kwargs)
+
+
 Logger = logging.getLoggerClass()
 class BBLogger(Logger, BBLoggerMixin):
     def __init__(self, name, *args, **kwargs):
@@ -85,26 +104,6 @@ class BBLoggerAdapter(logging.LoggerAdapter, BBLoggerMixin):
         self.setup_bblogger(logger.name)
         super().__init__(logger, *args, **kwargs)
 
-    if sys.version_info < (3, 6):
-        # These properties were added in Python 3.6. Add them in older versions
-        # for compatibility
-        @property
-        def manager(self):
-            return self.logger.manager
-
-        @manager.setter
-        def manager(self, value):
-            self.logger.manager = value
-
-        @property
-        def name(self):
-            return self.logger.name
-
-        def __repr__(self):
-            logger = self.logger
-            level = logger.getLevelName(logger.getEffectiveLevel())
-            return '<%s %s (%s)>' % (self.__class__.__name__, logger.name, level)
-
 logging.LoggerAdapter = BBLoggerAdapter
 
 logger = logging.getLogger("BitBake")
@@ -130,9 +129,25 @@ sys.modules['bb.fetch'] = sys.modules['bb.fetch2']
 
 # Messaging convenience functions
 def plain(*args):
+    """
+    Prints a message at "plain" level (higher level than a ``bb.note()``).
+
+    Arguments:
+
+    -  ``args``: one or more strings to print.
+    """
     mainlogger.plain(''.join(args))
 
 def debug(lvl, *args):
+    """
+    Prints a debug message.
+
+    Arguments:
+
+    -  ``lvl``: debug level. Higher value increases the debug level
+       (determined by ``bitbake -D``).
+    -  ``args``: one or more strings to print.
+    """
     if isinstance(lvl, str):
         mainlogger.warning("Passed invalid debug level '%s' to bb.debug", lvl)
         args = (lvl,) + args
@@ -140,27 +155,81 @@ def debug(lvl, *args):
     mainlogger.bbdebug(lvl, ''.join(args))
 
 def note(*args):
+    """
+    Prints a message at "note" level.
+
+    Arguments:
+
+    -  ``args``: one or more strings to print.
+    """
     mainlogger.info(''.join(args))
 
-#
-# A higher prioity note which will show on the console but isn't a warning
-#
-# Something is happening the user should be aware of but they probably did
-# something to make it happen
-#
 def verbnote(*args):
+    """
+    A higher priority note which will show on the console but isn't a warning.
+
+    Use in contexts when something is happening the user should be aware of but
+    they probably did something to make it happen.
+
+    Arguments:
+
+    -  ``args``: one or more strings to print.
+    """
     mainlogger.verbnote(''.join(args))
 
 #
 # Warnings - things the user likely needs to pay attention to and fix
 #
 def warn(*args):
+    """
+    Prints a warning message.
+
+    Arguments:
+
+    -  ``args``: one or more strings to print.
+    """
     mainlogger.warning(''.join(args))
 
+def warnonce(*args):
+    """
+    Prints a warning message like ``bb.warn()``, but only prints the message
+    once.
+
+    Arguments:
+
+    -  ``args``: one or more strings to print.
+    """
+    mainlogger.warnonce(''.join(args))
+
 def error(*args, **kwargs):
+    """
+    Prints an error message.
+
+    Arguments:
+
+    -  ``args``: one or more strings to print.
+    """
     mainlogger.error(''.join(args), extra=kwargs)
 
+def erroronce(*args):
+    """
+    Prints an error message like ``bb.error()``, but only prints the message
+    once.
+
+    Arguments:
+
+    -  ``args``: one or more strings to print.
+    """
+    mainlogger.erroronce(''.join(args))
+
 def fatal(*args, **kwargs):
+    """
+    Prints an error message and stops the BitBake execution.
+
+    Arguments:
+
+    -  ``args``: one or more strings to print.
+    """
     mainlogger.critical(''.join(args), extra=kwargs)
     raise BBHandledException()
 
@@ -189,7 +258,6 @@ def deprecated(func, name=None, advice=""):
 # For compatibility
 def deprecate_import(current, modulename, fromlist, renames = None):
     """Import objects from one module into another, wrapping them with a DeprecationWarning"""
-    import sys
 
     module = __import__(modulename, fromlist = fromlist)
     for position, objname in enumerate(fromlist):
@@ -203,3 +271,14 @@ def deprecate_import(current, modulename, fromlist, renames = None):
 
         setattr(sys.modules[current], newname, newobj)
 
+TaskData = namedtuple("TaskData", [
+    "pn",
+    "taskname",
+    "fn",
+    "deps",
+    "provides",
+    "taskhash",
+    "unihash",
+    "hashfn",
+    "taskhash_deps",
+])
diff --git a/bitbake/lib/bb/acl.py b/bitbake/lib/bb/acl.py
new file mode 100755
index 0000000000..e9dbdb617f
--- /dev/null
+++ b/bitbake/lib/bb/acl.py
@@ -0,0 +1,213 @@
+#! /usr/bin/env python3
+#
+# Copyright 2023 by Garmin Ltd. or its subsidiaries
+#
+# SPDX-License-Identifier: MIT
+
+
+import sys
+import ctypes
+import os
+import errno
+import pwd
+import grp
+
+libacl = ctypes.CDLL("libacl.so.1", use_errno=True)
+
+
+ACL_TYPE_ACCESS = 0x8000
+ACL_TYPE_DEFAULT = 0x4000
+
+ACL_FIRST_ENTRY = 0
+ACL_NEXT_ENTRY = 1
+
+ACL_UNDEFINED_TAG = 0x00
+ACL_USER_OBJ = 0x01
+ACL_USER = 0x02
+ACL_GROUP_OBJ = 0x04
+ACL_GROUP = 0x08
+ACL_MASK = 0x10
+ACL_OTHER = 0x20
+
+ACL_READ = 0x04
+ACL_WRITE = 0x02
+ACL_EXECUTE = 0x01
+
+acl_t = ctypes.c_void_p
+acl_entry_t = ctypes.c_void_p
+acl_permset_t = ctypes.c_void_p
+acl_perm_t = ctypes.c_uint
+
+acl_tag_t = ctypes.c_int
+
+libacl.acl_free.argtypes = [acl_t]
+
+
+def acl_free(acl):
+    libacl.acl_free(acl)
+
+
+libacl.acl_get_file.restype = acl_t
+libacl.acl_get_file.argtypes = [ctypes.c_char_p, ctypes.c_uint]
+
+
+def acl_get_file(path, typ):
+    acl = libacl.acl_get_file(os.fsencode(path), typ)
+    if acl is None:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err), str(path))
+
+    return acl
+
+
+libacl.acl_get_entry.argtypes = [acl_t, ctypes.c_int, ctypes.c_void_p]
+
+
+def acl_get_entry(acl, entry_id):
+    entry = acl_entry_t()
+    ret = libacl.acl_get_entry(acl, entry_id, ctypes.byref(entry))
+    if ret < 0:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+
+    if ret == 0:
+        return None
+
+    return entry
+
+
+libacl.acl_get_tag_type.argtypes = [acl_entry_t, ctypes.c_void_p]
+
+
+def acl_get_tag_type(entry_d):
+    tag = acl_tag_t()
+    ret = libacl.acl_get_tag_type(entry_d, ctypes.byref(tag))
+    if ret < 0:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+    return tag.value
+
+
+libacl.acl_get_qualifier.restype = ctypes.c_void_p
+libacl.acl_get_qualifier.argtypes = [acl_entry_t]
+
+
+def acl_get_qualifier(entry_d):
+    ret = libacl.acl_get_qualifier(entry_d)
+    if ret is None:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+    return ctypes.c_void_p(ret)
+
+
+libacl.acl_get_permset.argtypes = [acl_entry_t, ctypes.c_void_p]
+
+
+def acl_get_permset(entry_d):
+    permset = acl_permset_t()
+    ret = libacl.acl_get_permset(entry_d, ctypes.byref(permset))
+    if ret < 0:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+
+    return permset
+
+
+libacl.acl_get_perm.argtypes = [acl_permset_t, acl_perm_t]
+
+
+def acl_get_perm(permset_d, perm):
+    ret = libacl.acl_get_perm(permset_d, perm)
+    if ret < 0:
+        err = ctypes.get_errno()
+        raise OSError(err, os.strerror(err))
+    return bool(ret)
+
+
+class Entry(object):
+    def __init__(self, tag, qualifier, mode):
+        self.tag = tag
+        self.qualifier = qualifier
+        self.mode = mode
+
+    def __str__(self):
+        typ = ""
+        qual = ""
+        if self.tag == ACL_USER:
+            typ = "user"
+            qual = pwd.getpwuid(self.qualifier).pw_name
+        elif self.tag == ACL_GROUP:
+            typ = "group"
+            qual = grp.getgrgid(self.qualifier).gr_name
+        elif self.tag == ACL_USER_OBJ:
+            typ = "user"
+        elif self.tag == ACL_GROUP_OBJ:
+            typ = "group"
+        elif self.tag == ACL_MASK:
+            typ = "mask"
+        elif self.tag == ACL_OTHER:
+            typ = "other"
+
+        r = "r" if self.mode & ACL_READ else "-"
+        w = "w" if self.mode & ACL_WRITE else "-"
+        x = "x" if self.mode & ACL_EXECUTE else "-"
+
+        return f"{typ}:{qual}:{r}{w}{x}"
+
+
+class ACL(object):
+    def __init__(self, acl):
+        self.acl = acl
+
+    def __del__(self):
+        acl_free(self.acl)
+
+    def entries(self):
+        entry_id = ACL_FIRST_ENTRY
+        while True:
+            entry = acl_get_entry(self.acl, entry_id)
+            if entry is None:
+                break
+
+            permset = acl_get_permset(entry)
+
+            mode = 0
+            for m in (ACL_READ, ACL_WRITE, ACL_EXECUTE):
+                if acl_get_perm(permset, m):
+                    mode |= m
+
+            qualifier = None
+            tag = acl_get_tag_type(entry)
+
+            if tag == ACL_USER or tag == ACL_GROUP:
+                qual = acl_get_qualifier(entry)
+                qualifier = ctypes.cast(qual, ctypes.POINTER(ctypes.c_int))[0]
+
+            yield Entry(tag, qualifier, mode)
+
+            entry_id = ACL_NEXT_ENTRY
+
+    @classmethod
+    def from_path(cls, path, typ):
+        acl = acl_get_file(path, typ)
+        return cls(acl)
+
+
+def main():
+    import argparse
+    from pathlib import Path
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", help="File Path", type=Path)
+
+    args = parser.parse_args()
+
+    acl = ACL.from_path(args.path, ACL_TYPE_ACCESS)
+    for entry in acl.entries():
+        print(str(entry))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/bitbake/lib/bb/asyncrpc/__init__.py b/bitbake/lib/bb/asyncrpc/__init__.py
new file mode 100644
index 0000000000..a4371643d7
--- /dev/null
+++ b/bitbake/lib/bb/asyncrpc/__init__.py
@@ -0,0 +1,16 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+
+from .client import AsyncClient, Client
+from .serv import AsyncServer, AsyncServerConnection
+from .connection import DEFAULT_MAX_CHUNK
+from .exceptions import (
+    ClientError,
+    ServerError,
+    ConnectionClosedError,
+    InvokeError,
+)
diff --git a/bitbake/lib/bb/asyncrpc/client.py b/bitbake/lib/bb/asyncrpc/client.py
new file mode 100644
index 0000000000..17b72033b9
--- /dev/null
+++ b/bitbake/lib/bb/asyncrpc/client.py
@@ -0,0 +1,271 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import abc
+import asyncio
+import json
+import os
+import socket
+import sys
+import re
+import contextlib
+from threading import Thread
+from .connection import StreamConnection, WebsocketConnection, DEFAULT_MAX_CHUNK
+from .exceptions import ConnectionClosedError, InvokeError
+
+UNIX_PREFIX = "unix://"
+WS_PREFIX = "ws://"
+WSS_PREFIX = "wss://"
+
+ADDR_TYPE_UNIX = 0
+ADDR_TYPE_TCP = 1
+ADDR_TYPE_WS = 2
+
+WEBSOCKETS_MIN_VERSION = (9, 1)
+# Need websockets 10 with python 3.10+
+if sys.version_info >= (3, 10, 0):
+    WEBSOCKETS_MIN_VERSION = (10, 0)
+
+
+def parse_address(addr):
+    if addr.startswith(UNIX_PREFIX):
+        return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX) :],))
+    elif addr.startswith(WS_PREFIX) or addr.startswith(WSS_PREFIX):
+        return (ADDR_TYPE_WS, (addr,))
+    else:
+        m = re.match(r"\[(?P<host>[^\]]*)\]:(?P<port>\d+)$", addr)
+        if m is not None:
+            host = m.group("host")
+            port = m.group("port")
+        else:
+            host, port = addr.split(":")
+
+        return (ADDR_TYPE_TCP, (host, int(port)))
+
+
+class AsyncClient(object):
+    def __init__(
+        self,
+        proto_name,
+        proto_version,
+        logger,
+        timeout=30,
+        server_headers=False,
+        headers={},
+    ):
+        self.socket = None
+        self.max_chunk = DEFAULT_MAX_CHUNK
+        self.proto_name = proto_name
+        self.proto_version = proto_version
+        self.logger = logger
+        self.timeout = timeout
+        self.needs_server_headers = server_headers
+        self.server_headers = {}
+        self.headers = headers
+
+    async def connect_tcp(self, address, port):
+        async def connect_sock():
+            reader, writer = await asyncio.open_connection(address, port)
+            return StreamConnection(reader, writer, self.timeout, self.max_chunk)
+
+        self._connect_sock = connect_sock
+
+    async def connect_unix(self, path):
+        async def connect_sock():
+            # AF_UNIX has path length issues so chdir here to workaround
+            cwd = os.getcwd()
+            try:
+                os.chdir(os.path.dirname(path))
+                # The socket must be opened synchronously so that CWD doesn't get
+                # changed out from underneath us so we pass as a sock into asyncio
+                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM, 0)
+                sock.connect(os.path.basename(path))
+            finally:
+                os.chdir(cwd)
+            reader, writer = await asyncio.open_unix_connection(sock=sock)
+            return StreamConnection(reader, writer, self.timeout, self.max_chunk)
+
+        self._connect_sock = connect_sock
+
+    async def connect_websocket(self, uri):
+        import websockets
+
+        try:
+            version = tuple(
+                int(v)
+                for v in websockets.__version__.split(".")[
+                    0 : len(WEBSOCKETS_MIN_VERSION)
+                ]
+            )
+        except ValueError:
+            raise ImportError(
+                f"Unable to parse websockets version '{websockets.__version__}'"
+            )
+
+        if version < WEBSOCKETS_MIN_VERSION:
+            min_ver_str = ".".join(str(v) for v in WEBSOCKETS_MIN_VERSION)
+            raise ImportError(
+                f"Websockets version {websockets.__version__} is less than minimum required version {min_ver_str}"
+            )
+
+        async def connect_sock():
+            try:
+                websocket = await websockets.connect(
+                    uri,
+                    ping_interval=None,
+                    open_timeout=self.timeout,
+                )
+            except asyncio.exceptions.TimeoutError:
+                raise ConnectionError("Timeout while connecting to websocket")
+            except (OSError, websockets.InvalidHandshake, websockets.InvalidURI) as exc:
+                raise ConnectionError(f"Could not connect to websocket: {exc}") from exc
+            return WebsocketConnection(websocket, self.timeout)
+
+        self._connect_sock = connect_sock
+
+    async def setup_connection(self):
+        # Send headers
+        await self.socket.send("%s %s" % (self.proto_name, self.proto_version))
+        await self.socket.send(
+            "needs-headers: %s" % ("true" if self.needs_server_headers else "false")
+        )
+        for k, v in self.headers.items():
+            await self.socket.send("%s: %s" % (k, v))
+
+        # End of headers
+        await self.socket.send("")
+
+        self.server_headers = {}
+        if self.needs_server_headers:
+            while True:
+                line = await self.socket.recv()
+                if not line:
+                    # End headers
+                    break
+                tag, value = line.split(":", 1)
+                self.server_headers[tag.lower()] = value.strip()
+
+    async def get_header(self, tag, default):
+        await self.connect()
+        return self.server_headers.get(tag, default)
+
+    async def connect(self):
+        if self.socket is None:
+            self.socket = await self._connect_sock()
+            await self.setup_connection()
+
+    async def disconnect(self):
+        if self.socket is not None:
+            await self.socket.close()
+            self.socket = None
+
+    async def close(self):
+        await self.disconnect()
+
+    async def _send_wrapper(self, proc):
+        count = 0
+        while True:
+            try:
+                await self.connect()
+                return await proc()
+            except (
+                OSError,
+                ConnectionError,
+                ConnectionClosedError,
+                json.JSONDecodeError,
+                UnicodeDecodeError,
+            ) as e:
+                self.logger.warning("Error talking to server: %s" % e)
+                if count >= 3:
+                    if not isinstance(e, ConnectionError):
+                        raise ConnectionError(str(e))
+                    raise e
+                await self.close()
+                count += 1
+
+    def check_invoke_error(self, msg):
+        if isinstance(msg, dict) and "invoke-error" in msg:
+            raise InvokeError(msg["invoke-error"]["message"])
+
+    async def invoke(self, msg):
+        async def proc():
+            await self.socket.send_message(msg)
+            return await self.socket.recv_message()
+
+        result = await self._send_wrapper(proc)
+        self.check_invoke_error(result)
+        return result
+
+    async def ping(self):
+        return await self.invoke({"ping": {}})
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.close()
+
+
+class Client(object):
+    def __init__(self):
+        self.client = self._get_async_client()
+        self.loop = asyncio.new_event_loop()
+
+        # Override any pre-existing loop.
+        # Without this, the PR server export selftest triggers a hang
+        # when running with Python 3.7.  The drawback is that there is
+        # potential for issues if the PR and hash equiv (or some new)
+        # clients need to both be instantiated in the same process.
+        # This should be revisited if/when Python 3.9 becomes the
+        # minimum required version for BitBake, as it seems not
+        # required (but harmless) with it.
+        asyncio.set_event_loop(self.loop)
+
+        self._add_methods("connect_tcp", "ping")
+
+    @abc.abstractmethod
+    def _get_async_client(self):
+        pass
+
+    def _get_downcall_wrapper(self, downcall):
+        def wrapper(*args, **kwargs):
+            return self.loop.run_until_complete(downcall(*args, **kwargs))
+
+        return wrapper
+
+    def _add_methods(self, *methods):
+        for m in methods:
+            downcall = getattr(self.client, m)
+            setattr(self, m, self._get_downcall_wrapper(downcall))
+
+    def connect_unix(self, path):
+        self.loop.run_until_complete(self.client.connect_unix(path))
+        self.loop.run_until_complete(self.client.connect())
+
+    @property
+    def max_chunk(self):
+        return self.client.max_chunk
+
+    @max_chunk.setter
+    def max_chunk(self, value):
+        self.client.max_chunk = value
+
+    def disconnect(self):
+        self.loop.run_until_complete(self.client.close())
+
+    def close(self):
+        if self.loop:
+            self.loop.run_until_complete(self.client.close())
+            self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+            self.loop.close()
+        self.loop = None
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close()
+        return False
diff --git a/bitbake/lib/bb/asyncrpc/connection.py b/bitbake/lib/bb/asyncrpc/connection.py
new file mode 100644
index 0000000000..7f0cf6ba96
--- /dev/null
+++ b/bitbake/lib/bb/asyncrpc/connection.py
@@ -0,0 +1,146 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import asyncio
+import itertools
+import json
+from datetime import datetime
+from .exceptions import ClientError, ConnectionClosedError
+
+
+# The Python async server defaults to a 64K receive buffer, so we hardcode our
+# maximum chunk size. It would be better if the client and server reported to
+# each other what the maximum chunk sizes were, but that will slow down the
+# connection setup with a round trip delay so I'd rather not do that unless it
+# is necessary
+DEFAULT_MAX_CHUNK = 32 * 1024
+
+
+def chunkify(msg, max_chunk):
+    if len(msg) < max_chunk - 1:
+        yield "".join((msg, "\n"))
+    else:
+        yield "".join((json.dumps({"chunk-stream": None}), "\n"))
+
+        args = [iter(msg)] * (max_chunk - 1)
+        for m in map("".join, itertools.zip_longest(*args, fillvalue="")):
+            yield "".join(itertools.chain(m, "\n"))
+        yield "\n"
+
+
+def json_serialize(obj):
+    if isinstance(obj, datetime):
+        return obj.isoformat()
+    raise TypeError("Type %s not serializeable" % type(obj))
+
+
+class StreamConnection(object):
+    def __init__(self, reader, writer, timeout, max_chunk=DEFAULT_MAX_CHUNK):
+        self.reader = reader
+        self.writer = writer
+        self.timeout = timeout
+        self.max_chunk = max_chunk
+
+    @property
+    def address(self):
+        return self.writer.get_extra_info("peername")
+
+    async def send_message(self, msg):
+        for c in chunkify(json.dumps(msg, default=json_serialize), self.max_chunk):
+            self.writer.write(c.encode("utf-8"))
+        await self.writer.drain()
+
+    async def recv_message(self):
+        l = await self.recv()
+
+        m = json.loads(l)
+        if not m:
+            return m
+
+        if "chunk-stream" in m:
+            lines = []
+            while True:
+                l = await self.recv()
+                if not l:
+                    break
+                lines.append(l)
+
+            m = json.loads("".join(lines))
+
+        return m
+
+    async def send(self, msg):
+        self.writer.write(("%s\n" % msg).encode("utf-8"))
+        await self.writer.drain()
+
+    async def recv(self):
+        if self.timeout < 0:
+            line = await self.reader.readline()
+        else:
+            try:
+                line = await asyncio.wait_for(self.reader.readline(), self.timeout)
+            except asyncio.TimeoutError:
+                raise ConnectionError("Timed out waiting for data")
+
+        if not line:
+            raise ConnectionClosedError("Connection closed")
+
+        line = line.decode("utf-8")
+
+        if not line.endswith("\n"):
+            raise ConnectionError("Bad message %r" % (line))
+
+        return line.rstrip()
+
+    async def close(self):
+        self.reader = None
+        if self.writer is not None:
+            self.writer.close()
+            self.writer = None
+
+
+class WebsocketConnection(object):
+    def __init__(self, socket, timeout):
+        self.socket = socket
+        self.timeout = timeout
+
+    @property
+    def address(self):
+        return ":".join(str(s) for s in self.socket.remote_address)
+
+    async def send_message(self, msg):
+        await self.send(json.dumps(msg, default=json_serialize))
+
+    async def recv_message(self):
+        m = await self.recv()
+        return json.loads(m)
+
+    async def send(self, msg):
+        import websockets.exceptions
+
+        try:
+            await self.socket.send(msg)
+        except websockets.exceptions.ConnectionClosed:
+            raise ConnectionClosedError("Connection closed")
+
+    async def recv(self):
+        import websockets.exceptions
+
+        try:
+            if self.timeout < 0:
+                return await self.socket.recv()
+
+            try:
+                return await asyncio.wait_for(self.socket.recv(), self.timeout)
+            except asyncio.TimeoutError:
+                raise ConnectionError("Timed out waiting for data")
+        except websockets.exceptions.ConnectionClosed:
+            raise ConnectionClosedError("Connection closed")
+
+    async def close(self):
+        if self.socket is not None:
+            await self.socket.close()
+            self.socket = None
diff --git a/bitbake/lib/bb/asyncrpc/exceptions.py b/bitbake/lib/bb/asyncrpc/exceptions.py
new file mode 100644
index 0000000000..ae1043a38b
--- /dev/null
+++ b/bitbake/lib/bb/asyncrpc/exceptions.py
@@ -0,0 +1,21 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+
+class ClientError(Exception):
+    pass
+
+
+class InvokeError(Exception):
+    pass
+
+
+class ServerError(Exception):
+    pass
+
+
+class ConnectionClosedError(Exception):
+    pass
diff --git a/bitbake/lib/bb/asyncrpc/serv.py b/bitbake/lib/bb/asyncrpc/serv.py
new file mode 100644
index 0000000000..667217c5c1
--- /dev/null
+++ b/bitbake/lib/bb/asyncrpc/serv.py
@@ -0,0 +1,410 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import abc
+import asyncio
+import json
+import os
+import signal
+import socket
+import sys
+import multiprocessing
+import logging
+from .connection import StreamConnection, WebsocketConnection
+from .exceptions import ClientError, ServerError, ConnectionClosedError, InvokeError
+
+
+class ClientLoggerAdapter(logging.LoggerAdapter):
+    def process(self, msg, kwargs):
+        return f"[Client {self.extra['address']}] {msg}", kwargs
+
+
+class AsyncServerConnection(object):
+    # If a handler returns this object (e.g. `return self.NO_RESPONSE`), no
+    # return message will be automatically be sent back to the client
+    NO_RESPONSE = object()
+
+    def __init__(self, socket, proto_name, logger):
+        self.socket = socket
+        self.proto_name = proto_name
+        self.handlers = {
+            "ping": self.handle_ping,
+        }
+        self.logger = ClientLoggerAdapter(
+            logger,
+            {
+                "address": socket.address,
+            },
+        )
+        self.client_headers = {}
+
+    async def close(self):
+        await self.socket.close()
+
+    async def handle_headers(self, headers):
+        return {}
+
+    async def process_requests(self):
+        try:
+            self.logger.info("Client %r connected" % (self.socket.address,))
+
+            # Read protocol and version
+            client_protocol = await self.socket.recv()
+            if not client_protocol:
+                return
+
+            (client_proto_name, client_proto_version) = client_protocol.split()
+            if client_proto_name != self.proto_name:
+                self.logger.debug("Rejecting invalid protocol %s" % (self.proto_name))
+                return
+
+            self.proto_version = tuple(int(v) for v in client_proto_version.split("."))
+            if not self.validate_proto_version():
+                self.logger.debug(
+                    "Rejecting invalid protocol version %s" % (client_proto_version)
+                )
+                return
+
+            # Read headers
+            self.client_headers = {}
+            while True:
+                header = await self.socket.recv()
+                if not header:
+                    # Empty line. End of headers
+                    break
+                tag, value = header.split(":", 1)
+                self.client_headers[tag.lower()] = value.strip()
+
+            if self.client_headers.get("needs-headers", "false") == "true":
+                for k, v in (await self.handle_headers(self.client_headers)).items():
+                    await self.socket.send("%s: %s" % (k, v))
+                await self.socket.send("")
+
+            # Handle messages
+            while True:
+                d = await self.socket.recv_message()
+                if d is None:
+                    break
+                try:
+                    response = await self.dispatch_message(d)
+                except InvokeError as e:
+                    await self.socket.send_message(
+                        {"invoke-error": {"message": str(e)}}
+                    )
+                    break
+
+                if response is not self.NO_RESPONSE:
+                    await self.socket.send_message(response)
+
+        except ConnectionClosedError as e:
+            self.logger.info(str(e))
+        except (ClientError, ConnectionError) as e:
+            self.logger.error(str(e))
+        finally:
+            await self.close()
+
+    async def dispatch_message(self, msg):
+        for k in self.handlers.keys():
+            if k in msg:
+                self.logger.debug("Handling %s" % k)
+                return await self.handlers[k](msg[k])
+
+        raise ClientError("Unrecognized command %r" % msg)
+
+    async def handle_ping(self, request):
+        return {"alive": True}
+
+
+class StreamServer(object):
+    def __init__(self, handler, logger):
+        self.handler = handler
+        self.logger = logger
+        self.closed = False
+
+    async def handle_stream_client(self, reader, writer):
+        # writer.transport.set_write_buffer_limits(0)
+        socket = StreamConnection(reader, writer, -1)
+        if self.closed:
+            await socket.close()
+            return
+
+        await self.handler(socket)
+
+    async def stop(self):
+        self.closed = True
+
+
+class TCPStreamServer(StreamServer):
+    def __init__(self, host, port, handler, logger, *, reuseport=False):
+        super().__init__(handler, logger)
+        self.host = host
+        self.port = port
+        self.reuseport = reuseport
+
+    def start(self, loop):
+        self.server = loop.run_until_complete(
+            asyncio.start_server(
+                self.handle_stream_client,
+                self.host,
+                self.port,
+                reuse_port=self.reuseport,
+            )
+        )
+
+        for s in self.server.sockets:
+            self.logger.debug("Listening on %r" % (s.getsockname(),))
+            # Newer python does this automatically. Do it manually here for
+            # maximum compatibility
+            s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
+            s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
+
+            # Enable keep alives. This prevents broken client connections
+            # from persisting on the server for long periods of time.
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 30)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 15)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 4)
+
+        name = self.server.sockets[0].getsockname()
+        if self.server.sockets[0].family == socket.AF_INET6:
+            self.address = "[%s]:%d" % (name[0], name[1])
+        else:
+            self.address = "%s:%d" % (name[0], name[1])
+
+        return [self.server.wait_closed()]
+
+    async def stop(self):
+        await super().stop()
+        self.server.close()
+
+    def cleanup(self):
+        pass
+
+
+class UnixStreamServer(StreamServer):
+    def __init__(self, path, handler, logger):
+        super().__init__(handler, logger)
+        self.path = path
+
+    def start(self, loop):
+        cwd = os.getcwd()
+        try:
+            # Work around path length limits in AF_UNIX
+            os.chdir(os.path.dirname(self.path))
+            self.server = loop.run_until_complete(
+                asyncio.start_unix_server(
+                    self.handle_stream_client, os.path.basename(self.path)
+                )
+            )
+        finally:
+            os.chdir(cwd)
+
+        self.logger.debug("Listening on %r" % self.path)
+        self.address = "unix://%s" % os.path.abspath(self.path)
+        return [self.server.wait_closed()]
+
+    async def stop(self):
+        await super().stop()
+        self.server.close()
+
+    def cleanup(self):
+        os.unlink(self.path)
+
+
+class WebsocketsServer(object):
+    def __init__(self, host, port, handler, logger, *, reuseport=False):
+        self.host = host
+        self.port = port
+        self.handler = handler
+        self.logger = logger
+        self.reuseport = reuseport
+
+    def start(self, loop):
+        import websockets.server
+
+        self.server = loop.run_until_complete(
+            websockets.server.serve(
+                self.client_handler,
+                self.host,
+                self.port,
+                ping_interval=None,
+                reuse_port=self.reuseport,
+            )
+        )
+
+        for s in self.server.sockets:
+            self.logger.debug("Listening on %r" % (s.getsockname(),))
+
+            # Enable keep alives. This prevents broken client connections
+            # from persisting on the server for long periods of time.
+            s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 30)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 15)
+            s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 4)
+
+        name = self.server.sockets[0].getsockname()
+        if self.server.sockets[0].family == socket.AF_INET6:
+            self.address = "ws://[%s]:%d" % (name[0], name[1])
+        else:
+            self.address = "ws://%s:%d" % (name[0], name[1])
+
+        return [self.server.wait_closed()]
+
+    async def stop(self):
+        self.server.close()
+
+    def cleanup(self):
+        pass
+
+    async def client_handler(self, websocket):
+        socket = WebsocketConnection(websocket, -1)
+        await self.handler(socket)
+
+
+class AsyncServer(object):
+    def __init__(self, logger):
+        self.logger = logger
+        self.loop = None
+        self.run_tasks = []
+
+    def start_tcp_server(self, host, port, *, reuseport=False):
+        self.server = TCPStreamServer(
+            host,
+            port,
+            self._client_handler,
+            self.logger,
+            reuseport=reuseport,
+        )
+
+    def start_unix_server(self, path):
+        self.server = UnixStreamServer(path, self._client_handler, self.logger)
+
+    def start_websocket_server(self, host, port, reuseport=False):
+        self.server = WebsocketsServer(
+            host,
+            port,
+            self._client_handler,
+            self.logger,
+            reuseport=reuseport,
+        )
+
+    async def _client_handler(self, socket):
+        address = socket.address
+        try:
+            client = self.accept_client(socket)
+            await client.process_requests()
+        except Exception as e:
+            import traceback
+
+            self.logger.error(
+                "Error from client %s: %s" % (address, str(e)), exc_info=True
+            )
+            traceback.print_exc()
+        finally:
+            self.logger.debug("Client %s disconnected", address)
+            await socket.close()
+
+    @abc.abstractmethod
+    def accept_client(self, socket):
+        pass
+
+    async def stop(self):
+        self.logger.debug("Stopping server")
+        await self.server.stop()
+
+    def start(self):
+        tasks = self.server.start(self.loop)
+        self.address = self.server.address
+        return tasks
+
+    def signal_handler(self):
+        self.logger.debug("Got exit signal")
+        self.loop.create_task(self.stop())
+
+    def _serve_forever(self, tasks):
+        try:
+            self.loop.add_signal_handler(signal.SIGTERM, self.signal_handler)
+            self.loop.add_signal_handler(signal.SIGINT, self.signal_handler)
+            self.loop.add_signal_handler(signal.SIGQUIT, self.signal_handler)
+            signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGTERM])
+
+            self.loop.run_until_complete(asyncio.gather(*tasks))
+
+            self.logger.debug("Server shutting down")
+        finally:
+            self.server.cleanup()
+
+    def serve_forever(self):
+        """
+        Serve requests in the current process
+        """
+        self._create_loop()
+        tasks = self.start()
+        self._serve_forever(tasks)
+        self.loop.close()
+
+    def _create_loop(self):
+        # Create loop and override any loop that may have existed in
+        # a parent process.  It is possible that the usecases of
+        # serve_forever might be constrained enough to allow using
+        # get_event_loop here, but better safe than sorry for now.
+        self.loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(self.loop)
+
+    def serve_as_process(self, *, prefunc=None, args=(), log_level=None):
+        """
+        Serve requests in a child process
+        """
+
+        def run(queue):
+            # Create loop and override any loop that may have existed
+            # in a parent process.  Without doing this and instead
+            # using get_event_loop, at the very minimum the hashserv
+            # unit tests will hang when running the second test.
+            # This happens since get_event_loop in the spawned server
+            # process for the second testcase ends up with the loop
+            # from the hashserv client created in the unit test process
+            # when running the first testcase.  The problem is somewhat
+            # more general, though, as any potential use of asyncio in
+            # Cooker could create a loop that needs to replaced in this
+            # new process.
+            self._create_loop()
+            try:
+                self.address = None
+                tasks = self.start()
+            finally:
+                # Always put the server address to wake up the parent task
+                queue.put(self.address)
+                queue.close()
+
+            if prefunc is not None:
+                prefunc(self, *args)
+
+            if log_level is not None:
+                self.logger.setLevel(log_level)
+
+            self._serve_forever(tasks)
+
+            self.loop.run_until_complete(self.loop.shutdown_asyncgens())
+            self.loop.close()
+
+        queue = multiprocessing.Queue()
+
+        # Temporarily block SIGTERM. The server process will inherit this
+        # block which will ensure it doesn't receive the SIGTERM until the
+        # handler is ready for it
+        mask = signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGTERM])
+        try:
+            self.process = multiprocessing.Process(target=run, args=(queue,))
+            self.process.start()
+
+            self.address = queue.get()
+            queue.close()
+            queue.join_thread()
+
+            return self.process
+        finally:
+            signal.pthread_sigmask(signal.SIG_SETMASK, mask)
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index f4f897e41a..40839a81b5 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -20,10 +20,12 @@ import itertools
 import time
 import re
 import stat
+import datetime
 import bb
 import bb.msg
 import bb.process
 import bb.progress
+from io import StringIO
 from bb import data, event, utils
 
 bblogger = logging.getLogger('BitBake')
@@ -176,7 +178,9 @@ class StdoutNoopContextManager:
 
     @property
     def name(self):
-        return sys.stdout.name
+        if "name" in dir(sys.stdout):
+            return sys.stdout.name
+        return "<mem>"
 
 
 def exec_func(func, d, dirs = None):
@@ -193,6 +197,8 @@ def exec_func(func, d, dirs = None):
         for cdir in d.expand(cleandirs).split():
             bb.utils.remove(cdir, True)
             bb.utils.mkdirhier(cdir)
+            if cdir == oldcwd:
+                os.chdir(cdir)
 
     if flags and dirs is None:
         dirs = flags.get('dirs')
@@ -295,9 +301,25 @@ def exec_func_python(func, d, runfile, cwd=None):
         lineno = int(d.getVarFlag(func, "lineno", False))
         bb.methodpool.insert_method(func, text, fn, lineno - 1)
 
-        comp = utils.better_compile(code, func, "exec_python_func() autogenerated")
-        utils.better_exec(comp, {"d": d}, code, "exec_python_func() autogenerated")
+        if verboseStdoutLogging:
+            sys.stdout.flush()
+            sys.stderr.flush()
+            currout = sys.stdout
+            currerr = sys.stderr
+            sys.stderr = sys.stdout = execio = StringIO()
+        comp = utils.better_compile(code, func, "exec_func_python() autogenerated")
+        utils.better_exec(comp, {"d": d}, code, "exec_func_python() autogenerated")
     finally:
+        if verboseStdoutLogging:
+            execio.flush()
+            logger.plain("%s" % execio.getvalue())
+            sys.stdout = currout
+            sys.stderr = currerr
+            execio.close()
+        # We want any stdout/stderr to be printed before any other log messages to make debugging
+        # more accurate. In some cases we seem to lose stdout/stderr entirely in logging tests without this.
+        sys.stdout.flush()
+        sys.stderr.flush()
         bb.debug(2, "Python function %s finished" % func)
 
         if cwd and olddir:
@@ -375,7 +397,7 @@ def create_progress_handler(func, progress, logfile, d):
         # Use specified regex
         return bb.progress.OutOfProgressHandler(d, regex=progress.split(':', 1)[1], outfile=logfile)
     elif progress.startswith("custom:"):
-        # Use a custom progress handler that was injected via OE_EXTRA_IMPORTS or __builtins__
+        # Use a custom progress handler that was injected via other means
         import functools
         from types import ModuleType
 
@@ -436,7 +458,11 @@ exit $ret
         if fakerootcmd:
             cmd = [fakerootcmd, runfile]
 
-    if verboseStdoutLogging:
+    # We only want to output to logger via LogTee if stdout is sys.__stdout__ (which will either
+    # be real stdout or subprocess PIPE or similar). In other cases we are being run "recursively",
+    # ie. inside another function, in which case stdout is already being captured so we don't
+    # want to Tee here as output would be printed twice, and out of order.
+    if verboseStdoutLogging and sys.stdout == sys.__stdout__:
         logfile = LogTee(logger, StdoutNoopContextManager())
     else:
         logfile = StdoutNoopContextManager()
@@ -565,10 +591,8 @@ exit $ret
 def _task_data(fn, task, d):
     localdata = bb.data.createCopy(d)
     localdata.setVar('BB_FILENAME', fn)
-    localdata.setVar('BB_CURRENTTASK', task[3:])
     localdata.setVar('OVERRIDES', 'task-%s:%s' %
                      (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False)))
-    localdata.finalize()
     bb.data.expandKeys(localdata)
     return localdata
 
@@ -579,7 +603,7 @@ def _exec_task(fn, task, d, quieterr):
     running it with its own local metadata, and with some useful variables set.
     """
     if not d.getVarFlag(task, 'task', False):
-        event.fire(TaskInvalid(task, d), d)
+        event.fire(TaskInvalid(task, fn, d), d)
         logger.error("No such task: %s" % task)
         return 1
 
@@ -615,7 +639,8 @@ def _exec_task(fn, task, d, quieterr):
     logorder = os.path.join(tempdir, 'log.task_order')
     try:
         with open(logorder, 'a') as logorderfile:
-            logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase))
+            timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S.%f")
+            logorderfile.write('{0} {1} ({2}): {3}\n'.format(timestamp, task, os.getpid(), logbase))
     except OSError:
         logger.exception("Opening log file '%s'", logorder)
         pass
@@ -682,47 +707,55 @@ def _exec_task(fn, task, d, quieterr):
     try:
         try:
             event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata)
-        except (bb.BBHandledException, SystemExit):
-            return 1
 
-        try:
             for func in (prefuncs or '').split():
                 exec_func(func, localdata)
             exec_func(task, localdata)
             for func in (postfuncs or '').split():
                 exec_func(func, localdata)
-        except bb.BBHandledException:
-            event.fire(TaskFailed(task, fn, logfn, localdata, True), localdata)
-            return 1
-        except Exception as exc:
-            if quieterr:
-                event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata)
-            else:
-                errprinted = errchk.triggered
+        finally:
+            # Need to flush and close the logs before sending events where the
+            # UI may try to look at the logs.
+            sys.stdout.flush()
+            sys.stderr.flush()
+
+            bblogger.removeHandler(handler)
+
+            # Restore the backup fds
+            os.dup2(osi[0], osi[1])
+            os.dup2(oso[0], oso[1])
+            os.dup2(ose[0], ose[1])
+
+            # Close the backup fds
+            os.close(osi[0])
+            os.close(oso[0])
+            os.close(ose[0])
+
+            logfile.close()
+            if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
+                logger.debug2("Zero size logfn %s, removing", logfn)
+                bb.utils.remove(logfn)
+                bb.utils.remove(loglink)
+    except (Exception, SystemExit) as exc:
+        handled = False
+        if isinstance(exc, bb.BBHandledException):
+            handled = True
+
+        if quieterr:
+            if not handled:
+                logger.warning(str(exc))
+            event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata)
+        else:
+            errprinted = errchk.triggered
+            # If the output is already on stdout, we've printed the information in the
+            # logs once already so don't duplicate
+            if verboseStdoutLogging or handled:
+                errprinted = True
+            if not handled:
                 logger.error(str(exc))
-                event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata)
-            return 1
-    finally:
-        sys.stdout.flush()
-        sys.stderr.flush()
-
-        bblogger.removeHandler(handler)
-
-        # Restore the backup fds
-        os.dup2(osi[0], osi[1])
-        os.dup2(oso[0], oso[1])
-        os.dup2(ose[0], ose[1])
-
-        # Close the backup fds
-        os.close(osi[0])
-        os.close(oso[0])
-        os.close(ose[0])
+            event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata)
+        return 1
 
-        logfile.close()
-        if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
-            logger.debug2("Zero size logfn %s, removing", logfn)
-            bb.utils.remove(logfn)
-            bb.utils.remove(loglink)
     event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata)
 
     if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False):
@@ -760,132 +793,92 @@ def exec_task(fn, task, d, profile = False):
             event.fire(failedevent, d)
         return 1
 
-def stamp_internal(taskname, d, file_name, baseonly=False, noextra=False):
+def _get_cleanmask(taskname, mcfn):
     """
-    Internal stamp helper function
-    Makes sure the stamp directory exists
+    Internal stamp helper function to generate stamp cleaning mask
     Returns the stamp path+filename
 
     In the bitbake core, d can be a CacheData and file_name will be set.
     When called in task context, d will be a data store, file_name will not be set
     """
-    taskflagname = taskname
-    if taskname.endswith("_setscene") and taskname != "do_setscene":
-        taskflagname = taskname.replace("_setscene", "")
-
-    if file_name:
-        stamp = d.stamp[file_name]
-        extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
-    else:
-        stamp = d.getVar('STAMP')
-        file_name = d.getVar('BB_FILENAME')
-        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
+    cleanmask = bb.parse.siggen.stampcleanmask_mcfn(taskname, mcfn)
+    taskflagname = taskname.replace("_setscene", "")
+    if cleanmask:
+        return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")]
+    return []
+
+def clean_stamp_mcfn(task, mcfn):
+    cleanmask = _get_cleanmask(task, mcfn)
+    for mask in cleanmask:
+        for name in glob.glob(mask):
+            # Preserve sigdata files in the stamps directory
+            if "sigdata" in name or "sigbasedata" in name:
+                continue
+            # Preserve taint files in the stamps directory
+            if name.endswith('.taint'):
+                continue
+            os.unlink(name)
 
-    if baseonly:
-        return stamp
-    if noextra:
-        extrainfo = ""
+def clean_stamp(task, d):
+    mcfn = d.getVar('BB_FILENAME')
+    clean_stamp_mcfn(task, mcfn)
 
-    if not stamp:
-        return
+def make_stamp_mcfn(task, mcfn):
 
-    stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo)
+    basestamp = bb.parse.siggen.stampfile_mcfn(task, mcfn)
 
-    stampdir = os.path.dirname(stamp)
+    stampdir = os.path.dirname(basestamp)
     if cached_mtime_noerror(stampdir) == 0:
         bb.utils.mkdirhier(stampdir)
 
-    return stamp
+    clean_stamp_mcfn(task, mcfn)
 
-def stamp_cleanmask_internal(taskname, d, file_name):
-    """
-    Internal stamp helper function to generate stamp cleaning mask
-    Returns the stamp path+filename
+    # Remove the file and recreate to force timestamp
+    # change on broken NFS filesystems
+    if basestamp:
+        bb.utils.remove(basestamp)
+        open(basestamp, "w").close()
 
-    In the bitbake core, d can be a CacheData and file_name will be set.
-    When called in task context, d will be a data store, file_name will not be set
+def make_stamp(task, d):
     """
-    taskflagname = taskname
-    if taskname.endswith("_setscene") and taskname != "do_setscene":
-        taskflagname = taskname.replace("_setscene", "")
-
-    if file_name:
-        stamp = d.stampclean[file_name]
-        extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
-    else:
-        stamp = d.getVar('STAMPCLEAN')
-        file_name = d.getVar('BB_FILENAME')
-        extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or ""
+    Creates/updates a stamp for a given task
+    """
+    mcfn = d.getVar('BB_FILENAME')
 
-    if not stamp:
-        return []
+    make_stamp_mcfn(task, mcfn)
 
-    cleanmask = bb.parse.siggen.stampcleanmask(stamp, file_name, taskname, extrainfo)
+    # If we're in task context, write out a signature file for each task
+    # as it completes
+    if not task.endswith("_setscene"):
+        stampbase = bb.parse.siggen.stampfile_base(mcfn)
+        bb.parse.siggen.dump_sigtask(mcfn, task, stampbase, True)
 
-    return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")]
 
-def make_stamp(task, d, file_name = None):
-    """
-    Creates/updates a stamp for a given task
-    (d can be a data dict or dataCache)
-    """
-    cleanmask = stamp_cleanmask_internal(task, d, file_name)
+def find_stale_stamps(task, mcfn):
+    current = bb.parse.siggen.stampfile_mcfn(task, mcfn)
+    current2 = bb.parse.siggen.stampfile_mcfn(task + "_setscene", mcfn)
+    cleanmask = _get_cleanmask(task, mcfn)
+    found = []
     for mask in cleanmask:
         for name in glob.glob(mask):
-            # Preserve sigdata files in the stamps directory
             if "sigdata" in name or "sigbasedata" in name:
                 continue
-            # Preserve taint files in the stamps directory
             if name.endswith('.taint'):
                 continue
-            os.unlink(name)
-
-    stamp = stamp_internal(task, d, file_name)
-    # Remove the file and recreate to force timestamp
-    # change on broken NFS filesystems
-    if stamp:
-        bb.utils.remove(stamp)
-        open(stamp, "w").close()
-
-    # If we're in task context, write out a signature file for each task
-    # as it completes
-    if not task.endswith("_setscene") and task != "do_setscene" and not file_name:
-        stampbase = stamp_internal(task, d, None, True)
-        file_name = d.getVar('BB_FILENAME')
-        bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True)
-
-def del_stamp(task, d, file_name = None):
-    """
-    Removes a stamp for a given task
-    (d can be a data dict or dataCache)
-    """
-    stamp = stamp_internal(task, d, file_name)
-    bb.utils.remove(stamp)
+            if name == current or name == current2:
+                continue
+            logger.debug2("Stampfile %s does not match %s or %s" % (name, current, current2))
+            found.append(name)
+    return found
 
-def write_taint(task, d, file_name = None):
+def write_taint(task, d):
     """
     Creates a "taint" file which will force the specified task and its
     dependents to be re-run the next time by influencing the value of its
     taskhash.
-    (d can be a data dict or dataCache)
     """
-    import uuid
-    if file_name:
-        taintfn = d.stamp[file_name] + '.' + task + '.taint'
-    else:
-        taintfn = d.getVar('STAMP') + '.' + task + '.taint'
-    bb.utils.mkdirhier(os.path.dirname(taintfn))
-    # The specific content of the taint file is not really important,
-    # we just need it to be random, so a random UUID is used
-    with open(taintfn, 'w') as taintf:
-        taintf.write(str(uuid.uuid4()))
-
-def stampfile(taskname, d, file_name = None, noextra=False):
-    """
-    Return the stamp for a given task
-    (d can be a data dict or dataCache)
-    """
-    return stamp_internal(taskname, d, file_name, noextra=noextra)
+    mcfn = d.getVar('BB_FILENAME')
+    bb.parse.siggen.invalidate_task(task, mcfn)
 
 def add_tasks(tasklist, d):
     task_deps = d.getVar('_task_deps', False)
@@ -910,6 +903,11 @@ def add_tasks(tasklist, d):
                 task_deps[name] = {}
             if name in flags:
                 deptask = d.expand(flags[name])
+                if name in ['noexec', 'fakeroot', 'nostamp']:
+                    if deptask != '1':
+                        bb.warn("In a future version of BitBake, setting the '{}' flag to something other than '1' "
+                                "will result in the flag not being set. See YP bug #13808.".format(name))
+
                 task_deps[name][task] = deptask
         getTask('mcdepends')
         getTask('depends')
@@ -934,9 +932,13 @@ def add_tasks(tasklist, d):
     # don't assume holding a reference
     d.setVar('_task_deps', task_deps)
 
+def ensure_task_prefix(name):
+    if name[:3] != "do_":
+        name = "do_" + name
+    return name
+
 def addtask(task, before, after, d):
-    if task[:3] != "do_":
-        task = "do_" + task
+    task = ensure_task_prefix(task)
 
     d.setVarFlag(task, "task", 1)
     bbtasks = d.getVar('__BBTASKS', False) or []
@@ -948,19 +950,20 @@ def addtask(task, before, after, d):
     if after is not None:
         # set up deps for function
         for entry in after.split():
+            entry = ensure_task_prefix(entry)
             if entry not in existing:
                 existing.append(entry)
     d.setVarFlag(task, "deps", existing)
     if before is not None:
         # set up things that depend on this func
         for entry in before.split():
+            entry = ensure_task_prefix(entry)
             existing = d.getVarFlag(entry, "deps", False) or []
             if task not in existing:
                 d.setVarFlag(entry, "deps", [task] + existing)
 
 def deltask(task, d):
-    if task[:3] != "do_":
-        task = "do_" + task
+    task = ensure_task_prefix(task)
 
     bbtasks = d.getVar('__BBTASKS', False) or []
     if task in bbtasks:
@@ -1008,6 +1011,8 @@ def tasksbetween(task_start, task_end, d):
     def follow_chain(task, endtask, chain=None):
         if not chain:
             chain = []
+        if task in chain:
+            bb.fatal("Circular task dependencies as %s depends on itself via the chain %s" % (task, " -> ".join(chain)))
         chain.append(task)
         for othertask in tasks:
             if othertask == task:
@@ -1023,3 +1028,9 @@ def tasksbetween(task_start, task_end, d):
         chain.pop()
     follow_chain(task_start, task_end)
     return outtasks
+
+def listtasks(d):
+    """
+    Return the list of tasks in the current recipe.
+    """
+    return tuple(d.getVar('__BBTASKS', False) or ())
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index aea2b8bc11..2361c5684d 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -19,14 +19,16 @@
 import os
 import logging
 import pickle
-from collections import defaultdict, Mapping
+from collections import defaultdict
+from collections.abc import Mapping
 import bb.utils
 from bb import PrefixLoggerAdapter
 import re
+import shutil
 
 logger = logging.getLogger("BitBake.Cache")
 
-__cache_version__ = "154"
+__cache_version__ = "156"
 
 def getCacheFile(path, filename, mc, data_hash):
     mcspec = ''
@@ -53,12 +55,12 @@ class RecipeInfoCommon(object):
 
     @classmethod
     def pkgvar(cls, var, packages, metadata):
-        return dict((pkg, cls.depvar("%s_%s" % (var, pkg), metadata))
+        return dict((pkg, cls.depvar("%s:%s" % (var, pkg), metadata))
                     for pkg in packages)
 
     @classmethod
     def taskvar(cls, var, tasks, metadata):
-        return dict((task, cls.getvar("%s_task-%s" % (var, task), metadata))
+        return dict((task, cls.getvar("%s:task-%s" % (var, task), metadata))
                     for task in tasks)
 
     @classmethod
@@ -103,7 +105,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
 
         self.tasks = metadata.getVar('__BBTASKS', False)
 
-        self.basetaskhashes = self.taskvar('BB_BASEHASH', self.tasks, metadata)
+        self.basetaskhashes = metadata.getVar('__siggen_basehashes', False) or {}
         self.hashfilename = self.getvar('BB_HASHFILENAME', metadata)
 
         self.task_deps = metadata.getVar('_task_deps', False) or {'tasks': [], 'parents': {}}
@@ -126,6 +128,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         self.inherits         = self.getvar('__inherit_cache', metadata, expand=False)
         self.fakerootenv      = self.getvar('FAKEROOTENV', metadata)
         self.fakerootdirs     = self.getvar('FAKEROOTDIRS', metadata)
+        self.fakerootlogs     = self.getvar('FAKEROOTLOGS', metadata)
         self.fakerootnoenv    = self.getvar('FAKEROOTNOENV', metadata)
         self.extradepsfunc    = self.getvar('calculate_extra_depends', metadata)
 
@@ -163,6 +166,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
         cachedata.fakerootenv = {}
         cachedata.fakerootnoenv = {}
         cachedata.fakerootdirs = {}
+        cachedata.fakerootlogs = {}
         cachedata.extradepsfunc = {}
 
     def add_cacheData(self, cachedata, fn):
@@ -212,7 +216,7 @@ class CoreRecipeInfo(RecipeInfoCommon):
 
         # Collect files we may need for possible world-dep
         # calculations
-        if not self.not_world:
+        if not bb.utils.to_boolean(self.not_world):
             cachedata.possible_world.append(fn)
         #else:
         #    logger.debug2("EXCLUDE FROM WORLD: %s", fn)
@@ -231,17 +235,116 @@ class CoreRecipeInfo(RecipeInfoCommon):
         cachedata.fakerootenv[fn] = self.fakerootenv
         cachedata.fakerootnoenv[fn] = self.fakerootnoenv
         cachedata.fakerootdirs[fn] = self.fakerootdirs
+        cachedata.fakerootlogs[fn] = self.fakerootlogs
         cachedata.extradepsfunc[fn] = self.extradepsfunc
 
+
+class SiggenRecipeInfo(RecipeInfoCommon):
+    __slots__ = ()
+
+    classname = "SiggenRecipeInfo"
+    cachefile = "bb_cache_" + classname +".dat"
+    # we don't want to show this information in graph files so don't set cachefields
+    #cachefields = []
+
+    def __init__(self, filename, metadata):
+        self.siggen_gendeps = metadata.getVar("__siggen_gendeps", False)
+        self.siggen_varvals = metadata.getVar("__siggen_varvals", False)
+        self.siggen_taskdeps = metadata.getVar("__siggen_taskdeps", False)
+
+    @classmethod
+    def init_cacheData(cls, cachedata):
+        cachedata.siggen_taskdeps = {}
+        cachedata.siggen_gendeps = {}
+        cachedata.siggen_varvals = {}
+
+    def add_cacheData(self, cachedata, fn):
+        cachedata.siggen_gendeps[fn] = self.siggen_gendeps
+        cachedata.siggen_varvals[fn] = self.siggen_varvals
+        cachedata.siggen_taskdeps[fn] = self.siggen_taskdeps
+
+    # The siggen variable data is large and impacts:
+    #  - bitbake's overall memory usage
+    #  - the amount of data sent over IPC between parsing processes and the server
+    #  - the size of the cache files on disk
+    #  - the size of "sigdata" hash information files on disk
+    # The data consists of strings (some large) or frozenset lists of variables
+    # As such, we a) deplicate the data here and b) pass references to the object at second
+    # access (e.g. over IPC or saving into pickle).
+
+    store = {}
+    save_map = {}
+    save_count = 1
+    restore_map = {}
+    restore_count = {}
+
+    @classmethod
+    def reset(cls):
+        # Needs to be called before starting new streamed data in a given process 
+        # (e.g. writing out the cache again)
+        cls.save_map = {}
+        cls.save_count = 1
+        cls.restore_map = {}
+
+    @classmethod
+    def _save(cls, deps):
+        ret = []
+        if not deps:
+            return deps
+        for dep in deps:
+            fs = deps[dep]
+            if fs is None:
+                ret.append((dep, None, None))
+            elif fs in cls.save_map:
+                ret.append((dep, None, cls.save_map[fs]))
+            else:
+                cls.save_map[fs] = cls.save_count
+                ret.append((dep, fs, cls.save_count))
+                cls.save_count = cls.save_count + 1
+        return ret
+
+    @classmethod
+    def _restore(cls, deps, pid):
+        ret = {}
+        if not deps:
+            return deps
+        if pid not in cls.restore_map:
+            cls.restore_map[pid] = {}
+        map = cls.restore_map[pid]
+        for dep, fs, mapnum in deps:
+            if fs is None and mapnum is None:
+                ret[dep] = None
+            elif fs is None:
+                ret[dep] = map[mapnum]
+            else:
+                try:
+                    fs = cls.store[fs]
+                except KeyError:
+                    cls.store[fs] = fs
+                map[mapnum] = fs
+                ret[dep] = fs
+        return ret
+
+    def __getstate__(self):
+        ret = {}
+        for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]:
+            ret[key] = self._save(self.__dict__[key])
+        ret['pid'] = os.getpid()
+        return ret
+
+    def __setstate__(self, state):
+        pid = state['pid']
+        for key in ["siggen_gendeps", "siggen_taskdeps", "siggen_varvals"]:
+            setattr(self, key, self._restore(state[key], pid))
+
+
 def virtualfn2realfn(virtualfn):
     """
     Convert a virtual file name to a real one + the associated subclass keyword
     """
     mc = ""
     if virtualfn.startswith('mc:') and virtualfn.count(':') >= 2:
-        elems = virtualfn.split(':')
-        mc = elems[1]
-        virtualfn = ":".join(elems[2:])
+        (_, mc, virtualfn) = virtualfn.split(':', 2)
 
     fn = virtualfn
     cls = ""
@@ -264,7 +367,7 @@ def realfn2virtual(realfn, cls, mc):
 
 def variant2virtual(realfn, variant):
     """
-    Convert a real filename + the associated subclass keyword to a virtual filename
+    Convert a real filename + a variant to a virtual filename
     """
     if variant == "":
         return realfn
@@ -275,104 +378,26 @@ def variant2virtual(realfn, variant):
         return "mc:" + elems[1] + ":" + realfn
     return "virtual:" + variant + ":" + realfn
 
-def parse_recipe(bb_data, bbfile, appends, mc=''):
-    """
-    Parse a recipe
-    """
-
-    chdir_back = False
-
-    bb_data.setVar("__BBMULTICONFIG", mc)
-
-    # expand tmpdir to include this topdir
-    bb_data.setVar('TMPDIR', bb_data.getVar('TMPDIR') or "")
-    bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
-    oldpath = os.path.abspath(os.getcwd())
-    bb.parse.cached_mtime_noerror(bbfile_loc)
-
-    # The ConfHandler first looks if there is a TOPDIR and if not
-    # then it would call getcwd().
-    # Previously, we chdir()ed to bbfile_loc, called the handler
-    # and finally chdir()ed back, a couple of thousand times. We now
-    # just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet.
-    if not bb_data.getVar('TOPDIR', False):
-        chdir_back = True
-        bb_data.setVar('TOPDIR', bbfile_loc)
-    try:
-        if appends:
-            bb_data.setVar('__BBAPPEND', " ".join(appends))
-        bb_data = bb.parse.handle(bbfile, bb_data)
-        if chdir_back:
-            os.chdir(oldpath)
-        return bb_data
-    except:
-        if chdir_back:
-            os.chdir(oldpath)
-        raise
-
-
-
-class NoCache(object):
-
-    def __init__(self, databuilder):
-        self.databuilder = databuilder
-        self.data = databuilder.data
-
-    def loadDataFull(self, virtualfn, appends):
-        """
-        Return a complete set of data for fn.
-        To do this, we need to parse the file.
-        """
-        logger.debug("Parsing %s (full)" % virtualfn)
-        (fn, virtual, mc) = virtualfn2realfn(virtualfn)
-        bb_data = self.load_bbfile(virtualfn, appends, virtonly=True)
-        return bb_data[virtual]
-
-    def load_bbfile(self, bbfile, appends, virtonly = False, mc=None):
-        """
-        Load and parse one .bb build file
-        Return the data and whether parsing resulted in the file being skipped
-        """
-
-        if virtonly:
-            (bbfile, virtual, mc) = virtualfn2realfn(bbfile)
-            bb_data = self.databuilder.mcdata[mc].createCopy()
-            bb_data.setVar("__ONLYFINALISE", virtual or "default")
-            datastores = parse_recipe(bb_data, bbfile, appends, mc)
-            return datastores
-
-        if mc is not None:
-            bb_data = self.databuilder.mcdata[mc].createCopy()
-            return parse_recipe(bb_data, bbfile, appends, mc)
-
-        bb_data = self.data.createCopy()
-        datastores = parse_recipe(bb_data, bbfile, appends)
-
-        for mc in self.databuilder.mcdata:
-            if not mc:
-                continue
-            bb_data = self.databuilder.mcdata[mc].createCopy()
-            newstores = parse_recipe(bb_data, bbfile, appends, mc)
-            for ns in newstores:
-                datastores["mc:%s:%s" % (mc, ns)] = newstores[ns]
-
-        return datastores
-
-class Cache(NoCache):
+#
+# Cooker calls cacheValid on its recipe list, then either calls loadCached
+# from it's main thread or parse from separate processes to generate an up to
+# date cache
+#
+class Cache(object):
     """
     BitBake Cache implementation
     """
     def __init__(self, databuilder, mc, data_hash, caches_array):
-        super().__init__(databuilder)
-        data = databuilder.data
+        self.databuilder = databuilder
+        self.data = databuilder.data
 
         # Pass caches_array information into Cache Constructor
         # It will be used later for deciding whether we
         # need extra cache file dump/load support
         self.mc = mc
-        self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else "default"), logger)
+        self.logger = PrefixLoggerAdapter("Cache: %s: " % (mc if mc else ''), logger)
         self.caches_array = caches_array
-        self.cachedir = data.getVar("CACHE")
+        self.cachedir = self.data.getVar("CACHE")
         self.clean = set()
         self.checked = set()
         self.depends_cache = {}
@@ -382,20 +407,12 @@ class Cache(NoCache):
         self.filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
 
         if self.cachedir in [None, '']:
-            self.has_cache = False
-            self.logger.info("Not using a cache. "
-                             "Set CACHE = <directory> to enable.")
-            return
-
-        self.has_cache = True
+            bb.fatal("Please ensure CACHE is set to the cache directory for BitBake to use")
 
     def getCacheFile(self, cachefile):
         return getCacheFile(self.cachedir, cachefile, self.mc, self.data_hash)
 
     def prepare_cache(self, progress):
-        if not self.has_cache:
-            return 0
-
         loaded = 0
 
         self.cachefile = self.getCacheFile("bb_cache.dat")
@@ -424,7 +441,7 @@ class Cache(NoCache):
         else:
             symlink = os.path.join(self.cachedir, "bb_cache.dat")
 
-        if os.path.exists(symlink):
+        if os.path.exists(symlink) or os.path.islink(symlink):
             bb.utils.remove(symlink)
         try:
             os.symlink(os.path.basename(self.cachefile), symlink)
@@ -434,9 +451,6 @@ class Cache(NoCache):
         return loaded
 
     def cachesize(self):
-        if not self.has_cache:
-            return 0
-
         cachesize = 0
         for cache_class in self.caches_array:
             cachefile = self.getCacheFile(cache_class.cachefile)
@@ -498,11 +512,11 @@ class Cache(NoCache):
 
         return len(self.depends_cache)
 
-    def parse(self, filename, appends):
+    def parse(self, filename, appends, layername):
         """Parse the specified filename, returning the recipe information"""
         self.logger.debug("Parsing %s", filename)
         infos = []
-        datastores = self.load_bbfile(filename, appends, mc=self.mc)
+        datastores = self.databuilder.parseRecipeVariants(filename, appends, mc=self.mc, layername=layername)
         depends = []
         variants = []
         # Process the "real" fn last so we can store variants list
@@ -524,43 +538,19 @@ class Cache(NoCache):
 
         return infos
 
-    def load(self, filename, appends):
+    def loadCached(self, filename, appends):
         """Obtain the recipe information for the specified filename,
-        using cached values if available, otherwise parsing.
-
-        Note that if it does parse to obtain the info, it will not
-        automatically add the information to the cache or to your
-        CacheData.  Use the add or add_info method to do so after
-        running this, or use loadData instead."""
-        cached = self.cacheValid(filename, appends)
-        if cached:
-            infos = []
-            # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo]
-            info_array = self.depends_cache[filename]
-            for variant in info_array[0].variants:
-                virtualfn = variant2virtual(filename, variant)
-                infos.append((virtualfn, self.depends_cache[virtualfn]))
-        else:
-            return self.parse(filename, appends, configdata, self.caches_array)
-
-        return cached, infos
-
-    def loadData(self, fn, appends, cacheData):
-        """Load the recipe info for the specified filename,
-        parsing and adding to the cache if necessary, and adding
-        the recipe information to the supplied CacheData instance."""
-        skipped, virtuals = 0, 0
+        using cached values.
+        """
 
-        cached, infos = self.load(fn, appends)
-        for virtualfn, info_array in infos:
-            if info_array[0].skipped:
-                self.logger.debug("Skipping %s: %s", virtualfn, info_array[0].skipreason)
-                skipped += 1
-            else:
-                self.add_info(virtualfn, info_array, cacheData, not cached)
-                virtuals += 1
+        infos = []
+        # info_array item is a list of [CoreRecipeInfo, XXXRecipeInfo]
+        info_array = self.depends_cache[filename]
+        for variant in info_array[0].variants:
+            virtualfn = variant2virtual(filename, variant)
+            infos.append((virtualfn, self.depends_cache[virtualfn]))
 
-        return cached, skipped, virtuals
+        return infos
 
     def cacheValid(self, fn, appends):
         """
@@ -569,10 +559,6 @@ class Cache(NoCache):
         """
         if fn not in self.checked:
             self.cacheValidUpdate(fn, appends)
-
-        # Is cache enabled?
-        if not self.has_cache:
-            return False
         if fn in self.clean:
             return True
         return False
@@ -582,10 +568,6 @@ class Cache(NoCache):
         Is the cache valid for fn?
         Make thorough (slower) checks including timestamps.
         """
-        # Is cache enabled?
-        if not self.has_cache:
-            return False
-
         self.checked.add(fn)
 
         # File isn't in depends_cache
@@ -636,7 +618,7 @@ class Cache(NoCache):
                 for f in flist:
                     if not f:
                         continue
-                    f, exist = f.split(":")
+                    f, exist = f.rsplit(":", 1)
                     if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
                         self.logger.debug2("%s's file checksum list file %s changed",
                                              fn, f)
@@ -692,10 +674,6 @@ class Cache(NoCache):
         Save the cache
         Called from the parser when complete (or exiting)
         """
-
-        if not self.has_cache:
-            return
-
         if self.cacheclean:
             self.logger.debug2("Cache is clean, not saving.")
             return
@@ -716,6 +694,7 @@ class Cache(NoCache):
                             p.dump(info)
 
         del self.depends_cache
+        SiggenRecipeInfo.reset()
 
     @staticmethod
     def mtime(cachefile):
@@ -738,26 +717,11 @@ class Cache(NoCache):
             if watcher:
                 watcher(info_array[0].file_depends)
 
-        if not self.has_cache:
-            return
-
         if (info_array[0].skipped or 'SRCREVINACTION' not in info_array[0].pv) and not info_array[0].nocache:
             if parsed:
                 self.cacheclean = False
             self.depends_cache[filename] = info_array
 
-    def add(self, file_name, data, cacheData, parsed=None):
-        """
-        Save data we need into the cache
-        """
-
-        realfn = virtualfn2realfn(file_name)[0]
-
-        info_array = []
-        for cache_class in self.caches_array:
-            info_array.append(cache_class(realfn, data))
-        self.add_info(file_name, info_array, cacheData, parsed)
-
 class MulticonfigCache(Mapping):
     def __init__(self, databuilder, data_hash, caches_array):
         def progress(p):
@@ -794,6 +758,7 @@ class MulticonfigCache(Mapping):
         loaded = 0
 
         for c in self.__caches.values():
+            SiggenRecipeInfo.reset()
             loaded += c.prepare_cache(progress)
             previous_progress = current_progress
 
@@ -814,25 +779,6 @@ class MulticonfigCache(Mapping):
         for k in self.__caches:
             yield k
 
-def init(cooker):
-    """
-    The Objective: Cache the minimum amount of data possible yet get to the
-    stage of building packages (i.e. tryBuild) without reparsing any .bb files.
-
-    To do this, we intercept getVar calls and only cache the variables we see
-    being accessed. We rely on the cache getVar calls being made for all
-    variables bitbake might need to use to reach this stage. For each cached
-    file we need to track:
-
-    * Its mtime
-    * The mtimes of all its dependencies
-    * Whether it caused a parse.SkipRecipe exception
-
-    Files causing parsing errors are evicted from the cache.
-
-    """
-    return Cache(cooker.configuration.data, cooker.configuration.data_hash)
-
 
 class CacheData(object):
     """
@@ -871,11 +817,10 @@ class MultiProcessCache(object):
         self.cachedata = self.create_cachedata()
         self.cachedata_extras = self.create_cachedata()
 
-    def init_cache(self, d, cache_file_name=None):
-        cachedir = (d.getVar("PERSISTENT_DIR") or
-                    d.getVar("CACHE"))
-        if cachedir in [None, '']:
+    def init_cache(self, cachedir, cache_file_name=None):
+        if not cachedir:
             return
+
         bb.utils.mkdirhier(cachedir)
         self.cachefile = os.path.join(cachedir,
                                       cache_file_name or self.__class__.cache_file_name)
@@ -902,10 +847,24 @@ class MultiProcessCache(object):
         data = [{}]
         return data
 
+    def clear_cache(self):
+        if not self.cachefile:
+            bb.fatal("Can't clear invalid cachefile")
+
+        self.cachedata = self.create_cachedata()
+        self.cachedata_extras = self.create_cachedata()
+        with bb.utils.fileslocked([self.cachefile + ".lock"]):
+            bb.utils.remove(self.cachefile)
+            bb.utils.remove(self.cachefile + "-*")
+
     def save_extras(self):
         if not self.cachefile:
             return
 
+        have_data = any(self.cachedata_extras)
+        if not have_data:
+            return
+
         glf = bb.utils.lockfile(self.cachefile + ".lock", shared=True)
 
         i = os.getpid()
@@ -940,6 +899,8 @@ class MultiProcessCache(object):
 
         data = self.cachedata
 
+        have_data = False
+
         for f in [y for y in os.listdir(os.path.dirname(self.cachefile)) if y.startswith(os.path.basename(self.cachefile) + '-')]:
             f = os.path.join(os.path.dirname(self.cachefile), f)
             try:
@@ -954,12 +915,14 @@ class MultiProcessCache(object):
                 os.unlink(f)
                 continue
 
+            have_data = True
             self.merge_data(extradata, data)
             os.unlink(f)
 
-        with open(self.cachefile, "wb") as f:
-            p = pickle.Pickler(f, -1)
-            p.dump([data, self.__class__.CACHE_VERSION])
+        if have_data:
+            with open(self.cachefile, "wb") as f:
+                p = pickle.Pickler(f, -1)
+                p.dump([data, self.__class__.CACHE_VERSION])
 
         bb.utils.unlockfile(glf)
 
@@ -1015,3 +978,11 @@ class SimpleCache(object):
             p.dump([data, self.cacheversion])
 
         bb.utils.unlockfile(glf)
+
+    def copyfile(self, target):
+        if not self.cachefile:
+            return
+
+        glf = bb.utils.lockfile(self.cachefile + ".lock")
+        shutil.copy(self.cachefile, target)
+        bb.utils.unlockfile(glf)
diff --git a/bitbake/lib/bb/checksum.py b/bitbake/lib/bb/checksum.py
index 1d50a26426..3fb39a303e 100644
--- a/bitbake/lib/bb/checksum.py
+++ b/bitbake/lib/bb/checksum.py
@@ -11,10 +11,13 @@ import os
 import stat
 import bb.utils
 import logging
+import re
 from bb.cache import MultiProcessCache
 
 logger = logging.getLogger("BitBake.Cache")
 
+filelist_regex = re.compile(r'(?:(?<=:True)|(?<=:False))\s+')
+
 # mtime cache (non-persistent)
 # based upon the assumption that files do not change during bitbake run
 class FileMtimeCache(object):
@@ -50,6 +53,7 @@ class FileChecksumCache(MultiProcessCache):
         MultiProcessCache.__init__(self)
 
     def get_checksum(self, f):
+        f = os.path.normpath(f)
         entry = self.cachedata[0].get(f)
         cmtime = self.mtime_cache.cached_mtime(f)
         if entry:
@@ -84,22 +88,36 @@ class FileChecksumCache(MultiProcessCache):
                 return None
             return checksum
 
+        #
+        # Changing the format of file-checksums is problematic as both OE and Bitbake have
+        # knowledge of them. We need to encode a new piece of data, the portion of the path
+        # we care about from a checksum perspective. This means that files that change subdirectory
+        # are tracked by the task hashes. To do this, we do something horrible and put a "/./" into
+        # the path. The filesystem handles it but it gives us a marker to know which subsection
+        # of the path to cache.
+        #
         def checksum_dir(pth):
             # Handle directories recursively
             if pth == "/":
                 bb.fatal("Refusing to checksum /")
+            pth = pth.rstrip("/")
             dirchecksums = []
             for root, dirs, files in os.walk(pth, topdown=True):
                 [dirs.remove(d) for d in list(dirs) if d in localdirsexclude]
                 for name in files:
-                    fullpth = os.path.join(root, name)
+                    fullpth = os.path.join(root, name).replace(pth, os.path.join(pth, "."))
                     checksum = checksum_file(fullpth)
                     if checksum:
                         dirchecksums.append((fullpth, checksum))
             return dirchecksums
 
         checksums = []
-        for pth in filelist.split():
+        for pth in filelist_regex.split(filelist):
+            if not pth:
+                continue
+            pth = pth.strip()
+            if not pth:
+                continue
             exist = pth.split(":")[1]
             if exist == "False":
                 continue
@@ -124,3 +142,28 @@ class FileChecksumCache(MultiProcessCache):
 
         checksums.sort(key=operator.itemgetter(1))
         return checksums
+
+class RevisionsCache(MultiProcessCache):
+    cache_file_name = "local_srcrevisions.dat"
+    CACHE_VERSION = 1
+
+    def __init__(self):
+        MultiProcessCache.__init__(self)
+
+    def get_revs(self):
+        return self.cachedata[0]
+
+    def get_rev(self, k):
+        if k in self.cachedata_extras[0]:
+            return self.cachedata_extras[0][k]
+        if k in self.cachedata[0]:
+            return self.cachedata[0][k]
+        return None
+
+    def set_rev(self, k, v):
+        self.cachedata[0][k] = v
+        self.cachedata_extras[0][k] = v
+
+    def merge_data(self, source, dest):
+        for h in source[0]:
+            dest[0][h] = source[0][h]
diff --git a/bitbake/lib/bb/codeparser.py b/bitbake/lib/bb/codeparser.py
index 25a7ac69d3..4f70cf7fe7 100644
--- a/bitbake/lib/bb/codeparser.py
+++ b/bitbake/lib/bb/codeparser.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -25,6 +27,7 @@ import ast
 import sys
 import codegen
 import logging
+import inspect
 import bb.pysh as pysh
 import bb.utils, bb.data
 import hashlib
@@ -56,10 +59,56 @@ def check_indent(codestr):
 
     return codestr
 
-# A custom getstate/setstate using tuples is actually worth 15% cachesize by
-# avoiding duplication of the attribute names!
+modulecode_deps = {}
 
+def add_module_functions(fn, functions, namespace):
+    import os
+    fstat = os.stat(fn)
+    fixedhash = fn + ":" + str(fstat.st_size) +  ":" + str(fstat.st_mtime)
+    for f in functions:
+        name = "%s.%s" % (namespace, f)
+        parser = PythonParser(name, logger)
+        try:
+            parser.parse_python(None, filename=fn, lineno=1, fixedhash=fixedhash+f, func=functions[f])
+            #bb.warn("Cached %s" % f)
+        except KeyError:
+            try:
+                targetfn = inspect.getsourcefile(functions[f])
+            except TypeError:
+                # Builtin
+                continue
+            if fn != targetfn:
+                # Skip references to other modules outside this file
+                #bb.warn("Skipping %s" % name)
+                continue
+            try:
+                lines, lineno = inspect.getsourcelines(functions[f])
+            except TypeError:
+                # Builtin
+                continue
+            src = "".join(lines)
+            parser.parse_python(src, filename=fn, lineno=lineno, fixedhash=fixedhash+f, func=functions[f])
+            #bb.warn("Not cached %s" % f)
+        execs = parser.execs.copy()
+        # Expand internal module exec references
+        for e in parser.execs:
+            if e in functions:
+                execs.remove(e)
+                execs.add(namespace + "." + e)
+        visitorcode = None
+        if hasattr(functions[f], 'visitorcode'):
+            visitorcode = getattr(functions[f], "visitorcode")
+        modulecode_deps[name] = [parser.references.copy(), execs, parser.var_execs.copy(), parser.contains.copy(), parser.extra, visitorcode]
+        #bb.warn("%s: %s\nRefs:%s Execs: %s %s %s" % (name, fn, parser.references, parser.execs, parser.var_execs, parser.contains))
+
+def update_module_dependencies(d):
+    for mod in modulecode_deps:
+        excludes = set((d.getVarFlag(mod, "vardepsexclude") or "").split())
+        if excludes:
+            modulecode_deps[mod] = [modulecode_deps[mod][0] - excludes, modulecode_deps[mod][1] - excludes, modulecode_deps[mod][2] - excludes, modulecode_deps[mod][3], modulecode_deps[mod][4], modulecode_deps[mod][5]]
 
+# A custom getstate/setstate using tuples is actually worth 15% cachesize by
+# avoiding duplication of the attribute names!
 class SetCache(object):
     def __init__(self):
         self.setcache = {}
@@ -79,21 +128,22 @@ class SetCache(object):
 codecache = SetCache()
 
 class pythonCacheLine(object):
-    def __init__(self, refs, execs, contains):
+    def __init__(self, refs, execs, contains, extra):
         self.refs = codecache.internSet(refs)
         self.execs = codecache.internSet(execs)
         self.contains = {}
         for c in contains:
             self.contains[c] = codecache.internSet(contains[c])
+        self.extra = extra
 
     def __getstate__(self):
-        return (self.refs, self.execs, self.contains)
+        return (self.refs, self.execs, self.contains, self.extra)
 
     def __setstate__(self, state):
-        (refs, execs, contains) = state
-        self.__init__(refs, execs, contains)
+        (refs, execs, contains, extra) = state
+        self.__init__(refs, execs, contains, extra)
     def __hash__(self):
-        l = (hash(self.refs), hash(self.execs))
+        l = (hash(self.refs), hash(self.execs), hash(self.extra))
         for c in sorted(self.contains.keys()):
             l = l + (c, hash(self.contains[c]))
         return hash(l)
@@ -122,7 +172,7 @@ class CodeParserCache(MultiProcessCache):
     # so that an existing cache gets invalidated. Additionally you'll need
     # to increment __cache_version__ in cache.py in order to ensure that old
     # recipe caches don't trigger "Taskhash mismatch" errors.
-    CACHE_VERSION = 11
+    CACHE_VERSION = 14
 
     def __init__(self):
         MultiProcessCache.__init__(self)
@@ -136,8 +186,8 @@ class CodeParserCache(MultiProcessCache):
         self.pythoncachelines = {}
         self.shellcachelines = {}
 
-    def newPythonCacheLine(self, refs, execs, contains):
-        cacheline = pythonCacheLine(refs, execs, contains)
+    def newPythonCacheLine(self, refs, execs, contains, extra):
+        cacheline = pythonCacheLine(refs, execs, contains, extra)
         h = hash(cacheline)
         if h in self.pythoncachelines:
             return self.pythoncachelines[h]
@@ -152,12 +202,12 @@ class CodeParserCache(MultiProcessCache):
         self.shellcachelines[h] = cacheline
         return cacheline
 
-    def init_cache(self, d):
+    def init_cache(self, cachedir):
         # Check if we already have the caches
         if self.pythoncache:
             return
 
-        MultiProcessCache.init_cache(self, d)
+        MultiProcessCache.init_cache(self, cachedir)
 
         # cachedata gets re-assigned in the parent
         self.pythoncache = self.cachedata[0]
@@ -169,8 +219,8 @@ class CodeParserCache(MultiProcessCache):
 
 codeparsercache = CodeParserCache()
 
-def parser_cache_init(d):
-    codeparsercache.init_cache(d)
+def parser_cache_init(cachedir):
+    codeparsercache.init_cache(cachedir)
 
 def parser_cache_save():
     codeparsercache.save_extras()
@@ -195,6 +245,10 @@ class BufferedLogger(Logger):
                 self.target.handle(record)
         self.buffer = []
 
+class DummyLogger():
+    def flush(self):
+        return
+
 class PythonParser():
     getvars = (".getVar", ".appendVar", ".prependVar", "oe.utils.conditional")
     getvarflags = (".getVarFlag", ".appendVarFlag", ".prependVarFlag")
@@ -212,26 +266,34 @@ class PythonParser():
             funcstr = codegen.to_source(func)
             argstr = codegen.to_source(arg)
         except TypeError:
-            self.log.debug(2, 'Failed to convert function and argument to source form')
+            self.log.debug2('Failed to convert function and argument to source form')
         else:
-            self.log.debug(1, self.unhandled_message % (funcstr, argstr))
+            self.log.debug(self.unhandled_message % (funcstr, argstr))
 
     def visit_Call(self, node):
         name = self.called_node_name(node.func)
-        if name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs or name in self.containsanyfuncs):
-            if isinstance(node.args[0], ast.Str):
-                varname = node.args[0].s
-                if name in self.containsfuncs and isinstance(node.args[1], ast.Str):
+        if name and name in modulecode_deps and modulecode_deps[name][5]:
+            visitorcode = modulecode_deps[name][5]
+            contains, execs, warn = visitorcode(name, node.args)
+            for i in contains:
+                self.contains[i] = contains[i]
+            self.execs |= execs
+            if warn:
+                self.warn(node.func, warn)
+        elif name and (name.endswith(self.getvars) or name.endswith(self.getvarflags) or name in self.containsfuncs or name in self.containsanyfuncs):
+            if isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str):
+                varname = node.args[0].value
+                if name in self.containsfuncs and isinstance(node.args[1], ast.Constant):
                     if varname not in self.contains:
                         self.contains[varname] = set()
-                    self.contains[varname].add(node.args[1].s)
-                elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Str):
+                    self.contains[varname].add(node.args[1].value)
+                elif name in self.containsanyfuncs and isinstance(node.args[1], ast.Constant):
                     if varname not in self.contains:
                         self.contains[varname] = set()
-                    self.contains[varname].update(node.args[1].s.split())
+                    self.contains[varname].update(node.args[1].value.split())
                 elif name.endswith(self.getvarflags):
-                    if isinstance(node.args[1], ast.Str):
-                        self.references.add('%s[%s]' % (varname, node.args[1].s))
+                    if isinstance(node.args[1], ast.Constant):
+                        self.references.add('%s[%s]' % (varname, node.args[1].value))
                     else:
                         self.warn(node.func, node.args[1])
                 else:
@@ -239,8 +301,8 @@ class PythonParser():
             else:
                 self.warn(node.func, node.args[0])
         elif name and name.endswith(".expand"):
-            if isinstance(node.args[0], ast.Str):
-                value = node.args[0].s
+            if isinstance(node.args[0], ast.Constant):
+                value = node.args[0].value
                 d = bb.data.init()
                 parser = d.expandWithRefs(value, self.name)
                 self.references |= parser.references
@@ -250,8 +312,8 @@ class PythonParser():
                         self.contains[varname] = set()
                     self.contains[varname] |= parser.contains[varname]
         elif name in self.execfuncs:
-            if isinstance(node.args[0], ast.Str):
-                self.var_execs.add(node.args[0].s)
+            if isinstance(node.args[0], ast.Constant):
+                self.var_execs.add(node.args[0].value)
             else:
                 self.warn(node.func, node.args[0])
         elif name and isinstance(node.func, (ast.Name, ast.Attribute)):
@@ -276,16 +338,24 @@ class PythonParser():
         self.contains = {}
         self.execs = set()
         self.references = set()
-        self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, log)
+        self._log = log
+        # Defer init as expensive
+        self.log = DummyLogger()
 
         self.unhandled_message = "in call of %s, argument '%s' is not a string literal"
         self.unhandled_message = "while parsing %s, %s" % (name, self.unhandled_message)
 
-    def parse_python(self, node, lineno=0, filename="<string>"):
-        if not node or not node.strip():
+    # For the python module code it is expensive to have the function text so it is
+    # uses a different fixedhash to cache against. We can take the hit on obtaining the
+    # text if it isn't in the cache.
+    def parse_python(self, node, lineno=0, filename="<string>", fixedhash=None, func=None):
+        if not fixedhash and (not node or not node.strip()):
             return
 
-        h = bbhash(str(node))
+        if fixedhash:
+            h = fixedhash
+        else:
+            h = bbhash(str(node))
 
         if h in codeparsercache.pythoncache:
             self.references = set(codeparsercache.pythoncache[h].refs)
@@ -293,6 +363,7 @@ class PythonParser():
             self.contains = {}
             for i in codeparsercache.pythoncache[h].contains:
                 self.contains[i] = set(codeparsercache.pythoncache[h].contains[i])
+            self.extra = codeparsercache.pythoncache[h].extra
             return
 
         if h in codeparsercache.pythoncacheextras:
@@ -301,8 +372,15 @@ class PythonParser():
             self.contains = {}
             for i in codeparsercache.pythoncacheextras[h].contains:
                 self.contains[i] = set(codeparsercache.pythoncacheextras[h].contains[i])
+            self.extra = codeparsercache.pythoncacheextras[h].extra
             return
 
+        if fixedhash and not node:
+            raise KeyError
+
+        # Need to parse so take the hit on the real log buffer
+        self.log = BufferedLogger('BitBake.Data.PythonParser', logging.DEBUG, self._log)
+
         # We can't add to the linenumbers for compile, we can pad to the correct number of blank lines though
         node = "\n" * int(lineno) + node
         code = compile(check_indent(str(node)), filename, "exec",
@@ -312,16 +390,27 @@ class PythonParser():
             if n.__class__.__name__ == "Call":
                 self.visit_Call(n)
 
+        if func is not None:
+            self.references |= getattr(func, "bb_vardeps", set())
+            self.references -= getattr(func, "bb_vardepsexclude", set())
+
         self.execs.update(self.var_execs)
+        self.extra = None
+        if fixedhash:
+            self.extra = bbhash(str(node))
 
-        codeparsercache.pythoncacheextras[h] = codeparsercache.newPythonCacheLine(self.references, self.execs, self.contains)
+        codeparsercache.pythoncacheextras[h] = codeparsercache.newPythonCacheLine(self.references, self.execs, self.contains, self.extra)
 
 class ShellParser():
     def __init__(self, name, log):
         self.funcdefs = set()
         self.allexecs = set()
         self.execs = set()
-        self.log = BufferedLogger('BitBake.Data.%s' % name, logging.DEBUG, log)
+        self._name = name
+        self._log = log
+        # Defer init as expensive
+        self.log = DummyLogger()
+
         self.unhandled_template = "unable to handle non-literal command '%s'"
         self.unhandled_template = "while parsing %s, %s" % (name, self.unhandled_template)
 
@@ -340,6 +429,9 @@ class ShellParser():
             self.execs = set(codeparsercache.shellcacheextras[h].execs)
             return self.execs
 
+        # Need to parse so take the hit on the real log buffer
+        self.log = BufferedLogger('BitBake.Data.%s' % self._name, logging.DEBUG, self._log)
+
         self._parse_shell(value)
         self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)
 
@@ -426,19 +518,34 @@ class ShellParser():
         """
 
         words = list(words)
-        for word in list(words):
+        for word in words:
             wtree = pyshlex.make_wordtree(word[1])
             for part in wtree:
                 if not isinstance(part, list):
                     continue
 
-                if part[0] in ('`', '$('):
-                    command = pyshlex.wordtree_as_string(part[1:-1])
-                    self._parse_shell(command)
-
-                    if word[0] in ("cmd_name", "cmd_word"):
-                        if word in words:
-                            words.remove(word)
+                candidates = [part]
+
+                # If command is of type:
+                #
+                #   var="... $(cmd [...]) ..."
+                #
+                # Then iterate on what's between the quotes and if we find a
+                # list, make that what we check for below.
+                if len(part) >= 3 and part[0] == '"':
+                    for p in part[1:-1]:
+                        if isinstance(p, list):
+                            candidates.append(p)
+
+                for candidate in candidates:
+                    if len(candidate) >= 2:
+                        if candidate[0] in ('`', '$('):
+                            command = pyshlex.wordtree_as_string(candidate[1:-1])
+                            self._parse_shell(command)
+
+                            if word[0] in ("cmd_name", "cmd_word"):
+                                if word in words:
+                                    words.remove(word)
 
         usetoken = False
         for word in words:
@@ -450,7 +557,7 @@ class ShellParser():
 
                 cmd = word[1]
                 if cmd.startswith("$"):
-                    self.log.debug(1, self.unhandled_template % cmd)
+                    self.log.debug(self.unhandled_template % cmd)
                 elif cmd == "eval":
                     command = " ".join(word for _, word in words[1:])
                     self._parse_shell(command)
diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py
index dd77cdd6e2..59a979ee90 100644
--- a/bitbake/lib/bb/command.py
+++ b/bitbake/lib/bb/command.py
@@ -20,9 +20,11 @@ Commands are queued in a CommandQueue
 
 from collections import OrderedDict, defaultdict
 
+import io
 import bb.event
 import bb.cooker
 import bb.remotedata
+import bb.parse
 
 class DataStoreConnectionHandle(object):
     def __init__(self, dsindex=0):
@@ -50,23 +52,32 @@ class Command:
     """
     A queue of asynchronous commands for bitbake
     """
-    def __init__(self, cooker):
+    def __init__(self, cooker, process_server):
         self.cooker = cooker
         self.cmds_sync = CommandsSync()
         self.cmds_async = CommandsAsync()
         self.remotedatastores = None
 
-        # FIXME Add lock for this
+        self.process_server = process_server
+        # Access with locking using process_server.{get/set/clear}_async_cmd()
         self.currentAsyncCommand = None
 
-    def runCommand(self, commandline, ro_only = False):
+    def runCommand(self, commandline, process_server, ro_only=False):
         command = commandline.pop(0)
 
         # Ensure cooker is ready for commands
-        if command != "updateConfig" and command != "setFeatures":
-            self.cooker.init_configdata()
-            if not self.remotedatastores:
-                self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker)
+        if command not in ["updateConfig", "setFeatures", "ping"]:
+            try:
+                self.cooker.init_configdata()
+                if not self.remotedatastores:
+                    self.remotedatastores = bb.remotedata.RemoteDatastores(self.cooker)
+            except (Exception, SystemExit) as exc:
+                import traceback
+                if isinstance(exc, bb.BBHandledException):
+                    # We need to start returning real exceptions here. Until we do, we can't
+                    # tell if an exception is an instance of bb.BBHandledException
+                    return None, "bb.BBHandledException()\n" + traceback.format_exc()
+                return None, traceback.format_exc()
 
         if hasattr(CommandsSync, command):
             # Can run synchronous commands straight away
@@ -75,7 +86,6 @@ class Command:
                 if not hasattr(command_method, 'readonly') or not getattr(command_method, 'readonly'):
                     return None, "Not able to execute not readonly commands in readonly mode"
             try:
-                self.cooker.process_inotify_updates()
                 if getattr(command_method, 'needconfig', True):
                     self.cooker.updateCacheSync()
                 result = command_method(self, commandline)
@@ -90,61 +100,57 @@ class Command:
                 return None, traceback.format_exc()
             else:
                 return result, None
-        if self.currentAsyncCommand is not None:
-            return None, "Busy (%s in progress)" % self.currentAsyncCommand[0]
         if command not in CommandsAsync.__dict__:
             return None, "No such command"
-        self.currentAsyncCommand = (command, commandline)
-        self.cooker.idleCallBackRegister(self.cooker.runCommands, self.cooker)
+        if not process_server.set_async_cmd((command, commandline)):
+            return None, "Busy (%s in progress)" % self.process_server.get_async_cmd()[0]
+        self.cooker.idleCallBackRegister(self.runAsyncCommand, process_server)
         return True, None
 
-    def runAsyncCommand(self):
+    def runAsyncCommand(self, _, process_server, halt):
         try:
-            self.cooker.process_inotify_updates()
-            if self.cooker.state in (bb.cooker.state.error, bb.cooker.state.shutdown, bb.cooker.state.forceshutdown):
+            if self.cooker.state in (bb.cooker.State.ERROR, bb.cooker.State.SHUTDOWN, bb.cooker.State.FORCE_SHUTDOWN):
                 # updateCache will trigger a shutdown of the parser
                 # and then raise BBHandledException triggering an exit
                 self.cooker.updateCache()
-                return False
-            if self.currentAsyncCommand is not None:
-                (command, options) = self.currentAsyncCommand
+                return bb.server.process.idleFinish("Cooker in error state")
+            cmd = process_server.get_async_cmd()
+            if cmd is not None:
+                (command, options) = cmd
                 commandmethod = getattr(CommandsAsync, command)
                 needcache = getattr( commandmethod, "needcache" )
-                if needcache and self.cooker.state != bb.cooker.state.running:
+                if needcache and self.cooker.state != bb.cooker.State.RUNNING:
                     self.cooker.updateCache()
                     return True
                 else:
                     commandmethod(self.cmds_async, self, options)
                     return False
             else:
-                return False
+                return bb.server.process.idleFinish("Nothing to do, no async command?")
         except KeyboardInterrupt as exc:
-            self.finishAsyncCommand("Interrupted")
-            return False
+            return bb.server.process.idleFinish("Interrupted")
         except SystemExit as exc:
             arg = exc.args[0]
             if isinstance(arg, str):
-                self.finishAsyncCommand(arg)
+                return bb.server.process.idleFinish(arg)
             else:
-                self.finishAsyncCommand("Exited with %s" % arg)
-            return False
+                return bb.server.process.idleFinish("Exited with %s" % arg)
         except Exception as exc:
             import traceback
             if isinstance(exc, bb.BBHandledException):
-                self.finishAsyncCommand("")
+                return bb.server.process.idleFinish("")
             else:
-                self.finishAsyncCommand(traceback.format_exc())
-            return False
+                return bb.server.process.idleFinish(traceback.format_exc())
 
     def finishAsyncCommand(self, msg=None, code=None):
+        self.cooker.finishcommand()
+        self.process_server.clear_async_cmd()
         if msg or msg == "":
             bb.event.fire(CommandFailed(msg), self.cooker.data)
         elif code:
             bb.event.fire(CommandExit(code), self.cooker.data)
         else:
             bb.event.fire(CommandCompleted(), self.cooker.data)
-        self.currentAsyncCommand = None
-        self.cooker.finishcommand()
 
     def reset(self):
         if self.remotedatastores:
@@ -157,6 +163,14 @@ class CommandsSync:
     These must not influence any running synchronous command.
     """
 
+    def ping(self, command, params):
+        """
+        Allow a UI to check the server is still alive
+        """
+        return "Still alive!"
+    ping.needconfig = False
+    ping.readonly = True
+
     def stateShutdown(self, command, params):
         """
         Trigger cooker 'shutdown' mode
@@ -294,6 +308,11 @@ class CommandsSync:
         return ret
     getLayerPriorities.readonly = True
 
+    def revalidateCaches(self, command, params):
+        """Called by UI clients when metadata may have changed"""
+        command.cooker.revalidateCaches()
+    revalidateCaches.needconfig = False
+
     def getRecipes(self, command, params):
         try:
             mc = params[0]
@@ -402,15 +421,30 @@ class CommandsSync:
         return command.cooker.recipecaches[mc].pkg_dp
     getDefaultPreference.readonly = True
 
+
     def getSkippedRecipes(self, command, params):
+        """
+        Get the map of skipped recipes for the specified multiconfig/mc name (`params[0]`).
+
+        Invoked by `bb.tinfoil.Tinfoil.get_skipped_recipes`
+
+        :param command: Internally used parameter.
+        :param params: Parameter array. params[0] is multiconfig/mc name. If not given, then default mc '' is assumed.
+        :return: Dict whose keys are virtualfns and values are `bb.cooker.SkippedPackage`
+        """
+        try:
+            mc = params[0]
+        except IndexError:
+            mc = ''
+
         # Return list sorted by reverse priority order
         import bb.cache
         def sortkey(x):
             vfn, _ = x
-            realfn, _, mc = bb.cache.virtualfn2realfn(vfn)
-            return (-command.cooker.collections[mc].calc_bbfile_priority(realfn)[0], vfn)
+            realfn, _, item_mc = bb.cache.virtualfn2realfn(vfn)
+            return -command.cooker.collections[item_mc].calc_bbfile_priority(realfn)[0], vfn
 
-        skipdict = OrderedDict(sorted(command.cooker.skiplist.items(), key=sortkey))
+        skipdict = OrderedDict(sorted(command.cooker.skiplist_by_mc[mc].items(), key=sortkey))
         return list(skipdict.items())
     getSkippedRecipes.readonly = True
 
@@ -500,6 +534,17 @@ class CommandsSync:
         d = command.remotedatastores[dsindex].varhistory
         return getattr(d, method)(*args, **kwargs)
 
+    def dataStoreConnectorVarHistCmdEmit(self, command, params):
+        dsindex = params[0]
+        var = params[1]
+        oval = params[2]
+        val = params[3]
+        d = command.remotedatastores[params[4]]
+
+        o = io.StringIO()
+        command.remotedatastores[dsindex].varhistory.emit(var, oval, val, o, d)
+        return o.getvalue()
+
     def dataStoreConnectorIncHistCmd(self, command, params):
         dsindex = params[0]
         method = params[1]
@@ -521,8 +566,8 @@ class CommandsSync:
         and return a datastore object representing the environment
         for the recipe.
         """
-        fn = params[0]
-        mc = bb.runqueue.mc_from_tid(fn)
+        virtualfn = params[0]
+        (fn, cls, mc) = bb.cache.virtualfn2realfn(virtualfn)
         appends = params[1]
         appendlist = params[2]
         if len(params) > 3:
@@ -537,6 +582,7 @@ class CommandsSync:
                 appendfiles = command.cooker.collections[mc].get_file_appends(fn)
         else:
             appendfiles = []
+        layername = command.cooker.collections[mc].calc_bbfile_priority(fn)[2]
         # We are calling bb.cache locally here rather than on the server,
         # but that's OK because it doesn't actually need anything from
         # the server barring the global datastore (which we have a remote
@@ -544,15 +590,21 @@ class CommandsSync:
         if config_data:
             # We have to use a different function here if we're passing in a datastore
             # NOTE: we took a copy above, so we don't do it here again
-            envdata = bb.cache.parse_recipe(config_data, fn, appendfiles, mc)['']
+            envdata = command.cooker.databuilder._parse_recipe(config_data, fn, appendfiles, mc, layername)[cls]
         else:
             # Use the standard path
-            parser = bb.cache.NoCache(command.cooker.databuilder)
-            envdata = parser.loadDataFull(fn, appendfiles)
+            envdata = command.cooker.databuilder.parseRecipe(virtualfn, appendfiles, layername)
         idx = command.remotedatastores.store(envdata)
         return DataStoreConnectionHandle(idx)
     parseRecipeFile.readonly = True
 
+    def finalizeData(self, command, params):
+        newdata = command.cooker.data.createCopy()
+        bb.data.expandKeys(newdata)
+        bb.parse.ast.runAnonFuncs(newdata)
+        idx = command.remotedatastores.store(newdata)
+        return DataStoreConnectionHandle(idx)
+
 class CommandsAsync:
     """
     A class of asynchronous commands
@@ -647,6 +699,16 @@ class CommandsAsync:
         command.finishAsyncCommand()
     findFilesMatchingInDir.needcache = False
 
+    def testCookerCommandEvent(self, command, params):
+        """
+        Dummy command used by OEQA selftest to test tinfoil without IO
+        """
+        pattern = params[0]
+
+        command.cooker.testCookerCommandEvent(pattern)
+        command.finishAsyncCommand()
+    testCookerCommandEvent.needcache = False
+
     def findConfigFilePath(self, command, params):
         """
         Find the path of the requested configuration file
@@ -711,7 +773,7 @@ class CommandsAsync:
         """
         event = params[0]
         bb.event.fire(eval(event), command.cooker.data)
-        command.currentAsyncCommand = None
+        process_server.clear_async_cmd()
     triggerEvent.needcache = False
 
     def resetCooker(self, command, params):
@@ -738,7 +800,14 @@ class CommandsAsync:
         (mc, pn) = bb.runqueue.split_mc(params[0])
         taskname = params[1]
         sigs = params[2]
+        bb.siggen.check_siggen_version(bb.siggen)
         res = bb.siggen.find_siginfo(pn, taskname, sigs, command.cooker.databuilder.mcdata[mc])
         bb.event.fire(bb.event.FindSigInfoResult(res), command.cooker.databuilder.mcdata[mc])
         command.finishAsyncCommand()
     findSigInfo.needcache = False
+
+    def getTaskSignatures(self, command, params):
+        res = command.cooker.getTaskSignatures(params[0], params[1])
+        bb.event.fire(bb.event.GetTaskSignatureResult(res), command.cooker.data)
+        command.finishAsyncCommand()
+    getTaskSignatures.needcache = True
diff --git a/bitbake/lib/bb/compress/_pipecompress.py b/bitbake/lib/bb/compress/_pipecompress.py
new file mode 100644
index 0000000000..4a403d62cf
--- /dev/null
+++ b/bitbake/lib/bb/compress/_pipecompress.py
@@ -0,0 +1,196 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Helper library to implement streaming compression and decompression using an
+# external process
+#
+# This library should be used directly by end users; a wrapper library for the
+# specific compression tool should be created
+
+import builtins
+import io
+import os
+import subprocess
+
+
+def open_wrap(
+    cls, filename, mode="rb", *, encoding=None, errors=None, newline=None, **kwargs
+):
+    """
+    Open a compressed file in binary or text mode.
+
+    Users should not call this directly. A specific compression library can use
+    this helper to provide it's own "open" command
+
+    The filename argument can be an actual filename (a str or bytes object), or
+    an existing file object to read from or write to.
+
+    The mode argument can be "r", "rb", "w", "wb", "x", "xb", "a" or "ab" for
+    binary mode, or "rt", "wt", "xt" or "at" for text mode. The default mode is
+    "rb".
+
+    For binary mode, this function is equivalent to the cls constructor:
+    cls(filename, mode). In this case, the encoding, errors and newline
+    arguments must not be provided.
+
+    For text mode, a cls object is created, and wrapped in an
+    io.TextIOWrapper instance with the specified encoding, error handling
+    behavior, and line ending(s).
+    """
+    if "t" in mode:
+        if "b" in mode:
+            raise ValueError("Invalid mode: %r" % (mode,))
+    else:
+        if encoding is not None:
+            raise ValueError("Argument 'encoding' not supported in binary mode")
+        if errors is not None:
+            raise ValueError("Argument 'errors' not supported in binary mode")
+        if newline is not None:
+            raise ValueError("Argument 'newline' not supported in binary mode")
+
+    file_mode = mode.replace("t", "")
+    if isinstance(filename, (str, bytes, os.PathLike, int)):
+        binary_file = cls(filename, file_mode, **kwargs)
+    elif hasattr(filename, "read") or hasattr(filename, "write"):
+        binary_file = cls(None, file_mode, fileobj=filename, **kwargs)
+    else:
+        raise TypeError("filename must be a str or bytes object, or a file")
+
+    if "t" in mode:
+        return io.TextIOWrapper(
+            binary_file, encoding, errors, newline, write_through=True
+        )
+    else:
+        return binary_file
+
+
+class CompressionError(OSError):
+    pass
+
+
+class PipeFile(io.RawIOBase):
+    """
+    Class that implements generically piping to/from a compression program
+
+    Derived classes should add the function get_compress() and get_decompress()
+    that return the required commands. Input will be piped into stdin and the
+    (de)compressed output should be written to stdout, e.g.:
+
+        class FooFile(PipeCompressionFile):
+            def get_decompress(self):
+                return ["fooc", "--decompress", "--stdout"]
+
+            def get_compress(self):
+                return ["fooc", "--compress", "--stdout"]
+
+    """
+
+    READ = 0
+    WRITE = 1
+
+    def __init__(self, filename=None, mode="rb", *, stderr=None, fileobj=None):
+        if "t" in mode or "U" in mode:
+            raise ValueError("Invalid mode: {!r}".format(mode))
+
+        if not "b" in mode:
+            mode += "b"
+
+        if mode.startswith("r"):
+            self.mode = self.READ
+        elif mode.startswith("w"):
+            self.mode = self.WRITE
+        else:
+            raise ValueError("Invalid mode %r" % mode)
+
+        if fileobj is not None:
+            self.fileobj = fileobj
+        else:
+            self.fileobj = builtins.open(filename, mode or "rb")
+
+        if self.mode == self.READ:
+            self.p = subprocess.Popen(
+                self.get_decompress(),
+                stdin=self.fileobj,
+                stdout=subprocess.PIPE,
+                stderr=stderr,
+                close_fds=True,
+            )
+            self.pipe = self.p.stdout
+        else:
+            self.p = subprocess.Popen(
+                self.get_compress(),
+                stdin=subprocess.PIPE,
+                stdout=self.fileobj,
+                stderr=stderr,
+                close_fds=True,
+            )
+            self.pipe = self.p.stdin
+
+        self.__closed = False
+
+    def _check_process(self):
+        if self.p is None:
+            return
+
+        returncode = self.p.wait()
+        if returncode:
+            raise CompressionError("Process died with %d" % returncode)
+        self.p = None
+
+    def close(self):
+        if self.closed:
+            return
+
+        self.pipe.close()
+        if self.p is not None:
+            self._check_process()
+        self.fileobj.close()
+
+        self.__closed = True
+
+    @property
+    def closed(self):
+        return self.__closed
+
+    def fileno(self):
+        return self.pipe.fileno()
+
+    def flush(self):
+        self.pipe.flush()
+
+    def isatty(self):
+        return self.pipe.isatty()
+
+    def readable(self):
+        return self.mode == self.READ
+
+    def writable(self):
+        return self.mode == self.WRITE
+
+    def readinto(self, b):
+        if self.mode != self.READ:
+            import errno
+
+            raise OSError(
+                errno.EBADF, "read() on write-only %s object" % self.__class__.__name__
+            )
+        size = self.pipe.readinto(b)
+        if size == 0:
+            self._check_process()
+        return size
+
+    def write(self, data):
+        if self.mode != self.WRITE:
+            import errno
+
+            raise OSError(
+                errno.EBADF, "write() on read-only %s object" % self.__class__.__name__
+            )
+        data = self.pipe.write(data)
+
+        if not data:
+            self._check_process()
+
+        return data
diff --git a/bitbake/lib/bb/compress/lz4.py b/bitbake/lib/bb/compress/lz4.py
new file mode 100644
index 0000000000..2a64681c86
--- /dev/null
+++ b/bitbake/lib/bb/compress/lz4.py
@@ -0,0 +1,19 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb.compress._pipecompress
+
+
+def open(*args, **kwargs):
+    return bb.compress._pipecompress.open_wrap(LZ4File, *args, **kwargs)
+
+
+class LZ4File(bb.compress._pipecompress.PipeFile):
+    def get_compress(self):
+        return ["lz4", "-z", "-c"]
+
+    def get_decompress(self):
+        return ["lz4", "-d", "-c"]
diff --git a/bitbake/lib/bb/compress/zstd.py b/bitbake/lib/bb/compress/zstd.py
new file mode 100644
index 0000000000..cdbbe9d60f
--- /dev/null
+++ b/bitbake/lib/bb/compress/zstd.py
@@ -0,0 +1,30 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import bb.compress._pipecompress
+import shutil
+
+
+def open(*args, **kwargs):
+    return bb.compress._pipecompress.open_wrap(ZstdFile, *args, **kwargs)
+
+
+class ZstdFile(bb.compress._pipecompress.PipeFile):
+    def __init__(self, *args, num_threads=1, compresslevel=3, **kwargs):
+        self.num_threads = num_threads
+        self.compresslevel = compresslevel
+        super().__init__(*args, **kwargs)
+
+    def _get_zstd(self):
+        if self.num_threads == 1 or not shutil.which("pzstd"):
+            return ["zstd"]
+        return ["pzstd", "-p", "%d" % self.num_threads]
+
+    def get_compress(self):
+        return self._get_zstd() + ["-c", "-%d" % self.compresslevel]
+
+    def get_decompress(self):
+        return self._get_zstd() + ["-d", "-c"]
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index f4ab797edf..1810bcc604 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -8,22 +8,20 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
-
+import enum
 import sys, os, glob, os.path, re, time
 import itertools
 import logging
 import multiprocessing
-import sre_constants
 import threading
 from io import StringIO, UnsupportedOperation
 from contextlib import closing
 from collections import defaultdict, namedtuple
-import bb, bb.exceptions, bb.command
+import bb, bb.command
 from bb import utils, data, parse, event, cache, providers, taskdata, runqueue, build
 import queue
 import signal
 import prserv.serv
-import pyinotify
 import json
 import pickle
 import codecs
@@ -50,16 +48,15 @@ class CollectionError(bb.BBHandledException):
     Exception raised when layer configuration is incorrect
     """
 
-class state:
-    initial, parsing, running, shutdown, forceshutdown, stopped, error = list(range(7))
 
-    @classmethod
-    def get_name(cls, code):
-        for name in dir(cls):
-            value = getattr(cls, name)
-            if type(value) == type(cls.initial) and value == code:
-                return name
-        raise ValueError("Invalid status code: %s" % code)
+class State(enum.Enum):
+    INITIAL = 0,
+    PARSING = 1,
+    RUNNING = 2,
+    SHUTDOWN = 3,
+    FORCE_SHUTDOWN = 4,
+    STOPPED = 5,
+    ERROR = 6
 
 
 class SkippedPackage:
@@ -81,7 +78,7 @@ class SkippedPackage:
 
 
 class CookerFeatures(object):
-    _feature_list = [HOB_EXTRA_CACHES, BASEDATASTORE_TRACKING, SEND_SANITYEVENTS] = list(range(3))
+    _feature_list = [HOB_EXTRA_CACHES, BASEDATASTORE_TRACKING, SEND_SANITYEVENTS, RECIPE_SIGGEN_INFO] = list(range(4))
 
     def __init__(self):
         self._features=set()
@@ -104,12 +101,15 @@ class CookerFeatures(object):
 
 class EventWriter:
     def __init__(self, cooker, eventfile):
-        self.file_inited = None
         self.cooker = cooker
         self.eventfile = eventfile
         self.event_queue = []
 
-    def write_event(self, event):
+    def write_variables(self):
+        with open(self.eventfile, "a") as f:
+            f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
+
+    def send(self, event):
         with open(self.eventfile, "a") as f:
             try:
                 str_event = codecs.encode(pickle.dumps(event), 'base64').decode('utf-8')
@@ -119,28 +119,6 @@ class EventWriter:
                 import traceback
                 print(err, traceback.format_exc())
 
-    def send(self, event):
-        if self.file_inited:
-            # we have the file, just write the event
-            self.write_event(event)
-        else:
-            # init on bb.event.BuildStarted
-            name = "%s.%s" % (event.__module__, event.__class__.__name__)
-            if name in ("bb.event.BuildStarted", "bb.cooker.CookerExit"):
-                with open(self.eventfile, "w") as f:
-                    f.write("%s\n" % json.dumps({ "allvariables" : self.cooker.getAllKeysWithFlags(["doc", "func"])}))
-
-                self.file_inited = True
-
-                # write pending events
-                for evt in self.event_queue:
-                    self.write_event(evt)
-
-                # also write the current event
-                self.write_event(event)
-            else:
-                # queue all events until the file is inited
-                self.event_queue.append(event)
 
 #============================================================================#
 # BBCooker
@@ -150,43 +128,34 @@ class BBCooker:
     Manages one bitbake build run
     """
 
-    def __init__(self, featureSet=None, idleCallBackRegister=None):
+    def __init__(self, featureSet=None, server=None):
         self.recipecaches = None
+        self.baseconfig_valid = False
+        self.parsecache_valid = False
         self.eventlog = None
-        self.skiplist = {}
+        # The skiplists, one per multiconfig
+        self.skiplist_by_mc = defaultdict(dict)
         self.featureset = CookerFeatures()
         if featureSet:
             for f in featureSet:
                 self.featureset.setFeature(f)
 
+        self.orig_syspath = sys.path.copy()
+        self.orig_sysmodules = [*sys.modules]
+
         self.configuration = bb.cookerdata.CookerConfiguration()
 
-        self.idleCallBackRegister = idleCallBackRegister
+        self.process_server = server
+        self.idleCallBackRegister = None
+        self.waitIdle = None
+        if server:
+            self.idleCallBackRegister = server.register_idle_function
+            self.waitIdle = server.wait_for_idle
 
         bb.debug(1, "BBCooker starting %s" % time.time())
-        sys.stdout.flush()
-
-        self.configwatcher = pyinotify.WatchManager()
-        bb.debug(1, "BBCooker pyinotify1 %s" % time.time())
-        sys.stdout.flush()
-
-        self.configwatcher.bbseen = set()
-        self.configwatcher.bbwatchedfiles = set()
-        self.confignotifier = pyinotify.Notifier(self.configwatcher, self.config_notifications)
-        bb.debug(1, "BBCooker pyinotify2 %s" % time.time())
-        sys.stdout.flush()
-        self.watchmask = pyinotify.IN_CLOSE_WRITE | pyinotify.IN_CREATE | pyinotify.IN_DELETE | \
-                         pyinotify.IN_DELETE_SELF | pyinotify.IN_MODIFY | pyinotify.IN_MOVE_SELF | \
-                         pyinotify.IN_MOVED_FROM | pyinotify.IN_MOVED_TO
-        self.watcher = pyinotify.WatchManager()
-        bb.debug(1, "BBCooker pyinotify3 %s" % time.time())
-        sys.stdout.flush()
-        self.watcher.bbseen = set()
-        self.watcher.bbwatchedfiles = set()
-        self.notifier = pyinotify.Notifier(self.watcher, self.notifications)
-
-        bb.debug(1, "BBCooker pyinotify complete %s" % time.time())
-        sys.stdout.flush()
+
+        self.configwatched = {}
+        self.parsewatched = {}
 
         # If being called by something like tinfoil, we need to clean cached data
         # which may now be invalid
@@ -197,14 +166,6 @@ class BBCooker:
         self.hashserv = None
         self.hashservaddr = None
 
-        self.inotify_modified_files = []
-
-        def _process_inotify_updates(server, cooker, abort):
-            cooker.process_inotify_updates()
-            return 1.0
-
-        self.idleCallBackRegister(_process_inotify_updates, self)
-
         # TOSTOP must not be set or our children will hang when they output
         try:
             fd = sys.stdout.fileno()
@@ -218,8 +179,8 @@ class BBCooker:
         except UnsupportedOperation:
             pass
 
-        self.command = bb.command.Command(self)
-        self.state = state.initial
+        self.command = bb.command.Command(self, self.process_server)
+        self.state = State.INITIAL
 
         self.parser = None
 
@@ -228,108 +189,68 @@ class BBCooker:
         signal.signal(signal.SIGHUP, self.sigterm_exception)
 
         bb.debug(1, "BBCooker startup complete %s" % time.time())
-        sys.stdout.flush()
 
     def init_configdata(self):
         if not hasattr(self, "data"):
             self.initConfigurationData()
             bb.debug(1, "BBCooker parsed base configuration %s" % time.time())
-            sys.stdout.flush()
             self.handlePRServ()
 
-    def process_inotify_updates(self):
-        for n in [self.confignotifier, self.notifier]:
-            if n.check_events(timeout=0):
-                # read notified events and enqeue them
-                n.read_events()
-                n.process_events()
-
-    def config_notifications(self, event):
-        if event.maskname == "IN_Q_OVERFLOW":
-            bb.warn("inotify event queue overflowed, invalidating caches.")
-            self.parsecache_valid = False
-            self.baseconfig_valid = False
-            bb.parse.clear_cache()
-            return
-        if not event.pathname in self.configwatcher.bbwatchedfiles:
-            return
-        if not event.pathname in self.inotify_modified_files:
-            self.inotify_modified_files.append(event.pathname)
-        self.baseconfig_valid = False
-
-    def notifications(self, event):
-        if event.maskname == "IN_Q_OVERFLOW":
-            bb.warn("inotify event queue overflowed, invalidating caches.")
-            self.parsecache_valid = False
-            bb.parse.clear_cache()
-            return
-        if event.pathname.endswith("bitbake-cookerdaemon.log") \
-                or event.pathname.endswith("bitbake.lock"):
-            return
-        if not event.pathname in self.inotify_modified_files:
-            self.inotify_modified_files.append(event.pathname)
-        self.parsecache_valid = False
+    def _baseconfig_set(self, value):
+        if value and not self.baseconfig_valid:
+            bb.server.process.serverlog("Base config valid")
+        elif not value and self.baseconfig_valid:
+            bb.server.process.serverlog("Base config invalidated")
+        self.baseconfig_valid = value
+
+    def _parsecache_set(self, value):
+        if value and not self.parsecache_valid:
+            bb.server.process.serverlog("Parse cache valid")
+        elif not value and self.parsecache_valid:
+            bb.server.process.serverlog("Parse cache invalidated")
+        self.parsecache_valid = value
+
+    def add_filewatch(self, deps, configwatcher=False):
+        if configwatcher:
+            watcher = self.configwatched
+        else:
+            watcher = self.parsewatched
 
-    def add_filewatch(self, deps, watcher=None, dirs=False):
-        if not watcher:
-            watcher = self.watcher
         for i in deps:
-            watcher.bbwatchedfiles.add(i[0])
-            if dirs:
-                f = i[0]
-            else:
-                f = os.path.dirname(i[0])
-            if f in watcher.bbseen:
-                continue
-            watcher.bbseen.add(f)
-            watchtarget = None
-            while True:
-                # We try and add watches for files that don't exist but if they did, would influence
-                # the parser. The parent directory of these files may not exist, in which case we need
-                # to watch any parent that does exist for changes.
-                try:
-                    watcher.add_watch(f, self.watchmask, quiet=False)
-                    if watchtarget:
-                        watcher.bbwatchedfiles.add(watchtarget)
-                    break
-                except pyinotify.WatchManagerError as e:
-                    if 'ENOENT' in str(e):
-                        watchtarget = f
-                        f = os.path.dirname(f)
-                        if f in watcher.bbseen:
-                            break
-                        watcher.bbseen.add(f)
-                        continue
-                    if 'ENOSPC' in str(e):
-                        providerlog.error("No space left on device or exceeds fs.inotify.max_user_watches?")
-                        providerlog.error("To check max_user_watches: sysctl -n fs.inotify.max_user_watches.")
-                        providerlog.error("To modify max_user_watches: sysctl -n -w fs.inotify.max_user_watches=<value>.")
-                        providerlog.error("Root privilege is required to modify max_user_watches.")
-                    raise
+            f = i[0]
+            mtime = i[1]
+            watcher[f] = mtime
 
     def sigterm_exception(self, signum, stackframe):
         if signum == signal.SIGTERM:
             bb.warn("Cooker received SIGTERM, shutting down...")
         elif signum == signal.SIGHUP:
             bb.warn("Cooker received SIGHUP, shutting down...")
-        self.state = state.forceshutdown
+        self.state = State.FORCE_SHUTDOWN
+        bb.event._should_exit.set()
 
     def setFeatures(self, features):
         # we only accept a new feature set if we're in state initial, so we can reset without problems
-        if not self.state in [state.initial, state.shutdown, state.forceshutdown, state.stopped, state.error]:
+        if not self.state in [State.INITIAL, State.SHUTDOWN, State.FORCE_SHUTDOWN, State.STOPPED, State.ERROR]:
             raise Exception("Illegal state for feature set change")
         original_featureset = list(self.featureset)
         for feature in features:
             self.featureset.setFeature(feature)
         bb.debug(1, "Features set %s (was %s)" % (original_featureset, list(self.featureset)))
-        if (original_featureset != list(self.featureset)) and self.state != state.error and hasattr(self, "data"):
+        if (original_featureset != list(self.featureset)) and self.state != State.ERROR and hasattr(self, "data"):
             self.reset()
 
     def initConfigurationData(self):
-
-        self.state = state.initial
+        self.state = State.INITIAL
         self.caches_array = []
 
+        sys.path = self.orig_syspath.copy()
+        for mod in [*sys.modules]:
+            if mod not in self.orig_sysmodules:
+                del sys.modules[mod]
+
+        self.configwatched = {}
+
         # Need to preserve BB_CONSOLELOG over resets
         consolelog = None
         if hasattr(self, "data"):
@@ -338,12 +259,12 @@ class BBCooker:
         if CookerFeatures.BASEDATASTORE_TRACKING in self.featureset:
             self.enableDataTracking()
 
-        all_extra_cache_names = []
+        caches_name_array = ['bb.cache:CoreRecipeInfo']
         # We hardcode all known cache types in a single place, here.
         if CookerFeatures.HOB_EXTRA_CACHES in self.featureset:
-            all_extra_cache_names.append("bb.cache_extra:HobRecipeInfo")
-
-        caches_name_array = ['bb.cache:CoreRecipeInfo'] + all_extra_cache_names
+            caches_name_array.append("bb.cache_extra:HobRecipeInfo")
+        if CookerFeatures.RECIPE_SIGGEN_INFO in self.featureset:
+            caches_name_array.append("bb.cache:SiggenRecipeInfo")
 
         # At least CoreRecipeInfo will be loaded, so caches_array will never be empty!
         # This is the entry point, no further check needed!
@@ -359,9 +280,12 @@ class BBCooker:
         self.databuilder = bb.cookerdata.CookerDataBuilder(self.configuration, False)
         self.databuilder.parseBaseConfiguration()
         self.data = self.databuilder.data
-        self.data_hash = self.databuilder.data_hash
         self.extraconfigdata = {}
 
+        eventlog = self.data.getVar("BB_DEFAULT_EVENTLOG")
+        if not self.configuration.writeeventlog and eventlog:
+            self.setupEventLog(eventlog)
+
         if consolelog:
             self.data.setVar("BB_CONSOLELOG", consolelog)
 
@@ -371,31 +295,48 @@ class BBCooker:
             self.disableDataTracking()
 
         for mc in self.databuilder.mcdata.values():
-            mc.renameVar("__depends", "__base_depends")
-            self.add_filewatch(mc.getVar("__base_depends", False), self.configwatcher)
+            self.add_filewatch(mc.getVar("__base_depends", False), configwatcher=True)
 
-        self.baseconfig_valid = True
-        self.parsecache_valid = False
+        self._baseconfig_set(True)
+        self._parsecache_set(False)
 
     def handlePRServ(self):
         # Setup a PR Server based on the new configuration
         try:
             self.prhost = prserv.serv.auto_start(self.data)
         except prserv.serv.PRServiceConfigError as e:
-            bb.fatal("Unable to start PR Server, exitting")
+            bb.fatal("Unable to start PR Server, exiting, check the bitbake-cookerdaemon.log")
 
         if self.data.getVar("BB_HASHSERVE") == "auto":
             # Create a new hash server bound to a unix domain socket
             if not self.hashserv:
                 dbfile = (self.data.getVar("PERSISTENT_DIR") or self.data.getVar("CACHE")) + "/hashserv.db"
+                upstream = self.data.getVar("BB_HASHSERVE_UPSTREAM") or None
+                if upstream:
+                    try:
+                        with hashserv.create_client(upstream) as client:
+                            client.ping()
+                    except ImportError as e:
+                        bb.fatal(""""Unable to use hash equivalence server at '%s' due to missing or incorrect python module:
+%s
+Please install the needed module on the build host, or use an environment containing it (e.g a pip venv or OpenEmbedded's buildtools tarball).
+You can also remove the BB_HASHSERVE_UPSTREAM setting, but this may result in significantly longer build times as bitbake will be unable to reuse prebuilt sstate artefacts."""
+                                 % (upstream, repr(e)))
+                    except ConnectionError as e:
+                        bb.warn("Unable to connect to hash equivalence server at '%s', please correct or remove BB_HASHSERVE_UPSTREAM:\n%s"
+                                 % (upstream, repr(e)))
+                        upstream = None
+
                 self.hashservaddr = "unix://%s/hashserve.sock" % self.data.getVar("TOPDIR")
-                self.hashserv = hashserv.create_server(self.hashservaddr, dbfile, sync=False)
-                self.hashserv.process = multiprocessing.Process(target=self.hashserv.serve_forever)
-                self.hashserv.process.start()
-            self.data.setVar("BB_HASHSERVE", self.hashservaddr)
-            self.databuilder.origdata.setVar("BB_HASHSERVE", self.hashservaddr)
-            self.databuilder.data.setVar("BB_HASHSERVE", self.hashservaddr)
+                self.hashserv = hashserv.create_server(
+                    self.hashservaddr,
+                    dbfile,
+                    sync=False,
+                    upstream=upstream,
+                )
+                self.hashserv.serve_as_process(log_level=logging.WARNING)
             for mc in self.databuilder.mcdata:
+                self.databuilder.mcorigdata[mc].setVar("BB_HASHSERVE", self.hashservaddr)
                 self.databuilder.mcdata[mc].setVar("BB_HASHSERVE", self.hashservaddr)
 
         bb.parse.init_parser(self.data)
@@ -410,6 +351,34 @@ class BBCooker:
         if hasattr(self, "data"):
             self.data.disableTracking()
 
+    def revalidateCaches(self):
+        bb.parse.clear_cache()
+
+        clean = True
+        for f in self.configwatched:
+            if not bb.parse.check_mtime(f, self.configwatched[f]):
+                bb.server.process.serverlog("Found %s changed, invalid cache" % f)
+                self._baseconfig_set(False)
+                self._parsecache_set(False)
+                clean = False
+                break
+
+        if clean:
+            for f in self.parsewatched:
+                if not bb.parse.check_mtime(f, self.parsewatched[f]):
+                    bb.server.process.serverlog("Found %s changed, invalid cache" % f)
+                    self._parsecache_set(False)
+                    clean = False
+                    break
+
+        if not clean:
+            bb.parse.BBHandler.cached_statements = {}
+
+        # If writes were made to any of the data stores, we need to recalculate the data
+        # store cache
+        if hasattr(self, "databuilder"):
+            self.databuilder.calc_datastore_hashes()
+
     def parseConfiguration(self):
         self.updateCacheSync()
 
@@ -428,8 +397,24 @@ class BBCooker:
             self.recipecaches[mc] = bb.cache.CacheData(self.caches_array)
 
         self.handleCollections(self.data.getVar("BBFILE_COLLECTIONS"))
-
-        self.parsecache_valid = False
+        self.collections = {}
+        for mc in self.multiconfigs:
+            self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
+
+        self._parsecache_set(False)
+
+    def setupEventLog(self, eventlog):
+        if self.eventlog and self.eventlog[0] != eventlog:
+            bb.event.unregister_UIHhandler(self.eventlog[1])
+            self.eventlog = None
+        if not self.eventlog or self.eventlog[0] != eventlog:
+            # we log all events to a file if so directed
+            # register the log file writer as UI Handler
+            if not os.path.exists(os.path.dirname(eventlog)):
+                bb.utils.mkdirhier(os.path.dirname(eventlog))
+            writer = EventWriter(self, eventlog)
+            EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event'])
+            self.eventlog = (eventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer)), writer)
 
     def updateConfigOpts(self, options, environment, cmdline):
         self.ui_cmdline = cmdline
@@ -450,14 +435,7 @@ class BBCooker:
                 setattr(self.configuration, o, options[o])
 
         if self.configuration.writeeventlog:
-            if self.eventlog and self.eventlog[0] != self.configuration.writeeventlog:
-                bb.event.unregister_UIHhandler(self.eventlog[1])
-            if not self.eventlog or self.eventlog[0] != self.configuration.writeeventlog:
-                # we log all events to a file if so directed
-                # register the log file writer as UI Handler
-                writer = EventWriter(self, self.configuration.writeeventlog)
-                EventLogWriteHandler = namedtuple('EventLogWriteHandler', ['event'])
-                self.eventlog = (self.configuration.writeeventlog, bb.event.register_UIHhandler(EventLogWriteHandler(writer)))
+            self.setupEventLog(self.configuration.writeeventlog)
 
         bb.msg.loggerDefaultLogLevel = self.configuration.default_loglevel
         bb.msg.loggerDefaultDomains = self.configuration.debug_domains
@@ -487,37 +465,37 @@ class BBCooker:
         # Now update all the variables not in the datastore to match
         self.configuration.env = environment
 
+        self.revalidateCaches()
         if not clean:
             logger.debug("Base environment change, triggering reparse")
             self.reset()
 
-    def runCommands(self, server, data, abort):
-        """
-        Run any queued asynchronous command
-        This is done by the idle handler so it runs in true context rather than
-        tied to any UI.
-        """
-
-        return self.command.runAsyncCommand()
-
     def showVersions(self):
 
-        (latest_versions, preferred_versions) = self.findProviders()
+        (latest_versions, preferred_versions, required) = self.findProviders()
 
-        logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version")
-        logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================")
+        logger.plain("%-35s %25s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version", "Required Version")
+        logger.plain("%-35s %25s %25s %25s\n", "===========", "==============", "=================", "================")
 
         for p in sorted(self.recipecaches[''].pkg_pn):
-            pref = preferred_versions[p]
+            preferred = preferred_versions[p]
             latest = latest_versions[p]
+            requiredstr = ""
+            preferredstr = ""
+            if required[p]:
+                if preferred[0] is not None:
+                    requiredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
+                else:
+                    bb.fatal("REQUIRED_VERSION of package %s not available" % p)
+            else:
+                preferredstr = preferred[0][0] + ":" + preferred[0][1] + '-' + preferred[0][2]
 
-            prefstr = pref[0][0] + ":" + pref[0][1] + '-' + pref[0][2]
             lateststr = latest[0][0] + ":" + latest[0][1] + "-" + latest[0][2]
 
-            if pref == latest:
-                prefstr = ""
+            if preferred == latest:
+                preferredstr = ""
 
-            logger.plain("%-35s %25s %25s", p, lateststr, prefstr)
+            logger.plain("%-35s %25s %25s %25s", p, lateststr, preferredstr, requiredstr)
 
     def showEnvironment(self, buildfile=None, pkgs_to_build=None):
         """
@@ -533,6 +511,8 @@ class BBCooker:
         if not orig_tracking:
             self.enableDataTracking()
             self.reset()
+            # reset() resets to the UI requested value so we have to redo this
+            self.enableDataTracking()
 
         def mc_base(p):
             if p.startswith('mc:'):
@@ -556,21 +536,21 @@ class BBCooker:
                 if pkgs_to_build[0] in set(ignore.split()):
                     bb.fatal("%s is in ASSUME_PROVIDED" % pkgs_to_build[0])
 
-                taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.abort, allowincomplete=True)
+                taskdata, runlist = self.buildTaskData(pkgs_to_build, None, self.configuration.halt, allowincomplete=True)
 
                 mc = runlist[0][0]
                 fn = runlist[0][3]
 
         if fn:
             try:
-                bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
-                envdata = bb_caches[mc].loadDataFull(fn, self.collections[mc].get_file_appends(fn))
+                layername = self.collections[mc].calc_bbfile_priority(fn)[2]
+                envdata = self.databuilder.parseRecipe(fn, self.collections[mc].get_file_appends(fn), layername)
             except Exception as e:
                 parselog.exception("Unable to read %s", fn)
                 raise
         else:
             if not mc in self.databuilder.mcdata:
-                bb.fatal('Not multiconfig named "%s" found' % mc)
+                bb.fatal('No multiconfig named "%s" found' % mc)
             envdata = self.databuilder.mcdata[mc]
             data.expandKeys(envdata)
             parse.ast.runAnonFuncs(envdata)
@@ -585,7 +565,7 @@ class BBCooker:
             data.emit_env(env, envdata, True)
             logger.plain(env.getvalue())
 
-        # emit the metadata which isnt valid shell
+        # emit the metadata which isn't valid shell
         for e in sorted(envdata.keys()):
             if envdata.getVarFlag(e, 'func', False) and envdata.getVarFlag(e, 'python', False):
                 logger.plain("\npython %s () {\n%s}\n", e, envdata.getVar(e, False))
@@ -594,7 +574,7 @@ class BBCooker:
             self.disableDataTracking()
             self.reset()
 
-    def buildTaskData(self, pkgs_to_build, task, abort, allowincomplete=False):
+    def buildTaskData(self, pkgs_to_build, task, halt, allowincomplete=False):
         """
         Prepare a runqueue and taskdata object for iteration over pkgs_to_build
         """
@@ -641,8 +621,8 @@ class BBCooker:
         localdata = {}
 
         for mc in self.multiconfigs:
-            taskdata[mc] = bb.taskdata.TaskData(abort, skiplist=self.skiplist, allowincomplete=allowincomplete)
-            localdata[mc] = data.createCopy(self.databuilder.mcdata[mc])
+            taskdata[mc] = bb.taskdata.TaskData(halt, skiplist=self.skiplist_by_mc[mc], allowincomplete=allowincomplete)
+            localdata[mc] = bb.data.createCopy(self.databuilder.mcdata[mc])
             bb.data.expandKeys(localdata[mc])
 
         current = 0
@@ -690,19 +670,18 @@ class BBCooker:
                     taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
                     mcdeps |= set(taskdata[mc].get_mcdepends())
                 new = False
-                for mc in self.multiconfigs:
-                    for k in mcdeps:
-                        if k in seen:
-                            continue
-                        l = k.split(':')
-                        depmc = l[2]
-                        if depmc not in self.multiconfigs:
-                            bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc))
-                        else:
-                            logger.debug("Adding providers for multiconfig dependency %s" % l[3])
-                            taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
-                            seen.add(k)
-                            new = True
+                for k in mcdeps:
+                    if k in seen:
+                        continue
+                    l = k.split(':')
+                    depmc = l[2]
+                    if depmc not in self.multiconfigs:
+                        bb.fatal("Multiconfig dependency %s depends on nonexistent multiconfig configuration named configuration %s" % (k,depmc))
+                    else:
+                        logger.debug("Adding providers for multiconfig dependency %s" % l[3])
+                        taskdata[depmc].add_provider(localdata[depmc], self.recipecaches[depmc], l[3])
+                        seen.add(k)
+                        new = True
 
         for mc in self.multiconfigs:
             taskdata[mc].add_unresolved(localdata[mc], self.recipecaches[mc])
@@ -710,14 +689,14 @@ class BBCooker:
         bb.event.fire(bb.event.TreeDataPreparationCompleted(len(fulltargetlist)), self.data)
         return taskdata, runlist
 
-    def prepareTreeData(self, pkgs_to_build, task):
+    def prepareTreeData(self, pkgs_to_build, task, halt=False):
         """
         Prepare a runqueue and taskdata object for iteration over pkgs_to_build
         """
 
-        # We set abort to False here to prevent unbuildable targets raising
+        # We set halt to False here to prevent unbuildable targets raising
         # an exception when we're just generating data
-        taskdata, runlist = self.buildTaskData(pkgs_to_build, task, False, allowincomplete=True)
+        taskdata, runlist = self.buildTaskData(pkgs_to_build, task, halt, allowincomplete=True)
 
         return runlist, taskdata
 
@@ -731,7 +710,7 @@ class BBCooker:
         if not task.startswith("do_"):
             task = "do_%s" % task
 
-        runlist, taskdata = self.prepareTreeData(pkgs_to_build, task)
+        runlist, taskdata = self.prepareTreeData(pkgs_to_build, task, halt=True)
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
         rq.rqdata.prepare()
         return self.buildDependTree(rq, taskdata)
@@ -792,7 +771,9 @@ class BBCooker:
             for dep in rq.rqdata.runtaskentries[tid].depends:
                 (depmc, depfn, _, deptaskfn) = bb.runqueue.split_tid_mcfn(dep)
                 deppn = self.recipecaches[depmc].pkg_fn[deptaskfn]
-                depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, bb.runqueue.taskname_from_tid(dep)))
+                if depmc:
+                    depmc = "mc:" + depmc + ":"
+                depend_tree["tdepends"][dotname].append("%s%s.%s" % (depmc, deppn, bb.runqueue.taskname_from_tid(dep)))
             if taskfn not in seen_fns:
                 seen_fns.append(taskfn)
                 packages = []
@@ -924,10 +905,11 @@ class BBCooker:
 
         depgraph = self.generateTaskDepTreeData(pkgs_to_build, task)
 
-        with open('pn-buildlist', 'w') as f:
-            for pn in depgraph["pn"]:
-                f.write(pn + "\n")
-        logger.info("PN build list saved to 'pn-buildlist'")
+        pns = depgraph["pn"].keys()
+        if pns:
+            with open('pn-buildlist', 'w') as f:
+                f.write("%s\n" % "\n".join(sorted(pns)))
+            logger.info("PN build list saved to 'pn-buildlist'")
 
         # Remove old format output files to ensure no confusion with stale data
         try:
@@ -961,7 +943,7 @@ class BBCooker:
         for mc in self.multiconfigs:
             # First get list of recipes, including skipped
             recipefns = list(self.recipecaches[mc].pkg_fn.keys())
-            recipefns.extend(self.skiplist.keys())
+            recipefns.extend(self.skiplist_by_mc[mc].keys())
 
             # Work out list of bbappends that have been applied
             applied_appends = []
@@ -980,13 +962,7 @@ class BBCooker:
                                                                         '\n  '.join(appends_without_recipes[mc])))
 
         if msgs:
-            msg = "\n".join(msgs)
-            warn_only = self.databuilder.mcdata[mc].getVar("BB_DANGLINGAPPENDS_WARNONLY", \
-                False) or "no"
-            if warn_only.lower() in ("1", "yes", "true"):
-                bb.warn(msg)
-            else:
-                bb.fatal(msg)
+            bb.fatal("\n".join(msgs))
 
     def handlePrefProviders(self):
 
@@ -1056,6 +1032,11 @@ class BBCooker:
         if matches:
             bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
 
+    def testCookerCommandEvent(self, filepattern):
+        # Dummy command used by OEQA selftest to test tinfoil without IO
+        matches = ["A", "B"]
+        bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
+
     def findProviders(self, mc=''):
         return bb.providers.findProviders(self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
 
@@ -1063,10 +1044,16 @@ class BBCooker:
         if pn in self.recipecaches[mc].providers:
             filenames = self.recipecaches[mc].providers[pn]
             eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.databuilder.mcdata[mc], self.recipecaches[mc])
-            filename = eligible[0]
+            if eligible is not None:
+                filename = eligible[0]
+            else:
+                filename = None
             return None, None, None, filename
         elif pn in self.recipecaches[mc].pkg_pn:
-            return bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+            (latest, latest_f, preferred_ver, preferred_file, required) = bb.providers.findBestProvider(pn, self.databuilder.mcdata[mc], self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
+            if required and preferred_file is None:
+                return None, None, None, None
+            return (latest, latest_f, preferred_ver, preferred_file)
         else:
             return None, None, None, None
 
@@ -1211,15 +1198,15 @@ class BBCooker:
                                     except bb.utils.VersionStringException as vse:
                                         bb.fatal('Error parsing LAYERRECOMMENDS_%s: %s' % (c, str(vse)))
                                     if not res:
-                                        parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec, layerver)
+                                        parselog.debug3("Layer '%s' recommends version %s of layer '%s', but version %s is currently enabled in your configuration. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec, layerver)
                                         continue
                                 else:
-                                    parselog.debug(3,"Layer '%s' recommends version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec)
+                                    parselog.debug3("Layer '%s' recommends version %s of layer '%s', which exists in your configuration but does not specify a version. Check that you are using the correct matching versions/branches of these two layers.", c, opstr, rec)
                                     continue
-                            parselog.debug(3,"Layer '%s' recommends layer '%s', so we are adding it", c, rec)
+                            parselog.debug3("Layer '%s' recommends layer '%s', so we are adding it", c, rec)
                             collection_depends[c].append(rec)
                         else:
-                            parselog.debug(3,"Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec)
+                            parselog.debug3("Layer '%s' recommends layer '%s', but this layer is not enabled in your configuration", c, rec)
 
             # Recursively work out collection priorities based on dependencies
             def calc_layer_priority(collection):
@@ -1231,7 +1218,7 @@ class BBCooker:
                         if depprio > max_depprio:
                             max_depprio = depprio
                     max_depprio += 1
-                    parselog.debug(1, "Calculated priority of layer %s as %d", collection, max_depprio)
+                    parselog.debug("Calculated priority of layer %s as %d", collection, max_depprio)
                     collection_priorities[collection] = max_depprio
 
             # Calculate all layer priorities using calc_layer_priority and store in bbfile_config_priorities
@@ -1243,7 +1230,7 @@ class BBCooker:
                     errors = True
                     continue
                 elif regex == "":
-                    parselog.debug(1, "BBFILE_PATTERN_%s is empty" % c)
+                    parselog.debug("BBFILE_PATTERN_%s is empty" % c)
                     cre = re.compile('^NULL$')
                     errors = False
                 else:
@@ -1290,8 +1277,8 @@ class BBCooker:
         if bf.startswith("/") or bf.startswith("../"):
             bf = os.path.abspath(bf)
 
-        self.collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
-        filelist, masked, searchdirs = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
+        collections = {mc: CookerCollectFiles(self.bbfile_config_priorities, mc)}
+        filelist, masked, searchdirs = collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
         try:
             os.stat(bf)
             bf = os.path.abspath(bf)
@@ -1355,9 +1342,10 @@ class BBCooker:
         self.buildSetVars()
         self.reset_mtime_caches()
 
-        bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.data_hash, self.caches_array)
+        bb_caches = bb.cache.MulticonfigCache(self.databuilder, self.databuilder.data_hash, self.caches_array)
 
-        infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn))
+        layername = self.collections[mc].calc_bbfile_priority(fn)[2]
+        infos = bb_caches[mc].parse(fn, self.collections[mc].get_file_appends(fn), layername)
         infos = dict(infos)
 
         fn = bb.cache.realfn2virtual(fn, cls, mc)
@@ -1383,14 +1371,16 @@ class BBCooker:
         self.recipecaches[mc].rundeps[fn] = defaultdict(list)
         self.recipecaches[mc].runrecs[fn] = defaultdict(list)
 
+        bb.parse.siggen.setup_datacache(self.recipecaches)
+
         # Invalidate task for target if force mode active
         if self.configuration.force:
             logger.verbose("Invalidate task %s, %s", task, fn)
-            bb.parse.siggen.invalidate_task(task, self.recipecaches[mc], fn)
+            bb.parse.siggen.invalidate_task(task, fn)
 
         # Setup taskdata structure
         taskdata = {}
-        taskdata[mc] = bb.taskdata.TaskData(self.configuration.abort)
+        taskdata[mc] = bb.taskdata.TaskData(self.configuration.halt)
         taskdata[mc].add_provider(self.databuilder.mcdata[mc], self.recipecaches[mc], item)
 
         if quietlog:
@@ -1400,21 +1390,24 @@ class BBCooker:
         buildname = self.databuilder.mcdata[mc].getVar("BUILDNAME")
         if fireevents:
             bb.event.fire(bb.event.BuildStarted(buildname, [item]), self.databuilder.mcdata[mc])
+            if self.eventlog:
+                self.eventlog[2].write_variables()
+            bb.event.enable_heartbeat()
 
         # Execute the runqueue
         runlist = [[mc, item, task, fn]]
 
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
 
-        def buildFileIdle(server, rq, abort):
+        def buildFileIdle(server, rq, halt):
 
             msg = None
             interrupted = 0
-            if abort or self.state == state.forceshutdown:
+            if halt or self.state == State.FORCE_SHUTDOWN:
                 rq.finish_runqueue(True)
                 msg = "Forced shutdown"
                 interrupted = 2
-            elif self.state == state.shutdown:
+            elif self.state == State.SHUTDOWN:
                 rq.finish_runqueue(False)
                 msg = "Stopped build"
                 interrupted = 1
@@ -1425,41 +1418,71 @@ class BBCooker:
                 failures += len(exc.args)
                 retval = False
             except SystemExit as exc:
-                self.command.finishAsyncCommand(str(exc))
                 if quietlog:
                     bb.runqueue.logger.setLevel(rqloglevel)
-                return False
+                return bb.server.process.idleFinish(str(exc))
 
             if not retval:
                 if fireevents:
                     bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item, failures, interrupted), self.databuilder.mcdata[mc])
-                self.command.finishAsyncCommand(msg)
+                    bb.event.disable_heartbeat()
                 # We trashed self.recipecaches above
-                self.parsecache_valid = False
+                self._parsecache_set(False)
                 self.configuration.limited_deps = False
                 bb.parse.siggen.reset(self.data)
                 if quietlog:
                     bb.runqueue.logger.setLevel(rqloglevel)
-                return False
+                return bb.server.process.idleFinish(msg)
             if retval is True:
                 return True
             return retval
 
         self.idleCallBackRegister(buildFileIdle, rq)
 
+    def getTaskSignatures(self, target, tasks):
+        sig = []
+        getAllTaskSignatures = False
+
+        if not tasks:
+            tasks = ["do_build"]
+            getAllTaskSignatures = True
+
+        for task in tasks:
+            taskdata, runlist = self.buildTaskData(target, task, self.configuration.halt)
+            rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
+            rq.rqdata.prepare()
+
+            for l in runlist:
+                mc, pn, taskname, fn = l
+
+                taskdep = rq.rqdata.dataCaches[mc].task_deps[fn]
+                for t in taskdep['tasks']:
+                    if t in taskdep['nostamp'] or "setscene" in t:
+                        continue
+                    tid = bb.runqueue.build_tid(mc, fn, t)
+
+                    if t in task or getAllTaskSignatures:
+                        try:
+                            sig.append([pn, t, rq.rqdata.get_task_unihash(tid)])
+                        except KeyError:
+                            sig.append(self.getTaskSignatures(target, [t])[0])
+
+        return sig
+
     def buildTargets(self, targets, task):
         """
         Attempt to build the targets specified
         """
 
-        def buildTargetsIdle(server, rq, abort):
+        def buildTargetsIdle(server, rq, halt):
             msg = None
             interrupted = 0
-            if abort or self.state == state.forceshutdown:
+            if halt or self.state == State.FORCE_SHUTDOWN:
+                bb.event._should_exit.set()
                 rq.finish_runqueue(True)
                 msg = "Forced shutdown"
                 interrupted = 2
-            elif self.state == state.shutdown:
+            elif self.state == State.SHUTDOWN:
                 rq.finish_runqueue(False)
                 msg = "Stopped build"
                 interrupted = 1
@@ -1470,16 +1493,16 @@ class BBCooker:
                 failures += len(exc.args)
                 retval = False
             except SystemExit as exc:
-                self.command.finishAsyncCommand(str(exc))
-                return False
+                return bb.server.process.idleFinish(str(exc))
 
             if not retval:
                 try:
                     for mc in self.multiconfigs:
                         bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets, failures, interrupted), self.databuilder.mcdata[mc])
                 finally:
-                    self.command.finishAsyncCommand(msg)
-                return False
+                    bb.event.disable_heartbeat()
+                return bb.server.process.idleFinish(msg)
+
             if retval is True:
                 return True
             return retval
@@ -1498,7 +1521,7 @@ class BBCooker:
 
         bb.event.fire(bb.event.BuildInit(packages), self.data)
 
-        taskdata, runlist = self.buildTaskData(targets, task, self.configuration.abort)
+        taskdata, runlist = self.buildTaskData(targets, task, self.configuration.halt)
 
         buildname = self.data.getVar("BUILDNAME", False)
 
@@ -1511,6 +1534,9 @@ class BBCooker:
 
         for mc in self.multiconfigs:
             bb.event.fire(bb.event.BuildStarted(buildname, ntargets), self.databuilder.mcdata[mc])
+        if self.eventlog:
+            self.eventlog[2].write_variables()
+        bb.event.enable_heartbeat()
 
         rq = bb.runqueue.RunQueue(self, self.data, self.recipecaches, taskdata, runlist)
         if 'universe' in targets:
@@ -1520,7 +1546,13 @@ class BBCooker:
 
 
     def getAllKeysWithFlags(self, flaglist):
+        def dummy_autorev(d):
+            return
+
         dump = {}
+        # Horrible but for now we need to avoid any sideeffects of autorev being called
+        saved = bb.fetch2.get_autorev
+        bb.fetch2.get_autorev = dummy_autorev
         for k in self.data.keys():
             try:
                 expand = True
@@ -1540,20 +1572,14 @@ class BBCooker:
                             dump[k][d] = None
             except Exception as e:
                 print(e)
+        bb.fetch2.get_autorev = saved
         return dump
 
 
     def updateCacheSync(self):
-        if self.state == state.running:
+        if self.state == State.RUNNING:
             return
 
-        # reload files for which we got notifications
-        for p in self.inotify_modified_files:
-            bb.parse.update_cache(p)
-            if p in bb.parse.BBHandler.cached_statements:
-                del bb.parse.BBHandler.cached_statements[p]
-        self.inotify_modified_files = []
-
         if not self.baseconfig_valid:
             logger.debug("Reloading base configuration data")
             self.initConfigurationData()
@@ -1561,19 +1587,22 @@ class BBCooker:
 
     # This is called for all async commands when self.state != running
     def updateCache(self):
-        if self.state == state.running:
+        if self.state == State.RUNNING:
             return
 
-        if self.state in (state.shutdown, state.forceshutdown, state.error):
+        if self.state in (State.SHUTDOWN, State.FORCE_SHUTDOWN, State.ERROR):
             if hasattr(self.parser, 'shutdown'):
-                self.parser.shutdown(clean=False, force = True)
+                self.parser.shutdown(clean=False)
                 self.parser.final_cleanup()
             raise bb.BBHandledException()
 
-        if self.state != state.parsing:
+        if self.state != State.PARSING:
             self.updateCacheSync()
 
-        if self.state != state.parsing and not self.parsecache_valid:
+        if self.state != State.PARSING and not self.parsecache_valid:
+            bb.server.process.serverlog("Parsing started")
+            self.parsewatched = {}
+
             bb.parse.siggen.reset(self.data)
             self.parseConfiguration ()
             if CookerFeatures.SEND_SANITYEVENTS in self.featureset:
@@ -1587,37 +1616,35 @@ class BBCooker:
                 for dep in self.configuration.extra_assume_provided:
                     self.recipecaches[mc].ignored_dependencies.add(dep)
 
-            self.collections = {}
-
             mcfilelist = {}
             total_masked = 0
             searchdirs = set()
             for mc in self.multiconfigs:
-                self.collections[mc] = CookerCollectFiles(self.bbfile_config_priorities, mc)
                 (filelist, masked, search) = self.collections[mc].collect_bbfiles(self.databuilder.mcdata[mc], self.databuilder.mcdata[mc])
 
                 mcfilelist[mc] = filelist
                 total_masked += masked
                 searchdirs |= set(search)
 
-            # Add inotify watches for directories searched for bb/bbappend files
+            # Add mtimes for directories searched for bb/bbappend files
             for dirent in searchdirs:
-                self.add_filewatch([[dirent]], dirs=True)
+                self.add_filewatch([(dirent, bb.parse.cached_mtime_noerror(dirent))])
 
             self.parser = CookerParser(self, mcfilelist, total_masked)
-            self.parsecache_valid = True
+            self._parsecache_set(True)
 
-        self.state = state.parsing
+        self.state = State.PARSING
 
         if not self.parser.parse_next():
-            collectlog.debug(1, "parsing complete")
+            bb.server.process.serverlog("Parsing completed")
+            collectlog.debug("parsing complete")
             if self.parser.error:
                 raise bb.BBHandledException()
             self.show_appends_with_no_recipes()
             self.handlePrefProviders()
             for mc in self.multiconfigs:
                 self.recipecaches[mc].bbfile_priority = self.collections[mc].collection_priorities(self.recipecaches[mc].pkg_fn, self.parser.mcfilelist[mc], self.data)
-            self.state = state.running
+            self.state = State.RUNNING
 
             # Send an event listing all stamps reachable after parsing
             # which the metadata may use to clean up stale data
@@ -1633,7 +1660,7 @@ class BBCooker:
         # Return a copy, don't modify the original
         pkgs_to_build = pkgs_to_build[:]
 
-        if len(pkgs_to_build) == 0:
+        if not pkgs_to_build:
             raise NothingToBuild
 
         ignore = (self.data.getVar("ASSUME_PROVIDED") or "").split()
@@ -1655,7 +1682,7 @@ class BBCooker:
 
         if 'universe' in pkgs_to_build:
             parselog.verbnote("The \"universe\" target is only intended for testing and may produce errors.")
-            parselog.debug(1, "collating packages for \"universe\"")
+            parselog.debug("collating packages for \"universe\"")
             pkgs_to_build.remove('universe')
             for mc in self.multiconfigs:
                 for t in self.recipecaches[mc].universe_target:
@@ -1680,26 +1707,36 @@ class BBCooker:
     def post_serve(self):
         self.shutdown(force=True)
         prserv.serv.auto_shutdown()
+        if hasattr(bb.parse, "siggen"):
+            bb.parse.siggen.exit()
         if self.hashserv:
             self.hashserv.process.terminate()
             self.hashserv.process.join()
         if hasattr(self, "data"):
             bb.event.fire(CookerExit(), self.data)
 
-    def shutdown(self, force = False):
+    def shutdown(self, force=False):
         if force:
-            self.state = state.forceshutdown
+            self.state = State.FORCE_SHUTDOWN
+            bb.event._should_exit.set()
         else:
-            self.state = state.shutdown
+            self.state = State.SHUTDOWN
 
         if self.parser:
-            self.parser.shutdown(clean=not force, force=force)
+            self.parser.shutdown(clean=False)
             self.parser.final_cleanup()
 
     def finishcommand(self):
-        self.state = state.initial
+        if hasattr(self.parser, 'shutdown'):
+            self.parser.shutdown(clean=False)
+            self.parser.final_cleanup()
+        self.state = State.INITIAL
+        bb.event._should_exit.clear()
 
     def reset(self):
+        if hasattr(bb.parse, "siggen"):
+            bb.parse.siggen.exit()
+        self.finishcommand()
         self.initConfigurationData()
         self.handlePRServ()
 
@@ -1711,9 +1748,9 @@ class BBCooker:
         if hasattr(self, "data"):
            self.databuilder.reset()
            self.data = self.databuilder.data
-        self.parsecache_valid = False
-        self.baseconfig_valid = False
-
+        # In theory tinfoil could have modified the base data before parsing,
+        # ideally need to track if anything did modify the datastore
+        self._parsecache_set(False)
 
 class CookerExit(bb.event.Event):
     """
@@ -1728,16 +1765,16 @@ class CookerCollectFiles(object):
     def __init__(self, priorities, mc=''):
         self.mc = mc
         self.bbappends = []
-        # Priorities is a list of tupples, with the second element as the pattern.
+        # Priorities is a list of tuples, with the second element as the pattern.
         # We need to sort the list with the longest pattern first, and so on to
         # the shortest.  This allows nested layers to be properly evaluated.
         self.bbfile_config_priorities = sorted(priorities, key=lambda tup: tup[1], reverse=True)
 
     def calc_bbfile_priority(self, filename):
-        for _, _, regex, pri in self.bbfile_config_priorities:
+        for layername, _, regex, pri in self.bbfile_config_priorities:
             if regex.match(filename):
-                return pri, regex
-        return 0, None
+                return pri, regex, layername
+        return 0, None, None
 
     def get_bbfiles(self):
         """Get list of default .bb files by reading out the current directory"""
@@ -1756,7 +1793,7 @@ class CookerCollectFiles(object):
             for ignored in ('SCCS', 'CVS', '.svn'):
                 if ignored in dirs:
                     dirs.remove(ignored)
-            found += [os.path.join(dir, f) for f in files if (f.endswith(['.bb', '.bbappend']))]
+            found += [os.path.join(dir, f) for f in files if (f.endswith(('.bb', '.bbappend')))]
 
         return found
 
@@ -1764,7 +1801,7 @@ class CookerCollectFiles(object):
         """Collect all available .bb build files"""
         masked = 0
 
-        collectlog.debug(1, "collecting .bb files")
+        collectlog.debug("collecting .bb files")
 
         files = (config.getVar( "BBFILES") or "").split()
 
@@ -1772,16 +1809,16 @@ class CookerCollectFiles(object):
         files.sort( key=lambda fileitem: self.calc_bbfile_priority(fileitem)[0] )
         config.setVar("BBFILES_PRIORITIZED", " ".join(files))
 
-        if not len(files):
+        if not files:
             files = self.get_bbfiles()
 
-        if not len(files):
+        if not files:
             collectlog.error("no recipe files to build, check your BBPATH and BBFILES?")
             bb.event.fire(CookerExit(), eventdata)
 
-        # We need to track where we look so that we can add inotify watches. There
-        # is no nice way to do this, this is horrid. We intercept the os.listdir()
-        # (or os.scandir() for python 3.6+) calls while we run glob().
+        # We need to track where we look so that we can know when the cache is invalid. There
+        # is no nice way to do this, this is horrid. We intercept the os.listdir() and os.scandir()
+        # calls while we run glob().
         origlistdir = os.listdir
         if hasattr(os, 'scandir'):
             origscandir = os.scandir
@@ -1835,7 +1872,7 @@ class CookerCollectFiles(object):
                 try:
                     re.compile(mask)
                     bbmasks.append(mask)
-                except sre_constants.error:
+                except re.error:
                     collectlog.critical("BBMASK contains an invalid regular expression, ignoring: %s" % mask)
 
             # Then validate the combined regular expressions. This should never
@@ -1843,7 +1880,7 @@ class CookerCollectFiles(object):
             bbmask = "|".join(bbmasks)
             try:
                 bbmask_compiled = re.compile(bbmask)
-            except sre_constants.error:
+            except re.error:
                 collectlog.critical("BBMASK is not a valid regular expression, ignoring: %s" % bbmask)
                 bbmask = None
 
@@ -1851,7 +1888,7 @@ class CookerCollectFiles(object):
         bbappend = []
         for f in newfiles:
             if bbmask and bbmask_compiled.search(f):
-                collectlog.debug(1, "skipping masked file %s", f)
+                collectlog.debug("skipping masked file %s", f)
                 masked += 1
                 continue
             if f.endswith('.bb'):
@@ -1859,7 +1896,7 @@ class CookerCollectFiles(object):
             elif f.endswith('.bbappend'):
                 bbappend.append(f)
             else:
-                collectlog.debug(1, "skipping %s: unknown file extension", f)
+                collectlog.debug("skipping %s: unknown file extension", f)
 
         # Build a list of .bbappend files for each .bb file
         for f in bbappend:
@@ -1910,7 +1947,7 @@ class CookerCollectFiles(object):
         # Calculate priorities for each file
         for p in pkgfns:
             realfn, cls, mc = bb.cache.virtualfn2realfn(p)
-            priorities[p], regex = self.calc_bbfile_priority(realfn)
+            priorities[p], regex, _ = self.calc_bbfile_priority(realfn)
             if regex in unmatched_regex:
                 matched_regex.add(regex)
                 unmatched_regex.remove(regex)
@@ -1961,15 +1998,30 @@ class ParsingFailure(Exception):
         Exception.__init__(self, realexception, recipe)
 
 class Parser(multiprocessing.Process):
-    def __init__(self, jobs, results, quit, init, profile):
+    def __init__(self, jobs, results, quit, profile):
         self.jobs = jobs
         self.results = results
         self.quit = quit
-        self.init = init
         multiprocessing.Process.__init__(self)
         self.context = bb.utils.get_context().copy()
         self.handlers = bb.event.get_class_handlers().copy()
         self.profile = profile
+        self.queue_signals = False
+        self.signal_received = []
+        self.signal_threadlock = threading.Lock()
+
+    def catch_sig(self, signum, frame):
+        if self.queue_signals:
+            self.signal_received.append(signum)
+        else:
+            self.handle_sig(signum, frame)
+
+    def handle_sig(self, signum, frame):
+        if signum == signal.SIGTERM:
+            signal.signal(signal.SIGTERM, signal.SIG_DFL)
+            os.kill(os.getpid(), signal.SIGTERM)
+        elif signum == signal.SIGINT:
+            signal.default_int_handler(signum, frame)
 
     def run(self):
 
@@ -1989,38 +2041,50 @@ class Parser(multiprocessing.Process):
             prof.dump_stats(logfile)
 
     def realrun(self):
-        if self.init:
-            self.init()
+        # Signal handling here is hard. We must not terminate any process or thread holding the write
+        # lock for the event stream as it will not be released, ever, and things will hang.
+        # Python handles signals in the main thread/process but they can be raised from any thread and
+        # we want to defer processing of any SIGTERM/SIGINT signal until we're outside the critical section
+        # and don't hold the lock (see server/process.py). We therefore always catch the signals (so any
+        # new thread should also do so) and we defer handling but we handle with the local thread lock
+        # held (a threading lock, not a multiprocessing one) so that no other thread in the process
+        # can be in the critical section.
+        signal.signal(signal.SIGTERM, self.catch_sig)
+        signal.signal(signal.SIGHUP, signal.SIG_DFL)
+        signal.signal(signal.SIGINT, self.catch_sig)
+        bb.utils.set_process_name(multiprocessing.current_process().name)
+        multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
+        multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
 
         pending = []
-        while True:
-            try:
-                self.quit.get_nowait()
-            except queue.Empty:
-                pass
-            else:
-                self.results.close()
-                self.results.join_thread()
-                break
+        havejobs = True
+        try:
+            while havejobs or pending:
+                if self.quit.is_set():
+                    break
 
-            if pending:
-                result = pending.pop()
-            else:
+                job = None
                 try:
                     job = self.jobs.pop()
                 except IndexError:
-                    self.results.close()
-                    self.results.join_thread()
-                    break
-                result = self.parse(*job)
-                # Clear the siggen cache after parsing to control memory usage, its huge
-                bb.parse.siggen.postparsing_clean_cache()
-            try:
-                self.results.put(result, timeout=0.25)
-            except queue.Full:
-                pending.append(result)
+                    havejobs = False
+                if job:
+                    result = self.parse(*job)
+                    # Clear the siggen cache after parsing to control memory usage, its huge
+                    bb.parse.siggen.postparsing_clean_cache()
+                    pending.append(result)
+
+                if pending:
+                    try:
+                        result = pending.pop()
+                        self.results.put(result, timeout=0.05)
+                    except queue.Full:
+                        pending.append(result)
+        finally:
+            self.results.close()
+            self.results.join_thread()
 
-    def parse(self, mc, cache, filename, appends):
+    def parse(self, mc, cache, filename, appends, layername):
         try:
             origfilter = bb.event.LogHandler.filter
             # Record the filename we're parsing into any events generated
@@ -2034,17 +2098,16 @@ class Parser(multiprocessing.Process):
             bb.event.set_class_handlers(self.handlers.copy())
             bb.event.LogHandler.filter = parse_filter
 
-            return True, mc, cache.parse(filename, appends)
+            return True, mc, cache.parse(filename, appends, layername)
         except Exception as exc:
             tb = sys.exc_info()[2]
             exc.recipe = filename
-            exc.traceback = list(bb.exceptions.extract_traceback(tb, context=3))
-            return True, exc
+            return True, None, exc
         # Need to turn BaseExceptions into Exceptions here so we gracefully shutdown
         # and for example a worker thread doesn't just exit on its own in response to
         # a SystemExit event for example.
         except BaseException as exc:
-            return True, ParsingFailure(exc, filename)
+            return True, None, ParsingFailure(exc, filename)
         finally:
             bb.event.LogHandler.filter = origfilter
 
@@ -2053,7 +2116,7 @@ class CookerParser(object):
         self.mcfilelist = mcfilelist
         self.cooker = cooker
         self.cfgdata = cooker.data
-        self.cfghash = cooker.data_hash
+        self.cfghash = cooker.databuilder.data_hash
         self.cfgbuilder = cooker.databuilder
 
         # Accounting statistics
@@ -2074,10 +2137,11 @@ class CookerParser(object):
         for mc in self.cooker.multiconfigs:
             for filename in self.mcfilelist[mc]:
                 appends = self.cooker.collections[mc].get_file_appends(filename)
+                layername = self.cooker.collections[mc].calc_bbfile_priority(filename)[2]
                 if not self.bb_caches[mc].cacheValid(filename, appends):
-                    self.willparse.add((mc, self.bb_caches[mc], filename, appends))
+                    self.willparse.add((mc, self.bb_caches[mc], filename, appends, layername))
                 else:
-                    self.fromcache.add((mc, self.bb_caches[mc], filename, appends))
+                    self.fromcache.add((mc, self.bb_caches[mc], filename, appends, layername))
 
         self.total = len(self.fromcache) + len(self.willparse)
         self.toparse = len(self.willparse)
@@ -2086,6 +2150,7 @@ class CookerParser(object):
         self.num_processes = min(int(self.cfgdata.getVar("BB_NUMBER_PARSE_THREADS") or
                                  multiprocessing.cpu_count()), self.toparse)
 
+        bb.cache.SiggenRecipeInfo.reset()
         self.start()
         self.haveshutdown = False
         self.syncthread = None
@@ -2095,15 +2160,8 @@ class CookerParser(object):
         self.processes = []
         if self.toparse:
             bb.event.fire(bb.event.ParseStarted(self.toparse), self.cfgdata)
-            def init():
-                signal.signal(signal.SIGTERM, signal.SIG_DFL)
-                signal.signal(signal.SIGHUP, signal.SIG_DFL)
-                signal.signal(signal.SIGINT, signal.SIG_IGN)
-                bb.utils.set_process_name(multiprocessing.current_process().name)
-                multiprocessing.util.Finalize(None, bb.codeparser.parser_cache_save, exitpriority=1)
-                multiprocessing.util.Finalize(None, bb.fetch.fetcher_parse_save, exitpriority=1)
-
-            self.parser_quit = multiprocessing.Queue(maxsize=self.num_processes)
+
+            self.parser_quit = multiprocessing.Event()
             self.result_queue = multiprocessing.Queue()
 
             def chunkify(lst,n):
@@ -2111,14 +2169,14 @@ class CookerParser(object):
             self.jobs = chunkify(list(self.willparse), self.num_processes)
 
             for i in range(0, self.num_processes):
-                parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, init, self.cooker.configuration.profile)
+                parser = Parser(self.jobs[i], self.result_queue, self.parser_quit, self.cooker.configuration.profile)
                 parser.start()
                 self.process_names.append(parser.name)
                 self.processes.append(parser)
 
             self.results = itertools.chain(self.results, self.parse_generator())
 
-    def shutdown(self, clean=True, force=False):
+    def shutdown(self, clean=True, eventmsg="Parsing halted due to errors"):
         if not self.toparse:
             return
         if self.haveshutdown:
@@ -2132,9 +2190,9 @@ class CookerParser(object):
                                             self.total)
 
             bb.event.fire(event, self.cfgdata)
-
-        for process in self.processes:
-            self.parser_quit.put(None)
+        else:
+            bb.event.fire(bb.event.ParseError(eventmsg), self.cfgdata)
+            bb.error("Parsing halted due to errors, see error messages above")
 
         # Cleanup the queue before call process.join(), otherwise there might be
         # deadlocks.
@@ -2144,106 +2202,152 @@ class CookerParser(object):
             except queue.Empty:
                 break
 
-        for process in self.processes:
-            if force:
-                process.join(.1)
-                process.terminate()
-            else:
-                process.join()
-
-        self.parser_quit.close()
-        # Allow data left in the cancel queue to be discarded
-        self.parser_quit.cancel_join_thread()
-
         def sync_caches():
             for c in self.bb_caches.values():
+                bb.cache.SiggenRecipeInfo.reset()
                 c.sync()
 
-        sync = threading.Thread(target=sync_caches, name="SyncThread")
-        self.syncthread = sync
-        sync.start()
+        self.syncthread = threading.Thread(target=sync_caches, name="SyncThread")
+        self.syncthread.start()
+
+        self.parser_quit.set()
+
+        for process in self.processes:
+            process.join(0.5)
+
+        for process in self.processes:
+            if process.exitcode is None:
+                os.kill(process.pid, signal.SIGINT)
+
+        for process in self.processes:
+            process.join(0.5)
+
+        for process in self.processes:
+            if process.exitcode is None:
+                process.terminate()
+
+        for process in self.processes:
+            process.join()
+            # clean up zombies
+            process.close()
+
+        bb.codeparser.parser_cache_save()
         bb.codeparser.parser_cache_savemerge()
+        bb.cache.SiggenRecipeInfo.reset()
         bb.fetch.fetcher_parse_done()
         if self.cooker.configuration.profile:
             profiles = []
             for i in self.process_names:
                 logfile = "profile-parse-%s.log" % i
-                if os.path.exists(logfile):
+                if os.path.exists(logfile) and os.path.getsize(logfile):
                     profiles.append(logfile)
 
-            pout = "profile-parse.log.processed"
-            bb.utils.process_profilelog(profiles, pout = pout)
-            print("Processed parsing statistics saved to %s" % (pout))
+            if profiles:
+                pout = "profile-parse.log.processed"
+                bb.utils.process_profilelog(profiles, pout = pout)
+                print("Processed parsing statistics saved to %s" % (pout))
 
     def final_cleanup(self):
         if self.syncthread:
             self.syncthread.join()
 
     def load_cached(self):
-        for mc, cache, filename, appends in self.fromcache:
-            cached, infos = cache.load(filename, appends)
-            yield not cached, mc, infos
+        for mc, cache, filename, appends, layername in self.fromcache:
+            infos = cache.loadCached(filename, appends)
+            yield False, mc, infos
 
     def parse_generator(self):
-        while True:
+        empty = False
+        while self.processes or not empty:
+            for process in self.processes.copy():
+                if not process.is_alive():
+                    process.join()
+                    self.processes.remove(process)
+
             if self.parsed >= self.toparse:
                 break
 
             try:
                 result = self.result_queue.get(timeout=0.25)
             except queue.Empty:
-                pass
+                empty = True
+                yield None, None, None
             else:
-                value = result[1]
-                if isinstance(value, BaseException):
-                    raise value
-                else:
-                    yield result
+                empty = False
+                yield result
+
+        if not (self.parsed >= self.toparse):
+            raise bb.parse.ParseError("Not all recipes parsed, parser thread killed/died? Exiting.", None)
+
 
     def parse_next(self):
         result = []
         parsed = None
         try:
             parsed, mc, result = next(self.results)
+            if isinstance(result, BaseException):
+                # Turn exceptions back into exceptions
+                raise result
+            if parsed is None:
+                # Timeout, loop back through the main loop
+                return True
+
         except StopIteration:
             self.shutdown()
             return False
         except bb.BBHandledException as exc:
             self.error += 1
-            logger.error('Failed to parse recipe: %s' % exc.recipe)
-            self.shutdown(clean=False, force=True)
+            logger.debug('Failed to parse recipe: %s' % exc.recipe)
+            self.shutdown(clean=False)
             return False
         except ParsingFailure as exc:
             self.error += 1
-            logger.error('Unable to parse %s: %s' %
-                     (exc.recipe, bb.exceptions.to_string(exc.realexception)))
-            self.shutdown(clean=False, force=True)
+
+            exc_desc = str(exc)
+            if isinstance(exc, SystemExit) and not isinstance(exc.code, str):
+                exc_desc = 'Exited with "%d"' % exc.code
+
+            logger.error('Unable to parse %s: %s' % (exc.recipe, exc_desc))
+            self.shutdown(clean=False)
             return False
         except bb.parse.ParseError as exc:
             self.error += 1
             logger.error(str(exc))
-            self.shutdown(clean=False, force=True)
+            self.shutdown(clean=False, eventmsg=str(exc))
             return False
         except bb.data_smart.ExpansionError as exc:
+            def skip_frames(f, fn_prefix):
+                while f and f.tb_frame.f_code.co_filename.startswith(fn_prefix):
+                    f = f.tb_next
+                return f
+
             self.error += 1
             bbdir = os.path.dirname(__file__) + os.sep
-            etype, value, _ = sys.exc_info()
-            tb = list(itertools.dropwhile(lambda e: e.filename.startswith(bbdir), exc.traceback))
+            etype, value, tb = sys.exc_info()
+
+            # Remove any frames where the code comes from bitbake. This
+            # prevents deep (and pretty useless) backtraces for expansion error
+            tb = skip_frames(tb, bbdir)
+            cur = tb
+            while cur:
+                cur.tb_next = skip_frames(cur.tb_next, bbdir)
+                cur = cur.tb_next
+
             logger.error('ExpansionError during parsing %s', value.recipe,
                          exc_info=(etype, value, tb))
-            self.shutdown(clean=False, force=True)
+            self.shutdown(clean=False)
             return False
         except Exception as exc:
             self.error += 1
-            etype, value, tb = sys.exc_info()
+            _, value, _ = sys.exc_info()
             if hasattr(value, "recipe"):
                 logger.error('Unable to parse %s' % value.recipe,
-                            exc_info=(etype, value, exc.traceback))
+                            exc_info=sys.exc_info())
             else:
                 # Most likely, an exception occurred during raising an exception
                 import traceback
                 logger.error('Exception during parse: %s' % traceback.format_exc())
-            self.shutdown(clean=False, force=True)
+            self.shutdown(clean=False)
             return False
 
         self.current += 1
@@ -2259,17 +2363,19 @@ class CookerParser(object):
         for virtualfn, info_array in result:
             if info_array[0].skipped:
                 self.skipped += 1
-                self.cooker.skiplist[virtualfn] = SkippedPackage(info_array[0])
+                self.cooker.skiplist_by_mc[mc][virtualfn] = SkippedPackage(info_array[0])
             self.bb_caches[mc].add_info(virtualfn, info_array, self.cooker.recipecaches[mc],
                                         parsed=parsed, watcher = self.cooker.add_filewatch)
         return True
 
     def reparse(self, filename):
+        bb.cache.SiggenRecipeInfo.reset()
         to_reparse = set()
         for mc in self.cooker.multiconfigs:
-            to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename)))
+            layername = self.cooker.collections[mc].calc_bbfile_priority(filename)[2]
+            to_reparse.add((mc, filename, self.cooker.collections[mc].get_file_appends(filename), layername))
 
-        for mc, filename, appends in to_reparse:
-            infos = self.bb_caches[mc].parse(filename, appends)
+        for mc, filename, appends, layername in to_reparse:
+            infos = self.bb_caches[mc].parse(filename, appends, layername)
             for vfn, info_array in infos:
                 self.cooker.recipecaches[mc].add_from_recipeinfo(vfn, info_array)
diff --git a/bitbake/lib/bb/cookerdata.py b/bitbake/lib/bb/cookerdata.py
index 1c1e008c6b..65c153a5bb 100644
--- a/bitbake/lib/bb/cookerdata.py
+++ b/bitbake/lib/bb/cookerdata.py
@@ -1,3 +1,4 @@
+
 #
 # Copyright (C) 2003, 2004  Chris Larson
 # Copyright (C) 2003, 2004  Phil Blundell
@@ -57,7 +58,7 @@ class ConfigParameters(object):
 
     def updateToServer(self, server, environment):
         options = {}
-        for o in ["abort", "force", "invalidate_stamp",
+        for o in ["halt", "force", "invalidate_stamp",
                   "dry_run", "dump_signatures",
                   "extra_assume_provided", "profile",
                   "prefile", "postfile", "server_timeout",
@@ -86,7 +87,7 @@ class ConfigParameters(object):
                 action['msg'] = "Only one target can be used with the --environment option."
             elif self.options.buildfile and len(self.options.pkgs_to_build) > 0:
                 action['msg'] = "No target should be used with the --environment and --buildfile options."
-            elif len(self.options.pkgs_to_build) > 0:
+            elif self.options.pkgs_to_build:
                 action['action'] = ["showEnvironmentTarget", self.options.pkgs_to_build]
             else:
                 action['action'] = ["showEnvironment", self.options.buildfile]
@@ -124,7 +125,7 @@ class CookerConfiguration(object):
         self.prefile = []
         self.postfile = []
         self.cmd = None
-        self.abort = True
+        self.halt = True
         self.force = False
         self.profile = False
         self.nosetscene = False
@@ -160,12 +161,7 @@ def catch_parse_error(func):
     def wrapped(fn, *args):
         try:
             return func(fn, *args)
-        except IOError as exc:
-            import traceback
-            parselog.critical(traceback.format_exc())
-            parselog.critical("Unable to parse %s: %s" % (fn, exc))
-            raise bb.BBHandledException()
-        except bb.data_smart.ExpansionError as exc:
+        except Exception as exc:
             import traceback
 
             bbdir = os.path.dirname(__file__) + os.sep
@@ -177,14 +173,11 @@ def catch_parse_error(func):
                     break
             parselog.critical("Unable to parse %s" % fn, exc_info=(exc_class, exc, tb))
             raise bb.BBHandledException()
-        except bb.parse.ParseError as exc:
-            parselog.critical(str(exc))
-            raise bb.BBHandledException()
     return wrapped
 
 @catch_parse_error
 def parse_config_file(fn, data, include=True):
-    return bb.parse.handle(fn, data, include)
+    return bb.parse.handle(fn, data, include, baseconfig=True)
 
 @catch_parse_error
 def _inherit(bbclass, data):
@@ -210,7 +203,7 @@ def findConfigFile(configfile, data):
 
 #
 # We search for a conf/bblayers.conf under an entry in BBPATH or in cwd working
-# up to /. If that fails, we search for a conf/bitbake.conf in BBPATH.
+# up to /. If that fails, bitbake would fall back to cwd.
 #
 
 def findTopdir():
@@ -223,11 +216,8 @@ def findTopdir():
     layerconf = findConfigFile("bblayers.conf", d)
     if layerconf:
         return os.path.dirname(os.path.dirname(layerconf))
-    if bbpath:
-        bitbakeconf = bb.utils.which(bbpath, "conf/bitbake.conf")
-        if bitbakeconf:
-            return os.path.dirname(os.path.dirname(bitbakeconf))
-    return None
+
+    return os.path.abspath(os.getcwd())
 
 class CookerDataBuilder(object):
 
@@ -250,10 +240,14 @@ class CookerDataBuilder(object):
         self.savedenv = bb.data.init()
         for k in cookercfg.env:
             self.savedenv.setVar(k, cookercfg.env[k])
+            if k in bb.data_smart.bitbake_renamed_vars:
+                bb.error('Shell environment variable %s has been renamed to %s' % (k, bb.data_smart.bitbake_renamed_vars[k]))
+                bb.fatal("Exiting to allow enviroment variables to be corrected")
 
         filtered_keys = bb.utils.approved_variables()
         bb.data.inheritFromOS(self.basedata, self.savedenv, filtered_keys)
         self.basedata.setVar("BB_ORIGENV", self.savedenv)
+        self.basedata.setVar("__bbclasstype", "global")
 
         if worker:
             self.basedata.setVar("BB_WORKERCONTEXT", "1")
@@ -261,15 +255,22 @@ class CookerDataBuilder(object):
         self.data = self.basedata
         self.mcdata = {}
 
-    def parseBaseConfiguration(self):
+    def calc_datastore_hashes(self):
         data_hash = hashlib.sha256()
+        data_hash.update(self.data.get_hash().encode('utf-8'))
+        multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
+        for config in multiconfig:
+            data_hash.update(self.mcdata[config].get_hash().encode('utf-8'))
+        self.data_hash = data_hash.hexdigest()
+
+    def parseBaseConfiguration(self, worker=False):
+        mcdata = {}
         try:
             self.data = self.parseConfigurationFiles(self.prefiles, self.postfiles)
 
-            if self.data.getVar("BB_WORKERCONTEXT", False) is None:
-                bb.fetch.fetcher_init(self.data)
+            servercontext = self.data.getVar("BB_WORKERCONTEXT", False) is None and not worker
+            bb.fetch.fetcher_init(self.data, servercontext)
             bb.parse.init_parser(self.data)
-            bb.codeparser.parser_cache_init(self.data)
 
             bb.event.fire(bb.event.ConfigParsed(), self.data)
 
@@ -286,44 +287,66 @@ class CookerDataBuilder(object):
                 bb.event.fire(bb.event.ConfigParsed(), self.data)
 
             bb.parse.init_parser(self.data)
-            data_hash.update(self.data.get_hash().encode('utf-8'))
-            self.mcdata[''] = self.data
+            mcdata[''] = self.data
 
             multiconfig = (self.data.getVar("BBMULTICONFIG") or "").split()
             for config in multiconfig:
-                mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
-                bb.event.fire(bb.event.ConfigParsed(), mcdata)
-                self.mcdata[config] = mcdata
-                data_hash.update(mcdata.get_hash().encode('utf-8'))
+                if config[0].isdigit():
+                    bb.fatal("Multiconfig name '%s' is invalid as multiconfigs cannot start with a digit" % config)
+                parsed_mcdata = self.parseConfigurationFiles(self.prefiles, self.postfiles, config)
+                bb.event.fire(bb.event.ConfigParsed(), parsed_mcdata)
+                mcdata[config] = parsed_mcdata
             if multiconfig:
-                bb.event.fire(bb.event.MultiConfigParsed(self.mcdata), self.data)
+                bb.event.fire(bb.event.MultiConfigParsed(mcdata), self.data)
 
-            self.data_hash = data_hash.hexdigest()
-        except (SyntaxError, bb.BBHandledException):
-            raise bb.BBHandledException()
         except bb.data_smart.ExpansionError as e:
             logger.error(str(e))
             raise bb.BBHandledException()
-        except Exception:
-            logger.exception("Error parsing configuration files")
+
+        bb.codeparser.update_module_dependencies(self.data)
+
+        # Handle obsolete variable names
+        d = self.data
+        renamedvars = d.getVarFlags('BB_RENAMED_VARIABLES') or {}
+        renamedvars.update(bb.data_smart.bitbake_renamed_vars)
+        issues = False
+        for v in renamedvars:
+            if d.getVar(v) != None or d.hasOverrides(v):
+                issues = True
+                loginfo = {}
+                history = d.varhistory.get_variable_refs(v)
+                for h in history:
+                    for line in history[h]:
+                        loginfo = {'file' : h, 'line' : line}
+                        bb.data.data_smart._print_rename_error(v, loginfo, renamedvars)
+                if not history:
+                    bb.data.data_smart._print_rename_error(v, loginfo, renamedvars)
+        if issues:
             raise bb.BBHandledException()
 
+        for mc in mcdata:
+            mcdata[mc].renameVar("__depends", "__base_depends")
+            mcdata[mc].setVar("__bbclasstype", "recipe")
+
         # Create a copy so we can reset at a later date when UIs disconnect
-        self.origdata = self.data
-        self.data = bb.data.createCopy(self.origdata)
-        self.mcdata[''] = self.data
+        self.mcorigdata = mcdata
+        for mc in mcdata:
+            self.mcdata[mc] = bb.data.createCopy(mcdata[mc])
+        self.data = self.mcdata['']
+        self.calc_datastore_hashes()
 
     def reset(self):
         # We may not have run parseBaseConfiguration() yet
-        if not hasattr(self, 'origdata'):
+        if not hasattr(self, 'mcorigdata'):
             return
-        self.data = bb.data.createCopy(self.origdata)
-        self.mcdata[''] = self.data
+        for mc in self.mcorigdata:
+            self.mcdata[mc] = bb.data.createCopy(self.mcorigdata[mc])
+        self.data = self.mcdata['']
 
     def _findLayerConf(self, data):
         return findConfigFile("bblayers.conf", data)
 
-    def parseConfigurationFiles(self, prefiles, postfiles, mc = "default"):
+    def parseConfigurationFiles(self, prefiles, postfiles, mc = ""):
         data = bb.data.createCopy(self.basedata)
         data.setVar("BB_CURRENT_MC", mc)
 
@@ -333,15 +356,23 @@ class CookerDataBuilder(object):
 
         layerconf = self._findLayerConf(data)
         if layerconf:
-            parselog.debug(2, "Found bblayers.conf (%s)", layerconf)
+            parselog.debug2("Found bblayers.conf (%s)", layerconf)
             # By definition bblayers.conf is in conf/ of TOPDIR.
             # We may have been called with cwd somewhere else so reset TOPDIR
             data.setVar("TOPDIR", os.path.dirname(os.path.dirname(layerconf)))
             data = parse_config_file(layerconf, data)
 
+            if not data.getVar("BB_CACHEDIR"):
+                data.setVar("BB_CACHEDIR", "${TOPDIR}/cache")
+
+            bb.codeparser.parser_cache_init(data.getVar("BB_CACHEDIR"))
+
             layers = (data.getVar('BBLAYERS') or "").split()
             broken_layers = []
 
+            if not layers:
+                bb.fatal("The bblayers.conf file doesn't contain any BBLAYERS definition")
+
             data = bb.data.createCopy(data)
             approved = bb.utils.approved_variables()
 
@@ -357,8 +388,10 @@ class CookerDataBuilder(object):
                 parselog.critical("Please check BBLAYERS in %s" % (layerconf))
                 raise bb.BBHandledException()
 
+            layerseries = None
+            compat_entries = {}
             for layer in layers:
-                parselog.debug(2, "Adding layer %s", layer)
+                parselog.debug2("Adding layer %s", layer)
                 if 'HOME' in approved and '~' in layer:
                     layer = os.path.expanduser(layer)
                 if layer.endswith('/'):
@@ -369,8 +402,27 @@ class CookerDataBuilder(object):
                 data.expandVarref('LAYERDIR')
                 data.expandVarref('LAYERDIR_RE')
 
+                # Sadly we can't have nice things.
+                # Some layers think they're going to be 'clever' and copy the values from
+                # another layer, e.g. using ${LAYERSERIES_COMPAT_core}. The whole point of
+                # this mechanism is to make it clear which releases a layer supports and
+                # show when a layer master branch is bitrotting and is unmaintained.
+                # We therefore avoid people doing this here.
+                collections = (data.getVar('BBFILE_COLLECTIONS') or "").split()
+                for c in collections:
+                    compat_entry = data.getVar("LAYERSERIES_COMPAT_%s" % c)
+                    if compat_entry:
+                        compat_entries[c] = set(compat_entry.split())
+                        data.delVar("LAYERSERIES_COMPAT_%s" % c)
+                if not layerseries:
+                    layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
+                    if layerseries:
+                        data.delVar("LAYERSERIES_CORENAMES")
+
             data.delVar('LAYERDIR_RE')
             data.delVar('LAYERDIR')
+            for c in compat_entries:
+                data.setVar("LAYERSERIES_COMPAT_%s" % c, " ".join(sorted(compat_entries[c])))
 
             bbfiles_dynamic = (data.getVar('BBFILES_DYNAMIC') or "").split()
             collections = (data.getVar('BBFILE_COLLECTIONS') or "").split()
@@ -389,26 +441,38 @@ class CookerDataBuilder(object):
             if invalid:
                 bb.fatal("BBFILES_DYNAMIC entries must be of the form {!}<collection name>:<filename pattern>, not:\n    %s" % "\n    ".join(invalid))
 
-            layerseries = set((data.getVar("LAYERSERIES_CORENAMES") or "").split())
             collections_tmp = collections[:]
             for c in collections:
                 collections_tmp.remove(c)
                 if c in collections_tmp:
                     bb.fatal("Found duplicated BBFILE_COLLECTIONS '%s', check bblayers.conf or layer.conf to fix it." % c)
-                compat = set((data.getVar("LAYERSERIES_COMPAT_%s" % c) or "").split())
+
+                compat = set()
+                if c in compat_entries:
+                    compat = compat_entries[c]
+                if compat and not layerseries:
+                    bb.fatal("No core layer found to work with layer '%s'. Missing entry in bblayers.conf?" % c)
                 if compat and not (compat & layerseries):
                     bb.fatal("Layer %s is not compatible with the core layer which only supports these series: %s (layer is compatible with %s)"
                               % (c, " ".join(layerseries), " ".join(compat)))
                 elif not compat and not data.getVar("BB_WORKERCONTEXT"):
                     bb.warn("Layer %s should set LAYERSERIES_COMPAT_%s in its conf/layer.conf file to list the core layer names it is compatible with." % (c, c))
 
+            data.setVar("LAYERSERIES_CORENAMES", " ".join(sorted(layerseries)))
+
         if not data.getVar("BBPATH"):
             msg = "The BBPATH variable is not set"
             if not layerconf:
                 msg += (" and bitbake did not find a conf/bblayers.conf file in"
                         " the expected location.\nMaybe you accidentally"
                         " invoked bitbake from the wrong directory?")
-            raise SystemExit(msg)
+            bb.fatal(msg)
+
+        if not data.getVar("TOPDIR"):
+            data.setVar("TOPDIR", os.path.abspath(os.getcwd()))
+        if not data.getVar("BB_CACHEDIR"):
+            data.setVar("BB_CACHEDIR", "${TOPDIR}/cache")
+        bb.codeparser.parser_cache_init(data.getVar("BB_CACHEDIR"))
 
         data = parse_config_file(os.path.join("conf", "bitbake.conf"), data)
 
@@ -421,7 +485,7 @@ class CookerDataBuilder(object):
         for bbclass in bbclasses:
             data = _inherit(bbclass, data)
 
-        # Nomally we only register event handlers at the end of parsing .bb files
+        # Normally we only register event handlers at the end of parsing .bb files
         # We register any handlers we've found so far here...
         for var in data.getVar('__BBHANDLERS', False) or []:
             handlerfn = data.getVarFlag(var, "filename", False)
@@ -435,3 +499,54 @@ class CookerDataBuilder(object):
 
         return data
 
+    @staticmethod
+    def _parse_recipe(bb_data, bbfile, appends, mc, layername):
+        bb_data.setVar("__BBMULTICONFIG", mc)
+        bb_data.setVar("FILE_LAYERNAME", layername)
+
+        bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
+        bb.parse.cached_mtime_noerror(bbfile_loc)
+
+        if appends:
+            bb_data.setVar('__BBAPPEND', " ".join(appends))
+
+        return bb.parse.handle(bbfile, bb_data)
+
+    def parseRecipeVariants(self, bbfile, appends, virtonly=False, mc=None, layername=None):
+        """
+        Load and parse one .bb build file
+        Return the data and whether parsing resulted in the file being skipped
+        """
+
+        if virtonly:
+            (bbfile, virtual, mc) = bb.cache.virtualfn2realfn(bbfile)
+            bb_data = self.mcdata[mc].createCopy()
+            bb_data.setVar("__ONLYFINALISE", virtual or "default")
+            return self._parse_recipe(bb_data, bbfile, appends, mc, layername)
+
+        if mc is not None:
+            bb_data = self.mcdata[mc].createCopy()
+            return self._parse_recipe(bb_data, bbfile, appends, mc, layername)
+
+        bb_data = self.data.createCopy()
+        datastores = self._parse_recipe(bb_data, bbfile, appends, '', layername)
+
+        for mc in self.mcdata:
+            if not mc:
+                continue
+            bb_data = self.mcdata[mc].createCopy()
+            newstores = self._parse_recipe(bb_data, bbfile, appends, mc, layername)
+            for ns in newstores:
+                datastores["mc:%s:%s" % (mc, ns)] = newstores[ns]
+
+        return datastores
+
+    def parseRecipe(self, virtualfn, appends, layername):
+        """
+        Return a complete set of data for fn.
+        To do this, we need to parse the file.
+        """
+        logger.debug("Parsing %s (full)" % virtualfn)
+        (fn, virtual, mc) = bb.cache.virtualfn2realfn(virtualfn)
+        datastores = self.parseRecipeVariants(virtualfn, appends, virtonly=True, layername=layername)
+        return datastores[virtual]
diff --git a/bitbake/lib/bb/daemonize.py b/bitbake/lib/bb/daemonize.py
index c187fcfc6c..7689404436 100644
--- a/bitbake/lib/bb/daemonize.py
+++ b/bitbake/lib/bb/daemonize.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -74,26 +76,26 @@ def createDaemon(function, logfile):
     with open('/dev/null', 'r') as si:
         os.dup2(si.fileno(), sys.stdin.fileno())
 
-    try:
-        so = open(logfile, 'a+')
-        os.dup2(so.fileno(), sys.stdout.fileno())
-        os.dup2(so.fileno(), sys.stderr.fileno())
-    except io.UnsupportedOperation:
-        sys.stdout = open(logfile, 'a+')
+    with open(logfile, 'a+') as so:
+        try:
+            os.dup2(so.fileno(), sys.stdout.fileno())
+            os.dup2(so.fileno(), sys.stderr.fileno())
+        except io.UnsupportedOperation:
+            sys.stdout = so
 
-    # Have stdout and stderr be the same so log output matches chronologically
-    # and there aren't two seperate buffers
-    sys.stderr = sys.stdout
+        # Have stdout and stderr be the same so log output matches chronologically
+        # and there aren't two separate buffers
+        sys.stderr = sys.stdout
 
-    try:
-        function()
-    except Exception as e:
-        traceback.print_exc()
-    finally:
-        bb.event.print_ui_queue()
-        # os._exit() doesn't flush open files like os.exit() does. Manually flush
-        # stdout and stderr so that any logging output will be seen, particularly
-        # exception tracebacks.
-        sys.stdout.flush()
-        sys.stderr.flush()
-        os._exit(0)
+        try:
+            function()
+        except Exception as e:
+            traceback.print_exc()
+        finally:
+            bb.event.print_ui_queue()
+            # os._exit() doesn't flush open files like os.exit() does. Manually flush
+            # stdout and stderr so that any logging output will be seen, particularly
+            # exception tracebacks.
+            sys.stdout.flush()
+            sys.stderr.flush()
+            os._exit(0)
diff --git a/bitbake/lib/bb/data.py b/bitbake/lib/bb/data.py
index 97022853ca..f672a84451 100644
--- a/bitbake/lib/bb/data.py
+++ b/bitbake/lib/bb/data.py
@@ -4,14 +4,16 @@ BitBake 'Data' implementations
 Functions for interacting with the data structure used by the
 BitBake build tools.
 
-The expandKeys and update_data are the most expensive
-operations. At night the cookie monster came by and
+expandKeys and datastore iteration are the most expensive
+operations. Updating overrides is now "on the fly" but still based
+on the idea of the cookie monster introduced by zecke:
+"At night the cookie monster came by and
 suggested 'give me cookies on setting the variables and
 things will work out'. Taking this suggestion into account
 applying the skills from the not yet passed 'Entwurf und
 Analyse von Algorithmen' lecture and the cookie
 monster seems to be right. We will track setVar more carefully
-to have faster update_data and expandKeys operations.
+to have faster datastore operations."
 
 This is a trade-off between speed and memory again but
 the speed is more critical here.
@@ -26,11 +28,6 @@ the speed is more critical here.
 
 import sys, os, re
 import hashlib
-if sys.argv[0][-5:] == "pydoc":
-    path = os.path.dirname(os.path.dirname(sys.argv[1]))
-else:
-    path = os.path.dirname(os.path.dirname(sys.argv[0]))
-sys.path.insert(0, path)
 from itertools import groupby
 
 from bb import data_smart
@@ -70,10 +67,6 @@ def keys(d):
     """Return a list of keys in d"""
     return d.keys()
 
-
-__expand_var_regexp__ = re.compile(r"\${[^{}]+}")
-__expand_python_regexp__ = re.compile(r"\${@.+?}")
-
 def expand(s, d, varname = None):
     """Variable expansion using the data store"""
     return d.expand(s, varname)
@@ -121,8 +114,8 @@ def emit_var(var, o=sys.__stdout__, d = init(), all=False):
     if d.getVarFlag(var, 'python', False) and func:
         return False
 
-    export = d.getVarFlag(var, "export", False)
-    unexport = d.getVarFlag(var, "unexport", False)
+    export = bb.utils.to_boolean(d.getVarFlag(var, "export"))
+    unexport = bb.utils.to_boolean(d.getVarFlag(var, "unexport"))
     if not all and not export and not unexport and not func:
         return False
 
@@ -195,8 +188,8 @@ def emit_env(o=sys.__stdout__, d = init(), all=False):
 
 def exported_keys(d):
     return (key for key in d.keys() if not key.startswith('__') and
-                                      d.getVarFlag(key, 'export', False) and
-                                      not d.getVarFlag(key, 'unexport', False))
+                                      bb.utils.to_boolean(d.getVarFlag(key, 'export')) and
+                                      not bb.utils.to_boolean(d.getVarFlag(key, 'unexport')))
 
 def exported_vars(d):
     k = list(exported_keys(d))
@@ -226,7 +219,7 @@ def emit_func(func, o=sys.__stdout__, d = init()):
         deps = newdeps
         seen |= deps
         newdeps = set()
-        for dep in deps:
+        for dep in sorted(deps):
             if d.getVarFlag(dep, "func", False) and not d.getVarFlag(dep, "python", False):
                emit_var(dep, o, d, False) and o.write('\n')
                newdeps |=  bb.codeparser.ShellParser(dep, logger).parse_shell(d.getVar(dep))
@@ -268,65 +261,72 @@ def emit_func_python(func, o=sys.__stdout__, d = init()):
                newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
         newdeps -= seen
 
-def update_data(d):
-    """Performs final steps upon the datastore, including application of overrides"""
-    d.finalize(parent = True)
+def build_dependencies(key, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparsedata):
+    def handle_contains(value, contains, exclusions, d):
+        newvalue = []
+        if value:
+            newvalue.append(str(value))
+        for k in sorted(contains):
+            if k in exclusions or k in ignored_vars:
+                continue
+            l = (d.getVar(k) or "").split()
+            for item in sorted(contains[k]):
+                for word in item.split():
+                    if not word in l:
+                        newvalue.append("\n%s{%s} = Unset" % (k, item))
+                        break
+                else:
+                    newvalue.append("\n%s{%s} = Set" % (k, item))
+        return "".join(newvalue)
+
+    def handle_remove(value, deps, removes, d):
+        for r in sorted(removes):
+            r2 = d.expandWithRefs(r, None)
+            value += "\n_remove of %s" % r
+            deps |= r2.references
+            deps = deps | (keys & r2.execs)
+            value = handle_contains(value, r2.contains, exclusions, d)
+        return value
 
-def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
     deps = set()
     try:
+        if key in mod_funcs:
+            exclusions = set()
+            moddep = bb.codeparser.modulecode_deps[key]
+            value = handle_contains(moddep[4], moddep[3], exclusions, d)
+            return frozenset((moddep[0] | keys & moddep[1]) - ignored_vars), value
+
         if key[-1] == ']':
             vf = key[:-1].split('[')
+            if vf[1] == "vardepvalueexclude":
+                return deps, ""
             value, parser = d.getVarFlag(vf[0], vf[1], False, retparser=True)
             deps |= parser.references
             deps = deps | (keys & parser.execs)
-            return deps, value
+            deps -= ignored_vars
+            return frozenset(deps), value
         varflags = d.getVarFlags(key, ["vardeps", "vardepvalue", "vardepsexclude", "exports", "postfuncs", "prefuncs", "lineno", "filename"]) or {}
         vardeps = varflags.get("vardeps")
-
-        def handle_contains(value, contains, d):
-            newvalue = ""
-            for k in sorted(contains):
-                l = (d.getVar(k) or "").split()
-                for item in sorted(contains[k]):
-                    for word in item.split():
-                        if not word in l:
-                            newvalue += "\n%s{%s} = Unset" % (k, item)
-                            break
-                    else:
-                        newvalue += "\n%s{%s} = Set" % (k, item)
-            if not newvalue:
-                return value
-            if not value:
-                return newvalue
-            return value + newvalue
-
-        def handle_remove(value, deps, removes, d):
-            for r in sorted(removes):
-                r2 = d.expandWithRefs(r, None)
-                value += "\n_remove of %s" % r
-                deps |= r2.references
-                deps = deps | (keys & r2.execs)
-            return value
+        exclusions = varflags.get("vardepsexclude", "").split()
 
         if "vardepvalue" in varflags:
             value = varflags.get("vardepvalue")
         elif varflags.get("func"):
             if varflags.get("python"):
-                value = d.getVarFlag(key, "_content", False)
+                value = codeparsedata.getVarFlag(key, "_content", False)
                 parser = bb.codeparser.PythonParser(key, logger)
                 parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno"))
                 deps = deps | parser.references
                 deps = deps | (keys & parser.execs)
-                value = handle_contains(value, parser.contains, d)
+                value = handle_contains(value, parser.contains, exclusions, d)
             else:
-                value, parsedvar = d.getVarFlag(key, "_content", False, retparser=True)
+                value, parsedvar = codeparsedata.getVarFlag(key, "_content", False, retparser=True)
                 parser = bb.codeparser.ShellParser(key, logger)
                 parser.parse_shell(parsedvar.value)
                 deps = deps | shelldeps
                 deps = deps | parsedvar.references
                 deps = deps | (keys & parser.execs) | (keys & parsedvar.execs)
-                value = handle_contains(value, parsedvar.contains, d)
+                value = handle_contains(value, parsedvar.contains, exclusions, d)
                 if hasattr(parsedvar, "removes"):
                     value = handle_remove(value, deps, parsedvar.removes, d)
             if vardeps is None:
@@ -341,7 +341,7 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
             value, parser = d.getVarFlag(key, "_content", False, retparser=True)
             deps |= parser.references
             deps = deps | (keys & parser.execs)
-            value = handle_contains(value, parser.contains, d)
+            value = handle_contains(value, parser.contains, exclusions, d)
             if hasattr(parser, "removes"):
                 value = handle_remove(value, deps, parser.removes, d)
 
@@ -361,43 +361,50 @@ def build_dependencies(key, keys, shelldeps, varflagsexcl, d):
                 deps |= set(varfdeps)
 
         deps |= set((vardeps or "").split())
-        deps -= set(varflags.get("vardepsexclude", "").split())
+        deps -= set(exclusions)
+        deps -= ignored_vars
     except bb.parse.SkipRecipe:
         raise
     except Exception as e:
         bb.warn("Exception during build_dependencies for %s" % key)
         raise
-    return deps, value
+    return frozenset(deps), value
     #bb.note("Variable %s references %s and calls %s" % (key, str(deps), str(execs)))
     #d.setVarFlag(key, "vardeps", deps)
 
-def generate_dependencies(d, whitelist):
+def generate_dependencies(d, ignored_vars):
 
-    keys = set(key for key in d if not key.startswith("__"))
-    shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
+    mod_funcs = set(bb.codeparser.modulecode_deps.keys())
+    keys = set(key for key in d if not key.startswith("__")) | mod_funcs
+    shelldeps = set(key for key in d.getVar("__exportlist", False) if bb.utils.to_boolean(d.getVarFlag(key, "export")) and not bb.utils.to_boolean(d.getVarFlag(key, "unexport")))
     varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS')
 
+    codeparserd = d.createCopy()
+    for forced in (d.getVar('BB_HASH_CODEPARSER_VALS') or "").split():
+        key, value = forced.split("=", 1)
+        codeparserd.setVar(key, value)
+
     deps = {}
     values = {}
 
     tasklist = d.getVar('__BBTASKS', False) or []
     for task in tasklist:
-        deps[task], values[task] = build_dependencies(task, keys, shelldeps, varflagsexcl, d)
+        deps[task], values[task] = build_dependencies(task, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparserd)
         newdeps = deps[task]
         seen = set()
         while newdeps:
-            nextdeps = newdeps - whitelist
+            nextdeps = newdeps
             seen |= nextdeps
             newdeps = set()
             for dep in nextdeps:
                 if dep not in deps:
-                    deps[dep], values[dep] = build_dependencies(dep, keys, shelldeps, varflagsexcl, d)
+                    deps[dep], values[dep] = build_dependencies(dep, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparserd)
                 newdeps |=  deps[dep]
             newdeps -= seen
         #print "For %s: %s" % (task, str(deps[task]))
     return tasklist, deps, values
 
-def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
+def generate_dependency_hash(tasklist, gendeps, lookupcache, ignored_vars, fn):
     taskdeps = {}
     basehash = {}
 
@@ -406,9 +413,10 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
 
         if data is None:
             bb.error("Task %s from %s seems to be empty?!" % (task, fn))
-            data = ''
+            data = []
+        else:
+            data = [data]
 
-        gendeps[task] -= whitelist
         newdeps = gendeps[task]
         seen = set()
         while newdeps:
@@ -416,27 +424,24 @@ def generate_dependency_hash(tasklist, gendeps, lookupcache, whitelist, fn):
             seen |= nextdeps
             newdeps = set()
             for dep in nextdeps:
-                if dep in whitelist:
-                    continue
-                gendeps[dep] -= whitelist
                 newdeps |= gendeps[dep]
             newdeps -= seen
 
         alldeps = sorted(seen)
         for dep in alldeps:
-            data = data + dep
+            data.append(dep)
             var = lookupcache[dep]
             if var is not None:
-                data = data + str(var)
+                data.append(str(var))
         k = fn + ":" + task
-        basehash[k] = hashlib.sha256(data.encode("utf-8")).hexdigest()
-        taskdeps[task] = alldeps
+        basehash[k] = hashlib.sha256("".join(data).encode("utf-8")).hexdigest()
+        taskdeps[task] = frozenset(seen)
 
     return taskdeps, basehash
 
 def inherits_class(klass, d):
     val = d.getVar('__inherit_cache', False) or []
-    needle = os.path.join('classes', '%s.bbclass' % klass)
+    needle = '/%s.bbclass' % klass
     for v in val:
         if v.endswith(needle):
             return True
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index 2328c334ac..8e7dd98384 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -16,8 +16,11 @@ BitBake build tools.
 #
 # Based on functions from the base bb module, Copyright 2003 Holger Schurig
 
-import copy, re, sys, traceback
-from collections import MutableMapping
+import builtins
+import copy
+import re
+import sys
+from collections.abc import MutableMapping
 import logging
 import hashlib
 import bb, bb.codeparser
@@ -26,13 +29,25 @@ from bb.COW  import COWDictBase
 
 logger = logging.getLogger("BitBake.Data")
 
-__setvar_keyword__ = ["_append", "_prepend", "_remove"]
-__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>_append|_prepend|_remove)(_(?P<add>[^A-Z]*))?$')
-__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~]+?}")
-__expand_python_regexp__ = re.compile(r"\${@.+?}")
+__setvar_keyword__ = [":append", ":prepend", ":remove"]
+__setvar_regexp__ = re.compile(r'(?P<base>.*?)(?P<keyword>:append|:prepend|:remove)(:(?P<add>[^A-Z]*))?$')
+__expand_var_regexp__ = re.compile(r"\${[a-zA-Z0-9\-_+./~:]+}")
+__expand_python_regexp__ = re.compile(r"\${@(?:{.*?}|.)+?}")
 __whitespace_split__ = re.compile(r'(\s)')
 __override_regexp__ = re.compile(r'[a-z0-9]+')
 
+bitbake_renamed_vars = {
+    "BB_ENV_WHITELIST": "BB_ENV_PASSTHROUGH",
+    "BB_ENV_EXTRAWHITE": "BB_ENV_PASSTHROUGH_ADDITIONS",
+    "BB_HASHBASE_WHITELIST": "BB_BASEHASH_IGNORE_VARS",
+    "BB_HASHCONFIG_WHITELIST": "BB_HASHCONFIG_IGNORE_VARS",
+    "BB_HASHTASK_WHITELIST": "BB_TASKHASH_IGNORE_TASKS",
+    "BB_SETSCENE_ENFORCE_WHITELIST": "BB_SETSCENE_ENFORCE_IGNORE_TASKS",
+    "MULTI_PROVIDER_WHITELIST": "BB_MULTI_PROVIDER_ALLOWED",
+    "BB_STAMP_WHITELIST": "is a deprecated variable and support has been removed",
+    "BB_STAMP_POLICY": "is a deprecated variable and support has been removed",
+}
+
 def infer_caller_details(loginfo, parent = False, varval = True):
     """Save the caller the trouble of specifying everything."""
     # Save effort.
@@ -80,68 +95,79 @@ def infer_caller_details(loginfo, parent = False, varval = True):
             loginfo['func'] = func
 
 class VariableParse:
-    def __init__(self, varname, d, val = None):
+    def __init__(self, varname, d, unexpanded_value = None, val = None):
         self.varname = varname
         self.d = d
         self.value = val
+        self.unexpanded_value = unexpanded_value
 
         self.references = set()
         self.execs = set()
         self.contains = {}
 
     def var_sub(self, match):
-            key = match.group()[2:-1]
-            if self.varname and key:
-                if self.varname == key:
-                    raise Exception("variable %s references itself!" % self.varname)
-            var = self.d.getVarFlag(key, "_content")
-            self.references.add(key)
-            if var is not None:
-                return var
-            else:
-                return match.group()
+        key = match.group()[2:-1]
+        if self.varname and key:
+            if self.varname == key:
+                raise Exception("variable %s references itself!" % self.varname)
+        var = self.d.getVarFlag(key, "_content")
+        self.references.add(key)
+        if var is not None:
+            return var
+        else:
+            return match.group()
 
     def python_sub(self, match):
-            if isinstance(match, str):
-                code = match
-            else:
-                code = match.group()[3:-1]
+        if isinstance(match, str):
+            code = match
+        else:
+            code = match.group()[3:-1]
 
-            if self.varname:
-                varname = 'Var <%s>' % self.varname
-            else:
-                varname = '<expansion>'
-            codeobj = compile(code.strip(), varname, "eval")
-
-            parser = bb.codeparser.PythonParser(self.varname, logger)
-            parser.parse_python(code)
-            if self.varname:
-                vardeps = self.d.getVarFlag(self.varname, "vardeps")
-                if vardeps is None:
-                    parser.log.flush()
-            else:
-                parser.log.flush()
-            self.references |= parser.references
-            self.execs |= parser.execs
+        # Do not run code that contains one or more unexpanded variables
+        # instead return the code with the characters we removed put back
+        if __expand_var_regexp__.findall(code):
+            return "${@" + code + "}"
 
-            for k in parser.contains:
-                if k not in self.contains:
-                    self.contains[k] = parser.contains[k].copy()
-                else:
-                    self.contains[k].update(parser.contains[k])
-            value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
-            return str(value)
+        if self.varname:
+            varname = 'Var <%s>' % self.varname
+        else:
+            varname = '<expansion>'
+        codeobj = compile(code.strip(), varname, "eval")
+
+        parser = bb.codeparser.PythonParser(self.varname, logger)
+        parser.parse_python(code)
+        if self.varname:
+            vardeps = self.d.getVarFlag(self.varname, "vardeps")
+            if vardeps is None:
+                parser.log.flush()
+        else:
+            parser.log.flush()
+        self.references |= parser.references
+        self.execs |= parser.execs
 
+        for k in parser.contains:
+            if k not in self.contains:
+                self.contains[k] = parser.contains[k].copy()
+            else:
+                self.contains[k].update(parser.contains[k])
+        value = utils.better_eval(codeobj, DataContext(self.d), {'d' : self.d})
+        return str(value)
 
 class DataContext(dict):
+    excluded = set([i for i in dir(builtins) if not i.startswith('_')] + ['oe'])
+
     def __init__(self, metadata, **kwargs):
         self.metadata = metadata
         dict.__init__(self, **kwargs)
         self['d'] = metadata
+        self.context = set(bb.utils.get_context())
 
     def __missing__(self, key):
+        if key in self.excluded or key in self.context:
+            raise KeyError(key)
+
         value = self.metadata.getVar(key)
-        if value is None or self.metadata.getVarFlag(key, 'func', False):
+        if value is None:
             raise KeyError(key)
         else:
             return value
@@ -151,6 +177,7 @@ class ExpansionError(Exception):
         self.expression = expression
         self.variablename = varname
         self.exception = exception
+        self.varlist = [varname or expression or ""]
         if varname:
             if expression:
                 self.msg = "Failure expanding variable %s, expression was %s which triggered exception %s: %s" % (varname, expression, type(exception).__name__, exception)
@@ -160,8 +187,14 @@ class ExpansionError(Exception):
             self.msg = "Failure expanding expression %s which triggered exception %s: %s" % (expression, type(exception).__name__, exception)
         Exception.__init__(self, self.msg)
         self.args = (varname, expression, exception)
+
+    def addVar(self, varname):
+        if varname:
+            self.varlist.append(varname)
+
     def __str__(self):
-        return self.msg
+        chain = "\nThe variable dependency chain for the failure is: " + " -> ".join(self.varlist)
+        return self.msg + chain
 
 class IncludeHistory(object):
     def __init__(self, parent = None, filename = '[TOP LEVEL]'):
@@ -239,12 +272,9 @@ class VariableHistory(object):
             return
         if 'op' not in loginfo or not loginfo['op']:
             loginfo['op'] = 'set'
-        if 'detail' in loginfo:
-            loginfo['detail'] = str(loginfo['detail'])
         if 'variable' not in loginfo or 'file' not in loginfo:
             raise ValueError("record() missing variable or file.")
         var = loginfo['variable']
-
         if var not in self.variables:
             self.variables[var] = []
         if not isinstance(self.variables[var], list):
@@ -277,7 +307,7 @@ class VariableHistory(object):
             for (r, override) in d.overridedata[var]:
                 for event in self.variable(r):
                     loginfo = event.copy()
-                    if 'flag' in loginfo and not loginfo['flag'].startswith("_"):
+                    if 'flag' in loginfo and not loginfo['flag'].startswith(("_", ":")):
                         continue
                     loginfo['variable'] = var
                     loginfo['op'] = 'override[%s]:%s' % (override, loginfo['op'])
@@ -303,7 +333,8 @@ class VariableHistory(object):
                     flag = '[%s] ' % (event['flag'])
                 else:
                     flag = ''
-                o.write("#   %s %s:%s%s\n#     %s\"%s\"\n" % (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n#     ', event['detail'])))
+                o.write("#   %s %s:%s%s\n#     %s\"%s\"\n" % \
+                    (event['op'], event['file'], event['line'], display_func, flag, re.sub('\n', '\n#     ', str(event['detail']))))
             if len(history) > 1:
                 o.write("# pre-expansion value:\n")
                 o.write('#   "%s"\n' % (commentVal))
@@ -329,6 +360,16 @@ class VariableHistory(object):
                 lines.append(line)
         return lines
 
+    def get_variable_refs(self, var):
+        """Return a dict of file/line references"""
+        var_history = self.variable(var)
+        refs = {}
+        for event in var_history:
+            if event['file'] not in refs:
+                refs[event['file']] = []
+            refs[event['file']].append(event['line'])
+        return refs
+
     def get_variable_items_files(self, var):
         """
         Use variable history to map items added to a list variable and
@@ -342,12 +383,12 @@ class VariableHistory(object):
         for event in history:
             if 'flag' in event:
                 continue
-            if event['op'] == '_remove':
+            if event['op'] == ':remove':
                 continue
             if isset and event['op'] == 'set?':
                 continue
             isset = True
-            items = d.expand(event['detail']).split()
+            items = d.expand(str(event['detail'])).split()
             for item in items:
                 # This is a little crude but is belt-and-braces to avoid us
                 # having to handle every possible operation type specifically
@@ -363,6 +404,23 @@ class VariableHistory(object):
             else:
                 self.variables[var] = []
 
+def _print_rename_error(var, loginfo, renamedvars, fullvar=None):
+    info = ""
+    if "file" in loginfo:
+        info = " file: %s" % loginfo["file"]
+    if "line" in loginfo:
+        info += " line: %s" % loginfo["line"]
+    if fullvar and fullvar != var:
+        info += " referenced as: %s" % fullvar
+    if info:
+        info = " (%s)" % info.strip()
+    renameinfo = renamedvars[var]
+    if " " in renameinfo:
+        # A space signals a string to display instead of a rename
+        bb.erroronce('Variable %s %s%s' % (var, renameinfo, info))
+    else:
+        bb.erroronce('Variable %s has been renamed to %s%s' % (var, renameinfo, info))
+
 class DataSmart(MutableMapping):
     def __init__(self):
         self.dict = {}
@@ -370,6 +428,8 @@ class DataSmart(MutableMapping):
         self.inchistory = IncludeHistory()
         self.varhistory = VariableHistory(self)
         self._tracking = False
+        self._var_renames = {}
+        self._var_renames.update(bitbake_renamed_vars)
 
         self.expand_cache = {}
 
@@ -391,9 +451,9 @@ class DataSmart(MutableMapping):
     def expandWithRefs(self, s, varname):
 
         if not isinstance(s, str): # sanity check
-            return VariableParse(varname, self, s)
+            return VariableParse(varname, self, s, s)
 
-        varparse = VariableParse(varname, self)
+        varparse = VariableParse(varname, self, s)
 
         while s.find('${') != -1:
             olds = s
@@ -403,14 +463,17 @@ class DataSmart(MutableMapping):
                     s = __expand_python_regexp__.sub(varparse.python_sub, s)
                 except SyntaxError as e:
                     # Likely unmatched brackets, just don't expand the expression
-                    if e.msg != "EOL while scanning string literal":
+                    if e.msg != "EOL while scanning string literal" and not e.msg.startswith("unterminated string literal"):
                         raise
                 if s == olds:
                     break
-            except ExpansionError:
+            except ExpansionError as e:
+                e.addVar(varname)
                 raise
             except bb.parse.SkipRecipe:
                 raise
+            except bb.BBHandledException:
+                raise
             except Exception as exc:
                 tb = sys.exc_info()[2]
                 raise ExpansionError(varname, s, exc).with_traceback(tb) from exc
@@ -422,24 +485,19 @@ class DataSmart(MutableMapping):
     def expand(self, s, varname = None):
         return self.expandWithRefs(s, varname).value
 
-    def finalize(self, parent = False):
-        return
-
-    def internal_finalize(self, parent = False):
-        """Performs final steps upon the datastore, including application of overrides"""
-        self.overrides = None
-
     def need_overrides(self):
         if self.overrides is not None:
             return
         if self.inoverride:
             return
+        overrride_stack = []
         for count in range(5):
             self.inoverride = True
             # Can end up here recursively so setup dummy values
             self.overrides = []
             self.overridesset = set()
             self.overrides = (self.getVar("OVERRIDES") or "").split(":") or []
+            overrride_stack.append(self.overrides)
             self.overridesset = set(self.overrides)
             self.inoverride = False
             self.expand_cache = {}
@@ -449,7 +507,7 @@ class DataSmart(MutableMapping):
             self.overrides = newoverrides
             self.overridesset = set(self.overrides)
         else:
-            bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work.")
+            bb.fatal("Overrides could not be expanded into a stable state after 5 iterations, overrides must be being referenced by other overridden variables in some recursive fashion. Please provide your configuration to bitbake-devel so we can laugh, er, I mean try and understand how to make it work. The list of failing override expansions: %s" % "\n".join(str(s) for s in overrride_stack))
 
     def initVar(self, var):
         self.expand_cache = {}
@@ -460,27 +518,44 @@ class DataSmart(MutableMapping):
         dest = self.dict
         while dest:
             if var in dest:
-                return dest[var], self.overridedata.get(var, None)
+                return dest[var]
 
             if "_data" not in dest:
                 break
             dest = dest["_data"]
-        return None, self.overridedata.get(var, None)
+        return None
 
     def _makeShadowCopy(self, var):
         if var in self.dict:
             return
 
-        local_var, _ = self._findVar(var)
+        local_var = self._findVar(var)
 
         if local_var:
             self.dict[var] = copy.copy(local_var)
         else:
             self.initVar(var)
 
+    def hasOverrides(self, var):
+        return var in self.overridedata
 
     def setVar(self, var, value, **loginfo):
         #print("var=" + str(var) + "  val=" + str(value))
+
+        if not var.startswith("__anon_") and ("_append" in var or "_prepend" in var or "_remove" in var):
+            info = "%s" % var
+            if "file" in loginfo:
+                info += " file: %s" % loginfo["file"]
+            if "line" in loginfo:
+                info += " line: %s" % loginfo["line"]
+            bb.fatal("Variable %s contains an operation using the old override syntax. Please convert this layer/metadata before attempting to use with a newer bitbake." % info)
+
+        shortvar = var.split(":", 1)[0]
+        if shortvar in self._var_renames:
+            _print_rename_error(shortvar, loginfo, self._var_renames, fullvar=var)
+            # Mark that we have seen a renamed variable
+            self.setVar("_FAILPARSINGERRORHANDLED", True)
+
         self.expand_cache = {}
         parsing=False
         if 'parsing' in loginfo:
@@ -505,12 +580,10 @@ class DataSmart(MutableMapping):
             else:
                 loginfo['op'] = keyword
             self.varhistory.record(**loginfo)
-            # todo make sure keyword is not __doc__ or __module__
             # pay the cookie monster
 
             # more cookies for the cookie monster
-            if '_' in var:
-                self._setvar_update_overrides(base, **loginfo)
+            self._setvar_update_overrides(base, **loginfo)
 
             if base in self.overridevars:
                 self._setvar_update_overridevars(var, value)
@@ -520,27 +593,27 @@ class DataSmart(MutableMapping):
             self._makeShadowCopy(var)
 
         if not parsing:
-            if "_append" in self.dict[var]:
-                del self.dict[var]["_append"]
-            if "_prepend" in self.dict[var]:
-                del self.dict[var]["_prepend"]
-            if "_remove" in self.dict[var]:
-                del self.dict[var]["_remove"]
+            if ":append" in self.dict[var]:
+                del self.dict[var][":append"]
+            if ":prepend" in self.dict[var]:
+                del self.dict[var][":prepend"]
+            if ":remove" in self.dict[var]:
+                del self.dict[var][":remove"]
             if var in self.overridedata:
                 active = []
                 self.need_overrides()
                 for (r, o) in self.overridedata[var]:
                     if o in self.overridesset:
                         active.append(r)
-                    elif "_" in o:
-                        if set(o.split("_")).issubset(self.overridesset):
+                    elif ":" in o:
+                        if set(o.split(":")).issubset(self.overridesset):
                             active.append(r)
                 for a in active:
                     self.delVar(a)
                 del self.overridedata[var]
 
         # more cookies for the cookie monster
-        if '_' in var:
+        if ':' in var:
             self._setvar_update_overrides(var, **loginfo)
 
         # setting var
@@ -562,12 +635,13 @@ class DataSmart(MutableMapping):
                 nextnew.update(vardata.references)
                 nextnew.update(vardata.contains.keys())
             new = nextnew
-        self.internal_finalize(True)
+        self.overrides = None
+        self.expand_cache = {}
 
     def _setvar_update_overrides(self, var, **loginfo):
         # aka pay the cookie monster
-        override = var[var.rfind('_')+1:]
-        shortvar = var[:var.rfind('_')]
+        override = var[var.rfind(':')+1:]
+        shortvar = var[:var.rfind(':')]
         while override and __override_regexp__.match(override):
             if shortvar not in self.overridedata:
                 self.overridedata[shortvar] = []
@@ -576,9 +650,9 @@ class DataSmart(MutableMapping):
                 self.overridedata[shortvar] = list(self.overridedata[shortvar])
                 self.overridedata[shortvar].append([var, override])
             override = None
-            if "_" in shortvar:
-                override = var[shortvar.rfind('_')+1:]
-                shortvar = var[:shortvar.rfind('_')]
+            if ":" in shortvar:
+                override = var[shortvar.rfind(':')+1:]
+                shortvar = var[:shortvar.rfind(':')]
                 if len(shortvar) == 0:
                     override = None
 
@@ -602,10 +676,11 @@ class DataSmart(MutableMapping):
             self.varhistory.record(**loginfo)
             self.setVar(newkey, val, ignore=True, parsing=True)
 
-        for i in (__setvar_keyword__):
-            src = self.getVarFlag(key, i, False)
-            if src is None:
+        srcflags = self.getVarFlags(key, False, True) or {}
+        for i in srcflags:
+            if i not in (__setvar_keyword__):
                 continue
+            src = srcflags[i]
 
             dest = self.getVarFlag(newkey, i, False) or []
             dest.extend(src)
@@ -617,7 +692,7 @@ class DataSmart(MutableMapping):
                 self.overridedata[newkey].append([v.replace(key, newkey), o])
                 self.renameVar(v, v.replace(key, newkey))
 
-        if '_' in newkey and val is None:
+        if ':' in newkey and val is None:
             self._setvar_update_overrides(newkey, **loginfo)
 
         loginfo['variable'] = key
@@ -629,12 +704,12 @@ class DataSmart(MutableMapping):
     def appendVar(self, var, value, **loginfo):
         loginfo['op'] = 'append'
         self.varhistory.record(**loginfo)
-        self.setVar(var + "_append", value, ignore=True, parsing=True)
+        self.setVar(var + ":append", value, ignore=True, parsing=True)
 
     def prependVar(self, var, value, **loginfo):
         loginfo['op'] = 'prepend'
         self.varhistory.record(**loginfo)
-        self.setVar(var + "_prepend", value, ignore=True, parsing=True)
+        self.setVar(var + ":prepend", value, ignore=True, parsing=True)
 
     def delVar(self, var, **loginfo):
         self.expand_cache = {}
@@ -645,10 +720,10 @@ class DataSmart(MutableMapping):
         self.dict[var] = {}
         if var in self.overridedata:
             del self.overridedata[var]
-        if '_' in var:
-            override = var[var.rfind('_')+1:]
-            shortvar = var[:var.rfind('_')]
-            while override and override.islower():
+        if ':' in var:
+            override = var[var.rfind(':')+1:]
+            shortvar = var[:var.rfind(':')]
+            while override and __override_regexp__.match(override):
                 try:
                     if shortvar in self.overridedata:
                         # Force CoW by recreating the list first
@@ -657,15 +732,23 @@ class DataSmart(MutableMapping):
                 except ValueError as e:
                     pass
                 override = None
-                if "_" in shortvar:
-                    override = var[shortvar.rfind('_')+1:]
-                    shortvar = var[:shortvar.rfind('_')]
+                if ":" in shortvar:
+                    override = var[shortvar.rfind(':')+1:]
+                    shortvar = var[:shortvar.rfind(':')]
                     if len(shortvar) == 0:
                          override = None
 
     def setVarFlag(self, var, flag, value, **loginfo):
         self.expand_cache = {}
 
+        if var == "BB_RENAMED_VARIABLES":
+            self._var_renames[flag] = value
+
+        if var in self._var_renames:
+            _print_rename_error(var, loginfo, self._var_renames)
+            # Mark that we have seen a renamed variable
+            self.setVar("_FAILPARSINGERRORHANDLED", True)
+
         if 'op' not in loginfo:
             loginfo['op'] = "set"
         loginfo['flag'] = flag
@@ -674,7 +757,7 @@ class DataSmart(MutableMapping):
             self._makeShadowCopy(var)
         self.dict[var][flag] = value
 
-        if flag == "_defaultval" and '_' in var:
+        if flag == "_defaultval" and ':' in var:
             self._setvar_update_overrides(var, **loginfo)
         if flag == "_defaultval" and var in self.overridevars:
             self._setvar_update_overridevars(var, value)
@@ -695,22 +778,27 @@ class DataSmart(MutableMapping):
                 return None
             cachename = var + "[" + flag + "]"
 
+        if not expand and retparser and cachename in self.expand_cache:
+            return self.expand_cache[cachename].unexpanded_value, self.expand_cache[cachename]
+
         if expand and cachename in self.expand_cache:
             return self.expand_cache[cachename].value
 
-        local_var, overridedata = self._findVar(var)
+        local_var = self._findVar(var)
         value = None
         removes = set()
-        if flag == "_content" and overridedata is not None and not parsing:
+        if flag == "_content" and not parsing:
+            overridedata = self.overridedata.get(var, None)
+        if flag == "_content" and not parsing and overridedata is not None:
             match = False
             active = {}
             self.need_overrides()
             for (r, o) in overridedata:
-                # What about double overrides both with "_" in the name?
+                # FIXME What about double overrides both with "_" in the name?
                 if o in self.overridesset:
                     active[o] = r
-                elif "_" in o:
-                    if set(o.split("_")).issubset(self.overridesset):
+                elif ":" in o:
+                    if set(o.split(":")).issubset(self.overridesset):
                         active[o] = r
 
             mod = True
@@ -718,10 +806,10 @@ class DataSmart(MutableMapping):
                 mod = False
                 for o in self.overrides:
                     for a in active.copy():
-                        if a.endswith("_" + o):
+                        if a.endswith(":" + o):
                             t = active[a]
                             del active[a]
-                            active[a.replace("_" + o, "")] = t
+                            active[a.replace(":" + o, "")] = t
                             mod = True
                         elif a == o:
                             match = active[a]
@@ -738,33 +826,35 @@ class DataSmart(MutableMapping):
                 value = copy.copy(local_var[flag])
             elif flag == "_content" and "_defaultval" in local_var and not noweakdefault:
                 value = copy.copy(local_var["_defaultval"])
+            elif "_defaultval_flag_"+flag in local_var and not noweakdefault:
+                value = copy.copy(local_var["_defaultval_flag_"+flag])
 
 
-        if flag == "_content" and local_var is not None and "_append" in local_var and not parsing:
-            if not value:
-                value = ""
+        if flag == "_content" and local_var is not None and ":append" in local_var and not parsing:
             self.need_overrides()
-            for (r, o) in local_var["_append"]:
+            for (r, o) in local_var[":append"]:
                 match = True
                 if o:
-                    for o2 in o.split("_"):
+                    for o2 in o.split(":"):
                         if not o2 in self.overrides:
                             match = False                            
                 if match:
+                    if value is None:
+                        value = ""
                     value = value + r
 
-        if flag == "_content" and local_var is not None and "_prepend" in local_var and not parsing:
-            if not value:
-                value = ""
+        if flag == "_content" and local_var is not None and ":prepend" in local_var and not parsing:
             self.need_overrides()
-            for (r, o) in local_var["_prepend"]:
+            for (r, o) in local_var[":prepend"]:
 
                 match = True
                 if o:
-                    for o2 in o.split("_"):
+                    for o2 in o.split(":"):
                         if not o2 in self.overrides:
                             match = False                            
                 if match:
+                    if value is None:
+                        value = ""
                     value = r + value
 
         parser = None
@@ -773,12 +863,12 @@ class DataSmart(MutableMapping):
         if expand:
             value = parser.value
 
-        if value and flag == "_content" and local_var is not None and "_remove" in local_var and not parsing:
+        if value and flag == "_content" and local_var is not None and ":remove" in local_var and not parsing:
             self.need_overrides()
-            for (r, o) in local_var["_remove"]:
+            for (r, o) in local_var[":remove"]:
                 match = True
                 if o:
-                    for o2 in o.split("_"):
+                    for o2 in o.split(":"):
                         if not o2 in self.overrides:
                             match = False                            
                 if match:
@@ -791,7 +881,7 @@ class DataSmart(MutableMapping):
                     expanded_removes[r] = self.expand(r).split()
 
                 parser.removes = set()
-                val = ""
+                val = []
                 for v in __whitespace_split__.split(parser.value):
                     skip = False
                     for r in removes:
@@ -800,8 +890,8 @@ class DataSmart(MutableMapping):
                             skip = True
                     if skip:
                         continue
-                    val = val + v
-                parser.value = val
+                    val.append(v)
+                parser.value = "".join(val)
                 if expand:
                     value = parser.value
 
@@ -816,7 +906,7 @@ class DataSmart(MutableMapping):
     def delVarFlag(self, var, flag, **loginfo):
         self.expand_cache = {}
 
-        local_var, _ = self._findVar(var)
+        local_var = self._findVar(var)
         if not local_var:
             return
         if not var in self.dict:
@@ -829,6 +919,8 @@ class DataSmart(MutableMapping):
             self.varhistory.record(**loginfo)
 
             del self.dict[var][flag]
+            if ("_defaultval_flag_" + flag) in self.dict[var]:
+                del self.dict[var]["_defaultval_flag_" + flag]
 
     def appendVarFlag(self, var, flag, value, **loginfo):
         loginfo['op'] = 'append'
@@ -859,21 +951,26 @@ class DataSmart(MutableMapping):
             self.dict[var][i] = flags[i]
 
     def getVarFlags(self, var, expand = False, internalflags=False):
-        local_var, _ = self._findVar(var)
+        local_var = self._findVar(var)
         flags = {}
 
         if local_var:
-            for i in local_var:
-                if i.startswith("_") and not internalflags:
+            for i, val in local_var.items():
+                if i.startswith("_defaultval_flag_") and not internalflags:
+                    i = i[len("_defaultval_flag_"):]
+                    if i not in local_var:
+                        flags[i] = val
+                elif i.startswith(("_", ":")) and not internalflags:
                     continue
-                flags[i] = local_var[i]
+                else:
+                    flags[i] = val
+
                 if expand and i in expand:
                     flags[i] = self.expand(flags[i], var + "[" + i + "]")
         if len(flags) == 0:
             return None
         return flags
 
-
     def delVarFlags(self, var, **loginfo):
         self.expand_cache = {}
         if not var in self.dict:
@@ -905,6 +1002,7 @@ class DataSmart(MutableMapping):
         data.inchistory = self.inchistory.copy()
 
         data._tracking = self._tracking
+        data._var_renames = self._var_renames
 
         data.overrides = None
         data.overridevars = copy.copy(self.overridevars)
@@ -927,7 +1025,7 @@ class DataSmart(MutableMapping):
         value = self.getVar(variable, False)
         for key in keys:
             referrervalue = self.getVar(key, False)
-            if referrervalue and ref in referrervalue:
+            if referrervalue and isinstance(referrervalue, str) and ref in referrervalue:
                 self.setVar(key, referrervalue.replace(ref, value))
 
     def localkeys(self):
@@ -962,8 +1060,8 @@ class DataSmart(MutableMapping):
             for (r, o) in self.overridedata[var]:
                 if o in self.overridesset:
                     overrides.add(var)
-                elif "_" in o:
-                    if set(o.split("_")).issubset(self.overridesset):
+                elif ":" in o:
+                    if set(o.split(":")).issubset(self.overridesset):
                         overrides.add(var)
 
         for k in keylist(self.dict):
@@ -993,10 +1091,10 @@ class DataSmart(MutableMapping):
         d = self.createCopy()
         bb.data.expandKeys(d)
 
-        config_whitelist = set((d.getVar("BB_HASHCONFIG_WHITELIST") or "").split())
+        config_ignore_vars = set((d.getVar("BB_HASHCONFIG_IGNORE_VARS") or "").split())
         keys = set(key for key in iter(d) if not key.startswith("__"))
         for key in keys:
-            if key in config_whitelist:
+            if key in config_ignore_vars:
                 continue
 
             value = d.getVar(key, False) or ""
@@ -1022,5 +1120,10 @@ class DataSmart(MutableMapping):
                     value = d.getVar(i, False) or ""
                     data.update({i:value})
 
+        moddeps = bb.codeparser.modulecode_deps
+        for dep in sorted(moddeps):
+            # Ignore visitor code, sort sets
+            data.update({'moddep[%s]' % dep : [sorted(moddeps[dep][0]), sorted(moddeps[dep][1]), sorted(moddeps[dep][2]), sorted(moddeps[dep][3]), moddeps[dep][4]]})
+
         data_str = str([(k, data[k]) for k in sorted(data.keys())])
         return hashlib.sha256(data_str.encode("utf-8")).hexdigest()
diff --git a/bitbake/lib/bb/event.py b/bitbake/lib/bb/event.py
index 23e1f3187b..b29f0a5568 100644
--- a/bitbake/lib/bb/event.py
+++ b/bitbake/lib/bb/event.py
@@ -19,7 +19,6 @@ import sys
 import threading
 import traceback
 
-import bb.exceptions
 import bb.utils
 
 # This is the pid for which we should generate the event. This is set when
@@ -40,7 +39,7 @@ class HeartbeatEvent(Event):
     """Triggered at regular time intervals of 10 seconds. Other events can fire much more often
        (runQueueTaskStarted when there are many short tasks) or not at all for long periods
        of time (again runQueueTaskStarted, when there is just one long-running task), so this
-       event is more suitable for doing some task-independent work occassionally."""
+       event is more suitable for doing some task-independent work occasionally."""
     def __init__(self, time):
         Event.__init__(self)
         self.time = time
@@ -68,29 +67,39 @@ _catchall_handlers = {}
 _eventfilter = None
 _uiready = False
 _thread_lock = threading.Lock()
-_thread_lock_enabled = False
-
-if hasattr(__builtins__, '__setitem__'):
-    builtins = __builtins__
-else:
-    builtins = __builtins__.__dict__
+_heartbeat_enabled = False
+_should_exit = threading.Event()
 
 def enable_threadlock():
-    global _thread_lock_enabled
-    _thread_lock_enabled = True
+    # Always needed now
+    return
 
 def disable_threadlock():
-    global _thread_lock_enabled
-    _thread_lock_enabled = False
+    # Always needed now
+    return
+
+def enable_heartbeat():
+    global _heartbeat_enabled
+    _heartbeat_enabled = True
+
+def disable_heartbeat():
+    global _heartbeat_enabled
+    _heartbeat_enabled = False
+
+#
+# In long running code, this function should be called periodically
+# to check if we should exit due to an interuption (.e.g Ctrl+C from the UI)
+#
+def check_for_interrupts(d):
+    global _should_exit
+    if _should_exit.is_set():
+        bb.warn("Exiting due to interrupt.")
+        raise bb.BBHandledException()
 
 def execute_handler(name, handler, event, d):
     event.data = d
-    addedd = False
-    if 'd' not in builtins:
-        builtins['d'] = d
-        addedd = True
     try:
-        ret = handler(event)
+        ret = handler(event, d)
     except (bb.parse.SkipRecipe, bb.BBHandledException):
         raise
     except Exception:
@@ -104,8 +113,7 @@ def execute_handler(name, handler, event, d):
         raise
     finally:
         del event.data
-        if addedd:
-            del builtins['d']
+
 
 def fire_class_handlers(event, d):
     if isinstance(event, logging.LogRecord):
@@ -118,7 +126,7 @@ def fire_class_handlers(event, d):
             if _eventfilter:
                 if not _eventfilter(name, handler, event, d):
                     continue
-            if d and not name in (d.getVar("__BBHANDLERS_MC") or []):
+            if d is not None and not name in (d.getVar("__BBHANDLERS_MC") or set()):
                 continue
             execute_handler(name, handler, event, d)
 
@@ -132,8 +140,14 @@ def print_ui_queue():
     if not _uiready:
         from bb.msg import BBLogFormatter
         # Flush any existing buffered content
-        sys.stdout.flush()
-        sys.stderr.flush()
+        try:
+            sys.stdout.flush()
+        except:
+            pass
+        try:
+            sys.stderr.flush()
+        except:
+            pass
         stdout = logging.StreamHandler(sys.stdout)
         stderr = logging.StreamHandler(sys.stderr)
         formatter = BBLogFormatter("%(levelname)s: %(message)s")
@@ -174,36 +188,38 @@ def print_ui_queue():
 
 def fire_ui_handlers(event, d):
     global _thread_lock
-    global _thread_lock_enabled
 
     if not _uiready:
         # No UI handlers registered yet, queue up the messages
         ui_queue.append(event)
         return
 
-    if _thread_lock_enabled:
-        _thread_lock.acquire()
-
-    errors = []
-    for h in _ui_handlers:
-        #print "Sending event %s" % event
-        try:
-             if not _ui_logfilters[h].filter(event):
-                 continue
-             # We use pickle here since it better handles object instances
-             # which xmlrpc's marshaller does not. Events *must* be serializable
-             # by pickle.
-             if hasattr(_ui_handlers[h].event, "sendpickle"):
-                _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
-             else:
-                _ui_handlers[h].event.send(event)
-        except:
-            errors.append(h)
-    for h in errors:
-        del _ui_handlers[h]
+    with bb.utils.lock_timeout_nocheck(_thread_lock) as lock:
+        if not lock:
+            # If we can't get the lock, we may be recursively called, queue and return
+            ui_queue.append(event)
+            return
 
-    if _thread_lock_enabled:
-        _thread_lock.release()
+        errors = []
+        for h in _ui_handlers:
+            #print "Sending event %s" % event
+            try:
+                 if not _ui_logfilters[h].filter(event):
+                     continue
+                 # We use pickle here since it better handles object instances
+                 # which xmlrpc's marshaller does not. Events *must* be serializable
+                 # by pickle.
+                 if hasattr(_ui_handlers[h].event, "sendpickle"):
+                    _ui_handlers[h].event.sendpickle((pickle.dumps(event)))
+                 else:
+                    _ui_handlers[h].event.send(event)
+            except:
+                errors.append(h)
+        for h in errors:
+            del _ui_handlers[h]
+
+    while ui_queue:
+        fire_ui_handlers(ui_queue.pop(), d)
 
 def fire(event, d):
     """Fire off an Event"""
@@ -232,26 +248,31 @@ noop = lambda _: None
 def register(name, handler, mask=None, filename=None, lineno=None, data=None):
     """Register an Event handler"""
 
-    if data and data.getVar("BB_CURRENT_MC"):
+    if data is not None and data.getVar("BB_CURRENT_MC"):
         mc = data.getVar("BB_CURRENT_MC")
         name = '%s%s' % (mc.replace('-', '_'), name)
 
     # already registered
     if name in _handlers:
+        if data is not None:
+            bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
+            bbhands_mc.add(name)
+            data.setVar("__BBHANDLERS_MC", bbhands_mc)
         return AlreadyRegistered
 
     if handler is not None:
         # handle string containing python code
         if isinstance(handler, str):
-            tmp = "def %s(e):\n%s" % (name, handler)
+            tmp = "def %s(e, d):\n%s" % (name, handler)
+            # Inject empty lines to make code match lineno in filename
+            if lineno is not None:
+                tmp = "\n" * (lineno-1) + tmp
             try:
                 code = bb.methodpool.compile_cache(tmp)
                 if not code:
                     if filename is None:
-                        filename = "%s(e)" % name
+                        filename = "%s(e, d)" % name
                     code = compile(tmp, filename, "exec", ast.PyCF_ONLY_AST)
-                    if lineno is not None:
-                        ast.increment_lineno(code, lineno-1)
                     code = compile(code, filename, "exec")
                     bb.methodpool.compile_cache_add(tmp, code)
             except SyntaxError:
@@ -274,16 +295,16 @@ def register(name, handler, mask=None, filename=None, lineno=None, data=None):
                     _event_handler_map[m] = {}
                 _event_handler_map[m][name] = True
 
-        if data:
-            bbhands_mc = (data.getVar("__BBHANDLERS_MC") or [])
-            bbhands_mc.append(name)
+        if data is not None:
+            bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
+            bbhands_mc.add(name)
             data.setVar("__BBHANDLERS_MC", bbhands_mc)
 
         return Registered
 
 def remove(name, handler, data=None):
     """Remove an Event handler"""
-    if data:
+    if data is not None:
         if data.getVar("BB_CURRENT_MC"):
             mc = data.getVar("BB_CURRENT_MC")
             name = '%s%s' % (mc.replace('-', '_'), name)
@@ -295,8 +316,8 @@ def remove(name, handler, data=None):
         if name in _event_handler_map[event]:
             _event_handler_map[event].pop(name)
 
-    if data:
-        bbhands_mc = (data.getVar("__BBHANDLERS_MC") or [])
+    if data is not None:
+        bbhands_mc = (data.getVar("__BBHANDLERS_MC") or set())
         if name in bbhands_mc:
             bbhands_mc.remove(name)
             data.setVar("__BBHANDLERS_MC", bbhands_mc)
@@ -313,21 +334,23 @@ def set_eventfilter(func):
     _eventfilter = func
 
 def register_UIHhandler(handler, mainui=False):
-    bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1
-    _ui_handlers[_ui_handler_seq] = handler
-    level, debug_domains = bb.msg.constructLogOptions()
-    _ui_logfilters[_ui_handler_seq] = UIEventFilter(level, debug_domains)
-    if mainui:
-        global _uiready
-        _uiready = _ui_handler_seq
-    return _ui_handler_seq
+    with bb.utils.lock_timeout(_thread_lock):
+        bb.event._ui_handler_seq = bb.event._ui_handler_seq + 1
+        _ui_handlers[_ui_handler_seq] = handler
+        level, debug_domains = bb.msg.constructLogOptions()
+        _ui_logfilters[_ui_handler_seq] = UIEventFilter(level, debug_domains)
+        if mainui:
+            global _uiready
+            _uiready = _ui_handler_seq
+        return _ui_handler_seq
 
 def unregister_UIHhandler(handlerNum, mainui=False):
     if mainui:
         global _uiready
         _uiready = False
-    if handlerNum in _ui_handlers:
-        del _ui_handlers[handlerNum]
+    with bb.utils.lock_timeout(_thread_lock):
+        if handlerNum in _ui_handlers:
+            del _ui_handlers[handlerNum]
     return
 
 def get_uihandler():
@@ -408,6 +431,16 @@ class RecipeEvent(Event):
         self.fn = fn
         Event.__init__(self)
 
+class RecipePreDeferredInherits(RecipeEvent):
+    """
+    Called before deferred inherits are processed so code can snoop on class extensions for example
+    Limitations: It won't see inherits of inherited classes and the data is unexpanded
+    """
+    def __init__(self, fn, inherits):
+        self.fn = fn
+        self.inherits = inherits
+        Event.__init__(self)
+
 class RecipePreFinalise(RecipeEvent):
     """ Recipe Parsing Complete but not yet finalised"""
 
@@ -482,7 +515,7 @@ class BuildCompleted(BuildBase, OperationCompleted):
         BuildBase.__init__(self, n, p, failures)
 
 class DiskFull(Event):
-    """Disk full case build aborted"""
+    """Disk full case build halted"""
     def __init__(self, dev, type, freespace, mountpoint):
         Event.__init__(self)
         self._dev = dev
@@ -666,6 +699,17 @@ class ReachableStamps(Event):
         Event.__init__(self)
         self.stamps = stamps
 
+class StaleSetSceneTasks(Event):
+    """
+    An event listing setscene tasks which are 'stale' and will
+    be rerun. The metadata may use to clean up stale data.
+    tasks is a mapping of tasks and matching stale stamps.
+    """
+
+    def __init__(self, tasks):
+        Event.__init__(self)
+        self.tasks = tasks
+
 class FilesMatchingFound(Event):
     """
     Event when a list of files matching the supplied pattern has
@@ -732,13 +776,7 @@ class LogHandler(logging.Handler):
 
     def emit(self, record):
         if record.exc_info:
-            etype, value, tb = record.exc_info
-            if hasattr(tb, 'tb_next'):
-                tb = list(bb.exceptions.extract_traceback(tb, context=3))
-            # Need to turn the value into something the logging system can pickle
-            record.bb_exc_info = (etype, value, tb)
-            record.bb_exc_formatted = bb.exceptions.format_exception(etype, value, tb, limit=5)
-            value = str(value)
+            record.bb_exc_formatted = traceback.format_exception(*record.exc_info)
             record.exc_info = None
         fire(record, None)
 
@@ -749,7 +787,7 @@ class LogHandler(logging.Handler):
 class MetadataEvent(Event):
     """
     Generic event that target for OE-Core classes
-    to report information during asynchrous execution
+    to report information during asynchronous execution
     """
     def __init__(self, eventtype, eventdata):
         Event.__init__(self)
@@ -830,3 +868,19 @@ class FindSigInfoResult(Event):
     def __init__(self, result):
         Event.__init__(self)
         self.result = result
+
+class GetTaskSignatureResult(Event):
+    """
+    Event to return results from GetTaskSignatures command
+    """
+    def __init__(self, sig):
+        Event.__init__(self)
+        self.sig = sig
+
+class ParseError(Event):
+    """
+    Event to indicate parse failed
+    """
+    def __init__(self, msg):
+        super().__init__()
+        self._msg = msg
diff --git a/bitbake/lib/bb/exceptions.py b/bitbake/lib/bb/exceptions.py
deleted file mode 100644
index ecbad59970..0000000000
--- a/bitbake/lib/bb/exceptions.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#
-# SPDX-License-Identifier: GPL-2.0-only
-#
-
-import inspect
-import traceback
-import bb.namedtuple_with_abc
-from collections import namedtuple
-
-
-class TracebackEntry(namedtuple.abc):
-    """Pickleable representation of a traceback entry"""
-    _fields = 'filename lineno function args code_context index'
-    _header = '  File "{0.filename}", line {0.lineno}, in {0.function}{0.args}'
-
-    def format(self, formatter=None):
-        if not self.code_context:
-            return self._header.format(self) + '\n'
-
-        formatted = [self._header.format(self) + ':\n']
-
-        for lineindex, line in enumerate(self.code_context):
-            if formatter:
-                line = formatter(line)
-
-            if lineindex == self.index:
-                formatted.append('    >%s' % line)
-            else:
-                formatted.append('     %s' % line)
-        return formatted
-
-    def __str__(self):
-        return ''.join(self.format())
-
-def _get_frame_args(frame):
-    """Get the formatted arguments and class (if available) for a frame"""
-    arginfo = inspect.getargvalues(frame)
-
-    try:
-        if not arginfo.args:
-            return '', None
-    # There have been reports from the field of python 2.6 which doesn't 
-    # return a namedtuple here but simply a tuple so fallback gracefully if
-    # args isn't present.
-    except AttributeError:
-        return '', None
-
-    firstarg = arginfo.args[0]
-    if firstarg == 'self':
-        self = arginfo.locals['self']
-        cls = self.__class__.__name__
-
-        arginfo.args.pop(0)
-        del arginfo.locals['self']
-    else:
-        cls = None
-
-    formatted = inspect.formatargvalues(*arginfo)
-    return formatted, cls
-
-def extract_traceback(tb, context=1):
-    frames = inspect.getinnerframes(tb, context)
-    for frame, filename, lineno, function, code_context, index in frames:
-        formatted_args, cls = _get_frame_args(frame)
-        if cls:
-            function = '%s.%s' % (cls, function)
-        yield TracebackEntry(filename, lineno, function, formatted_args,
-                             code_context, index)
-
-def format_extracted(extracted, formatter=None, limit=None):
-    if limit:
-        extracted = extracted[-limit:]
-
-    formatted = []
-    for tracebackinfo in extracted:
-        formatted.extend(tracebackinfo.format(formatter))
-    return formatted
-
-
-def format_exception(etype, value, tb, context=1, limit=None, formatter=None):
-    formatted = ['Traceback (most recent call last):\n']
-
-    if hasattr(tb, 'tb_next'):
-        tb = extract_traceback(tb, context)
-
-    formatted.extend(format_extracted(tb, formatter, limit))
-    formatted.extend(traceback.format_exception_only(etype, value))
-    return formatted
-
-def to_string(exc):
-    if isinstance(exc, SystemExit):
-        if not isinstance(exc.code, str):
-            return 'Exited with "%d"' % exc.code
-    return str(exc)
diff --git a/bitbake/lib/bb/fetch2/README b/bitbake/lib/bb/fetch2/README
new file mode 100644
index 0000000000..67b787ef47
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/README
@@ -0,0 +1,57 @@
+There are expectations of users of the fetcher code. This file attempts to document
+some of the constraints that are present. Some are obvious, some are less so. It is
+documented in the context of how OE uses it but the API calls are generic.
+
+a) network access for sources is only expected to happen in the do_fetch task.
+   This is not enforced or tested but is required so that we can:
+
+   i) audit the sources used (i.e. for license/manifest reasons) 
+   ii) support offline builds with a suitable cache
+   iii) allow work to continue even with downtime upstream
+   iv) allow for changes upstream in incompatible ways
+   v) allow rebuilding of the software in X years time
+
+b) network access is not expected in do_unpack task.
+
+c) you can take DL_DIR and use it as a mirror for offline builds.
+
+d) access to the network is only made when explicitly configured in recipes
+   (e.g. use of AUTOREV, or use of git tags which change revision).
+
+e) fetcher output is deterministic (i.e. if you fetch configuration XXX now it 
+   will match in future exactly in a clean build with a new DL_DIR).
+   One specific pain point example are git tags. They can be replaced and change
+   so the git fetcher has to resolve them with the network. We use git revisions
+   where possible to avoid this and ensure determinism.
+
+f) network access is expected to work with the standard linux proxy variables
+   so that access behind firewalls works (the fetcher sets these in the 
+   environment but only in the do_fetch tasks).
+
+g) access during parsing has to be minimal, a "git ls-remote" for an AUTOREV 
+   git recipe might be ok but you can't expect to checkout a git tree.
+
+h) we need to provide revision information during parsing such that a version
+   for the recipe can be constructed.
+
+i) versions are expected to be able to increase in a way which sorts allowing 
+   package feeds to operate (see PR server required for git revisions to sort).
+
+j) API to query for possible version upgrades of a url is highly desireable to 
+   allow our automated upgrage code to function (it is implied this does always 
+   have network access).
+
+k) Where fixes or changes to behaviour in the fetcher are made, we ask that 
+   test cases are added (run with "bitbake-selftest bb.tests.fetch"). We do 
+   have fairly extensive test coverage of the fetcher as it is the only way
+   to track all of its corner cases, it still doesn't give entire coverage 
+   though sadly.
+   
+l) If using tools during parse time, they will have to be in ASSUME_PROVIDED
+   in OE's context as we can't build git-native, then parse a recipe and use
+   git ls-remote.
+
+Not all fetchers support all features, autorev is optional and doesn't make
+sense for some. Upgrade detection means different things in different contexts
+too.
+
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 19169d780f..0ad987c596 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -23,17 +23,18 @@ import collections
 import subprocess
 import pickle
 import errno
-import bb.persist_data, bb.utils
+import bb.utils
 import bb.checksum
 import bb.process
 import bb.event
 
 __version__ = "2"
 _checksum_cache = bb.checksum.FileChecksumCache()
+_revisions_cache = bb.checksum.RevisionsCache()
 
 logger = logging.getLogger("BitBake.Fetcher")
 
-CHECKSUM_LIST = [ "md5", "sha256", "sha1", "sha384", "sha512" ]
+CHECKSUM_LIST = [ "goh1", "md5", "sha256", "sha1", "sha384", "sha512" ]
 SHOWN_CHECKSUM_LIST = ["sha256"]
 
 class BBFetchException(Exception):
@@ -113,7 +114,7 @@ class MissingParameterError(BBFetchException):
         self.args = (missing, url)
 
 class ParameterError(BBFetchException):
-    """Exception raised when a url cannot be proccessed due to invalid parameters."""
+    """Exception raised when a url cannot be processed due to invalid parameters."""
     def __init__(self, message, url):
         msg = "URL: '%s' has invalid parameters. %s" % (url, message)
         self.url = url
@@ -182,7 +183,7 @@ class URI(object):
     Some notes about relative URIs: while it's specified that
     a URI beginning with <scheme>:// should either be directly
     followed by a hostname or a /, the old URI handling of the
-    fetch2 library did not comform to this. Therefore, this URI
+    fetch2 library did not conform to this. Therefore, this URI
     class has some kludges to make sure that URIs are parsed in
     a way comforming to bitbake's current usage. This URI class
     supports the following:
@@ -199,7 +200,7 @@ class URI(object):
      file://hostname/absolute/path.diff (would be IETF compliant)
 
     Note that the last case only applies to a list of
-    "whitelisted" schemes (currently only file://), that requires
+    explicitly allowed schemes (currently only file://), that requires
     its URIs to not have a network location.
     """
 
@@ -237,7 +238,7 @@ class URI(object):
         # to RFC compliant URL format. E.g.:
         #   file://foo.diff -> file:foo.diff
         if urlp.scheme in self._netloc_forbidden:
-            uri = re.sub("(?<=:)//(?!/)", "", uri, 1)
+            uri = re.sub(r"(?<=:)//(?!/)", "", uri, count=1)
             reparse = 1
 
         if reparse:
@@ -290,12 +291,12 @@ class URI(object):
 
     def _param_str_split(self, string, elmdelim, kvdelim="="):
         ret = collections.OrderedDict()
-        for k, v in [x.split(kvdelim, 1) for x in string.split(elmdelim) if x]:
+        for k, v in [x.split(kvdelim, 1) if kvdelim in x else (x, None) for x in string.split(elmdelim) if x]:
             ret[k] = v
         return ret
 
     def _param_str_join(self, dict_, elmdelim, kvdelim="="):
-        return elmdelim.join([kvdelim.join([k, v]) for k, v in dict_.items()])
+        return elmdelim.join([kvdelim.join([k, v]) if v else k for k, v in dict_.items()])
 
     @property
     def hostport(self):
@@ -352,6 +353,14 @@ def decodeurl(url):
     user, password, parameters).
     """
 
+    uri = URI(url)
+    path = uri.path if uri.path else "/"
+    return uri.scheme, uri.hostport, path, uri.username, uri.password, uri.params
+
+def decodemirrorurl(url):
+    """Decodes a mirror URL into the tokens (scheme, network location, path,
+    user, password, parameters).
+    """
     m = re.compile('(?P<type>[^:]*)://((?P<user>[^/;]+)@)?(?P<location>[^;]+)(;(?P<parm>.*))?').match(url)
     if not m:
         raise MalformedUrl(url)
@@ -370,6 +379,9 @@ def decodeurl(url):
     elif type.lower() == 'file':
         host = ""
         path = location
+        if user:
+            path = user + '@' + path
+            user = ""
     else:
         host = location
         path = "/"
@@ -388,7 +400,7 @@ def decodeurl(url):
             if s:
                 if not '=' in s:
                     raise MalformedUrl(url, "The URL: '%s' is invalid: parameter %s does not specify a value (missing '=')" % (url, s))
-                s1, s2 = s.split('=')
+                s1, s2 = s.split('=', 1)
                 p[s1] = s2
 
     return type, host, urllib.parse.unquote(path), user, pswd, p
@@ -402,34 +414,37 @@ def encodeurl(decoded):
 
     if not type:
         raise MissingParameterError('type', "encoded from the data %s" % str(decoded))
-    url = '%s://' % type
+    uri = URI()
+    uri.scheme = type
     if user and type != "file":
-        url += "%s" % user
+        uri.username = user
         if pswd:
-            url += ":%s" % pswd
-        url += "@"
+            uri.password = pswd
     if host and type != "file":
-        url += "%s" % host
+        uri.hostname = host
     if path:
         # Standardise path to ensure comparisons work
         while '//' in path:
             path = path.replace("//", "/")
-        url += "%s" % urllib.parse.quote(path)
+        uri.path = path
+        if type == "file":
+            # Use old not IETF compliant style
+            uri.relative = False
     if p:
-        for parm in p:
-            url += ";%s=%s" % (parm, p[parm])
+        uri.params = p
 
-    return url
+    return str(uri)
 
 def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
     if not ud.url or not uri_find or not uri_replace:
         logger.error("uri_replace: passed an undefined value, not replacing")
         return None
-    uri_decoded = list(decodeurl(ud.url))
-    uri_find_decoded = list(decodeurl(uri_find))
-    uri_replace_decoded = list(decodeurl(uri_replace))
+    uri_decoded = list(decodemirrorurl(ud.url))
+    uri_find_decoded = list(decodemirrorurl(uri_find))
+    uri_replace_decoded = list(decodemirrorurl(uri_replace))
     logger.debug2("For url %s comparing %s to %s" % (uri_decoded, uri_find_decoded, uri_replace_decoded))
     result_decoded = ['', '', '', '', '', {}]
+    # 0 - type, 1 - host, 2 - path, 3 - user,  4- pswd, 5 - params
     for loc, i in enumerate(uri_find_decoded):
         result_decoded[loc] = uri_decoded[loc]
         regexp = i
@@ -449,6 +464,9 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
                 for l in replacements:
                     uri_replace_decoded[loc][k] = uri_replace_decoded[loc][k].replace(l, replacements[l])
                 result_decoded[loc][k] = uri_replace_decoded[loc][k]
+        elif (loc == 3 or loc == 4) and uri_replace_decoded[loc]:
+            # User/password in the replacement is just a straight replacement
+            result_decoded[loc] = uri_replace_decoded[loc]
         elif (re.match(regexp, uri_decoded[loc])):
             if not uri_replace_decoded[loc]:
                 result_decoded[loc] = ""
@@ -456,7 +474,7 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
                 for k in replacements:
                     uri_replace_decoded[loc] = uri_replace_decoded[loc].replace(k, replacements[k])
                 #bb.note("%s %s %s" % (regexp, uri_replace_decoded[loc], uri_decoded[loc]))
-                result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], 1)
+                result_decoded[loc] = re.sub(regexp, uri_replace_decoded[loc], uri_decoded[loc], count=1)
             if loc == 2:
                 # Handle path manipulations
                 basename = None
@@ -465,10 +483,18 @@ def uri_replace(ud, uri_find, uri_replace, replacements, d, mirrortarball=None):
                     basename = os.path.basename(mirrortarball)
                     # Kill parameters, they make no sense for mirror tarballs
                     uri_decoded[5] = {}
+                    uri_find_decoded[5] = {}
                 elif ud.localpath and ud.method.supports_checksum(ud):
                     basename = os.path.basename(ud.localpath)
-                if basename and not result_decoded[loc].endswith(basename):
-                    result_decoded[loc] = os.path.join(result_decoded[loc], basename)
+                if basename:
+                    uri_basename = os.path.basename(uri_decoded[loc])
+                    # Prefix with a slash as a sentinel in case
+                    # result_decoded[loc] does not contain one.
+                    path = "/" + result_decoded[loc]
+                    if uri_basename and basename != uri_basename and path.endswith("/" + uri_basename):
+                        result_decoded[loc] = path[1:-len(uri_basename)] + basename
+                    elif not path.endswith("/" + basename):
+                        result_decoded[loc] = os.path.join(path[1:], basename)
         else:
             return None
     result = encodeurl(result_decoded)
@@ -481,18 +507,23 @@ methods = []
 urldata_cache = {}
 saved_headrevs = {}
 
-def fetcher_init(d):
+def fetcher_init(d, servercontext=True):
     """
     Called to initialize the fetchers once the configuration data is known.
     Calls before this must not hit the cache.
     """
 
-    revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
+    _checksum_cache.init_cache(d.getVar("BB_CACHEDIR"))
+    _revisions_cache.init_cache(d.getVar("BB_CACHEDIR"))
+
+    if not servercontext:
+        return
+
     try:
         # fetcher_init is called multiple times, so make sure we only save the
         # revs the first time it is called.
         if not bb.fetch2.saved_headrevs:
-            bb.fetch2.saved_headrevs = dict(revs)
+            bb.fetch2.saved_headrevs = _revisions_cache.get_revs()
     except:
         pass
 
@@ -502,11 +533,10 @@ def fetcher_init(d):
         logger.debug("Keeping SRCREV cache due to cache policy of: %s", srcrev_policy)
     elif srcrev_policy == "clear":
         logger.debug("Clearing SRCREV cache due to cache policy of: %s", srcrev_policy)
-        revs.clear()
+        _revisions_cache.clear_cache()
     else:
         raise FetchError("Invalid SRCREV cache policy of: %s" % srcrev_policy)
 
-    _checksum_cache.init_cache(d)
 
     for m in methods:
         if hasattr(m, "init"):
@@ -514,9 +544,11 @@ def fetcher_init(d):
 
 def fetcher_parse_save():
     _checksum_cache.save_extras()
+    _revisions_cache.save_extras()
 
 def fetcher_parse_done():
     _checksum_cache.save_merge()
+    _revisions_cache.save_merge()
 
 def fetcher_compare_revisions(d):
     """
@@ -524,7 +556,7 @@ def fetcher_compare_revisions(d):
     when bitbake was started and return true if they have changed.
     """
 
-    headrevs = dict(bb.persist_data.persist('BB_URI_HEADREVS', d))
+    headrevs = _revisions_cache.get_revs()
     return headrevs != bb.fetch2.saved_headrevs
 
 def mirror_from_string(data):
@@ -534,7 +566,7 @@ def mirror_from_string(data):
         bb.warn('Invalid mirror data %s, should have paired members.' % data)
     return list(zip(*[iter(mirrors)]*2))
 
-def verify_checksum(ud, d, precomputed={}):
+def verify_checksum(ud, d, precomputed={}, localpath=None, fatal_nochecksum=True):
     """
     verify the MD5 and SHA256 checksum for downloaded src
 
@@ -548,20 +580,25 @@ def verify_checksum(ud, d, precomputed={}):
     file against those in the recipe each time, rather than only after
     downloading. See https://bugzilla.yoctoproject.org/show_bug.cgi?id=5571.
     """
-
     if ud.ignore_checksums or not ud.method.supports_checksum(ud):
         return {}
 
+    if localpath is None:
+        localpath = ud.localpath
+
     def compute_checksum_info(checksum_id):
         checksum_name = getattr(ud, "%s_name" % checksum_id)
 
         if checksum_id in precomputed:
             checksum_data = precomputed[checksum_id]
         else:
-            checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(ud.localpath)
+            checksum_data = getattr(bb.utils, "%s_file" % checksum_id)(localpath)
 
         checksum_expected = getattr(ud, "%s_expected" % checksum_id)
 
+        if checksum_expected == '':
+            checksum_expected = None
+
         return {
             "id": checksum_id,
             "name": checksum_name,
@@ -581,17 +618,13 @@ def verify_checksum(ud, d, precomputed={}):
             checksum_lines = ["SRC_URI[%s] = \"%s\"" % (ci["name"], ci["data"])]
 
     # If no checksum has been provided
-    if ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
+    if fatal_nochecksum and ud.method.recommends_checksum(ud) and all(ci["expected"] is None for ci in checksum_infos):
         messages = []
         strict = d.getVar("BB_STRICT_CHECKSUM") or "0"
 
         # If strict checking enabled and neither sum defined, raise error
         if strict == "1":
-            messages.append("No checksum specified for '%s', please add at " \
-                            "least one to the recipe:" % ud.localpath)
-            messages.extend(checksum_lines)
-            logger.error("\n".join(messages))
-            raise NoChecksumError("Missing SRC_URI checksum", ud.url)
+            raise NoChecksumError("\n".join(checksum_lines))
 
         bb.event.fire(MissingChecksumEvent(ud.url, **checksum_event), d)
 
@@ -612,8 +645,8 @@ def verify_checksum(ud, d, precomputed={}):
 
     for ci in checksum_infos:
         if ci["expected"] and ci["expected"] != ci["data"]:
-            messages.append("File: '%s' has %s checksum %s when %s was " \
-                            "expected" % (ud.localpath, ci["id"], ci["data"], ci["expected"]))
+            messages.append("File: '%s' has %s checksum '%s' when '%s' was " \
+                            "expected" % (localpath, ci["id"], ci["data"], ci["expected"]))
             bad_checksum = ci["data"]
 
     if bad_checksum:
@@ -731,13 +764,16 @@ def subprocess_setup():
     # SIGPIPE errors are known issues with gzip/bash
     signal.signal(signal.SIGPIPE, signal.SIG_DFL)
 
-def get_autorev(d):
-    #  only not cache src rev in autorev case
+def mark_recipe_nocache(d):
     if d.getVar('BB_SRCREV_POLICY') != "cache":
         d.setVar('BB_DONT_CACHE', '1')
+
+def get_autorev(d):
+    mark_recipe_nocache(d)
+    d.setVar("__BBAUTOREV_SEEN", True)
     return "AUTOINC"
 
-def get_srcrev(d, method_name='sortable_revision'):
+def _get_srcrev(d, method_name='sortable_revision'):
     """
     Return the revision string, usually for use in the version string (PV) of the current package
     Most packages usually only have one SCM so we just pass on the call.
@@ -751,23 +787,34 @@ def get_srcrev(d, method_name='sortable_revision'):
     that fetcher provides a method with the given name and the same signature as sortable_revision.
     """
 
+    d.setVar("__BBSRCREV_SEEN", "1")
+    recursion = d.getVar("__BBINSRCREV")
+    if recursion:
+        raise FetchError("There are recursive references in fetcher variables, likely through SRC_URI")
+    d.setVar("__BBINSRCREV", True)
+
     scms = []
+    revs = []
     fetcher = Fetch(d.getVar('SRC_URI').split(), d)
     urldata = fetcher.ud
     for u in urldata:
         if urldata[u].method.supports_srcrev():
             scms.append(u)
 
-    if len(scms) == 0:
-        raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
+    if not scms:
+        d.delVar("__BBINSRCREV")
+        return "", revs
 
-    if len(scms) == 1 and len(urldata[scms[0]].names) == 1:
-        autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].names[0])
+
+    if len(scms) == 1:
+        autoinc, rev = getattr(urldata[scms[0]].method, method_name)(urldata[scms[0]], d, urldata[scms[0]].name)
+        revs.append(rev)
         if len(rev) > 10:
             rev = rev[:10]
+        d.delVar("__BBINSRCREV")
         if autoinc:
-            return "AUTOINC+" + rev
-        return rev
+            return "AUTOINC+" + rev, revs
+        return rev, revs
 
     #
     # Mutiple SCMs are in SRC_URI so we resort to SRCREV_FORMAT
@@ -781,12 +828,12 @@ def get_srcrev(d, method_name='sortable_revision'):
     seenautoinc = False
     for scm in scms:
         ud = urldata[scm]
-        for name in ud.names:
-            autoinc, rev = getattr(ud.method, method_name)(ud, d, name)
-            seenautoinc = seenautoinc or autoinc
-            if len(rev) > 10:
-                rev = rev[:10]
-            name_to_rev[name] = rev
+        autoinc, rev = getattr(ud.method, method_name)(ud, d, ud.name)
+        revs.append(rev)
+        seenautoinc = seenautoinc or autoinc
+        if len(rev) > 10:
+            rev = rev[:10]
+        name_to_rev[ud.name] = rev
     # Replace names by revisions in the SRCREV_FORMAT string. The approach used
     # here can handle names being prefixes of other names and names appearing
     # as substrings in revisions (in which case the name should not be
@@ -799,12 +846,71 @@ def get_srcrev(d, method_name='sortable_revision'):
     if seenautoinc:
         format = "AUTOINC+" + format
 
-    return format
+    d.delVar("__BBINSRCREV")
+    return format, revs
+
+def get_hashvalue(d, method_name='sortable_revision'):
+    pkgv, revs = _get_srcrev(d, method_name=method_name)
+    return " ".join(revs)
+
+def get_pkgv_string(d, method_name='sortable_revision'):
+    pkgv, revs = _get_srcrev(d, method_name=method_name)
+    return pkgv
+
+def get_srcrev(d, method_name='sortable_revision'):
+    pkgv, revs = _get_srcrev(d, method_name=method_name)
+    if not pkgv:
+        raise FetchError("SRCREV was used yet no valid SCM was found in SRC_URI")
+    return pkgv
 
 def localpath(url, d):
     fetcher = bb.fetch2.Fetch([url], d)
     return fetcher.localpath(url)
 
+# Need to export PATH as binary could be in metadata paths
+# rather than host provided
+# Also include some other variables.
+FETCH_EXPORT_VARS = ['HOME', 'PATH',
+                     'HTTP_PROXY', 'http_proxy',
+                     'HTTPS_PROXY', 'https_proxy',
+                     'FTP_PROXY', 'ftp_proxy',
+                     'FTPS_PROXY', 'ftps_proxy',
+                     'NO_PROXY', 'no_proxy',
+                     'ALL_PROXY', 'all_proxy',
+                     'GIT_PROXY_COMMAND',
+                     'GIT_SSH',
+                     'GIT_SSH_COMMAND',
+                     'GIT_SSL_CAINFO',
+                     'GIT_SMART_HTTP',
+                     'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
+                     'SOCKS5_USER', 'SOCKS5_PASSWD',
+                     'DBUS_SESSION_BUS_ADDRESS',
+                     'P4CONFIG',
+                     'SSL_CERT_FILE',
+                     'NODE_EXTRA_CA_CERTS',
+                     'AWS_PROFILE',
+                     'AWS_ACCESS_KEY_ID',
+                     'AWS_SECRET_ACCESS_KEY',
+                     'AWS_ROLE_ARN',
+                     'AWS_WEB_IDENTITY_TOKEN_FILE',
+                     'AWS_DEFAULT_REGION',
+                     'AWS_SESSION_TOKEN',
+                     'GIT_CACHE_PATH',
+                     'REMOTE_CONTAINERS_IPC',
+                     'GITHUB_TOKEN',
+                     'SSL_CERT_DIR']
+
+def get_fetcher_environment(d):
+    newenv = {}
+    origenv = d.getVar("BB_ORIGENV")
+    for name in bb.fetch2.FETCH_EXPORT_VARS:
+        value = d.getVar(name)
+        if not value and origenv:
+            value = origenv.getVar(name)
+        if value:
+            newenv[name] = value
+    return newenv
+
 def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
     """
     Run cmd returning the command output
@@ -813,25 +919,7 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
     Optionally remove the files/directories listed in cleanup upon failure
     """
 
-    # Need to export PATH as binary could be in metadata paths
-    # rather than host provided
-    # Also include some other variables.
-    # FIXME: Should really include all export varaiables?
-    exportvars = ['HOME', 'PATH',
-                  'HTTP_PROXY', 'http_proxy',
-                  'HTTPS_PROXY', 'https_proxy',
-                  'FTP_PROXY', 'ftp_proxy',
-                  'FTPS_PROXY', 'ftps_proxy',
-                  'NO_PROXY', 'no_proxy',
-                  'ALL_PROXY', 'all_proxy',
-                  'GIT_PROXY_COMMAND',
-                  'GIT_SSH',
-                  'GIT_SSL_CAINFO',
-                  'GIT_SMART_HTTP',
-                  'SSH_AUTH_SOCK', 'SSH_AGENT_PID',
-                  'SOCKS5_USER', 'SOCKS5_PASSWD',
-                  'DBUS_SESSION_BUS_ADDRESS',
-                  'P4CONFIG']
+    exportvars = FETCH_EXPORT_VARS
 
     if not cleanup:
         cleanup = []
@@ -868,14 +956,17 @@ def runfetchcmd(cmd, d, quiet=False, cleanup=None, log=None, workdir=None):
         (output, errors) = bb.process.run(cmd, log=log, shell=True, stderr=subprocess.PIPE, cwd=workdir)
         success = True
     except bb.process.NotFoundError as e:
-        error_message = "Fetch command %s" % (e.command)
+        error_message = "Fetch command %s not found" % (e.command)
     except bb.process.ExecutionError as e:
         if e.stdout:
             output = "output:\n%s\n%s" % (e.stdout, e.stderr)
         elif e.stderr:
             output = "output:\n%s" % e.stderr
         else:
-            output = "no output"
+            if log:
+                output = "see logfile for output"
+            else:
+                output = "no output"
         error_message = "Fetch command %s failed with exit code %s, %s" % (e.command, e.exitcode, output)
     except bb.process.CmdError as e:
         error_message = "Fetch command %s could not be run:\n%s" % (e.command, e.msg)
@@ -937,6 +1028,7 @@ def build_mirroruris(origud, mirrors, ld):
 
                 try:
                     newud = FetchData(newuri, ld)
+                    newud.ignore_checksums = True
                     newud.setup_localpath(ld)
                 except bb.fetch2.BBFetchException as e:
                     logger.debug("Mirror fetch failure for url %s (original url: %s)" % (newuri, origud.url))
@@ -1000,6 +1092,10 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
         # If that tarball is a local file:// we need to provide a symlink to it
         dldir = ld.getVar("DL_DIR")
 
+        if bb.utils.to_boolean(ld.getVar("BB_FETCH_PREMIRRORONLY")):
+            ld = ld.createCopy()
+            ld.setVar("BB_NO_NETWORK", "1")
+
         if origud.mirrortarballs and os.path.basename(ud.localpath) in origud.mirrortarballs and os.path.basename(ud.localpath) != os.path.basename(origud.localpath):
             # Create donestamp in old format to avoid triggering a re-download
             if ud.donestamp:
@@ -1021,7 +1117,10 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
                     origud.method.build_mirror_data(origud, ld)
             return origud.localpath
         # Otherwise the result is a local file:// and we symlink to it
-        ensure_symlink(ud.localpath, origud.localpath)
+        # This may also be a link to a shallow archive
+        # When using shallow mode, add a symlink to the original fullshallow
+        # path to ensure a valid symlink even in the `PREMIRRORS` case
+        origud.method.update_mirror_links(ud, origud)
         update_stamp(origud, ld)
         return ud.localpath
 
@@ -1046,7 +1145,8 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
             logger.debug("Mirror fetch failure for url %s (original url: %s)" % (ud.url, origud.url))
             logger.debug(str(e))
         try:
-            ud.method.clean(ud, ld)
+            if ud.method.cleanup_upon_failure():
+                ud.method.clean(ud, ld)
         except UnboundLocalError:
             pass
         return False
@@ -1054,23 +1154,6 @@ def try_mirror_url(fetch, origud, ud, ld, check = False):
         if ud.lockfile and ud.lockfile != origud.lockfile:
             bb.utils.unlockfile(lf)
 
-
-def ensure_symlink(target, link_name):
-    if not os.path.exists(link_name):
-        if os.path.islink(link_name):
-            # Broken symbolic link
-            os.unlink(link_name)
-
-        # In case this is executing without any file locks held (as is
-        # the case for file:// URLs), two tasks may end up here at the
-        # same time, in which case we do not want the second task to
-        # fail when the link has already been created by the first task.
-        try:
-            os.symlink(target, link_name)
-        except FileExistsError:
-            pass
-
-
 def try_mirrors(fetch, d, origud, mirrors, check = False):
     """
     Try to use a mirrored version of the sources.
@@ -1099,7 +1182,7 @@ def trusted_network(d, url):
     if bb.utils.to_boolean(d.getVar("BB_NO_NETWORK")):
         return True
 
-    pkgname = d.expand(d.getVar('PN', False))
+    pkgname = d.getVar('PN')
     trusted_hosts = None
     if pkgname:
         trusted_hosts = d.getVarFlag('BB_ALLOWED_NETWORKS', pkgname, False)
@@ -1140,11 +1223,11 @@ def srcrev_internal_helper(ud, d, name):
     pn = d.getVar("PN")
     attempts = []
     if name != '' and pn:
-        attempts.append("SRCREV_%s_pn-%s" % (name, pn))
+        attempts.append("SRCREV_%s:pn-%s" % (name, pn))
     if name != '':
         attempts.append("SRCREV_%s" % name)
     if pn:
-        attempts.append("SRCREV_pn-%s" % pn)
+        attempts.append("SRCREV:pn-%s" % pn)
     attempts.append("SRCREV")
 
     for a in attempts:
@@ -1152,23 +1235,21 @@ def srcrev_internal_helper(ud, d, name):
         if srcrev and srcrev != "INVALID":
             break
 
-    if 'rev' in ud.parm and 'tag' in ud.parm:
-        raise FetchError("Please specify a ;rev= parameter or a ;tag= parameter in the url %s but not both." % (ud.url))
-
-    if 'rev' in ud.parm or 'tag' in ud.parm:
-        if 'rev' in ud.parm:
-            parmrev = ud.parm['rev']
-        else:
-            parmrev = ud.parm['tag']
+    if 'rev' in ud.parm:
+        parmrev = ud.parm['rev']
         if srcrev == "INVALID" or not srcrev:
             return parmrev
         if srcrev != parmrev:
             raise FetchError("Conflicting revisions (%s from SRCREV and %s from the url) found, please specify one valid value" % (srcrev, parmrev))
         return parmrev
 
+    if 'tag' in ud.parm and (srcrev == "INVALID" or not srcrev):
+        return ud.parm['tag']
+
     if srcrev == "INVALID" or not srcrev:
         raise FetchError("Please set a valid SRCREV for url %s (possible key names are %s, or use a ;rev=X URL parameter)" % (str(attempts), ud.url), ud.url)
     if srcrev == "AUTOINC":
+        d.setVar("__BBAUTOREV_ACTED_UPON", True)
         srcrev = ud.method.latest_revision(ud, d, name)
 
     return srcrev
@@ -1180,23 +1261,21 @@ def get_checksum_file_list(d):
     SRC_URI as a space-separated string
     """
     fetch = Fetch([], d, cache = False, localonly = True)
-
-    dl_dir = d.getVar('DL_DIR')
     filelist = []
     for u in fetch.urls:
         ud = fetch.ud[u]
-
         if ud and isinstance(ud.method, local.Local):
-            paths = ud.method.localpaths(ud, d)
+            found = False
+            paths = ud.method.localfile_searchpaths(ud, d)
             for f in paths:
-                pth = ud.decodedurl
-                if f.startswith(dl_dir):
-                    # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else
-                    if os.path.exists(f):
-                        bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN'), os.path.basename(f)))
-                    else:
-                        bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN'), os.path.basename(f)))
+                pth = ud.path
+                if os.path.exists(f):
+                    found = True
                 filelist.append(f + ":" + str(os.path.exists(f)))
+            if not found:
+                bb.fatal(("Unable to get checksum for %s SRC_URI entry %s: file could not be found"
+                            "\nThe following paths were searched:"
+                            "\n%s") % (d.getVar('PN'), os.path.basename(f), '\n'.join(paths)))
 
     return " ".join(filelist)
 
@@ -1234,28 +1313,28 @@ class FetchData(object):
         self.setup = False
 
         def configure_checksum(checksum_id):
+            checksum_plain_name = "%ssum" % checksum_id
             if "name" in self.parm:
                 checksum_name = "%s.%ssum" % (self.parm["name"], checksum_id)
             else:
-                checksum_name = "%ssum" % checksum_id
-
-            setattr(self, "%s_name" % checksum_id, checksum_name)
+                checksum_name = checksum_plain_name
 
             if checksum_name in self.parm:
                 checksum_expected = self.parm[checksum_name]
-            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3"]:
+            elif checksum_plain_name in self.parm:
+                checksum_expected = self.parm[checksum_plain_name]
+                checksum_name = checksum_plain_name
+            elif self.type not in ["http", "https", "ftp", "ftps", "sftp", "s3", "az", "crate", "gs", "gomod", "npm"]:
                 checksum_expected = None
             else:
                 checksum_expected = d.getVarFlag("SRC_URI", checksum_name)
 
+            setattr(self, "%s_name" % checksum_id, checksum_name)
             setattr(self, "%s_expected" % checksum_id, checksum_expected)
 
-        for checksum_id in CHECKSUM_LIST:
-            configure_checksum(checksum_id)
-
-        self.ignore_checksums = False
-
-        self.names = self.parm.get("name",'default').split(',')
+        self.name = self.parm.get("name",'default')
+        if "," in self.name:
+            raise ParameterError("The fetcher no longer supports multiple name parameters in a single url", self.url)
 
         self.method = None
         for m in methods:
@@ -1276,6 +1355,11 @@ class FetchData(object):
         if hasattr(self.method, "urldata_init"):
             self.method.urldata_init(self, d)
 
+        for checksum_id in CHECKSUM_LIST:
+            configure_checksum(checksum_id)
+
+        self.ignore_checksums = False
+
         if "localpath" in self.parm:
             # if user sets localpath for file, use it instead.
             self.localpath = self.parm["localpath"]
@@ -1302,13 +1386,7 @@ class FetchData(object):
         self.lockfile = basepath + '.lock'
 
     def setup_revisions(self, d):
-        self.revisions = {}
-        for name in self.names:
-            self.revisions[name] = srcrev_internal_helper(self, d, name)
-
-        # add compatibility code for non name specified case
-        if len(self.names) == 1:
-            self.revision = self.revisions[self.names[0]]
+        self.revision = srcrev_internal_helper(self, d, self.name)
 
     def setup_localpath(self, d):
         if not self.localpath:
@@ -1355,6 +1433,9 @@ class FetchMethod(object):
         Is localpath something that can be represented by a checksum?
         """
 
+        # We cannot compute checksums for None
+        if urldata.localpath is None:
+            return False
         # We cannot compute checksums for directories
         if os.path.isdir(urldata.localpath):
             return False
@@ -1367,6 +1448,12 @@ class FetchMethod(object):
         """
         return False
 
+    def cleanup_upon_failure(self):
+        """
+        When a fetch fails, should clean() be called?
+        """
+        return True
+
     def verify_donestamp(self, ud, d):
         """
         Verify the donestamp file
@@ -1427,37 +1514,40 @@ class FetchMethod(object):
                      (file, urldata.parm.get('unpack')))
 
         base, ext = os.path.splitext(file)
-        if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz']:
+        if ext in ['.gz', '.bz2', '.Z', '.xz', '.lz', '.zst']:
             efile = os.path.join(rootdir, os.path.basename(base))
         else:
             efile = file
         cmd = None
 
         if unpack:
+            tar_cmd = 'tar --extract --no-same-owner'
+            if 'striplevel' in urldata.parm:
+                tar_cmd += ' --strip-components=%s' %  urldata.parm['striplevel']
             if file.endswith('.tar'):
-                cmd = 'tar x --no-same-owner -f %s' % file
+                cmd = '%s -f %s' % (tar_cmd, file)
             elif file.endswith('.tgz') or file.endswith('.tar.gz') or file.endswith('.tar.Z'):
-                cmd = 'tar xz --no-same-owner -f %s' % file
+                cmd = '%s -z -f %s' % (tar_cmd, file)
             elif file.endswith('.tbz') or file.endswith('.tbz2') or file.endswith('.tar.bz2'):
-                cmd = 'bzip2 -dc %s | tar x --no-same-owner -f -' % file
+                cmd = 'bzip2 -dc %s | %s -f -' % (file, tar_cmd)
             elif file.endswith('.gz') or file.endswith('.Z') or file.endswith('.z'):
                 cmd = 'gzip -dc %s > %s' % (file, efile)
             elif file.endswith('.bz2'):
                 cmd = 'bzip2 -dc %s > %s' % (file, efile)
             elif file.endswith('.txz') or file.endswith('.tar.xz'):
-                cmd = 'xz -dc %s | tar x --no-same-owner -f -' % file
+                cmd = 'xz -dc %s | %s -f -' % (file, tar_cmd)
             elif file.endswith('.xz'):
                 cmd = 'xz -dc %s > %s' % (file, efile)
             elif file.endswith('.tar.lz'):
-                cmd = 'lzip -dc %s | tar x --no-same-owner -f -' % file
+                cmd = 'lzip -dc %s | %s -f -' % (file, tar_cmd)
             elif file.endswith('.lz'):
                 cmd = 'lzip -dc %s > %s' % (file, efile)
             elif file.endswith('.tar.7z'):
-                cmd = '7z x -so %s | tar x --no-same-owner -f -' % file
+                cmd = '7z x -so %s | %s -f -' % (file, tar_cmd)
             elif file.endswith('.7z'):
                 cmd = '7za x -y %s 1>/dev/null' % file
             elif file.endswith('.tzst') or file.endswith('.tar.zst'):
-                cmd = 'zstd --decompress --stdout %s | tar x --no-same-owner -f -' % file
+                cmd = 'zstd --decompress --stdout %s | %s -f -' % (file, tar_cmd)
             elif file.endswith('.zst'):
                 cmd = 'zstd --decompress --stdout %s > %s' % (file, efile)
             elif file.endswith('.zip') or file.endswith('.jar'):
@@ -1483,14 +1573,14 @@ class FetchMethod(object):
                 datafile = None
                 if output:
                     for line in output.decode().splitlines():
-                        if line.startswith('data.tar.'):
+                        if line.startswith('data.tar.') or line == 'data.tar':
                             datafile = line
                             break
                     else:
-                        raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar.* file", urldata.url)
+                        raise UnpackError("Unable to unpack deb/ipk package - does not contain data.tar* file", urldata.url)
                 else:
                     raise UnpackError("Unable to unpack deb/ipk package - could not list contents", urldata.url)
-                cmd = 'ar x %s %s && tar --no-same-owner -xpf %s && rm %s' % (file, datafile, datafile, datafile)
+                cmd = 'ar x %s %s && %s -p -f %s && rm %s' % (file, datafile, tar_cmd, datafile, datafile)
 
         # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
         if 'subdir' in urldata.parm:
@@ -1506,6 +1596,7 @@ class FetchMethod(object):
             unpackdir = rootdir
 
         if not unpack or not cmd:
+            urldata.unpack_tracer.unpack("file-copy", unpackdir)
             # If file == dest, then avoid any copies, as we already put the file into dest!
             dest = os.path.join(unpackdir, os.path.basename(file))
             if file != dest and not (os.path.exists(dest) and os.path.samefile(file, dest)):
@@ -1519,7 +1610,9 @@ class FetchMethod(object):
                     if urlpath.find("/") != -1:
                         destdir = urlpath.rsplit("/", 1)[0] + '/'
                         bb.utils.mkdirhier("%s/%s" % (unpackdir, destdir))
-                cmd = 'cp -fpPRH "%s" "%s"' % (file, destdir)
+                cmd = 'cp --force --preserve=timestamps --no-dereference --recursive -H "%s" "%s"' % (file, destdir)
+        else:
+            urldata.unpack_tracer.unpack("archive-extract", unpackdir)
 
         if not cmd:
             return
@@ -1546,6 +1639,28 @@ class FetchMethod(object):
         """
         bb.utils.remove(urldata.localpath)
 
+    def ensure_symlink(self, target, link_name):
+        if not os.path.exists(link_name):
+            dirname = os.path.dirname(link_name)
+            bb.utils.mkdirhier(dirname)
+            if os.path.islink(link_name):
+                # Broken symbolic link
+                os.unlink(link_name)
+
+            # In case this is executing without any file locks held (as is
+            # the case for file:// URLs), two tasks may end up here at the
+            # same time, in which case we do not want the second task to
+            # fail when the link has already been created by the first task.
+            try:
+                os.symlink(target, link_name)
+            except FileExistsError:
+                pass
+
+    def update_mirror_links(self, ud, origud):
+        # For local file:// results, create a symlink to them
+        # This may also be a link to a shallow archive
+        self.ensure_symlink(ud.localpath, origud.localpath)
+
     def try_premirror(self, urldata, d):
         """
         Should premirrors be used?
@@ -1573,13 +1688,13 @@ class FetchMethod(object):
         if not hasattr(self, "_latest_revision"):
             raise ParameterError("The fetcher for this URL does not support _latest_revision", ud.url)
 
-        revs = bb.persist_data.persist('BB_URI_HEADREVS', d)
         key = self.generate_revision_key(ud, d, name)
-        try:
-            return revs[key]
-        except KeyError:
-            revs[key] = rev = self._latest_revision(ud, d, name)
-            return rev
+
+        rev = _revisions_cache.get_rev(key)
+        if rev is None:
+            rev = self._latest_revision(ud, d, name)
+            _revisions_cache.set_rev(key, rev)
+        return rev
 
     def sortable_revision(self, ud, d, name):
         latest_rev = self._build_revision(ud, d, name)
@@ -1611,12 +1726,61 @@ class FetchMethod(object):
         """
         return []
 
+
+class DummyUnpackTracer(object):
+    """
+    Abstract API definition for a class that traces unpacked source files back
+    to their respective upstream SRC_URI entries, for software composition
+    analysis, license compliance and detailed SBOM generation purposes.
+    User may load their own unpack tracer class (instead of the dummy
+    one) by setting the BB_UNPACK_TRACER_CLASS config parameter.
+    """
+    def start(self, unpackdir, urldata_dict, d):
+        """
+        Start tracing the core Fetch.unpack process, using an index to map
+        unpacked files to each SRC_URI entry.
+        This method is called by Fetch.unpack and it may receive nested calls by
+        gitsm and npmsw fetchers, that expand SRC_URI entries by adding implicit
+        URLs and by recursively calling Fetch.unpack from new (nested) Fetch
+        instances.
+        """
+        return
+    def start_url(self, url):
+        """Start tracing url unpack process.
+        This method is called by Fetch.unpack before the fetcher-specific unpack
+        method starts, and it may receive nested calls by gitsm and npmsw
+        fetchers.
+        """
+        return
+    def unpack(self, unpack_type, destdir):
+        """
+        Set unpack_type and destdir for current url.
+        This method is called by the fetcher-specific unpack method after url
+        tracing started.
+        """
+        return
+    def finish_url(self, url):
+        """Finish tracing url unpack process and update the file index.
+        This method is called by Fetch.unpack after the fetcher-specific unpack
+        method finished its job, and it may receive nested calls by gitsm
+        and npmsw fetchers.
+        """
+        return
+    def complete(self):
+        """
+        Finish tracing the Fetch.unpack process, and check if all nested
+        Fecth.unpack calls (if any) have been completed; if so, save collected
+        metadata.
+        """
+        return
+
+
 class Fetch(object):
     def __init__(self, urls, d, cache = True, localonly = False, connection_cache = None):
         if localonly and cache:
             raise Exception("bb.fetch2.Fetch.__init__: cannot set cache and localonly at same time")
 
-        if len(urls) == 0:
+        if not urls:
             urls = d.getVar("SRC_URI").split()
         self.urls = urls
         self.d = d
@@ -1631,10 +1795,30 @@ class Fetch(object):
         if key in urldata_cache:
             self.ud = urldata_cache[key]
 
+        # the unpack_tracer object needs to be made available to possible nested
+        # Fetch instances (when those are created by gitsm and npmsw fetchers)
+        # so we set it as a global variable
+        global unpack_tracer
+        try:
+            unpack_tracer
+        except NameError:
+            class_path = d.getVar("BB_UNPACK_TRACER_CLASS")
+            if class_path:
+                # use user-defined unpack tracer class
+                import importlib
+                module_name, _, class_name = class_path.rpartition(".")
+                module = importlib.import_module(module_name)
+                class_ = getattr(module, class_name)
+                unpack_tracer = class_()
+            else:
+                # fall back to the dummy/abstract class
+                unpack_tracer = DummyUnpackTracer()
+
         for url in urls:
             if url not in self.ud:
                 try:
                     self.ud[url] = FetchData(url, d, localonly)
+                    self.ud[url].unpack_tracer = unpack_tracer
                 except NonLocalMethod:
                     if localonly:
                         self.ud[url] = None
@@ -1648,7 +1832,7 @@ class Fetch(object):
             self.ud[url] = FetchData(url, self.d)
 
         self.ud[url].setup_localpath(self.d)
-        return self.d.expand(self.ud[url].localpath)
+        return self.ud[url].localpath
 
     def localpaths(self):
         """
@@ -1673,6 +1857,7 @@ class Fetch(object):
         network = self.d.getVar("BB_NO_NETWORK")
         premirroronly = bb.utils.to_boolean(self.d.getVar("BB_FETCH_PREMIRRORONLY"))
 
+        checksum_missing_messages = []
         for u in urls:
             ud = self.ud[u]
             ud.setup_localpath(self.d)
@@ -1684,7 +1869,6 @@ class Fetch(object):
 
             try:
                 self.d.setVar("BB_NO_NETWORK", network)
-
                 if m.verify_donestamp(ud, self.d) and not m.need_update(ud, self.d):
                     done = True
                 elif m.try_premirror(ud, self.d):
@@ -1701,23 +1885,28 @@ class Fetch(object):
                             logger.debug(str(e))
                             done = False
 
+                d = self.d
                 if premirroronly:
-                    self.d.setVar("BB_NO_NETWORK", "1")
+                    # Only disable the network in a copy
+                    d = bb.data.createCopy(self.d)
+                    d.setVar("BB_NO_NETWORK", "1")
 
                 firsterr = None
-                verified_stamp = m.verify_donestamp(ud, self.d)
-                if not done and (not verified_stamp or m.need_update(ud, self.d)):
+                verified_stamp = False
+                if done:
+                    verified_stamp = m.verify_donestamp(ud, d)
+                if not done and (not verified_stamp or m.need_update(ud, d)):
                     try:
-                        if not trusted_network(self.d, ud.url):
+                        if not trusted_network(d, ud.url):
                             raise UntrustedUrl(ud.url)
                         logger.debug("Trying Upstream")
-                        m.download(ud, self.d)
+                        m.download(ud, d)
                         if hasattr(m, "build_mirror_data"):
-                            m.build_mirror_data(ud, self.d)
+                            m.build_mirror_data(ud, d)
                         done = True
                         # early checksum verify, so that if checksum mismatched,
                         # fetcher still have chance to fetch from mirror
-                        m.update_donestamp(ud, self.d)
+                        m.update_donestamp(ud, d)
 
                     except bb.fetch2.NetworkAccess:
                         raise
@@ -1735,18 +1924,18 @@ class Fetch(object):
                             logger.debug(str(e))
                         firsterr = e
                         # Remove any incomplete fetch
-                        if not verified_stamp:
-                            m.clean(ud, self.d)
+                        if not verified_stamp and m.cleanup_upon_failure():
+                            m.clean(ud, d)
                         logger.debug("Trying MIRRORS")
-                        mirrors = mirror_from_string(self.d.getVar('MIRRORS'))
-                        done = m.try_mirrors(self, ud, self.d, mirrors)
+                        mirrors = mirror_from_string(d.getVar('MIRRORS'))
+                        done = m.try_mirrors(self, ud, d, mirrors)
 
-                if not done or not m.done(ud, self.d):
+                if not done or not m.done(ud, d):
                     if firsterr:
                         logger.error(str(firsterr))
                     raise FetchError("Unable to fetch URL from any source.", u)
 
-                m.update_donestamp(ud, self.d)
+                m.update_donestamp(ud, d)
 
             except IOError as e:
                 if e.errno in [errno.ESTALE]:
@@ -1754,17 +1943,28 @@ class Fetch(object):
                     raise ChecksumError("Stale Error Detected")
 
             except BBFetchException as e:
-                if isinstance(e, ChecksumError):
+                if isinstance(e, NoChecksumError):
+                    (message, _) = e.args
+                    checksum_missing_messages.append(message)
+                    continue
+                elif isinstance(e, ChecksumError):
                     logger.error("Checksum failure fetching %s" % u)
                 raise
 
             finally:
                 if ud.lockfile:
                     bb.utils.unlockfile(lf)
+        if checksum_missing_messages:
+            logger.error("Missing SRC_URI checksum, please add those to the recipe: \n%s", "\n".join(checksum_missing_messages))
+            raise BBFetchException("There was some missing checksums in the recipe")
 
     def checkstatus(self, urls=None):
         """
-        Check all urls exist upstream
+        Check all URLs exist upstream.
+
+        Returns None if the URLs exist, raises FetchError if the check wasn't
+        successful but there wasn't an error (such as file not found), and
+        raises other exceptions in error cases.
         """
 
         if not urls:
@@ -1787,7 +1987,7 @@ class Fetch(object):
                     ret = m.try_mirrors(self, ud, self.d, mirrors, True)
 
             if not ret:
-                raise FetchError("URL %s doesn't work" % u, u)
+                raise FetchError("URL doesn't work", u)
 
     def unpack(self, root, urls=None):
         """
@@ -1797,6 +1997,8 @@ class Fetch(object):
         if not urls:
             urls = self.urls
 
+        unpack_tracer.start(root, self.ud, self.d)
+
         for u in urls:
             ud = self.ud[u]
             ud.setup_localpath(self.d)
@@ -1804,11 +2006,15 @@ class Fetch(object):
             if ud.lockfile:
                 lf = bb.utils.lockfile(ud.lockfile)
 
+            unpack_tracer.start_url(u)
             ud.method.unpack(ud, root, self.d)
+            unpack_tracer.finish_url(u)
 
             if ud.lockfile:
                 bb.utils.unlockfile(lf)
 
+        unpack_tracer.complete()
+
     def clean(self, urls=None):
         """
         Clean files that the fetcher gets or places
@@ -1908,6 +2114,10 @@ from . import repo
 from . import clearcase
 from . import npm
 from . import npmsw
+from . import az
+from . import crate
+from . import gcp
+from . import gomod
 
 methods.append(local.Local())
 methods.append(wget.Wget())
@@ -1927,3 +2137,8 @@ methods.append(repo.Repo())
 methods.append(clearcase.ClearCase())
 methods.append(npm.Npm())
 methods.append(npmsw.NpmShrinkWrap())
+methods.append(az.Az())
+methods.append(crate.Crate())
+methods.append(gcp.GCP())
+methods.append(gomod.GoMod())
+methods.append(gomod.GoModGit())
diff --git a/bitbake/lib/bb/fetch2/az.py b/bitbake/lib/bb/fetch2/az.py
new file mode 100644
index 0000000000..1d3664f213
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/az.py
@@ -0,0 +1,98 @@
+"""
+BitBake 'Fetch' Azure Storage implementation
+
+"""
+
+# Copyright (C) 2021 Alejandro Hernandez Samaniego
+#
+# Based on bb.fetch2.wget:
+# Copyright (C) 2003, 2004  Chris Larson
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import shlex
+import os
+import bb
+from   bb.fetch2 import FetchError
+from   bb.fetch2 import logger
+from   bb.fetch2.wget import Wget
+
+
+class Az(Wget):
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched from Azure Storage
+        """
+        return ud.type in ['az']
+
+
+    def checkstatus(self, fetch, ud, d, try_again=True):
+
+        # checkstatus discards parameters either way, we need to do this before adding the SAS
+        ud.url = ud.url.replace('az://','https://').split(';')[0]
+
+        az_sas = d.getVar('AZ_SAS')
+        if az_sas and az_sas not in ud.url:
+            if not az_sas.startswith('?'):
+                raise FetchError("When using AZ_SAS, it must start with a '?' character to mark the start of the query-parameters.")
+            ud.url += az_sas
+
+        return Wget.checkstatus(self, fetch, ud, d, try_again)
+
+    # Override download method, include retries
+    def download(self, ud, d, retries=3):
+        """Fetch urls"""
+
+        # If were reaching the account transaction limit we might be refused a connection,
+        # retrying allows us to avoid false negatives since the limit changes over time
+        fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5'
+
+        # We need to provide a localpath to avoid wget using the SAS
+        # ud.localfile either has the downloadfilename or ud.path
+        localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
+        bb.utils.mkdirhier(os.path.dirname(localpath))
+        fetchcmd += " -O %s" % shlex.quote(localpath)
+
+
+        if ud.user and ud.pswd:
+            fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
+
+        # Check if a Shared Access Signature was given and use it
+        az_sas = d.getVar('AZ_SAS')
+
+        if az_sas:
+            if not az_sas.startswith('?'):
+                raise FetchError("When using AZ_SAS, it must start with a '?' character to mark the start of the query-parameters.")
+            azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas)
+        else:
+            azuri = '%s%s%s' % ('https://', ud.host, ud.path)
+
+        dldir = d.getVar("DL_DIR")
+        if os.path.exists(ud.localpath):
+            # file exists, but we didnt complete it.. trying again.
+            fetchcmd += " -c -P %s '%s'" % (dldir, azuri)
+        else:
+            fetchcmd += " -P %s '%s'" % (dldir, azuri)
+
+        try:
+            self._runwget(ud, d, fetchcmd, False)
+        except FetchError as e:
+            # Azure fails on handshake sometimes when using wget after some stress, producing a
+            # FetchError from the fetcher, if the artifact exists retyring should succeed
+            if 'Unable to establish SSL connection' in str(e):
+                logger.debug2('Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries)
+                self.download(ud, d, retries -1)
+
+        # Sanity check since wget can pretend it succeed when it didn't
+        # Also, this used to happen if sourceforge sent us to the mirror page
+        if not os.path.exists(ud.localpath):
+            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri)
+
+        if os.path.getsize(ud.localpath) == 0:
+            os.remove(ud.localpath)
+            raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri)
+
+        return True
diff --git a/bitbake/lib/bb/fetch2/clearcase.py b/bitbake/lib/bb/fetch2/clearcase.py
index 1a9c863769..17500daf95 100644
--- a/bitbake/lib/bb/fetch2/clearcase.py
+++ b/bitbake/lib/bb/fetch2/clearcase.py
@@ -108,7 +108,7 @@ class ClearCase(FetchMethod):
                                                 ud.module.replace("/", "."),
                                                 ud.label.replace("/", "."))
 
-        ud.viewname         = "%s-view%s" % (ud.identifier, d.getVar("DATETIME", d, True))
+        ud.viewname         = "%s-view%s" % (ud.identifier, d.getVar("DATETIME"))
         ud.csname           = "%s-config-spec" % (ud.identifier)
         ud.ccasedir         = os.path.join(d.getVar("DL_DIR"), ud.type)
         ud.viewdir          = os.path.join(ud.ccasedir, ud.viewname)
@@ -130,8 +130,6 @@ class ClearCase(FetchMethod):
         self.debug("configspecfile  = %s" % ud.configspecfile)
         self.debug("localfile       = %s" % ud.localfile)
 
-        ud.localfile = os.path.join(d.getVar("DL_DIR"), ud.localfile)
-
     def _build_ccase_command(self, ud, command):
         """
         Build up a commandline based on ud
@@ -196,7 +194,7 @@ class ClearCase(FetchMethod):
 
     def need_update(self, ud, d):
         if ("LATEST" in ud.label) or (ud.customspec and "LATEST" in ud.customspec):
-            ud.identifier += "-%s" % d.getVar("DATETIME",d, True)
+            ud.identifier += "-%s" % d.getVar("DATETIME")
             return True
         if os.path.exists(ud.localpath):
             return False
diff --git a/bitbake/lib/bb/fetch2/crate.py b/bitbake/lib/bb/fetch2/crate.py
new file mode 100644
index 0000000000..e611736f06
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/crate.py
@@ -0,0 +1,150 @@
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+"""
+BitBake 'Fetch' implementation for crates.io
+"""
+
+# Copyright (C) 2016 Doug Goldstein
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import hashlib
+import json
+import os
+import subprocess
+import bb
+from   bb.fetch2 import logger, subprocess_setup, UnpackError
+from   bb.fetch2.wget import Wget
+
+
+class Crate(Wget):
+
+    """Class to fetch crates via wget"""
+
+    def _cargo_bitbake_path(self, rootdir):
+        return os.path.join(rootdir, "cargo_home", "bitbake")
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url is for this fetcher
+        """
+        return ud.type in ['crate']
+
+    def recommends_checksum(self, urldata):
+        return True
+
+    def urldata_init(self, ud, d):
+        """
+        Sets up to download the respective crate from crates.io
+        """
+
+        if ud.type == 'crate':
+            self._crate_urldata_init(ud, d)
+
+        super(Crate, self).urldata_init(ud, d)
+
+    def _crate_urldata_init(self, ud, d):
+        """
+        Sets up the download for a crate
+        """
+
+        # URL syntax is: crate://NAME/VERSION
+        # break the URL apart by /
+        parts = ud.url.split('/')
+        if len(parts) < 5:
+            raise bb.fetch2.ParameterError("Invalid URL: Must be crate://HOST/NAME/VERSION", ud.url)
+
+        # version is expected to be the last token
+        # but ignore possible url parameters which will be used
+        # by the top fetcher class
+        version = parts[-1].split(";")[0]
+        # second to last field is name
+        name = parts[-2]
+        # host (this is to allow custom crate registries to be specified
+        host = '/'.join(parts[2:-2])
+
+        # if using upstream just fix it up nicely
+        if host == 'crates.io':
+            host = 'crates.io/api/v1/crates'
+
+        ud.url = "https://%s/%s/%s/download" % (host, name, version)
+        ud.versionsurl = "https://%s/%s/versions" % (host, name)
+        ud.parm['downloadfilename'] = "%s-%s.crate" % (name, version)
+        if 'name' not in ud.parm:
+            ud.parm['name'] = '%s-%s' % (name, version)
+
+        logger.debug2("Fetching %s to %s" % (ud.url, ud.parm['downloadfilename']))
+
+    def unpack(self, ud, rootdir, d):
+        """
+        Uses the crate to build the necessary paths for cargo to utilize it
+        """
+        if ud.type == 'crate':
+            return self._crate_unpack(ud, rootdir, d)
+        else:
+            super(Crate, self).unpack(ud, rootdir, d)
+
+    def _crate_unpack(self, ud, rootdir, d):
+        """
+        Unpacks a crate
+        """
+        thefile = ud.localpath
+
+        # possible metadata we need to write out
+        metadata = {}
+
+        # change to the rootdir to unpack but save the old working dir
+        save_cwd = os.getcwd()
+        os.chdir(rootdir)
+
+        bp = d.getVar('BP')
+        if bp == ud.parm.get('name'):
+            cmd = "tar -xz --no-same-owner -f %s" % thefile
+            ud.unpack_tracer.unpack("crate-extract", rootdir)
+        else:
+            cargo_bitbake = self._cargo_bitbake_path(rootdir)
+            ud.unpack_tracer.unpack("cargo-extract", cargo_bitbake)
+
+            cmd = "tar -xz --no-same-owner -f %s -C %s" % (thefile, cargo_bitbake)
+
+            # ensure we've got these paths made
+            bb.utils.mkdirhier(cargo_bitbake)
+
+            # generate metadata necessary
+            with open(thefile, 'rb') as f:
+                # get the SHA256 of the original tarball
+                tarhash = hashlib.sha256(f.read()).hexdigest()
+
+            metadata['files'] = {}
+            metadata['package'] = tarhash
+
+        path = d.getVar('PATH')
+        if path:
+            cmd = "PATH=\"%s\" %s" % (path, cmd)
+        bb.note("Unpacking %s to %s/" % (thefile, os.getcwd()))
+
+        ret = subprocess.call(cmd, preexec_fn=subprocess_setup, shell=True)
+
+        os.chdir(save_cwd)
+
+        if ret != 0:
+            raise UnpackError("Unpack command %s failed with return value %s" % (cmd, ret), ud.url)
+
+        # if we have metadata to write out..
+        if len(metadata) > 0:
+            cratepath = os.path.splitext(os.path.basename(thefile))[0]
+            bbpath = self._cargo_bitbake_path(rootdir)
+            mdfile = '.cargo-checksum.json'
+            mdpath = os.path.join(bbpath, cratepath, mdfile)
+            with open(mdpath, "w") as f:
+                json.dump(metadata, f)
+
+    def latest_versionstring(self, ud, d):
+        from functools import cmp_to_key
+        json_data = json.loads(self._fetch_index(ud.versionsurl, ud, d))
+        versions = [(0, i["num"], "") for i in json_data["versions"]]
+        versions = sorted(versions, key=cmp_to_key(bb.utils.vercmp))
+
+        return (versions[-1][1], "")
diff --git a/bitbake/lib/bb/fetch2/gcp.py b/bitbake/lib/bb/fetch2/gcp.py
new file mode 100644
index 0000000000..86546d40bf
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/gcp.py
@@ -0,0 +1,102 @@
+"""
+BitBake 'Fetch' implementation for Google Cloup Platform Storage.
+
+Class for fetching files from Google Cloud Storage using the
+Google Cloud Storage Python Client. The GCS Python Client must
+be correctly installed, configured and authenticated prior to use.
+Additionally, gsutil must also be installed.
+
+"""
+
+# Copyright (C) 2023, Snap Inc.
+#
+# Based in part on bb.fetch2.s3:
+#    Copyright (C) 2017 Andre McCurdy
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Based on functions from the base bb module, Copyright 2003 Holger Schurig
+
+import os
+import bb
+import urllib.parse, urllib.error
+from bb.fetch2 import FetchMethod
+from bb.fetch2 import FetchError
+from bb.fetch2 import logger
+
+class GCP(FetchMethod):
+    """
+    Class to fetch urls via GCP's Python API.
+    """
+    def __init__(self):
+        self.gcp_client = None
+
+    def supports(self, ud, d):
+        """
+        Check to see if a given url can be fetched with GCP.
+        """
+        return ud.type in ['gs']
+
+    def recommends_checksum(self, urldata):
+        return True
+
+    def urldata_init(self, ud, d):
+        if 'downloadfilename' in ud.parm:
+            ud.basename = ud.parm['downloadfilename']
+        else:
+            ud.basename = os.path.basename(ud.path)
+
+        ud.localfile = ud.basename
+
+    def get_gcp_client(self):
+        from google.cloud import storage
+        self.gcp_client = storage.Client(project=None)
+
+    def download(self, ud, d):
+        """
+        Fetch urls using the GCP API.
+        Assumes localpath was called first.
+        """
+        from google.api_core.exceptions import NotFound
+        logger.debug2(f"Trying to download gs://{ud.host}{ud.path} to {ud.localpath}")
+        if self.gcp_client is None:
+            self.get_gcp_client()
+
+        bb.fetch2.check_network_access(d, "blob.download_to_filename", f"gs://{ud.host}{ud.path}")
+
+        # Path sometimes has leading slash, so strip it
+        path = ud.path.lstrip("/")
+        blob = self.gcp_client.bucket(ud.host).blob(path)
+        try:
+            blob.download_to_filename(ud.localpath)
+        except NotFound:
+            raise FetchError("The GCP API threw a NotFound exception")
+
+        # Additional sanity checks copied from the wget class (although there
+        # are no known issues which mean these are required, treat the GCP API
+        # tool with a little healthy suspicion).
+        if not os.path.exists(ud.localpath):
+            raise FetchError(f"The GCP API returned success for gs://{ud.host}{ud.path} but {ud.localpath} doesn't exist?!")
+
+        if os.path.getsize(ud.localpath) == 0:
+            os.remove(ud.localpath)
+            raise FetchError(f"The downloaded file for gs://{ud.host}{ud.path} resulted in a zero size file?! Deleting and failing since this isn't right.")
+
+        return True
+
+    def checkstatus(self, fetch, ud, d):
+        """
+        Check the status of a URL.
+        """
+        logger.debug2(f"Checking status of gs://{ud.host}{ud.path}")
+        if self.gcp_client is None:
+            self.get_gcp_client()
+
+        bb.fetch2.check_network_access(d, "gcp_client.bucket(ud.host).blob(path).exists()", f"gs://{ud.host}{ud.path}")
+
+        # Path sometimes has leading slash, so strip it
+        path = ud.path.lstrip("/")
+        if self.gcp_client.bucket(ud.host).blob(path).exists() == False:
+            raise FetchError(f"The GCP API reported that gs://{ud.host}{ud.path} does not exist")
+        else:
+            return True
diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py
index e3ba80a3f5..14ec45a3f6 100644
--- a/bitbake/lib/bb/fetch2/git.py
+++ b/bitbake/lib/bb/fetch2/git.py
@@ -9,15 +9,6 @@ Supported SRC_URI options are:
 - branch
    The git branch to retrieve from. The default is "master"
 
-   This option also supports multiple branch fetching, with branches
-   separated by commas.  In multiple branches case, the name option
-   must have the same number of names to match the branches, which is
-   used to specify the SRC_REV for the branch
-   e.g:
-   SRC_URI="git://some.host/somepath;branch=branchX,branchY;name=nameX,nameY"
-   SRCREV_nameX = "xxxxxxxxxxxxxxxxxxxx"
-   SRCREV_nameY = "YYYYYYYYYYYYYYYYYYYY"
-
 - tag
     The git tag to retrieve. The default is "master"
 
@@ -44,13 +35,27 @@ Supported SRC_URI options are:
 
 - nobranch
    Don't check the SHA validation for branch. set this option for the recipe
-   referring to commit which is valid in tag instead of branch.
+   referring to commit which is valid in any namespace (branch, tag, ...)
+   instead of branch.
    The default is "0", set nobranch=1 if needed.
 
+- subpath
+   Limit the checkout to a specific subpath of the tree.
+   By default, checkout the whole tree, set subpath=<path> if needed
+
+- destsuffix
+   The name of the path in which to place the checkout.
+   By default, the path is git/, set destsuffix=<suffix> if needed
+
 - usehead
    For local git:// urls to use the current branch HEAD as the revision for use with
    AUTOREV. Implies nobranch.
 
+- lfs
+    Enable the checkout to use LFS for large files. This will download all LFS files
+    in the download step, as the unpack step does not have network access.
+    The default is "1", set lfs=0 to skip.
+
 """
 
 # Copyright (C) 2005 Richard Purdie
@@ -64,15 +69,22 @@ import fnmatch
 import os
 import re
 import shlex
+import shutil
 import subprocess
 import tempfile
+import urllib
 import bb
 import bb.progress
+from contextlib import contextmanager
 from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import runfetchcmd
 from   bb.fetch2 import logger
+from   bb.fetch2 import trusted_network
 
 
+sha1_re = re.compile(r'^[0-9a-f]{40}$')
+slash_re = re.compile(r"/+")
+
 class GitProgressHandler(bb.progress.LineFilterProgressHandler):
     """Extract progress information from git output"""
     def __init__(self, d):
@@ -130,6 +142,9 @@ class Git(FetchMethod):
     def supports_checksum(self, urldata):
         return False
 
+    def cleanup_upon_failure(self):
+        return False
+
     def urldata_init(self, ud, d):
         """
         init git specific variable within url data
@@ -141,6 +156,11 @@ class Git(FetchMethod):
             ud.proto = 'file'
         else:
             ud.proto = "git"
+        if ud.host == "github.com" and ud.proto == "git":
+            # github stopped supporting git protocol
+            # https://github.blog/2021-09-01-improving-git-protocol-security-github/#no-more-unauthenticated-git
+            ud.proto = "https"
+            bb.warn("URL: %s uses git protocol which is no longer supported by github. Please change to ;protocol=https in the url." % ud.url)
 
         if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
             raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
@@ -162,18 +182,25 @@ class Git(FetchMethod):
         ud.bareclone = ud.parm.get("bareclone","0") == "1"
         if ud.bareclone:
             ud.nocheckout = 1
-  
-        ud.unresolvedrev = {}
-        branches = ud.parm.get("branch", "master").split(',')
-        if len(branches) != len(ud.names):
-            raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
 
-        ud.cloneflags = "-s -n"
+        ud.unresolvedrev = ""
+        ud.branch = ud.parm.get("branch", "")
+        if not ud.branch and not ud.nobranch:
+            raise bb.fetch2.ParameterError("The url does not set any branch parameter or set nobranch=1.", ud.url)
+
+        ud.noshared = d.getVar("BB_GIT_NOSHARED") == "1"
+
+        ud.cloneflags = "-n"
+        if not ud.noshared:
+            ud.cloneflags += " -s"
         if ud.bareclone:
             ud.cloneflags += " --mirror"
 
+        ud.shallow_skip_fast = False
         ud.shallow = d.getVar("BB_GIT_SHALLOW") == "1"
         ud.shallow_extra_refs = (d.getVar("BB_GIT_SHALLOW_EXTRA_REFS") or "").split()
+        if 'tag' in ud.parm:
+            ud.shallow_extra_refs.append("refs/tags/" + ud.parm['tag'])
 
         depth_default = d.getVar("BB_GIT_SHALLOW_DEPTH")
         if depth_default is not None:
@@ -190,32 +217,27 @@ class Git(FetchMethod):
 
         revs_default = d.getVar("BB_GIT_SHALLOW_REVS")
         ud.shallow_revs = []
-        ud.branches = {}
-        for pos, name in enumerate(ud.names):
-            branch = branches[pos]
-            ud.branches[name] = branch
-            ud.unresolvedrev[name] = branch
-
-            shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % name)
-            if shallow_depth is not None:
-                try:
-                    shallow_depth = int(shallow_depth or 0)
-                except ValueError:
-                    raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
-                else:
-                    if shallow_depth < 0:
-                        raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (name, shallow_depth))
-                    ud.shallow_depths[name] = shallow_depth
-
-            revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % name)
-            if revs is not None:
-                ud.shallow_revs.extend(revs.split())
-            elif revs_default is not None:
-                ud.shallow_revs.extend(revs_default.split())
-
-        if (ud.shallow and
-                not ud.shallow_revs and
-                all(ud.shallow_depths[n] == 0 for n in ud.names)):
+
+        ud.unresolvedrev = ud.branch
+
+        shallow_depth = d.getVar("BB_GIT_SHALLOW_DEPTH_%s" % ud.name)
+        if shallow_depth is not None:
+            try:
+                shallow_depth = int(shallow_depth or 0)
+            except ValueError:
+                raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
+            else:
+                if shallow_depth < 0:
+                    raise bb.fetch2.FetchError("Invalid depth for BB_GIT_SHALLOW_DEPTH_%s: %s" % (ud.name, shallow_depth))
+                ud.shallow_depths[ud.name] = shallow_depth
+
+        revs = d.getVar("BB_GIT_SHALLOW_REVS_%s" % ud.name)
+        if revs is not None:
+            ud.shallow_revs.extend(revs.split())
+        elif revs_default is not None:
+            ud.shallow_revs.extend(revs_default.split())
+
+        if ud.shallow and not ud.shallow_revs and ud.shallow_depths[ud.name] == 0:
             # Shallow disabled for this URL
             ud.shallow = False
 
@@ -224,10 +246,9 @@ class Git(FetchMethod):
             # rev of this repository. This will get resolved into a revision
             # later. If an actual revision happens to have also been provided
             # then this setting will be overridden.
-            for name in ud.names:
-                ud.unresolvedrev[name] = 'HEAD'
+            ud.unresolvedrev = 'HEAD'
 
-        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c core.fsyncobjectfiles=0"
+        ud.basecmd = d.getVar("FETCHCMD_git") or "git -c gc.autoDetach=false -c core.pager=cat -c safe.bareRepository=all -c clone.defaultRemoteName=origin"
 
         write_tarballs = d.getVar("BB_GENERATE_MIRROR_TARBALLS") or "0"
         ud.write_tarballs = write_tarballs != "0" or ud.rebaseable
@@ -235,24 +256,22 @@ class Git(FetchMethod):
 
         ud.setup_revisions(d)
 
-        for name in ud.names:
-            # Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
-            if not ud.revisions[name] or len(ud.revisions[name]) != 40  or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
-                if ud.revisions[name]:
-                    ud.unresolvedrev[name] = ud.revisions[name]
-                ud.revisions[name] = self.latest_revision(ud, d, name)
+        # Ensure any revision that doesn't look like a SHA-1 is translated into one
+        if not sha1_re.match(ud.revision or ''):
+            if ud.revision:
+                ud.unresolvedrev = ud.revision
+            ud.revision = self.latest_revision(ud, d, ud.name)
 
-        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_'))
+        gitsrcname = '%s%s' % (ud.host.replace(':', '.'), ud.path.replace('/', '.').replace('*', '.').replace(' ','_').replace('(', '_').replace(')', '_'))
         if gitsrcname.startswith('.'):
             gitsrcname = gitsrcname[1:]
 
-        # for rebaseable git repo, it is necessary to keep mirror tar ball
-        # per revision, so that even the revision disappears from the
+        # For a rebaseable git repo, it is necessary to keep a mirror tar ball
+        # per revision, so that even if the revision disappears from the
         # upstream repo in the future, the mirror will remain intact and still
-        # contains the revision
+        # contain the revision
         if ud.rebaseable:
-            for name in ud.names:
-                gitsrcname = gitsrcname + '_' + ud.revisions[name]
+            gitsrcname = gitsrcname + '_' + ud.revision
 
         dl_dir = d.getVar("DL_DIR")
         gitdir = d.getVar("GITDIR") or (dl_dir + "/git2")
@@ -270,15 +289,14 @@ class Git(FetchMethod):
             if ud.shallow_revs:
                 tarballname = "%s_%s" % (tarballname, "_".join(sorted(ud.shallow_revs)))
 
-            for name, revision in sorted(ud.revisions.items()):
-                tarballname = "%s_%s" % (tarballname, ud.revisions[name][:7])
-                depth = ud.shallow_depths[name]
-                if depth:
-                    tarballname = "%s-%s" % (tarballname, depth)
+            tarballname = "%s_%s" % (tarballname, ud.revision[:7])
+            depth = ud.shallow_depths[ud.name]
+            if depth:
+                tarballname = "%s-%s" % (tarballname, depth)
 
             shallow_refs = []
             if not ud.nobranch:
-                shallow_refs.extend(ud.branches.values())
+                shallow_refs.append(ud.branch)
             if ud.shallow_extra_refs:
                 shallow_refs.extend(r.replace('refs/heads/', '').replace('*', 'ALL') for r in ud.shallow_extra_refs)
             if shallow_refs:
@@ -293,16 +311,29 @@ class Git(FetchMethod):
         return ud.clonedir
 
     def need_update(self, ud, d):
-        return self.clonedir_need_update(ud, d) or self.shallow_tarball_need_update(ud) or self.tarball_need_update(ud)
+        return self.clonedir_need_update(ud, d) \
+                or self.shallow_tarball_need_update(ud) \
+                or self.tarball_need_update(ud) \
+                or self.lfs_need_update(ud, d)
 
     def clonedir_need_update(self, ud, d):
         if not os.path.exists(ud.clonedir):
             return True
         if ud.shallow and ud.write_shallow_tarballs and self.clonedir_need_shallow_revs(ud, d):
             return True
-        for name in ud.names:
-            if not self._contains_ref(ud, d, name, ud.clonedir):
-                return True
+        if not self._contains_ref(ud, d, ud.name, ud.clonedir):
+            return True
+        return False
+
+    def lfs_need_update(self, ud, d):
+        if not self._need_lfs(ud):
+            return False
+
+        if self.clonedir_need_update(ud, d):
+            return True
+
+        if not self._lfs_objects_downloaded(ud, d, ud.clonedir):
+            return True
         return False
 
     def clonedir_need_shallow_revs(self, ud, d):
@@ -319,11 +350,28 @@ class Git(FetchMethod):
     def tarball_need_update(self, ud):
         return ud.write_tarballs and not os.path.exists(ud.fullmirror)
 
+    def update_mirror_links(self, ud, origud):
+        super().update_mirror_links(ud, origud)
+        # When using shallow mode, add a symlink to the original fullshallow
+        # path to ensure a valid symlink even in the `PREMIRRORS` case
+        if ud.shallow and not os.path.exists(origud.fullshallow):
+            self.ensure_symlink(ud.localpath, origud.fullshallow)
+
     def try_premirror(self, ud, d):
         # If we don't do this, updating an existing checkout with only premirrors
         # is not possible
         if bb.utils.to_boolean(d.getVar("BB_FETCH_PREMIRRORONLY")):
             return True
+        # If the url is not in trusted network, that is, BB_NO_NETWORK is set to 0
+        # and BB_ALLOWED_NETWORKS does not contain the host that ud.url uses, then
+        # we need to try premirrors first as using upstream is destined to fail.
+        if not trusted_network(d, ud.url):
+            return True
+        # the following check is to ensure incremental fetch in downloads, this is
+        # because the premirror might be old and does not contain the new rev required,
+        # and this will cause a total removal and new clone. So if we can reach to
+        # network, we prefer upstream over premirror, though the premirror might contain
+        # the new rev.
         if os.path.exists(ud.clonedir):
             return False
         return True
@@ -337,21 +385,76 @@ class Git(FetchMethod):
         if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
             ud.localpath = ud.fullshallow
             return
-        elif os.path.exists(ud.fullmirror) and not os.path.exists(ud.clonedir):
-            bb.utils.mkdirhier(ud.clonedir)
-            runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
-
+        elif os.path.exists(ud.fullmirror) and self.need_update(ud, d):
+            if not os.path.exists(ud.clonedir):
+                bb.utils.mkdirhier(ud.clonedir)
+                runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=ud.clonedir)
+            else:
+                tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
+                runfetchcmd("tar -xzf %s" % ud.fullmirror, d, workdir=tmpdir)
+                output = runfetchcmd("%s remote" % ud.basecmd, d, quiet=True, workdir=ud.clonedir)
+                if 'mirror' in output:
+                    runfetchcmd("%s remote rm mirror" % ud.basecmd, d, workdir=ud.clonedir)
+                runfetchcmd("%s remote add --mirror=fetch mirror %s" % (ud.basecmd, tmpdir), d, workdir=ud.clonedir)
+                fetch_cmd = "LANG=C %s fetch -f --update-head-ok  --progress mirror " % (ud.basecmd)
+                runfetchcmd(fetch_cmd, d, workdir=ud.clonedir)
         repourl = self._get_repo_url(ud)
 
+        needs_clone = False
+        if os.path.exists(ud.clonedir):
+            # The directory may exist, but not be the top level of a bare git
+            # repository in which case it needs to be deleted and re-cloned.
+            try:
+                # Since clones can be bare, use --absolute-git-dir instead of --show-toplevel
+                output = runfetchcmd("LANG=C %s rev-parse --absolute-git-dir" % ud.basecmd, d, workdir=ud.clonedir)
+                toplevel = output.rstrip()
+
+                if not bb.utils.path_is_descendant(toplevel, ud.clonedir):
+                    logger.warning("Top level directory '%s' is not a descendant of '%s'. Re-cloning", toplevel, ud.clonedir)
+                    needs_clone = True
+            except bb.fetch2.FetchError as e:
+                logger.warning("Unable to get top level for %s (not a git directory?): %s", ud.clonedir, e)
+                needs_clone = True
+            except FileNotFoundError as e:
+                logger.warning("%s", e)
+                needs_clone = True
+
+            if needs_clone:
+                shutil.rmtree(ud.clonedir)
+        else:
+            needs_clone = True
+
         # If the repo still doesn't exist, fallback to cloning it
-        if not os.path.exists(ud.clonedir):
-            # We do this since git will use a "-l" option automatically for local urls where possible
+        if needs_clone:
+            # We do this since git will use a "-l" option automatically for local urls where possible,
+            # but it doesn't work when git/objects is a symlink, only works when it is a directory.
             if repourl.startswith("file://"):
-                repourl = repourl[7:]
+                repourl_path = repourl[7:]
+                objects = os.path.join(repourl_path, 'objects')
+                if os.path.isdir(objects) and not os.path.islink(objects):
+                    repourl = repourl_path
             clone_cmd = "LANG=C %s clone --bare --mirror %s %s --progress" % (ud.basecmd, shlex.quote(repourl), ud.clonedir)
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, clone_cmd, ud.url)
             progresshandler = GitProgressHandler(d)
+
+            # Try creating a fast initial shallow clone
+            # Enabling ud.shallow_skip_fast will skip this
+            # If the Git error "Server does not allow request for unadvertised object"
+            # occurs, shallow_skip_fast is enabled automatically.
+            # This may happen if the Git server does not allow the request
+            # or if the Git client has issues with this functionality.
+            if ud.shallow and not ud.shallow_skip_fast:
+                try:
+                    self.clone_shallow_with_tarball(ud, d)
+                    # When the shallow clone has succeeded, use the shallow tarball
+                    ud.localpath = ud.fullshallow
+                    return
+                except:
+                    logger.warning("Creating fast initial shallow clone failed, try initial regular clone now.")
+
+            # When skipping fast initial shallow or the fast inital shallow clone failed:
+            # Try again with an initial regular clone
             runfetchcmd(clone_cmd, d, log=progresshandler)
 
         # Update the checkout if needed
@@ -361,7 +464,11 @@ class Git(FetchMethod):
               runfetchcmd("%s remote rm origin" % ud.basecmd, d, workdir=ud.clonedir)
 
             runfetchcmd("%s remote add --mirror=fetch origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=ud.clonedir)
-            fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
+
+            if ud.nobranch:
+                fetch_cmd = "LANG=C %s fetch -f --progress %s refs/*:refs/*" % (ud.basecmd, shlex.quote(repourl))
+            else:
+                fetch_cmd = "LANG=C %s fetch -f --progress %s refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" % (ud.basecmd, shlex.quote(repourl))
             if ud.proto.lower() != 'file':
                 bb.fetch2.check_network_access(d, fetch_cmd, ud.url)
             progresshandler = GitProgressHandler(d)
@@ -375,138 +482,206 @@ class Git(FetchMethod):
                 if exc.errno != errno.ENOENT:
                     raise
 
-        for name in ud.names:
-            if not self._contains_ref(ud, d, name, ud.clonedir):
-                raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revisions[name], ud.branches[name]))
+        if not self._contains_ref(ud, d, ud.name, ud.clonedir):
+            raise bb.fetch2.FetchError("Unable to find revision %s in branch %s even from upstream" % (ud.revision, ud.branch))
 
         if ud.shallow and ud.write_shallow_tarballs:
             missing_rev = self.clonedir_need_shallow_revs(ud, d)
             if missing_rev:
                 raise bb.fetch2.FetchError("Unable to find revision %s even from upstream" % missing_rev)
 
-        if self._contains_lfs(ud, d, ud.clonedir) and self._need_lfs(ud):
-            # Unpack temporary working copy, use it to run 'git checkout' to force pre-fetching
-            # of all LFS blobs needed at the the srcrev.
-            #
-            # It would be nice to just do this inline here by running 'git-lfs fetch'
-            # on the bare clonedir, but that operation requires a working copy on some
-            # releases of Git LFS.
-            tmpdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
-            try:
-                # Do the checkout. This implicitly involves a Git LFS fetch.
-                self.unpack(ud, tmpdir, d)
-
-                # Scoop up a copy of any stuff that Git LFS downloaded. Merge them into
-                # the bare clonedir.
-                #
-                # As this procedure is invoked repeatedly on incremental fetches as
-                # a recipe's SRCREV is bumped throughout its lifetime, this will
-                # result in a gradual accumulation of LFS blobs in <ud.clonedir>/lfs
-                # corresponding to all the blobs reachable from the different revs
-                # fetched across time.
-                #
-                # Only do this if the unpack resulted in a .git/lfs directory being
-                # created; this only happens if at least one blob needed to be
-                # downloaded.
-                if os.path.exists(os.path.join(tmpdir, "git", ".git", "lfs")):
-                    runfetchcmd("tar -cf - lfs | tar -xf - -C %s" % ud.clonedir, d, workdir="%s/git/.git" % tmpdir)
-            finally:
-                bb.utils.remove(tmpdir, recurse=True)
+        if self.lfs_need_update(ud, d):
+            self.lfs_fetch(ud, d, ud.clonedir, ud.revision)
+
+    def lfs_fetch(self, ud, d, clonedir, revision, fetchall=False, progresshandler=None):
+        """Helper method for fetching Git LFS data"""
+        try:
+            if self._need_lfs(ud) and self._contains_lfs(ud, d, clonedir) and len(revision):
+                self._ensure_git_lfs(d, ud)
+
+                # Using worktree with the revision because .lfsconfig may exists
+                worktree_add_cmd = "%s worktree add wt %s" % (ud.basecmd, revision)
+                runfetchcmd(worktree_add_cmd, d, log=progresshandler, workdir=clonedir)
+                lfs_fetch_cmd = "%s lfs fetch %s" % (ud.basecmd, "--all" if fetchall else "")
+                runfetchcmd(lfs_fetch_cmd, d, log=progresshandler, workdir=(clonedir + "/wt"))
+                worktree_rem_cmd = "%s worktree remove -f wt" % ud.basecmd
+                runfetchcmd(worktree_rem_cmd, d, log=progresshandler, workdir=clonedir)
+        except:
+            logger.warning("Fetching LFS did not succeed.")
+
+    @contextmanager
+    def create_atomic(self, filename):
+        """Create as a temp file and move atomically into position to avoid races"""
+        fd, tfile = tempfile.mkstemp(dir=os.path.dirname(filename))
+        try:
+            yield tfile
+            umask = os.umask(0o666)
+            os.umask(umask)
+            os.chmod(tfile, (0o666 & ~umask))
+            os.rename(tfile, filename)
+        finally:
+            os.close(fd)
 
     def build_mirror_data(self, ud, d):
         if ud.shallow and ud.write_shallow_tarballs:
             if not os.path.exists(ud.fullshallow):
                 if os.path.islink(ud.fullshallow):
                     os.unlink(ud.fullshallow)
-                tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
-                shallowclone = os.path.join(tempdir, 'git')
-                try:
-                    self.clone_shallow_local(ud, shallowclone, d)
-
-                    logger.info("Creating tarball of git repository")
-                    runfetchcmd("tar -czf %s ." % ud.fullshallow, d, workdir=shallowclone)
-                    runfetchcmd("touch %s.done" % ud.fullshallow, d)
-                finally:
-                    bb.utils.remove(tempdir, recurse=True)
+                self.clone_shallow_with_tarball(ud, d)
         elif ud.write_tarballs and not os.path.exists(ud.fullmirror):
             if os.path.islink(ud.fullmirror):
                 os.unlink(ud.fullmirror)
 
             logger.info("Creating tarball of git repository")
-            runfetchcmd("tar -czf %s ." % ud.fullmirror, d, workdir=ud.clonedir)
+            with self.create_atomic(ud.fullmirror) as tfile:
+                mtime = runfetchcmd("{} log --all -1 --format=%cD".format(ud.basecmd), d,
+                        quiet=True, workdir=ud.clonedir)
+                runfetchcmd("tar -czf %s --owner oe:0 --group oe:0 --mtime \"%s\" ."
+                        % (tfile, mtime), d, workdir=ud.clonedir)
             runfetchcmd("touch %s.done" % ud.fullmirror, d)
 
+    def clone_shallow_with_tarball(self, ud, d):
+        ret = False
+        tempdir = tempfile.mkdtemp(dir=d.getVar('DL_DIR'))
+        shallowclone = os.path.join(tempdir, 'git')
+        try:
+            try:
+                self.clone_shallow_local(ud, shallowclone, d)
+            except:
+                logger.warning("Fast shallow clone failed, try to skip fast mode now.")
+                bb.utils.remove(tempdir, recurse=True)
+                os.mkdir(tempdir)
+                ud.shallow_skip_fast = True
+                self.clone_shallow_local(ud, shallowclone, d)
+            logger.info("Creating tarball of git repository")
+            with self.create_atomic(ud.fullshallow) as tfile:
+                runfetchcmd("tar -czf %s ." % tfile, d, workdir=shallowclone)
+            runfetchcmd("touch %s.done" % ud.fullshallow, d)
+            ret = True
+        finally:
+            bb.utils.remove(tempdir, recurse=True)
+
+        return ret
+
     def clone_shallow_local(self, ud, dest, d):
-        """Clone the repo and make it shallow.
+        """
+        Shallow fetch from ud.clonedir (${DL_DIR}/git2/<gitrepo> by default):
+        - For BB_GIT_SHALLOW_DEPTH: git fetch --depth <depth> rev
+        - For BB_GIT_SHALLOW_REVS: git fetch --shallow-exclude=<revs> rev
+        """
 
-        The upstream url of the new clone isn't set at this time, as it'll be
-        set correctly when unpacked."""
-        runfetchcmd("%s clone %s %s %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, dest), d)
+        progresshandler = GitProgressHandler(d)
+        repourl = self._get_repo_url(ud)
+        bb.utils.mkdirhier(dest)
+        init_cmd = "%s init -q" % ud.basecmd
+        if ud.bareclone:
+            init_cmd += " --bare"
+        runfetchcmd(init_cmd, d, workdir=dest)
+        # Use repourl when creating a fast initial shallow clone
+        # Prefer already existing full bare clones if available
+        if not ud.shallow_skip_fast and not os.path.exists(ud.clonedir):
+            remote = shlex.quote(repourl)
+        else:
+            remote = ud.clonedir
+        runfetchcmd("%s remote add origin %s" % (ud.basecmd, remote), d, workdir=dest)
 
-        to_parse, shallow_branches = [], []
-        for name in ud.names:
-            revision = ud.revisions[name]
-            depth = ud.shallow_depths[name]
-            if depth:
-                to_parse.append('%s~%d^{}' % (revision, depth - 1))
+        # Check the histories which should be excluded
+        shallow_exclude = ''
+        for revision in ud.shallow_revs:
+            shallow_exclude += " --shallow-exclude=%s" % revision
 
-            # For nobranch, we need a ref, otherwise the commits will be
-            # removed, and for non-nobranch, we truncate the branch to our
-            # srcrev, to avoid keeping unnecessary history beyond that.
-            branch = ud.branches[name]
-            if ud.nobranch:
-                ref = "refs/shallow/%s" % name
-            elif ud.bareclone:
-                ref = "refs/heads/%s" % branch
-            else:
-                ref = "refs/remotes/origin/%s" % branch
+        revision = ud.revision
+        depth = ud.shallow_depths[ud.name]
 
-            shallow_branches.append(ref)
-            runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
+        # The --depth and --shallow-exclude can't be used together
+        if depth and shallow_exclude:
+            raise bb.fetch2.FetchError("BB_GIT_SHALLOW_REVS is set, but BB_GIT_SHALLOW_DEPTH is not 0.")
 
-        # Map srcrev+depths to revisions
-        parsed_depths = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join(to_parse)), d, workdir=dest)
+        # For nobranch, we need a ref, otherwise the commits will be
+        # removed, and for non-nobranch, we truncate the branch to our
+        # srcrev, to avoid keeping unnecessary history beyond that.
+        branch = ud.branch
+        if ud.nobranch:
+            ref = "refs/shallow/%s" % ud.name
+        elif ud.bareclone:
+            ref = "refs/heads/%s" % branch
+        else:
+            ref = "refs/remotes/origin/%s" % branch
+
+        fetch_cmd = "%s fetch origin %s" % (ud.basecmd, revision)
+        if depth:
+            fetch_cmd += " --depth %s" % depth
+
+        if shallow_exclude:
+            fetch_cmd += shallow_exclude
 
-        # Resolve specified revisions
-        parsed_revs = runfetchcmd("%s rev-parse %s" % (ud.basecmd, " ".join('"%s^{}"' % r for r in ud.shallow_revs)), d, workdir=dest)
-        shallow_revisions = parsed_depths.splitlines() + parsed_revs.splitlines()
+        # Advertise the revision for lower version git such as 2.25.1:
+        # error: Server does not allow request for unadvertised object.
+        # The ud.clonedir is a local temporary dir, will be removed when
+        # fetch is done, so we can do anything on it.
+        adv_cmd = 'git branch -f advertise-%s %s' % (revision, revision)
+        if ud.shallow_skip_fast:
+            runfetchcmd(adv_cmd, d, workdir=ud.clonedir)
+
+        runfetchcmd(fetch_cmd, d, workdir=dest)
+        runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
+        # Fetch Git LFS data
+        self.lfs_fetch(ud, d, dest, ud.revision)
 
         # Apply extra ref wildcards
-        all_refs = runfetchcmd('%s for-each-ref "--format=%%(refname)"' % ud.basecmd,
-                               d, workdir=dest).splitlines()
+        all_refs_remote = runfetchcmd("%s ls-remote origin 'refs/*'" % ud.basecmd, \
+                                        d, workdir=dest).splitlines()
+        all_refs = []
+        for line in all_refs_remote:
+            all_refs.append(line.split()[-1])
+        extra_refs = []
         for r in ud.shallow_extra_refs:
             if not ud.bareclone:
                 r = r.replace('refs/heads/', 'refs/remotes/origin/')
 
             if '*' in r:
                 matches = filter(lambda a: fnmatch.fnmatchcase(a, r), all_refs)
-                shallow_branches.extend(matches)
+                extra_refs.extend(matches)
             else:
-                shallow_branches.append(r)
+                extra_refs.append(r)
+
+        for ref in extra_refs:
+            ref_fetch = ref.replace('refs/heads/', '').replace('refs/remotes/origin/', '').replace('refs/tags/', '')
+            runfetchcmd("%s fetch origin --depth 1 %s" % (ud.basecmd, ref_fetch), d, workdir=dest)
+            revision = runfetchcmd("%s rev-parse FETCH_HEAD" % ud.basecmd, d, workdir=dest)
+            runfetchcmd("%s update-ref %s %s" % (ud.basecmd, ref, revision), d, workdir=dest)
 
-        # Make the repository shallow
-        shallow_cmd = [self.make_shallow_path, '-s']
-        for b in shallow_branches:
-            shallow_cmd.append('-r')
-            shallow_cmd.append(b)
-        shallow_cmd.extend(shallow_revisions)
-        runfetchcmd(subprocess.list2cmdline(shallow_cmd), d, workdir=dest)
+        # The url is local ud.clonedir, set it to upstream one
+        runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=dest)
 
     def unpack(self, ud, destdir, d):
         """ unpack the downloaded src to destdir"""
 
-        subdir = ud.parm.get("subpath", "")
-        if subdir != "":
-            readpathspec = ":%s" % subdir
-            def_destsuffix = "%s/" % os.path.basename(subdir.rstrip('/'))
-        else:
-            readpathspec = ""
-            def_destsuffix = "git/"
+        subdir = ud.parm.get("subdir")
+        subpath = ud.parm.get("subpath")
+        readpathspec = ""
+        def_destsuffix = (d.getVar("BB_GIT_DEFAULT_DESTSUFFIX") or "git") + "/"
+
+        if subpath:
+            readpathspec = ":%s" % subpath
+            def_destsuffix = "%s/" % os.path.basename(subpath.rstrip('/'))
+
+        if subdir:
+            # If 'subdir' param exists, create a dir and use it as destination for unpack cmd
+            if os.path.isabs(subdir):
+                if not os.path.realpath(subdir).startswith(os.path.realpath(destdir)):
+                    raise bb.fetch2.UnpackError("subdir argument isn't a subdirectory of unpack root %s" % destdir, ud.url)
+                destdir = subdir
+            else:
+                destdir = os.path.join(destdir, subdir)
+            def_destsuffix = ""
 
         destsuffix = ud.parm.get("destsuffix", def_destsuffix)
         destdir = ud.destdir = os.path.join(destdir, destsuffix)
         if os.path.exists(destdir):
             bb.utils.prunedir(destdir)
+        if not ud.bareclone:
+            ud.unpack_tracer.unpack("git", destdir)
 
         need_lfs = self._need_lfs(ud)
 
@@ -516,13 +691,12 @@ class Git(FetchMethod):
         source_found = False
         source_error = []
 
-        if not source_found:
-            clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
-            if clonedir_is_up_to_date:
-                runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
-                source_found = True
-            else:
-                source_error.append("clone directory not available or not up to date: " + ud.clonedir)
+        clonedir_is_up_to_date = not self.clonedir_need_update(ud, d)
+        if clonedir_is_up_to_date:
+            runfetchcmd("%s clone %s %s/ %s" % (ud.basecmd, ud.cloneflags, ud.clonedir, destdir), d)
+            source_found = True
+        else:
+            source_error.append("clone directory not available or not up to date: " + ud.clonedir)
 
         if not source_found:
             if ud.shallow:
@@ -538,28 +712,43 @@ class Git(FetchMethod):
         if not source_found:
             raise bb.fetch2.UnpackError("No up to date source found: " + "; ".join(source_error), ud.url)
 
+        # If there is a tag parameter in the url and we also have a fixed srcrev, check the tag
+        # matches the revision
+        if 'tag' in ud.parm and sha1_re.match(ud.revision):
+            output = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.parm['tag']), d, workdir=destdir)
+            output = output.strip()
+            if output != ud.revision:
+                # It is possible ud.revision is the revision on an annotated tag which won't match the output of rev-list
+                # If it resolves to the same thing there isn't a problem.
+                output2 = runfetchcmd("%s rev-list -n 1 %s" % (ud.basecmd, ud.revision), d, workdir=destdir)
+                output2 = output2.strip()
+                if output != output2:
+                    raise bb.fetch2.FetchError("The revision the git tag '%s' resolved to didn't match the SRCREV in use (%s vs %s)" % (ud.parm['tag'], output, ud.revision), ud.url)
+
         repourl = self._get_repo_url(ud)
         runfetchcmd("%s remote set-url origin %s" % (ud.basecmd, shlex.quote(repourl)), d, workdir=destdir)
 
         if self._contains_lfs(ud, d, destdir):
-            if need_lfs and not self._find_git_lfs(d):
-                raise bb.fetch2.FetchError("Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 to ignore it)" % (repourl))
-            elif not need_lfs:
+            if not need_lfs:
                 bb.note("Repository %s has LFS content but it is not being fetched" % (repourl))
+            else:
+                self._ensure_git_lfs(d, ud)
+
+                runfetchcmd("%s lfs install --local" % ud.basecmd, d, workdir=destdir)
 
         if not ud.nocheckout:
-            if subdir != "":
-                runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revisions[ud.names[0]], readpathspec), d,
+            if subpath:
+                runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.revision, readpathspec), d,
                             workdir=destdir)
                 runfetchcmd("%s checkout-index -q -f -a" % ud.basecmd, d, workdir=destdir)
             elif not ud.nobranch:
-                branchname =  ud.branches[ud.names[0]]
+                branchname =  ud.branch
                 runfetchcmd("%s checkout -B %s %s" % (ud.basecmd, branchname, \
-                            ud.revisions[ud.names[0]]), d, workdir=destdir)
+                            ud.revision), d, workdir=destdir)
                 runfetchcmd("%s branch %s --set-upstream-to origin/%s" % (ud.basecmd, branchname, \
                             branchname), d, workdir=destdir)
             else:
-                runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=destdir)
+                runfetchcmd("%s checkout %s" % (ud.basecmd, ud.revision), d, workdir=destdir)
 
         return True
 
@@ -573,8 +762,13 @@ class Git(FetchMethod):
             clonedir = os.path.realpath(ud.localpath)
             to_remove.append(clonedir)
 
+        # Remove shallow mirror tarball
+        if ud.shallow:
+            to_remove.append(ud.fullshallow)
+            to_remove.append(ud.fullshallow + ".done")
+
         for r in to_remove:
-            if os.path.exists(r):
+            if os.path.exists(r) or os.path.islink(r):
                 bb.note('Removing %s' % r)
                 bb.utils.remove(r, True)
 
@@ -585,10 +779,10 @@ class Git(FetchMethod):
         cmd = ""
         if ud.nobranch:
             cmd = "%s log --pretty=oneline -n 1 %s -- 2> /dev/null | wc -l" % (
-                ud.basecmd, ud.revisions[name])
+                ud.basecmd, ud.revision)
         else:
             cmd =  "%s branch --contains %s --list %s 2> /dev/null | wc -l" % (
-                ud.basecmd, ud.revisions[name], ud.branches[name])
+                ud.basecmd, ud.revision, ud.branch)
         try:
             output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
         except bb.fetch2.FetchError:
@@ -597,6 +791,37 @@ class Git(FetchMethod):
             raise bb.fetch2.FetchError("The command '%s' gave output with more then 1 line unexpectedly, output: '%s'" % (cmd, output))
         return output.split()[0] != "0"
 
+    def _lfs_objects_downloaded(self, ud, d, wd):
+        """
+        Verifies whether the LFS objects for requested revisions have already been downloaded
+        """
+        # Bail out early if this repository doesn't use LFS
+        if not self._contains_lfs(ud, d, wd):
+            return True
+
+        self._ensure_git_lfs(d, ud)
+
+        # The Git LFS specification specifies ([1]) the LFS folder layout so it should be safe to check for file
+        # existence.
+        # [1] https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md#intercepting-git
+        cmd = "%s lfs ls-files -l %s" \
+                % (ud.basecmd, ud.revision)
+        output = runfetchcmd(cmd, d, quiet=True, workdir=wd).rstrip()
+        # Do not do any further matching if no objects are managed by LFS
+        if not output:
+            return True
+
+        # Match all lines beginning with the hexadecimal OID
+        oid_regex = re.compile("^(([a-fA-F0-9]{2})([a-fA-F0-9]{2})[A-Fa-f0-9]+)")
+        for line in output.split("\n"):
+            oid = re.search(oid_regex, line)
+            if not oid:
+                bb.warn("git lfs ls-files output '%s' did not match expected format." % line)
+            if not os.path.exists(os.path.join(wd, "lfs", "objects", oid.group(2), oid.group(3), oid.group(1))):
+                return False
+
+        return True
+
     def _need_lfs(self, ud):
         return ud.parm.get("lfs", "1") == "1"
 
@@ -604,20 +829,8 @@ class Git(FetchMethod):
         """
         Check if the repository has 'lfs' (large file) content
         """
-
-        if not ud.nobranch:
-            branchname = ud.branches[ud.names[0]]
-        else:
-            branchname = "master"
-
-        # The bare clonedir doesn't use the remote names; it has the branch immediately.
-        if wd == ud.clonedir:
-            refname = ud.branches[ud.names[0]]
-        else:
-            refname = "origin/%s" % ud.branches[ud.names[0]]
-
         cmd = "%s grep lfs %s:.gitattributes | wc -l" % (
-            ud.basecmd, refname)
+            ud.basecmd, ud.revision)
 
         try:
             output = runfetchcmd(cmd, d, quiet=True, workdir=wd)
@@ -627,12 +840,14 @@ class Git(FetchMethod):
             pass
         return False
 
-    def _find_git_lfs(self, d):
+    def _ensure_git_lfs(self, d, ud):
         """
-        Return True if git-lfs can be found, False otherwise.
+        Ensures that git-lfs is available, raising a FetchError if it isn't.
         """
-        import shutil
-        return shutil.which("git-lfs", path=d.getVar('PATH')) is not None
+        if shutil.which("git-lfs", path=d.getVar('PATH')) is None:
+            raise bb.fetch2.FetchError(
+                "Repository %s has LFS content, install git-lfs on host to download (or set lfs=0 "
+                "to ignore it)" % self._get_repo_url(ud))
 
     def _get_repo_url(self, ud):
         """
@@ -640,22 +855,21 @@ class Git(FetchMethod):
         """
         # Note that we do not support passwords directly in the git urls. There are several
         # reasons. SRC_URI can be written out to things like buildhistory and people don't
-        # want to leak passwords like that. Its also all too easy to share metadata without 
+        # want to leak passwords like that. Its also all too easy to share metadata without
         # removing the password. ssh keys, ~/.netrc and ~/.ssh/config files can be used as
         # alternatives so we will not take patches adding password support here.
         if ud.user:
             username = ud.user + '@'
         else:
             username = ""
-        return "%s://%s%s%s" % (ud.proto, username, ud.host, ud.path)
+        return "%s://%s%s%s" % (ud.proto, username, ud.host, urllib.parse.quote(ud.path))
 
     def _revision_key(self, ud, d, name):
         """
         Return a unique key for the url
         """
         # Collapse adjacent slashes
-        slash_re = re.compile(r"/+")
-        return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev[name]
+        return "git:" + ud.host + slash_re.sub(".", ud.path) + ud.unresolvedrev
 
     def _lsremote(self, ud, d, search):
         """
@@ -687,21 +901,27 @@ class Git(FetchMethod):
         """
         Compute the HEAD revision for the url
         """
+        if not d.getVar("__BBSRCREV_SEEN"):
+            raise bb.fetch2.FetchError("Recipe uses a floating tag/branch '%s' for repo '%s' without a fixed SRCREV yet doesn't call bb.fetch2.get_srcrev() (use SRCPV in PV for OE)." % (ud.unresolvedrev, ud.host+ud.path))
+
+        # Ensure we mark as not cached
+        bb.fetch2.mark_recipe_nocache(d)
+
         output = self._lsremote(ud, d, "")
         # Tags of the form ^{} may not work, need to fallback to other form
-        if ud.unresolvedrev[name][:5] == "refs/" or ud.usehead:
-            head = ud.unresolvedrev[name]
-            tag = ud.unresolvedrev[name]
+        if ud.unresolvedrev[:5] == "refs/" or ud.usehead:
+            head = ud.unresolvedrev
+            tag = ud.unresolvedrev
         else:
-            head = "refs/heads/%s" % ud.unresolvedrev[name]
-            tag = "refs/tags/%s" % ud.unresolvedrev[name]
+            head = "refs/heads/%s" % ud.unresolvedrev
+            tag = "refs/tags/%s" % ud.unresolvedrev
         for s in [head, tag + "^{}", tag]:
             for l in output.strip().split('\n'):
                 sha1, ref = l.split()
                 if s == ref:
                     return sha1
         raise bb.fetch2.FetchError("Unable to resolve '%s' in upstream git repository in git ls-remote output for %s" % \
-            (ud.unresolvedrev[name], ud.host+ud.path))
+            (ud.unresolvedrev, ud.host+ud.path))
 
     def latest_versionstring(self, ud, d):
         """
@@ -711,60 +931,63 @@ class Git(FetchMethod):
         """
         pupver = ('', '')
 
-        tagregex = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
         try:
             output = self._lsremote(ud, d, "refs/tags/*")
         except (bb.fetch2.FetchError, bb.fetch2.NetworkAccess) as e:
             bb.note("Could not list remote: %s" % str(e))
             return pupver
 
+        rev_tag_re = re.compile(r"([0-9a-f]{40})\s+refs/tags/(.*)")
+        pver_re = re.compile(d.getVar('UPSTREAM_CHECK_GITTAGREGEX') or r"(?P<pver>([0-9][\.|_]?)+)")
+        nonrel_re = re.compile(r"(alpha|beta|rc|final)+")
+
         verstring = ""
-        revision = ""
         for line in output.split("\n"):
             if not line:
                 break
 
-            tag_head = line.split("/")[-1]
+            m = rev_tag_re.match(line)
+            if not m:
+                continue
+
+            (revision, tag) = m.groups()
+
             # Ignore non-released branches
-            m = re.search(r"(alpha|beta|rc|final)+", tag_head)
-            if m:
+            if nonrel_re.search(tag):
                 continue
 
             # search for version in the line
-            tag = tagregex.search(tag_head)
-            if tag is None:
+            m = pver_re.search(tag)
+            if not m:
                 continue
 
-            tag = tag.group('pver')
-            tag = tag.replace("_", ".")
+            pver = m.group('pver').replace("_", ".")
 
-            if verstring and bb.utils.vercmp(("0", tag, ""), ("0", verstring, "")) < 0:
+            if verstring and bb.utils.vercmp(("0", pver, ""), ("0", verstring, "")) < 0:
                 continue
 
-            verstring = tag
-            revision = line.split()[0]
+            verstring = pver
             pupver = (verstring, revision)
 
         return pupver
 
     def _build_revision(self, ud, d, name):
-        return ud.revisions[name]
+        return ud.revision
 
     def gitpkgv_revision(self, ud, d, name):
         """
         Return a sortable revision number by counting commits in the history
         Based on gitpkgv.bblass in meta-openembedded
         """
-        rev = self._build_revision(ud, d, name)
+        rev = ud.revision
         localpath = ud.localpath
         rev_file = os.path.join(localpath, "oe-gitpkgv_" + rev)
         if not os.path.exists(localpath):
             commits = None
         else:
             if not os.path.exists(rev_file) or not os.path.getsize(rev_file):
-                from pipes import quote
                 commits = bb.fetch2.runfetchcmd(
-                        "git rev-list %s -- | wc -l" % quote(rev),
+                        "git rev-list %s -- | wc -l" % shlex.quote(rev),
                         d, quiet=True).strip().lstrip('0')
                 if commits:
                     open(rev_file, "w").write("%d\n" % int(commits))
diff --git a/bitbake/lib/bb/fetch2/gitsm.py b/bitbake/lib/bb/fetch2/gitsm.py
index a4527bf364..5869e1b99b 100644
--- a/bitbake/lib/bb/fetch2/gitsm.py
+++ b/bitbake/lib/bb/fetch2/gitsm.py
@@ -62,36 +62,35 @@ class GitSM(Git):
             return modules
 
         # Collect the defined submodules, and their attributes
-        for name in ud.names:
+        try:
+            gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revision), d, quiet=True, workdir=workdir)
+        except:
+            # No submodules to update
+            gitmodules = ""
+
+        for m, md in parse_gitmodules(gitmodules).items():
             try:
-                gitmodules = runfetchcmd("%s show %s:.gitmodules" % (ud.basecmd, ud.revisions[name]), d, quiet=True, workdir=workdir)
+                module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revision, md['path']), d, quiet=True, workdir=workdir)
             except:
-                # No submodules to update
+                # If the command fails, we don't have a valid file to check.  If it doesn't
+                # fail -- it still might be a failure, see next check...
+                module_hash = ""
+
+            if not module_hash:
+                logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
                 continue
 
-            for m, md in parse_gitmodules(gitmodules).items():
-                try:
-                    module_hash = runfetchcmd("%s ls-tree -z -d %s %s" % (ud.basecmd, ud.revisions[name], md['path']), d, quiet=True, workdir=workdir)
-                except:
-                    # If the command fails, we don't have a valid file to check.  If it doesn't
-                    # fail -- it still might be a failure, see next check...
-                    module_hash = ""
-
-                if not module_hash:
-                    logger.debug("submodule %s is defined, but is not initialized in the repository. Skipping", m)
-                    continue
-
-                submodules.append(m)
-                paths[m] = md['path']
-                revision[m] = ud.revisions[name]
-                uris[m] = md['url']
-                subrevision[m] = module_hash.split()[2]
-
-                # Convert relative to absolute uri based on parent uri
-                if uris[m].startswith('..'):
-                    newud = copy.copy(ud)
-                    newud.path = os.path.realpath(os.path.join(newud.path, uris[m]))
-                    uris[m] = Git._get_repo_url(self, newud)
+            submodules.append(m)
+            paths[m] = md['path']
+            revision[m] = ud.revision
+            uris[m] = md['url']
+            subrevision[m] = module_hash.split()[2]
+
+            # Convert relative to absolute uri based on parent uri
+            if  uris[m].startswith('..') or uris[m].startswith('./'):
+                newud = copy.copy(ud)
+                newud.path = os.path.normpath(os.path.join(newud.path, uris[m]))
+                uris[m] = Git._get_repo_url(self, newud)
 
         for module in submodules:
             # Translate the module url into a SRC_URI
@@ -115,10 +114,21 @@ class GitSM(Git):
                     # This has to be a file reference
                     proto = "file"
                     url = "gitsm://" + uris[module]
+            if url.endswith("{}{}".format(ud.host, ud.path)):
+                raise bb.fetch2.FetchError("Submodule refers to the parent repository. This will cause deadlock situation in current version of Bitbake." \
+                                           "Consider using git fetcher instead.")
 
             url += ';protocol=%s' % proto
             url += ";name=%s" % module
             url += ";subpath=%s" % module
+            url += ";nobranch=1"
+            url += ";lfs=%s" % ("1" if self._need_lfs(ud) else "0")
+            # Note that adding "user=" here to give credentials to the
+            # submodule is not supported. Since using SRC_URI to give git://
+            # URL a password is not supported, one have to use one of the
+            # recommended way (eg. ~/.netrc or SSH config) which does specify
+            # the user (See comment in git.py).
+            # So, we will not take patches adding "user=" support here.
 
             ld = d.createCopy()
             # Not necessary to set SRC_URI, since we're passing the URI to
@@ -136,20 +146,26 @@ class GitSM(Git):
 
         return submodules != []
 
+    def call_process_submodules(self, ud, d, extra_check, subfunc):
+        # If we're using a shallow mirror tarball it needs to be
+        # unpacked temporarily so that we can examine the .gitmodules file
+        # Unpack even when ud.clonedir is not available,
+        # which may occur during a fast shallow clone
+        unpack = extra_check or not os.path.exists(ud.clonedir)
+        if ud.shallow and os.path.exists(ud.fullshallow) and unpack:
+            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
+            try:
+                runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
+                self.process_submodules(ud, tmpdir, subfunc, d)
+            finally:
+                shutil.rmtree(tmpdir)
+        else:
+            self.process_submodules(ud, ud.clonedir, subfunc, d)
+
     def need_update(self, ud, d):
         if Git.need_update(self, ud, d):
             return True
 
-        try:
-            # Check for the nugget dropped by the download operation
-            known_srcrevs = runfetchcmd("%s config --get-all bitbake.srcrev" % \
-                                        (ud.basecmd), d, workdir=ud.clonedir)
-
-            if ud.revisions[ud.names[0]] in known_srcrevs.split():
-                return False
-        except bb.fetch2.FetchError:
-            pass
-
         need_update_list = []
         def need_update_submodule(ud, url, module, modpath, workdir, d):
             url += ";bareclone=1;nobranch=1"
@@ -163,22 +179,9 @@ class GitSM(Git):
                 logger.error('gitsm: submodule update check failed: %s %s' % (type(e).__name__, str(e)))
                 need_update_result = True
 
-        # If we're using a shallow mirror tarball it needs to be unpacked
-        # temporarily so that we can examine the .gitmodules file
-        if ud.shallow and os.path.exists(ud.fullshallow) and not os.path.exists(ud.clonedir):
-            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
-            runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
-            self.process_submodules(ud, tmpdir, need_update_submodule, d)
-            shutil.rmtree(tmpdir)
-        else:
-            self.process_submodules(ud, ud.clonedir, need_update_submodule, d)
-            if len(need_update_list) == 0:
-                # We already have the required commits of all submodules. Drop
-                # a nugget so we don't need to check again.
-                runfetchcmd("%s config --add bitbake.srcrev %s" % \
-                            (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
-
-        if len(need_update_list) > 0:
+        self.call_process_submodules(ud, d, not os.path.exists(ud.clonedir), need_update_submodule)
+
+        if need_update_list:
             logger.debug('gitsm: Submodules requiring update: %s' % (' '.join(need_update_list)))
             return True
 
@@ -199,19 +202,7 @@ class GitSM(Git):
                 raise
 
         Git.download(self, ud, d)
-
-        # If we're using a shallow mirror tarball it needs to be unpacked
-        # temporarily so that we can examine the .gitmodules file
-        if ud.shallow and os.path.exists(ud.fullshallow) and self.need_update(ud, d):
-            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
-            runfetchcmd("tar -xzf %s" % ud.fullshallow, d, workdir=tmpdir)
-            self.process_submodules(ud, tmpdir, download_submodule, d)
-            shutil.rmtree(tmpdir)
-        else:
-            self.process_submodules(ud, ud.clonedir, download_submodule, d)
-            # Drop a nugget for the srcrev we've fetched (used by need_update)
-            runfetchcmd("%s config --add bitbake.srcrev %s" % \
-                        (ud.basecmd, ud.revisions[ud.names[0]]), d, workdir=ud.clonedir)
+        self.call_process_submodules(ud, d, self.need_update(ud, d), download_submodule)
 
     def unpack(self, ud, destdir, d):
         def unpack_submodules(ud, url, module, modpath, workdir, d):
@@ -225,6 +216,10 @@ class GitSM(Git):
 
             try:
                 newfetch = Fetch([url], d, cache=False)
+                # modpath is needed by unpack tracer to calculate submodule
+                # checkout dir
+                new_ud = newfetch.ud[url]
+                new_ud.modpath = modpath
                 newfetch.unpack(root=os.path.dirname(os.path.join(repo_conf, 'modules', module)))
             except Exception as e:
                 logger.error('gitsm: submodule unpack failed: %s %s' % (type(e).__name__, str(e)))
@@ -250,13 +245,27 @@ class GitSM(Git):
         ret = self.process_submodules(ud, ud.destdir, unpack_submodules, d)
 
         if not ud.bareclone and ret:
-            # All submodules should already be downloaded and configured in the tree.  This simply sets
-            # up the configuration and checks out the files.  The main project config should remain
-            # unmodified, and no download from the internet should occur.
-            runfetchcmd("%s submodule update --recursive --no-fetch" % (ud.basecmd), d, quiet=True, workdir=ud.destdir)
+            cmdprefix = ""
+            # Avoid LFS smudging (replacing the LFS pointers with the actual content) when LFS shouldn't be used but git-lfs is installed.
+            if not self._need_lfs(ud):
+                cmdprefix = "GIT_LFS_SKIP_SMUDGE=1 "
+            runfetchcmd("%s%s submodule update --recursive --no-fetch" % (cmdprefix, ud.basecmd), d, quiet=True, workdir=ud.destdir)
+    def clean(self, ud, d):
+        def clean_submodule(ud, url, module, modpath, workdir, d):
+            url += ";bareclone=1;nobranch=1"
+            try:
+                newfetch = Fetch([url], d, cache=False)
+                newfetch.clean()
+            except Exception as e:
+                logger.warning('gitsm: submodule clean failed: %s %s' % (type(e).__name__, str(e)))
+
+        self.call_process_submodules(ud, d, True, clean_submodule)
+
+        # Clean top git dir
+        Git.clean(self, ud, d)
 
     def implicit_urldata(self, ud, d):
-        import shutil, subprocess, tempfile
+        import subprocess
 
         urldata = []
         def add_submodule(ud, url, module, modpath, workdir, d):
@@ -264,14 +273,6 @@ class GitSM(Git):
             newfetch = Fetch([url], d, cache=False)
             urldata.extend(newfetch.expanded_urldata())
 
-        # If we're using a shallow mirror tarball it needs to be unpacked
-        # temporarily so that we can examine the .gitmodules file
-        if ud.shallow and os.path.exists(ud.fullshallow) and ud.method.need_update(ud, d):
-            tmpdir = tempfile.mkdtemp(dir=d.getVar("DL_DIR"))
-            subprocess.check_call("tar -xzf %s" % ud.fullshallow, cwd=tmpdir, shell=True)
-            self.process_submodules(ud, tmpdir, add_submodule, d)
-            shutil.rmtree(tmpdir)
-        else:
-            self.process_submodules(ud, ud.clonedir, add_submodule, d)
+        self.call_process_submodules(ud, d, ud.method.need_update(ud, d), add_submodule)
 
         return urldata
diff --git a/bitbake/lib/bb/fetch2/gomod.py b/bitbake/lib/bb/fetch2/gomod.py
new file mode 100644
index 0000000000..53c1d8d115
--- /dev/null
+++ b/bitbake/lib/bb/fetch2/gomod.py
@@ -0,0 +1,273 @@
+"""
+BitBake 'Fetch' implementation for Go modules
+
+The gomod/gomodgit fetchers are used to download Go modules to the module cache
+from a module proxy or directly from a version control repository.
+
+Example SRC_URI:
+
+SRC_URI += "gomod://golang.org/x/net;version=v0.9.0;sha256sum=..."
+SRC_URI += "gomodgit://golang.org/x/net;version=v0.9.0;repo=go.googlesource.com/net;srcrev=..."
+
+Required SRC_URI parameters:
+
+- version
+    The version of the module.
+
+Optional SRC_URI parameters:
+
+- mod
+    Fetch and unpack the go.mod file only instead of the complete module.
+    The go command may need to download go.mod files for many different modules
+    when computing the build list, and go.mod files are much smaller than
+    module zip files.
+    The default is "0", set mod=1 for the go.mod file only.
+
+- sha256sum
+    The checksum of the module zip file, or the go.mod file in case of fetching
+    only the go.mod file. Alternatively, set the SRC_URI varible flag for
+    "module@version.sha256sum".
+
+- protocol
+    The method used when fetching directly from a version control repository.
+    The default is "https" for git.
+
+- repo
+    The URL when fetching directly from a version control repository. Required
+    when the URL is different from the module path.
+
+- srcrev
+    The revision identifier used when fetching directly from a version control
+    repository. Alternatively, set the SRCREV varible for "module@version".
+
+- subdir
+    The module subdirectory when fetching directly from a version control
+    repository. Required when the module is not located in the root of the
+    repository.
+
+Related variables:
+
+- GO_MOD_PROXY
+    The module proxy used by the fetcher.
+
+- GO_MOD_CACHE_DIR
+    The directory where the module cache is located.
+    This must match the exported GOMODCACHE variable for the go command to find
+    the downloaded modules.
+
+See the Go modules reference, https://go.dev/ref/mod, for more information
+about the module cache, module proxies and version control systems.
+"""
+
+import hashlib
+import os
+import re
+import shutil
+import subprocess
+import zipfile
+
+import bb
+from bb.fetch2 import FetchError
+from bb.fetch2 import MissingParameterError
+from bb.fetch2 import runfetchcmd
+from bb.fetch2 import subprocess_setup
+from bb.fetch2.git import Git
+from bb.fetch2.wget import Wget
+
+
+def escape(path):
+    """Escape capital letters using exclamation points."""
+    return re.sub(r'([A-Z])', lambda m: '!' + m.group(1).lower(), path)
+
+
+class GoMod(Wget):
+    """Class to fetch Go modules from a Go module proxy via wget"""
+
+    def supports(self, ud, d):
+        """Check to see if a given URL is for this fetcher."""
+        return ud.type == 'gomod'
+
+    def urldata_init(self, ud, d):
+        """Set up to download the module from the module proxy.
+
+        Set up to download the module zip file to the module cache directory
+        and unpack the go.mod file (unless downloading only the go.mod file):
+
+        cache/download/<module>/@v/<version>.zip: The module zip file.
+        cache/download/<module>/@v/<version>.mod: The go.mod file.
+        """
+
+        proxy = d.getVar('GO_MOD_PROXY') or 'proxy.golang.org'
+        moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
+
+        if 'version' not in ud.parm:
+            raise MissingParameterError('version', ud.url)
+
+        module = ud.host
+        if ud.path != '/':
+            module += ud.path
+        ud.parm['module'] = module
+        version = ud.parm['version']
+
+        # Set URL and filename for wget download
+        if ud.parm.get('mod', '0') == '1':
+            ext = '.mod'
+        else:
+            ext = '.zip'
+        path = escape(f"{module}/@v/{version}{ext}")
+        ud.url = bb.fetch2.encodeurl(
+            ('https', proxy, '/' + path, None, None, None))
+        ud.parm['downloadfilename'] =  f"{module.replace('/', '.')}@{version}{ext}"
+
+        # Set name for checksum verification
+        ud.parm['name'] = f"{module}@{version}"
+
+        # Set path for unpack
+        ud.parm['unpackpath'] = os.path.join(moddir, 'cache/download', path)
+
+        super().urldata_init(ud, d)
+
+    def unpack(self, ud, rootdir, d):
+        """Unpack the module in the module cache."""
+
+        # Unpack the module zip file or go.mod file
+        unpackpath = os.path.join(rootdir, ud.parm['unpackpath'])
+        unpackdir = os.path.dirname(unpackpath)
+        bb.utils.mkdirhier(unpackdir)
+        ud.unpack_tracer.unpack("file-copy", unpackdir)
+        cmd = f"cp {ud.localpath} {unpackpath}"
+        path = d.getVar('PATH')
+        if path:
+            cmd = f"PATH={path} {cmd}"
+        name = os.path.basename(unpackpath)
+        bb.note(f"Unpacking {name} to {unpackdir}/")
+        subprocess.check_call(cmd, shell=True, preexec_fn=subprocess_setup)
+
+        if name.endswith('.zip'):
+            # Unpack the go.mod file from the zip file
+            module = ud.parm['module']
+            name = name.rsplit('.', 1)[0] + '.mod'
+            bb.note(f"Unpacking {name} to {unpackdir}/")
+            with zipfile.ZipFile(ud.localpath) as zf:
+                with open(os.path.join(unpackdir, name), mode='wb') as mf:
+                    try:
+                        f = module + '@' + ud.parm['version'] + '/go.mod'
+                        shutil.copyfileobj(zf.open(f), mf)
+                    except KeyError:
+                        # If the module does not have a go.mod file, synthesize
+                        # one containing only a module statement.
+                        mf.write(f'module {module}\n'.encode())
+
+
+class GoModGit(Git):
+    """Class to fetch Go modules directly from a git repository"""
+
+    def supports(self, ud, d):
+        """Check to see if a given URL is for this fetcher."""
+        return ud.type == 'gomodgit'
+
+    def urldata_init(self, ud, d):
+        """Set up to download the module from the git repository.
+
+        Set up to download the git repository to the module cache directory and
+        unpack the module zip file and the go.mod file:
+
+        cache/vcs/<hash>:                         The bare git repository.
+        cache/download/<module>/@v/<version>.zip: The module zip file.
+        cache/download/<module>/@v/<version>.mod: The go.mod file.
+        """
+
+        moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
+
+        if 'version' not in ud.parm:
+            raise MissingParameterError('version', ud.url)
+
+        module = ud.host
+        if ud.path != '/':
+            module += ud.path
+        ud.parm['module'] = module
+
+        # Set host, path and srcrev for git download
+        if 'repo' in ud.parm:
+            repo = ud.parm['repo']
+            idx = repo.find('/')
+            if idx != -1:
+                ud.host = repo[:idx]
+                ud.path = repo[idx:]
+            else:
+                ud.host = repo
+                ud.path = ''
+        if 'protocol' not in ud.parm:
+            ud.parm['protocol'] = 'https'
+        ud.name = f"{module}@{ud.parm['version']}"
+        srcrev = d.getVar('SRCREV_' + ud.name)
+        if srcrev:
+            if 'srcrev' not in ud.parm:
+                ud.parm['srcrev'] = srcrev
+        else:
+            if 'srcrev' in ud.parm:
+                d.setVar('SRCREV_' + ud.name, ud.parm['srcrev'])
+        if 'branch' not in ud.parm:
+            ud.parm['nobranch'] = '1'
+
+        # Set subpath, subdir and bareclone for git unpack
+        if 'subdir' in ud.parm:
+            ud.parm['subpath'] = ud.parm['subdir']
+        key = f"git3:{ud.parm['protocol']}://{ud.host}{ud.path}".encode()
+        ud.parm['key'] = key
+        ud.parm['subdir'] = os.path.join(moddir, 'cache/vcs',
+                                         hashlib.sha256(key).hexdigest())
+        ud.parm['bareclone'] = '1'
+
+        super().urldata_init(ud, d)
+
+    def unpack(self, ud, rootdir, d):
+        """Unpack the module in the module cache."""
+
+        # Unpack the bare git repository
+        super().unpack(ud, rootdir, d)
+
+        moddir = d.getVar('GO_MOD_CACHE_DIR') or 'pkg/mod'
+
+        # Create the info file
+        module = ud.parm['module']
+        repodir = os.path.join(rootdir, ud.parm['subdir'])
+        with open(repodir + '.info', 'wb') as f:
+            f.write(ud.parm['key'])
+
+        # Unpack the go.mod file from the repository
+        unpackdir = os.path.join(rootdir, moddir, 'cache/download',
+                                 escape(module), '@v')
+        bb.utils.mkdirhier(unpackdir)
+        srcrev = ud.parm['srcrev']
+        version = ud.parm['version']
+        escaped_version = escape(version)
+        cmd = f"git ls-tree -r --name-only '{srcrev}'"
+        if 'subpath' in ud.parm:
+            cmd += f" '{ud.parm['subpath']}'"
+        files = runfetchcmd(cmd, d, workdir=repodir).split()
+        name = escaped_version + '.mod'
+        bb.note(f"Unpacking {name} to {unpackdir}/")
+        with open(os.path.join(unpackdir, name), mode='wb') as mf:
+            f = 'go.mod'
+            if 'subpath' in ud.parm:
+                f = os.path.join(ud.parm['subpath'], f)
+            if f in files:
+                cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
+                subprocess.check_call(cmd, stdout=mf, cwd=repodir,
+                                      preexec_fn=subprocess_setup)
+            else:
+                # If the module does not have a go.mod file, synthesize one
+                # containing only a module statement.
+                mf.write(f'module {module}\n'.encode())
+
+        # Synthesize the module zip file from the repository
+        name = escaped_version + '.zip'
+        bb.note(f"Unpacking {name} to {unpackdir}/")
+        with zipfile.ZipFile(os.path.join(unpackdir, name), mode='w') as zf:
+            prefix = module + '@' + version + '/'
+            for f in files:
+                cmd = ['git', 'cat-file', 'blob', srcrev + ':' + f]
+                data = subprocess.check_output(cmd, cwd=repodir,
+                                               preexec_fn=subprocess_setup)
+                zf.writestr(prefix + f, data)
diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py
index 063e13008a..cbff8c490c 100644
--- a/bitbake/lib/bb/fetch2/hg.py
+++ b/bitbake/lib/bb/fetch2/hg.py
@@ -242,6 +242,7 @@ class Hg(FetchMethod):
         revflag = "-r %s" % ud.revision
         subdir = ud.parm.get("destsuffix", ud.module)
         codir = "%s/%s" % (destdir, subdir)
+        ud.unpack_tracer.unpack("hg", codir)
 
         scmdata = ud.parm.get("scmdata", "")
         if scmdata != "nokeep":
diff --git a/bitbake/lib/bb/fetch2/local.py b/bitbake/lib/bb/fetch2/local.py
index e7d1c8c58f..fda56a564e 100644
--- a/bitbake/lib/bb/fetch2/local.py
+++ b/bitbake/lib/bb/fetch2/local.py
@@ -29,11 +29,10 @@ class Local(FetchMethod):
 
     def urldata_init(self, ud, d):
         # We don't set localfile as for this fetcher the file is already local!
-        ud.decodedurl = urllib.parse.unquote(ud.url.split("://")[1].split(";")[0])
-        ud.basename = os.path.basename(ud.decodedurl)
-        ud.basepath = ud.decodedurl
+        ud.basename = os.path.basename(ud.path)
+        ud.basepath = ud.path
         ud.needdonestamp = False
-        if "*" in ud.decodedurl:
+        if "*" in ud.path:
             raise bb.fetch2.ParameterError("file:// urls using globbing are no longer supported. Please place the files in a directory and reference that instead.", ud.url)
         return
 
@@ -41,28 +40,24 @@ class Local(FetchMethod):
         """
         Return the local filename of a given url assuming a successful fetch.
         """
-        return self.localpaths(urldata, d)[-1]
+        return self.localfile_searchpaths(urldata, d)[-1]
 
-    def localpaths(self, urldata, d):
+    def localfile_searchpaths(self, urldata, d):
         """
         Return the local filename of a given url assuming a successful fetch.
         """
         searched = []
-        path = urldata.decodedurl
+        path = urldata.path
         newpath = path
         if path[0] == "/":
+            logger.debug2("Using absolute %s" % (path))
             return [path]
         filespath = d.getVar('FILESPATH')
         if filespath:
             logger.debug2("Searching for %s in paths:\n    %s" % (path, "\n    ".join(filespath.split(":"))))
             newpath, hist = bb.utils.which(filespath, path, history=True)
+            logger.debug2("Using %s for %s" % (newpath, path))
             searched.extend(hist)
-        if not os.path.exists(newpath):
-            dldirfile = os.path.join(d.getVar("DL_DIR"), path)
-            logger.debug2("Defaulting to %s for %s" % (dldirfile, path))
-            bb.utils.mkdirhier(os.path.dirname(dldirfile))
-            searched.append(dldirfile)
-            return searched
         return searched
 
     def need_update(self, ud, d):
@@ -78,9 +73,7 @@ class Local(FetchMethod):
             filespath = d.getVar('FILESPATH')
             if filespath:
                 locations = filespath.split(":")
-            locations.append(d.getVar("DL_DIR"))
-
-            msg = "Unable to find file " + urldata.url + " anywhere. The paths that were searched were:\n    " + "\n    ".join(locations)
+            msg = "Unable to find file " + urldata.url + " anywhere to download to " + urldata.localpath + ". The paths that were searched were:\n    " + "\n    ".join(locations)
             raise FetchError(msg)
 
         return True
diff --git a/bitbake/lib/bb/fetch2/npm.py b/bitbake/lib/bb/fetch2/npm.py
index 47898509ff..e469d66768 100644
--- a/bitbake/lib/bb/fetch2/npm.py
+++ b/bitbake/lib/bb/fetch2/npm.py
@@ -42,19 +42,27 @@ from bb.utils import is_semver
 
 def npm_package(package):
     """Convert the npm package name to remove unsupported character"""
-    # Scoped package names (with the @) use the same naming convention
-    # as the 'npm pack' command.
-    if package.startswith("@"):
-        return re.sub("/", "-", package[1:])
-    return package
+    # For scoped package names ('@user/package') the '/' is replaced by a '-'.
+    # This is similar to what 'npm pack' does, but 'npm pack' also strips the
+    # leading '@', which can lead to ambiguous package names.
+    name = re.sub("/", "-", package)
+    name = name.lower()
+    name = re.sub(r"[^\-a-z0-9@]", "", name)
+    name = name.strip("-")
+    return name
+
 
 def npm_filename(package, version):
     """Get the filename of a npm package"""
     return npm_package(package) + "-" + version + ".tgz"
 
-def npm_localfile(package, version):
+def npm_localfile(package, version=None):
     """Get the local filename of a npm package"""
-    return os.path.join("npm2", npm_filename(package, version))
+    if version is not None:
+        filename = npm_filename(package, version)
+    else:
+        filename = package
+    return os.path.join("npm2", filename)
 
 def npm_integrity(integrity):
     """
@@ -69,41 +77,67 @@ def npm_unpack(tarball, destdir, d):
     bb.utils.mkdirhier(destdir)
     cmd = "tar --extract --gzip --file=%s" % shlex.quote(tarball)
     cmd += " --no-same-owner"
+    cmd += " --delay-directory-restore"
     cmd += " --strip-components=1"
     runfetchcmd(cmd, d, workdir=destdir)
+    runfetchcmd("chmod -R +X '%s'" % (destdir), d, quiet=True, workdir=destdir)
 
 class NpmEnvironment(object):
     """
     Using a npm config file seems more reliable than using cli arguments.
     This class allows to create a controlled environment for npm commands.
     """
-    def __init__(self, d, configs=None):
+    def __init__(self, d, configs=[], npmrc=None):
         self.d = d
-        self.configs = configs
+
+        self.user_config = tempfile.NamedTemporaryFile(mode="w", buffering=1)
+
+        hn = self._home_npmrc(d)
+        if hn is not None:
+            with open(hn, 'r') as hnf:
+                self.user_config.write(hnf.read())
+
+        for key, value in configs:
+            self.user_config.write("%s=%s\n" % (key, value))
+
+        if npmrc:
+            self.global_config_name = npmrc
+        else:
+            self.global_config_name = "/dev/null"
+
+    def __del__(self):
+        if self.user_config:
+            self.user_config.close()
+
+    def _home_npmrc(self, d):
+        """Function to return user's HOME .npmrc file (or None if it doesn't exist)"""
+        home_npmrc_file = os.path.join(os.environ.get("HOME"), ".npmrc")
+        if d.getVar("BB_USE_HOME_NPMRC") == "1" and os.path.exists(home_npmrc_file):
+            bb.warn(f"BB_USE_HOME_NPMRC flag set and valid .npmrc detected - "\
+                    f"npm fetcher will use {home_npmrc_file}")
+            return home_npmrc_file
+        return None
 
     def run(self, cmd, args=None, configs=None, workdir=None):
         """Run npm command in a controlled environment"""
         with tempfile.TemporaryDirectory() as tmpdir:
             d = bb.data.createCopy(self.d)
+            d.setVar("PATH", d.getVar("PATH"))  # PATH might contain $HOME - evaluate it before patching
             d.setVar("HOME", tmpdir)
 
-            cfgfile = os.path.join(tmpdir, "npmrc")
-
             if not workdir:
                 workdir = tmpdir
 
             def _run(cmd):
-                cmd = "NPM_CONFIG_USERCONFIG=%s " % cfgfile + cmd
-                cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % cfgfile + cmd
+                cmd = "NPM_CONFIG_USERCONFIG=%s " % (self.user_config.name) + cmd
+                cmd = "NPM_CONFIG_GLOBALCONFIG=%s " % (self.global_config_name) + cmd
                 return runfetchcmd(cmd, d, workdir=workdir)
 
-            if self.configs:
-                for key, value in self.configs:
-                    _run("npm config set %s %s" % (key, shlex.quote(value)))
-
             if configs:
+                bb.warn("Use of configs argument of NpmEnvironment.run() function"
+                    " is deprecated. Please use args argument instead.")
                 for key, value in configs:
-                    _run("npm config set %s %s" % (key, shlex.quote(value)))
+                    cmd += " --%s=%s" % (key, shlex.quote(value))
 
             if args:
                 for key, value in args:
@@ -142,12 +176,12 @@ class Npm(FetchMethod):
             raise ParameterError("Invalid 'version' parameter", ud.url)
 
         # Extract the 'registry' part of the url
-        ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0])
+        ud.registry = re.sub(r"^npm://", "https://", ud.url.split(";")[0])
 
         # Using the 'downloadfilename' parameter as local filename
         # or the npm package name.
         if "downloadfilename" in ud.parm:
-            ud.localfile = d.expand(ud.parm["downloadfilename"])
+            ud.localfile = npm_localfile(ud.parm["downloadfilename"])
         else:
             ud.localfile = npm_localfile(ud.package, ud.version)
 
@@ -165,14 +199,14 @@ class Npm(FetchMethod):
 
     def _resolve_proxy_url(self, ud, d):
         def _npm_view():
-            configs = []
-            configs.append(("json", "true"))
-            configs.append(("registry", ud.registry))
+            args = []
+            args.append(("json", "true"))
+            args.append(("registry", ud.registry))
             pkgver = shlex.quote(ud.package + "@" + ud.version)
             cmd = ud.basecmd + " view %s" % pkgver
             env = NpmEnvironment(d)
             check_network_access(d, cmd, ud.registry)
-            view_string = env.run(cmd, configs=configs)
+            view_string = env.run(cmd, args=args)
 
             if not view_string:
                 raise FetchError("Unavailable package %s" % pkgver, ud.url)
@@ -280,6 +314,7 @@ class Npm(FetchMethod):
         destsuffix = ud.parm.get("destsuffix", "npm")
         destdir = os.path.join(rootdir, destsuffix)
         npm_unpack(ud.localpath, destdir, d)
+        ud.unpack_tracer.unpack("npm", destdir)
 
     def clean(self, ud, d):
         """Clean any existing full or partial download"""
diff --git a/bitbake/lib/bb/fetch2/npmsw.py b/bitbake/lib/bb/fetch2/npmsw.py
index 0c3511d8ab..2f9599ee9e 100644
--- a/bitbake/lib/bb/fetch2/npmsw.py
+++ b/bitbake/lib/bb/fetch2/npmsw.py
@@ -24,34 +24,39 @@ import bb
 from bb.fetch2 import Fetch
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import ParameterError
+from bb.fetch2 import runfetchcmd
 from bb.fetch2 import URI
 from bb.fetch2.npm import npm_integrity
 from bb.fetch2.npm import npm_localfile
 from bb.fetch2.npm import npm_unpack
 from bb.utils import is_semver
+from bb.utils import lockfile
+from bb.utils import unlockfile
 
 def foreach_dependencies(shrinkwrap, callback=None, dev=False):
     """
         Run a callback for each dependencies of a shrinkwrap file.
         The callback is using the format:
-            callback(name, params, deptree)
+            callback(name, data, location)
         with:
             name = the package name (string)
-            params = the package parameters (dictionary)
-            deptree = the package dependency tree (array of strings)
+            data = the package data (dictionary)
+            location = the location of the package (string)
     """
-    def _walk_deps(deps, deptree):
-        for name in deps:
-            subtree = [*deptree, name]
-            _walk_deps(deps[name].get("dependencies", {}), subtree)
-            if callback is not None:
-                if deps[name].get("dev", False) and not dev:
-                    continue
-                elif deps[name].get("bundled", False):
-                    continue
-                callback(name, deps[name], subtree)
-
-    _walk_deps(shrinkwrap.get("dependencies", {}), [])
+    packages = shrinkwrap.get("packages")
+    if not packages:
+        raise FetchError("Invalid shrinkwrap file format")
+
+    for location, data in packages.items():
+        # Skip empty main and local link target packages
+        if not location.startswith('node_modules/'):
+            continue
+        elif not dev and data.get("dev", False):
+            continue
+        elif data.get("inBundle", False):
+            continue
+        name = location.split('node_modules/')[-1]
+        callback(name, data, location)
 
 class NpmShrinkWrap(FetchMethod):
     """Class to fetch all package from a shrinkwrap file"""
@@ -72,19 +77,28 @@ class NpmShrinkWrap(FetchMethod):
         # Resolve the dependencies
         ud.deps = []
 
-        def _resolve_dependency(name, params, deptree):
+        def _resolve_dependency(name, params, destsuffix):
             url = None
             localpath = None
             extrapaths = []
-            destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
-            destsuffix = os.path.join(*destsubdirs)
+            unpack = True
 
-            integrity = params.get("integrity", None)
-            resolved = params.get("resolved", None)
-            version = params.get("version", None)
+            integrity = params.get("integrity")
+            resolved = params.get("resolved")
+            version = params.get("version")
+            link = params.get("link", False)
+
+            # Handle link sources
+            if link:
+                localpath = resolved
+                unpack = False
 
             # Handle registry sources
-            if is_semver(version) and resolved and integrity:
+            elif version and is_semver(version) and integrity:
+                # Handle duplicate dependencies without url
+                if not resolved:
+                    return
+
                 localfile = npm_localfile(name, version)
 
                 uri = URI(resolved)
@@ -108,10 +122,10 @@ class NpmShrinkWrap(FetchMethod):
                 extrapaths.append(resolvefile)
 
             # Handle http tarball sources
-            elif version.startswith("http") and integrity:
-                localfile = os.path.join("npm2", os.path.basename(version))
+            elif resolved.startswith("http") and integrity:
+                localfile = npm_localfile(os.path.basename(resolved))
 
-                uri = URI(version)
+                uri = URI(resolved)
                 uri.params["downloadfilename"] = localfile
 
                 checksum_name, checksum_expected = npm_integrity(integrity)
@@ -121,8 +135,12 @@ class NpmShrinkWrap(FetchMethod):
 
                 localpath = os.path.join(d.getVar("DL_DIR"), localfile)
 
+            # Handle local tarball sources
+            elif resolved.startswith("file"):
+                localpath = resolved[5:]
+
             # Handle git sources
-            elif version.startswith("git"):
+            elif resolved.startswith("git"):
                 regex = re.compile(r"""
                     ^
                     git\+
@@ -134,29 +152,31 @@ class NpmShrinkWrap(FetchMethod):
                     $
                     """, re.VERBOSE)
 
-                match = regex.match(version)
-
+                match = regex.match(resolved)
                 if not match:
-                    raise ParameterError("Invalid git url: %s" % version, ud.url)
+                    raise ParameterError("Invalid git url: %s" % resolved, ud.url)
 
                 groups = match.groupdict()
 
                 uri = URI("git://" + str(groups["url"]))
                 uri.params["protocol"] = str(groups["protocol"])
                 uri.params["rev"] = str(groups["rev"])
+                uri.params["nobranch"] = "1"
                 uri.params["destsuffix"] = destsuffix
 
                 url = str(uri)
 
-            # local tarball sources and local link sources are unsupported
             else:
                 raise ParameterError("Unsupported dependency: %s" % name, ud.url)
 
+            # name is needed by unpack tracer for module mapping
             ud.deps.append({
+                "name": name,
                 "url": url,
                 "localpath": localpath,
                 "extrapaths": extrapaths,
                 "destsuffix": destsuffix,
+                "unpack": unpack,
             })
 
         try:
@@ -177,17 +197,23 @@ class NpmShrinkWrap(FetchMethod):
         # This fetcher resolves multiple URIs from a shrinkwrap file and then
         # forwards it to a proxy fetcher. The management of the donestamp file,
         # the lockfile and the checksums are forwarded to the proxy fetcher.
-        ud.proxy = Fetch([dep["url"] for dep in ud.deps], data)
+        shrinkwrap_urls = [dep["url"] for dep in ud.deps if dep["url"]]
+        if shrinkwrap_urls:
+            ud.proxy = Fetch(shrinkwrap_urls, data)
         ud.needdonestamp = False
 
     @staticmethod
     def _foreach_proxy_method(ud, handle):
         returns = []
-        for proxy_url in ud.proxy.urls:
-            proxy_ud = ud.proxy.ud[proxy_url]
-            proxy_d = ud.proxy.d
-            proxy_ud.setup_localpath(proxy_d)
-            returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
+        #Check if there are dependencies before try to fetch them
+        if len(ud.deps) > 0:
+            for proxy_url in ud.proxy.urls:
+                proxy_ud = ud.proxy.ud[proxy_url]
+                proxy_d = ud.proxy.d
+                proxy_ud.setup_localpath(proxy_d)
+                lf = lockfile(proxy_ud.lockfile)
+                returns.append(handle(proxy_ud.method, proxy_ud, proxy_d))
+                unlockfile(lf)
         return returns
 
     def verify_donestamp(self, ud, d):
@@ -220,10 +246,11 @@ class NpmShrinkWrap(FetchMethod):
 
     def unpack(self, ud, rootdir, d):
         """Unpack the downloaded dependencies"""
-        destdir = d.getVar("S")
+        destdir = rootdir
         destsuffix = ud.parm.get("destsuffix")
         if destsuffix:
             destdir = os.path.join(rootdir, destsuffix)
+        ud.unpack_tracer.unpack("npm-shrinkwrap", destdir)
 
         bb.utils.mkdirhier(destdir)
         bb.utils.copyfile(ud.shrinkwrap_file,
@@ -237,7 +264,16 @@ class NpmShrinkWrap(FetchMethod):
 
         for dep in manual:
             depdestdir = os.path.join(destdir, dep["destsuffix"])
-            npm_unpack(dep["localpath"], depdestdir, d)
+            if dep["url"]:
+                npm_unpack(dep["localpath"], depdestdir, d)
+            else:
+                depsrcdir= os.path.join(destdir, dep["localpath"])
+                if dep["unpack"]:
+                    npm_unpack(depsrcdir, depdestdir, d)
+                else:
+                    bb.utils.mkdirhier(depdestdir)
+                    cmd = 'cp -fpPRH "%s/." .' % (depsrcdir)
+                    runfetchcmd(cmd, d, workdir=depdestdir)
 
     def clean(self, ud, d):
         """Clean any existing full or partial download"""
diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py
index d9ce44390c..495ac8a30a 100644
--- a/bitbake/lib/bb/fetch2/osc.py
+++ b/bitbake/lib/bb/fetch2/osc.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 """
@@ -9,6 +11,7 @@ Based on the svn "Fetch" implementation.
 
 import logging
 import os
+import re
 import  bb
 from    bb.fetch2 import FetchMethod
 from    bb.fetch2 import FetchError
@@ -36,6 +39,7 @@ class Osc(FetchMethod):
         # Create paths to osc checkouts
         oscdir = d.getVar("OSCDIR") or (d.getVar("DL_DIR") + "/osc")
         relpath = self._strip_leading_slashes(ud.path)
+        ud.oscdir = oscdir
         ud.pkgdir = os.path.join(oscdir, ud.host)
         ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
 
@@ -43,13 +47,13 @@ class Osc(FetchMethod):
             ud.revision = ud.parm['rev']
         else:
             pv = d.getVar("PV", False)
-            rev = bb.fetch2.srcrev_internal_helper(ud, d)
+            rev = bb.fetch2.srcrev_internal_helper(ud, d, '')
             if rev:
                 ud.revision = rev
             else:
                 ud.revision = ""
 
-        ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision))
+        ud.localfile = d.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), relpath.replace('/', '.'), ud.revision))
 
     def _buildosccommand(self, ud, d, command):
         """
@@ -59,26 +63,49 @@ class Osc(FetchMethod):
 
         basecmd = d.getVar("FETCHCMD_osc") or "/usr/bin/env osc"
 
-        proto = ud.parm.get('protocol', 'ocs')
+        proto = ud.parm.get('protocol', 'https')
 
         options = []
 
         config = "-c %s" % self.generate_config(ud, d)
 
-        if ud.revision:
+        if getattr(ud, 'revision', ''):
             options.append("-r %s" % ud.revision)
 
         coroot = self._strip_leading_slashes(ud.path)
 
         if command == "fetch":
-            osccmd = "%s %s co %s/%s %s" % (basecmd, config, coroot, ud.module, " ".join(options))
+            osccmd = "%s %s -A %s://%s co %s/%s %s" % (basecmd, config, proto, ud.host, coroot, ud.module, " ".join(options))
         elif command == "update":
-            osccmd = "%s %s up %s" % (basecmd, config, " ".join(options))
+            osccmd = "%s %s -A %s://%s up %s" % (basecmd, config, proto, ud.host, " ".join(options))
+        elif command == "api_source":
+            osccmd = "%s %s -A %s://%s api source/%s/%s" % (basecmd, config, proto, ud.host, coroot, ud.module)
         else:
             raise FetchError("Invalid osc command %s" % command, ud.url)
 
         return osccmd
 
+    def _latest_revision(self, ud, d, name):
+        """
+        Fetch latest revision for the given package
+        """
+        api_source_cmd = self._buildosccommand(ud, d, "api_source")
+
+        output = runfetchcmd(api_source_cmd, d)
+        match = re.match(r'<directory ?.* rev="(\d+)".*>', output)
+        if match is None:
+            raise FetchError("Unable to parse osc response", ud.url)
+        return match.groups()[0]
+
+    def _revision_key(self, ud, d, name):
+        """
+        Return a unique key for the url
+        """
+        # Collapse adjacent slashes
+        slash_re = re.compile(r"/+")
+        rev = getattr(ud, 'revision', "latest")
+        return "osc:%s%s.%s.%s" % (ud.host, slash_re.sub(".", ud.path), name, rev)
+
     def download(self, ud, d):
         """
         Fetch url
@@ -86,7 +113,7 @@ class Osc(FetchMethod):
 
         logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
 
-        if os.access(os.path.join(d.getVar('OSCDIR'), ud.path, ud.module), os.R_OK):
+        if os.access(ud.moddir, os.R_OK):
             oscupdatecmd = self._buildosccommand(ud, d, "update")
             logger.info("Update "+ ud.url)
             # update sources there
@@ -114,20 +141,23 @@ class Osc(FetchMethod):
         Generate a .oscrc to be used for this run.
         """
 
-        config_path = os.path.join(d.getVar('OSCDIR'), "oscrc")
+        config_path = os.path.join(ud.oscdir, "oscrc")
+        if not os.path.exists(ud.oscdir):
+            bb.utils.mkdirhier(ud.oscdir)
+
         if (os.path.exists(config_path)):
             os.remove(config_path)
 
         f = open(config_path, 'w')
+        proto = ud.parm.get('protocol', 'https')
         f.write("[general]\n")
-        f.write("apisrv = %s\n" % ud.host)
-        f.write("scheme = http\n")
+        f.write("apiurl = %s://%s\n" % (proto, ud.host))
         f.write("su-wrapper = su -c\n")
         f.write("build-root = %s\n" % d.getVar('WORKDIR'))
         f.write("urllist = %s\n" % d.getVar("OSCURLLIST"))
         f.write("extra-pkgs = gzip\n")
         f.write("\n")
-        f.write("[%s]\n" % ud.host)
+        f.write("[%s://%s]\n" % (proto, ud.host))
         f.write("user = %s\n" % ud.parm["user"])
         f.write("pass = %s\n" % ud.parm["pswd"])
         f.close()
diff --git a/bitbake/lib/bb/fetch2/perforce.py b/bitbake/lib/bb/fetch2/perforce.py
index e2a41a4a12..3b6fa4b1ec 100644
--- a/bitbake/lib/bb/fetch2/perforce.py
+++ b/bitbake/lib/bb/fetch2/perforce.py
@@ -134,7 +134,7 @@ class Perforce(FetchMethod):
 
         ud.setup_revisions(d)
 
-        ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleandedmodule, ud.revision))
+        ud.localfile = d.expand('%s_%s_%s_%s.tar.gz' % (cleanedhost, cleanedpath, cleanedmodule, ud.revision))
 
     def _buildp4command(self, ud, d, command, depot_filename=None):
         """
diff --git a/bitbake/lib/bb/fetch2/s3.py b/bitbake/lib/bb/fetch2/s3.py
index ffca73c8e4..22c0538139 100644
--- a/bitbake/lib/bb/fetch2/s3.py
+++ b/bitbake/lib/bb/fetch2/s3.py
@@ -18,10 +18,47 @@ The aws tool must be correctly installed and configured prior to use.
 import os
 import bb
 import urllib.request, urllib.parse, urllib.error
+import re
 from bb.fetch2 import FetchMethod
 from bb.fetch2 import FetchError
 from bb.fetch2 import runfetchcmd
 
+def convertToBytes(value, unit):
+    value = float(value)
+    if (unit == "KiB"):
+        value = value*1024.0;
+    elif (unit == "MiB"):
+        value = value*1024.0*1024.0;
+    elif (unit == "GiB"):
+        value = value*1024.0*1024.0*1024.0;
+    return value
+
+class S3ProgressHandler(bb.progress.LineFilterProgressHandler):
+    """
+    Extract progress information from s3 cp output, e.g.:
+    Completed 5.1 KiB/8.8 GiB (12.0 MiB/s) with 1 file(s) remaining
+    """
+    def __init__(self, d):
+        super(S3ProgressHandler, self).__init__(d)
+        # Send an initial progress event so the bar gets shown
+        self._fire_progress(0)
+
+    def writeline(self, line):
+        percs = re.findall(r'^Completed (\d+.{0,1}\d*) (\w+)\/(\d+.{0,1}\d*) (\w+) (\(.+\)) with\s+', line)
+        if percs:
+            completed = (percs[-1][0])
+            completedUnit = (percs[-1][1])
+            total = (percs[-1][2])
+            totalUnit = (percs[-1][3])
+            completed = convertToBytes(completed, completedUnit)
+            total = convertToBytes(total, totalUnit)
+            progress = (completed/total)*100.0
+            rate = percs[-1][4]
+            self.update(progress, rate)
+            return False
+        return True
+
+
 class S3(FetchMethod):
     """Class to fetch urls via 'aws s3'"""
 
@@ -40,7 +77,7 @@ class S3(FetchMethod):
         else:
             ud.basename = os.path.basename(ud.path)
 
-        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.localfile = ud.basename
 
         ud.basecmd = d.getVar("FETCHCMD_s3") or "/usr/bin/env aws s3"
 
@@ -52,7 +89,9 @@ class S3(FetchMethod):
 
         cmd = '%s cp s3://%s%s %s' % (ud.basecmd, ud.host, ud.path, ud.localpath)
         bb.fetch2.check_network_access(d, cmd, ud.url)
-        runfetchcmd(cmd, d)
+
+        progresshandler = S3ProgressHandler(d)
+        runfetchcmd(cmd, d, False, log=progresshandler)
 
         # Additional sanity checks copied from the wget class (although there
         # are no known issues which mean these are required, treat the aws cli
diff --git a/bitbake/lib/bb/fetch2/sftp.py b/bitbake/lib/bb/fetch2/sftp.py
index f87f292e5d..bee71a0d0d 100644
--- a/bitbake/lib/bb/fetch2/sftp.py
+++ b/bitbake/lib/bb/fetch2/sftp.py
@@ -77,7 +77,7 @@ class SFTP(FetchMethod):
         else:
             ud.basename = os.path.basename(ud.path)
 
-        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.localfile = ud.basename
 
     def download(self, ud, d):
         """Fetch urls"""
@@ -103,7 +103,7 @@ class SFTP(FetchMethod):
         if path[:3] == '/~/':
             path = path[3:]
 
-        remote = '%s%s:%s' % (user, urlo.hostname, path)
+        remote = '"%s%s:%s"' % (user, urlo.hostname, path)
 
         cmd = '%s %s %s %s' % (basecmd, port, remote, lpath)
 
diff --git a/bitbake/lib/bb/fetch2/ssh.py b/bitbake/lib/bb/fetch2/ssh.py
index 2c8557e1f8..2a0f2cb44b 100644
--- a/bitbake/lib/bb/fetch2/ssh.py
+++ b/bitbake/lib/bb/fetch2/ssh.py
@@ -32,6 +32,7 @@ IETF secsh internet draft:
 
 import re, os
 from bb.fetch2 import check_network_access, FetchMethod, ParameterError, runfetchcmd
+import urllib
 
 
 __pattern__ = re.compile(r'''
@@ -40,9 +41,9 @@ __pattern__ = re.compile(r'''
  (                   # Optional username/password block
   (?P<user>\S+)      # username
   (:(?P<pass>\S+))?  # colon followed by the password (optional)
- )?
  (?P<cparam>(;[^;]+)*)?  # connection parameters block (optional)
  @
+ )?
  (?P<host>\S+?)          # non-greedy match of the host
  (:(?P<port>[0-9]+))?    # colon followed by the port (optional)
  /
@@ -70,9 +71,9 @@ class SSH(FetchMethod):
                 "git:// prefix with protocol=ssh", urldata.url)
         m = __pattern__.match(urldata.url)
         path = m.group('path')
+        path = urllib.parse.unquote(path)
         host = m.group('host')
-        urldata.localpath = os.path.join(d.getVar('DL_DIR'),
-                os.path.basename(os.path.normpath(path)))
+        urldata.localfile = os.path.basename(os.path.normpath(path))
 
     def download(self, urldata, d):
         dldir = d.getVar('DL_DIR')
@@ -96,6 +97,11 @@ class SSH(FetchMethod):
             fr += '@%s' % host
         else:
             fr = host
+
+        if path[0] != '~':
+            path = '/%s' % path
+        path = urllib.parse.unquote(path)
+
         fr += ':%s' % path
 
         cmd = 'scp -B -r %s %s %s/' % (
@@ -108,3 +114,41 @@ class SSH(FetchMethod):
 
         runfetchcmd(cmd, d)
 
+    def checkstatus(self, fetch, urldata, d):
+        """
+        Check the status of the url
+        """
+        m = __pattern__.match(urldata.url)
+        path = m.group('path')
+        host = m.group('host')
+        port = m.group('port')
+        user = m.group('user')
+        password = m.group('pass')
+
+        if port:
+            portarg = '-P %s' % port
+        else:
+            portarg = ''
+
+        if user:
+            fr = user
+            if password:
+                fr += ':%s' % password
+            fr += '@%s' % host
+        else:
+            fr = host
+
+        if path[0] != '~':
+            path = '/%s' % path
+        path = urllib.parse.unquote(path)
+
+        cmd = 'ssh -o BatchMode=true %s %s [ -f %s ]' % (
+            portarg,
+            fr,
+            path
+        )
+
+        check_network_access(d, cmd, urldata.url)
+        runfetchcmd(cmd, d)
+
+        return True
diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py
index 8856ef1c62..0852108e7d 100644
--- a/bitbake/lib/bb/fetch2/svn.py
+++ b/bitbake/lib/bb/fetch2/svn.py
@@ -57,7 +57,12 @@ class Svn(FetchMethod):
         if 'rev' in ud.parm:
             ud.revision = ud.parm['rev']
 
-        ud.localfile = d.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision))
+        # Whether to use the @REV peg-revision syntax in the svn command or not
+        ud.pegrevision = True
+        if 'nopegrevision' in ud.parm:
+            ud.pegrevision = False
+
+        ud.localfile = d.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ["0", "1"][ud.pegrevision]))
 
     def _buildsvncommand(self, ud, d, command):
         """
@@ -86,7 +91,7 @@ class Svn(FetchMethod):
         if command == "info":
             svncmd = "%s info %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
         elif command == "log1":
-            svncmd = "%s log --limit 1 %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
+            svncmd = "%s log --limit 1 --quiet %s %s://%s/%s/" % (ud.basecmd, " ".join(options), proto, svnroot, ud.module)
         else:
             suffix = ""
 
@@ -98,7 +103,8 @@ class Svn(FetchMethod):
 
             if ud.revision:
                 options.append("-r %s" % ud.revision)
-                suffix = "@%s" % (ud.revision)
+                if ud.pegrevision:
+                    suffix = "@%s" % (ud.revision)
 
             if command == "fetch":
                 transportuser = ud.parm.get("transportuser", "")
@@ -204,3 +210,6 @@ class Svn(FetchMethod):
 
     def _build_revision(self, ud, d):
         return ud.revision
+
+    def supports_checksum(self, urldata):
+        return False
diff --git a/bitbake/lib/bb/fetch2/wget.py b/bitbake/lib/bb/fetch2/wget.py
index 6d82f3af07..7e43d3bc97 100644
--- a/bitbake/lib/bb/fetch2/wget.py
+++ b/bitbake/lib/bb/fetch2/wget.py
@@ -26,7 +26,6 @@ from   bb.fetch2 import FetchMethod
 from   bb.fetch2 import FetchError
 from   bb.fetch2 import logger
 from   bb.fetch2 import runfetchcmd
-from   bb.utils import export_proxies
 from   bs4 import BeautifulSoup
 from   bs4 import SoupStrainer
 
@@ -52,18 +51,19 @@ class WgetProgressHandler(bb.progress.LineFilterProgressHandler):
 
 
 class Wget(FetchMethod):
+    """Class to fetch urls via 'wget'"""
 
-    # CDNs like CloudFlare may do a 'browser integrity test' which can fail
-    # with the standard wget/urllib User-Agent, so pretend to be a modern
-    # browser.
-    user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0"
+    def check_certs(self, d):
+        """
+        Should certificates be checked?
+        """
+        return (d.getVar("BB_CHECK_SSL_CERTS") or "1") != "0"
 
-    """Class to fetch urls via 'wget'"""
     def supports(self, ud, d):
         """
         Check to see if a given url can be fetched with wget.
         """
-        return ud.type in ['http', 'https', 'ftp']
+        return ud.type in ['http', 'https', 'ftp', 'ftps']
 
     def recommends_checksum(self, urldata):
         return True
@@ -78,11 +78,17 @@ class Wget(FetchMethod):
         else:
             ud.basename = os.path.basename(ud.path)
 
-        ud.localfile = d.expand(urllib.parse.unquote(ud.basename))
+        ud.localfile = ud.basename
         if not ud.localfile:
-            ud.localfile = d.expand(urllib.parse.unquote(ud.host + ud.path).replace("/", "."))
+            ud.localfile = ud.host + ud.path.replace("/", ".")
 
-        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget -t 2 -T 30 --passive-ftp --no-check-certificate"
+        self.basecmd = d.getVar("FETCHCMD_wget") or "/usr/bin/env wget --tries=2 --timeout=100"
+
+        if ud.type == 'ftp' or ud.type == 'ftps':
+            self.basecmd += " --passive-ftp"
+
+        if not self.check_certs(d):
+            self.basecmd += " --no-check-certificate"
 
     def _runwget(self, ud, d, command, quiet, workdir=None):
 
@@ -90,39 +96,53 @@ class Wget(FetchMethod):
 
         logger.debug2("Fetching %s using command '%s'" % (ud.url, command))
         bb.fetch2.check_network_access(d, command, ud.url)
-        runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
+        runfetchcmd(command + ' --progress=dot --verbose', d, quiet, log=progresshandler, workdir=workdir)
 
     def download(self, ud, d):
         """Fetch urls"""
 
         fetchcmd = self.basecmd
 
-        if 'downloadfilename' in ud.parm:
-            localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile)
-            bb.utils.mkdirhier(os.path.dirname(localpath))
-            fetchcmd += " -O %s" % shlex.quote(localpath)
+        dldir = os.path.realpath(d.getVar("DL_DIR"))
+        localpath = os.path.join(dldir, ud.localfile) + ".tmp"
+        bb.utils.mkdirhier(os.path.dirname(localpath))
+        fetchcmd += " --output-document=%s" % shlex.quote(localpath)
 
         if ud.user and ud.pswd:
-            fetchcmd += " --user=%s --password=%s --auth-no-challenge" % (ud.user, ud.pswd)
+            fetchcmd += " --auth-no-challenge"
+            if ud.parm.get("redirectauth", "1") == "1":
+                # An undocumented feature of wget is that if the
+                # username/password are specified on the URI, wget will only
+                # send the Authorization header to the first host and not to
+                # any hosts that it is redirected to.  With the increasing
+                # usage of temporary AWS URLs, this difference now matters as
+                # AWS will reject any request that has authentication both in
+                # the query parameters (from the redirect) and in the
+                # Authorization header.
+                fetchcmd += " --user=%s --password=%s" % (ud.user, ud.pswd)
 
         uri = ud.url.split(";")[0]
-        if os.path.exists(ud.localpath):
-            # file exists, but we didnt complete it.. trying again..
-            fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % uri)
-        else:
-            fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % uri)
-
+        fetchcmd += " --continue --directory-prefix=%s '%s'" % (dldir, uri)
         self._runwget(ud, d, fetchcmd, False)
 
         # Sanity check since wget can pretend it succeed when it didn't
         # Also, this used to happen if sourceforge sent us to the mirror page
-        if not os.path.exists(ud.localpath):
-            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri)
+        if not os.path.exists(localpath):
+            raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, localpath), uri)
 
-        if os.path.getsize(ud.localpath) == 0:
-            os.remove(ud.localpath)
+        if os.path.getsize(localpath) == 0:
+            os.remove(localpath)
             raise FetchError("The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (uri), uri)
 
+        # Try and verify any checksum now, meaning if it isn't correct, we don't remove the
+        # original file, which might be a race (imagine two recipes referencing the same
+        # source, one with an incorrect checksum)
+        bb.fetch2.verify_checksum(ud, d, localpath=localpath, fatal_nochecksum=False)
+
+        # Remove the ".tmp" and move the file into position atomically
+        # Our lock prevents multiple writers but mirroring code may grab incomplete files
+        os.rename(localpath, localpath[:-4])
+
         return True
 
     def checkstatus(self, fetch, ud, d, try_again=True):
@@ -209,12 +229,17 @@ class Wget(FetchMethod):
                         # We let the request fail and expect it to be
                         # tried once more ("try_again" in check_status()),
                         # with the dead connection removed from the cache.
-                        # If it still fails, we give up, which can happend for bad
+                        # If it still fails, we give up, which can happen for bad
                         # HTTP proxy settings.
                         fetch.connection_cache.remove_connection(h.host, h.port)
                     raise urllib.error.URLError(err)
                 else:
-                    r = h.getresponse()
+                    try:
+                        r = h.getresponse()
+                    except TimeoutError as e:
+                        if fetch.connection_cache:
+                            fetch.connection_cache.remove_connection(h.host, h.port)
+                        raise TimeoutError(e)
 
                 # Pick apart the HTTPResponse object to get the addinfourl
                 # object initialized properly.
@@ -275,71 +300,115 @@ class Wget(FetchMethod):
 
         class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
             """
-            urllib2.HTTPRedirectHandler resets the method to GET on redirect,
-            when we want to follow redirects using the original method.
+            urllib2.HTTPRedirectHandler before 3.13 has two flaws:
+            
+            It resets the method to GET on redirect when we want to follow
+            redirects using the original method (typically HEAD). This was fixed
+            in 759e8e7.
+
+            It also doesn't handle 308 (Permanent Redirect). This was fixed in
+            c379bc5.
+
+            Until we depend on Python 3.13 onwards, copy the redirect_request
+            method to fix these issues.
             """
             def redirect_request(self, req, fp, code, msg, headers, newurl):
-                newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
-                newreq.get_method = req.get_method
-                return newreq
-        exported_proxies = export_proxies(d)
-
-        handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback]
-        if exported_proxies:
-            handlers.append(urllib.request.ProxyHandler())
-        handlers.append(CacheHTTPHandler())
-        # Since Python 2.7.9 ssl cert validation is enabled by default
-        # see PEP-0476, this causes verification errors on some https servers
-        # so disable by default.
-        import ssl
-        if hasattr(ssl, '_create_unverified_context'):
-            handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context()))
-        opener = urllib.request.build_opener(*handlers)
-
-        try:
-            uri = ud.url.split(";")[0]
-            r = urllib.request.Request(uri)
-            r.get_method = lambda: "HEAD"
-            # Some servers (FusionForge, as used on Alioth) require that the
-            # optional Accept header is set.
-            r.add_header("Accept", "*/*")
-            r.add_header("User-Agent", self.user_agent)
-            def add_basic_auth(login_str, request):
-                '''Adds Basic auth to http request, pass in login:password as string'''
-                import base64
-                encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
-                authheader = "Basic %s" % encodeuser
-                r.add_header("Authorization", authheader)
-
-            if ud.user and ud.pswd:
-                add_basic_auth(ud.user + ':' + ud.pswd, r)
+                m = req.get_method()
+                if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
+                    or code in (301, 302, 303) and m == "POST")):
+                    raise urllib.HTTPError(req.full_url, code, msg, headers, fp)
+
+                # Strictly (according to RFC 2616), 301 or 302 in response to
+                # a POST MUST NOT cause a redirection without confirmation
+                # from the user (of urllib.request, in this case).  In practice,
+                # essentially all clients do redirect in this case, so we do
+                # the same.
+
+                # Be conciliant with URIs containing a space.  This is mainly
+                # redundant with the more complete encoding done in http_error_302(),
+                # but it is kept for compatibility with other callers.
+                newurl = newurl.replace(' ', '%20')
+
+                CONTENT_HEADERS = ("content-length", "content-type")
+                newheaders = {k: v for k, v in req.headers.items()
+                            if k.lower() not in CONTENT_HEADERS}
+                return urllib.request.Request(newurl,
+                            method="HEAD" if m == "HEAD" else "GET",
+                            headers=newheaders,
+                            origin_req_host=req.origin_req_host,
+                            unverifiable=True)
+
+            http_error_308 = urllib.request.HTTPRedirectHandler.http_error_302
+
+        # We need to update the environment here as both the proxy and HTTPS
+        # handlers need variables set. The proxy needs http_proxy and friends to
+        # be set, and HTTPSHandler ends up calling into openssl to load the
+        # certificates. In buildtools configurations this will be looking at the
+        # wrong place for certificates by default: we set SSL_CERT_FILE to the
+        # right location in the buildtools environment script but as BitBake
+        # prunes prunes the environment this is lost. When binaries are executed
+        # runfetchcmd ensures these values are in the environment, but this is
+        # pure Python so we need to update the environment.
+        #
+        # Avoid tramping the environment too much by using bb.utils.environment
+        # to scope the changes to the build_opener request, which is when the
+        # environment lookups happen.
+        newenv = bb.fetch2.get_fetcher_environment(d)
+
+        with bb.utils.environment(**newenv):
+            import ssl
+
+            if self.check_certs(d):
+                context = ssl.create_default_context()
+            else:
+                context = ssl._create_unverified_context()
+
+            handlers = [FixedHTTPRedirectHandler,
+                        HTTPMethodFallback,
+                        urllib.request.ProxyHandler(),
+                        CacheHTTPHandler(),
+                        urllib.request.HTTPSHandler(context=context)]
+            opener = urllib.request.build_opener(*handlers)
 
             try:
-                import netrc
-                n = netrc.netrc()
-                login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
-                add_basic_auth("%s:%s" % (login, password), r)
-            except (TypeError, ImportError, IOError, netrc.NetrcParseError):
-                pass
-
-            with opener.open(r) as response:
-                pass
-        except urllib.error.URLError as e:
-            if try_again:
-                logger.debug2("checkstatus: trying again")
-                return self.checkstatus(fetch, ud, d, False)
-            else:
-                # debug for now to avoid spamming the logs in e.g. remote sstate searches
-                logger.debug2("checkstatus() urlopen failed: %s" % e)
-                return False
-        except ConnectionResetError as e:
-            if try_again:
-                logger.debug2("checkstatus: trying again")
-                return self.checkstatus(fetch, ud, d, False)
-            else:
-                # debug for now to avoid spamming the logs in e.g. remote sstate searches
-                logger.debug2("checkstatus() urlopen failed: %s" % e)
-                return False
+                parts = urllib.parse.urlparse(ud.url.split(";")[0])
+                uri = "{}://{}{}".format(parts.scheme, parts.netloc, parts.path)
+                r = urllib.request.Request(uri)
+                r.get_method = lambda: "HEAD"
+                # Some servers (FusionForge, as used on Alioth) require that the
+                # optional Accept header is set.
+                r.add_header("Accept", "*/*")
+                r.add_header("User-Agent", "bitbake/{}".format(bb.__version__))
+                def add_basic_auth(login_str, request):
+                    '''Adds Basic auth to http request, pass in login:password as string'''
+                    import base64
+                    encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
+                    authheader = "Basic %s" % encodeuser
+                    r.add_header("Authorization", authheader)
+
+                if ud.user and ud.pswd:
+                    add_basic_auth(ud.user + ':' + ud.pswd, r)
+
+                try:
+                    import netrc
+                    auth_data = netrc.netrc().authenticators(urllib.parse.urlparse(uri).hostname)
+                    if auth_data:
+                        login, _, password = auth_data
+                        add_basic_auth("%s:%s" % (login, password), r)
+                except (FileNotFoundError, netrc.NetrcParseError):
+                    pass
+
+                with opener.open(r, timeout=100) as response:
+                    pass
+            except (urllib.error.URLError, ConnectionResetError, TimeoutError) as e:
+                if try_again:
+                    logger.debug2("checkstatus: trying again")
+                    return self.checkstatus(fetch, ud, d, False)
+                else:
+                    # debug for now to avoid spamming the logs in e.g. remote sstate searches
+                    logger.debug2("checkstatus() urlopen failed for %s: %s" % (uri,e))
+                    return False
+
         return True
 
     def _parse_path(self, regex, s):
@@ -416,7 +485,7 @@ class Wget(FetchMethod):
         f = tempfile.NamedTemporaryFile()
         with tempfile.TemporaryDirectory(prefix="wget-index-") as workdir, tempfile.NamedTemporaryFile(dir=workdir, prefix="wget-listing-") as f:
             fetchcmd = self.basecmd
-            fetchcmd += " -O " + f.name + " --user-agent='" + self.user_agent + "' '" + uri + "'"
+            fetchcmd += " --output-document=%s '%s'" % (f.name, uri)
             try:
                 self._runwget(ud, d, fetchcmd, True, workdir=workdir)
                 fetchresult = f.read()
@@ -472,7 +541,7 @@ class Wget(FetchMethod):
         version_dir = ['', '', '']
         version = ['', '', '']
 
-        dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))")
+        dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])*(\d+))")
         s = dirver_regex.search(dirver)
         if s:
             version_dir[1] = s.group('ver')
@@ -548,7 +617,7 @@ class Wget(FetchMethod):
 
         # src.rpm extension was added only for rpm package. Can be removed if the rpm
         # packaged will always be considered as having to be manually upgraded
-        psuffix_regex = r"(tar\.gz|tgz|tar\.bz2|zip|xz|tar\.lz|rpm|bz2|orig\.tar\.gz|tar\.xz|src\.tar\.gz|src\.tgz|svnr\d+\.tar\.bz2|stable\.tar\.gz|src\.rpm)"
+        psuffix_regex = r"(tar\.\w+|tgz|zip|xz|rpm|bz2|orig\.tar\.\w+|src\.tar\.\w+|src\.tgz|svnr\d+\.tar\.\w+|stable\.tar\.\w+|src\.rpm)"
 
         # match name, version and archive type of a package
         package_regex_comp = re.compile(r"(?P<name>%s?\.?v?)(?P<pver>%s)(?P<arch>%s)?[\.-](?P<type>%s$)"
@@ -576,13 +645,17 @@ class Wget(FetchMethod):
 
         sanity check to ensure same name and type.
         """
-        package = ud.path.split("/")[-1]
+        if 'downloadfilename' in ud.parm:
+            package = ud.parm['downloadfilename']
+        else:
+            package = ud.path.split("/")[-1]
         current_version = ['', d.getVar('PV'), '']
 
         """possible to have no version in pkg name, such as spectrum-fw"""
         if not re.search(r"\d+", package):
             current_version[1] = re.sub('_', '.', current_version[1])
             current_version[1] = re.sub('-', '.', current_version[1])
+            bb.debug(3, "latest_versionstring: no version found in %s" % package)
             return (current_version[1], '')
 
         package_regex = self._init_regexes(package, ud, d)
@@ -599,10 +672,10 @@ class Wget(FetchMethod):
             # search for version matches on folders inside the path, like:
             # "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
             dirver_regex = re.compile(r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
-            m = dirver_regex.search(path)
+            m = dirver_regex.findall(path)
             if m:
                 pn = d.getVar('PN')
-                dirver = m.group('dirver')
+                dirver = m[-1][0]
 
                 dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn)))
                 if not dirver_pn_regex.search(dirver):
diff --git a/bitbake/lib/bb/main.py b/bitbake/lib/bb/main.py
index 06bad495ac..bca8ebfa09 100755
--- a/bitbake/lib/bb/main.py
+++ b/bitbake/lib/bb/main.py
@@ -12,11 +12,12 @@
 import os
 import sys
 import logging
-import optparse
+import argparse
 import warnings
 import fcntl
 import time
 import traceback
+import datetime
 
 import bb
 from bb import event
@@ -43,18 +44,18 @@ def present_options(optionlist):
     else:
         return optionlist[0]
 
-class BitbakeHelpFormatter(optparse.IndentedHelpFormatter):
-    def format_option(self, option):
+class BitbakeHelpFormatter(argparse.HelpFormatter):
+    def _get_help_string(self, action):
         # We need to do this here rather than in the text we supply to
         # add_option() because we don't want to call list_extension_modules()
         # on every execution (since it imports all of the modules)
         # Note also that we modify option.help rather than the returned text
         # - this is so that we don't have to re-format the text ourselves
-        if option.dest == 'ui':
+        if action.dest == 'ui':
             valid_uis = list_extension_modules(bb.ui, 'main')
-            option.help = option.help.replace('@CHOICES@', present_options(valid_uis))
+            return action.help.replace('@CHOICES@', present_options(valid_uis))
 
-        return optparse.IndentedHelpFormatter.format_option(self, option)
+        return action.help
 
 def list_extension_modules(pkg, checkattr):
     """
@@ -112,189 +113,209 @@ def _showwarning(message, category, filename, lineno, file=None, line=None):
         warnlog.warning(s)
 
 warnings.showwarning = _showwarning
-warnings.filterwarnings("ignore")
-warnings.filterwarnings("default", module="(<string>$|(oe|bb)\.)")
-warnings.filterwarnings("ignore", category=PendingDeprecationWarning)
-warnings.filterwarnings("ignore", category=ImportWarning)
-warnings.filterwarnings("ignore", category=DeprecationWarning, module="<string>$")
-warnings.filterwarnings("ignore", message="With-statements now directly support multiple context managers")
-
 
 def create_bitbake_parser():
-    parser = optparse.OptionParser(
-        formatter=BitbakeHelpFormatter(),
-        version="BitBake Build Tool Core version %s" % bb.__version__,
-        usage="""%prog [options] [recipename/target recipe:do_task ...]
-
-    Executes the specified task (default is 'build') for a given set of target recipes (.bb files).
-    It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
-    will provide the layer, BBFILES and other configuration information.""")
-
-    parser.add_option("-b", "--buildfile", action="store", dest="buildfile", default=None,
-                      help="Execute tasks from a specific .bb recipe directly. WARNING: Does "
-                           "not handle any dependencies from other recipes.")
-
-    parser.add_option("-k", "--continue", action="store_false", dest="abort", default=True,
-                      help="Continue as much as possible after an error. While the target that "
-                           "failed and anything depending on it cannot be built, as much as "
-                           "possible will be built before stopping.")
-
-    parser.add_option("-f", "--force", action="store_true", dest="force", default=False,
-                      help="Force the specified targets/task to run (invalidating any "
-                           "existing stamp file).")
-
-    parser.add_option("-c", "--cmd", action="store", dest="cmd",
-                      help="Specify the task to execute. The exact options available "
-                           "depend on the metadata. Some examples might be 'compile'"
-                           " or 'populate_sysroot' or 'listtasks' may give a list of "
-                           "the tasks available.")
-
-    parser.add_option("-C", "--clear-stamp", action="store", dest="invalidate_stamp",
-                      help="Invalidate the stamp for the specified task such as 'compile' "
-                           "and then run the default task for the specified target(s).")
-
-    parser.add_option("-r", "--read", action="append", dest="prefile", default=[],
-                      help="Read the specified file before bitbake.conf.")
-
-    parser.add_option("-R", "--postread", action="append", dest="postfile", default=[],
-                      help="Read the specified file after bitbake.conf.")
-
-    parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
-                      help="Enable tracing of shell tasks (with 'set -x'). "
-                           "Also print bb.note(...) messages to stdout (in "
-                           "addition to writing them to ${T}/log.do_<task>).")
-
-    parser.add_option("-D", "--debug", action="count", dest="debug", default=0,
-                      help="Increase the debug level. You can specify this "
-                           "more than once. -D sets the debug level to 1, "
-                           "where only bb.debug(1, ...) messages are printed "
-                           "to stdout; -DD sets the debug level to 2, where "
-                           "both bb.debug(1, ...) and bb.debug(2, ...) "
-                           "messages are printed; etc. Without -D, no debug "
-                           "messages are printed. Note that -D only affects "
-                           "output to stdout. All debug messages are written "
-                           "to ${T}/log.do_taskname, regardless of the debug "
-                           "level.")
-
-    parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0,
-                      help="Output less log message data to the terminal. You can specify this more than once.")
-
-    parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False,
-                      help="Don't execute, just go through the motions.")
-
-    parser.add_option("-S", "--dump-signatures", action="append", dest="dump_signatures",
-                      default=[], metavar="SIGNATURE_HANDLER",
-                      help="Dump out the signature construction information, with no task "
-                           "execution. The SIGNATURE_HANDLER parameter is passed to the "
-                           "handler. Two common values are none and printdiff but the handler "
-                           "may define more/less. none means only dump the signature, printdiff"
-                           " means compare the dumped signature with the cached one.")
-
-    parser.add_option("-p", "--parse-only", action="store_true",
-                      dest="parse_only", default=False,
-                      help="Quit after parsing the BB recipes.")
-
-    parser.add_option("-s", "--show-versions", action="store_true",
-                      dest="show_versions", default=False,
-                      help="Show current and preferred versions of all recipes.")
-
-    parser.add_option("-e", "--environment", action="store_true",
-                      dest="show_environment", default=False,
-                      help="Show the global or per-recipe environment complete with information"
-                           " about where variables were set/changed.")
-
-    parser.add_option("-g", "--graphviz", action="store_true", dest="dot_graph", default=False,
-                      help="Save dependency tree information for the specified "
-                           "targets in the dot syntax.")
-
-    parser.add_option("-I", "--ignore-deps", action="append",
-                      dest="extra_assume_provided", default=[],
-                      help="Assume these dependencies don't exist and are already provided "
-                           "(equivalent to ASSUME_PROVIDED). Useful to make dependency "
-                           "graphs more appealing")
-
-    parser.add_option("-l", "--log-domains", action="append", dest="debug_domains", default=[],
-                      help="Show debug logging for the specified logging domains")
-
-    parser.add_option("-P", "--profile", action="store_true", dest="profile", default=False,
-                      help="Profile the command and save reports.")
+    parser = argparse.ArgumentParser(
+        description="""\
+            It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which
+            will provide the layer, BBFILES and other configuration information.
+            """,
+        formatter_class=BitbakeHelpFormatter,
+        allow_abbrev=False,
+        add_help=False, # help is manually added below in a specific argument group
+    )
+
+    general_group  = parser.add_argument_group('General options')
+    task_group     = parser.add_argument_group('Task control options')
+    exec_group     = parser.add_argument_group('Execution control options')
+    logging_group  = parser.add_argument_group('Logging/output control options')
+    server_group   = parser.add_argument_group('Server options')
+    config_group   = parser.add_argument_group('Configuration options')
+
+    general_group.add_argument("targets", nargs="*", metavar="recipename/target",
+                        help="Execute the specified task (default is 'build') for these target "
+                             "recipes (.bb files).")
+
+    general_group.add_argument("-s", "--show-versions", action="store_true",
+                        help="Show current and preferred versions of all recipes.")
+
+    general_group.add_argument("-e", "--environment", action="store_true",
+                        dest="show_environment",
+                        help="Show the global or per-recipe environment complete with information"
+                             " about where variables were set/changed.")
+
+    general_group.add_argument("-g", "--graphviz", action="store_true", dest="dot_graph",
+                        help="Save dependency tree information for the specified "
+                             "targets in the dot syntax.")
 
     # @CHOICES@ is substituted out by BitbakeHelpFormatter above
-    parser.add_option("-u", "--ui", action="store", dest="ui",
-                      default=os.environ.get('BITBAKE_UI', 'knotty'),
-                      help="The user interface to use (@CHOICES@ - default %default).")
-
-    parser.add_option("", "--token", action="store", dest="xmlrpctoken",
-                      default=os.environ.get("BBTOKEN"),
-                      help="Specify the connection token to be used when connecting "
-                           "to a remote server.")
-
-    parser.add_option("", "--revisions-changed", action="store_true",
-                      dest="revisions_changed", default=False,
-                      help="Set the exit code depending on whether upstream floating "
-                           "revisions have changed or not.")
-
-    parser.add_option("", "--server-only", action="store_true",
-                      dest="server_only", default=False,
-                      help="Run bitbake without a UI, only starting a server "
-                           "(cooker) process.")
-
-    parser.add_option("-B", "--bind", action="store", dest="bind", default=False,
-                      help="The name/address for the bitbake xmlrpc server to bind to.")
-
-    parser.add_option("-T", "--idle-timeout", type=float, dest="server_timeout",
-                      default=os.getenv("BB_SERVER_TIMEOUT"),
-                      help="Set timeout to unload bitbake server due to inactivity, "
-                           "set to -1 means no unload, "
-                           "default: Environment variable BB_SERVER_TIMEOUT.")
-
-    parser.add_option("", "--no-setscene", action="store_true",
-                      dest="nosetscene", default=False,
-                      help="Do not run any setscene tasks. sstate will be ignored and "
-                           "everything needed, built.")
-
-    parser.add_option("", "--skip-setscene", action="store_true",
-                      dest="skipsetscene", default=False,
-                      help="Skip setscene tasks if they would be executed. Tasks previously "
-                           "restored from sstate will be kept, unlike --no-setscene")
-
-    parser.add_option("", "--setscene-only", action="store_true",
-                      dest="setsceneonly", default=False,
-                      help="Only run setscene tasks, don't run any real tasks.")
-
-    parser.add_option("", "--remote-server", action="store", dest="remote_server",
-                      default=os.environ.get("BBSERVER"),
-                      help="Connect to the specified server.")
-
-    parser.add_option("-m", "--kill-server", action="store_true",
-                      dest="kill_server", default=False,
-                      help="Terminate any running bitbake server.")
-
-    parser.add_option("", "--observe-only", action="store_true",
-                      dest="observe_only", default=False,
-                      help="Connect to a server as an observing-only client.")
-
-    parser.add_option("", "--status-only", action="store_true",
-                      dest="status_only", default=False,
-                      help="Check the status of the remote bitbake server.")
-
-    parser.add_option("-w", "--write-log", action="store", dest="writeeventlog",
-                      default=os.environ.get("BBEVENTLOG"),
-                      help="Writes the event log of the build to a bitbake event json file. "
-                           "Use '' (empty string) to assign the name automatically.")
-
-    parser.add_option("", "--runall", action="append", dest="runall",
-                      help="Run the specified task for any recipe in the taskgraph of the specified target (even if it wouldn't otherwise have run).")
-
-    parser.add_option("", "--runonly", action="append", dest="runonly",
-                      help="Run only the specified task within the taskgraph of the specified targets (and any task dependencies those tasks may have).")
+    general_group.add_argument("-u", "--ui",
+                        default=os.environ.get('BITBAKE_UI', 'knotty'),
+                        help="The user interface to use (@CHOICES@ - default %(default)s).")
+
+    general_group.add_argument("--version", action="store_true",
+                        help="Show programs version and exit.")
+
+    general_group.add_argument('-h', '--help', action='help',
+                        help='Show this help message and exit.')
+
+
+    task_group.add_argument("-f", "--force", action="store_true",
+                        help="Force the specified targets/task to run (invalidating any "
+                             "existing stamp file).")
+
+    task_group.add_argument("-c", "--cmd",
+                        help="Specify the task to execute. The exact options available "
+                             "depend on the metadata. Some examples might be 'compile'"
+                             " or 'populate_sysroot' or 'listtasks' may give a list of "
+                             "the tasks available.")
+
+    task_group.add_argument("-C", "--clear-stamp", dest="invalidate_stamp",
+                        help="Invalidate the stamp for the specified task such as 'compile' "
+                             "and then run the default task for the specified target(s).")
+
+    task_group.add_argument("--runall", action="append", default=[],
+                        help="Run the specified task for any recipe in the taskgraph of the "
+                             "specified target (even if it wouldn't otherwise have run).")
+
+    task_group.add_argument("--runonly", action="append",
+                        help="Run only the specified task within the taskgraph of the "
+                             "specified targets (and any task dependencies those tasks may have).")
+
+    task_group.add_argument("--no-setscene", action="store_true",
+                        dest="nosetscene",
+                        help="Do not run any setscene tasks. sstate will be ignored and "
+                            "everything needed, built.")
+
+    task_group.add_argument("--skip-setscene", action="store_true",
+                        dest="skipsetscene",
+                        help="Skip setscene tasks if they would be executed. Tasks previously "
+                            "restored from sstate will be kept, unlike --no-setscene.")
+
+    task_group.add_argument("--setscene-only", action="store_true",
+                        dest="setsceneonly",
+                        help="Only run setscene tasks, don't run any real tasks.")
+
+
+    exec_group.add_argument("-n", "--dry-run", action="store_true",
+                        help="Don't execute, just go through the motions.")
+
+    exec_group.add_argument("-p", "--parse-only", action="store_true",
+                        help="Quit after parsing the BB recipes.")
+
+    exec_group.add_argument("-k", "--continue", action="store_false", dest="halt",
+                        help="Continue as much as possible after an error. While the target that "
+                             "failed and anything depending on it cannot be built, as much as "
+                             "possible will be built before stopping.")
+
+    exec_group.add_argument("-P", "--profile", action="store_true",
+                        help="Profile the command and save reports.")
+
+    exec_group.add_argument("-S", "--dump-signatures", action="append",
+                        default=[], metavar="SIGNATURE_HANDLER",
+                        help="Dump out the signature construction information, with no task "
+                             "execution. The SIGNATURE_HANDLER parameter is passed to the "
+                             "handler. Two common values are none and printdiff but the handler "
+                             "may define more/less. none means only dump the signature, printdiff"
+                             " means recursively compare the dumped signature with the most recent"
+                             " one in a local build or sstate cache (can be used to find out why tasks re-run"
+                             " when that is not expected)")
+
+    exec_group.add_argument("--revisions-changed", action="store_true",
+                        help="Set the exit code depending on whether upstream floating "
+                            "revisions have changed or not.")
+
+    exec_group.add_argument("-b", "--buildfile",
+                        help="Execute tasks from a specific .bb recipe directly. WARNING: Does "
+                             "not handle any dependencies from other recipes.")
+
+    logging_group.add_argument("-D", "--debug", action="count", default=0,
+                        help="Increase the debug level. You can specify this "
+                             "more than once. -D sets the debug level to 1, "
+                             "where only bb.debug(1, ...) messages are printed "
+                             "to stdout; -DD sets the debug level to 2, where "
+                             "both bb.debug(1, ...) and bb.debug(2, ...) "
+                             "messages are printed; etc. Without -D, no debug "
+                             "messages are printed. Note that -D only affects "
+                             "output to stdout. All debug messages are written "
+                             "to ${T}/log.do_taskname, regardless of the debug "
+                             "level.")
+
+    logging_group.add_argument("-l", "--log-domains", action="append", dest="debug_domains",
+                        default=[],
+                        help="Show debug logging for the specified logging domains.")
+
+    logging_group.add_argument("-v", "--verbose", action="store_true",
+                        help="Enable tracing of shell tasks (with 'set -x'). "
+                             "Also print bb.note(...) messages to stdout (in "
+                             "addition to writing them to ${T}/log.do_<task>).")
+
+    logging_group.add_argument("-q", "--quiet", action="count", default=0,
+                        help="Output less log message data to the terminal. You can specify this "
+                             "more than once.")
+
+    logging_group.add_argument("-w", "--write-log", dest="writeeventlog",
+                        default=os.environ.get("BBEVENTLOG"),
+                        help="Writes the event log of the build to a bitbake event json file. "
+                            "Use '' (empty string) to assign the name automatically.")
+
+
+    server_group.add_argument("-B", "--bind", default=False,
+                        help="The name/address for the bitbake xmlrpc server to bind to.")
+
+    server_group.add_argument("-T", "--idle-timeout", type=float, dest="server_timeout",
+                        default=os.getenv("BB_SERVER_TIMEOUT"),
+                        help="Set timeout to unload bitbake server due to inactivity, "
+                             "set to -1 means no unload, "
+                             "default: Environment variable BB_SERVER_TIMEOUT.")
+
+    server_group.add_argument("--remote-server",
+                        default=os.environ.get("BBSERVER"),
+                        help="Connect to the specified server.")
+
+    server_group.add_argument("-m", "--kill-server", action="store_true",
+                        help="Terminate any running bitbake server.")
+
+    server_group.add_argument("--token", dest="xmlrpctoken",
+                        default=os.environ.get("BBTOKEN"),
+                        help="Specify the connection token to be used when connecting "
+                             "to a remote server.")
+
+    server_group.add_argument("--observe-only", action="store_true",
+                        help="Connect to a server as an observing-only client.")
+
+    server_group.add_argument("--status-only", action="store_true",
+                        help="Check the status of the remote bitbake server.")
+
+    server_group.add_argument("--server-only", action="store_true",
+                        help="Run bitbake without a UI, only starting a server "
+                            "(cooker) process.")
+
+
+    config_group.add_argument("-r", "--read", action="append", dest="prefile", default=[],
+                        help="Read the specified file before bitbake.conf.")
+
+    config_group.add_argument("-R", "--postread", action="append", dest="postfile", default=[],
+                        help="Read the specified file after bitbake.conf.")
+
+
+    config_group.add_argument("-I", "--ignore-deps", action="append",
+                        dest="extra_assume_provided", default=[],
+                        help="Assume these dependencies don't exist and are already provided "
+                             "(equivalent to ASSUME_PROVIDED). Useful to make dependency "
+                             "graphs more appealing.")
+
     return parser
 
 
 class BitBakeConfigParameters(cookerdata.ConfigParameters):
     def parseCommandLine(self, argv=sys.argv):
         parser = create_bitbake_parser()
-        options, targets = parser.parse_args(argv)
+        options = parser.parse_intermixed_args(argv[1:])
+
+        if options.version:
+            print("BitBake Build Tool Core version %s" % bb.__version__)
+            sys.exit(0)
 
         if options.quiet and options.verbose:
             parser.error("options --quiet and --verbose are mutually exclusive")
@@ -326,7 +347,7 @@ class BitBakeConfigParameters(cookerdata.ConfigParameters):
         else:
             options.xmlrpcinterface = (None, 0)
 
-        return options, targets[1:]
+        return options, options.targets
 
 
 def bitbake_main(configParams, configuration):
@@ -391,6 +412,9 @@ def bitbake_main(configParams, configuration):
 
     return 1
 
+def timestamp():
+    return datetime.datetime.now().strftime('%H:%M:%S.%f')
+
 def setup_bitbake(configParams, extrafeatures=None):
     # Ensure logging messages get sent to the UI as events
     handler = bb.event.LogHandler()
@@ -398,6 +422,11 @@ def setup_bitbake(configParams, extrafeatures=None):
         # In status only mode there are no logs and no UI
         logger.addHandler(handler)
 
+    if configParams.dump_signatures:
+        if extrafeatures is None:
+            extrafeatures = []
+        extrafeatures.append(bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO)
+
     if configParams.server_only:
         featureset = []
         ui_module = None
@@ -425,7 +454,7 @@ def setup_bitbake(configParams, extrafeatures=None):
         retries = 8
         while retries:
             try:
-                topdir, lock = lockBitbake()
+                topdir, lock, lockfile = lockBitbake()
                 sockname = topdir + "/bitbake.sock"
                 if lock:
                     if configParams.status_only or configParams.kill_server:
@@ -436,18 +465,22 @@ def setup_bitbake(configParams, extrafeatures=None):
                     logger.info("Starting bitbake server...")
                     # Clear the event queue since we already displayed messages
                     bb.event.ui_queue = []
-                    server = bb.server.process.BitBakeServer(lock, sockname, featureset, configParams.server_timeout, configParams.xmlrpcinterface)
+                    server = bb.server.process.BitBakeServer(lock, sockname, featureset, configParams.server_timeout, configParams.xmlrpcinterface, configParams.profile)
 
                 else:
                     logger.info("Reconnecting to bitbake server...")
                     if not os.path.exists(sockname):
-                        logger.info("Previous bitbake instance shutting down?, waiting to retry...")
+                        logger.info("Previous bitbake instance shutting down?, waiting to retry... (%s)" % timestamp())
+                        procs = bb.server.process.get_lockfile_process_msg(lockfile)
+                        if procs:
+                            logger.info("Processes holding bitbake.lock (missing socket %s):\n%s" % (sockname, procs))
+                        logger.info("Directory listing: %s" % (str(os.listdir(topdir))))
                         i = 0
                         lock = None
                         # Wait for 5s or until we can get the lock
                         while not lock and i < 50:
                             time.sleep(0.1)
-                            _, lock = lockBitbake()
+                            _, lock, _ = lockBitbake()
                             i += 1
                         if lock:
                             bb.utils.unlockfile(lock)
@@ -466,9 +499,9 @@ def setup_bitbake(configParams, extrafeatures=None):
                 retries -= 1
                 tryno = 8 - retries
                 if isinstance(e, (bb.server.process.ProcessTimeout, BrokenPipeError, EOFError, SystemExit)):
-                    logger.info("Retrying server connection (#%d)..." % tryno)
+                    logger.info("Retrying server connection (#%d)... (%s)" % (tryno, timestamp()))
                 else:
-                    logger.info("Retrying server connection (#%d)... (%s)" % (tryno, traceback.format_exc()))
+                    logger.info("Retrying server connection (#%d)... (%s, %s)" % (tryno, traceback.format_exc(), timestamp()))
 
             if not retries:
                 bb.fatal("Unable to connect to bitbake server, or start one (server startup failures would be in bitbake-cookerdaemon.log).")
@@ -497,5 +530,5 @@ def lockBitbake():
         bb.error("Unable to find conf/bblayers.conf or conf/bitbake.conf. BBPATH is unset and/or not in a build directory?")
         raise BBMainFatal
     lockfile = topdir + "/bitbake.lock"
-    return topdir, bb.utils.lockfile(lockfile, False, False)
+    return topdir, bb.utils.lockfile(lockfile, False, False), lockfile
 
diff --git a/bitbake/lib/bb/monitordisk.py b/bitbake/lib/bb/monitordisk.py
index 98f2109ed2..f928210351 100644
--- a/bitbake/lib/bb/monitordisk.py
+++ b/bitbake/lib/bb/monitordisk.py
@@ -76,7 +76,12 @@ def getDiskData(BBDirs):
             return None
 
         action = pathSpaceInodeRe.group(1)
-        if action not in ("ABORT", "STOPTASKS", "WARN"):
+        if action == "ABORT":
+            # Emit a deprecation warning
+            logger.warnonce("The BB_DISKMON_DIRS \"ABORT\" action has been renamed to \"HALT\", update configuration")
+            action = "HALT"
+
+        if action not in ("HALT", "STOPTASKS", "WARN"):
             printErr("Unknown disk space monitor action: %s" % action)
             return None
 
@@ -177,7 +182,7 @@ class diskMonitor:
                     # use them to avoid printing too many warning messages
                     self.preFreeS = {}
                     self.preFreeI = {}
-                    # This is for STOPTASKS and ABORT, to avoid printing the message
+                    # This is for STOPTASKS and HALT, to avoid printing the message
                     # repeatedly while waiting for the tasks to finish
                     self.checked = {}
                     for k in self.devDict:
@@ -219,8 +224,8 @@ class diskMonitor:
                         self.checked[k] = True
                         rq.finish_runqueue(False)
                         bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
-                    elif action == "ABORT" and not self.checked[k]:
-                        logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
+                    elif action == "HALT" and not self.checked[k]:
+                        logger.error("Immediately halt since the disk space monitor action is \"HALT\"!")
                         self.checked[k] = True
                         rq.finish_runqueue(True)
                         bb.event.fire(bb.event.DiskFull(dev, 'disk', freeSpace, path), self.configuration)
@@ -229,9 +234,10 @@ class diskMonitor:
                 freeInode = st.f_favail
 
                 if minInode and freeInode < minInode:
-                    # Some filesystems use dynamic inodes so can't run out
-                    # (e.g. btrfs). This is reported by the inode count being 0.
-                    if st.f_files == 0:
+                    # Some filesystems use dynamic inodes so can't run out.
+                    # This is reported by the inode count being 0 (btrfs) or the free
+                    # inode count being -1 (cephfs).
+                    if st.f_files == 0 or st.f_favail == -1:
                         self.devDict[k][2] = None
                         continue
                     # Always show warning, the self.checked would always be False if the action is WARN
@@ -245,8 +251,8 @@ class diskMonitor:
                         self.checked[k] = True
                         rq.finish_runqueue(False)
                         bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
-                    elif action  == "ABORT" and not self.checked[k]:
-                        logger.error("Immediately abort since the disk space monitor action is \"ABORT\"!")
+                    elif action  == "HALT" and not self.checked[k]:
+                        logger.error("Immediately halt since the disk space monitor action is \"HALT\"!")
                         self.checked[k] = True
                         rq.finish_runqueue(True)
                         bb.event.fire(bb.event.DiskFull(dev, 'inode', freeInode, path), self.configuration)
diff --git a/bitbake/lib/bb/msg.py b/bitbake/lib/bb/msg.py
index 291b38ff7f..4f616ff42e 100644
--- a/bitbake/lib/bb/msg.py
+++ b/bitbake/lib/bb/msg.py
@@ -30,7 +30,9 @@ class BBLogFormatter(logging.Formatter):
     PLAIN = logging.INFO + 1
     VERBNOTE = logging.INFO + 2
     ERROR = logging.ERROR
+    ERRORONCE = logging.ERROR - 1
     WARNING = logging.WARNING
+    WARNONCE = logging.WARNING - 1
     CRITICAL = logging.CRITICAL
 
     levelnames = {
@@ -42,7 +44,9 @@ class BBLogFormatter(logging.Formatter):
         PLAIN  : '',
         VERBNOTE: 'NOTE',
         WARNING : 'WARNING',
+        WARNONCE : 'WARNING',
         ERROR   : 'ERROR',
+        ERRORONCE   : 'ERROR',
         CRITICAL: 'ERROR',
     }
 
@@ -58,7 +62,9 @@ class BBLogFormatter(logging.Formatter):
         PLAIN   : BASECOLOR,
         VERBNOTE: BASECOLOR,
         WARNING : YELLOW,
+        WARNONCE : YELLOW,
         ERROR   : RED,
+        ERRORONCE : RED,
         CRITICAL: RED,
     }
 
@@ -83,10 +89,6 @@ class BBLogFormatter(logging.Formatter):
             msg = logging.Formatter.format(self, record)
         if hasattr(record, 'bb_exc_formatted'):
             msg += '\n' + ''.join(record.bb_exc_formatted)
-        elif hasattr(record, 'bb_exc_info'):
-            etype, value, tb = record.bb_exc_info
-            formatted = bb.exceptions.format_exception(etype, value, tb, limit=5)
-            msg += '\n' + ''.join(formatted)
         return msg
 
     def colorize(self, record):
@@ -121,6 +123,22 @@ class BBLogFilter(object):
             return True
         return False
 
+class LogFilterShowOnce(logging.Filter):
+    def __init__(self):
+        self.seen_warnings = set()
+        self.seen_errors = set()
+
+    def filter(self, record):
+        if record.levelno == bb.msg.BBLogFormatter.WARNONCE:
+            if record.msg in self.seen_warnings:
+                return False
+            self.seen_warnings.add(record.msg)
+        if record.levelno == bb.msg.BBLogFormatter.ERRORONCE:
+            if record.msg in self.seen_errors:
+                return False
+            self.seen_errors.add(record.msg)
+        return True
+
 class LogFilterGEQLevel(logging.Filter):
     def __init__(self, level):
         self.strlevel = str(level)
@@ -206,8 +224,9 @@ def logger_create(name, output=sys.stderr, level=logging.INFO, preserve_handlers
     """Standalone logger creation function"""
     logger = logging.getLogger(name)
     console = logging.StreamHandler(output)
+    console.addFilter(bb.msg.LogFilterShowOnce())
     format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-    if color == 'always' or (color == 'auto' and output.isatty()):
+    if color == 'always' or (color == 'auto' and output.isatty() and os.environ.get('NO_COLOR', '') == ''):
         format.enable_color()
     console.setFormatter(format)
     if preserve_handlers:
@@ -293,10 +312,17 @@ def setLoggingConfig(defaultconfig, userconfigfile=None):
 
     # Convert all level parameters to integers in case users want to use the
     # bitbake defined level names
-    for h in logconfig["handlers"].values():
+    for name, h in logconfig["handlers"].items():
         if "level" in h:
             h["level"] = bb.msg.stringToLevel(h["level"])
 
+        # Every handler needs its own instance of the once filter.
+        once_filter_name = name + ".showonceFilter"
+        logconfig.setdefault("filters", {})[once_filter_name] = {
+            "()": "bb.msg.LogFilterShowOnce",
+        }
+        h.setdefault("filters", []).append(once_filter_name)
+
     for l in logconfig["loggers"].values():
         if "level" in l:
             l["level"] = bb.msg.stringToLevel(l["level"])
diff --git a/bitbake/lib/bb/parse/__init__.py b/bitbake/lib/bb/parse/__init__.py
index c01807ba87..d428d8a4b4 100644
--- a/bitbake/lib/bb/parse/__init__.py
+++ b/bitbake/lib/bb/parse/__init__.py
@@ -49,20 +49,32 @@ class SkipPackage(SkipRecipe):
 __mtime_cache = {}
 def cached_mtime(f):
     if f not in __mtime_cache:
-        __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+        res = os.stat(f)
+        __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino)
     return __mtime_cache[f]
 
 def cached_mtime_noerror(f):
     if f not in __mtime_cache:
         try:
-            __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+            res = os.stat(f)
+            __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino)
         except OSError:
             return 0
     return __mtime_cache[f]
 
+def check_mtime(f, mtime):
+    try:
+        res = os.stat(f)
+        current_mtime = (res.st_mtime_ns, res.st_size, res.st_ino)
+        __mtime_cache[f] = current_mtime
+    except OSError:
+        current_mtime = 0
+    return current_mtime == mtime
+
 def update_mtime(f):
     try:
-        __mtime_cache[f] = os.stat(f)[stat.ST_MTIME]
+        res = os.stat(f)
+        __mtime_cache[f] = (res.st_mtime_ns, res.st_size, res.st_ino)
     except OSError:
         if f in __mtime_cache:
             del __mtime_cache[f]
@@ -99,12 +111,12 @@ def supports(fn, data):
             return 1
     return 0
 
-def handle(fn, data, include = 0):
+def handle(fn, data, include=0, baseconfig=False):
     """Call the handler that is appropriate for this file"""
     for h in handlers:
         if h['supports'](fn, data):
             with data.inchistory.include(fn):
-                return h['handle'](fn, data, include)
+                return h['handle'](fn, data, include, baseconfig)
     raise ParseError("not a BitBake file", fn)
 
 def init(fn, data):
@@ -113,6 +125,8 @@ def init(fn, data):
             return h['init'](data)
 
 def init_parser(d):
+    if hasattr(bb.parse, "siggen"):
+        bb.parse.siggen.exit()
     bb.parse.siggen = bb.siggen.init(d)
 
 def resolve_file(fn, d):
@@ -162,4 +176,41 @@ def get_file_depends(d):
         dep_files.append(os.path.abspath(fn))
     return " ".join(dep_files)
 
+def vardeps(*varnames):
+    """
+    Function decorator that can be used to instruct the bitbake dependency
+    parsing to add a dependency on the specified variables names
+
+    Example:
+
+        @bb.parse.vardeps("FOO", "BAR")
+        def my_function():
+            ...
+
+    """
+    def inner(f):
+        if not hasattr(f, "bb_vardeps"):
+            f.bb_vardeps = set()
+        f.bb_vardeps |= set(varnames)
+        return f
+    return inner
+
+def vardepsexclude(*varnames):
+    """
+    Function decorator that can be used to instruct the bitbake dependency
+    parsing to ignore dependencies on the specified variable names in the code
+
+    Example:
+
+        @bb.parse.vardepsexclude("FOO", "BAR")
+        def my_function():
+            ...
+    """
+    def inner(f):
+        if not hasattr(f, "bb_vardepsexclude"):
+            f.bb_vardepsexclude = set()
+        f.bb_vardepsexclude |= set(varnames)
+        return f
+    return inner
+
 from bb.parse.parse_py import __version__, ConfHandler, BBHandler
diff --git a/bitbake/lib/bb/parse/ast.py b/bitbake/lib/bb/parse/ast.py
index 50a88f7da7..49a0788038 100644
--- a/bitbake/lib/bb/parse/ast.py
+++ b/bitbake/lib/bb/parse/ast.py
@@ -9,6 +9,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import sys
 import bb
 from bb import methodpool
 from bb.parse import logger
@@ -42,6 +43,21 @@ class IncludeNode(AstNode):
         else:
             bb.parse.ConfHandler.include(self.filename, s, self.lineno, data, False)
 
+class IncludeAllNode(AstNode):
+    def __init__(self, filename, lineno, what_file):
+        AstNode.__init__(self, filename, lineno)
+        self.what_file = what_file
+
+    def eval(self, data):
+        """
+        Include the file and evaluate the statements
+        """
+        s = data.expand(self.what_file)
+        logger.debug2("CONF %s:%s: including %s", self.filename, self.lineno, s)
+
+        for path in data.getVar("BBPATH").split(":"):
+            bb.parse.ConfHandler.include(self.filename, os.path.join(path, s), self.lineno, data, False)
+
 class ExportNode(AstNode):
     def __init__(self, filename, lineno, var):
         AstNode.__init__(self, filename, lineno)
@@ -130,9 +146,16 @@ class DataNode(AstNode):
         else:
             val = groupd["value"]
 
+        if ":append" in key or ":remove" in key or ":prepend" in key:
+            if op in ["append", "prepend", "postdot", "predot", "ques"]:
+                bb.warn(key + " " + groupd[op] + " is not a recommended operator combination, please replace it.")
+
         flag = None
         if 'flag' in groupd and groupd['flag'] is not None:
-            flag = groupd['flag']
+            if groupd["lazyques"]:
+                flag = "_defaultval_flag_"+groupd['flag']
+            else:
+                flag = groupd['flag']
         elif groupd["lazyques"]:
             flag = "_defaultval"
 
@@ -145,7 +168,7 @@ class DataNode(AstNode):
             data.setVar(key, val, parsing=True, **loginfo)
 
 class MethodNode(AstNode):
-    tr_tbl = str.maketrans('/.+-@%&', '_______')
+    tr_tbl = str.maketrans('/.+-@%&~', '________')
 
     def __init__(self, filename, lineno, func_name, body, python, fakeroot):
         AstNode.__init__(self, filename, lineno)
@@ -206,10 +229,12 @@ class ExportFuncsNode(AstNode):
 
     def eval(self, data):
 
+        sentinel = "    # Export function set\n"
         for func in self.n:
             calledfunc = self.classname + "_" + func
 
-            if data.getVar(func, False) and not data.getVarFlag(func, 'export_func', False):
+            basevar = data.getVar(func, False)
+            if basevar and sentinel not in basevar:
                 continue
 
             if data.getVar(func, False):
@@ -219,29 +244,30 @@ class ExportFuncsNode(AstNode):
             for flag in [ "func", "python" ]:
                 if data.getVarFlag(calledfunc, flag, False):
                     data.setVarFlag(func, flag, data.getVarFlag(calledfunc, flag, False))
-            for flag in [ "dirs" ]:
+            for flag in ["dirs", "cleandirs", "fakeroot"]:
                 if data.getVarFlag(func, flag, False):
                     data.setVarFlag(calledfunc, flag, data.getVarFlag(func, flag, False))
             data.setVarFlag(func, "filename", "autogenerated")
             data.setVarFlag(func, "lineno", 1)
 
             if data.getVarFlag(calledfunc, "python", False):
-                data.setVar(func, "    bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True)
+                data.setVar(func, sentinel + "    bb.build.exec_func('" + calledfunc + "', d)\n", parsing=True)
             else:
                 if "-" in self.classname:
                    bb.fatal("The classname %s contains a dash character and is calling an sh function %s using EXPORT_FUNCTIONS. Since a dash is illegal in sh function names, this cannot work, please rename the class or don't use EXPORT_FUNCTIONS." % (self.classname, calledfunc))
-                data.setVar(func, "    " + calledfunc + "\n", parsing=True)
-            data.setVarFlag(func, 'export_func', '1')
+                data.setVar(func, sentinel + "    " + calledfunc + "\n", parsing=True)
 
 class AddTaskNode(AstNode):
-    def __init__(self, filename, lineno, func, before, after):
+    def __init__(self, filename, lineno, tasks, before, after):
         AstNode.__init__(self, filename, lineno)
-        self.func = func
+        self.tasks = tasks
         self.before = before
         self.after = after
 
     def eval(self, data):
-        bb.build.addtask(self.func, self.before, self.after, data)
+        tasks = self.tasks.split()
+        for task in tasks:
+            bb.build.addtask(task, self.before, self.after, data)
 
 class DelTaskNode(AstNode):
     def __init__(self, filename, lineno, tasks):
@@ -265,6 +291,41 @@ class BBHandlerNode(AstNode):
             data.setVarFlag(h, "handler", 1)
         data.setVar('__BBHANDLERS', bbhands)
 
+class PyLibNode(AstNode):
+    def __init__(self, filename, lineno, libdir, namespace):
+        AstNode.__init__(self, filename, lineno)
+        self.libdir = libdir
+        self.namespace = namespace
+
+    def eval(self, data):
+        global_mods = (data.getVar("BB_GLOBAL_PYMODULES") or "").split()
+        for m in global_mods:
+            if m not in bb.utils._context:
+                bb.utils._context[m] = __import__(m)
+
+        libdir = data.expand(self.libdir)
+        if libdir not in sys.path:
+            sys.path.append(libdir)
+        try:
+            bb.utils._context[self.namespace] = __import__(self.namespace)
+            toimport = getattr(bb.utils._context[self.namespace], "BBIMPORTS", [])
+            for i in toimport:
+                bb.utils._context[self.namespace] = __import__(self.namespace + "." + i)
+                mod = getattr(bb.utils._context[self.namespace], i)
+                fn = getattr(mod, "__file__")
+                funcs = {}
+                for f in dir(mod):
+                    if f.startswith("_"):
+                        continue
+                    fcall = getattr(mod, f)
+                    if not callable(fcall):
+                        continue
+                    funcs[f] = fcall
+                bb.codeparser.add_module_functions(fn, funcs, "%s.%s" % (self.namespace, i))
+
+        except AttributeError as e:
+            bb.error("Error importing OE modules: %s" % str(e))
+
 class InheritNode(AstNode):
     def __init__(self, filename, lineno, classes):
         AstNode.__init__(self, filename, lineno)
@@ -273,9 +334,68 @@ class InheritNode(AstNode):
     def eval(self, data):
         bb.parse.BBHandler.inherit(self.classes, self.filename, self.lineno, data)
 
+class InheritDeferredNode(AstNode):
+    def __init__(self, filename, lineno, classes):
+        AstNode.__init__(self, filename, lineno)
+        self.inherit = (classes, filename, lineno)
+
+    def eval(self, data):
+        bb.parse.BBHandler.inherit_defer(*self.inherit, data)
+
+class AddFragmentsNode(AstNode):
+    def __init__(self, filename, lineno, fragments_path_prefix, fragments_variable, flagged_variables_list_variable, builtin_fragments_variable):
+        AstNode.__init__(self, filename, lineno)
+        self.fragments_path_prefix = fragments_path_prefix
+        self.fragments_variable = fragments_variable
+        self.flagged_variables_list_variable = flagged_variables_list_variable
+        self.builtin_fragments_variable = builtin_fragments_variable
+
+    def eval(self, data):
+        # No need to use mark_dependency since we would only match a fragment
+        # from a specific layer and there can only be a single layer with a
+        # given namespace.
+        def find_fragment(layers, layerid, full_fragment_name):
+           for layerpath in layers.split():
+               candidate_fragment_path = os.path.join(layerpath, full_fragment_name)
+               if os.path.exists(candidate_fragment_path) and bb.utils.get_file_layer(candidate_fragment_path, data) == layerid:
+                   return candidate_fragment_path
+           return None
+
+        def check_and_set_builtin_fragment(fragment, data, builtin_fragments):
+            prefix, value = fragment.split('/', 1)
+            if prefix in builtin_fragments.keys():
+                data.setVar(builtin_fragments[prefix], value)
+                return True
+            return False
+
+        fragments = data.getVar(self.fragments_variable)
+        layers = data.getVar('BBLAYERS')
+        flagged_variables = data.getVar(self.flagged_variables_list_variable).split()
+        builtin_fragments = {f[0]:f[1] for f in [f.split(':') for f in data.getVar(self.builtin_fragments_variable).split()] }
+
+        if not fragments:
+            return
+        for f in fragments.split():
+            if check_and_set_builtin_fragment(f, data, builtin_fragments):
+                continue
+            layerid, fragment_name = f.split('/', 1)
+            full_fragment_name = data.expand("{}/{}.conf".format(self.fragments_path_prefix, fragment_name))
+            fragment_path = find_fragment(layers, layerid, full_fragment_name)
+            if fragment_path:
+                bb.parse.ConfHandler.include(self.filename, fragment_path, self.lineno, data, "include fragment")
+                for flagged_var in flagged_variables:
+                    val = data.getVar(flagged_var)
+                    data.setVarFlag(flagged_var, f, val)
+                    data.setVar(flagged_var, None)
+            else:
+                bb.error("Could not find fragment {} in enabled layers: {}".format(f, layers))
+
 def handleInclude(statements, filename, lineno, m, force):
     statements.append(IncludeNode(filename, lineno, m.group(1), force))
 
+def handleIncludeAll(statements, filename, lineno, m):
+    statements.append(IncludeAllNode(filename, lineno, m.group(1)))
+
 def handleExport(statements, filename, lineno, m):
     statements.append(ExportNode(filename, lineno, m.group(1)))
 
@@ -297,38 +417,81 @@ def handlePythonMethod(statements, filename, lineno, funcname, modulename, body)
 def handleExportFuncs(statements, filename, lineno, m, classname):
     statements.append(ExportFuncsNode(filename, lineno, m.group(1), classname))
 
-def handleAddTask(statements, filename, lineno, m):
-    func = m.group("func")
-    before = m.group("before")
-    after = m.group("after")
-    if func is None:
-        return
-
-    statements.append(AddTaskNode(filename, lineno, func, before, after))
+def handleAddTask(statements, filename, lineno, tasks, before, after):
+    statements.append(AddTaskNode(filename, lineno, tasks, before, after))
 
-def handleDelTask(statements, filename, lineno, m):
-    func = m.group(1)
-    if func is None:
-        return
-
-    statements.append(DelTaskNode(filename, lineno, func))
+def handleDelTask(statements, filename, lineno, tasks):
+    statements.append(DelTaskNode(filename, lineno, tasks))
 
 def handleBBHandlers(statements, filename, lineno, m):
     statements.append(BBHandlerNode(filename, lineno, m.group(1)))
 
+def handlePyLib(statements, filename, lineno, m):
+    statements.append(PyLibNode(filename, lineno, m.group(1), m.group(2)))
+
 def handleInherit(statements, filename, lineno, m):
     classes = m.group(1)
     statements.append(InheritNode(filename, lineno, classes))
 
+def handleInheritDeferred(statements, filename, lineno, m):
+    classes = m.group(1)
+    statements.append(InheritDeferredNode(filename, lineno, classes))
+
+def handleAddFragments(statements, filename, lineno, m):
+    fragments_path_prefix = m.group(1)
+    fragments_variable = m.group(2)
+    flagged_variables_list_variable = m.group(3)
+    builtin_fragments_variable = m.group(4)
+    statements.append(AddFragmentsNode(filename, lineno, fragments_path_prefix, fragments_variable, flagged_variables_list_variable, builtin_fragments_variable))
+
 def runAnonFuncs(d):
     code = []
     for funcname in d.getVar("__BBANONFUNCS", False) or []:
         code.append("%s(d)" % funcname)
     bb.utils.better_exec("\n".join(code), {"d": d})
 
+# Handle recipe level PREFERRED_PROVIDERs
+def handleVirtRecipeProviders(tasklist, d):
+    depends = (d.getVar("DEPENDS") or "").split()
+    virtprovs = (d.getVar("BB_RECIPE_VIRTUAL_PROVIDERS") or "").split()
+    newdeps = []
+    for dep in depends:
+        if dep in virtprovs:
+            newdep = d.getVar("PREFERRED_PROVIDER_" + dep)
+            if not newdep:
+                 bb.fatal("Error, recipe virtual provider PREFERRED_PROVIDER_%s not set" % dep)
+            newdeps.append(newdep)
+        else:
+            newdeps.append(dep)
+    d.setVar("DEPENDS", " ".join(newdeps))
+    for task in tasklist:
+        taskdeps = (d.getVarFlag(task, "depends") or "").split()
+        remapped = []
+        for entry in taskdeps:
+            r, t = entry.split(":")
+            if r in virtprovs:
+                r = d.getVar("PREFERRED_PROVIDER_" + r)
+            remapped.append("%s:%s" % (r, t))
+        d.setVarFlag(task, "depends", " ".join(remapped))
+
 def finalize(fn, d, variant = None):
     saved_handlers = bb.event.get_handlers().copy()
     try:
+        # Found renamed variables. Exit immediately
+        if d.getVar("_FAILPARSINGERRORHANDLED", False) == True:
+            raise bb.BBHandledException()
+
+        inherits = [x[0] for x in (d.getVar('__BBDEFINHERITS', False) or [('',)])]
+        bb.event.fire(bb.event.RecipePreDeferredInherits(fn, inherits), d)
+
+        while True:
+            inherits = d.getVar('__BBDEFINHERITS', False) or []
+            if not inherits:
+                break
+            inherit, filename, lineno = inherits.pop(0)
+            d.setVar('__BBDEFINHERITS', inherits)
+            bb.parse.BBHandler.inherit(inherit, filename, lineno, d, deferred=True)
+
         for var in d.getVar('__BBHANDLERS', False) or []:
             # try to add the handler
             handlerfn = d.getVarFlag(var, "filename", False)
@@ -347,12 +510,16 @@ def finalize(fn, d, variant = None):
 
         tasklist = d.getVar('__BBTASKS', False) or []
         bb.event.fire(bb.event.RecipeTaskPreProcess(fn, list(tasklist)), d)
+        handleVirtRecipeProviders(tasklist, d)
         bb.build.add_tasks(tasklist, d)
 
         bb.parse.siggen.finalise(fn, d, variant)
 
         d.setVar('BBINCLUDED', bb.parse.get_file_depends(d))
 
+        if d.getVar('__BBAUTOREV_SEEN') and d.getVar('__BBSRCREV_SEEN') and not d.getVar("__BBAUTOREV_ACTED_UPON"):
+            bb.fatal("AUTOREV/SRCPV set too late for the fetcher to work properly, please set the variables earlier in parsing. Erroring instead of later obtuse build failures.")
+
         bb.event.fire(bb.event.RecipeParsed(fn), d)
     finally:
         bb.event.set_handlers(saved_handlers)
@@ -414,7 +581,7 @@ def multi_finalize(fn, d):
                 d.setVar("BBEXTENDVARIANT", variantmap[name])
             else:
                 d.setVar("PN", "%s-%s" % (pn, name))
-            bb.parse.BBHandler.inherit(extendedmap[name], fn, 0, d)
+            bb.parse.BBHandler.inherit_defer(extendedmap[name], fn, 0, d)
 
         safe_d.setVar("BBCLASSEXTEND", extended)
         _create_variants(datastores, extendedmap.keys(), extendfunc, onlyfinalise)
diff --git a/bitbake/lib/bb/parse/parse_py/BBHandler.py b/bitbake/lib/bb/parse/parse_py/BBHandler.py
index f8988b8631..008fec2308 100644
--- a/bitbake/lib/bb/parse/parse_py/BBHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/BBHandler.py
@@ -19,14 +19,12 @@ from . import ConfHandler
 from .. import resolve_file, ast, logger, ParseError
 from .ConfHandler import include, init
 
-# For compatibility
-bb.deprecate_import(__name__, "bb.parse", ["vars_from_file"])
-
-__func_start_regexp__    = re.compile(r"(((?P<py>python(?=(\s|\()))|(?P<fr>fakeroot(?=\s)))\s*)*(?P<func>[\w\.\-\+\{\}\$]+)?\s*\(\s*\)\s*{$" )
+__func_start_regexp__    = re.compile(r"(((?P<py>python(?=(\s|\()))|(?P<fr>fakeroot(?=\s)))\s*)*(?P<func>[\w\.\-\+\{\}\$:]+)?\s*\(\s*\)\s*{$" )
 __inherit_regexp__       = re.compile(r"inherit\s+(.+)" )
+__inherit_def_regexp__   = re.compile(r"inherit_defer\s+(.+)" )
 __export_func_regexp__   = re.compile(r"EXPORT_FUNCTIONS\s+(.+)" )
-__addtask_regexp__       = re.compile(r"addtask\s+(?P<func>\w+)\s*((before\s*(?P<before>((.*(?=after))|(.*))))|(after\s*(?P<after>((.*(?=before))|(.*)))))*")
-__deltask_regexp__       = re.compile(r"deltask\s+(.+)")
+__addtask_regexp__       = re.compile(r"addtask\s+([^#\n]+)(?P<comment>#.*|.*?)")
+__deltask_regexp__       = re.compile(r"deltask\s+([^#\n]+)(?P<comment>#.*|.*?)")
 __addhandler_regexp__    = re.compile(r"addhandler\s+(.+)" )
 __def_regexp__           = re.compile(r"def\s+(\w+).*:" )
 __python_func_regexp__   = re.compile(r"(\s+.*)|(^$)|(^#)" )
@@ -36,6 +34,7 @@ __infunc__ = []
 __inpython__ = False
 __body__   = []
 __classname__ = ""
+__residue__ = []
 
 cached_statements = {}
 
@@ -43,31 +42,56 @@ def supports(fn, d):
     """Return True if fn has a supported extension"""
     return os.path.splitext(fn)[-1] in [".bb", ".bbclass", ".inc"]
 
-def inherit(files, fn, lineno, d):
+def inherit_defer(expression, fn, lineno, d):
+    inherit = (expression, fn, lineno)
+    inherits = d.getVar('__BBDEFINHERITS', False) or []
+    inherits.append(inherit)
+    d.setVar('__BBDEFINHERITS', inherits)
+
+def inherit(files, fn, lineno, d, deferred=False):
     __inherit_cache = d.getVar('__inherit_cache', False) or []
+    #if "${" in files and not deferred:
+    #    bb.warn("%s:%s has non deferred conditional inherit" % (fn, lineno))
     files = d.expand(files).split()
     for file in files:
-        if not os.path.isabs(file) and not file.endswith(".bbclass"):
-            file = os.path.join('classes', '%s.bbclass' % file)
-
-        if not os.path.isabs(file):
-            bbpath = d.getVar("BBPATH")
-            abs_fn, attempts = bb.utils.which(bbpath, file, history=True)
-            for af in attempts:
-                if af != abs_fn:
-                    bb.parse.mark_dependency(d, af)
-            if abs_fn:
-                file = abs_fn
+        defer = (d.getVar("BB_DEFER_BBCLASSES") or "").split()
+        if not deferred and file in defer:
+            inherit_defer(file, fn, lineno, d)
+            continue
+        classtype = d.getVar("__bbclasstype", False)
+        origfile = file
+        for t in ["classes-" + classtype, "classes"]:
+            file = origfile
+            if not os.path.isabs(file) and not file.endswith(".bbclass"):
+                file = os.path.join(t, '%s.bbclass' % file)
+
+            if not os.path.isabs(file):
+                bbpath = d.getVar("BBPATH")
+                abs_fn, attempts = bb.utils.which(bbpath, file, history=True)
+                for af in attempts:
+                    if af != abs_fn:
+                        bb.parse.mark_dependency(d, af)
+                if abs_fn:
+                    file = abs_fn
+
+            if os.path.exists(file):
+                break
+
+        if not os.path.exists(file):
+            raise ParseError("Could not inherit file %s" % (file), fn, lineno)
 
         if not file in __inherit_cache:
             logger.debug("Inheriting %s (from %s:%d)" % (file, fn, lineno))
             __inherit_cache.append( file )
             d.setVar('__inherit_cache', __inherit_cache)
-            include(fn, file, lineno, d, "inherit")
+            try:
+                bb.parse.handle(file, d, True)
+            except (IOError, OSError) as exc:
+                raise ParseError("Could not inherit file %s: %s" % (fn, exc.strerror), fn, lineno)
             __inherit_cache = d.getVar('__inherit_cache', False) or []
 
 def get_statements(filename, absolute_filename, base_name):
-    global cached_statements
+    global cached_statements, __residue__, __body__
 
     try:
         return cached_statements[absolute_filename]
@@ -87,12 +111,17 @@ def get_statements(filename, absolute_filename, base_name):
             # add a blank line to close out any python definition
             feeder(lineno, "", filename, base_name, statements, eof=True)
 
+        if __residue__:
+            raise ParseError("Unparsed lines %s: %s" % (filename, str(__residue__)), filename, lineno)
+        if __body__:
+            raise ParseError("Unparsed lines from unclosed function %s: %s" % (filename, str(__body__)), filename, lineno)
+
         if filename.endswith(".bbclass") or filename.endswith(".inc"):
             cached_statements[absolute_filename] = statements
         return statements
 
-def handle(fn, d, include):
-    global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __infunc__, __body__, __residue__, __classname__
+def handle(fn, d, include, baseconfig=False):
+    global __infunc__, __body__, __residue__, __classname__
     __body__ = []
     __infunc__ = []
     __classname__ = ""
@@ -144,7 +173,7 @@ def handle(fn, d, include):
     return d
 
 def feeder(lineno, s, fn, root, statements, eof=False):
-    global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __def_regexp__, __python_func_regexp__, __inpython__, __infunc__, __body__, bb, __residue__, __classname__
+    global __inpython__, __infunc__, __body__, __residue__, __classname__
 
     # Check tabs in python functions:
     # - def py_funcname(): covered by __inpython__
@@ -181,10 +210,10 @@ def feeder(lineno, s, fn, root, statements, eof=False):
 
     if s and s[0] == '#':
         if len(__residue__) != 0 and __residue__[0][0] != "#":
-            bb.fatal("There is a comment on line %s of file %s (%s) which is in the middle of a multiline expression.\nBitbake used to ignore these but no longer does so, please fix your metadata as errors are likely as a result of this change." % (lineno, fn, s))
+            bb.fatal("There is a comment on line %s of file %s:\n'''\n%s\n'''\nwhich is in the middle of a multiline expression. This syntax is invalid, please correct it." % (lineno, fn, s))
 
     if len(__residue__) != 0 and __residue__[0][0] == "#" and (not s or s[0] != "#"):
-        bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
+        bb.fatal("There is a confusing multiline partially commented expression on line %s of file %s:\n%s\nPlease clarify whether this is all a comment or should be parsed." % (lineno - len(__residue__), fn, "\n".join(__residue__)))
 
     if s and s[-1] == '\\':
         __residue__.append(s[:-1])
@@ -220,29 +249,38 @@ def feeder(lineno, s, fn, root, statements, eof=False):
 
     m = __addtask_regexp__.match(s)
     if m:
-        if len(m.group().split()) == 2:
-            # Check and warn for "addtask task1 task2"
-            m2 = re.match(r"addtask\s+(?P<func>\w+)(?P<ignores>.*)", s)
-            if m2 and m2.group('ignores'):
-                logger.warning('addtask ignored: "%s"' % m2.group('ignores'))
-
-        # Check and warn for "addtask task1 before task2 before task3", the
-        # similar to "after"
-        taskexpression = s.split()
-        for word in ('before', 'after'):
-            if taskexpression.count(word) > 1:
-                logger.warning("addtask contained multiple '%s' keywords, only one is supported" % word)
+        after = ""
+        before = ""
+
+        # This code splits on 'before' and 'after' instead of on whitespace so we can defer
+        # evaluation to as late as possible.
+        tasks = m.group(1).split(" before ")[0].split(" after ")[0]
+
+        for exp in m.group(1).split(" before "):
+            exp2 = exp.split(" after ")
+            if len(exp2) > 1:
+                after = after + " ".join(exp2[1:])
 
-        # Check and warn for having task with exprssion as part of task name
+        for exp in m.group(1).split(" after "):
+            exp2 = exp.split(" before ")
+            if len(exp2) > 1:
+                before = before + " ".join(exp2[1:])
+
+        # Check and warn for having task with a keyword as part of task name
+        taskexpression = s.split()
         for te in taskexpression:
             if any( ( "%s_" % keyword ) in te for keyword in bb.data_smart.__setvar_keyword__ ):
                 raise ParseError("Task name '%s' contains a keyword which is not recommended/supported.\nPlease rename the task not to include the keyword.\n%s" % (te, ("\n".join(map(str, bb.data_smart.__setvar_keyword__)))), fn)
-        ast.handleAddTask(statements, fn, lineno, m)
+
+        if tasks is not None:
+            ast.handleAddTask(statements, fn, lineno, tasks, before, after)
         return
 
     m = __deltask_regexp__.match(s)
     if m:
-        ast.handleDelTask(statements, fn, lineno, m)
+        task = m.group(1)
+        if task is not None:
+            ast.handleDelTask(statements, fn, lineno, task)
         return
 
     m = __addhandler_regexp__.match(s)
@@ -255,7 +293,12 @@ def feeder(lineno, s, fn, root, statements, eof=False):
         ast.handleInherit(statements, fn, lineno, m)
         return
 
-    return ConfHandler.feeder(lineno, s, fn, statements)
+    m = __inherit_def_regexp__.match(s)
+    if m:
+        ast.handleInheritDeferred(statements, fn, lineno, m)
+        return
+
+    return ConfHandler.feeder(lineno, s, fn, statements, conffile=False)
 
 # Add us to the handlers list
 from .. import handlers
diff --git a/bitbake/lib/bb/parse/parse_py/ConfHandler.py b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
index f171c5c932..9ddbae123d 100644
--- a/bitbake/lib/bb/parse/parse_py/ConfHandler.py
+++ b/bitbake/lib/bb/parse/parse_py/ConfHandler.py
@@ -20,10 +20,10 @@ from bb.parse import ParseError, resolve_file, ast, logger, handle
 __config_regexp__  = re.compile( r"""
     ^
     (?P<exp>export\s+)?
-    (?P<var>[a-zA-Z0-9\-_+.${}/~]+?)
-    (\[(?P<flag>[a-zA-Z0-9\-_+.]+)\])?
+    (?P<var>[a-zA-Z0-9\-_+.${}/~:]*?)
+    (\[(?P<flag>[a-zA-Z0-9\-_+.][a-zA-Z0-9\-_+.@/]*)\])?
 
-    \s* (
+    (?P<whitespace>\s*) (
         (?P<colon>:=) |
         (?P<lazyques>\?\?=) |
         (?P<ques>\?=) |
@@ -32,7 +32,7 @@ __config_regexp__  = re.compile( r"""
         (?P<predot>=\.) |
         (?P<postdot>\.=) |
         =
-    ) \s*
+    ) (?P<whitespace2>\s*)
 
     (?!'[^']*'[^']*'$)
     (?!\"[^\"]*\"[^\"]*\"$)
@@ -43,15 +43,15 @@ __config_regexp__  = re.compile( r"""
     """, re.X)
 __include_regexp__ = re.compile( r"include\s+(.+)" )
 __require_regexp__ = re.compile( r"require\s+(.+)" )
+__includeall_regexp__ = re.compile( r"include_all\s+(.+)" )
 __export_regexp__ = re.compile( r"export\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
 __unset_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)$" )
-__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.]+)\]$" )
+__unset_flag_regexp__ = re.compile( r"unset\s+([a-zA-Z0-9\-_+.${}/~]+)\[([a-zA-Z0-9\-_+.][a-zA-Z0-9\-_+.@]+)\]$" )
+__addpylib_regexp__      = re.compile(r"addpylib\s+(.+)\s+(.+)" )
+__addfragments_regexp__  = re.compile(r"addfragments\s+(.+)\s+(.+)\s+(.+)\s+(.+)" )
 
 def init(data):
-    topdir = data.getVar('TOPDIR', False)
-    if not topdir:
-        data.setVar('TOPDIR', os.getcwd())
-
+    return
 
 def supports(fn, d):
     return fn[-5:] == ".conf"
@@ -105,12 +105,12 @@ def include_single_file(parentfn, fn, lineno, data, error_out):
 # We have an issue where a UI might want to enforce particular settings such as
 # an empty DISTRO variable. If configuration files do something like assigning
 # a weak default, it turns out to be very difficult to filter out these changes,
-# particularly when the weak default might appear half way though parsing a chain 
+# particularly when the weak default might appear half way though parsing a chain
 # of configuration files. We therefore let the UIs hook into configuration file
 # parsing. This turns out to be a hard problem to solve any other way.
 confFilters = []
 
-def handle(fn, data, include):
+def handle(fn, data, include, baseconfig=False):
     init(data)
 
     if include == 0:
@@ -128,21 +128,26 @@ def handle(fn, data, include):
             s = f.readline()
             if not s:
                 break
+            origlineno = lineno
+            origline = s
             w = s.strip()
             # skip empty lines
             if not w:
                 continue
             s = s.rstrip()
             while s[-1] == '\\':
-                s2 = f.readline().rstrip()
+                line = f.readline()
+                origline += line
+                s2 = line.rstrip()
                 lineno = lineno + 1
                 if (not s2 or s2 and s2[0] != "#") and s[0] == "#" :
-                    bb.fatal("There is a confusing multiline, partially commented expression on line %s of file %s (%s).\nPlease clarify whether this is all a comment or should be parsed." % (lineno, fn, s))
+                    bb.fatal("There is a confusing multiline, partially commented expression starting on line %s of file %s:\n%s\nPlease clarify whether this is all a comment or should be parsed." % (origlineno, fn, origline))
+
                 s = s[:-1] + s2
             # skip comments
             if s[0] == '#':
                 continue
-            feeder(lineno, s, abs_fn, statements)
+            feeder(lineno, s, abs_fn, statements, baseconfig=baseconfig)
 
     # DONE WITH PARSING... time to evaluate
     data.setVar('FILE', abs_fn)
@@ -150,17 +155,21 @@ def handle(fn, data, include):
     if oldfile:
         data.setVar('FILE', oldfile)
 
-    f.close()
-
     for f in confFilters:
         f(fn, data)
 
     return data
 
-def feeder(lineno, s, fn, statements):
+# baseconfig is set for the bblayers/layer.conf cookerdata config parsing
+# The function is also used by BBHandler, conffile would be False
+def feeder(lineno, s, fn, statements, baseconfig=False, conffile=True):
     m = __config_regexp__.match(s)
     if m:
         groupd = m.groupdict()
+        if groupd['var'] == "":
+            raise ParseError("Empty variable name in assignment: '%s'" % s, fn, lineno);
+        if not groupd['whitespace'] or not groupd['whitespace2']:
+            logger.warning("%s:%s has a lack of whitespace around the assignment: '%s'" % (fn, lineno, s))
         ast.handleData(statements, fn, lineno, groupd)
         return
 
@@ -174,6 +183,11 @@ def feeder(lineno, s, fn, statements):
         ast.handleInclude(statements, fn, lineno, m, True)
         return
 
+    m = __includeall_regexp__.match(s)
+    if m:
+        ast.handleIncludeAll(statements, fn, lineno, m)
+        return
+
     m = __export_regexp__.match(s)
     if m:
         ast.handleExport(statements, fn, lineno, m)
@@ -189,6 +203,16 @@ def feeder(lineno, s, fn, statements):
         ast.handleUnsetFlag(statements, fn, lineno, m)
         return
 
+    m = __addpylib_regexp__.match(s)
+    if baseconfig and conffile and m:
+        ast.handlePyLib(statements, fn, lineno, m)
+        return
+
+    m = __addfragments_regexp__.match(s)
+    if m:
+        ast.handleAddFragments(statements, fn, lineno, m)
+        return
+
     raise ParseError("unparsed line: '%s'" % s, fn, lineno);
 
 # Add us to the handlers list
diff --git a/bitbake/lib/bb/persist_data.py b/bitbake/lib/bb/persist_data.py
deleted file mode 100644
index c6a209fb3f..0000000000
--- a/bitbake/lib/bb/persist_data.py
+++ /dev/null
@@ -1,301 +0,0 @@
-"""BitBake Persistent Data Store
-
-Used to store data in a central location such that other threads/tasks can
-access them at some future date.  Acts as a convenience wrapper around sqlite,
-currently, providing a key/value store accessed by 'domain'.
-"""
-
-# Copyright (C) 2007        Richard Purdie
-# Copyright (C) 2010        Chris Larson <chris_larson@mentor.com>
-#
-# SPDX-License-Identifier: GPL-2.0-only
-#
-
-import collections
-import contextlib
-import functools
-import logging
-import os.path
-import sqlite3
-import sys
-import warnings
-from collections import Mapping
-
-sqlversion = sqlite3.sqlite_version_info
-if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
-    raise Exception("sqlite3 version 3.3.0 or later is required.")
-
-
-logger = logging.getLogger("BitBake.PersistData")
-
-@functools.total_ordering
-class SQLTable(collections.MutableMapping):
-    class _Decorators(object):
-        @staticmethod
-        def retry(*, reconnect=True):
-            """
-            Decorator that restarts a function if a database locked sqlite
-            exception occurs. If reconnect is True, the database connection
-            will be closed and reopened each time a failure occurs
-            """
-            def retry_wrapper(f):
-                def wrap_func(self, *args, **kwargs):
-                    # Reconnect if necessary
-                    if self.connection is None and reconnect:
-                        self.reconnect()
-
-                    count = 0
-                    while True:
-                        try:
-                            return f(self, *args, **kwargs)
-                        except sqlite3.OperationalError as exc:
-                            if count < 500 and ('is locked' in str(exc) or 'locking protocol' in str(exc)):
-                                count = count + 1
-                                if reconnect:
-                                    self.reconnect()
-                                continue
-                            raise
-                return wrap_func
-            return retry_wrapper
-
-        @staticmethod
-        def transaction(f):
-            """
-            Decorator that starts a database transaction and creates a database
-            cursor for performing queries. If no exception is thrown, the
-            database results are commited. If an exception occurs, the database
-            is rolled back. In all cases, the cursor is closed after the
-            function ends.
-
-            Note that the cursor is passed as an extra argument to the function
-            after `self` and before any of the normal arguments
-            """
-            def wrap_func(self, *args, **kwargs):
-                # Context manager will COMMIT the database on success,
-                # or ROLLBACK on an exception
-                with self.connection:
-                    # Automatically close the cursor when done
-                    with contextlib.closing(self.connection.cursor()) as cursor:
-                        return f(self, cursor, *args, **kwargs)
-            return wrap_func
-
-    """Object representing a table/domain in the database"""
-    def __init__(self, cachefile, table):
-        self.cachefile = cachefile
-        self.table = table
-
-        self.connection = None
-        self._execute_single("CREATE TABLE IF NOT EXISTS %s(key TEXT PRIMARY KEY NOT NULL, value TEXT);" % table)
-
-    @_Decorators.retry(reconnect=False)
-    @_Decorators.transaction
-    def _setup_database(self, cursor):
-        cursor.execute("pragma synchronous = off;")
-        # Enable WAL and keep the autocheckpoint length small (the default is
-        # usually 1000). Persistent caches are usually read-mostly, so keeping
-        # this short will keep readers running quickly
-        cursor.execute("pragma journal_mode = WAL;")
-        cursor.execute("pragma wal_autocheckpoint = 100;")
-
-    def reconnect(self):
-        if self.connection is not None:
-            self.connection.close()
-        self.connection = sqlite3.connect(self.cachefile, timeout=5)
-        self.connection.text_factory = str
-        self._setup_database()
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def _execute_single(self, cursor, *query):
-        """
-        Executes a single query and discards the results. This correctly closes
-        the database cursor when finished
-        """
-        cursor.execute(*query)
-
-    @_Decorators.retry()
-    def _row_iter(self, f, *query):
-        """
-        Helper function that returns a row iterator. Each time __next__ is
-        called on the iterator, the provided function is evaluated to determine
-        the return value
-        """
-        class CursorIter(object):
-            def __init__(self, cursor):
-                self.cursor = cursor
-
-            def __iter__(self):
-                return self
-
-            def __next__(self):
-                row = self.cursor.fetchone()
-                if row is None:
-                    self.cursor.close()
-                    raise StopIteration
-                return f(row)
-
-            def __enter__(self):
-                return self
-
-            def __exit__(self, typ, value, traceback):
-                self.cursor.close()
-                return False
-
-        cursor = self.connection.cursor()
-        try:
-            cursor.execute(*query)
-            return CursorIter(cursor)
-        except:
-            cursor.close()
-
-    def __enter__(self):
-        self.connection.__enter__()
-        return self
-
-    def __exit__(self, *excinfo):
-        self.connection.__exit__(*excinfo)
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def __getitem__(self, cursor, key):
-        cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
-        row = cursor.fetchone()
-        if row is not None:
-            return row[1]
-        raise KeyError(key)
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def __delitem__(self, cursor, key):
-        if key not in self:
-            raise KeyError(key)
-        cursor.execute("DELETE from %s where key=?;" % self.table, [key])
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def __setitem__(self, cursor, key, value):
-        if not isinstance(key, str):
-            raise TypeError('Only string keys are supported')
-        elif not isinstance(value, str):
-            raise TypeError('Only string values are supported')
-
-        # Ensure the entire transaction (including SELECT) executes under write lock
-        cursor.execute("BEGIN EXCLUSIVE")
-
-        cursor.execute("SELECT * from %s where key=?;" % self.table, [key])
-        row = cursor.fetchone()
-        if row is not None:
-            cursor.execute("UPDATE %s SET value=? WHERE key=?;" % self.table, [value, key])
-        else:
-            cursor.execute("INSERT into %s(key, value) values (?, ?);" % self.table, [key, value])
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def __contains__(self, cursor, key):
-        cursor.execute('SELECT * from %s where key=?;' % self.table, [key])
-        return cursor.fetchone() is not None
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def __len__(self, cursor):
-        cursor.execute("SELECT COUNT(key) FROM %s;" % self.table)
-        row = cursor.fetchone()
-        if row is not None:
-            return row[0]
-
-    def __iter__(self):
-        return self._row_iter(lambda row: row[0], "SELECT key from %s;" % self.table)
-
-    def __lt__(self, other):
-        if not isinstance(other, Mapping):
-            raise NotImplemented
-
-        return len(self) < len(other)
-
-    def get_by_pattern(self, pattern):
-        return self._row_iter(lambda row: row[1], "SELECT * FROM %s WHERE key LIKE ?;" %
-                              self.table, [pattern])
-
-    def values(self):
-        return list(self.itervalues())
-
-    def itervalues(self):
-        return self._row_iter(lambda row: row[0], "SELECT value FROM %s;" %
-                              self.table)
-
-    def items(self):
-        return list(self.iteritems())
-
-    def iteritems(self):
-        return self._row_iter(lambda row: (row[0], row[1]), "SELECT * FROM %s;" %
-                              self.table)
-
-    @_Decorators.retry()
-    @_Decorators.transaction
-    def clear(self, cursor):
-        cursor.execute("DELETE FROM %s;" % self.table)
-
-    def has_key(self, key):
-        return key in self
-
-
-class PersistData(object):
-    """Deprecated representation of the bitbake persistent data store"""
-    def __init__(self, d):
-        warnings.warn("Use of PersistData is deprecated.  Please use "
-                      "persist(domain, d) instead.",
-                      category=DeprecationWarning,
-                      stacklevel=2)
-
-        self.data = persist(d)
-        logger.debug("Using '%s' as the persistent data cache",
-                     self.data.filename)
-
-    def addDomain(self, domain):
-        """
-        Add a domain (pending deprecation)
-        """
-        return self.data[domain]
-
-    def delDomain(self, domain):
-        """
-        Removes a domain and all the data it contains
-        """
-        del self.data[domain]
-
-    def getKeyValues(self, domain):
-        """
-        Return a list of key + value pairs for a domain
-        """
-        return list(self.data[domain].items())
-
-    def getValue(self, domain, key):
-        """
-        Return the value of a key for a domain
-        """
-        return self.data[domain][key]
-
-    def setValue(self, domain, key, value):
-        """
-        Sets the value of a key for a domain
-        """
-        self.data[domain][key] = value
-
-    def delValue(self, domain, key):
-        """
-        Deletes a key/value pair
-        """
-        del self.data[domain][key]
-
-def persist(domain, d):
-    """Convenience factory for SQLTable objects based upon metadata"""
-    import bb.utils
-    cachedir = (d.getVar("PERSISTENT_DIR") or
-                d.getVar("CACHE"))
-    if not cachedir:
-        logger.critical("Please set the 'PERSISTENT_DIR' or 'CACHE' variable")
-        sys.exit(1)
-
-    bb.utils.mkdirhier(cachedir)
-    cachefile = os.path.join(cachedir, "bb_persist_data.sqlite3")
-    return SQLTable(cachefile, domain)
diff --git a/bitbake/lib/bb/process.py b/bitbake/lib/bb/process.py
index 7c3995cce5..4c7b6d39df 100644
--- a/bitbake/lib/bb/process.py
+++ b/bitbake/lib/bb/process.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -60,7 +62,7 @@ class Popen(subprocess.Popen):
         "close_fds": True,
         "preexec_fn": subprocess_setup,
         "stdout": subprocess.PIPE,
-        "stderr": subprocess.STDOUT,
+        "stderr": subprocess.PIPE,
         "stdin": subprocess.PIPE,
         "shell": False,
     }
@@ -142,7 +144,7 @@ def _logged_communicate(pipe, log, input, extrafiles):
         while pipe.poll() is None:
             read_all_pipes(log, rin, outdata, errdata)
 
-        # Pocess closed, drain all pipes...
+        # Process closed, drain all pipes...
         read_all_pipes(log, rin, outdata, errdata)
     finally:
         log.flush()
@@ -181,5 +183,8 @@ def run(cmd, input=None, log=None, extrafiles=None, **options):
             stderr = stderr.decode("utf-8")
 
     if pipe.returncode != 0:
+        if log:
+            # Don't duplicate the output in the exception if logging it
+            raise ExecutionError(cmd, pipe.returncode, None, None)
         raise ExecutionError(cmd, pipe.returncode, stdout, stderr)
     return stdout, stderr
diff --git a/bitbake/lib/bb/progress.py b/bitbake/lib/bb/progress.py
index d051ba0198..9518be77fb 100644
--- a/bitbake/lib/bb/progress.py
+++ b/bitbake/lib/bb/progress.py
@@ -94,12 +94,15 @@ class LineFilterProgressHandler(ProgressHandler):
         while True:
             breakpos = self._linebuffer.find('\n') + 1
             if breakpos == 0:
-                break
+                # for the case when the line with progress ends with only '\r'
+                breakpos = self._linebuffer.find('\r') + 1
+                if breakpos == 0:
+                    break
             line = self._linebuffer[:breakpos]
             self._linebuffer = self._linebuffer[breakpos:]
             # Drop any line feeds and anything that precedes them
             lbreakpos = line.rfind('\r') + 1
-            if lbreakpos:
+            if lbreakpos and lbreakpos != breakpos:
                 line = line[lbreakpos:]
             if self.writeline(filter_color(line)):
                 super().write(line)
@@ -145,7 +148,7 @@ class MultiStageProgressReporter:
     for tasks made up of python code spread across multiple
     classes / functions - the progress reporter object can
     be passed around or stored at the object level and calls
-    to next_stage() and update() made whereever needed.
+    to next_stage() and update() made wherever needed.
     """
     def __init__(self, d, stage_weights, debug=False):
         """
diff --git a/bitbake/lib/bb/providers.py b/bitbake/lib/bb/providers.py
index b5a6cd0090..e11a4637d1 100644
--- a/bitbake/lib/bb/providers.py
+++ b/bitbake/lib/bb/providers.py
@@ -38,16 +38,17 @@ def findProviders(cfgData, dataCache, pkg_pn = None):
     localdata = data.createCopy(cfgData)
     bb.data.expandKeys(localdata)
 
+    required = {}
     preferred_versions = {}
     latest_versions = {}
 
     for pn in pkg_pn:
-        (last_ver, last_file, pref_ver, pref_file) = findBestProvider(pn, localdata, dataCache, pkg_pn)
+        (last_ver, last_file, pref_ver, pref_file, req) = findBestProvider(pn, localdata, dataCache, pkg_pn)
         preferred_versions[pn] = (pref_ver, pref_file)
         latest_versions[pn] = (last_ver, last_file)
+        required[pn] = req
 
-    return (latest_versions, preferred_versions)
-
+    return (latest_versions, preferred_versions, required)
 
 def allProviders(dataCache):
     """
@@ -59,7 +60,6 @@ def allProviders(dataCache):
         all_providers[pn].append((ver, fn))
     return all_providers
 
-
 def sortPriorities(pn, dataCache, pkg_pn = None):
     """
     Reorder pkg_pn by file priority and default preference
@@ -87,6 +87,21 @@ def sortPriorities(pn, dataCache, pkg_pn = None):
 
     return tmp_pn
 
+def versionVariableMatch(cfgData, keyword, pn):
+    """
+    Return the value of the <keyword>_VERSION variable if set.
+    """
+
+    # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
+    # hence we do this manually rather than use OVERRIDES
+    ver = cfgData.getVar("%s_VERSION:pn-%s" % (keyword, pn))
+    if not ver:
+        ver = cfgData.getVar("%s_VERSION_%s" % (keyword, pn))
+    if not ver:
+        ver = cfgData.getVar("%s_VERSION" % keyword)
+
+    return ver
+
 def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
     """
     Check if the version pe,pv,pr is the preferred one.
@@ -102,19 +117,28 @@ def preferredVersionMatch(pe, pv, pr, preferred_e, preferred_v, preferred_r):
 
 def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     """
-    Find the first provider in pkg_pn with a PREFERRED_VERSION set.
+    Find the first provider in pkg_pn with REQUIRED_VERSION or PREFERRED_VERSION set.
     """
 
     preferred_file = None
     preferred_ver = None
+    required = False
 
-    # pn can contain '_', e.g. gcc-cross-x86_64 and an override cannot
-    # hence we do this manually rather than use OVERRIDES
-    preferred_v = cfgData.getVar("PREFERRED_VERSION_pn-%s" % pn)
-    if not preferred_v:
-        preferred_v = cfgData.getVar("PREFERRED_VERSION_%s" % pn)
-    if not preferred_v:
-        preferred_v = cfgData.getVar("PREFERRED_VERSION")
+    required_v = versionVariableMatch(cfgData, "REQUIRED", pn)
+    preferred_v = versionVariableMatch(cfgData, "PREFERRED", pn)
+
+    itemstr = ""
+    if item:
+        itemstr = " (for item %s)" % item
+
+    if required_v is not None:
+        if preferred_v is not None:
+            logger.warning("REQUIRED_VERSION and PREFERRED_VERSION for package %s%s are both set using REQUIRED_VERSION %s", pn, itemstr, required_v)
+        else:
+            logger.debug("REQUIRED_VERSION is set for package %s%s", pn, itemstr)
+        # REQUIRED_VERSION always takes precedence over PREFERRED_VERSION
+        preferred_v = required_v
+        required = True
 
     if preferred_v:
         m = re.match(r'(\d+:)*(.*)(_.*)*', preferred_v)
@@ -147,11 +171,9 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
             pv_str = preferred_v
         if not (preferred_e is None):
             pv_str = '%s:%s' % (preferred_e, pv_str)
-        itemstr = ""
-        if item:
-            itemstr = " (for item %s)" % item
         if preferred_file is None:
-            logger.warn("preferred version %s of %s not available%s", pv_str, pn, itemstr)
+            if not required:
+                logger.warning("preferred version %s of %s not available%s", pv_str, pn, itemstr)
             available_vers = []
             for file_set in pkg_pn:
                 for f in file_set:
@@ -163,12 +185,16 @@ def findPreferredProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
                         available_vers.append(ver_str)
             if available_vers:
                 available_vers.sort()
-                logger.warn("versions of %s available: %s", pn, ' '.join(available_vers))
+                logger.warning("versions of %s available: %s", pn, ' '.join(available_vers))
+            if required:
+                logger.error("required version %s of %s not available%s", pv_str, pn, itemstr)
         else:
-            logger.debug("selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
-
-    return (preferred_ver, preferred_file)
+            if required:
+                logger.debug("selecting %s as REQUIRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
+            else:
+                logger.debug("selecting %s as PREFERRED_VERSION %s of package %s%s", preferred_file, pv_str, pn, itemstr)
 
+    return (preferred_ver, preferred_file, required)
 
 def findLatestProvider(pn, cfgData, dataCache, file_set):
     """
@@ -189,7 +215,6 @@ def findLatestProvider(pn, cfgData, dataCache, file_set):
 
     return (latest, latest_f)
 
-
 def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     """
     If there is a PREFERRED_VERSION, find the highest-priority bbfile
@@ -198,17 +223,16 @@ def findBestProvider(pn, cfgData, dataCache, pkg_pn = None, item = None):
     """
 
     sortpkg_pn = sortPriorities(pn, dataCache, pkg_pn)
-    # Find the highest priority provider with a PREFERRED_VERSION set
-    (preferred_ver, preferred_file) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
+    # Find the highest priority provider with a REQUIRED_VERSION or PREFERRED_VERSION set
+    (preferred_ver, preferred_file, required) = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn, item)
     # Find the latest version of the highest priority provider
     (latest, latest_f) = findLatestProvider(pn, cfgData, dataCache, sortpkg_pn[0])
 
-    if preferred_file is None:
+    if not required and preferred_file is None:
         preferred_file = latest_f
         preferred_ver = latest
 
-    return (latest, latest_f, preferred_ver, preferred_file)
-
+    return (latest, latest_f, preferred_ver, preferred_file, required)
 
 def _filterProviders(providers, item, cfgData, dataCache):
     """
@@ -234,10 +258,13 @@ def _filterProviders(providers, item, cfgData, dataCache):
 
     logger.debug("providers for %s are: %s", item, list(sorted(pkg_pn.keys())))
 
-    # First add PREFERRED_VERSIONS
+    # First add REQUIRED_VERSIONS or PREFERRED_VERSIONS
     for pn in sorted(pkg_pn):
         sortpkg_pn[pn] = sortPriorities(pn, dataCache, pkg_pn)
-        preferred_versions[pn] = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
+        preferred_ver, preferred_file, required = findPreferredProvider(pn, cfgData, dataCache, sortpkg_pn[pn], item)
+        if required and preferred_file is None:
+            return eligible
+        preferred_versions[pn] = (preferred_ver, preferred_file)
         if preferred_versions[pn][1]:
             eligible.append(preferred_versions[pn][1])
 
@@ -249,7 +276,6 @@ def _filterProviders(providers, item, cfgData, dataCache):
         eligible.append(preferred_versions[pn][1])
 
     if not eligible:
-        logger.error("no eligible providers for %s", item)
         return eligible
 
     # If pn == item, give it a slight default preference
@@ -266,7 +292,6 @@ def _filterProviders(providers, item, cfgData, dataCache):
 
     return eligible
 
-
 def filterProviders(providers, item, cfgData, dataCache):
     """
     Take a list of providers and filter/reorder according to the
@@ -371,8 +396,8 @@ def getRuntimeProviders(dataCache, rdepend):
         return rproviders
 
     # Only search dynamic packages if we can't find anything in other variables
-    for pattern in dataCache.packages_dynamic:
-        pattern = pattern.replace(r'+', r"\+")
+    for pat_key in dataCache.packages_dynamic:
+        pattern = pat_key.replace(r'+', r"\+")
         if pattern in regexp_cache:
             regexp = regexp_cache[pattern]
         else:
@@ -383,12 +408,11 @@ def getRuntimeProviders(dataCache, rdepend):
                 raise
             regexp_cache[pattern] = regexp
         if regexp.match(rdepend):
-            rproviders += dataCache.packages_dynamic[pattern]
+            rproviders += dataCache.packages_dynamic[pat_key]
             logger.debug("Assuming %s is a dynamic package, but it may not exist" % rdepend)
 
     return rproviders
 
-
 def buildWorldTargetList(dataCache, task=None):
     """
     Build package list for "bitbake world"
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 54ef245a63..80f3d3282f 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -14,6 +14,7 @@ import os
 import sys
 import stat
 import errno
+import itertools
 import logging
 import re
 import bb
@@ -24,6 +25,7 @@ import pickle
 from multiprocessing import Process
 import shlex
 import pprint
+import time
 
 bblogger = logging.getLogger("BitBake")
 logger = logging.getLogger("BitBake.RunQueue")
@@ -85,15 +87,19 @@ class RunQueueStats:
     """
     Holds statistics on the tasks handled by the associated runQueue
     """
-    def __init__(self, total):
+    def __init__(self, total, setscene_total):
         self.completed = 0
         self.skipped = 0
         self.failed = 0
         self.active = 0
+        self.setscene_active = 0
+        self.setscene_covered = 0
+        self.setscene_notcovered = 0
+        self.setscene_total = setscene_total
         self.total = total
 
     def copy(self):
-        obj = self.__class__(self.total)
+        obj = self.__class__(self.total, self.setscene_total)
         obj.__dict__.update(self.__dict__)
         return obj
 
@@ -112,10 +118,18 @@ class RunQueueStats:
     def taskActive(self):
         self.active = self.active + 1
 
+    def updateCovered(self, covered, notcovered):
+        self.setscene_covered = covered
+        self.setscene_notcovered = notcovered
+
+    def updateActiveSetscene(self, active):
+        self.setscene_active = active
+
 # These values indicate the next step due to be run in the
 # runQueue state machine
 runQueuePrepare = 2
 runQueueSceneInit = 3
+runQueueDumpSigs = 4
 runQueueRunning = 6
 runQueueFailed = 7
 runQueueCleanUp = 8
@@ -143,11 +157,82 @@ class RunQueueScheduler(object):
         self.stamps = {}
         for tid in self.rqdata.runtaskentries:
             (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
-            self.stamps[tid] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
+            self.stamps[tid] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
             if tid in self.rq.runq_buildable:
-                self.buildable.append(tid)
+                self.buildable.add(tid)
 
         self.rev_prio_map = None
+        self.is_pressure_usable()
+
+    def is_pressure_usable(self):
+        """
+        If monitoring pressure, return True if pressure files can be open and read. For example
+        openSUSE /proc/pressure/* files have readable file permissions but when read the error EOPNOTSUPP (Operation not supported)
+        is returned.
+        """
+        if self.rq.max_cpu_pressure or self.rq.max_io_pressure or self.rq.max_memory_pressure:
+            try:
+                with open("/proc/pressure/cpu") as cpu_pressure_fds, \
+                    open("/proc/pressure/io") as io_pressure_fds, \
+                    open("/proc/pressure/memory") as memory_pressure_fds:
+
+                    self.prev_cpu_pressure = cpu_pressure_fds.readline().split()[4].split("=")[1]
+                    self.prev_io_pressure = io_pressure_fds.readline().split()[4].split("=")[1]
+                    self.prev_memory_pressure = memory_pressure_fds.readline().split()[4].split("=")[1]
+                    self.prev_pressure_time = time.time()
+                self.check_pressure = True
+            except:
+                bb.note("The /proc/pressure files can't be read. Continuing build without monitoring pressure")
+                self.check_pressure = False
+        else:
+            self.check_pressure = False
+
+    def exceeds_max_pressure(self):
+        """
+        Monitor the difference in total pressure at least once per second, if
+        BB_PRESSURE_MAX_{CPU|IO|MEMORY} are set, return True if above threshold.
+        """
+        if self.check_pressure:
+            with open("/proc/pressure/cpu") as cpu_pressure_fds, \
+                open("/proc/pressure/io") as io_pressure_fds, \
+                open("/proc/pressure/memory") as memory_pressure_fds:
+                # extract "total" from /proc/pressure/{cpu|io}
+                curr_cpu_pressure = cpu_pressure_fds.readline().split()[4].split("=")[1]
+                curr_io_pressure = io_pressure_fds.readline().split()[4].split("=")[1]
+                curr_memory_pressure = memory_pressure_fds.readline().split()[4].split("=")[1]
+                now = time.time()
+                tdiff = now - self.prev_pressure_time
+                psi_accumulation_interval = 1.0
+                cpu_pressure = (float(curr_cpu_pressure) - float(self.prev_cpu_pressure)) / tdiff
+                io_pressure = (float(curr_io_pressure) - float(self.prev_io_pressure)) / tdiff
+                memory_pressure = (float(curr_memory_pressure) - float(self.prev_memory_pressure)) / tdiff
+                exceeds_cpu_pressure =  self.rq.max_cpu_pressure and cpu_pressure > self.rq.max_cpu_pressure
+                exceeds_io_pressure =  self.rq.max_io_pressure and io_pressure > self.rq.max_io_pressure
+                exceeds_memory_pressure =  self.rq.max_memory_pressure and memory_pressure > self.rq.max_memory_pressure
+
+                if tdiff > psi_accumulation_interval:
+                    self.prev_cpu_pressure = curr_cpu_pressure
+                    self.prev_io_pressure = curr_io_pressure
+                    self.prev_memory_pressure = curr_memory_pressure
+                    self.prev_pressure_time = now
+
+            pressure_state = (exceeds_cpu_pressure, exceeds_io_pressure, exceeds_memory_pressure)
+            pressure_values = (round(cpu_pressure,1), self.rq.max_cpu_pressure, round(io_pressure,1), self.rq.max_io_pressure, round(memory_pressure,1), self.rq.max_memory_pressure)
+            if hasattr(self, "pressure_state") and pressure_state != self.pressure_state:
+                bb.note("Pressure status changed to CPU: %s, IO: %s, Mem: %s (CPU: %s/%s, IO: %s/%s, Mem: %s/%s) - using %s/%s bitbake threads" % (pressure_state + pressure_values + (len(self.rq.runq_running.difference(self.rq.runq_complete)), self.rq.number_tasks)))
+            self.pressure_state = pressure_state
+            return (exceeds_cpu_pressure or exceeds_io_pressure or exceeds_memory_pressure)
+        elif self.rq.max_loadfactor:
+            limit = False
+            loadfactor = float(os.getloadavg()[0]) / os.cpu_count()
+            # bb.warn("Comparing %s to %s" % (loadfactor, self.rq.max_loadfactor))
+            if loadfactor > self.rq.max_loadfactor:
+                limit = True
+            if hasattr(self, "loadfactor_limit") and limit != self.loadfactor_limit:
+                bb.note("Load average limiting set to %s as load average: %s - using %s/%s bitbake threads" % (limit, loadfactor, len(self.rq.runq_running.difference(self.rq.runq_complete)), self.rq.number_tasks))
+            self.loadfactor_limit = limit
+            return limit
+        return False
 
     def next_buildable_task(self):
         """
@@ -161,6 +246,12 @@ class RunQueueScheduler(object):
         if not buildable:
             return None
 
+        # Bitbake requires that at least one task be active. Only check for pressure if
+        # this is the case, otherwise the pressure limitation could result in no tasks
+        # being active and no new tasks started thereby, at times, breaking the scheduler.
+        if self.rq.stats.active and self.exceeds_max_pressure():
+            return None
+
         # Filter out tasks that have a max number of threads that have been exceeded
         skip_buildable = {}
         for running in self.rq.runq_running.difference(self.rq.runq_complete):
@@ -191,11 +282,11 @@ class RunQueueScheduler(object):
         best = None
         bestprio = None
         for tid in buildable:
-            taskname = taskname_from_tid(tid)
-            if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
-                continue
             prio = self.rev_prio_map[tid]
             if bestprio is None or bestprio > prio:
+                taskname = taskname_from_tid(tid)
+                if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
+                    continue
                 stamp = self.stamps[tid]
                 if stamp in self.rq.build_stamps.values():
                     continue
@@ -374,10 +465,9 @@ class RunQueueData:
         self.rq = rq
         self.warn_multi_bb = False
 
-        self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST") or ""
-        self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST") or "").split()
-        self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData, targets)
-        self.setscenewhitelist_checked = False
+        self.multi_provider_allowed = (cfgData.getVar("BB_MULTI_PROVIDER_ALLOWED") or "").split()
+        self.setscene_ignore_tasks = get_setscene_enforce_ignore_tasks(cfgData, targets)
+        self.setscene_ignore_tasks_checked = False
         self.setscene_enforce = (cfgData.getVar('BB_SETSCENE_ENFORCE') == "1")
         self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter()
 
@@ -387,7 +477,6 @@ class RunQueueData:
         self.runtaskentries = {}
 
     def runq_depends_names(self, ids):
-        import re
         ret = []
         for id in ids:
             nam = os.path.basename(id)
@@ -475,7 +564,7 @@ class RunQueueData:
                             msgs.append("  Task %s (dependent Tasks %s)\n" % (dep, self.runq_depends_names(self.runtaskentries[dep].depends)))
                         msgs.append("\n")
                     if len(valid_chains) > 10:
-                        msgs.append("Aborted dependency loops search after 10 matches.\n")
+                        msgs.append("Halted dependency loops search after 10 matches.\n")
                         raise TooManyLoops
                     continue
                 scan = False
@@ -536,7 +625,7 @@ class RunQueueData:
                         next_points.append(revdep)
                         task_done[revdep] = True
             endpoints = next_points
-            if len(next_points) == 0:
+            if not next_points:
                 break
 
         # Circular dependency sanity check
@@ -578,15 +667,18 @@ class RunQueueData:
 
         found = False
         for mc in self.taskData:
-            if len(taskData[mc].taskentries) > 0:
+            if taskData[mc].taskentries:
                 found = True
                 break
         if not found:
             # Nothing to do
             return 0
 
+        bb.parse.siggen.setup_datacache(self.dataCaches)
+
         self.init_progress_reporter.start()
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Step A - Work out a list of tasks to run
         #
@@ -632,9 +724,13 @@ class RunQueueData:
                 frommc = mcdependency[1]
                 mcdep = mcdependency[2]
                 deptask = mcdependency[4]
+                if mcdep not in taskData:
+                    bb.fatal("Multiconfig '%s' is referenced in multiconfig dependency '%s' but not enabled in BBMULTICONFIG?" % (mcdep, dep))
                 if mc == frommc:
                     fn = taskData[mcdep].build_targets[pn][0]
                     newdep = '%s:%s' % (fn,deptask)
+                    if newdep not in taskData[mcdep].taskentries:
+                        bb.fatal("Task mcdepends on non-existent task %s" % (newdep))
                     taskData[mc].taskentries[tid].tdepends.append(newdep)
 
         for mc in taskData:
@@ -733,6 +829,7 @@ class RunQueueData:
         #self.dump_data()
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Resolve recursive 'recrdeptask' dependencies (Part B)
         #
@@ -762,7 +859,7 @@ class RunQueueData:
         # Find the dependency chain endpoints
         endpoints = set()
         for tid in self.runtaskentries:
-            if len(deps[tid]) == 0:
+            if not deps[tid]:
                 endpoints.add(tid)
         # Iterate the chains collating dependencies
         while endpoints:
@@ -773,11 +870,11 @@ class RunQueueData:
                     cumulativedeps[dep].update(cumulativedeps[tid])
                     if tid in deps[dep]:
                         deps[dep].remove(tid)
-                    if len(deps[dep]) == 0:
+                    if not deps[dep]:
                         next.add(dep)
             endpoints = next
         #for tid in deps:
-        #    if len(deps[tid]) != 0:
+        #    if deps[tid]:
         #        bb.warn("Sanity test failure, dependencies left for %s (%s)" % (tid, deps[tid]))
 
         # Loop here since recrdeptasks can depend upon other recrdeptasks and we have to
@@ -829,6 +926,7 @@ class RunQueueData:
             self.runtaskentries[tid].depends.difference_update(recursivetasksselfref)
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         #self.dump_data()
 
@@ -867,7 +965,7 @@ class RunQueueData:
                     bb.debug(1, "Task %s is marked nostamp, cannot invalidate this task" % taskname)
             else:
                 logger.verbose("Invalidate task %s, %s", taskname, fn)
-                bb.parse.siggen.invalidate_task(taskname, self.dataCaches[mc], taskfn)
+                bb.parse.siggen.invalidate_task(taskname, taskfn)
 
         self.target_tids = []
         for (mc, target, task, fn) in self.targets:
@@ -910,47 +1008,54 @@ class RunQueueData:
                 mark_active(tid, 1)
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Step C - Prune all inactive tasks
         #
         # Once all active tasks are marked, prune the ones we don't need.
 
-        delcount = {}
-        for tid in list(self.runtaskentries.keys()):
-            if tid not in runq_build:
-                delcount[tid] = self.runtaskentries[tid]
-                del self.runtaskentries[tid]
-
         # Handle --runall
         if self.cooker.configuration.runall:
             # re-run the mark_active and then drop unused tasks from new list
-            runq_build = {}
 
-            for task in self.cooker.configuration.runall:
-                if not task.startswith("do_"):
-                    task = "do_{0}".format(task)
-                runall_tids = set()
-                for tid in list(self.runtaskentries):
-                    wanttid = "{0}:{1}".format(fn_from_tid(tid), task)
-                    if wanttid in delcount:
-                        self.runtaskentries[wanttid] = delcount[wanttid]
-                    if wanttid in self.runtaskentries:
-                        runall_tids.add(wanttid)
-
-                for tid in list(runall_tids):
-                    mark_active(tid,1)
-                    if self.cooker.configuration.force:
-                        invalidate_task(tid, False)
+            runall_tids = set()
+            added = True
+            while added:
+                reduced_tasklist = set(self.runtaskentries.keys())
+                for tid in list(self.runtaskentries.keys()):
+                    if tid not in runq_build:
+                       reduced_tasklist.remove(tid)
+                runq_build = {}
 
-            for tid in list(self.runtaskentries.keys()):
-                if tid not in runq_build:
-                    delcount[tid] = self.runtaskentries[tid]
-                    del self.runtaskentries[tid]
+                orig = runall_tids
+                runall_tids = set()
+                for task in self.cooker.configuration.runall:
+                    if not task.startswith("do_"):
+                        task = "do_{0}".format(task)
+                    for tid in reduced_tasklist:
+                        wanttid = "{0}:{1}".format(fn_from_tid(tid), task)
+                        if wanttid in self.runtaskentries:
+                            runall_tids.add(wanttid)
+
+                    for tid in list(runall_tids):
+                        mark_active(tid, 1)
+                        self.target_tids.append(tid)
+                        if self.cooker.configuration.force:
+                            invalidate_task(tid, False)
+                added = runall_tids - orig
+
+        delcount = set()
+        for tid in list(self.runtaskentries.keys()):
+            if tid not in runq_build:
+                delcount.add(tid)
+                del self.runtaskentries[tid]
 
-            if len(self.runtaskentries) == 0:
+        if self.cooker.configuration.runall:
+            if not self.runtaskentries:
                 bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the recipes of the taskgraphs of the targets %s" % (str(self.cooker.configuration.runall), str(self.targets)))
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Handle runonly
         if self.cooker.configuration.runonly:
@@ -960,19 +1065,19 @@ class RunQueueData:
             for task in self.cooker.configuration.runonly:
                 if not task.startswith("do_"):
                     task = "do_{0}".format(task)
-                runonly_tids = { k: v for k, v in self.runtaskentries.items() if taskname_from_tid(k) == task }
+                runonly_tids = [k for k in self.runtaskentries.keys() if taskname_from_tid(k) == task]
 
-                for tid in list(runonly_tids):
-                    mark_active(tid,1)
+                for tid in runonly_tids:
+                    mark_active(tid, 1)
                     if self.cooker.configuration.force:
                         invalidate_task(tid, False)
 
             for tid in list(self.runtaskentries.keys()):
                 if tid not in runq_build:
-                    delcount[tid] = self.runtaskentries[tid]
+                    delcount.add(tid)
                     del self.runtaskentries[tid]
 
-            if len(self.runtaskentries) == 0:
+            if not self.runtaskentries:
                 bb.msg.fatal("RunQueue", "Could not find any tasks with the tasknames %s to run within the taskgraphs of the targets %s" % (str(self.cooker.configuration.runonly), str(self.targets)))
 
         #
@@ -980,8 +1085,8 @@ class RunQueueData:
         #
 
         # Check to make sure we still have tasks to run
-        if len(self.runtaskentries) == 0:
-            if not taskData[''].abort:
+        if not self.runtaskentries:
+            if not taskData[''].halt:
                 bb.msg.fatal("RunQueue", "All buildable tasks have been run but the build is incomplete (--continue mode). Errors for the tasks that failed will have been printed above.")
             else:
                 bb.msg.fatal("RunQueue", "No active tasks and not in --continue mode?! Please report this bug.")
@@ -991,6 +1096,7 @@ class RunQueueData:
         logger.verbose("Assign Weightings")
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Generate a list of reverse dependencies to ease future calculations
         for tid in self.runtaskentries:
@@ -998,13 +1104,14 @@ class RunQueueData:
                 self.runtaskentries[dep].revdeps.add(tid)
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Identify tasks at the end of dependency chains
         # Error on circular dependency loops (length two)
         endpoints = []
         for tid in self.runtaskentries:
             revdeps = self.runtaskentries[tid].revdeps
-            if len(revdeps) == 0:
+            if not revdeps:
                 endpoints.append(tid)
             for dep in revdeps:
                 if dep in self.runtaskentries[tid].depends:
@@ -1014,12 +1121,14 @@ class RunQueueData:
         logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints))
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Calculate task weights
         # Check of higher length circular dependencies
         self.runq_weight = self.calculate_task_weights(endpoints)
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Sanity Check - Check for multiple tasks building the same provider
         for mc in self.dataCaches:
@@ -1040,7 +1149,7 @@ class RunQueueData:
             for prov in prov_list:
                 if len(prov_list[prov]) < 2:
                     continue
-                if prov in self.multi_provider_whitelist:
+                if prov in self.multi_provider_allowed:
                     continue
                 seen_pn = []
                 # If two versions of the same PN are being built its fatal, we don't support it.
@@ -1050,12 +1159,12 @@ class RunQueueData:
                         seen_pn.append(pn)
                     else:
                         bb.fatal("Multiple versions of %s are due to be built (%s). Only one version of a given PN should be built in any given build. You likely need to set PREFERRED_VERSION_%s to select the correct version or don't depend on multiple versions." % (pn, " ".join(prov_list[prov]), pn))
-                msg = "Multiple .bb files are due to be built which each provide %s:\n  %s" % (prov, "\n  ".join(prov_list[prov]))
+                msgs = ["Multiple .bb files are due to be built which each provide %s:\n  %s" % (prov, "\n  ".join(prov_list[prov]))]
                 #
                 # Construct a list of things which uniquely depend on each provider
                 # since this may help the user figure out which dependency is triggering this warning
                 #
-                msg += "\nA list of tasks depending on these providers is shown and may help explain where the dependency comes from."
+                msgs.append("\nA list of tasks depending on these providers is shown and may help explain where the dependency comes from.")
                 deplist = {}
                 commondeps = None
                 for provfn in prov_list[prov]:
@@ -1075,12 +1184,12 @@ class RunQueueData:
                         commondeps &= deps
                     deplist[provfn] = deps
                 for provfn in deplist:
-                    msg += "\n%s has unique dependees:\n  %s" % (provfn, "\n  ".join(deplist[provfn] - commondeps))
+                    msgs.append("\n%s has unique dependees:\n  %s" % (provfn, "\n  ".join(deplist[provfn] - commondeps)))
                 #
                 # Construct a list of provides and runtime providers for each recipe
                 # (rprovides has to cover RPROVIDES, PACKAGES, PACKAGES_DYNAMIC)
                 #
-                msg += "\nIt could be that one recipe provides something the other doesn't and should. The following provider and runtime provider differences may be helpful."
+                msgs.append("\nIt could be that one recipe provides something the other doesn't and should. The following provider and runtime provider differences may be helpful.")
                 provide_results = {}
                 rprovide_results = {}
                 commonprovs = None
@@ -1107,30 +1216,20 @@ class RunQueueData:
                     else:
                         commonrprovs &= rprovides
                     rprovide_results[provfn] = rprovides
-                #msg += "\nCommon provides:\n  %s" % ("\n  ".join(commonprovs))
-                #msg += "\nCommon rprovides:\n  %s" % ("\n  ".join(commonrprovs))
+                #msgs.append("\nCommon provides:\n  %s" % ("\n  ".join(commonprovs)))
+                #msgs.append("\nCommon rprovides:\n  %s" % ("\n  ".join(commonrprovs)))
                 for provfn in prov_list[prov]:
-                    msg += "\n%s has unique provides:\n  %s" % (provfn, "\n  ".join(provide_results[provfn] - commonprovs))
-                    msg += "\n%s has unique rprovides:\n  %s" % (provfn, "\n  ".join(rprovide_results[provfn] - commonrprovs))
+                    msgs.append("\n%s has unique provides:\n  %s" % (provfn, "\n  ".join(provide_results[provfn] - commonprovs)))
+                    msgs.append("\n%s has unique rprovides:\n  %s" % (provfn, "\n  ".join(rprovide_results[provfn] - commonrprovs)))
 
                 if self.warn_multi_bb:
-                    logger.verbnote(msg)
+                    logger.verbnote("".join(msgs))
                 else:
-                    logger.error(msg)
+                    logger.error("".join(msgs))
 
         self.init_progress_reporter.next_stage()
-
-        # Create a whitelist usable by the stamp checks
-        self.stampfnwhitelist = {}
-        for mc in self.taskData:
-            self.stampfnwhitelist[mc] = []
-            for entry in self.stampwhitelist.split():
-                if entry not in self.taskData[mc].build_targets:
-                    continue
-                fn = self.taskData.build_targets[entry][0]
-                self.stampfnwhitelist[mc].append(fn)
-
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Iterate over the task list looking for tasks with a 'setscene' function
         self.runq_setscene_tids = set()
@@ -1143,6 +1242,7 @@ class RunQueueData:
                 self.runq_setscene_tids.add(tid)
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Invalidate task if force mode active
         if self.cooker.configuration.force:
@@ -1159,6 +1259,7 @@ class RunQueueData:
                     invalidate_task(fn + ":" + st, True)
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         # Create and print to the logs a virtual/xxxx -> PN (fn) table
         for mc in taskData:
@@ -1171,30 +1272,45 @@ class RunQueueData:
                 bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCaches[mc])
 
         self.init_progress_reporter.next_stage()
+        bb.event.check_for_interrupts(self.cooker.data)
 
         bb.parse.siggen.set_setscene_tasks(self.runq_setscene_tids)
 
+        starttime = time.time()
+        lasttime = starttime
+
         # Iterate over the task list and call into the siggen code
         dealtwith = set()
         todeal = set(self.runtaskentries)
-        while len(todeal) > 0:
+        while todeal:
+            ready = set()
             for tid in todeal.copy():
-                if len(self.runtaskentries[tid].depends - dealtwith) == 0:
-                    dealtwith.add(tid)
-                    todeal.remove(tid)
-                    self.prepare_task_hash(tid)
+                if not (self.runtaskentries[tid].depends - dealtwith):
+                    self.runtaskentries[tid].taskhash_deps = bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
+                    # get_taskhash for a given tid *must* be called before get_unihash* below
+                    self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, self.dataCaches)
+                    ready.add(tid)
+            unihashes = bb.parse.siggen.get_unihashes(ready)
+            for tid in ready:
+                dealtwith.add(tid)
+                todeal.remove(tid)
+                self.runtaskentries[tid].unihash = unihashes[tid]
+
+            bb.event.check_for_interrupts(self.cooker.data)
+
+            if time.time() > (lasttime + 30):
+                lasttime = time.time()
+                hashequiv_logger.verbose("Initial setup loop progress: %s of %s in %s" % (len(todeal), len(self.runtaskentries), lasttime - starttime))
+
+        endtime = time.time()
+        if (endtime-starttime > 60):
+            hashequiv_logger.verbose("Initial setup loop took: %s" % (endtime-starttime))
 
         bb.parse.siggen.writeout_file_checksum_cache()
 
         #self.dump_data()
         return len(self.runtaskentries)
 
-    def prepare_task_hash(self, tid):
-        dc = bb.parse.siggen.get_data_caches(self.dataCaches, mc_from_tid(tid))
-        bb.parse.siggen.prep_taskhash(tid, self.runtaskentries[tid].depends, dc)
-        self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(tid, self.runtaskentries[tid].depends, dc)
-        self.runtaskentries[tid].unihash = bb.parse.siggen.get_unihash(tid)
-
     def dump_data(self):
         """
         Dump some debug information on the internal data structures
@@ -1218,7 +1334,6 @@ class RunQueue:
         self.cfgData = cfgData
         self.rqdata = RunQueueData(self, cooker, cfgData, dataCaches, taskData, targets)
 
-        self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY") or "perfile"
         self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION") or None
         self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID") or None
 
@@ -1237,30 +1352,40 @@ class RunQueue:
         self.worker = {}
         self.fakeworker = {}
 
+    @staticmethod
+    def send_pickled_data(worker, data, name):
+        msg = bytearray()
+        msg.extend(b"<" + name.encode() + b">")
+        pickled_data = pickle.dumps(data)
+        msg.extend(len(pickled_data).to_bytes(4, 'big'))
+        msg.extend(pickled_data)
+        msg.extend(b"</" + name.encode() + b">")
+        worker.stdin.write(msg)
+
     def _start_worker(self, mc, fakeroot = False, rqexec = None):
         logger.debug("Starting bitbake-worker")
         magic = "decafbad"
         if self.cooker.configuration.profile:
             magic = "decafbadbad"
+        fakerootlogs = None
+
+        workerscript = os.path.realpath(os.path.dirname(__file__) + "/../../bin/bitbake-worker")
         if fakeroot:
             magic = magic + "beef"
             mcdata = self.cooker.databuilder.mcdata[mc]
             fakerootcmd = shlex.split(mcdata.getVar("FAKEROOTCMD"))
             fakerootenv = (mcdata.getVar("FAKEROOTBASEENV") or "").split()
             env = os.environ.copy()
-            for key, value in (var.split('=') for var in fakerootenv):
+            for key, value in (var.split('=',1) for var in fakerootenv):
                 env[key] = value
-            worker = subprocess.Popen(fakerootcmd + ["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
+            worker = subprocess.Popen(fakerootcmd + [sys.executable, workerscript, magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE, env=env)
+            fakerootlogs = self.rqdata.dataCaches[mc].fakerootlogs
         else:
-            worker = subprocess.Popen(["bitbake-worker", magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+            worker = subprocess.Popen([sys.executable, workerscript, magic], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
         bb.utils.nonblockingfd(worker.stdout)
-        workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec)
+        workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec, fakerootlogs=fakerootlogs)
 
         workerdata = {
-            "taskdeps" : self.rqdata.dataCaches[mc].task_deps,
-            "fakerootenv" : self.rqdata.dataCaches[mc].fakerootenv,
-            "fakerootdirs" : self.rqdata.dataCaches[mc].fakerootdirs,
-            "fakerootnoenv" : self.rqdata.dataCaches[mc].fakerootnoenv,
             "sigdata" : bb.parse.siggen.get_taskdata(),
             "logdefaultlevel" : bb.msg.loggerDefaultLogLevel,
             "build_verbose_shell" : self.cooker.configuration.build_verbose_shell,
@@ -1274,9 +1399,9 @@ class RunQueue:
             "umask" : self.cfgData.getVar("BB_DEFAULT_UMASK"),
         }
 
-        worker.stdin.write(b"<cookerconfig>" + pickle.dumps(self.cooker.configuration) + b"</cookerconfig>")
-        worker.stdin.write(b"<extraconfigdata>" + pickle.dumps(self.cooker.extraconfigdata) + b"</extraconfigdata>")
-        worker.stdin.write(b"<workerdata>" + pickle.dumps(workerdata) + b"</workerdata>")
+        RunQueue.send_pickled_data(worker, self.cooker.configuration, "cookerconfig")
+        RunQueue.send_pickled_data(worker, self.cooker.extraconfigdata, "extraconfigdata")
+        RunQueue.send_pickled_data(worker, workerdata, "workerdata")
         worker.stdin.flush()
 
         return RunQueueWorker(worker, workerpipe)
@@ -1286,7 +1411,7 @@ class RunQueue:
             return
         logger.debug("Teardown for bitbake-worker")
         try:
-           worker.process.stdin.write(b"<quit></quit>")
+           RunQueue.send_pickled_data(worker.process, b"", "quit")
            worker.process.stdin.flush()
            worker.process.stdin.close()
         except IOError:
@@ -1298,12 +1423,12 @@ class RunQueue:
             continue
         worker.pipe.close()
 
-    def start_worker(self):
+    def start_worker(self, rqexec):
         if self.worker:
             self.teardown_workers()
         self.teardown = False
         for mc in self.rqdata.dataCaches:
-            self.worker[mc] = self._start_worker(mc)
+            self.worker[mc] = self._start_worker(mc, False, rqexec)
 
     def start_fakeworker(self, rqexec, mc):
         if not mc in self.fakeworker:
@@ -1345,15 +1470,7 @@ class RunQueue:
         if taskname is None:
             taskname = tn
 
-        if self.stamppolicy == "perfile":
-            fulldeptree = False
-        else:
-            fulldeptree = True
-            stampwhitelist = []
-            if self.stamppolicy == "whitelist":
-                stampwhitelist = self.rqdata.stampfnwhitelist[mc]
-
-        stampfile = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn)
+        stampfile = bb.parse.siggen.stampfile_mcfn(taskname, taskfn)
 
         # If the stamp is missing, it's not current
         if not os.access(stampfile, os.F_OK):
@@ -1365,7 +1482,7 @@ class RunQueue:
             logger.debug2("%s.%s is nostamp\n", fn, taskname)
             return False
 
-        if taskname != "do_setscene" and taskname.endswith("_setscene"):
+        if taskname.endswith("_setscene"):
             return True
 
         if cache is None:
@@ -1376,15 +1493,15 @@ class RunQueue:
         for dep in self.rqdata.runtaskentries[tid].depends:
             if iscurrent:
                 (mc2, fn2, taskname2, taskfn2) = split_tid_mcfn(dep)
-                stampfile2 = bb.build.stampfile(taskname2, self.rqdata.dataCaches[mc2], taskfn2)
-                stampfile3 = bb.build.stampfile(taskname2 + "_setscene", self.rqdata.dataCaches[mc2], taskfn2)
+                stampfile2 = bb.parse.siggen.stampfile_mcfn(taskname2, taskfn2)
+                stampfile3 = bb.parse.siggen.stampfile_mcfn(taskname2 + "_setscene", taskfn2)
                 t2 = get_timestamp(stampfile2)
                 t3 = get_timestamp(stampfile3)
                 if t3 and not t2:
                     continue
                 if t3 and t3 > t2:
                     continue
-                if fn == fn2 or (fulldeptree and fn2 not in stampwhitelist):
+                if fn == fn2:
                     if not t2:
                         logger.debug2('Stampfile %s does not exist', stampfile2)
                         iscurrent = False
@@ -1434,10 +1551,11 @@ class RunQueue:
         """
         Run the tasks in a queue prepared by rqdata.prepare()
         Upon failure, optionally try to recover the build using any alternate providers
-        (if the abort on failure configuration option isn't set)
+        (if the halt on failure configuration option isn't set)
         """
 
         retval = True
+        bb.event.check_for_interrupts(self.cooker.data)
 
         if self.state is runQueuePrepare:
             # NOTE: if you add, remove or significantly refactor the stages of this
@@ -1466,31 +1584,37 @@ class RunQueue:
 
             if not self.dm_event_handler_registered:
                  res = bb.event.register(self.dm_event_handler_name,
-                                         lambda x: self.dm.check(self) if self.state in [runQueueRunning, runQueueCleanUp] else False,
+                                         lambda x, y: self.dm.check(self) if self.state in [runQueueRunning, runQueueCleanUp] else False,
                                          ('bb.event.HeartbeatEvent',), data=self.cfgData)
                  self.dm_event_handler_registered = True
 
-            dump = self.cooker.configuration.dump_signatures
-            if dump:
+            self.rqdata.init_progress_reporter.next_stage()
+            self.rqexe = RunQueueExecute(self)
+
+            dumpsigs = self.cooker.configuration.dump_signatures
+            if dumpsigs:
                 self.rqdata.init_progress_reporter.finish()
-                if 'printdiff' in dump:
-                    invalidtasks = self.print_diffscenetasks()
-                self.dump_signatures(dump)
-                if 'printdiff' in dump:
-                    self.write_diffscenetasks(invalidtasks)
+                if 'printdiff' in dumpsigs:
+                    self.invalidtasks_dump = self.print_diffscenetasks()
+                self.state = runQueueDumpSigs
+
+        if self.state is runQueueDumpSigs:
+            dumpsigs = self.cooker.configuration.dump_signatures
+            retval = self.dump_signatures(dumpsigs)
+            if retval is False:
+                if 'printdiff' in dumpsigs:
+                    self.write_diffscenetasks(self.invalidtasks_dump)
                 self.state = runQueueComplete
 
         if self.state is runQueueSceneInit:
-            self.rqdata.init_progress_reporter.next_stage()
-            self.start_worker()
-            self.rqdata.init_progress_reporter.next_stage()
-            self.rqexe = RunQueueExecute(self)
+            self.start_worker(self.rqexe)
+            self.rqdata.init_progress_reporter.finish()
 
             # If we don't have any setscene functions, skip execution
-            if len(self.rqdata.runq_setscene_tids) == 0:
+            if not self.rqdata.runq_setscene_tids:
                 logger.info('No setscene tasks')
                 for tid in self.rqdata.runtaskentries:
-                    if len(self.rqdata.runtaskentries[tid].depends) == 0:
+                    if not self.rqdata.runtaskentries[tid].depends:
                         self.rqexe.setbuildable(tid)
                     self.rqexe.tasks_notcovered.add(tid)
                 self.rqexe.sqdone = True
@@ -1563,43 +1687,62 @@ class RunQueue:
         else:
             self.rqexe.finish()
 
-    def rq_dump_sigfn(self, fn, options):
-        bb_cache = bb.cache.NoCache(self.cooker.databuilder)
-        mc = bb.runqueue.mc_from_tid(fn)
-        the_data = bb_cache.loadDataFull(fn, self.cooker.collections[mc].get_file_appends(fn))
-        siggen = bb.parse.siggen
-        dataCaches = self.rqdata.dataCaches
-        siggen.dump_sigfn(fn, dataCaches, options)
+    def _rq_dump_sigtid(self, tids):
+        for tid in tids:
+            (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+            dataCaches = self.rqdata.dataCaches
+            bb.parse.siggen.dump_sigtask(taskfn, taskname, dataCaches[mc].stamp[taskfn], True)
 
     def dump_signatures(self, options):
-        fns = set()
-        bb.note("Reparsing files to collect dependency data")
+        if not hasattr(self, "dumpsigs_launched"):
+            if bb.cooker.CookerFeatures.RECIPE_SIGGEN_INFO not in self.cooker.featureset:
+                bb.fatal("The dump signatures functionality needs the RECIPE_SIGGEN_INFO feature enabled")
 
-        for tid in self.rqdata.runtaskentries:
-            fn = fn_from_tid(tid)
-            fns.add(fn)
-
-        max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1)
-        # We cannot use the real multiprocessing.Pool easily due to some local data
-        # that can't be pickled. This is a cheap multi-process solution.
-        launched = []
-        while fns:
-            if len(launched) < max_process:
-                p = Process(target=self.rq_dump_sigfn, args=(fns.pop(), options))
+            bb.note("Writing task signature files")
+
+            max_process = int(self.cfgData.getVar("BB_NUMBER_PARSE_THREADS") or os.cpu_count() or 1)
+            def chunkify(l, n):
+                return [l[i::n] for i in range(n)]
+            dumpsigs_tids = chunkify(list(self.rqdata.runtaskentries), max_process)
+
+            # We cannot use the real multiprocessing.Pool easily due to some local data
+            # that can't be pickled. This is a cheap multi-process solution.
+            self.dumpsigs_launched = []
+
+            for tids in dumpsigs_tids:
+                p = Process(target=self._rq_dump_sigtid, args=(tids, ))
                 p.start()
-                launched.append(p)
-            for q in launched:
-                # The finished processes are joined when calling is_alive()
-                if not q.is_alive():
-                    launched.remove(q)
-        for p in launched:
+                self.dumpsigs_launched.append(p)
+
+            return 1.0
+
+        for q in self.dumpsigs_launched:
+            # The finished processes are joined when calling is_alive()
+            if not q.is_alive():
+                self.dumpsigs_launched.remove(q)
+
+        if self.dumpsigs_launched:
+            return 1.0
+
+        for p in self.dumpsigs_launched:
                 p.join()
 
         bb.parse.siggen.dump_sigs(self.rqdata.dataCaches, options)
 
-        return
+        return False
 
     def print_diffscenetasks(self):
+        def get_root_invalid_tasks(task, taskdepends, valid, noexec, visited_invalid):
+            invalidtasks = []
+            for t in taskdepends[task].depends:
+                if t not in valid and t not in visited_invalid:
+                    invalidtasks.extend(get_root_invalid_tasks(t, taskdepends, valid, noexec, visited_invalid))
+                    visited_invalid.add(t)
+
+            direct_invalid = [t for t in taskdepends[task].depends if t not in valid]
+            if not direct_invalid and task not in noexec:
+                invalidtasks = [task]
+            return invalidtasks
 
         noexec = []
         tocheck = set()
@@ -1633,46 +1776,49 @@ class RunQueue:
                     valid_new.add(dep)
 
         invalidtasks = set()
-        for tid in self.rqdata.runtaskentries:
-            if tid not in valid_new and tid not in noexec:
-                invalidtasks.add(tid)
 
-        found = set()
-        processed = set()
-        for tid in invalidtasks:
+        toptasks = set(["{}:{}".format(t[3], t[2]) for t in self.rqdata.targets])
+        for tid in toptasks:
             toprocess = set([tid])
             while toprocess:
                 next = set()
+                visited_invalid = set()
                 for t in toprocess:
-                    for dep in self.rqdata.runtaskentries[t].depends:
-                        if dep in invalidtasks:
-                            found.add(tid)
-                        if dep not in processed:
-                            processed.add(dep)
+                    if t not in valid_new and t not in noexec:
+                        invalidtasks.update(get_root_invalid_tasks(t, self.rqdata.runtaskentries, valid_new, noexec, visited_invalid))
+                        continue
+                    if t in self.rqdata.runq_setscene_tids:
+                        for dep in self.rqexe.sqdata.sq_deps[t]:
                             next.add(dep)
+                        continue
+
+                    for dep in self.rqdata.runtaskentries[t].depends:
+                        next.add(dep)
+
                 toprocess = next
-                if tid in found:
-                    toprocess = set()
 
         tasklist = []
-        for tid in invalidtasks.difference(found):
+        for tid in invalidtasks:
             tasklist.append(tid)
 
         if tasklist:
             bb.plain("The differences between the current build and any cached tasks start at the following tasks:\n" + "\n".join(tasklist))
 
-        return invalidtasks.difference(found)
+        return invalidtasks
 
     def write_diffscenetasks(self, invalidtasks):
+        bb.siggen.check_siggen_version(bb.siggen)
 
         # Define recursion callback
         def recursecb(key, hash1, hash2):
             hashes = [hash1, hash2]
+            bb.debug(1, "Recursively looking for recipe {} hashes {}".format(key, hashes))
             hashfiles = bb.siggen.find_siginfo(key, None, hashes, self.cfgData)
+            bb.debug(1, "Found hashfiles:\n{}".format(hashfiles))
 
             recout = []
             if len(hashfiles) == 2:
-                out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb)
+                out2 = bb.siggen.compare_sigfiles(hashfiles[hash1]['path'], hashfiles[hash2]['path'], recursecb)
                 recout.extend(list('    ' + l for l in out2))
             else:
                 recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2))
@@ -1683,20 +1829,25 @@ class RunQueue:
         for tid in invalidtasks:
             (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
             pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
-            h = self.rqdata.runtaskentries[tid].hash
-            matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData)
+            h = self.rqdata.runtaskentries[tid].unihash
+            bb.debug(1, "Looking for recipe {} task {}".format(pn, taskname))
+            matches = bb.siggen.find_siginfo(pn, taskname, [], self.cooker.databuilder.mcdata[mc])
+            bb.debug(1, "Found hashfiles:\n{}".format(matches))
             match = None
-            for m in matches:
-                if h in m:
-                    match = m
+            for m in matches.values():
+                if h in m['path']:
+                    match = m['path']
             if match is None:
-                bb.fatal("Can't find a task we're supposed to have written out? (hash: %s)?" % h)
+                bb.fatal("Can't find a task we're supposed to have written out? (hash: %s tid: %s)?" % (h, tid))
             matches = {k : v for k, v in iter(matches.items()) if h not in k}
+            matches_local = {k : v for k, v in iter(matches.items()) if h not in k and not v['sstate']}
+            if matches_local:
+                matches = matches_local
             if matches:
-                latestmatch = sorted(matches.keys(), key=lambda f: matches[f])[-1]
+                latestmatch = matches[sorted(matches.keys(), key=lambda h: matches[h]['time'])[-1]]['path']
                 prevh = __find_sha256__.search(latestmatch).group(0)
                 output = bb.siggen.compare_sigfiles(latestmatch, match, recursecb)
-                bb.plain("\nTask %s:%s couldn't be used from the cache because:\n  We need hash %s, closest matching task was %s\n  " % (pn, taskname, h, prevh) + '\n  '.join(output))
+                bb.plain("\nTask %s:%s couldn't be used from the cache because:\n  We need hash %s, most recent matching task was %s\n  " % (pn, taskname, h, prevh) + '\n  '.join(output))
 
 
 class RunQueueExecute:
@@ -1709,6 +1860,10 @@ class RunQueueExecute:
 
         self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS") or 1)
         self.scheduler = self.cfgData.getVar("BB_SCHEDULER") or "speed"
+        self.max_cpu_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_CPU")
+        self.max_io_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_IO")
+        self.max_memory_pressure = self.cfgData.getVar("BB_PRESSURE_MAX_MEMORY")
+        self.max_loadfactor = self.cfgData.getVar("BB_LOADFACTOR_MAX")
 
         self.sq_buildable = set()
         self.sq_running = set()
@@ -1726,6 +1881,8 @@ class RunQueueExecute:
         self.build_stamps2 = []
         self.failed_tids = []
         self.sq_deferred = {}
+        self.sq_needed_harddeps = set()
+        self.sq_harddep_deferred = set()
 
         self.stampcache = {}
 
@@ -1733,17 +1890,39 @@ class RunQueueExecute:
         self.holdoff_need_update = True
         self.sqdone = False
 
-        self.stats = RunQueueStats(len(self.rqdata.runtaskentries))
-        self.sq_stats = RunQueueStats(len(self.rqdata.runq_setscene_tids))
-
-        for mc in rq.worker:
-            rq.worker[mc].pipe.setrunqueueexec(self)
-        for mc in rq.fakeworker:
-            rq.fakeworker[mc].pipe.setrunqueueexec(self)
+        self.stats = RunQueueStats(len(self.rqdata.runtaskentries), len(self.rqdata.runq_setscene_tids))
 
         if self.number_tasks <= 0:
              bb.fatal("Invalid BB_NUMBER_THREADS %s" % self.number_tasks)
 
+        lower_limit = 1.0
+        upper_limit = 1000000.0
+        if self.max_cpu_pressure:
+            self.max_cpu_pressure = float(self.max_cpu_pressure)
+            if self.max_cpu_pressure < lower_limit:
+                bb.fatal("Invalid BB_PRESSURE_MAX_CPU %s, minimum value is %s." % (self.max_cpu_pressure, lower_limit))
+            if self.max_cpu_pressure > upper_limit:
+                bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_CPU is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_cpu_pressure))
+
+        if self.max_io_pressure:
+            self.max_io_pressure = float(self.max_io_pressure)
+            if self.max_io_pressure < lower_limit:
+                bb.fatal("Invalid BB_PRESSURE_MAX_IO %s, minimum value is %s." % (self.max_io_pressure, lower_limit))
+            if self.max_io_pressure > upper_limit:
+                bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_IO is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_io_pressure))
+
+        if self.max_memory_pressure:
+            self.max_memory_pressure = float(self.max_memory_pressure)
+            if self.max_memory_pressure < lower_limit:
+                bb.fatal("Invalid BB_PRESSURE_MAX_MEMORY %s, minimum value is %s." % (self.max_memory_pressure, lower_limit))
+            if self.max_memory_pressure > upper_limit:
+                bb.warn("Your build will be largely unregulated since BB_PRESSURE_MAX_MEMORY is set to %s. It is very unlikely that such high pressure will be experienced." % (self.max_io_pressure))
+
+        if self.max_loadfactor:
+            self.max_loadfactor = float(self.max_loadfactor)
+            if self.max_loadfactor <= 0:
+                bb.fatal("Invalid BB_LOADFACTOR_MAX %s, needs to be greater than zero." % (self.max_loadfactor))
+            
         # List of setscene tasks which we've covered
         self.scenequeue_covered = set()
         # List of tasks which are covered (including setscene ones)
@@ -1753,11 +1932,6 @@ class RunQueueExecute:
         self.tasks_notcovered = set()
         self.scenequeue_notneeded = set()
 
-        # We can't skip specified target tasks which aren't setscene tasks
-        self.cantskip = set(self.rqdata.target_tids)
-        self.cantskip.difference_update(self.rqdata.runq_setscene_tids)
-        self.cantskip.intersection_update(self.rqdata.runtaskentries)
-
         schedulers = self.get_schedulers()
         for scheduler in schedulers:
             if self.scheduler == scheduler.name:
@@ -1768,11 +1942,29 @@ class RunQueueExecute:
             bb.fatal("Invalid scheduler '%s'.  Available schedulers: %s" %
                      (self.scheduler, ", ".join(obj.name for obj in schedulers)))
 
-        #if len(self.rqdata.runq_setscene_tids) > 0:
+        #if self.rqdata.runq_setscene_tids:
         self.sqdata = SQData()
-        build_scenequeue_data(self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self)
+        build_scenequeue_data(self.sqdata, self.rqdata, self)
+
+        update_scenequeue_data(self.sqdata.sq_revdeps, self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=True)
+
+        # Compute a list of 'stale' sstate tasks where the current hash does not match the one
+        # in any stamp files. Pass the list out to metadata as an event.
+        found = {}
+        for tid in self.rqdata.runq_setscene_tids:
+            (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+            stamps = bb.build.find_stale_stamps(taskname, taskfn)
+            if stamps:
+                if mc not in found:
+                    found[mc] = {}
+                found[mc][tid] = stamps
+        for mc in found:
+            event = bb.event.StaleSetSceneTasks(found[mc])
+            bb.event.fire(event, self.cooker.databuilder.mcdata[mc])
+
+        self.build_taskdepdata_cache()
 
-    def runqueue_process_waitpid(self, task, status):
+    def runqueue_process_waitpid(self, task, status, fakerootlog=None):
 
         # self.build_stamps[pid] may not exist when use shared work directory.
         if task in self.build_stamps:
@@ -1785,9 +1977,10 @@ class RunQueueExecute:
             else:
                 self.sq_task_complete(task)
             self.sq_live.remove(task)
+            self.stats.updateActiveSetscene(len(self.sq_live))
         else:
             if status != 0:
-                self.task_fail(task, status)
+                self.task_fail(task, status, fakerootlog=fakerootlog)
             else:
                 self.task_complete(task)
         return True
@@ -1795,20 +1988,20 @@ class RunQueueExecute:
     def finish_now(self):
         for mc in self.rq.worker:
             try:
-                self.rq.worker[mc].process.stdin.write(b"<finishnow></finishnow>")
+                RunQueue.send_pickled_data(self.rq.worker[mc].process, b"", "finishnow")
                 self.rq.worker[mc].process.stdin.flush()
             except IOError:
                 # worker must have died?
                 pass
         for mc in self.rq.fakeworker:
             try:
-                self.rq.fakeworker[mc].process.stdin.write(b"<finishnow></finishnow>")
+                RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, b"", "finishnow")
                 self.rq.fakeworker[mc].process.stdin.flush()
             except IOError:
                 # worker must have died?
                 pass
 
-        if len(self.failed_tids) != 0:
+        if self.failed_tids:
             self.rq.state = runQueueFailed
             return
 
@@ -1818,13 +2011,13 @@ class RunQueueExecute:
     def finish(self):
         self.rq.state = runQueueCleanUp
 
-        active = self.stats.active + self.sq_stats.active
+        active = self.stats.active + len(self.sq_live)
         if active > 0:
             bb.event.fire(runQueueExitWait(active), self.cfgData)
             self.rq.read_workers()
             return self.rq.active_fds()
 
-        if len(self.failed_tids) != 0:
+        if self.failed_tids:
             self.rq.state = runQueueFailed
             return True
 
@@ -1851,7 +2044,7 @@ class RunQueueExecute:
         return valid
 
     def can_start_task(self):
-        active = self.stats.active + self.sq_stats.active
+        active = self.stats.active + len(self.sq_live)
         can_start = active < self.number_tasks
         return can_start
 
@@ -1871,8 +2064,7 @@ class RunQueueExecute:
                 try:
                     module = __import__(modname, fromlist=(name,))
                 except ImportError as exc:
-                    logger.critical("Unable to import scheduler '%s' from '%s': %s" % (name, modname, exc))
-                    raise SystemExit(1)
+                    bb.fatal("Unable to import scheduler '%s' from '%s': %s" % (name, modname, exc))
                 else:
                     schedulers.add(getattr(module, name))
         return schedulers
@@ -1902,21 +2094,52 @@ class RunQueueExecute:
                 self.setbuildable(revdep)
                 logger.debug("Marking task %s as buildable", revdep)
 
+        found = None
+        for t in sorted(self.sq_deferred.copy()):
+            if self.sq_deferred[t] == task:
+                # Allow the next deferred task to run. Any other deferred tasks should be deferred after that task.
+                # We shouldn't allow all to run at once as it is prone to races.
+                if not found:
+                    bb.debug(1, "Deferred task %s now buildable" % t)
+                    del self.sq_deferred[t]
+                    update_scenequeue_data([t], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
+                    found = t
+                else:
+                    bb.debug(1, "Deferring %s after %s" % (t, found))
+                    self.sq_deferred[t] = found
+
     def task_complete(self, task):
         self.stats.taskCompleted()
         bb.event.fire(runQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
         self.task_completeoutright(task)
         self.runq_tasksrun.add(task)
 
-    def task_fail(self, task, exitcode):
+    def task_fail(self, task, exitcode, fakerootlog=None):
         """
         Called when a task has failed
         Updates the state engine with the failure
         """
         self.stats.taskFailed()
         self.failed_tids.append(task)
-        bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq), self.cfgData)
-        if self.rqdata.taskData[''].abort:
+
+        fakeroot_log = []
+        if fakerootlog and os.path.exists(fakerootlog):
+            with open(fakerootlog) as fakeroot_log_file:
+                fakeroot_failed = False
+                for line in reversed(fakeroot_log_file.readlines()):
+                    for fakeroot_error in ['mismatch', 'error', 'fatal']:
+                        if fakeroot_error in line.lower():
+                            fakeroot_failed = True
+                    if 'doing new pid setup and server start' in line:
+                        break
+                    fakeroot_log.append(line)
+
+            if not fakeroot_failed:
+                fakeroot_log = []
+
+        bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq, fakeroot_log=("".join(fakeroot_log) or None)), self.cfgData)
+
+        if self.rqdata.taskData[''].halt:
             self.rq.state = runQueueCleanUp
 
     def task_skip(self, task, reason):
@@ -1931,7 +2154,7 @@ class RunQueueExecute:
         err = False
         if not self.sqdone:
             logger.debug('We could skip tasks %s', "\n".join(sorted(self.scenequeue_covered)))
-            completeevent = sceneQueueComplete(self.sq_stats, self.rq)
+            completeevent = sceneQueueComplete(self.stats, self.rq)
             bb.event.fire(completeevent, self.cfgData)
         if self.sq_deferred:
             logger.error("Scenequeue had deferred entries: %s" % pprint.pformat(self.sq_deferred))
@@ -1943,6 +2166,10 @@ class RunQueueExecute:
             logger.error("Scenequeue had holdoff tasks: %s" % pprint.pformat(self.holdoff_tasks))
             err = True
 
+        for tid in self.scenequeue_covered.intersection(self.scenequeue_notcovered):
+            # No task should end up in both covered and uncovered, that is a bug.
+            logger.error("Setscene task %s in both covered and notcovered." % tid)
+
         for tid in self.rqdata.runq_setscene_tids:
             if tid not in self.scenequeue_covered and tid not in self.scenequeue_notcovered:
                 err = True
@@ -1961,7 +2188,7 @@ class RunQueueExecute:
             if x not in self.tasks_scenequeue_done:
                 logger.error("Task %s was never processed by the setscene code" % x)
                 err = True
-            if len(self.rqdata.runtaskentries[x].depends) == 0 and x not in self.runq_buildable:
+            if not self.rqdata.runtaskentries[x].depends and x not in self.runq_buildable:
                 logger.error("Task %s was never marked as buildable by the setscene code" % x)
                 err = True
         return err
@@ -1979,13 +2206,24 @@ class RunQueueExecute:
         if not hasattr(self, "sorted_setscene_tids"):
             # Don't want to sort this set every execution
             self.sorted_setscene_tids = sorted(self.rqdata.runq_setscene_tids)
+            # Resume looping where we left off when we returned to feed the mainloop
+            self.setscene_tids_generator = itertools.cycle(self.rqdata.runq_setscene_tids)
 
         task = None
         if not self.sqdone and self.can_start_task():
-            # Find the next setscene to run
-            for nexttask in self.sorted_setscene_tids:
-                if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values():
-                    if nexttask not in self.sqdata.unskippable and len(self.sqdata.sq_revdeps[nexttask]) > 0 and self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
+            loopcount = 0
+            # Find the next setscene to run, exit the loop when we've processed all tids or found something to execute
+            while loopcount < len(self.rqdata.runq_setscene_tids):
+                loopcount += 1
+                nexttask = next(self.setscene_tids_generator)
+                if nexttask in self.sq_buildable and nexttask not in self.sq_running and self.sqdata.stamps[nexttask] not in self.build_stamps.values() and nexttask not in self.sq_harddep_deferred:
+                    if nexttask in self.sq_deferred and self.sq_deferred[nexttask] not in self.runq_complete:
+                        # Skip deferred tasks quickly before the 'expensive' tests below - this is key to performant multiconfig builds
+                        continue
+                    if nexttask not in self.sqdata.unskippable and self.sqdata.sq_revdeps[nexttask] and \
+                            nexttask not in self.sq_needed_harddeps and \
+                            self.sqdata.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and \
+                            self.check_dependencies(nexttask, self.sqdata.sq_revdeps[nexttask]):
                         if nexttask not in self.rqdata.target_tids:
                             logger.debug2("Skipping setscene for task %s" % nexttask)
                             self.sq_task_skip(nexttask)
@@ -1993,13 +2231,25 @@ class RunQueueExecute:
                             if nexttask in self.sq_deferred:
                                 del self.sq_deferred[nexttask]
                             return True
+                    if nexttask in self.sqdata.sq_harddeps_rev and not self.sqdata.sq_harddeps_rev[nexttask].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+                        logger.debug2("Deferring %s due to hard dependencies" % nexttask)
+                        updated = False
+                        for dep in self.sqdata.sq_harddeps_rev[nexttask]:
+                            if dep not in self.sq_needed_harddeps:
+                                logger.debug2("Enabling task %s as it is a hard dependency" % dep)
+                                self.sq_buildable.add(dep)
+                                self.sq_needed_harddeps.add(dep)
+                                updated = True
+                        self.sq_harddep_deferred.add(nexttask)
+                        if updated:
+                            return True
+                        continue
                     # If covered tasks are running, need to wait for them to complete
                     for t in self.sqdata.sq_covered_tasks[nexttask]:
                         if t in self.runq_running and t not in self.runq_complete:
                             continue
                     if nexttask in self.sq_deferred:
-                        if self.sq_deferred[nexttask] not in self.runq_complete:
-                            continue
+                        # Deferred tasks that were still deferred were skipped above so we now need to process
                         logger.debug("Task %s no longer deferred" % nexttask)
                         del self.sq_deferred[nexttask]
                         valid = self.rq.validate_hashes(set([nexttask]), self.cooker.data, 0, False, summary=False)
@@ -2007,8 +2257,6 @@ class RunQueueExecute:
                             logger.debug("%s didn't become valid, skipping setscene" % nexttask)
                             self.sq_task_failoutright(nexttask)
                             return True
-                        else:
-                            self.sqdata.outrightfail.remove(nexttask)
                     if nexttask in self.sqdata.outrightfail:
                         logger.debug2('No package found, so skipping setscene task %s', nexttask)
                         self.sq_task_failoutright(nexttask)
@@ -2040,28 +2288,42 @@ class RunQueueExecute:
                 self.sq_task_failoutright(task)
                 return True
 
-            startevent = sceneQueueTaskStarted(task, self.sq_stats, self.rq)
+            startevent = sceneQueueTaskStarted(task, self.stats, self.rq)
             bb.event.fire(startevent, self.cfgData)
 
-            taskdepdata = self.sq_build_taskdepdata(task)
-
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
-            taskhash = self.rqdata.get_task_hash(task)
-            unihash = self.rqdata.get_task_unihash(task)
+            realfn = bb.cache.virtualfn2realfn(taskfn)[0]
+            runtask = {
+                'fn' : taskfn,
+                'task' : task,
+                'taskname' : taskname,
+                'taskhash' : self.rqdata.get_task_hash(task),
+                'unihash' : self.rqdata.get_task_unihash(task),
+                'quieterrors' : True,
+                'appends' : self.cooker.collections[mc].get_file_appends(taskfn),
+                'layername' : self.cooker.collections[mc].calc_bbfile_priority(realfn)[2],
+                'taskdepdata' : self.sq_build_taskdepdata(task),
+                'dry_run' : False,
+                'taskdep': taskdep,
+                'fakerootenv' : self.rqdata.dataCaches[mc].fakerootenv[taskfn],
+                'fakerootdirs' : self.rqdata.dataCaches[mc].fakerootdirs[taskfn],
+                'fakerootnoenv' : self.rqdata.dataCaches[mc].fakerootnoenv[taskfn]
+            }
+
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not self.cooker.configuration.dry_run:
                 if not mc in self.rq.fakeworker:
                     self.rq.start_fakeworker(self, mc)
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+                RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, runtask, "runtask")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, True, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, False)) + b"</runtask>")
+                RunQueue.send_pickled_data(self.rq.worker[mc].process, runtask, "runtask")
                 self.rq.worker[mc].process.stdin.flush()
 
-            self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
+            self.build_stamps[task] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
             self.build_stamps2.append(self.build_stamps[task])
             self.sq_running.add(task)
             self.sq_live.add(task)
-            self.sq_stats.taskActive()
+            self.stats.updateActiveSetscene(len(self.sq_live))
             if self.can_start_task():
                 return True
 
@@ -2092,9 +2354,9 @@ class RunQueueExecute:
         if task is not None:
             (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
 
-            if self.rqdata.setscenewhitelist is not None:
-                if self.check_setscenewhitelist(task):
-                    self.task_fail(task, "setscene whitelist")
+            if self.rqdata.setscene_ignore_tasks is not None:
+                if self.check_setscene_ignore_tasks(task):
+                    self.task_fail(task, "setscene ignore_tasks")
                     return True
 
             if task in self.tasks_covered:
@@ -2117,18 +2379,32 @@ class RunQueueExecute:
                 self.runq_running.add(task)
                 self.stats.taskActive()
                 if not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
-                    bb.build.make_stamp(taskname, self.rqdata.dataCaches[mc], taskfn)
+                    bb.build.make_stamp_mcfn(taskname, taskfn)
                 self.task_complete(task)
                 return True
             else:
                 startevent = runQueueTaskStarted(task, self.stats, self.rq)
                 bb.event.fire(startevent, self.cfgData)
 
-            taskdepdata = self.build_taskdepdata(task)
-
             taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
-            taskhash = self.rqdata.get_task_hash(task)
-            unihash = self.rqdata.get_task_unihash(task)
+            realfn = bb.cache.virtualfn2realfn(taskfn)[0]
+            runtask = {
+                'fn' : taskfn,
+                'task' : task,
+                'taskname' : taskname,
+                'taskhash' : self.rqdata.get_task_hash(task),
+                'unihash' : self.rqdata.get_task_unihash(task),
+                'quieterrors' : False,
+                'appends' : self.cooker.collections[mc].get_file_appends(taskfn),
+                'layername' : self.cooker.collections[mc].calc_bbfile_priority(realfn)[2],
+                'taskdepdata' : self.build_taskdepdata(task),
+                'dry_run' : self.rqdata.setscene_enforce,
+                'taskdep': taskdep,
+                'fakerootenv' : self.rqdata.dataCaches[mc].fakerootenv[taskfn],
+                'fakerootdirs' : self.rqdata.dataCaches[mc].fakerootdirs[taskfn],
+                'fakerootnoenv' : self.rqdata.dataCaches[mc].fakerootnoenv[taskfn]
+            }
+
             if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not (self.cooker.configuration.dry_run or self.rqdata.setscene_enforce):
                 if not mc in self.rq.fakeworker:
                     try:
@@ -2138,31 +2414,31 @@ class RunQueueExecute:
                         self.rq.state = runQueueFailed
                         self.stats.taskFailed()
                         return True
-                self.rq.fakeworker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+                RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, runtask, "runtask")
                 self.rq.fakeworker[mc].process.stdin.flush()
             else:
-                self.rq.worker[mc].process.stdin.write(b"<runtask>" + pickle.dumps((taskfn, task, taskname, taskhash, unihash, False, self.cooker.collections[mc].get_file_appends(taskfn), taskdepdata, self.rqdata.setscene_enforce)) + b"</runtask>")
+                RunQueue.send_pickled_data(self.rq.worker[mc].process, runtask, "runtask")
                 self.rq.worker[mc].process.stdin.flush()
 
-            self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
+            self.build_stamps[task] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
             self.build_stamps2.append(self.build_stamps[task])
             self.runq_running.add(task)
             self.stats.taskActive()
             if self.can_start_task():
                 return True
 
-        if self.stats.active > 0 or self.sq_stats.active > 0:
+        if self.stats.active > 0 or self.sq_live:
             self.rq.read_workers()
             return self.rq.active_fds()
 
         # No more tasks can be run. If we have deferred setscene tasks we should run them.
         if self.sq_deferred:
-            tid = self.sq_deferred.pop(list(self.sq_deferred.keys())[0])
-            logger.warning("Runqeueue deadlocked on deferred tasks, forcing task %s" % tid)
-            self.sq_task_failoutright(tid)
+            deferred_tid = list(self.sq_deferred.keys())[0]
+            blocking_tid = self.sq_deferred.pop(deferred_tid)
+            logger.warning("Runqueue deadlocked on deferred tasks, forcing task %s blocked by %s" % (deferred_tid, blocking_tid))
             return True
 
-        if len(self.failed_tids) != 0:
+        if self.failed_tids:
             self.rq.state = runQueueFailed
             return True
 
@@ -2195,6 +2471,25 @@ class RunQueueExecute:
             ret.add(dep)
         return ret
 
+    # Build the individual cache entries in advance once to save time
+    def build_taskdepdata_cache(self):
+        taskdepdata_cache = {}
+        for task in self.rqdata.runtaskentries:
+            (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
+            taskdepdata_cache[task] = bb.TaskData(
+                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn],
+                taskname = taskname,
+                fn = fn,
+                deps = self.filtermcdeps(task, mc, self.rqdata.runtaskentries[task].depends),
+                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn],
+                taskhash = self.rqdata.runtaskentries[task].hash,
+                unihash = self.rqdata.runtaskentries[task].unihash,
+                hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn],
+                taskhash_deps = self.rqdata.runtaskentries[task].taskhash_deps,
+            )
+
+        self.taskdepdata_cache = taskdepdata_cache
+
     # We filter out multiconfig dependencies from taskdepdata we pass to the tasks
     # as most code can't handle them
     def build_taskdepdata(self, task):
@@ -2206,15 +2501,11 @@ class RunQueueExecute:
         while next:
             additional = []
             for revdep in next:
-                (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep)
-                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
-                deps = self.rqdata.runtaskentries[revdep].depends
-                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
-                taskhash = self.rqdata.runtaskentries[revdep].hash
-                unihash = self.rqdata.runtaskentries[revdep].unihash
-                deps = self.filtermcdeps(task, mc, deps)
-                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash]
-                for revdep2 in deps:
+                self.taskdepdata_cache[revdep] = self.taskdepdata_cache[revdep]._replace(
+                    unihash=self.rqdata.runtaskentries[revdep].unihash
+                )
+                taskdepdata[revdep] = self.taskdepdata_cache[revdep]
+                for revdep2 in self.taskdepdata_cache[revdep].deps:
                     if revdep2 not in taskdepdata:
                         additional.append(revdep2)
             next = additional
@@ -2228,7 +2519,7 @@ class RunQueueExecute:
             return
 
         notcovered = set(self.scenequeue_notcovered)
-        notcovered |= self.cantskip
+        notcovered |= self.sqdata.cantskip
         for tid in self.scenequeue_notcovered:
             notcovered |= self.sqdata.sq_covered_tasks[tid]
         notcovered |= self.sqdata.unskippable.difference(self.rqdata.runq_setscene_tids)
@@ -2241,7 +2532,7 @@ class RunQueueExecute:
         covered.intersection_update(self.tasks_scenequeue_done)
 
         for tid in notcovered | covered:
-            if len(self.rqdata.runtaskentries[tid].depends) == 0:
+            if not self.rqdata.runtaskentries[tid].depends:
                 self.setbuildable(tid)
             elif self.rqdata.runtaskentries[tid].depends.issubset(self.runq_complete):
                  self.setbuildable(tid)
@@ -2273,10 +2564,16 @@ class RunQueueExecute:
             self.updated_taskhash_queue.remove((tid, unihash))
 
             if unihash != self.rqdata.runtaskentries[tid].unihash:
-                hashequiv_logger.verbose("Task %s unihash changed to %s" % (tid, unihash))
-                self.rqdata.runtaskentries[tid].unihash = unihash
-                bb.parse.siggen.set_unihash(tid, unihash)
-                toprocess.add(tid)
+                # Make sure we rehash any other tasks with the same task hash that we're deferred against.
+                torehash = [tid]
+                for deftid in self.sq_deferred:
+                    if self.sq_deferred[deftid] == tid:
+                        torehash.append(deftid)
+                for hashtid in torehash:
+                    hashequiv_logger.verbose("Task %s unihash changed to %s" % (hashtid, unihash))
+                    self.rqdata.runtaskentries[hashtid].unihash = unihash
+                    bb.parse.siggen.set_unihash(hashtid, unihash)
+                    toprocess.add(hashtid)
 
         # Work out all tasks which depend upon these
         total = set()
@@ -2294,23 +2591,33 @@ class RunQueueExecute:
         # Now iterate those tasks in dependency order to regenerate their taskhash/unihash
         next = set()
         for p in total:
-            if len(self.rqdata.runtaskentries[p].depends) == 0:
+            if not self.rqdata.runtaskentries[p].depends:
                 next.add(p)
             elif self.rqdata.runtaskentries[p].depends.isdisjoint(total):
                 next.add(p)
 
+        starttime = time.time()
+        lasttime = starttime
+
         # When an item doesn't have dependencies in total, we can process it. Drop items from total when handled
         while next:
             current = next.copy()
             next = set()
+            ready = {}
             for tid in current:
-                if len(self.rqdata.runtaskentries[p].depends) and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
+                if self.rqdata.runtaskentries[p].depends and not self.rqdata.runtaskentries[tid].depends.isdisjoint(total):
                     continue
+                # get_taskhash for a given tid *must* be called before get_unihash* below
+                ready[tid] = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, self.rqdata.dataCaches)
+
+            unihashes = bb.parse.siggen.get_unihashes(ready.keys())
+
+            for tid in ready:
                 orighash = self.rqdata.runtaskentries[tid].hash
-                dc = bb.parse.siggen.get_data_caches(self.rqdata.dataCaches, mc_from_tid(tid))
-                newhash = bb.parse.siggen.get_taskhash(tid, self.rqdata.runtaskentries[tid].depends, dc)
+                newhash = ready[tid]
                 origuni = self.rqdata.runtaskentries[tid].unihash
-                newuni = bb.parse.siggen.get_unihash(tid)
+                newuni = unihashes[tid]
+
                 # FIXME, need to check it can come from sstate at all for determinism?
                 remapped = False
                 if newuni == origuni:
@@ -2331,12 +2638,21 @@ class RunQueueExecute:
                 next |= self.rqdata.runtaskentries[tid].revdeps
                 total.remove(tid)
                 next.intersection_update(total)
+                bb.event.check_for_interrupts(self.cooker.data)
+
+                if time.time() > (lasttime + 30):
+                    lasttime = time.time()
+                    hashequiv_logger.verbose("Rehash loop slow progress: %s in %s" % (len(total), lasttime - starttime))
+
+        endtime = time.time()
+        if (endtime-starttime > 60):
+            hashequiv_logger.verbose("Rehash loop took more than 60s: %s" % (endtime-starttime))
 
         if changed:
             for mc in self.rq.worker:
-                self.rq.worker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>")
+                RunQueue.send_pickled_data(self.rq.worker[mc].process, bb.parse.siggen.get_taskhashes(), "newtaskhashes")
             for mc in self.rq.fakeworker:
-                self.rq.fakeworker[mc].process.stdin.write(b"<newtaskhashes>" + pickle.dumps(bb.parse.siggen.get_taskhashes()) + b"</newtaskhashes>")
+                RunQueue.send_pickled_data(self.rq.fakeworker[mc].process, bb.parse.siggen.get_taskhashes(), "newtaskhashes")
 
             hashequiv_logger.debug(pprint.pformat("Tasks changed:\n%s" % (changed)))
 
@@ -2370,7 +2686,7 @@ class RunQueueExecute:
                 self.tasks_scenequeue_done.remove(tid)
             for dep in self.sqdata.sq_covered_tasks[tid]:
                 if dep in self.runq_complete and dep not in self.runq_tasksrun:
-                    bb.error("Task %s marked as completed but now needing to rerun? Aborting build." % dep)
+                    bb.error("Task %s marked as completed but now needing to rerun? Halting build." % dep)
                     self.failed_tids.append(tid)
                     self.rq.state = runQueueCleanUp
                     return
@@ -2383,17 +2699,6 @@ class RunQueueExecute:
                 self.sq_buildable.remove(tid)
             if tid in self.sq_running:
                 self.sq_running.remove(tid)
-            harddepfail = False
-            for t in self.sqdata.sq_harddeps:
-                if tid in self.sqdata.sq_harddeps[t] and t in self.scenequeue_notcovered:
-                    harddepfail = True
-                    break
-            if not harddepfail and self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
-                if tid not in self.sq_buildable:
-                    self.sq_buildable.add(tid)
-            if len(self.sqdata.sq_revdeps[tid]) == 0:
-                self.sq_buildable.add(tid)
-
             if tid in self.sqdata.outrightfail:
                 self.sqdata.outrightfail.remove(tid)
             if tid in self.scenequeue_notcovered:
@@ -2404,7 +2709,7 @@ class RunQueueExecute:
                 self.scenequeue_notneeded.remove(tid)
 
             (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
-            self.sqdata.stamps[tid] = bb.build.stampfile(taskname + "_setscene", self.rqdata.dataCaches[mc], taskfn, noextra=True)
+            self.sqdata.stamps[tid] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
 
             if tid in self.stampcache:
                 del self.stampcache[tid]
@@ -2412,29 +2717,67 @@ class RunQueueExecute:
             if tid in self.build_stamps:
                 del self.build_stamps[tid]
 
-            update_tasks.append((tid, harddepfail, tid in self.sqdata.valid))
+            update_tasks.append(tid)
 
-        if update_tasks:
+        update_tasks2 = []
+        for tid in update_tasks:
+            harddepfail = False
+            for t in self.sqdata.sq_harddeps_rev[tid]:
+                if t in self.scenequeue_notcovered:
+                    harddepfail = True
+                    break
+            if not harddepfail and self.sqdata.sq_revdeps[tid].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+                if tid not in self.sq_buildable:
+                    self.sq_buildable.add(tid)
+            if not self.sqdata.sq_revdeps[tid]:
+                self.sq_buildable.add(tid)
+
+            update_tasks2.append((tid, harddepfail, tid in self.sqdata.valid))
+
+        if update_tasks2:
             self.sqdone = False
-            update_scenequeue_data([t[0] for t in update_tasks], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
+            for mc in sorted(self.sqdata.multiconfigs):
+                for tid in sorted([t[0] for t in update_tasks2]):
+                    if mc_from_tid(tid) != mc:
+                        continue
+                    h = pending_hash_index(tid, self.rqdata)
+                    if h in self.sqdata.hashes and tid != self.sqdata.hashes[h]:
+                        self.sq_deferred[tid] = self.sqdata.hashes[h]
+                        bb.note("Deferring %s after %s" % (tid, self.sqdata.hashes[h]))
+            update_scenequeue_data([t[0] for t in update_tasks2], self.sqdata, self.rqdata, self.rq, self.cooker, self.stampcache, self, summary=False)
 
-        for (tid, harddepfail, origvalid) in update_tasks:
+        for (tid, harddepfail, origvalid) in update_tasks2:
             if tid in self.sqdata.valid and not origvalid:
                 hashequiv_logger.verbose("Setscene task %s became valid" % tid)
             if harddepfail:
+                logger.debug2("%s has an unavailable hard dependency so skipping" % (tid))
                 self.sq_task_failoutright(tid)
 
         if changed:
+            self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered))
+            self.sq_needed_harddeps = set()
+            self.sq_harddep_deferred = set()
             self.holdoff_need_update = True
 
     def scenequeue_updatecounters(self, task, fail=False):
 
-        for dep in sorted(self.sqdata.sq_deps[task]):
-            if fail and task in self.sqdata.sq_harddeps and dep in self.sqdata.sq_harddeps[task]:
+        if fail and task in self.sqdata.sq_harddeps:
+            for dep in sorted(self.sqdata.sq_harddeps[task]):
+                if dep in self.scenequeue_covered or dep in self.scenequeue_notcovered:
+                    # dependency could be already processed, e.g. noexec setscene task
+                    continue
+                noexec, stamppresent = check_setscene_stamps(dep, self.rqdata, self.rq, self.stampcache)
+                if noexec or stamppresent:
+                    continue
                 logger.debug2("%s was unavailable and is a hard dependency of %s so skipping" % (task, dep))
                 self.sq_task_failoutright(dep)
                 continue
-            if self.sqdata.sq_revdeps[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+
+        # For performance, only compute allcovered once if needed
+        if self.sqdata.sq_deps[task]:
+            allcovered = self.scenequeue_covered | self.scenequeue_notcovered
+        for dep in sorted(self.sqdata.sq_deps[task]):
+            if self.sqdata.sq_revdeps[dep].issubset(allcovered):
                 if dep not in self.sq_buildable:
                     self.sq_buildable.add(dep)
 
@@ -2452,6 +2795,14 @@ class RunQueueExecute:
                         new.add(dep)
             next = new
 
+        # If this task was one which other setscene tasks have a hard dependency upon, we need
+        # to walk through the hard dependencies and allow execution of those which have completed dependencies.
+        if task in self.sqdata.sq_harddeps:
+            for dep in self.sq_harddep_deferred.copy():
+                if self.sqdata.sq_harddeps_rev[dep].issubset(self.scenequeue_covered | self.scenequeue_notcovered):
+                    self.sq_harddep_deferred.remove(dep)
+
+        self.stats.updateCovered(len(self.scenequeue_covered), len(self.scenequeue_notcovered))
         self.holdoff_need_update = True
 
     def sq_task_completeoutright(self, task):
@@ -2466,22 +2817,20 @@ class RunQueueExecute:
         self.scenequeue_updatecounters(task)
 
     def sq_check_taskfail(self, task):
-        if self.rqdata.setscenewhitelist is not None:
+        if self.rqdata.setscene_ignore_tasks is not None:
             realtask = task.split('_setscene')[0]
             (mc, fn, taskname, taskfn) = split_tid_mcfn(realtask)
             pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
-            if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
+            if not check_setscene_enforce_ignore_tasks(pn, taskname, self.rqdata.setscene_ignore_tasks):
                 logger.error('Task %s.%s failed' % (pn, taskname + "_setscene"))
                 self.rq.state = runQueueCleanUp
 
     def sq_task_complete(self, task):
-        self.sq_stats.taskCompleted()
-        bb.event.fire(sceneQueueTaskCompleted(task, self.sq_stats, self.rq), self.cfgData)
+        bb.event.fire(sceneQueueTaskCompleted(task, self.stats, self.rq), self.cfgData)
         self.sq_task_completeoutright(task)
 
     def sq_task_fail(self, task, result):
-        self.sq_stats.taskFailed()
-        bb.event.fire(sceneQueueTaskFailed(task, self.sq_stats, result, self), self.cfgData)
+        bb.event.fire(sceneQueueTaskFailed(task, self.stats, result, self), self.cfgData)
         self.scenequeue_notcovered.add(task)
         self.scenequeue_updatecounters(task, True)
         self.sq_check_taskfail(task)
@@ -2489,8 +2838,6 @@ class RunQueueExecute:
     def sq_task_failoutright(self, task):
         self.sq_running.add(task)
         self.sq_buildable.add(task)
-        self.sq_stats.taskSkipped()
-        self.sq_stats.taskCompleted()
         self.scenequeue_notcovered.add(task)
         self.scenequeue_updatecounters(task, True)
 
@@ -2498,8 +2845,6 @@ class RunQueueExecute:
         self.sq_running.add(task)
         self.sq_buildable.add(task)
         self.sq_task_completeoutright(task)
-        self.sq_stats.taskSkipped()
-        self.sq_stats.taskCompleted()
 
     def sq_build_taskdepdata(self, task):
         def getsetscenedeps(tid):
@@ -2525,12 +2870,19 @@ class RunQueueExecute:
             additional = []
             for revdep in next:
                 (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep)
-                pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
                 deps = getsetscenedeps(revdep)
-                provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
-                taskhash = self.rqdata.runtaskentries[revdep].hash
-                unihash = self.rqdata.runtaskentries[revdep].unihash
-                taskdepdata[revdep] = [pn, taskname, fn, deps, provides, taskhash, unihash]
+
+                taskdepdata[revdep] = bb.TaskData(
+                    pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn],
+                    taskname = taskname,
+                    fn = fn,
+                    deps = deps,
+                    provides = self.rqdata.dataCaches[mc].fn_provides[taskfn],
+                    taskhash = self.rqdata.runtaskentries[revdep].hash,
+                    unihash = self.rqdata.runtaskentries[revdep].unihash,
+                    hashfn = self.rqdata.dataCaches[mc].hashfn[taskfn],
+                    taskhash_deps = self.rqdata.runtaskentries[revdep].taskhash_deps,
+                )
                 for revdep2 in deps:
                     if revdep2 not in taskdepdata:
                         additional.append(revdep2)
@@ -2539,8 +2891,8 @@ class RunQueueExecute:
         #bb.note("Task %s: " % task + str(taskdepdata).replace("], ", "],\n"))
         return taskdepdata
 
-    def check_setscenewhitelist(self, tid):
-        # Check task that is going to run against the whitelist
+    def check_setscene_ignore_tasks(self, tid):
+        # Check task that is going to run against the ignore tasks list
         (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
         # Ignore covered tasks
         if tid in self.tasks_covered:
@@ -2554,14 +2906,15 @@ class RunQueueExecute:
             return False
 
         pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
-        if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
+        if not check_setscene_enforce_ignore_tasks(pn, taskname, self.rqdata.setscene_ignore_tasks):
             if tid in self.rqdata.runq_setscene_tids:
-                msg = 'Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname)
+                msg = ['Task %s.%s attempted to execute unexpectedly and should have been setscened' % (pn, taskname)]
             else:
-                msg = 'Task %s.%s attempted to execute unexpectedly' % (pn, taskname)
+                msg = ['Task %s.%s attempted to execute unexpectedly' % (pn, taskname)]
             for t in self.scenequeue_notcovered:
-                msg = msg + "\nTask %s, unihash %s, taskhash %s" % (t, self.rqdata.runtaskentries[t].unihash, self.rqdata.runtaskentries[t].hash)
-            logger.error(msg + '\nThis is usually due to missing setscene tasks. Those missing in this build were: %s' % pprint.pformat(self.scenequeue_notcovered))
+                msg.append("\nTask %s, unihash %s, taskhash %s" % (t, self.rqdata.runtaskentries[t].unihash, self.rqdata.runtaskentries[t].hash))
+            msg.append('\nThis is usually due to missing setscene tasks. Those missing in this build were: %s' % pprint.pformat(self.scenequeue_notcovered))
+            logger.error("".join(msg))
             return True
         return False
 
@@ -2573,6 +2926,7 @@ class SQData(object):
         self.sq_revdeps = {}
         # Injected inter-setscene task dependencies
         self.sq_harddeps = {}
+        self.sq_harddeps_rev = {}
         # Cache of stamp files so duplicates can't run in parallel
         self.stamps = {}
         # Setscene tasks directly depended upon by the build
@@ -2582,12 +2936,17 @@ class SQData(object):
         # A list of normal tasks a setscene task covers
         self.sq_covered_tasks = {}
 
-def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
+def build_scenequeue_data(sqdata, rqdata, sqrq):
 
     sq_revdeps = {}
     sq_revdeps_squash = {}
     sq_collated_deps = {}
 
+    # We can't skip specified target tasks which aren't setscene tasks
+    sqdata.cantskip = set(rqdata.target_tids)
+    sqdata.cantskip.difference_update(rqdata.runq_setscene_tids)
+    sqdata.cantskip.intersection_update(rqdata.runtaskentries)
+
     # We need to construct a dependency graph for the setscene functions. Intermediate
     # dependencies between the setscene tasks only complicate the code. This code
     # therefore aims to collapse the huge runqueue dependency tree into a smaller one
@@ -2600,7 +2959,7 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
     for tid in rqdata.runtaskentries:
         sq_revdeps[tid] = copy.copy(rqdata.runtaskentries[tid].revdeps)
         sq_revdeps_squash[tid] = set()
-        if (len(sq_revdeps[tid]) == 0) and tid not in rqdata.runq_setscene_tids:
+        if not sq_revdeps[tid] and tid not in rqdata.runq_setscene_tids:
             #bb.warn("Added endpoint %s" % (tid))
             endpoints[tid] = set()
 
@@ -2634,16 +2993,15 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
             sq_revdeps_squash[point] = set()
             if point in rqdata.runq_setscene_tids:
                 sq_revdeps_squash[point] = tasks
-                tasks = set()
                 continue
             for dep in rqdata.runtaskentries[point].depends:
                 if point in sq_revdeps[dep]:
                     sq_revdeps[dep].remove(point)
                 if tasks:
                     sq_revdeps_squash[dep] |= tasks
-                if len(sq_revdeps[dep]) == 0 and dep not in rqdata.runq_setscene_tids:
+                if not sq_revdeps[dep] and dep not in rqdata.runq_setscene_tids:
                     newendpoints[dep] = task
-        if len(newendpoints) != 0:
+        if newendpoints:
             process_endpoints(newendpoints)
 
     process_endpoints(endpoints)
@@ -2655,16 +3013,16 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
     # Take the build endpoints (no revdeps) and find the sstate tasks they depend upon
     new = True
     for tid in rqdata.runtaskentries:
-        if len(rqdata.runtaskentries[tid].revdeps) == 0:
+        if not rqdata.runtaskentries[tid].revdeps:
             sqdata.unskippable.add(tid)
-    sqdata.unskippable |= sqrq.cantskip
+    sqdata.unskippable |= sqdata.cantskip
     while new:
         new = False
         orig = sqdata.unskippable.copy()
         for tid in sorted(orig, reverse=True):
             if tid in rqdata.runq_setscene_tids:
                 continue
-            if len(rqdata.runtaskentries[tid].depends) == 0:
+            if not rqdata.runtaskentries[tid].depends:
                 # These are tasks which have no setscene tasks in their chain, need to mark as directly buildable
                 sqrq.setbuildable(tid)
             sqdata.unskippable |= rqdata.runtaskentries[tid].depends
@@ -2676,14 +3034,13 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
     rqdata.init_progress_reporter.next_stage(len(rqdata.runtaskentries))
 
     # Sanity check all dependencies could be changed to setscene task references
-    for taskcounter, tid in enumerate(rqdata.runtaskentries):
+    for tid in rqdata.runtaskentries:
         if tid in rqdata.runq_setscene_tids:
             pass
-        elif len(sq_revdeps_squash[tid]) != 0:
-            bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.")
+        elif sq_revdeps_squash[tid]:
+            bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, halting. Please report this problem.")
         else:
             del sq_revdeps_squash[tid]
-        rqdata.init_progress_reporter.update(taskcounter)
 
     rqdata.init_progress_reporter.next_stage()
 
@@ -2694,7 +3051,9 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
         (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
         realtid = tid + "_setscene"
         idepends = rqdata.taskData[mc].taskentries[realtid].idepends
-        sqdata.stamps[tid] = bb.build.stampfile(taskname + "_setscene", rqdata.dataCaches[mc], taskfn, noextra=True)
+        sqdata.stamps[tid] = bb.parse.siggen.stampfile_mcfn(taskname, taskfn, extrainfo=False)
+
+        sqdata.sq_harddeps_rev[tid] = set()
         for (depname, idependtask) in idepends:
 
             if depname not in rqdata.taskData[mc].build_targets:
@@ -2707,20 +3066,15 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
             if deptid not in rqdata.runtaskentries:
                 bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (realtid, depfn, idependtask))
 
+            logger.debug2("Adding hard setscene dependency %s for %s" % (deptid, tid))
+
             if not deptid in sqdata.sq_harddeps:
                 sqdata.sq_harddeps[deptid] = set()
             sqdata.sq_harddeps[deptid].add(tid)
-
-            sq_revdeps_squash[tid].add(deptid)
-            # Have to zero this to avoid circular dependencies
-            sq_revdeps_squash[deptid] = set()
+            sqdata.sq_harddeps_rev[tid].add(deptid)
 
     rqdata.init_progress_reporter.next_stage()
 
-    for task in sqdata.sq_harddeps:
-        for dep in sqdata.sq_harddeps[task]:
-            sq_revdeps_squash[dep].add(task)
-
     rqdata.init_progress_reporter.next_stage()
 
     #for tid in sq_revdeps_squash:
@@ -2744,16 +3098,47 @@ def build_scenequeue_data(sqdata, rqdata, rq, cooker, stampcache, sqrq):
     sqdata.multiconfigs = set()
     for tid in sqdata.sq_revdeps:
         sqdata.multiconfigs.add(mc_from_tid(tid))
-        if len(sqdata.sq_revdeps[tid]) == 0:
+        if not sqdata.sq_revdeps[tid]:
             sqrq.sq_buildable.add(tid)
 
-    rqdata.init_progress_reporter.finish()
+    rqdata.init_progress_reporter.next_stage()
 
     sqdata.noexec = set()
     sqdata.stamppresent = set()
     sqdata.valid = set()
 
-    update_scenequeue_data(sqdata.sq_revdeps, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True)
+    sqdata.hashes = {}
+    sqrq.sq_deferred = {}
+    for mc in sorted(sqdata.multiconfigs):
+        for tid in sorted(sqdata.sq_revdeps):
+            if mc_from_tid(tid) != mc:
+                continue
+            h = pending_hash_index(tid, rqdata)
+            if h not in sqdata.hashes:
+                sqdata.hashes[h] = tid
+            else:
+                sqrq.sq_deferred[tid] = sqdata.hashes[h]
+                bb.debug(1, "Deferring %s after %s" % (tid, sqdata.hashes[h]))
+
+def check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=False):
+
+    (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
+
+    taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
+
+    if 'noexec' in taskdep and taskname in taskdep['noexec']:
+        bb.build.make_stamp_mcfn(taskname + "_setscene", taskfn)
+        return True, False
+
+    if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
+        logger.debug2('Setscene stamp current for task %s', tid)
+        return False, True
+
+    if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
+        logger.debug2('Normal stamp current for task %s', tid)
+        return False, True
+
+    return False, False
 
 def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, summary=True):
 
@@ -2764,55 +3149,42 @@ def update_scenequeue_data(tids, sqdata, rqdata, rq, cooker, stampcache, sqrq, s
             sqdata.stamppresent.remove(tid)
         if tid in sqdata.valid:
             sqdata.valid.remove(tid)
+        if tid in sqdata.outrightfail:
+            sqdata.outrightfail.remove(tid)
 
-        (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
-
-        taskdep = rqdata.dataCaches[mc].task_deps[taskfn]
+        noexec, stamppresent = check_setscene_stamps(tid, rqdata, rq, stampcache, noexecstamp=True)
 
-        if 'noexec' in taskdep and taskname in taskdep['noexec']:
+        if noexec:
             sqdata.noexec.add(tid)
             sqrq.sq_task_skip(tid)
-            bb.build.make_stamp(taskname + "_setscene", rqdata.dataCaches[mc], taskfn)
-            continue
-
-        if rq.check_stamp_task(tid, taskname + "_setscene", cache=stampcache):
-            logger.debug2('Setscene stamp current for task %s', tid)
-            sqdata.stamppresent.add(tid)
-            sqrq.sq_task_skip(tid)
+            logger.debug2("%s is noexec so skipping setscene" % (tid))
             continue
 
-        if rq.check_stamp_task(tid, taskname, recurse = True, cache=stampcache):
-            logger.debug2('Normal stamp current for task %s', tid)
+        if stamppresent:
             sqdata.stamppresent.add(tid)
             sqrq.sq_task_skip(tid)
+            logger.debug2("%s has a valid stamp, skipping" % (tid))
             continue
 
         tocheck.add(tid)
 
     sqdata.valid |= rq.validate_hashes(tocheck, cooker.data, len(sqdata.stamppresent), False, summary=summary)
 
-    sqdata.hashes = {}
-    for mc in sorted(sqdata.multiconfigs):
-        for tid in sorted(sqdata.sq_revdeps):
-            if mc_from_tid(tid) != mc:
-                continue
-            if tid in sqdata.stamppresent:
-                continue
-            if tid in sqdata.valid:
-                continue
-            if tid in sqdata.noexec:
-                continue
-            if tid in sqrq.scenequeue_notcovered:
-                continue
-            sqdata.outrightfail.add(tid)
-
-            h = pending_hash_index(tid, rqdata)
-            if h not in sqdata.hashes:
-                sqdata.hashes[h] = tid
-            else:
-                sqrq.sq_deferred[tid] = sqdata.hashes[h]
-                bb.note("Deferring %s after %s" % (tid, sqdata.hashes[h]))
-
+    for tid in tids:
+        if tid in sqdata.stamppresent:
+            continue
+        if tid in sqdata.valid:
+            continue
+        if tid in sqdata.noexec:
+            continue
+        if tid in sqrq.scenequeue_covered:
+            continue
+        if tid in sqrq.scenequeue_notcovered:
+            continue
+        if tid in sqrq.sq_deferred:
+            continue
+        sqdata.outrightfail.add(tid)
+        logger.debug2("%s already handled (fallthrough), skipping" % (tid))
 
 class TaskFailure(Exception):
     """
@@ -2876,12 +3248,16 @@ class runQueueTaskFailed(runQueueEvent):
     """
     Event notifying a task failed
     """
-    def __init__(self, task, stats, exitcode, rq):
+    def __init__(self, task, stats, exitcode, rq, fakeroot_log=None):
         runQueueEvent.__init__(self, task, stats, rq)
         self.exitcode = exitcode
+        self.fakeroot_log = fakeroot_log
 
     def __str__(self):
-        return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode)
+        if self.fakeroot_log:
+            return "Task (%s) failed with exit code '%s' \nPseudo log:\n%s" % (self.taskstring, self.exitcode, self.fakeroot_log)
+        else:
+            return "Task (%s) failed with exit code '%s'" % (self.taskstring, self.exitcode)
 
 class sceneQueueTaskFailed(sceneQueueEvent):
     """
@@ -2933,18 +3309,16 @@ class runQueuePipe():
     """
     Abstraction for a pipe between a worker thread and the server
     """
-    def __init__(self, pipein, pipeout, d, rq, rqexec):
+    def __init__(self, pipein, pipeout, d, rq, rqexec, fakerootlogs=None):
         self.input = pipein
         if pipeout:
             pipeout.close()
         bb.utils.nonblockingfd(self.input)
-        self.queue = b""
+        self.queue = bytearray()
         self.d = d
         self.rq = rq
         self.rqexec = rqexec
-
-    def setrunqueueexec(self, rqexec):
-        self.rqexec = rqexec
+        self.fakerootlogs = fakerootlogs
 
     def read(self):
         for workers, name in [(self.rq.worker, "Worker"), (self.rq.fakeworker, "Fakeroot")]:
@@ -2956,13 +3330,13 @@ class runQueuePipe():
 
         start = len(self.queue)
         try:
-            self.queue = self.queue + (self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(512 * 1024) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise
         end = len(self.queue)
         found = True
-        while found and len(self.queue):
+        while found and self.queue:
             found = False
             index = self.queue.find(b"</event>")
             while index != -1 and self.queue.startswith(b"<event>"):
@@ -2987,7 +3361,11 @@ class runQueuePipe():
                     task, status = pickle.loads(self.queue[10:index])
                 except (ValueError, pickle.UnpicklingError, AttributeError, IndexError) as e:
                     bb.msg.fatal("RunQueue", "failed load pickle '%s': '%s'" % (e, self.queue[10:index]))
-                self.rqexec.runqueue_process_waitpid(task, status)
+                (_, _, _, taskfn) = split_tid_mcfn(task)
+                fakerootlog = None
+                if self.fakerootlogs and taskfn and taskfn in self.fakerootlogs:
+                    fakerootlog = self.fakerootlogs[taskfn]
+                self.rqexec.runqueue_process_waitpid(task, status, fakerootlog=fakerootlog)
                 found = True
                 self.queue = self.queue[index+11:]
                 index = self.queue.find(b"</exitcode>")
@@ -2996,16 +3374,16 @@ class runQueuePipe():
     def close(self):
         while self.read():
             continue
-        if len(self.queue) > 0:
+        if self.queue:
             print("Warning, worker left partial message: %s" % self.queue)
         self.input.close()
 
-def get_setscene_enforce_whitelist(d, targets):
+def get_setscene_enforce_ignore_tasks(d, targets):
     if d.getVar('BB_SETSCENE_ENFORCE') != '1':
         return None
-    whitelist = (d.getVar("BB_SETSCENE_ENFORCE_WHITELIST") or "").split()
+    ignore_tasks = (d.getVar("BB_SETSCENE_ENFORCE_IGNORE_TASKS") or "").split()
     outlist = []
-    for item in whitelist[:]:
+    for item in ignore_tasks[:]:
         if item.startswith('%:'):
             for (mc, target, task, fn) in targets:
                 outlist.append(target + ':' + item.split(':')[1])
@@ -3013,12 +3391,12 @@ def get_setscene_enforce_whitelist(d, targets):
             outlist.append(item)
     return outlist
 
-def check_setscene_enforce_whitelist(pn, taskname, whitelist):
+def check_setscene_enforce_ignore_tasks(pn, taskname, ignore_tasks):
     import fnmatch
-    if whitelist is not None:
+    if ignore_tasks is not None:
         item = '%s:%s' % (pn, taskname)
-        for whitelist_item in whitelist:
-            if fnmatch.fnmatch(item, whitelist_item):
+        for ignore_tasks in ignore_tasks:
+            if fnmatch.fnmatch(item, ignore_tasks):
                 return True
         return False
     return True
diff --git a/bitbake/lib/bb/server/process.py b/bitbake/lib/bb/server/process.py
index b27b4aefe0..4b35be62cd 100644
--- a/bitbake/lib/bb/server/process.py
+++ b/bitbake/lib/bb/server/process.py
@@ -26,6 +26,9 @@ import errno
 import re
 import datetime
 import pickle
+import traceback
+import gc
+import stat
 import bb.server.xmlrpcserver
 from bb import daemonize
 from multiprocessing import queues
@@ -35,9 +38,46 @@ logger = logging.getLogger('BitBake')
 class ProcessTimeout(SystemExit):
     pass
 
+def currenttime():
+    return datetime.datetime.now().strftime('%H:%M:%S.%f')
+
 def serverlog(msg):
-    print(str(os.getpid()) + " " +  datetime.datetime.now().strftime('%H:%M:%S.%f') + " " + msg)
-    sys.stdout.flush()
+    print(str(os.getpid()) + " " +  currenttime() + " " + msg)
+    #Seems a flush here triggers filesytem sync like behaviour and long hangs in the server
+    #sys.stdout.flush()
+
+#
+# When we have lockfile issues, try and find infomation about which process is
+# using the lockfile
+#
+def get_lockfile_process_msg(lockfile):
+    # Some systems may not have lsof available
+    procs = None
+    try:
+        procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError:
+        # File was deleted?
+        pass
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
+    if procs is None:
+        # Fall back to fuser if lsof is unavailable
+        try:
+            procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError:
+            # File was deleted?
+            pass
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
+    if procs:
+        return procs.decode("utf-8")
+    return None
+
+class idleFinish():
+    def __init__(self, msg):
+         self.msg = msg
 
 class ProcessServer():
     profile_filename = "profile.log"
@@ -56,12 +96,19 @@ class ProcessServer():
         self.maxuiwait = 30
         self.xmlrpc = False
 
+        self.idle = None
+        # Need a lock for _idlefuns changes
         self._idlefuns = {}
+        self._idlefuncsLock = threading.Lock()
+        self.idle_cond = threading.Condition(self._idlefuncsLock)
 
         self.bitbake_lock = lock
         self.bitbake_lock_name = lockname
         self.sock = sock
         self.sockname = sockname
+        # It is possible the directory may be renamed. Cache the inode of the socket file
+        # so we can tell if things changed.
+        self.sockinode = os.stat(self.sockname)[stat.ST_INO]
 
         self.server_timeout = server_timeout
         self.timeout = self.server_timeout
@@ -70,7 +117,9 @@ class ProcessServer():
     def register_idle_function(self, function, data):
         """Register a function to be called while the server is idle"""
         assert hasattr(function, '__call__')
-        self._idlefuns[function] = data
+        with bb.utils.lock_timeout(self._idlefuncsLock):
+            self._idlefuns[function] = data
+        serverlog("Registering idle function %s" % str(function))
 
     def run(self):
 
@@ -109,6 +158,31 @@ class ProcessServer():
 
         return ret
 
+    def _idle_check(self):
+        return len(self._idlefuns) == 0 and self.cooker.command.currentAsyncCommand is None
+
+    def wait_for_idle(self, timeout=30):
+        # Wait for the idle loop to have cleared
+        with bb.utils.lock_timeout(self._idlefuncsLock):
+            return self.idle_cond.wait_for(self._idle_check, timeout) is not False
+
+    def set_async_cmd(self, cmd):
+        with bb.utils.lock_timeout(self._idlefuncsLock):
+            ret = self.idle_cond.wait_for(self._idle_check, 30)
+            if ret is False:
+                return False
+            self.cooker.command.currentAsyncCommand = cmd
+            return True
+
+    def clear_async_cmd(self):
+        with bb.utils.lock_timeout(self._idlefuncsLock):
+            self.cooker.command.currentAsyncCommand = None
+            self.idle_cond.notify_all()
+
+    def get_async_cmd(self):
+        with bb.utils.lock_timeout(self._idlefuncsLock):
+            return self.cooker.command.currentAsyncCommand
+
     def main(self):
         self.cooker.pre_serve()
 
@@ -123,14 +197,19 @@ class ProcessServer():
             fds.append(self.xmlrpc)
         seendata = False
         serverlog("Entering server connection loop")
+        serverlog("Lockfile is: %s\nSocket is %s (%s)" % (self.bitbake_lock_name, self.sockname, os.path.exists(self.sockname)))
 
         def disconnect_client(self, fds):
-            serverlog("Disconnecting Client")
+            serverlog("Disconnecting Client (socket: %s)" % os.path.exists(self.sockname))
             if self.controllersock:
                 fds.remove(self.controllersock)
                 self.controllersock.close()
                 self.controllersock = False
             if self.haveui:
+                # Wait for the idle loop to have cleared (30s max)
+                if not self.wait_for_idle(30):
+                    serverlog("Idle loop didn't finish queued commands after 30s, exiting.")
+                    self.quit = True
                 fds.remove(self.command_channel)
                 bb.event.unregister_UIHhandler(self.event_handle, True)
                 self.command_channel_reply.writer.close()
@@ -142,12 +221,12 @@ class ProcessServer():
                 self.cooker.clientComplete()
                 self.haveui = False
             ready = select.select(fds,[],[],0)[0]
-            if newconnections:
+            if newconnections and not self.quit:
                 serverlog("Starting new client")
                 conn = newconnections.pop(-1)
                 fds.append(conn)
                 self.controllersock = conn
-            elif self.timeout is None and not ready:
+            elif not self.timeout and not ready:
                 serverlog("No timeout, exiting.")
                 self.quit = True
 
@@ -214,11 +293,14 @@ class ProcessServer():
                     continue
                 try:
                     serverlog("Running command %s" % command)
-                    self.command_channel_reply.send(self.cooker.command.runCommand(command))
-                    serverlog("Command Completed")
+                    reply = self.cooker.command.runCommand(command, self)
+                    serverlog("Sending reply %s" % repr(reply))
+                    self.command_channel_reply.send(reply)
+                    serverlog("Command Completed (socket: %s)" % os.path.exists(self.sockname))
                 except Exception as e:
-                   serverlog('Exception in server main event loop running command %s (%s)' % (command, str(e)))
-                   logger.exception('Exception in server main event loop running command %s (%s)' % (command, str(e)))
+                   stack = traceback.format_exc()
+                   serverlog('Exception in server main event loop running command %s (%s)' % (command, stack))
+                   logger.exception('Exception in server main event loop running command %s (%s)' % (command, stack))
 
             if self.xmlrpc in ready:
                 self.xmlrpc.handle_requests()
@@ -239,21 +321,42 @@ class ProcessServer():
                     bb.warn('Ignoring invalid BB_SERVER_TIMEOUT=%s, must be a float specifying seconds.' % self.timeout)
                 seendata = True
 
-            ready = self.idle_commands(.1, fds)
+            if not self.idle:
+                self.idle = threading.Thread(target=self.idle_thread)
+                self.idle.start()
+            elif self.idle and not self.idle.is_alive():
+                serverlog("Idle thread terminated, main thread exiting too")
+                bb.error("Idle thread terminated, main thread exiting too")
+                self.quit = True
 
-        if len(threading.enumerate()) != 1:
-            serverlog("More than one thread left?: " + str(threading.enumerate()))
+            nextsleep = 1.0
+            if self.xmlrpc:
+                nextsleep = self.xmlrpc.get_timeout(nextsleep)
+            try:
+                ready = select.select(fds,[],[],nextsleep)[0]
+            except InterruptedError:
+                # Ignore EINTR
+                ready = []
+
+        if self.idle:
+            self.idle.join()
 
-        serverlog("Exiting")
+        serverlog("Exiting (socket: %s)" % os.path.exists(self.sockname))
         # Remove the socket file so we don't get any more connections to avoid races
+        # The build directory could have been renamed so if the file isn't the one we created
+        # we shouldn't delete it.
         try:
-            os.unlink(self.sockname)
-        except:
-            pass
+            sockinode = os.stat(self.sockname)[stat.ST_INO]
+            if sockinode == self.sockinode:
+                os.unlink(self.sockname)
+            else:
+                serverlog("bitbake.sock inode mismatch (%s vs %s), not deleting." % (sockinode, self.sockinode))
+        except Exception as err:
+            serverlog("Removing socket file '%s' failed (%s)" % (self.sockname, err))
         self.sock.close()
 
         try:
-            self.cooker.shutdown(True)
+            self.cooker.shutdown(True, idle=False)
             self.cooker.notifier.stop()
             self.cooker.confignotifier.stop()
         except:
@@ -261,6 +364,9 @@ class ProcessServer():
 
         self.cooker.post_serve()
 
+        if len(threading.enumerate()) != 1:
+            serverlog("More than one thread left?: " + str(threading.enumerate()))
+
         # Flush logs before we release the lock
         sys.stdout.flush()
         sys.stderr.flush()
@@ -276,20 +382,21 @@ class ProcessServer():
             except FileNotFoundError:
                 return None
 
-        lockcontents = get_lock_contents(lockfile)
-        serverlog("Original lockfile contents: " + str(lockcontents))
-
         lock.close()
         lock = None
 
         while not lock:
             i = 0
             lock = None
+            if not os.path.exists(os.path.basename(lockfile)):
+                serverlog("Lockfile directory gone, exiting.")
+                return
+
             while not lock and i < 30:
                 lock = bb.utils.lockfile(lockfile, shared=False, retry=False, block=False)
                 if not lock:
                     newlockcontents = get_lock_contents(lockfile)
-                    if newlockcontents != lockcontents:
+                    if not newlockcontents[0].startswith([f"{os.getpid()}\n", f"{os.getpid()} "]):
                         # A new server was started, the lockfile contents changed, we can exit
                         serverlog("Lockfile now contains different contents, exiting: " + str(newlockcontents))
                         return
@@ -303,87 +410,95 @@ class ProcessServer():
                 return
 
             if not lock:
-                # Some systems may not have lsof available
-                procs = None
-                try:
-                    procs = subprocess.check_output(["lsof", '-w', lockfile], stderr=subprocess.STDOUT)
-                except subprocess.CalledProcessError:
-                    # File was deleted?
-                    continue
-                except OSError as e:
-                    if e.errno != errno.ENOENT:
-                        raise
-                if procs is None:
-                    # Fall back to fuser if lsof is unavailable
-                    try:
-                        procs = subprocess.check_output(["fuser", '-v', lockfile], stderr=subprocess.STDOUT)
-                    except subprocess.CalledProcessError:
-                        # File was deleted?
-                        continue
-                    except OSError as e:
-                        if e.errno != errno.ENOENT:
-                            raise
-
-                msg = "Delaying shutdown due to active processes which appear to be holding bitbake.lock"
+                procs = get_lockfile_process_msg(lockfile)
+                msg = ["Delaying shutdown due to active processes which appear to be holding bitbake.lock"]
                 if procs:
-                    msg += ":\n%s" % str(procs.decode("utf-8"))
-                serverlog(msg)
+                    msg.append(":\n%s" % procs)
+                serverlog("".join(msg))
 
-    def idle_commands(self, delay, fds=None):
-        nextsleep = delay
-        if not fds:
-            fds = []
-
-        for function, data in list(self._idlefuns.items()):
+    def idle_thread(self):
+        if self.cooker.configuration.profile:
             try:
-                retval = function(self, data, False)
-                if retval is False:
-                    del self._idlefuns[function]
-                    nextsleep = None
-                elif retval is True:
-                    nextsleep = None
-                elif isinstance(retval, float) and nextsleep:
-                    if (retval < nextsleep):
-                        nextsleep = retval
-                elif nextsleep is None:
-                    continue
-                else:
-                    fds = fds + retval
-            except SystemExit:
-                raise
-            except Exception as exc:
-                if not isinstance(exc, bb.BBHandledException):
-                    logger.exception('Running idle function')
-                del self._idlefuns[function]
-                self.quit = True
+                import cProfile as profile
+            except:
+                import profile
+            prof = profile.Profile()
 
-        # Create new heartbeat event?
-        now = time.time()
-        if now >= self.next_heartbeat:
-            # We might have missed heartbeats. Just trigger once in
-            # that case and continue after the usual delay.
-            self.next_heartbeat += self.heartbeat_seconds
-            if self.next_heartbeat <= now:
-                self.next_heartbeat = now + self.heartbeat_seconds
-            if hasattr(self.cooker, "data"):
-                heartbeat = bb.event.HeartbeatEvent(now)
-                bb.event.fire(heartbeat, self.cooker.data)
-        if nextsleep and now + nextsleep > self.next_heartbeat:
-            # Shorten timeout so that we we wake up in time for
-            # the heartbeat.
-            nextsleep = self.next_heartbeat - now
-
-        if nextsleep is not None:
-            if self.xmlrpc:
-                nextsleep = self.xmlrpc.get_timeout(nextsleep)
-            try:
-                return select.select(fds,[],[],nextsleep)[0]
-            except InterruptedError:
-                # Ignore EINTR
-                return []
+            ret = profile.Profile.runcall(prof, self.idle_thread_internal)
+
+            prof.dump_stats("profile-mainloop.log")
+            bb.utils.process_profilelog("profile-mainloop.log")
+            serverlog("Raw profiling information saved to profile-mainloop.log and processed statistics to profile-mainloop.log.processed")
         else:
-            return select.select(fds,[],[],0)[0]
+            self.idle_thread_internal()
+
+    def idle_thread_internal(self):
+        def remove_idle_func(function):
+            with bb.utils.lock_timeout(self._idlefuncsLock):
+                del self._idlefuns[function]
+                self.idle_cond.notify_all()
+
+        while not self.quit:
+            nextsleep = 1.0
+            fds = []
+
+            with bb.utils.lock_timeout(self._idlefuncsLock):
+                items = list(self._idlefuns.items())
 
+            for function, data in items:
+                try:
+                    retval = function(self, data, False)
+                    if isinstance(retval, idleFinish):
+                        serverlog("Removing idle function %s at idleFinish" % str(function))
+                        remove_idle_func(function)
+                        self.cooker.command.finishAsyncCommand(retval.msg)
+                        nextsleep = None
+                    elif retval is False:
+                        serverlog("Removing idle function %s" % str(function))
+                        remove_idle_func(function)
+                        nextsleep = None
+                    elif retval is True:
+                        nextsleep = None
+                    elif isinstance(retval, float) and nextsleep:
+                        if (retval < nextsleep):
+                            nextsleep = retval
+                    elif nextsleep is None:
+                        continue
+                    else:
+                        fds = fds + retval
+                except SystemExit:
+                    raise
+                except Exception as exc:
+                    if not isinstance(exc, bb.BBHandledException):
+                        logger.exception('Running idle function')
+                    remove_idle_func(function)
+                    serverlog("Exception %s broke the idle_thread, exiting" % traceback.format_exc())
+                    self.quit = True
+
+            # Create new heartbeat event?
+            now = time.time()
+            if items and bb.event._heartbeat_enabled and now >= self.next_heartbeat:
+                # We might have missed heartbeats. Just trigger once in
+                # that case and continue after the usual delay.
+                self.next_heartbeat += self.heartbeat_seconds
+                if self.next_heartbeat <= now:
+                    self.next_heartbeat = now + self.heartbeat_seconds
+                if hasattr(self.cooker, "data"):
+                    heartbeat = bb.event.HeartbeatEvent(now)
+                    try:
+                        bb.event.fire(heartbeat, self.cooker.data)
+                    except Exception as exc:
+                        if not isinstance(exc, bb.BBHandledException):
+                            logger.exception('Running heartbeat function')
+                        serverlog("Exception %s broke in idle_thread, exiting" % traceback.format_exc())
+                        self.quit = True
+            if nextsleep and bb.event._heartbeat_enabled and now + nextsleep > self.next_heartbeat:
+                # Shorten timeout so that we we wake up in time for
+                # the heartbeat.
+                nextsleep = self.next_heartbeat - now
+
+            if nextsleep is not None:
+                select.select(fds,[],[],nextsleep)[0]
 
 class ServerCommunicator():
     def __init__(self, connection, recv):
@@ -391,12 +506,18 @@ class ServerCommunicator():
         self.recv = recv
 
     def runCommand(self, command):
-        self.connection.send(command)
+        try:
+            self.connection.send(command)
+        except BrokenPipeError as e:
+            raise BrokenPipeError("bitbake-server might have died or been forcibly stopped, ie. OOM killed") from e
         if not self.recv.poll(30):
-            logger.info("No reply from server in 30s")
+            logger.info("No reply from server in 30s (for command %s at %s)" % (command[0], currenttime()))
             if not self.recv.poll(30):
-                raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s)")
-        ret, exc = self.recv.get()
+                raise ProcessTimeout("Timeout while waiting for a reply from the bitbake server (60s at %s)" % currenttime())
+        try:
+            ret, exc = self.recv.get()
+        except EOFError as e:
+            raise EOFError("bitbake-server might have died or been forcibly stopped, ie. OOM killed") from e
         # Should probably turn all exceptions in exc back into exceptions?
         # For now, at least handle BBHandledException
         if exc and ("BBHandledException" in exc or "SystemExit" in exc):
@@ -429,6 +550,7 @@ class BitBakeProcessServerConnection(object):
         self.socket_connection = sock
 
     def terminate(self):
+        self.events.close()
         self.socket_connection.close()
         self.connection.connection.close()
         self.connection.recv.close()
@@ -439,13 +561,14 @@ start_log_datetime_format = '%Y-%m-%d %H:%M:%S.%f'
 
 class BitBakeServer(object):
 
-    def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface):
+    def __init__(self, lock, sockname, featureset, server_timeout, xmlrpcinterface, profile):
 
         self.server_timeout = server_timeout
         self.xmlrpcinterface = xmlrpcinterface
         self.featureset = featureset
         self.sockname = sockname
         self.bitbake_lock = lock
+        self.profile = profile
         self.readypipe, self.readypipein = os.pipe()
 
         # Place the log in the builddirectory alongside the lock file
@@ -466,7 +589,7 @@ class BitBakeServer(object):
             try:
                 r = ready.get()
             except EOFError:
-                # Trap the child exitting/closing the pipe and error out
+                # Trap the child exiting/closing the pipe and error out
                 r = None
         if not r or r[0] != "r":
             ready.close()
@@ -509,9 +632,9 @@ class BitBakeServer(object):
         os.set_inheritable(self.bitbake_lock.fileno(), True)
         os.set_inheritable(self.readypipein, True)
         serverscript = os.path.realpath(os.path.dirname(__file__) + "/../../../bin/bitbake-server")
-        os.execl(sys.executable, "bitbake-server", serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
+        os.execl(sys.executable, sys.executable, serverscript, "decafbad", str(self.bitbake_lock.fileno()), str(self.readypipein), self.logfile, self.bitbake_lock.name, self.sockname,  str(self.server_timeout or 0), str(int(self.profile)), str(self.xmlrpcinterface[0]), str(self.xmlrpcinterface[1]))
 
-def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface):
+def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpcinterface, profile):
 
     import bb.cookerdata
     import bb.cooker
@@ -523,6 +646,7 @@ def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpc
 
         # Create server control socket
         if os.path.exists(sockname):
+            serverlog("WARNING: removing existing socket file '%s'" % sockname)
             os.unlink(sockname)
 
         sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
@@ -539,7 +663,8 @@ def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpc
         writer = ConnectionWriter(readypipeinfd)
         try:
             featureset = []
-            cooker = bb.cooker.BBCooker(featureset, server.register_idle_function)
+            cooker = bb.cooker.BBCooker(featureset, server)
+            cooker.configuration.profile = profile
         except bb.BBHandledException:
             return None
         writer.send("r")
@@ -549,7 +674,7 @@ def execServer(lockfd, readypipeinfd, lockname, sockname, server_timeout, xmlrpc
 
         server.run()
     finally:
-        # Flush any ,essages/errors to the logfile before exit
+        # Flush any messages/errors to the logfile before exit
         sys.stdout.flush()
         sys.stderr.flush()
 
@@ -654,23 +779,18 @@ class BBUIEventQueue:
         self.reader = ConnectionReader(readfd)
 
         self.t = threading.Thread()
-        self.t.setDaemon(True)
         self.t.run = self.startCallbackHandler
         self.t.start()
 
     def getEvent(self):
-        self.eventQueueLock.acquire()
-
-        if len(self.eventQueue) == 0:
-            self.eventQueueLock.release()
-            return None
-
-        item = self.eventQueue.pop(0)
+        with bb.utils.lock_timeout(self.eventQueueLock):
+            if len(self.eventQueue) == 0:
+                return None
 
-        if len(self.eventQueue) == 0:
-            self.eventQueueNotify.clear()
+            item = self.eventQueue.pop(0)
+            if len(self.eventQueue) == 0:
+                self.eventQueueNotify.clear()
 
-        self.eventQueueLock.release()
         return item
 
     def waitEvent(self, delay):
@@ -678,10 +798,9 @@ class BBUIEventQueue:
         return self.getEvent()
 
     def queue_event(self, event):
-        self.eventQueueLock.acquire()
-        self.eventQueue.append(event)
-        self.eventQueueNotify.set()
-        self.eventQueueLock.release()
+        with bb.utils.lock_timeout(self.eventQueueLock):
+            self.eventQueue.append(event)
+            self.eventQueueNotify.set()
 
     def send_event(self, event):
         self.queue_event(pickle.loads(event))
@@ -690,13 +809,17 @@ class BBUIEventQueue:
         bb.utils.set_process_name("UIEventQueue")
         while True:
             try:
-                self.reader.wait()
-                event = self.reader.get()
-                self.queue_event(event)
-            except EOFError:
+                ready = self.reader.wait(0.25)
+                if ready:
+                    event = self.reader.get()
+                    self.queue_event(event)
+            except (EOFError, OSError, TypeError):
                 # Easiest way to exit is to close the file descriptor to cause an exit
                 break
+
+    def close(self):
         self.reader.close()
+        self.t.join()
 
 class ConnectionReader(object):
 
@@ -711,7 +834,7 @@ class ConnectionReader(object):
         return self.reader.poll(timeout)
 
     def get(self):
-        with self.rlock:
+        with bb.utils.lock_timeout(self.rlock):
             res = self.reader.recv_bytes()
         return multiprocessing.reduction.ForkingPickler.loads(res)
 
@@ -730,10 +853,31 @@ class ConnectionWriter(object):
         # Why bb.event needs this I have no idea
         self.event = self
 
+    def _send(self, obj):
+        gc.disable()
+        with bb.utils.lock_timeout(self.wlock):
+            self.writer.send_bytes(obj)
+        gc.enable()
+
     def send(self, obj):
         obj = multiprocessing.reduction.ForkingPickler.dumps(obj)
-        with self.wlock:
-            self.writer.send_bytes(obj)
+        # See notes/code in CookerParser
+        # We must not terminate holding this lock else processes will hang.
+        # For SIGTERM, raising afterwards avoids this.
+        # For SIGINT, we don't want to have written partial data to the pipe.
+        # pthread_sigmask block/unblock would be nice but doesn't work, https://bugs.python.org/issue47139
+        process = multiprocessing.current_process()
+        if process and hasattr(process, "queue_signals"):
+            with bb.utils.lock_timeout(process.signal_threadlock):
+                process.queue_signals = True
+                self._send(obj)
+                process.queue_signals = False
+
+                while len(process.signal_received) > 0:
+                    sig = process.signal_received.pop()
+                    process.handle_sig(sig, None)
+        else:
+            self._send(obj)
 
     def fileno(self):
         return self.writer.fileno()
diff --git a/bitbake/lib/bb/server/xmlrpcserver.py b/bitbake/lib/bb/server/xmlrpcserver.py
index 2fa71be667..ebc271aca4 100644
--- a/bitbake/lib/bb/server/xmlrpcserver.py
+++ b/bitbake/lib/bb/server/xmlrpcserver.py
@@ -11,8 +11,11 @@ import hashlib
 import time
 import inspect
 from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
+import bb.server.xmlrpcclient
 
 import bb
+import bb.cooker
+import bb.event
 
 # This request handler checks if the request has a "Bitbake-token" header
 # field (this comes from the client side) and compares it with its internal
@@ -53,7 +56,7 @@ class BitBakeXMLRPCServer(SimpleXMLRPCServer):
 
     def __init__(self, interface, cooker, parent):
         # Use auto port configuration
-        if (interface[1] == -1):
+        if interface[1] == -1:
             interface = (interface[0], 0)
         SimpleXMLRPCServer.__init__(self, interface,
                                     requestHandler=BitBakeXMLRPCRequestHandler,
@@ -86,11 +89,12 @@ class BitBakeXMLRPCServer(SimpleXMLRPCServer):
     def handle_requests(self):
         self._handle_request_noblock()
 
-class BitBakeXMLRPCServerCommands():
+class BitBakeXMLRPCServerCommands:
 
     def __init__(self, server):
         self.server = server
         self.has_client = False
+        self.event_handle = None
 
     def registerEventHandler(self, host, port):
         """
@@ -99,8 +103,8 @@ class BitBakeXMLRPCServerCommands():
         s, t = bb.server.xmlrpcclient._create_server(host, port)
 
         # we don't allow connections if the cooker is running
-        if (self.server.cooker.state in [bb.cooker.state.parsing, bb.cooker.state.running]):
-            return None, "Cooker is busy: %s" % bb.cooker.state.get_name(self.server.cooker.state)
+        if self.server.cooker.state in [bb.cooker.State.PARSING, bb.cooker.State.RUNNING]:
+            return None, f"Cooker is busy: {self.server.cooker.state.name}"
 
         self.event_handle = bb.event.register_UIHhandler(s, True)
         return self.event_handle, 'OK'
@@ -117,7 +121,7 @@ class BitBakeXMLRPCServerCommands():
         """
         Run a cooker command on the server
         """
-        return self.server.cooker.command.runCommand(command, self.server.readonly)
+        return self.server.cooker.command.runCommand(command, self.server.parent, self.server.readonly)
 
     def getEventHandle(self):
         return self.event_handle
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 0d88c6ec68..a6163b55ea 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -11,6 +13,10 @@ import pickle
 import bb.data
 import difflib
 import simplediff
+import json
+import types
+from contextlib import contextmanager
+import bb.compress.zstd
 from bb.checksum import FileChecksumCache
 from bb import runqueue
 import hashserv
@@ -19,6 +25,35 @@ import hashserv.client
 logger = logging.getLogger('BitBake.SigGen')
 hashequiv_logger = logging.getLogger('BitBake.SigGen.HashEquiv')
 
+#find_siginfo and find_siginfo_version are set by the metadata siggen
+# The minimum version of the find_siginfo function we need
+find_siginfo_minversion = 2
+
+HASHSERV_ENVVARS = [
+    "SSL_CERT_DIR",
+    "SSL_CERT_FILE",
+    "NO_PROXY",
+    "HTTPS_PROXY",
+    "HTTP_PROXY"
+]
+
+def check_siggen_version(siggen):
+    if not hasattr(siggen, "find_siginfo_version"):
+        bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (no version found)")
+    if siggen.find_siginfo_version < siggen.find_siginfo_minversion:
+        bb.fatal("Siggen from metadata (OE-Core?) is too old, please update it (%s vs %s)" % (siggen.find_siginfo_version, siggen.find_siginfo_minversion))
+
+class SetEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, set) or isinstance(obj, frozenset):
+            return dict(_set_object=list(sorted(obj)))
+        return json.JSONEncoder.default(self, obj)
+
+def SetDecoder(dct):
+    if '_set_object' in dct:
+        return frozenset(dct['_set_object'])
+    return dct
+
 def init(d):
     siggens = [obj for obj in globals().values()
                       if type(obj) is type and issubclass(obj, SignatureGenerator)]
@@ -27,7 +62,6 @@ def init(d):
     for sg in siggens:
         if desired == sg.name:
             return sg(d)
-            break
     else:
         logger.error("Invalid signature generator '%s', using default 'noop'\n"
                      "Available generators: %s", desired,
@@ -39,11 +73,6 @@ class SignatureGenerator(object):
     """
     name = "noop"
 
-    # If the derived class supports multiconfig datacaches, set this to True
-    # The default is False for backward compatibility with derived signature
-    # generators that do not understand multiconfig caches
-    supports_multiconfig_datacaches = False
-
     def __init__(self, data):
         self.basehash = {}
         self.taskhash = {}
@@ -61,9 +90,39 @@ class SignatureGenerator(object):
     def postparsing_clean_cache(self):
         return
 
+    def setup_datacache(self, datacaches):
+        self.datacaches = datacaches
+
+    def setup_datacache_from_datastore(self, mcfn, d):
+        # In task context we have no cache so setup internal data structures
+        # from the fully parsed data store provided
+
+        mc = d.getVar("__BBMULTICONFIG", False) or ""
+        tasks = d.getVar('__BBTASKS', False)
+
+        self.datacaches = {}
+        self.datacaches[mc] = types.SimpleNamespace()
+        setattr(self.datacaches[mc], "stamp", {})
+        self.datacaches[mc].stamp[mcfn] = d.getVar('STAMP')
+        setattr(self.datacaches[mc], "stamp_extrainfo", {})
+        self.datacaches[mc].stamp_extrainfo[mcfn] = {}
+        for t in tasks:
+            flag = d.getVarFlag(t, "stamp-extra-info")
+            if flag:
+                self.datacaches[mc].stamp_extrainfo[mcfn][t] = flag
+
+    def get_cached_unihash(self, tid):
+        return None
+
     def get_unihash(self, tid):
+        unihash = self.get_cached_unihash(tid)
+        if unihash:
+            return unihash
         return self.taskhash[tid]
 
+    def get_unihashes(self, tids):
+        return {tid: self.get_unihash(tid) for tid in tids}
+
     def prep_taskhash(self, tid, deps, dataCaches):
         return
 
@@ -75,17 +134,51 @@ class SignatureGenerator(object):
         """Write/update the file checksum cache onto disk"""
         return
 
+    def stampfile_base(self, mcfn):
+        mc = bb.runqueue.mc_from_tid(mcfn)
+        return self.datacaches[mc].stamp[mcfn]
+
+    def stampfile_mcfn(self, taskname, mcfn, extrainfo=True):
+        mc = bb.runqueue.mc_from_tid(mcfn)
+        stamp = self.datacaches[mc].stamp[mcfn]
+        if not stamp:
+            return
+
+        stamp_extrainfo = ""
+        if extrainfo:
+            taskflagname = taskname
+            if taskname.endswith("_setscene"):
+                taskflagname = taskname.replace("_setscene", "")
+            stamp_extrainfo = self.datacaches[mc].stamp_extrainfo[mcfn].get(taskflagname) or ""
+
+        return self.stampfile(stamp, mcfn, taskname, stamp_extrainfo)
+
     def stampfile(self, stampbase, file_name, taskname, extrainfo):
         return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
 
+    def stampcleanmask_mcfn(self, taskname, mcfn):
+        mc = bb.runqueue.mc_from_tid(mcfn)
+        stamp = self.datacaches[mc].stamp[mcfn]
+        if not stamp:
+            return []
+
+        taskflagname = taskname
+        if taskname.endswith("_setscene"):
+            taskflagname = taskname.replace("_setscene", "")
+        stamp_extrainfo = self.datacaches[mc].stamp_extrainfo[mcfn].get(taskflagname) or ""
+
+        return self.stampcleanmask(stamp, mcfn, taskname, stamp_extrainfo)
+
     def stampcleanmask(self, stampbase, file_name, taskname, extrainfo):
         return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
 
-    def dump_sigtask(self, fn, task, stampbase, runtime):
+    def dump_sigtask(self, mcfn, task, stampbase, runtime):
         return
 
-    def invalidate_task(self, task, d, fn):
-        bb.build.del_stamp(task, d, fn)
+    def invalidate_task(self, task, mcfn):
+        mc = bb.runqueue.mc_from_tid(mcfn)
+        stamp = self.datacaches[mc].stamp[mcfn]
+        bb.utils.remove(stamp)
 
     def dump_sigs(self, dataCache, options):
         return
@@ -111,37 +204,13 @@ class SignatureGenerator(object):
     def set_setscene_tasks(self, setscene_tasks):
         return
 
-    @classmethod
-    def get_data_caches(cls, dataCaches, mc):
-        """
-        This function returns the datacaches that should be passed to signature
-        generator functions. If the signature generator supports multiconfig
-        caches, the entire dictionary of data caches is sent, otherwise a
-        special proxy is sent that support both index access to all
-        multiconfigs, and also direct access for the default multiconfig.
-
-        The proxy class allows code in this class itself to always use
-        multiconfig aware code (to ease maintenance), but derived classes that
-        are unaware of multiconfig data caches can still access the default
-        multiconfig as expected.
-
-        Do not override this function in derived classes; it will be removed in
-        the future when support for multiconfig data caches is mandatory
-        """
-        class DataCacheProxy(object):
-            def __init__(self):
-                pass
-
-            def __getitem__(self, key):
-                return dataCaches[key]
-
-            def __getattr__(self, name):
-                return getattr(dataCaches[mc], name)
-
-        if cls.supports_multiconfig_datacaches:
-            return dataCaches
+    def exit(self):
+        return
 
-        return DataCacheProxy()
+def build_pnid(mc, pn, taskname):
+    if mc:
+        return "mc:" + mc + ":" + pn + ":" + taskname
+    return pn + ":" + taskname
 
 class SignatureGeneratorBasic(SignatureGenerator):
     """
@@ -152,15 +221,12 @@ class SignatureGeneratorBasic(SignatureGenerator):
         self.basehash = {}
         self.taskhash = {}
         self.unihash = {}
-        self.taskdeps = {}
         self.runtaskdeps = {}
         self.file_checksum_values = {}
         self.taints = {}
-        self.gendeps = {}
-        self.lookupcache = {}
         self.setscenetasks = set()
-        self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST") or "").split())
-        self.taskwhitelist = None
+        self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS") or "").split())
+        self.taskhash_ignore_tasks = None
         self.init_rundepcheck(data)
         checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
         if checksum_cache_file:
@@ -175,21 +241,21 @@ class SignatureGeneratorBasic(SignatureGenerator):
         self.tidtopn = {}
 
     def init_rundepcheck(self, data):
-        self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST") or None
-        if self.taskwhitelist:
-            self.twl = re.compile(self.taskwhitelist)
+        self.taskhash_ignore_tasks = data.getVar("BB_TASKHASH_IGNORE_TASKS") or None
+        if self.taskhash_ignore_tasks:
+            self.twl = re.compile(self.taskhash_ignore_tasks)
         else:
             self.twl = None
 
-    def _build_data(self, fn, d):
+    def _build_data(self, mcfn, d):
 
         ignore_mismatch = ((d.getVar("BB_HASH_IGNORE_MISMATCH") or '') == '1')
-        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basewhitelist)
+        tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d, self.basehash_ignore_vars)
 
-        taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basewhitelist, fn)
+        taskdeps, basehash = bb.data.generate_dependency_hash(tasklist, gendeps, lookupcache, self.basehash_ignore_vars, mcfn)
 
         for task in tasklist:
-            tid = fn + ":" + task
+            tid = mcfn + ":" + task
             if not ignore_mismatch and tid in self.basehash and self.basehash[tid] != basehash[tid]:
                 bb.error("When reparsing %s, the basehash value changed from %s to %s. The metadata is not deterministic and this needs to be fixed." % (tid, self.basehash[tid], basehash[tid]))
                 bb.error("The following commands may help:")
@@ -200,11 +266,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 bb.error("%s -Sprintdiff\n" % cmd)
             self.basehash[tid] = basehash[tid]
 
-        self.taskdeps[fn] = taskdeps
-        self.gendeps[fn] = gendeps
-        self.lookupcache[fn] = lookupcache
-
-        return taskdeps
+        return taskdeps, gendeps, lookupcache
 
     def set_setscene_tasks(self, setscene_tasks):
         self.setscenetasks = set(setscene_tasks)
@@ -212,35 +274,47 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def finalise(self, fn, d, variant):
 
         mc = d.getVar("__BBMULTICONFIG", False) or ""
+        mcfn = fn
         if variant or mc:
-            fn = bb.cache.realfn2virtual(fn, variant, mc)
+            mcfn = bb.cache.realfn2virtual(fn, variant, mc)
 
         try:
-            taskdeps = self._build_data(fn, d)
+            taskdeps, gendeps, lookupcache = self._build_data(mcfn, d)
         except bb.parse.SkipRecipe:
             raise
         except:
-            bb.warn("Error during finalise of %s" % fn)
+            bb.warn("Error during finalise of %s" % mcfn)
             raise
 
+        basehashes = {}
+        for task in taskdeps:
+            basehashes[task] = self.basehash[mcfn + ":" + task]
+
+        d.setVar("__siggen_basehashes", basehashes)
+        d.setVar("__siggen_gendeps", gendeps)
+        d.setVar("__siggen_varvals", lookupcache)
+        d.setVar("__siggen_taskdeps", taskdeps)
+
         #Slow but can be useful for debugging mismatched basehashes
-        #for task in self.taskdeps[fn]:
-        #    self.dump_sigtask(fn, task, d.getVar("STAMP"), False)
+        #self.setup_datacache_from_datastore(mcfn, d)
+        #for task in taskdeps:
+        #    self.dump_sigtask(mcfn, task, d.getVar("STAMP"), False)
 
-        for task in taskdeps:
-            d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + ":" + task])
+    def setup_datacache_from_datastore(self, mcfn, d):
+        super().setup_datacache_from_datastore(mcfn, d)
 
-    def postparsing_clean_cache(self):
-        #
-        # After parsing we can remove some things from memory to reduce our memory footprint
-        #
-        self.gendeps = {}
-        self.lookupcache = {}
-        self.taskdeps = {}
+        mc = bb.runqueue.mc_from_tid(mcfn)
+        for attr in ["siggen_varvals", "siggen_taskdeps", "siggen_gendeps"]:
+            if not hasattr(self.datacaches[mc], attr):
+                setattr(self.datacaches[mc], attr, {})
+        self.datacaches[mc].siggen_varvals[mcfn] = d.getVar("__siggen_varvals")
+        self.datacaches[mc].siggen_taskdeps[mcfn] = d.getVar("__siggen_taskdeps")
+        self.datacaches[mc].siggen_gendeps[mcfn] = d.getVar("__siggen_gendeps")
 
     def rundep_check(self, fn, recipename, task, dep, depname, dataCaches):
         # Return True if we should keep the dependency, False to drop it
-        # We only manipulate the dependencies for packages not in the whitelist
+        # We only manipulate the dependencies for packages not in the ignore
+        # list
         if self.twl and not self.twl.search(recipename):
             # then process the actual dependencies
             if self.twl.search(depname):
@@ -258,38 +332,37 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
     def prep_taskhash(self, tid, deps, dataCaches):
 
-        (mc, _, task, fn) = bb.runqueue.split_tid_mcfn(tid)
+        (mc, _, task, mcfn) = bb.runqueue.split_tid_mcfn(tid)
 
         self.basehash[tid] = dataCaches[mc].basetaskhash[tid]
         self.runtaskdeps[tid] = []
         self.file_checksum_values[tid] = []
-        recipename = dataCaches[mc].pkg_fn[fn]
+        recipename = dataCaches[mc].pkg_fn[mcfn]
 
         self.tidtopn[tid] = recipename
+        # save hashfn for deps into siginfo?
+        for dep in deps:
+            (depmc, _, deptask, depmcfn) = bb.runqueue.split_tid_mcfn(dep)
+            dep_pn = dataCaches[depmc].pkg_fn[depmcfn]
 
-        for dep in sorted(deps, key=clean_basepath):
-            (depmc, _, _, depmcfn) = bb.runqueue.split_tid_mcfn(dep)
-            depname = dataCaches[depmc].pkg_fn[depmcfn]
-            if not self.supports_multiconfig_datacaches and mc != depmc:
-                # If the signature generator doesn't understand multiconfig
-                # data caches, any dependency not in the same multiconfig must
-                # be skipped for backward compatibility
-                continue
-            if not self.rundep_check(fn, recipename, task, dep, depname, dataCaches):
+            if not self.rundep_check(mcfn, recipename, task, dep, dep_pn, dataCaches):
                 continue
+
             if dep not in self.taskhash:
                 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?" % dep)
-            self.runtaskdeps[tid].append(dep)
 
-        if task in dataCaches[mc].file_checksums[fn]:
+            dep_pnid = build_pnid(depmc, dep_pn, deptask)
+            self.runtaskdeps[tid].append((dep_pnid, dep))
+
+        if task in dataCaches[mc].file_checksums[mcfn]:
             if self.checksum_cache:
-                checksums = self.checksum_cache.get_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude)
+                checksums = self.checksum_cache.get_checksums(dataCaches[mc].file_checksums[mcfn][task], recipename, self.localdirsexclude)
             else:
-                checksums = bb.fetch2.get_file_checksums(dataCaches[mc].file_checksums[fn][task], recipename, self.localdirsexclude)
+                checksums = bb.fetch2.get_file_checksums(dataCaches[mc].file_checksums[mcfn][task], recipename, self.localdirsexclude)
             for (f,cs) in checksums:
                 self.file_checksum_values[tid].append((f,cs))
 
-        taskdep = dataCaches[mc].task_deps[fn]
+        taskdep = dataCaches[mc].task_deps[mcfn]
         if 'nostamp' in taskdep and task in taskdep['nostamp']:
             # Nostamp tasks need an implicit taint so that they force any dependent tasks to run
             if tid in self.taints and self.taints[tid].startswith("nostamp:"):
@@ -300,32 +373,34 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 taint = str(uuid.uuid4())
                 self.taints[tid] = "nostamp:" + taint
 
-        taint = self.read_taint(fn, task, dataCaches[mc].stamp[fn])
+        taint = self.read_taint(mcfn, task, dataCaches[mc].stamp[mcfn])
         if taint:
             self.taints[tid] = taint
             logger.warning("%s is tainted from a forced run" % tid)
 
-        return
+        return set(dep for _, dep in self.runtaskdeps[tid])
 
     def get_taskhash(self, tid, deps, dataCaches):
 
         data = self.basehash[tid]
-        for dep in self.runtaskdeps[tid]:
-            data = data + self.get_unihash(dep)
+        for dep in sorted(self.runtaskdeps[tid]):
+            data += self.get_unihash(dep[1])
 
-        for (f, cs) in self.file_checksum_values[tid]:
+        for (f, cs) in sorted(self.file_checksum_values[tid], key=clean_checksum_file_path):
             if cs:
-                data = data + cs
+                if "/./" in f:
+                    data += "./" + f.split("/./")[1]
+                data += cs
 
         if tid in self.taints:
             if self.taints[tid].startswith("nostamp:"):
-                data = data + self.taints[tid][8:]
+                data += self.taints[tid][8:]
             else:
-                data = data + self.taints[tid]
+                data += self.taints[tid]
 
         h = hashlib.sha256(data.encode("utf-8")).hexdigest()
         self.taskhash[tid] = h
-        #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
+        #d.setVar("BB_TASKHASH:task-%s" % task, taskhash[task])
         return h
 
     def writeout_file_checksum_cache(self):
@@ -340,9 +415,9 @@ class SignatureGeneratorBasic(SignatureGenerator):
     def save_unitaskhashes(self):
         self.unihash_cache.save(self.unitaskhashes)
 
-    def dump_sigtask(self, fn, task, stampbase, runtime):
-
-        tid = fn + ":" + task
+    def dump_sigtask(self, mcfn, task, stampbase, runtime):
+        tid = mcfn + ":" + task
+        mc = bb.runqueue.mc_from_tid(mcfn)
         referencestamp = stampbase
         if isinstance(runtime, str) and runtime.startswith("customfile"):
             sigfile = stampbase
@@ -357,29 +432,34 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
         data = {}
         data['task'] = task
-        data['basewhitelist'] = self.basewhitelist
-        data['taskwhitelist'] = self.taskwhitelist
-        data['taskdeps'] = self.taskdeps[fn][task]
+        data['basehash_ignore_vars'] = self.basehash_ignore_vars
+        data['taskhash_ignore_tasks'] = self.taskhash_ignore_tasks
+        data['taskdeps'] = self.datacaches[mc].siggen_taskdeps[mcfn][task]
         data['basehash'] = self.basehash[tid]
         data['gendeps'] = {}
         data['varvals'] = {}
-        data['varvals'][task] = self.lookupcache[fn][task]
-        for dep in self.taskdeps[fn][task]:
-            if dep in self.basewhitelist:
+        data['varvals'][task] = self.datacaches[mc].siggen_varvals[mcfn][task]
+        for dep in self.datacaches[mc].siggen_taskdeps[mcfn][task]:
+            if dep in self.basehash_ignore_vars:
                 continue
-            data['gendeps'][dep] = self.gendeps[fn][dep]
-            data['varvals'][dep] = self.lookupcache[fn][dep]
+            data['gendeps'][dep] = self.datacaches[mc].siggen_gendeps[mcfn][dep]
+            data['varvals'][dep] = self.datacaches[mc].siggen_varvals[mcfn][dep]
 
         if runtime and tid in self.taskhash:
-            data['runtaskdeps'] = self.runtaskdeps[tid]
-            data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[tid]]
+            data['runtaskdeps'] = [dep[0] for dep in sorted(self.runtaskdeps[tid])]
+            data['file_checksum_values'] = []
+            for f,cs in sorted(self.file_checksum_values[tid], key=clean_checksum_file_path):
+                if "/./" in f:
+                    data['file_checksum_values'].append(("./" + f.split("/./")[1], cs))
+                else:
+                    data['file_checksum_values'].append((os.path.basename(f), cs))
             data['runtaskhashes'] = {}
-            for dep in data['runtaskdeps']:
-                data['runtaskhashes'][dep] = self.get_unihash(dep)
+            for dep in self.runtaskdeps[tid]:
+                data['runtaskhashes'][dep[0]] = self.get_unihash(dep[1])
             data['taskhash'] = self.taskhash[tid]
             data['unihash'] = self.get_unihash(tid)
 
-        taint = self.read_taint(fn, task, referencestamp)
+        taint = self.read_taint(mcfn, task, referencestamp)
         if taint:
             data['taint'] = taint
 
@@ -396,13 +476,13 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 bb.error("Taskhash mismatch %s versus %s for %s" % (computed_taskhash, self.taskhash[tid], tid))
                 sigfile = sigfile.replace(self.taskhash[tid], computed_taskhash)
 
-        fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
+        fd, tmpfile = bb.utils.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
         try:
-            with os.fdopen(fd, "wb") as stream:
-                p = pickle.dump(data, stream, -1)
-                stream.flush()
+            with bb.compress.zstd.open(fd, "wt", encoding="utf-8", num_threads=1) as f:
+                json.dump(data, f, sort_keys=True, separators=(",", ":"), cls=SetEncoder)
+                f.flush()
             os.chmod(tmpfile, 0o664)
-            os.rename(tmpfile, sigfile)
+            bb.utils.rename(tmpfile, sigfile)
         except (OSError, IOError) as err:
             try:
                 os.unlink(tmpfile)
@@ -410,18 +490,6 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 pass
             raise err
 
-    def dump_sigfn(self, fn, dataCaches, options):
-        if fn in self.taskdeps:
-            for task in self.taskdeps[fn]:
-                tid = fn + ":" + task
-                mc = bb.runqueue.mc_from_tid(tid)
-                if tid not in self.taskhash:
-                    continue
-                if dataCaches[mc].basetaskhash[tid] != self.basehash[tid]:
-                    bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % tid)
-                    bb.error("The mismatched hashes were %s and %s" % (dataCaches[mc].basetaskhash[tid], self.basehash[tid]))
-                self.dump_sigtask(fn, task, dataCaches[mc].stamp[fn], True)
-
 class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
     name = "basichash"
 
@@ -432,11 +500,11 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
         # If task is not in basehash, then error
         return self.basehash[tid]
 
-    def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False):
-        if taskname != "do_setscene" and taskname.endswith("_setscene"):
-            tid = fn + ":" + taskname[:-9]
+    def stampfile(self, stampbase, mcfn, taskname, extrainfo, clean=False):
+        if taskname.endswith("_setscene"):
+            tid = mcfn + ":" + taskname[:-9]
         else:
-            tid = fn + ":" + taskname
+            tid = mcfn + ":" + taskname
         if clean:
             h = "*"
         else:
@@ -444,29 +512,99 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
 
         return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.')
 
-    def stampcleanmask(self, stampbase, fn, taskname, extrainfo):
-        return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True)
+    def stampcleanmask(self, stampbase, mcfn, taskname, extrainfo):
+        return self.stampfile(stampbase, mcfn, taskname, extrainfo, clean=True)
+
+    def invalidate_task(self, task, mcfn):
+        bb.note("Tainting hash to force rebuild of task %s, %s" % (mcfn, task))
 
-    def invalidate_task(self, task, d, fn):
-        bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
-        bb.build.write_taint(task, d, fn)
+        mc = bb.runqueue.mc_from_tid(mcfn)
+        stamp = self.datacaches[mc].stamp[mcfn]
+
+        taintfn = stamp + '.' + task + '.taint'
+
+        import uuid
+        bb.utils.mkdirhier(os.path.dirname(taintfn))
+        # The specific content of the taint file is not really important,
+        # we just need it to be random, so a random UUID is used
+        with open(taintfn, 'w') as taintf:
+            taintf.write(str(uuid.uuid4()))
 
 class SignatureGeneratorUniHashMixIn(object):
     def __init__(self, data):
         self.extramethod = {}
+        # NOTE: The cache only tracks hashes that exist. Hashes that don't
+        # exist are always queried from the server since it is possible for
+        # hashes to appear over time, but much less likely for them to
+        # disappear
+        self.unihash_exists_cache = set()
+        self.username = None
+        self.password = None
+        self.env = {}
+
+        origenv = data.getVar("BB_ORIGENV")
+        for e in HASHSERV_ENVVARS:
+            value = data.getVar(e)
+            if not value and origenv:
+                value = origenv.getVar(e)
+            if value:
+                self.env[e] = value
         super().__init__(data)
 
     def get_taskdata(self):
-        return (self.server, self.method, self.extramethod) + super().get_taskdata()
+        return (self.server, self.method, self.extramethod, self.username, self.password, self.env) + super().get_taskdata()
 
     def set_taskdata(self, data):
-        self.server, self.method, self.extramethod = data[:3]
-        super().set_taskdata(data[3:])
+        self.server, self.method, self.extramethod, self.username, self.password, self.env = data[:6]
+        super().set_taskdata(data[6:])
+
+    def get_hashserv_creds(self):
+        if self.username and self.password:
+            return {
+                "username": self.username,
+                "password": self.password,
+            }
+
+        return {}
+
+    @contextmanager
+    def _client_env(self):
+        orig_env = os.environ.copy()
+        try:
+            for k, v in self.env.items():
+                os.environ[k] = v
+
+            yield
+        finally:
+            for k, v in self.env.items():
+                if k in orig_env:
+                    os.environ[k] = orig_env[k]
+                else:
+                    del os.environ[k]
 
+    @contextmanager
     def client(self):
-        if getattr(self, '_client', None) is None:
-            self._client = hashserv.create_client(self.server)
-        return self._client
+        with self._client_env():
+            if getattr(self, '_client', None) is None:
+                self._client = hashserv.create_client(self.server, **self.get_hashserv_creds())
+            yield self._client
+
+    def reset(self, data):
+        self.__close_clients()
+        return super().reset(data)
+
+    def exit(self):
+        self.__close_clients()
+        return super().exit()
+
+    def __close_clients(self):
+        with self._client_env():
+            if getattr(self, '_client', None) is not None:
+                self._client.close()
+                self._client = None
+            if getattr(self, '_client_pool', None) is not None:
+                self._client_pool.close()
+                self._client_pool = None
 
     def get_stampfile_hash(self, tid):
         if tid in self.taskhash:
@@ -499,7 +637,7 @@ class SignatureGeneratorUniHashMixIn(object):
             return None
         return unihash
 
-    def get_unihash(self, tid):
+    def get_cached_unihash(self, tid):
         taskhash = self.taskhash[tid]
 
         # If its not a setscene task we can return
@@ -514,40 +652,96 @@ class SignatureGeneratorUniHashMixIn(object):
             self.unihash[tid] = unihash
             return unihash
 
-        # In the absence of being able to discover a unique hash from the
-        # server, make it be equivalent to the taskhash. The unique "hash" only
-        # really needs to be a unique string (not even necessarily a hash), but
-        # making it match the taskhash has a few advantages:
-        #
-        # 1) All of the sstate code that assumes hashes can be the same
-        # 2) It provides maximal compatibility with builders that don't use
-        #    an equivalency server
-        # 3) The value is easy for multiple independent builders to derive the
-        #    same unique hash from the same input. This means that if the
-        #    independent builders find the same taskhash, but it isn't reported
-        #    to the server, there is a better chance that they will agree on
-        #    the unique hash.
-        unihash = taskhash
+        return None
 
-        try:
-            method = self.method
-            if tid in self.extramethod:
-                method = method + self.extramethod[tid]
-            data = self.client().get_unihash(method, self.taskhash[tid])
-            if data:
-                unihash = data
+    def _get_method(self, tid):
+        method = self.method
+        if tid in self.extramethod:
+            method = method + self.extramethod[tid]
+
+        return method
+
+    def unihashes_exist(self, query):
+        if len(query) == 0:
+            return {}
+
+        query_keys = []
+        result = {}
+        for key, unihash in query.items():
+            if unihash in self.unihash_exists_cache:
+                result[key] = True
+            else:
+                query_keys.append(key)
+
+        if query_keys:
+            with self.client() as client:
+                query_result = client.unihash_exists_batch(query[k] for k in query_keys)
+
+        for idx, key in enumerate(query_keys):
+            exists = query_result[idx]
+            if exists:
+                self.unihash_exists_cache.add(query[key])
+            result[key] = exists
+
+        return result
+
+    def get_unihash(self, tid):
+        return self.get_unihashes([tid])[tid]
+
+    def get_unihashes(self, tids):
+        """
+        For a iterable of tids, returns a dictionary that maps each tid to a
+        unihash
+        """
+        result = {}
+        query_tids = []
+
+        for tid in tids:
+            unihash = self.get_cached_unihash(tid)
+            if unihash:
+                result[tid] = unihash
+            else:
+                query_tids.append(tid)
+
+        if query_tids:
+            unihashes = []
+            try:
+                with self.client() as client:
+                    unihashes = client.get_unihash_batch((self._get_method(tid), self.taskhash[tid]) for tid in query_tids)
+            except (ConnectionError, FileNotFoundError) as e:
+                bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
+
+        for idx, tid in enumerate(query_tids):
+            # In the absence of being able to discover a unique hash from the
+            # server, make it be equivalent to the taskhash. The unique "hash" only
+            # really needs to be a unique string (not even necessarily a hash), but
+            # making it match the taskhash has a few advantages:
+            #
+            # 1) All of the sstate code that assumes hashes can be the same
+            # 2) It provides maximal compatibility with builders that don't use
+            #    an equivalency server
+            # 3) The value is easy for multiple independent builders to derive the
+            #    same unique hash from the same input. This means that if the
+            #    independent builders find the same taskhash, but it isn't reported
+            #    to the server, there is a better chance that they will agree on
+            #    the unique hash.
+            taskhash = self.taskhash[tid]
+
+            if unihashes and unihashes[idx]:
+                unihash = unihashes[idx]
                 # A unique hash equal to the taskhash is not very interesting,
                 # so it is reported it at debug level 2. If they differ, that
                 # is much more interesting, so it is reported at debug level 1
-                hashequiv_logger.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
+                hashequiv_logger.bbdebug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, tid, self.server))
             else:
                 hashequiv_logger.debug2('No reported unihash for %s:%s from %s' % (tid, taskhash, self.server))
-        except hashserv.client.HashConnectionError as e:
-            bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
+                unihash = taskhash
 
-        self.set_unihash(tid, unihash)
-        self.unihash[tid] = unihash
-        return unihash
+            self.set_unihash(tid, unihash)
+            self.unihash[tid] = unihash
+            result[tid] = unihash
+
+        return result
 
     def report_unihash(self, path, task, d):
         import importlib
@@ -556,14 +750,14 @@ class SignatureGeneratorUniHashMixIn(object):
         unihash = d.getVar('BB_UNIHASH')
         report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
         tempdir = d.getVar('T')
-        fn = d.getVar('BB_FILENAME')
-        tid = fn + ':do_' + task
+        mcfn = d.getVar('BB_FILENAME')
+        tid = mcfn + ':do_' + task
         key = tid + ':' + taskhash
 
         if self.setscenetasks and tid not in self.setscenetasks:
             return
 
-        # This can happen if locked sigs are in action. Detect and just abort
+        # This can happen if locked sigs are in action. Detect and just exit
         if taskhash != self.taskhash[tid]:
             return
 
@@ -611,17 +805,19 @@ class SignatureGeneratorUniHashMixIn(object):
                 if tid in self.extramethod:
                     method = method + self.extramethod[tid]
 
-                data = self.client().report_unihash(taskhash, method, outhash, unihash, extra_data)
+                with self.client() as client:
+                    data = client.report_unihash(taskhash, method, outhash, unihash, extra_data)
+
                 new_unihash = data['unihash']
 
                 if new_unihash != unihash:
                     hashequiv_logger.debug('Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
-                    bb.event.fire(bb.runqueue.taskUniHashUpdate(fn + ':do_' + task, new_unihash), d)
+                    bb.event.fire(bb.runqueue.taskUniHashUpdate(mcfn + ':do_' + task, new_unihash), d)
                     self.set_unihash(tid, new_unihash)
                     d.setVar('BB_UNIHASH', new_unihash)
                 else:
                     hashequiv_logger.debug('Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
-            except hashserv.client.HashConnectionError as e:
+            except (ConnectionError, FileNotFoundError) as e:
                 bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
         finally:
             if sigfile:
@@ -642,7 +838,9 @@ class SignatureGeneratorUniHashMixIn(object):
             if tid in self.extramethod:
                 method = method + self.extramethod[tid]
 
-            data = self.client().report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
+            with self.client() as client:
+                data = client.report_unihash_equiv(taskhash, method, wanted_unihash, extra_data)
+
             hashequiv_logger.verbose('Reported task %s as unihash %s to %s (%s)' % (tid, wanted_unihash, self.server, str(data)))
 
             if data is None:
@@ -661,7 +859,7 @@ class SignatureGeneratorUniHashMixIn(object):
                 # TODO: What to do here?
                 hashequiv_logger.verbose('Task %s unihash reported as unwanted hash %s' % (tid, finalunihash))
 
-        except hashserv.client.HashConnectionError as e:
+        except (ConnectionError, FileNotFoundError) as e:
             bb.warn('Error contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
 
         return False
@@ -676,19 +874,18 @@ class SignatureGeneratorTestEquivHash(SignatureGeneratorUniHashMixIn, SignatureG
         self.server = data.getVar('BB_HASHSERVE')
         self.method = "sstate_output_hash"
 
-#
-# Dummy class used for bitbake-selftest
-#
-class SignatureGeneratorTestMulticonfigDepends(SignatureGeneratorBasicHash):
-    name = "TestMulticonfigDepends"
-    supports_multiconfig_datacaches = True
+def clean_checksum_file_path(file_checksum_tuple):
+    f, cs = file_checksum_tuple
+    if "/./" in f:
+        return "./" + f.split("/./")[1]
+    return os.path.basename(f)
 
 def dump_this_task(outfile, d):
     import bb.parse
-    fn = d.getVar("BB_FILENAME")
+    mcfn = d.getVar("BB_FILENAME")
     task = "do_" + d.getVar("BB_CURRENTTASK")
-    referencestamp = bb.build.stamp_internal(task, d, None, True)
-    bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile:" + referencestamp)
+    referencestamp = bb.parse.siggen.stampfile_base(mcfn)
+    bb.parse.siggen.dump_sigtask(mcfn, task, outfile, "customfile:" + referencestamp)
 
 def init_colors(enable_color):
     """Initialise colour dict for passing to compare_sigfiles()"""
@@ -741,38 +938,15 @@ def list_inline_diff(oldlist, newlist, colors=None):
             ret.append(item)
     return '[%s]' % (', '.join(ret))
 
-def clean_basepath(basepath):
-    basepath, dir, recipe_task = basepath.rsplit("/", 2)
-    cleaned = dir + '/' + recipe_task
-
-    if basepath[0] == '/':
-        return cleaned
-
-    if basepath.startswith("mc:") and basepath.count(':') >= 2:
-        mc, mc_name, basepath = basepath.split(":", 2)
-        mc_suffix = ':mc:' + mc_name
-    else:
-        mc_suffix = ''
-
-    # mc stuff now removed from basepath. Whatever was next, if present will be the first
-    # suffix. ':/', recipe path start, marks the end of this. Something like
-    # 'virtual:a[:b[:c]]:/path...' (b and c being optional)
-    if basepath[0] != '/':
-        cleaned += ':' + basepath.split(':/', 1)[0]
-
-    return cleaned + mc_suffix
+# Handled renamed fields
+def handle_renames(data):
+    if 'basewhitelist' in data:
+        data['basehash_ignore_vars'] = data['basewhitelist']
+        del data['basewhitelist']
+    if 'taskwhitelist' in data:
+        data['taskhash_ignore_tasks'] = data['taskwhitelist']
+        del data['taskwhitelist']
 
-def clean_basepaths(a):
-    b = {}
-    for x in a:
-        b[clean_basepath(x)] = a[x]
-    return b
-
-def clean_basepaths_list(a):
-    b = []
-    for x in a:
-        b.append(clean_basepath(x))
-    return b
 
 def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
     output = []
@@ -794,20 +968,29 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
         formatparams.update(values)
         return formatstr.format(**formatparams)
 
-    with open(a, 'rb') as f:
-        p1 = pickle.Unpickler(f)
-        a_data = p1.load()
-    with open(b, 'rb') as f:
-        p2 = pickle.Unpickler(f)
-        b_data = p2.load()
-
-    def dict_diff(a, b, whitelist=set()):
+    try:
+        with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
+            a_data = json.load(f, object_hook=SetDecoder)
+    except (TypeError, OSError) as err:
+        bb.error("Failed to open sigdata file '%s': %s" % (a, str(err)))
+        raise err
+    try:
+        with bb.compress.zstd.open(b, "rt", encoding="utf-8", num_threads=1) as f:
+            b_data = json.load(f, object_hook=SetDecoder)
+    except (TypeError, OSError) as err:
+        bb.error("Failed to open sigdata file '%s': %s" % (b, str(err)))
+        raise err
+
+    for data in [a_data, b_data]:
+        handle_renames(data)
+
+    def dict_diff(a, b, ignored_vars=set()):
         sa = set(a.keys())
         sb = set(b.keys())
         common = sa & sb
         changed = set()
         for i in common:
-            if a[i] != b[i] and i not in whitelist:
+            if a[i] != b[i] and i not in ignored_vars:
                 changed.add(i)
         added = sb - sa
         removed = sa - sb
@@ -815,11 +998,11 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
 
     def file_checksums_diff(a, b):
         from collections import Counter
-        # Handle old siginfo format
-        if isinstance(a, dict):
-            a = [(os.path.basename(f), cs) for f, cs in a.items()]
-        if isinstance(b, dict):
-            b = [(os.path.basename(f), cs) for f, cs in b.items()]
+
+        # Convert lists back to tuples
+        a = [(f[0], f[1]) for f in a]
+        b = [(f[0], f[1]) for f in b]
+
         # Compare lists, ensuring we can handle duplicate filenames if they exist
         removedcount = Counter(a)
         removedcount.subtract(b)
@@ -846,15 +1029,15 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
         removed = [x[0] for x in removed]
         return changed, added, removed
 
-    if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
-        output.append(color_format("{color_title}basewhitelist changed{color_default} from '%s' to '%s'") % (a_data['basewhitelist'], b_data['basewhitelist']))
-        if a_data['basewhitelist'] and b_data['basewhitelist']:
-            output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
+    if 'basehash_ignore_vars' in a_data and a_data['basehash_ignore_vars'] != b_data['basehash_ignore_vars']:
+        output.append(color_format("{color_title}basehash_ignore_vars changed{color_default} from '%s' to '%s'") % (a_data['basehash_ignore_vars'], b_data['basehash_ignore_vars']))
+        if a_data['basehash_ignore_vars'] and b_data['basehash_ignore_vars']:
+            output.append("changed items: %s" % a_data['basehash_ignore_vars'].symmetric_difference(b_data['basehash_ignore_vars']))
 
-    if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
-        output.append(color_format("{color_title}taskwhitelist changed{color_default} from '%s' to '%s'") % (a_data['taskwhitelist'], b_data['taskwhitelist']))
-        if a_data['taskwhitelist'] and b_data['taskwhitelist']:
-            output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
+    if 'taskhash_ignore_tasks' in a_data and a_data['taskhash_ignore_tasks'] != b_data['taskhash_ignore_tasks']:
+        output.append(color_format("{color_title}taskhash_ignore_tasks changed{color_default} from '%s' to '%s'") % (a_data['taskhash_ignore_tasks'], b_data['taskhash_ignore_tasks']))
+        if a_data['taskhash_ignore_tasks'] and b_data['taskhash_ignore_tasks']:
+            output.append("changed items: %s" % a_data['taskhash_ignore_tasks'].symmetric_difference(b_data['taskhash_ignore_tasks']))
 
     if a_data['taskdeps'] != b_data['taskdeps']:
         output.append(color_format("{color_title}Task dependencies changed{color_default} from:\n%s\nto:\n%s") % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
@@ -862,23 +1045,23 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
     if a_data['basehash'] != b_data['basehash'] and not collapsed:
         output.append(color_format("{color_title}basehash changed{color_default} from %s to %s") % (a_data['basehash'], b_data['basehash']))
 
-    changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
+    changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basehash_ignore_vars'] & b_data['basehash_ignore_vars'])
     if changed:
-        for dep in changed:
+        for dep in sorted(changed):
             output.append(color_format("{color_title}List of dependencies for variable %s changed from '{color_default}%s{color_title}' to '{color_default}%s{color_title}'") % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
             if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
                 output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
     if added:
-        for dep in added:
+        for dep in sorted(added):
             output.append(color_format("{color_title}Dependency on variable %s was added") % (dep))
     if removed:
-        for dep in removed:
+        for dep in sorted(removed):
             output.append(color_format("{color_title}Dependency on Variable %s was removed") % (dep))
 
 
     changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
     if changed:
-        for dep in changed:
+        for dep in sorted(changed):
             oldval = a_data['varvals'][dep]
             newval = b_data['varvals'][dep]
             if newval and oldval and ('\n' in oldval or '\n' in newval):
@@ -902,9 +1085,9 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                 output.append(color_format("{color_title}Variable {var} value changed from '{color_default}{oldval}{color_title}' to '{color_default}{newval}{color_title}'{color_default}", var=dep, oldval=oldval, newval=newval))
 
     if not 'file_checksum_values' in a_data:
-         a_data['file_checksum_values'] = {}
+         a_data['file_checksum_values'] = []
     if not 'file_checksum_values' in b_data:
-         b_data['file_checksum_values'] = {}
+         b_data['file_checksum_values'] = []
 
     changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
     if changed:
@@ -931,11 +1114,11 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                 a = a_data['runtaskdeps'][idx]
                 b = b_data['runtaskdeps'][idx]
                 if a_data['runtaskhashes'][a] != b_data['runtaskhashes'][b] and not collapsed:
-                    changed.append("%s with hash %s\n changed to\n%s with hash %s" % (clean_basepath(a), a_data['runtaskhashes'][a], clean_basepath(b), b_data['runtaskhashes'][b]))
+                    changed.append("%s with hash %s\n changed to\n%s with hash %s" % (a, a_data['runtaskhashes'][a], b, b_data['runtaskhashes'][b]))
 
         if changed:
-            clean_a = clean_basepaths_list(a_data['runtaskdeps'])
-            clean_b = clean_basepaths_list(b_data['runtaskdeps'])
+            clean_a = a_data['runtaskdeps']
+            clean_b = b_data['runtaskdeps']
             if clean_a != clean_b:
                 output.append(color_format("{color_title}runtaskdeps changed:{color_default}\n%s") % list_inline_diff(clean_a, clean_b, colors))
             else:
@@ -948,7 +1131,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
         b = b_data['runtaskhashes']
         changed, added, removed = dict_diff(a, b)
         if added:
-            for dep in added:
+            for dep in sorted(added):
                 bdep_found = False
                 if removed:
                     for bdep in removed:
@@ -956,9 +1139,9 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                             #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
                             bdep_found = True
                 if not bdep_found:
-                    output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (clean_basepath(dep), b[dep]))
+                    output.append(color_format("{color_title}Dependency on task %s was added{color_default} with hash %s") % (dep, b[dep]))
         if removed:
-            for dep in removed:
+            for dep in sorted(removed):
                 adep_found = False
                 if added:
                     for adep in added:
@@ -966,11 +1149,11 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                             #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
                             adep_found = True
                 if not adep_found:
-                    output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (clean_basepath(dep), a[dep]))
+                    output.append(color_format("{color_title}Dependency on task %s was removed{color_default} with hash %s") % (dep, a[dep]))
         if changed:
-            for dep in changed:
+            for dep in sorted(changed):
                 if not collapsed:
-                    output.append(color_format("{color_title}Hash for dependent task %s changed{color_default} from %s to %s") % (clean_basepath(dep), a[dep], b[dep]))
+                    output.append(color_format("{color_title}Hash for task dependency %s changed{color_default} from %s to %s") % (dep, a[dep], b[dep]))
                 if callable(recursecb):
                     recout = recursecb(dep, a[dep], b[dep])
                     if recout:
@@ -980,6 +1163,7 @@ def compare_sigfiles(a, b, recursecb=None, color=False, collapsed=False):
                             # If a dependent hash changed, might as well print the line above and then defer to the changes in
                             # that hash since in all likelyhood, they're the same changes this task also saw.
                             output = [output[-1]] + recout
+                            break
 
     a_taint = a_data.get('taint', None)
     b_taint = b_data.get('taint', None)
@@ -1001,7 +1185,7 @@ def calc_basehash(sigdata):
         basedata = ''
 
     alldeps = sigdata['taskdeps']
-    for dep in alldeps:
+    for dep in sorted(alldeps):
         basedata = basedata + dep
         val = sigdata['varvals'][dep]
         if val is not None:
@@ -1017,6 +1201,8 @@ def calc_taskhash(sigdata):
 
     for c in sigdata['file_checksum_values']:
         if c[1]:
+            if "./" in c[0]:
+                data = data + c[0]
             data = data + c[1]
 
     if 'taint' in sigdata:
@@ -1031,32 +1217,37 @@ def calc_taskhash(sigdata):
 def dump_sigfile(a):
     output = []
 
-    with open(a, 'rb') as f:
-        p1 = pickle.Unpickler(f)
-        a_data = p1.load()
+    try:
+        with bb.compress.zstd.open(a, "rt", encoding="utf-8", num_threads=1) as f:
+            a_data = json.load(f, object_hook=SetDecoder)
+    except (TypeError, OSError) as err:
+        bb.error("Failed to open sigdata file '%s': %s" % (a, str(err)))
+        raise err
+
+    handle_renames(a_data)
 
-    output.append("basewhitelist: %s" % (a_data['basewhitelist']))
+    output.append("basehash_ignore_vars: %s" % (sorted(a_data['basehash_ignore_vars'])))
 
-    output.append("taskwhitelist: %s" % (a_data['taskwhitelist']))
+    output.append("taskhash_ignore_tasks: %s" % (sorted(a_data['taskhash_ignore_tasks'] or [])))
 
     output.append("Task dependencies: %s" % (sorted(a_data['taskdeps'])))
 
     output.append("basehash: %s" % (a_data['basehash']))
 
-    for dep in a_data['gendeps']:
-        output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep]))
+    for dep in sorted(a_data['gendeps']):
+        output.append("List of dependencies for variable %s is %s" % (dep, sorted(a_data['gendeps'][dep])))
 
-    for dep in a_data['varvals']:
+    for dep in sorted(a_data['varvals']):
         output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep]))
 
     if 'runtaskdeps' in a_data:
-        output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps']))
+        output.append("Tasks this task depends on: %s" % (sorted(a_data['runtaskdeps'])))
 
     if 'file_checksum_values' in a_data:
-        output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values']))
+        output.append("This task depends on the checksums of files: %s" % (sorted(a_data['file_checksum_values'])))
 
     if 'runtaskhashes' in a_data:
-        for dep in a_data['runtaskhashes']:
+        for dep in sorted(a_data['runtaskhashes']):
             output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]))
 
     if 'taint' in a_data:
diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py
index 47bad6d1fa..66545a65af 100644
--- a/bitbake/lib/bb/taskdata.py
+++ b/bitbake/lib/bb/taskdata.py
@@ -39,7 +39,7 @@ class TaskData:
     """
     BitBake Task Data implementation
     """
-    def __init__(self, abort = True, skiplist = None, allowincomplete = False):
+    def __init__(self, halt = True, skiplist = None, allowincomplete = False):
         self.build_targets = {}
         self.run_targets = {}
 
@@ -57,7 +57,7 @@ class TaskData:
         self.failed_rdeps = []
         self.failed_fns = []
 
-        self.abort = abort
+        self.halt = halt
         self.allowincomplete = allowincomplete
 
         self.skiplist = skiplist
@@ -328,7 +328,7 @@ class TaskData:
         try:
             self.add_provider_internal(cfgData, dataCache, item)
         except bb.providers.NoProvider:
-            if self.abort:
+            if self.halt:
                 raise
             self.remove_buildtarget(item)
 
@@ -451,12 +451,12 @@ class TaskData:
         for target in self.build_targets:
             if fn in self.build_targets[target]:
                 self.build_targets[target].remove(fn)
-                if len(self.build_targets[target]) == 0:
+                if not self.build_targets[target]:
                     self.remove_buildtarget(target, missing_list)
         for target in self.run_targets:
             if fn in self.run_targets[target]:
                 self.run_targets[target].remove(fn)
-                if len(self.run_targets[target]) == 0:
+                if not self.run_targets[target]:
                     self.remove_runtarget(target, missing_list)
 
     def remove_buildtarget(self, target, missing_list=None):
@@ -479,7 +479,7 @@ class TaskData:
                     fn = tid.rsplit(":",1)[0]
                     self.fail_fn(fn, missing_list)
 
-        if self.abort and target in self.external_targets:
+        if self.halt and target in self.external_targets:
             logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable dependency chain was: %s", target, missing_list)
             raise bb.providers.NoProvider(target)
 
@@ -516,7 +516,7 @@ class TaskData:
                     self.add_provider_internal(cfgData, dataCache, target)
                     added = added + 1
                 except bb.providers.NoProvider:
-                    if self.abort and target in self.external_targets and not self.allowincomplete:
+                    if self.halt and target in self.external_targets and not self.allowincomplete:
                         raise
                     if not self.allowincomplete:
                         self.remove_buildtarget(target)
diff --git a/bitbake/lib/bb/tests/codeparser.py b/bitbake/lib/bb/tests/codeparser.py
index 826a2d2f6d..c0d1362a0c 100644
--- a/bitbake/lib/bb/tests/codeparser.py
+++ b/bitbake/lib/bb/tests/codeparser.py
@@ -44,6 +44,7 @@ class VariableReferenceTest(ReferenceTest):
     def parseExpression(self, exp):
         parsedvar = self.d.expandWithRefs(exp, None)
         self.references = parsedvar.references
+        self.execs = parsedvar.execs
 
     def test_simple_reference(self):
         self.setEmptyVars(["FOO"])
@@ -61,6 +62,11 @@ class VariableReferenceTest(ReferenceTest):
         self.parseExpression("${@d.getVar('BAR') + 'foo'}")
         self.assertReferences(set(["BAR"]))
 
+    def test_python_exec_reference(self):
+        self.parseExpression("${@eval('3 * 5')}")
+        self.assertReferences(set())
+        self.assertExecs(set(["eval"]))
+
 class ShellReferenceTest(ReferenceTest):
 
     def parseExpression(self, exp):
@@ -100,6 +106,46 @@ ${D}${libdir}/pkgconfig/*.pc
         self.parseExpression("foo=$(echo bar)")
         self.assertExecs(set(["echo"]))
 
+    def test_assign_subshell_expansion_quotes(self):
+        self.parseExpression('foo="$(echo bar)"')
+        self.assertExecs(set(["echo"]))
+
+    def test_assign_subshell_expansion_nested(self):
+        self.parseExpression('foo="$(func1 "$(func2 bar$(func3))")"')
+        self.assertExecs(set(["func1", "func2", "func3"]))
+
+    def test_assign_subshell_expansion_multiple(self):
+        self.parseExpression('foo="$(func1 "$(func2)") $(func3)"')
+        self.assertExecs(set(["func1", "func2", "func3"]))
+
+    def test_assign_subshell_expansion_escaped_quotes(self):
+        self.parseExpression('foo="\\"fo\\"o$(func1)"')
+        self.assertExecs(set(["func1"]))
+
+    def test_assign_subshell_expansion_empty(self):
+        self.parseExpression('foo="bar$()foo"')
+        self.assertExecs(set())
+
+    def test_assign_subshell_backticks(self):
+        self.parseExpression("foo=`echo bar`")
+        self.assertExecs(set(["echo"]))
+
+    def test_assign_subshell_backticks_quotes(self):
+        self.parseExpression('foo="`echo bar`"')
+        self.assertExecs(set(["echo"]))
+
+    def test_assign_subshell_backticks_multiple(self):
+        self.parseExpression('foo="`func1 bar` `func2`"')
+        self.assertExecs(set(["func1", "func2"]))
+
+    def test_assign_subshell_backticks_escaped_quotes(self):
+        self.parseExpression('foo="\\"fo\\"o`func1`"')
+        self.assertExecs(set(["func1"]))
+
+    def test_assign_subshell_backticks_empty(self):
+        self.parseExpression('foo="bar``foo"')
+        self.assertExecs(set())
+
     def test_shell_unexpanded(self):
         self.setEmptyVars(["QT_BASE_NAME"])
         self.parseExpression('echo "${QT_BASE_NAME}"')
@@ -111,9 +157,9 @@ ${D}${libdir}/pkgconfig/*.pc
         self.assertExecs(set(["sed"]))
 
     def test_parameter_expansion_modifiers(self):
-        # - and + are also valid modifiers for parameter expansion, but are
+        # -,+ and : are also valid modifiers for parameter expansion, but are
         # valid characters in bitbake variable names, so are not included here
-        for i in ('=', ':-', ':=', '?', ':?', ':+', '#', '%', '##', '%%'):
+        for i in ('=', '?', '#', '%', '##', '%%'):
             name = "foo%sbar" % i
             self.parseExpression("${%s}" % name)
             self.assertNotIn(name, self.references)
@@ -318,7 +364,7 @@ d.getVar(a(), False)
             "filename": "example.bb",
         })
 
-        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
+        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
 
         self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"]))
 
@@ -365,7 +411,7 @@ esac
         self.d.setVarFlags("FOO", {"func": True})
         self.setEmptyVars(execs)
 
-        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
+        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
 
         self.assertEqual(deps, set(["somevar", "inverted"] + execs))
 
@@ -375,7 +421,7 @@ esac
         self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
         self.d.setVarFlag("FOO", "vardeps", "oe_libinstall")
 
-        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
+        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
 
         self.assertEqual(deps, set(["oe_libinstall"]))
 
@@ -384,7 +430,7 @@ esac
         self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
         self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}")
 
-        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), self.d)
+        deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
 
         self.assertEqual(deps, set(["oe_libinstall"]))
 
@@ -399,7 +445,7 @@ esac
         # Check dependencies
         self.d.setVar('ANOTHERVAR', expr)
         self.d.setVar('TESTVAR', 'anothervalue testval testval2')
-        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), self.d)
+        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
         self.assertEqual(sorted(values.splitlines()),
                          sorted([expr,
                           'TESTVAR{anothervalue} = Set',
@@ -412,11 +458,55 @@ esac
         # Check final value
         self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['anothervalue', 'yetanothervalue', 'lastone'])
 
+    def test_contains_vardeps_excluded(self):
+        # Check the ignored_vars option to build_dependencies is handled by contains functionality
+        varval = '${TESTVAR2} ${@bb.utils.filter("TESTVAR", "somevalue anothervalue", d)}'
+        self.d.setVar('ANOTHERVAR', varval)
+        self.d.setVar('TESTVAR', 'anothervalue testval testval2')
+        self.d.setVar('TESTVAR2', 'testval3')
+        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(["TESTVAR"]), self.d, self.d)
+        self.assertEqual(sorted(values.splitlines()), sorted([varval]))
+        self.assertEqual(deps, set(["TESTVAR2"]))
+        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
+
+        # Check the vardepsexclude flag is handled by contains functionality
+        self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR')
+        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
+        self.assertEqual(sorted(values.splitlines()), sorted([varval]))
+        self.assertEqual(deps, set(["TESTVAR2"]))
+        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
+
+    def test_contains_vardeps_override_operators(self):
+        # Check override operators handle dependencies correctly with the contains functionality
+        expr_plain = 'testval'
+        expr_prepend = '${@bb.utils.filter("TESTVAR1", "testval1", d)} '
+        expr_append = ' ${@bb.utils.filter("TESTVAR2", "testval2", d)}'
+        expr_remove = '${@bb.utils.contains("TESTVAR3", "no-testval", "testval", "", d)}'
+        # Check dependencies
+        self.d.setVar('ANOTHERVAR', expr_plain)
+        self.d.prependVar('ANOTHERVAR', expr_prepend)
+        self.d.appendVar('ANOTHERVAR', expr_append)
+        self.d.setVar('ANOTHERVAR:remove', expr_remove)
+        self.d.setVar('TESTVAR1', 'blah')
+        self.d.setVar('TESTVAR2', 'testval2')
+        self.d.setVar('TESTVAR3', 'no-testval')
+        deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
+        self.assertEqual(sorted(values.splitlines()),
+                         sorted([
+                          expr_prepend + expr_plain + expr_append,
+                          '_remove of ' + expr_remove,
+                          'TESTVAR1{testval1} = Unset',
+                          'TESTVAR2{testval2} = Set',
+                          'TESTVAR3{no-testval} = Set',
+                          ]))
+        # Check final value
+        self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval2'])
+
     #Currently no wildcard support
     #def test_vardeps_wildcards(self):
     #    self.d.setVar("oe_libinstall", "echo test")
     #    self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
     #    self.d.setVarFlag("FOO", "vardeps", "oe_*")
-    #    self.assertEquals(deps, set(["oe_libinstall"]))
+    #    self.assertEqual(deps, set(["oe_libinstall"]))
 
 
diff --git a/bitbake/lib/bb/tests/color.py b/bitbake/lib/bb/tests/color.py
index bf03750c69..bb70cb393d 100644
--- a/bitbake/lib/bb/tests/color.py
+++ b/bitbake/lib/bb/tests/color.py
@@ -20,7 +20,7 @@ class ProgressWatcher:
     def __init__(self):
         self._reports = []
 
-    def handle_event(self, event):
+    def handle_event(self, event, d):
         self._reports.append((event.progress, event.rate))
 
     def reports(self):
@@ -31,7 +31,7 @@ class ColorCodeTests(unittest.TestCase):
     def setUp(self):
         self.d = bb.data.init()
         self._progress_watcher = ProgressWatcher()
-        bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event)
+        bb.event.register("bb.build.TaskProgress", self._progress_watcher.handle_event, data=self.d)
 
     def tearDown(self):
         bb.event.remove("bb.build.TaskProgress", None)
diff --git a/bitbake/lib/bb/tests/compression.py b/bitbake/lib/bb/tests/compression.py
new file mode 100644
index 0000000000..16c297b315
--- /dev/null
+++ b/bitbake/lib/bb/tests/compression.py
@@ -0,0 +1,100 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+from pathlib import Path
+import bb.compress.lz4
+import bb.compress.zstd
+import contextlib
+import os
+import shutil
+import tempfile
+import unittest
+import subprocess
+
+
+class CompressionTests(object):
+    def setUp(self):
+        self._t = tempfile.TemporaryDirectory()
+        self.tmpdir = Path(self._t.name)
+        self.addCleanup(self._t.cleanup)
+
+    def _file_helper(self, mode_suffix, data):
+        tmp_file = self.tmpdir / "compressed"
+
+        with self.do_open(tmp_file, mode="w" + mode_suffix) as f:
+            f.write(data)
+
+        with self.do_open(tmp_file, mode="r" + mode_suffix) as f:
+            read_data = f.read()
+
+        self.assertEqual(read_data, data)
+
+    def test_text_file(self):
+        self._file_helper("t", "Hello")
+
+    def test_binary_file(self):
+        self._file_helper("b", "Hello".encode("utf-8"))
+
+    def _pipe_helper(self, mode_suffix, data):
+        rfd, wfd = os.pipe()
+        with open(rfd, "rb") as r, open(wfd, "wb") as w:
+            with self.do_open(r, mode="r" + mode_suffix) as decompress:
+                with self.do_open(w, mode="w" + mode_suffix) as compress:
+                    compress.write(data)
+                read_data = decompress.read()
+
+        self.assertEqual(read_data, data)
+
+    def test_text_pipe(self):
+        self._pipe_helper("t", "Hello")
+
+    def test_binary_pipe(self):
+        self._pipe_helper("b", "Hello".encode("utf-8"))
+
+    def test_bad_decompress(self):
+        tmp_file = self.tmpdir / "compressed"
+        with tmp_file.open("wb") as f:
+            f.write(b"\x00")
+
+        with self.assertRaises(OSError):
+            with self.do_open(tmp_file, mode="rb", stderr=subprocess.DEVNULL) as f:
+                data = f.read()
+
+
+class LZ4Tests(CompressionTests, unittest.TestCase):
+    def setUp(self):
+        if shutil.which("lz4") is None:
+            self.skipTest("'lz4' not found")
+        super().setUp()
+
+    @contextlib.contextmanager
+    def do_open(self, *args, **kwargs):
+        with bb.compress.lz4.open(*args, **kwargs) as f:
+            yield f
+
+
+class ZStdTests(CompressionTests, unittest.TestCase):
+    def setUp(self):
+        if shutil.which("zstd") is None:
+            self.skipTest("'zstd' not found")
+        super().setUp()
+
+    @contextlib.contextmanager
+    def do_open(self, *args, **kwargs):
+        with bb.compress.zstd.open(*args, **kwargs) as f:
+            yield f
+
+
+class PZStdTests(CompressionTests, unittest.TestCase):
+    def setUp(self):
+        if shutil.which("pzstd") is None:
+            self.skipTest("'pzstd' not found")
+        super().setUp()
+
+    @contextlib.contextmanager
+    def do_open(self, *args, **kwargs):
+        with bb.compress.zstd.open(*args, num_threads=2, **kwargs) as f:
+            yield f
diff --git a/bitbake/lib/bb/tests/cooker.py b/bitbake/lib/bb/tests/cooker.py
index c82d4b7b81..9e524ae345 100644
--- a/bitbake/lib/bb/tests/cooker.py
+++ b/bitbake/lib/bb/tests/cooker.py
@@ -1,6 +1,8 @@
 #
 # BitBake Tests for cooker.py
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
diff --git a/bitbake/lib/bb/tests/data.py b/bitbake/lib/bb/tests/data.py
index 1d4a64b109..a895f6a58e 100644
--- a/bitbake/lib/bb/tests/data.py
+++ b/bitbake/lib/bb/tests/data.py
@@ -60,6 +60,15 @@ class DataExpansions(unittest.TestCase):
         val = self.d.expand("${@5*12}")
         self.assertEqual(str(val), "60")
 
+    def test_python_snippet_w_dict(self):
+        val = self.d.expand("${@{ 'green': 1, 'blue': 2 }['green']}")
+        self.assertEqual(str(val), "1")
+
+    def test_python_unexpanded_multi(self):
+        self.d.setVar("bar", "${unsetvar}")
+        val = self.d.expand("${@2*2},${foo},${@d.getVar('foo') + ' ${bar}'},${foo}")
+        self.assertEqual(str(val), "4,value_of_foo,${@d.getVar('foo') + ' ${unsetvar}'},value_of_foo")
+
     def test_expand_in_python_snippet(self):
         val = self.d.expand("${@'boo ' + '${foo}'}")
         self.assertEqual(str(val), "boo value_of_foo")
@@ -68,6 +77,18 @@ class DataExpansions(unittest.TestCase):
         val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")
         self.assertEqual(str(val), "value_of_foo value_of_bar")
 
+    def test_python_snippet_function_reference(self):
+        self.d.setVar("TESTVAL", "testvalue")
+        self.d.setVar("testfunc", 'd.getVar("TESTVAL")')
+        context = bb.utils.get_context()
+        context["testfunc"] = lambda d: d.getVar("TESTVAL")
+        val = self.d.expand("${@testfunc(d)}")
+        self.assertEqual(str(val), "testvalue")
+
+    def test_python_snippet_builtin_metadata(self):
+        self.d.setVar("eval", "INVALID")
+        self.d.expand("${@eval('3')}")
+
     def test_python_unexpanded(self):
         self.d.setVar("bar", "${unsetvar}")
         val = self.d.expand("${@d.getVar('foo') + ' ${bar}'}")
@@ -245,35 +266,35 @@ class TestConcatOverride(unittest.TestCase):
 
     def test_prepend(self):
         self.d.setVar("TEST", "${VAL}")
-        self.d.setVar("TEST_prepend", "${FOO}:")
+        self.d.setVar("TEST:prepend", "${FOO}:")
         self.assertEqual(self.d.getVar("TEST"), "foo:val")
 
     def test_append(self):
         self.d.setVar("TEST", "${VAL}")
-        self.d.setVar("TEST_append", ":${BAR}")
+        self.d.setVar("TEST:append", ":${BAR}")
         self.assertEqual(self.d.getVar("TEST"), "val:bar")
 
     def test_multiple_append(self):
         self.d.setVar("TEST", "${VAL}")
-        self.d.setVar("TEST_prepend", "${FOO}:")
-        self.d.setVar("TEST_append", ":val2")
-        self.d.setVar("TEST_append", ":${BAR}")
+        self.d.setVar("TEST:prepend", "${FOO}:")
+        self.d.setVar("TEST:append", ":val2")
+        self.d.setVar("TEST:append", ":${BAR}")
         self.assertEqual(self.d.getVar("TEST"), "foo:val:val2:bar")
 
     def test_append_unset(self):
-        self.d.setVar("TEST_prepend", "${FOO}:")
-        self.d.setVar("TEST_append", ":val2")
-        self.d.setVar("TEST_append", ":${BAR}")
+        self.d.setVar("TEST:prepend", "${FOO}:")
+        self.d.setVar("TEST:append", ":val2")
+        self.d.setVar("TEST:append", ":${BAR}")
         self.assertEqual(self.d.getVar("TEST"), "foo::val2:bar")
 
     def test_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
-        self.d.setVar("TEST_remove", "val")
+        self.d.setVar("TEST:remove", "val")
         self.assertEqual(self.d.getVar("TEST"), " bar")
 
     def test_remove_cleared(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
-        self.d.setVar("TEST_remove", "val")
+        self.d.setVar("TEST:remove", "val")
         self.d.setVar("TEST", "${VAL} ${BAR}")
         self.assertEqual(self.d.getVar("TEST"), "val bar")
 
@@ -281,42 +302,42 @@ class TestConcatOverride(unittest.TestCase):
     # (including that whitespace is preserved)
     def test_remove_inactive_override(self):
         self.d.setVar("TEST", "${VAL} ${BAR}    123")
-        self.d.setVar("TEST_remove_inactiveoverride", "val")
+        self.d.setVar("TEST:remove:inactiveoverride", "val")
         self.assertEqual(self.d.getVar("TEST"), "val bar    123")
 
     def test_doubleref_remove(self):
         self.d.setVar("TEST", "${VAL} ${BAR}")
-        self.d.setVar("TEST_remove", "val")
+        self.d.setVar("TEST:remove", "val")
         self.d.setVar("TEST_TEST", "${TEST} ${TEST}")
         self.assertEqual(self.d.getVar("TEST_TEST"), " bar  bar")
 
     def test_empty_remove(self):
         self.d.setVar("TEST", "")
-        self.d.setVar("TEST_remove", "val")
+        self.d.setVar("TEST:remove", "val")
         self.assertEqual(self.d.getVar("TEST"), "")
 
     def test_remove_expansion(self):
         self.d.setVar("BAR", "Z")
         self.d.setVar("TEST", "${BAR}/X Y")
-        self.d.setVar("TEST_remove", "${BAR}/X")
+        self.d.setVar("TEST:remove", "${BAR}/X")
         self.assertEqual(self.d.getVar("TEST"), " Y")
 
     def test_remove_expansion_items(self):
         self.d.setVar("TEST", "A B C D")
         self.d.setVar("BAR", "B D")
-        self.d.setVar("TEST_remove", "${BAR}")
+        self.d.setVar("TEST:remove", "${BAR}")
         self.assertEqual(self.d.getVar("TEST"), "A  C ")
 
     def test_remove_preserve_whitespace(self):
         # When the removal isn't active, the original value should be preserved
         self.d.setVar("TEST", " A B")
-        self.d.setVar("TEST_remove", "C")
+        self.d.setVar("TEST:remove", "C")
         self.assertEqual(self.d.getVar("TEST"), " A B")
 
     def test_remove_preserve_whitespace2(self):
         # When the removal is active preserve the whitespace
         self.d.setVar("TEST", " A B")
-        self.d.setVar("TEST_remove", "B")
+        self.d.setVar("TEST:remove", "B")
         self.assertEqual(self.d.getVar("TEST"), " A ")
 
 class TestOverrides(unittest.TestCase):
@@ -329,81 +350,86 @@ class TestOverrides(unittest.TestCase):
         self.assertEqual(self.d.getVar("TEST"), "testvalue")
 
     def test_one_override(self):
-        self.d.setVar("TEST_bar", "testvalue2")
+        self.d.setVar("TEST:bar", "testvalue2")
         self.assertEqual(self.d.getVar("TEST"), "testvalue2")
 
     def test_one_override_unset(self):
-        self.d.setVar("TEST2_bar", "testvalue2")
+        self.d.setVar("TEST2:bar", "testvalue2")
 
         self.assertEqual(self.d.getVar("TEST2"), "testvalue2")
-        self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2_bar'])
+        self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST2', 'OVERRIDES', 'TEST2:bar'])
 
     def test_multiple_override(self):
-        self.d.setVar("TEST_bar", "testvalue2")
-        self.d.setVar("TEST_local", "testvalue3")
-        self.d.setVar("TEST_foo", "testvalue4")
+        self.d.setVar("TEST:bar", "testvalue2")
+        self.d.setVar("TEST:local", "testvalue3")
+        self.d.setVar("TEST:foo", "testvalue4")
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
-        self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST_foo', 'OVERRIDES', 'TEST_bar', 'TEST_local'])
+        self.assertCountEqual(list(self.d.keys()), ['TEST', 'TEST:foo', 'OVERRIDES', 'TEST:bar', 'TEST:local'])
 
     def test_multiple_combined_overrides(self):
-        self.d.setVar("TEST_local_foo_bar", "testvalue3")
+        self.d.setVar("TEST:local:foo:bar", "testvalue3")
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
     def test_multiple_overrides_unset(self):
-        self.d.setVar("TEST2_local_foo_bar", "testvalue3")
+        self.d.setVar("TEST2:local:foo:bar", "testvalue3")
         self.assertEqual(self.d.getVar("TEST2"), "testvalue3")
 
     def test_keyexpansion_override(self):
         self.d.setVar("LOCAL", "local")
-        self.d.setVar("TEST_bar", "testvalue2")
-        self.d.setVar("TEST_${LOCAL}", "testvalue3")
-        self.d.setVar("TEST_foo", "testvalue4")
+        self.d.setVar("TEST:bar", "testvalue2")
+        self.d.setVar("TEST:${LOCAL}", "testvalue3")
+        self.d.setVar("TEST:foo", "testvalue4")
         bb.data.expandKeys(self.d)
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
     def test_rename_override(self):
-        self.d.setVar("ALTERNATIVE_ncurses-tools_class-target", "a")
+        self.d.setVar("ALTERNATIVE:ncurses-tools:class-target", "a")
         self.d.setVar("OVERRIDES", "class-target")
-        self.d.renameVar("ALTERNATIVE_ncurses-tools", "ALTERNATIVE_lib32-ncurses-tools")
-        self.assertEqual(self.d.getVar("ALTERNATIVE_lib32-ncurses-tools"), "a")
+        self.d.renameVar("ALTERNATIVE:ncurses-tools", "ALTERNATIVE:lib32-ncurses-tools")
+        self.assertEqual(self.d.getVar("ALTERNATIVE:lib32-ncurses-tools"), "a")
 
     def test_underscore_override(self):
-        self.d.setVar("TEST_bar", "testvalue2")
-        self.d.setVar("TEST_some_val", "testvalue3")
-        self.d.setVar("TEST_foo", "testvalue4")
+        self.d.setVar("TEST:bar", "testvalue2")
+        self.d.setVar("TEST:some_val", "testvalue3")
+        self.d.setVar("TEST:foo", "testvalue4")
         self.d.setVar("OVERRIDES", "foo:bar:some_val")
         self.assertEqual(self.d.getVar("TEST"), "testvalue3")
 
+    # Test an override with _<numeric> in it based on a real world OE issue
+    def test_underscore_override_2(self):
+        self.d.setVar("TARGET_ARCH", "x86_64")
+        self.d.setVar("PN", "test-${TARGET_ARCH}")
+        self.d.setVar("VERSION", "1")
+        self.d.setVar("VERSION:pn-test-${TARGET_ARCH}", "2")
+        self.d.setVar("OVERRIDES", "pn-${PN}")
+        bb.data.expandKeys(self.d)
+        self.assertEqual(self.d.getVar("VERSION"), "2")
+
     def test_remove_with_override(self):
-        self.d.setVar("TEST_bar", "testvalue2")
-        self.d.setVar("TEST_some_val", "testvalue3 testvalue5")
-        self.d.setVar("TEST_some_val_remove", "testvalue3")
-        self.d.setVar("TEST_foo", "testvalue4")
+        self.d.setVar("TEST:bar", "testvalue2")
+        self.d.setVar("TEST:some_val", "testvalue3 testvalue5")
+        self.d.setVar("TEST:some_val:remove", "testvalue3")
+        self.d.setVar("TEST:foo", "testvalue4")
         self.d.setVar("OVERRIDES", "foo:bar:some_val")
         self.assertEqual(self.d.getVar("TEST"), " testvalue5")
 
     def test_append_and_override_1(self):
-        self.d.setVar("TEST_append", "testvalue2")
-        self.d.setVar("TEST_bar", "testvalue3")
+        self.d.setVar("TEST:append", "testvalue2")
+        self.d.setVar("TEST:bar", "testvalue3")
         self.assertEqual(self.d.getVar("TEST"), "testvalue3testvalue2")
 
     def test_append_and_override_2(self):
-        self.d.setVar("TEST_append_bar", "testvalue2")
+        self.d.setVar("TEST:append:bar", "testvalue2")
         self.assertEqual(self.d.getVar("TEST"), "testvaluetestvalue2")
 
     def test_append_and_override_3(self):
-        self.d.setVar("TEST_bar_append", "testvalue2")
+        self.d.setVar("TEST:bar:append", "testvalue2")
         self.assertEqual(self.d.getVar("TEST"), "testvalue2")
 
-    # Test an override with _<numeric> in it based on a real world OE issue
-    def test_underscore_override(self):
-        self.d.setVar("TARGET_ARCH", "x86_64")
-        self.d.setVar("PN", "test-${TARGET_ARCH}")
-        self.d.setVar("VERSION", "1")
-        self.d.setVar("VERSION_pn-test-${TARGET_ARCH}", "2")
-        self.d.setVar("OVERRIDES", "pn-${PN}")
-        bb.data.expandKeys(self.d)
-        self.assertEqual(self.d.getVar("VERSION"), "2")
+    def test_append_and_unused_override(self):
+        # Had a bug where an unused override append could return "" instead of None
+        self.d.setVar("BAR:append:unusedoverride", "testvalue2")
+        self.assertEqual(self.d.getVar("BAR"), None)
 
 class TestKeyExpansion(unittest.TestCase):
     def setUp(self):
@@ -424,17 +450,64 @@ class TestFlags(unittest.TestCase):
         self.d = bb.data.init()
         self.d.setVar("foo", "value of foo")
         self.d.setVarFlag("foo", "flag1", "value of flag1")
+        self.d.setVarFlag("foo", "_defaultval_flag_flag1", "default of flag1")
         self.d.setVarFlag("foo", "flag2", "value of flag2")
+        self.d.setVarFlag("foo", "_defaultval_flag_flag2", "default of flag2")
+        self.d.setVarFlag("foo", "flag3", "value of flag3")
+        self.d.setVarFlag("foo", "_defaultval_flag_flagnovalue", "default of flagnovalue")
 
     def test_setflag(self):
         self.assertEqual(self.d.getVarFlag("foo", "flag1", False), "value of flag1")
         self.assertEqual(self.d.getVarFlag("foo", "flag2", False), "value of flag2")
+        self.assertDictEqual(
+            self.d.getVarFlags("foo"),
+            {
+                "flag1": "value of flag1",
+                "flag2": "value of flag2",
+                "flag3": "value of flag3",
+                "flagnovalue": "default of flagnovalue",
+            }
+        )
+        self.assertDictEqual(
+            self.d.getVarFlags("foo", internalflags=True),
+            {
+                "_content": "value of foo",
+                "flag1": "value of flag1",
+                "flag2": "value of flag2",
+                "flag3": "value of flag3",
+                "_defaultval_flag_flag1": "default of flag1",
+                "_defaultval_flag_flag2": "default of flag2",
+                "_defaultval_flag_flagnovalue": "default of flagnovalue",
+            }
+        )
 
     def test_delflag(self):
         self.d.delVarFlag("foo", "flag2")
+        self.d.delVarFlag("foo", "flag3")
         self.assertEqual(self.d.getVarFlag("foo", "flag1", False), "value of flag1")
         self.assertEqual(self.d.getVarFlag("foo", "flag2", False), None)
-
+        self.assertDictEqual(
+            self.d.getVarFlags("foo"),
+            {
+                "flag1": "value of flag1",
+                "flagnovalue": "default of flagnovalue",
+            }
+        )
+        self.assertDictEqual(
+            self.d.getVarFlags("foo", internalflags=True),
+            {
+                "_content": "value of foo",
+                "flag1": "value of flag1",
+                "_defaultval_flag_flag1": "default of flag1",
+                "_defaultval_flag_flagnovalue": "default of flagnovalue",
+            }
+        )
+
+    def test_delvar(self):
+        self.d.delVar("foo")
+        self.assertEqual(self.d.getVarFlag("foo", "flag1", False), None)
+        self.assertEqual(self.d.getVarFlag("foo", "flag2", False), None)
+        self.assertEqual(self.d.getVarFlags("foo", internalflags=True), None)
 
 class Contains(unittest.TestCase):
     def setUp(self):
@@ -498,7 +571,7 @@ class TaskHash(unittest.TestCase):
         d.setVar("VAR", "val")
         # Adding an inactive removal shouldn't change the hash
         d.setVar("BAR", "notbar")
-        d.setVar("MYCOMMAND_remove", "${BAR}")
+        d.setVar("MYCOMMAND:remove", "${BAR}")
         nexthash = gettask_bashhash("mytask", d)
         self.assertEqual(orighash, nexthash)
 
diff --git a/bitbake/lib/bb/tests/event.py b/bitbake/lib/bb/tests/event.py
index 9ca7e9bc8e..ef61891d30 100644
--- a/bitbake/lib/bb/tests/event.py
+++ b/bitbake/lib/bb/tests/event.py
@@ -13,6 +13,7 @@ import pickle
 import threading
 import time
 import unittest
+import tempfile
 from unittest.mock import Mock
 from unittest.mock import call
 
@@ -157,7 +158,7 @@ class EventHandlingTest(unittest.TestCase):
                                  self._test_process.event_handler,
                                  event,
                                  None)
-        self._test_process.event_handler.assert_called_once_with(event)
+        self._test_process.event_handler.assert_called_once_with(event, None)
 
     def test_fire_class_handlers(self):
         """ Test fire_class_handlers method """
@@ -175,10 +176,10 @@ class EventHandlingTest(unittest.TestCase):
         bb.event.fire_class_handlers(event1, None)
         bb.event.fire_class_handlers(event2, None)
         bb.event.fire_class_handlers(event2, None)
-        expected_event_handler1 = [call(event1)]
-        expected_event_handler2 = [call(event1),
-                                   call(event2),
-                                   call(event2)]
+        expected_event_handler1 = [call(event1, None)]
+        expected_event_handler2 = [call(event1, None),
+                                   call(event2, None),
+                                   call(event2, None)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected_event_handler1)
         self.assertEqual(self._test_process.event_handler2.call_args_list,
@@ -205,7 +206,7 @@ class EventHandlingTest(unittest.TestCase):
         bb.event.fire_class_handlers(event2, None)
         bb.event.fire_class_handlers(event2, None)
         expected_event_handler1 = []
-        expected_event_handler2 = [call(event1)]
+        expected_event_handler2 = [call(event1, None)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected_event_handler1)
         self.assertEqual(self._test_process.event_handler2.call_args_list,
@@ -223,7 +224,7 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(result, bb.event.Registered)
         bb.event.fire_class_handlers(event1, None)
         bb.event.fire_class_handlers(event2, None)
-        expected = [call(event1), call(event2)]
+        expected = [call(event1, None), call(event2, None)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected)
 
@@ -237,7 +238,7 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(result, bb.event.Registered)
         bb.event.fire_class_handlers(event1, None)
         bb.event.fire_class_handlers(event2, None)
-        expected = [call(event1), call(event2), call(event1)]
+        expected = [call(event1, None), call(event2, None), call(event1, None)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected)
 
@@ -251,7 +252,7 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(result, bb.event.Registered)
         bb.event.fire_class_handlers(event1, None)
         bb.event.fire_class_handlers(event2, None)
-        expected = [call(event1), call(event2), call(event1), call(event2)]
+        expected = [call(event1,None), call(event2, None), call(event1, None), call(event2, None)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected)
 
@@ -359,9 +360,10 @@ class EventHandlingTest(unittest.TestCase):
 
         event1 = bb.event.ConfigParsed()
         bb.event.fire(event1, None)
-        expected = [call(event1)]
+        expected = [call(event1, None)]
         self.assertEqual(self._test_process.event_handler1.call_args_list,
                          expected)
+        expected = [call(event1)]
         self.assertEqual(self._test_ui1.event.send.call_args_list,
                          expected)
 
@@ -450,10 +452,9 @@ class EventHandlingTest(unittest.TestCase):
             and disable threadlocks tests """
         bb.event.fire(bb.event.OperationStarted(), None)
 
-    def test_enable_threadlock(self):
+    def test_event_threadlock(self):
         """ Test enable_threadlock method """
         self._set_threadlock_test_mockups()
-        bb.event.enable_threadlock()
         self._set_and_run_threadlock_test_workers()
         # Calls to UI handlers should be in order as all the registered
         # handlers for the event coming from the first worker should be
@@ -461,20 +462,6 @@ class EventHandlingTest(unittest.TestCase):
         self.assertEqual(self._threadlock_test_calls,
                          ["w1_ui1", "w1_ui2", "w2_ui1", "w2_ui2"])
 
-
-    def test_disable_threadlock(self):
-        """ Test disable_threadlock method """
-        self._set_threadlock_test_mockups()
-        bb.event.disable_threadlock()
-        self._set_and_run_threadlock_test_workers()
-        # Calls to UI handlers should be intertwined together. Thanks to the
-        # delay in the registered handlers for the event coming from the first
-        # worker, the event coming from the second worker starts being
-        # processed before finishing handling the first worker event.
-        self.assertEqual(self._threadlock_test_calls,
-                         ["w1_ui1", "w2_ui1", "w1_ui2", "w2_ui2"])
-
-
 class EventClassesTest(unittest.TestCase):
     """ Event classes test class """
 
@@ -482,6 +469,8 @@ class EventClassesTest(unittest.TestCase):
 
     def setUp(self):
         bb.event.worker_pid = EventClassesTest._worker_pid
+        self.d = bb.data.init()
+        bb.parse.siggen = bb.siggen.init(self.d)
 
     def test_Event(self):
         """ Test the Event base class """
@@ -964,3 +953,24 @@ class EventClassesTest(unittest.TestCase):
         event = bb.event.FindSigInfoResult(result)
         self.assertEqual(event.result, result)
         self.assertEqual(event.pid, EventClassesTest._worker_pid)
+
+    def test_lineno_in_eventhandler(self):
+        # The error lineno is 5, not 4 since the first line is '\n'
+        error_line = """
+# Comment line1
+# Comment line2
+python test_lineno_in_eventhandler() {
+    This is an error line
+}
+addhandler test_lineno_in_eventhandler
+test_lineno_in_eventhandler[eventmask] = "bb.event.ConfigParsed"
+"""
+
+        with self.assertLogs() as logs:
+            f = tempfile.NamedTemporaryFile(suffix = '.bb')
+            f.write(bytes(error_line, "utf-8"))
+            f.flush()
+            d = bb.parse.handle(f.name, self.d)['']
+
+        output = "".join(logs.output)
+        self.assertTrue(" line 5\n" in output)
diff --git a/bitbake/lib/bb/tests/fetch-testdata/debian/pool/main/m/minicom/index.html b/bitbake/lib/bb/tests/fetch-testdata/debian/pool/main/m/minicom/index.html
new file mode 100644
index 0000000000..4a1eb4de13
--- /dev/null
+++ b/bitbake/lib/bb/tests/fetch-testdata/debian/pool/main/m/minicom/index.html
@@ -0,0 +1,59 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<html>
+ <head>
+  <title>Index of /debian/pool/main/m/minicom</title>
+ </head>
+ <body>
+<h1>Index of /debian/pool/main/m/minicom</h1>
+  <table>
+   <tr><th valign="top"><img src="/icons/blank.gif" alt="[ICO]"></th><th><a href="?C=N;O=D">Name</a></th><th><a href="?C=M;O=A">Last modified</a></th><th><a href="?C=S;O=A">Size</a></th></tr>
+   <tr><th colspan="4"><hr></th></tr>
+<tr><td valign="top"><img src="/icons/back.gif" alt="[PARENTDIR]"></td><td><a href="/debian/pool/main/m/">Parent Directory</a></td><td>&nbsp;</td><td align="right">  - </td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1.debian.tar.xz">minicom_2.7-1+deb8u1.debian.tar.xz</a></td><td align="right">2017-04-24 08:22  </td><td align="right"> 14K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1.dsc">minicom_2.7-1+deb8u1.dsc</a></td><td align="right">2017-04-24 08:22  </td><td align="right">1.9K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1_amd64.deb">minicom_2.7-1+deb8u1_amd64.deb</a></td><td align="right">2017-04-25 21:10  </td><td align="right">257K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1_armel.deb">minicom_2.7-1+deb8u1_armel.deb</a></td><td align="right">2017-04-26 00:58  </td><td align="right">246K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1_armhf.deb">minicom_2.7-1+deb8u1_armhf.deb</a></td><td align="right">2017-04-26 00:58  </td><td align="right">245K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1+deb8u1_i386.deb">minicom_2.7-1+deb8u1_i386.deb</a></td><td align="right">2017-04-25 21:41  </td><td align="right">258K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1.debian.tar.xz">minicom_2.7-1.1.debian.tar.xz</a></td><td align="right">2017-04-22 09:34  </td><td align="right"> 14K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1.dsc">minicom_2.7-1.1.dsc</a></td><td align="right">2017-04-22 09:34  </td><td align="right">1.9K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_amd64.deb">minicom_2.7-1.1_amd64.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">261K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_arm64.deb">minicom_2.7-1.1_arm64.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">250K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_armel.deb">minicom_2.7-1.1_armel.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">255K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_armhf.deb">minicom_2.7-1.1_armhf.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">254K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_i386.deb">minicom_2.7-1.1_i386.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">266K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_mips.deb">minicom_2.7-1.1_mips.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">258K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_mips64el.deb">minicom_2.7-1.1_mips64el.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">259K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_mipsel.deb">minicom_2.7-1.1_mipsel.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">259K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_ppc64el.deb">minicom_2.7-1.1_ppc64el.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">253K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7-1.1_s390x.deb">minicom_2.7-1.1_s390x.deb</a></td><td align="right">2017-04-22 15:29  </td><td align="right">261K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_amd64.deb">minicom_2.7.1-1+b1_amd64.deb</a></td><td align="right">2018-05-06 08:14  </td><td align="right">262K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_arm64.deb">minicom_2.7.1-1+b1_arm64.deb</a></td><td align="right">2018-05-06 07:58  </td><td align="right">250K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_armel.deb">minicom_2.7.1-1+b1_armel.deb</a></td><td align="right">2018-05-06 08:45  </td><td align="right">253K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_armhf.deb">minicom_2.7.1-1+b1_armhf.deb</a></td><td align="right">2018-05-06 10:42  </td><td align="right">253K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_i386.deb">minicom_2.7.1-1+b1_i386.deb</a></td><td align="right">2018-05-06 08:55  </td><td align="right">266K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_mips.deb">minicom_2.7.1-1+b1_mips.deb</a></td><td align="right">2018-05-06 08:14  </td><td align="right">258K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_mipsel.deb">minicom_2.7.1-1+b1_mipsel.deb</a></td><td align="right">2018-05-06 12:13  </td><td align="right">259K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_ppc64el.deb">minicom_2.7.1-1+b1_ppc64el.deb</a></td><td align="right">2018-05-06 09:10  </td><td align="right">260K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b1_s390x.deb">minicom_2.7.1-1+b1_s390x.deb</a></td><td align="right">2018-05-06 08:14  </td><td align="right">257K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1+b2_mips64el.deb">minicom_2.7.1-1+b2_mips64el.deb</a></td><td align="right">2018-05-06 09:41  </td><td align="right">260K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1.debian.tar.xz">minicom_2.7.1-1.debian.tar.xz</a></td><td align="right">2017-08-13 15:40  </td><td align="right"> 14K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.7.1-1.dsc">minicom_2.7.1-1.dsc</a></td><td align="right">2017-08-13 15:40  </td><td align="right">1.8K</td></tr>
+<tr><td valign="top"><img src="/icons/compressed.gif" alt="[   ]"></td><td><a href="minicom_2.7.1.orig.tar.gz">minicom_2.7.1.orig.tar.gz</a></td><td align="right">2017-08-13 15:40  </td><td align="right">855K</td></tr>
+<tr><td valign="top"><img src="/icons/compressed.gif" alt="[   ]"></td><td><a href="minicom_2.7.orig.tar.gz">minicom_2.7.orig.tar.gz</a></td><td align="right">2014-01-01 09:36  </td><td align="right">843K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2.debian.tar.xz">minicom_2.8-2.debian.tar.xz</a></td><td align="right">2021-06-15 03:47  </td><td align="right"> 14K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2.dsc">minicom_2.8-2.dsc</a></td><td align="right">2021-06-15 03:47  </td><td align="right">1.8K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_amd64.deb">minicom_2.8-2_amd64.deb</a></td><td align="right">2021-06-15 03:58  </td><td align="right">280K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_arm64.deb">minicom_2.8-2_arm64.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">275K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_armel.deb">minicom_2.8-2_armel.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">271K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_armhf.deb">minicom_2.8-2_armhf.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">272K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_i386.deb">minicom_2.8-2_i386.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">285K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_mips64el.deb">minicom_2.8-2_mips64el.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">277K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_mipsel.deb">minicom_2.8-2_mipsel.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">278K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_ppc64el.deb">minicom_2.8-2_ppc64el.deb</a></td><td align="right">2021-06-15 04:13  </td><td align="right">286K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8-2_s390x.deb">minicom_2.8-2_s390x.deb</a></td><td align="right">2021-06-15 03:58  </td><td align="right">275K</td></tr>
+<tr><td valign="top"><img src="/icons/unknown.gif" alt="[   ]"></td><td><a href="minicom_2.8.orig.tar.bz2">minicom_2.8.orig.tar.bz2</a></td><td align="right">2021-01-03 12:44  </td><td align="right">598K</td></tr>
+   <tr><th colspan="4"><hr></th></tr>
+</table>
+<address>Apache Server at ftp.debian.org Port 80</address>
+</body></html>
diff --git a/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.10/index.html b/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.10/index.html
new file mode 100644
index 0000000000..4e41af6d6a
--- /dev/null
+++ b/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.10/index.html
@@ -0,0 +1,20 @@
+<!DOCTYPE html><html><head><meta http-equiv="content-type" content="text/html; charset=utf-8"><meta name="viewport" content="width=device-width"><style type="text/css">body,html {background:#fff;font-family:"Bitstream Vera Sans","Lucida Grande","Lucida Sans Unicode",Lucidux,Verdana,Lucida,sans-serif;}tr:nth-child(even) {background:#f4f4f4;}th,td {padding:0.1em 0.5em;}th {text-align:left;font-weight:bold;background:#eee;border-bottom:1px solid #aaa;}#list {border:1px solid #aaa;width:100%;}a {color:#a33;}a:hover {color:#e33;}</style>
+
+<title>Index of /sources/libxml2/2.10/</title>
+</head><body><h1>Index of /sources/libxml2/2.10/</h1>
+<table id="list"><thead><tr><th style="width:55%"><a href="?C=N&amp;O=A">File Name</a>&nbsp;<a href="?C=N&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:20%"><a href="?C=S&amp;O=A">File Size</a>&nbsp;<a href="?C=S&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:25%"><a href="?C=M&amp;O=A">Date</a>&nbsp;<a href="?C=M&amp;O=D">&nbsp;&darr;&nbsp;</a></th></tr></thead>
+<tbody><tr><td class="link"><a href="../">Parent directory/</a></td><td class="size">-</td><td class="date">-</td></tr>
+<tr><td class="link"><a href="LATEST-IS-2.10.3" title="LATEST-IS-2.10.3">LATEST-IS-2.10.3</a></td><td class="size">2.5 MiB</td><td class="date">2022-Oct-14 12:55</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.0.news" title="libxml2-2.10.0.news">libxml2-2.10.0.news</a></td><td class="size">7.1 KiB</td><td class="date">2022-Aug-17 11:55</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.0.sha256sum" title="libxml2-2.10.0.sha256sum">libxml2-2.10.0.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Aug-17 11:55</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.0.tar.xz" title="libxml2-2.10.0.tar.xz">libxml2-2.10.0.tar.xz</a></td><td class="size">2.6 MiB</td><td class="date">2022-Aug-17 11:55</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.1.news" title="libxml2-2.10.1.news">libxml2-2.10.1.news</a></td><td class="size">455 B</td><td class="date">2022-Aug-25 11:33</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.1.sha256sum" title="libxml2-2.10.1.sha256sum">libxml2-2.10.1.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Aug-25 11:33</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.1.tar.xz" title="libxml2-2.10.1.tar.xz">libxml2-2.10.1.tar.xz</a></td><td class="size">2.6 MiB</td><td class="date">2022-Aug-25 11:33</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.2.news" title="libxml2-2.10.2.news">libxml2-2.10.2.news</a></td><td class="size">309 B</td><td class="date">2022-Aug-29 14:56</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.2.sha256sum" title="libxml2-2.10.2.sha256sum">libxml2-2.10.2.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Aug-29 14:56</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.2.tar.xz" title="libxml2-2.10.2.tar.xz">libxml2-2.10.2.tar.xz</a></td><td class="size">2.5 MiB</td><td class="date">2022-Aug-29 14:56</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.3.news" title="libxml2-2.10.3.news">libxml2-2.10.3.news</a></td><td class="size">294 B</td><td class="date">2022-Oct-14 12:55</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.3.sha256sum" title="libxml2-2.10.3.sha256sum">libxml2-2.10.3.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Oct-14 12:55</td></tr>
+<tr><td class="link"><a href="libxml2-2.10.3.tar.xz" title="libxml2-2.10.3.tar.xz">libxml2-2.10.3.tar.xz</a></td><td class="size">2.5 MiB</td><td class="date">2022-Oct-14 12:55</td></tr>
+</tbody></table></body></html>
diff --git a/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.9/index.html b/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.9/index.html
new file mode 100644
index 0000000000..abdfdd0fa2
--- /dev/null
+++ b/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/2.9/index.html
@@ -0,0 +1,40 @@
+<!DOCTYPE html><html><head><meta http-equiv="content-type" content="text/html; charset=utf-8"><meta name="viewport" content="width=device-width"><style type="text/css">body,html {background:#fff;font-family:"Bitstream Vera Sans","Lucida Grande","Lucida Sans Unicode",Lucidux,Verdana,Lucida,sans-serif;}tr:nth-child(even) {background:#f4f4f4;}th,td {padding:0.1em 0.5em;}th {text-align:left;font-weight:bold;background:#eee;border-bottom:1px solid #aaa;}#list {border:1px solid #aaa;width:100%;}a {color:#a33;}a:hover {color:#e33;}</style>
+
+<title>Index of /sources/libxml2/2.9/</title>
+</head><body><h1>Index of /sources/libxml2/2.9/</h1>
+<table id="list"><thead><tr><th style="width:55%"><a href="?C=N&amp;O=A">File Name</a>&nbsp;<a href="?C=N&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:20%"><a href="?C=S&amp;O=A">File Size</a>&nbsp;<a href="?C=S&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:25%"><a href="?C=M&amp;O=A">Date</a>&nbsp;<a href="?C=M&amp;O=D">&nbsp;&darr;&nbsp;</a></th></tr></thead>
+<tbody><tr><td class="link"><a href="../">Parent directory/</a></td><td class="size">-</td><td class="date">-</td></tr>
+<tr><td class="link"><a href="LATEST-IS-2.9.14" title="LATEST-IS-2.9.14">LATEST-IS-2.9.14</a></td><td class="size">3.0 MiB</td><td class="date">2022-May-02 12:03</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.0.sha256sum" title="libxml2-2.9.0.sha256sum">libxml2-2.9.0.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:27</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.0.tar.xz" title="libxml2-2.9.0.tar.xz">libxml2-2.9.0.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:27</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.1.sha256sum" title="libxml2-2.9.1.sha256sum">libxml2-2.9.1.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:28</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.1.tar.xz" title="libxml2-2.9.1.tar.xz">libxml2-2.9.1.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:28</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.10.sha256sum" title="libxml2-2.9.10.sha256sum">libxml2-2.9.10.sha256sum</a></td><td class="size">88 B</td><td class="date">2022-Feb-14 18:42</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.10.tar.xz" title="libxml2-2.9.10.tar.xz">libxml2-2.9.10.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:42</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.11.sha256sum" title="libxml2-2.9.11.sha256sum">libxml2-2.9.11.sha256sum</a></td><td class="size">88 B</td><td class="date">2022-Feb-14 18:43</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.11.tar.xz" title="libxml2-2.9.11.tar.xz">libxml2-2.9.11.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:43</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.12.sha256sum" title="libxml2-2.9.12.sha256sum">libxml2-2.9.12.sha256sum</a></td><td class="size">88 B</td><td class="date">2022-Feb-14 18:45</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.12.tar.xz" title="libxml2-2.9.12.tar.xz">libxml2-2.9.12.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:45</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.13.news" title="libxml2-2.9.13.news">libxml2-2.9.13.news</a></td><td class="size">26.6 KiB</td><td class="date">2022-Feb-20 12:42</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.13.sha256sum" title="libxml2-2.9.13.sha256sum">libxml2-2.9.13.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-Feb-20 12:42</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.13.tar.xz" title="libxml2-2.9.13.tar.xz">libxml2-2.9.13.tar.xz</a></td><td class="size">3.1 MiB</td><td class="date">2022-Feb-20 12:42</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.14.news" title="libxml2-2.9.14.news">libxml2-2.9.14.news</a></td><td class="size">1.0 KiB</td><td class="date">2022-May-02 12:03</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.14.sha256sum" title="libxml2-2.9.14.sha256sum">libxml2-2.9.14.sha256sum</a></td><td class="size">174 B</td><td class="date">2022-May-02 12:03</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.14.tar.xz" title="libxml2-2.9.14.tar.xz">libxml2-2.9.14.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-May-02 12:03</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.2.sha256sum" title="libxml2-2.9.2.sha256sum">libxml2-2.9.2.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:30</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.2.tar.xz" title="libxml2-2.9.2.tar.xz">libxml2-2.9.2.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:30</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.3.sha256sum" title="libxml2-2.9.3.sha256sum">libxml2-2.9.3.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:31</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.3.tar.xz" title="libxml2-2.9.3.tar.xz">libxml2-2.9.3.tar.xz</a></td><td class="size">3.2 MiB</td><td class="date">2022-Feb-14 18:31</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.4.sha256sum" title="libxml2-2.9.4.sha256sum">libxml2-2.9.4.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:33</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.4.tar.xz" title="libxml2-2.9.4.tar.xz">libxml2-2.9.4.tar.xz</a></td><td class="size">2.9 MiB</td><td class="date">2022-Feb-14 18:33</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.5.sha256sum" title="libxml2-2.9.5.sha256sum">libxml2-2.9.5.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:35</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.5.tar.xz" title="libxml2-2.9.5.tar.xz">libxml2-2.9.5.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:35</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.6.sha256sum" title="libxml2-2.9.6.sha256sum">libxml2-2.9.6.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:36</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.6.tar.xz" title="libxml2-2.9.6.tar.xz">libxml2-2.9.6.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:36</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.7.sha256sum" title="libxml2-2.9.7.sha256sum">libxml2-2.9.7.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:37</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.7.tar.xz" title="libxml2-2.9.7.tar.xz">libxml2-2.9.7.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:37</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.8.sha256sum" title="libxml2-2.9.8.sha256sum">libxml2-2.9.8.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:39</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.8.tar.xz" title="libxml2-2.9.8.tar.xz">libxml2-2.9.8.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:39</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.9.sha256sum" title="libxml2-2.9.9.sha256sum">libxml2-2.9.9.sha256sum</a></td><td class="size">87 B</td><td class="date">2022-Feb-14 18:40</td></tr>
+<tr><td class="link"><a href="libxml2-2.9.9.tar.xz" title="libxml2-2.9.9.tar.xz">libxml2-2.9.9.tar.xz</a></td><td class="size">3.0 MiB</td><td class="date">2022-Feb-14 18:40</td></tr>
+</tbody></table></body></html>
diff --git a/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/index.html b/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/index.html
new file mode 100644
index 0000000000..c183e06a55
--- /dev/null
+++ b/bitbake/lib/bb/tests/fetch-testdata/software/libxml2/index.html
@@ -0,0 +1,19 @@
+<!DOCTYPE html><html><head><meta http-equiv="content-type" content="text/html; charset=utf-8"><meta name="viewport" content="width=device-width"><style type="text/css">body,html {background:#fff;font-family:"Bitstream Vera Sans","Lucida Grande","Lucida Sans Unicode",Lucidux,Verdana,Lucida,sans-serif;}tr:nth-child(even) {background:#f4f4f4;}th,td {padding:0.1em 0.5em;}th {text-align:left;font-weight:bold;background:#eee;border-bottom:1px solid #aaa;}#list {border:1px solid #aaa;width:100%;}a {color:#a33;}a:hover {color:#e33;}</style>
+
+<title>Index of /sources/libxml2/</title>
+</head><body><h1>Index of /sources/libxml2/</h1>
+<table id="list"><thead><tr><th style="width:55%"><a href="?C=N&amp;O=A">File Name</a>&nbsp;<a href="?C=N&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:20%"><a href="?C=S&amp;O=A">File Size</a>&nbsp;<a href="?C=S&amp;O=D">&nbsp;&darr;&nbsp;</a></th><th style="width:25%"><a href="?C=M&amp;O=A">Date</a>&nbsp;<a href="?C=M&amp;O=D">&nbsp;&darr;&nbsp;</a></th></tr></thead>
+<tbody><tr><td class="link"><a href="../">Parent directory/</a></td><td class="size">-</td><td class="date">-</td></tr>
+<tr><td class="link"><a href="2.0/" title="2.0">2.0/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:04</td></tr>
+<tr><td class="link"><a href="2.1/" title="2.1">2.1/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:04</td></tr>
+<tr><td class="link"><a href="2.10/" title="2.10">2.10/</a></td><td class="size">-</td><td class="date">2022-Oct-14 12:55</td></tr>
+<tr><td class="link"><a href="2.2/" title="2.2">2.2/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:04</td></tr>
+<tr><td class="link"><a href="2.3/" title="2.3">2.3/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:05</td></tr>
+<tr><td class="link"><a href="2.4/" title="2.4">2.4/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:05</td></tr>
+<tr><td class="link"><a href="2.5/" title="2.5">2.5/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:05</td></tr>
+<tr><td class="link"><a href="2.6/" title="2.6">2.6/</a></td><td class="size">-</td><td class="date">2009-Jul-14 13:05</td></tr>
+<tr><td class="link"><a href="2.7/" title="2.7">2.7/</a></td><td class="size">-</td><td class="date">2022-Feb-14 18:24</td></tr>
+<tr><td class="link"><a href="2.8/" title="2.8">2.8/</a></td><td class="size">-</td><td class="date">2022-Feb-14 18:26</td></tr>
+<tr><td class="link"><a href="2.9/" title="2.9">2.9/</a></td><td class="size">-</td><td class="date">2022-May-02 12:04</td></tr>
+<tr><td class="link"><a href="cache.json" title="cache.json">cache.json</a></td><td class="size">22.8 KiB</td><td class="date">2022-Oct-14 12:55</td></tr>
+</tbody></table></body></html>
diff --git a/bitbake/lib/bb/tests/fetch-testdata/software/miniupnp/download.php b/bitbake/lib/bb/tests/fetch-testdata/software/miniupnp/download.php
new file mode 100644
index 0000000000..e27ee134f2
--- /dev/null
+++ b/bitbake/lib/bb/tests/fetch-testdata/software/miniupnp/download.php
@@ -0,0 +1,3528 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+<title>MiniUPnP download zone</title>
+<link href="../css/miniupnp.css" rel="stylesheet" type="text/css"/>
+<meta name="description" content="files download of the miniupnp project"/>
+<meta name="keywords" content="upnp,download,openbsd,freebsd,linux,windows"/>
+<meta name="viewport" content="width=device-width" />
+<link href="rss.php" title="MiniUPnPd, MiniUPnPc and MiniSSDPd Files" type="application/rss+xml" rel="alternate" />
+<link rel="canonical" href="http://miniupnp.free.fr/files/" />
+<link rel="alternate" hreflang="fr" href="/files/index_fr.php" />
+<script async="async" src="//pagead2.googlesyndication.com/pagead/js/adsbygoogle.js" type="text/javascript"></script>
+<script type="text/javascript">
+     (adsbygoogle = window.adsbygoogle || []).push({
+          google_ad_client: "ca-pub-6883148866513192",
+          enable_page_level_ads: true
+     });
+</script>
+</head>
+<body>
+<h2>MiniUPnP Project</h2>
+
+<p align="center">
+<a href="../">Home</a> |
+<b>Downloads</b> |
+<a href="../devicelist.php">Compatibility list</a> |
+<a href="../libnatpmp.html">libnatpmp</a> |
+<a href="../minissdpd.html">MiniSSDPd</a> |
+<a href="../xchat-upnp.html">xchat upnp patch</a> |
+<a href="../search.html">Search</a> |
+<a href="https://miniupnp.tuxfamily.org/forum/">Forum</a>
+</p>
+<p align="center">
+<b>English</b> | <a href="/files/index_fr.php">Fran&ccedil;ais</a>
+</p>
+
+<div align="center">
+<script type="text/javascript"><!--
+google_ad_client = "pub-6883148866513192";
+/* 728x90, created 7/10/08 */
+google_ad_slot = "0774293141";
+google_ad_width = 728;
+google_ad_height = 90;
+//-->
+</script>
+<script type="text/javascript"
+src="https://pagead2.googlesyndication.com/pagead/show_ads.js">
+</script>
+</div>
+
+<h2>MiniUPnP download zone</h2>
+<p>
+Find on this page the source of miniupnp and 
+some related files. You will also find precompiled binaries
+of the UPnP client sample program for windows compiled using
+<a href="https://mingw.osdn.io/">MinGW</a>. There are also Windows
+binaries (including python module) automatically built using
+<a href="https://ci.appveyor.com/project/miniupnp/miniupnp/build/artifacts">AppVeyor</a>.
+</p>
+<p>If you just need one of the software installed on your machine,
+you probably don't need to download and compile the source files.
+It is very likely that a package/port already exists for
+your system/distribution. Refer to your system documentation
+to find how to search and install a package/port.
+Mac OS X does have port systems too : see
+<a href="http://www.macports.org/">MacPorts</a> or
+<a href="http://mxcl.github.com/homebrew/">Homebrew</a> or
+<a href="http://www.finkproject.org/">Fink</a>.
+</p>
+<p>
+The miniupnpc (client) sources have been successfully compiled
+under Windows XP/vista/7/10/etc. (using
+<a href="https://mingw.osdn.io/">MinGW</a>,
+<a href="https://www.mingw-w64.org/">Mingw-w64</a>
+or <a href="http://www.cygwin.com/">Cygwin</a>),
+Linux, OpenBSD, FreeBSD, NetBSD, DragonFlyBSD,
+Solaris, MacOS X and AmigaOS. <br/>
+The Makefile of the client is made for GNU make :
+check which version your system have 
+with the command "make --version". On some systems, such as OpenBSD, 
+you have to use "gmake". Under Windows with MinGW, GNU make is
+called "mingw32-make" and a sligthly modified version of the Makefile
+should be used : Makefile.mingw. Run "mingw32make.bat" to compile. <br/>
+If you have any compatibility problem, please post on the
+<a href="https://miniupnp.tuxfamily.org/forum/">forum</a>
+or contact me by email.
+</p>
+<!--
+<p>A devoted user compiled miniupnp<strong>c</strong> for
+Openwrt (currently Kamikaze 7.09)
+and his work is available here : 
+<a href="http://replay.waybackmachine.org/20081120030628/http://www.myantihero.net/pub/openwrt/packages/">http://myantihero.net/pub/openwrt/packages/</a>.</p>
+-->
+<p>Get miniupnpc under AmigaOS 4 on
+<a href="http://os4depot.net/index.php?function=showfile&amp;file=network/misc/miniupnpc.lha">OS4Depot</a>.
+</p>
+<p>
+Dario Meloni has made a Ruby Gem embedding miniupnpc :
+<a href="https://rubygems.org/gems/mupnp">https://rubygems.org/gems/mupnp</a>.
+</p>
+<p>
+The python module is available on pypi.org :
+<a href="https://pypi.org/project/miniupnpc/">pip install miniupnpc</a>.
+</p>
+<p>
+The daemon (starting in November 2006) compiles with BSD make under BSD
+and Solaris.<br/>
+To compile the daemon under linux, use "make -f Makefile.linux"<br/>
+To compile for <a href="http://openwrt.org/">OpenWRT</a>
+please read the README.openwrt file, or use the packages 
+<a href="https://openwrt.org/packages/pkgdata/miniupnpd">miniupnpc</a> and
+<a href="https://openwrt.org/packages/pkgdata/miniupnpd">miniupnpd</a>.
+<!-- The
+<a href="http://www.x-wrt.org/">X-Wrt</a> project is providing
+precompiled ipkg packages for OpenWrt for both OpenWrt 
+<a href="ftp://ftp.berlios.de/pub/xwrt/packages/">White Russian</a>
+and OpenWrt
+<a href="ftp://ftp.berlios.de/pub/xwrt/kamikaze/packages">kamikaze</a>. 
+Check
+<a href="ftp://ftp.berlios.de/pub/xwrt/">ftp://ftp.berlios.de/pub/xwrt/</a>.
+For White Russian, take a look at
+<a href="http://jackassofalltrades.com/openwrt/">this</a>. -->
+<br/>
+<a href="http://pfsense.com">pfSense</a> users are advised to use the 
+miniupnpd port available for their system. Recent versions of
+pfSense include MiniUPnPd in the base system.
+<br/>
+For <a href="http://en.wikipedia.org/wiki/WRT54G">Linksys WRT54G</a>
+and WRT54GL owners,
+<a href="http://sourceforge.net/projects/tarifa/">Tarifa firmware</a>
+is another alternative to get miniUPnPd running on the router.
+</p>
+<p>
+Please read README and
+LICENCE files included with the distribution for further informations.
+</p>
+<p>
+The MiniUPnP daemon (miniupnpd) is working under
+<a href="http://www.openbsd.org/">OpenBSD</a>,
+<a href="http://www.netbsd.org/">NetBSD</a>,
+<a href="http://www.freebsd.org/">FreeBSD</a>,
+<a href="http://www.dragonflybsd.org/">DragonFlyBSD</a>,
+<a href="http://www.apple.com/macosx/">Mac OS X</a> and
+(<a href="https://en.wikipedia.org/wiki/OpenSolaris">Open</a>)<a href="http://www.oracle.com/us/products/servers-storage/solaris/solaris11/overview/index.html">Solaris</a>
+with <a href="http://www.openbsd.org/faq/pf/">pf</a>,
+with <a href="https://en.wikipedia.org/wiki/IPFilter">IP Filter</a> or
+with <a href="http://en.wikipedia.org/wiki/Ipfirewall">ipfw</a>.
+The linux version uses either libiptc which permits to access
+<a href="http://netfilter.org/">netfilter</a>
+rules inside the kernel the same way as
+<a href="https://www.netfilter.org/projects/iptables/index.html">iptables</a>, or
+<a href="https://www.netfilter.org/projects/libnftnl/index.html">libnftnl</a>
+which is the equivalent for
+<a href="https://www.netfilter.org/projects/nftables/index.html">nftables</a>.
+</p>
+
+<p>Releases are now GPG signed with the key <a href="../A31ACAAF.asc">A31ACAAF</a>.
+Previous signing key was <a href="../A5C0863C.asc">A5C0863C</a>.
+Get it from your favorite
+<a href="https://pgp.mit.edu/pks/lookup?search=0xA31ACAAF&amp;op=index&amp;fingerprint=on">key server</a>.</p>
+
+<h4>REST API</h4>
+<p>You can use the REST API to get the latest releases available:</p>
+<ul>
+<li><a href="rest.php/tags/miniupnpd?count=1">rest.php/tags/miniupnpd?count=1</a>: latest miniupnpd.</li>
+<li><a href="rest.php/tags?count=1">rest.php/tags?count=1</a>: miniupnpc, miniupnpd and minissdpd.</li>
+</ul>
+
+<h4>You can help !</h4>
+<p>If you make a package/port for your favorite OS distribution,
+inform me so I can upload the package here or add a link to your 
+repository.
+</p>
+
+<h4>Latest files</h4>
+<table>
+<tr><th>name</th>
+<th>size</th>
+<th>date</th>
+<th>comment</th>
+<th><!-- Changelog --></th>
+<th><!-- Signature --></th>
+</tr>
+<tr>
+        <td class="filename"><a href='miniupnpc-2.3.2.tar.gz'>miniupnpc-2.3.2.tar.gz</a></td>
+        <td class="filesize">140137</td>
+        <td class="filedate">05/03/2025 10:31</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="changelog.php?file=miniupnpc-2.3.2.tar.gz">changelog</a></td>
+        <td><a href="miniupnpc-2.3.2.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='miniupnpd-2.3.7.tar.gz'>miniupnpd-2.3.7.tar.gz</a></td>
+        <td class="filesize">265329</td>
+        <td class="filedate">22/06/2024 22:31</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="changelog.php?file=miniupnpd-2.3.7.tar.gz">changelog</a></td>
+        <td><a href="miniupnpd-2.3.7.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='libnatpmp-20230423.tar.gz'>libnatpmp-20230423.tar.gz</a></td>
+        <td class="filesize">26506</td>
+        <td class="filedate">23/04/2023 11:02</td>
+        <td class="comment">latest libnatpmp source code</td>
+        <td><a href="changelog.php?file=libnatpmp-20230423.tar.gz">changelog</a></td>
+        <td><a href="libnatpmp-20230423.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='minissdpd-1.6.0.tar.gz'>minissdpd-1.6.0.tar.gz</a></td>
+        <td class="filesize">39077</td>
+        <td class="filedate">22/10/2022 18:41</td>
+        <td class="comment">MiniSSDPd release source code</td>
+        <td><a href="changelog.php?file=minissdpd-1.6.0.tar.gz">changelog</a></td>
+        <td><a href="minissdpd-1.6.0.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='upnpc-exe-win32-20220515.zip'>upnpc-exe-win32-20220515.zip</a></td>
+        <td class="filesize">69503</td>
+        <td class="filedate">15/05/2022 14:31</td>
+        <td class="comment">Windows executable</td>
+        <td><a href="changelog.php?file=upnpc-exe-win32-20220515.zip">changelog</a></td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='minissdpd-1.5.20211105.tar.gz'>minissdpd-1.5.20211105.tar.gz</a></td>
+        <td class="filesize">38870</td>
+        <td class="filedate">04/11/2021 23:34</td>
+        <td class="comment">latest MiniSSDPd source code</td>
+        <td><a href="changelog.php?file=minissdpd-1.5.20211105.tar.gz">changelog</a></td>
+        <td><a href="minissdpd-1.5.20211105.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='miniupnpc-2.1.20201016.tar.gz'>miniupnpc-2.1.20201016.tar.gz</a></td>
+        <td class="filesize">97682</td>
+        <td class="filedate">15/10/2020 22:31</td>
+        <td class="comment">latest MiniUPnP client source code</td>
+        <td><a href="changelog.php?file=miniupnpc-2.1.20201016.tar.gz">changelog</a></td>
+        <td><a href="miniupnpc-2.1.20201016.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='miniupnpd-2.1.20200510.tar.gz'>miniupnpd-2.1.20200510.tar.gz</a></td>
+        <td class="filesize">245426</td>
+        <td class="filedate">10/05/2020 18:23</td>
+        <td class="comment">latest MiniUPnP daemon source code</td>
+        <td><a href="changelog.php?file=miniupnpd-2.1.20200510.tar.gz">changelog</a></td>
+        <td><a href="miniupnpd-2.1.20200510.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='xchat-upnp20110811.patch'>xchat-upnp20110811.patch</a></td>
+        <td class="filesize">10329</td>
+        <td class="filedate">11/08/2011 15:18</td>
+        <td class="comment">Patch to add UPnP capabilities to xchat</td>
+        <td><a href="changelog.php?file=xchat-upnp20110811.patch">changelog</a></td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='minidlna_1.0.21.minissdp1.patch'>minidlna_1.0.21.minissdp1.patch</a></td>
+        <td class="filesize">7598</td>
+        <td class="filedate">25/07/2011 14:57</td>
+        <td class="comment">Patch for MiniDLNA to use miniSSDPD</td>
+        <td><a href="changelog.php?file=minidlna_1.0.21.minissdp1.patch">changelog</a></td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='miniupnpc-new20060630.tar.gz'>miniupnpc-new20060630.tar.gz</a></td>
+        <td class="filesize">14840</td>
+        <td class="filedate">04/11/2006 18:16</td>
+        <td class="comment">Jo&atilde;o Paulo Barraca version of the upnp client</td>
+        <td><a href="changelog.php?file=miniupnpc-new20060630.tar.gz">changelog</a></td>
+        <td></td>
+</tr>
+</table>
+
+<h4>All files</h4>
+<table>
+<tr><th>name</th>
+<th>size</th>
+<th>date</th>
+<th>comment</th>
+<th><!-- signature --></th>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.3.2.tar.gz'>miniupnpc-2.3.2.tar.gz</a></td>
+        <td class="filesize">140137</td>
+        <td class="filedate">05/03/2025 10:31:36 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.3.2.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.3.1.tar.gz'>miniupnpc-2.3.1.tar.gz</a></td>
+        <td class="filesize">139499</td>
+        <td class="filedate">23/02/2025 16:44:16 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.3.1.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.3.0.tar.gz'>miniupnpc-2.3.0.tar.gz</a></td>
+        <td class="filesize">105071</td>
+        <td class="filedate">10/01/2025 23:16:45 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.3.0.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.3.7.tar.gz'>miniupnpd-2.3.7.tar.gz</a></td>
+        <td class="filesize">265329</td>
+        <td class="filedate">22/06/2024 22:31:38 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.3.7.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.2.8.tar.gz'>miniupnpc-2.2.8.tar.gz</a></td>
+        <td class="filesize">104603</td>
+        <td class="filedate">08/06/2024 22:13:39 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.2.8.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.3.6.tar.gz'>miniupnpd-2.3.6.tar.gz</a></td>
+        <td class="filesize">263018</td>
+        <td class="filedate">19/03/2024 23:39:51 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.3.6.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.2.7.tar.gz'>miniupnpc-2.2.7.tar.gz</a></td>
+        <td class="filesize">104258</td>
+        <td class="filedate">19/03/2024 23:25:18 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.2.7.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.3.5.tar.gz'>miniupnpd-2.3.5.tar.gz</a></td>
+        <td class="filesize">261952</td>
+        <td class="filedate">02/03/2024 11:04:07 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.3.5.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.3.4.tar.gz'>miniupnpd-2.3.4.tar.gz</a></td>
+        <td class="filesize">260810</td>
+        <td class="filedate">04/01/2024 00:53:17 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.3.4.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.2.6.tar.gz'>miniupnpc-2.2.6.tar.gz</a></td>
+        <td class="filesize">103949</td>
+        <td class="filedate">04/01/2024 00:27:14 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.2.6.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.2.5.tar.gz'>miniupnpc-2.2.5.tar.gz</a></td>
+        <td class="filesize">103654</td>
+        <td class="filedate">11/06/2023 23:14:56 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.2.5.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20230423.tar.gz'>libnatpmp-20230423.tar.gz</a></td>
+        <td class="filesize">26506</td>
+        <td class="filedate">23/04/2023 11:02:09 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td><a href="libnatpmp-20230423.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.3.3.tar.gz'>miniupnpd-2.3.3.tar.gz</a></td>
+        <td class="filesize">260079</td>
+        <td class="filedate">17/02/2023 03:07:46 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.3.3.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.3.2.tar.gz'>miniupnpd-2.3.2.tar.gz</a></td>
+        <td class="filesize">259686</td>
+        <td class="filedate">19/01/2023 23:18:08 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.3.2.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.6.0.tar.gz'>minissdpd-1.6.0.tar.gz</a></td>
+        <td class="filesize">39077</td>
+        <td class="filedate">22/10/2022 18:41:54 +0000</td>
+        <td class="comment">MiniSSDPd release source code</td>
+        <td><a href="minissdpd-1.6.0.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.2.4.tar.gz'>miniupnpc-2.2.4.tar.gz</a></td>
+        <td class="filesize">102932</td>
+        <td class="filedate">21/10/2022 21:01:01 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.2.4.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.3.1.tar.gz'>miniupnpd-2.3.1.tar.gz</a></td>
+        <td class="filesize">258050</td>
+        <td class="filedate">16/10/2022 05:58:44 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.3.1.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20220515.zip'>upnpc-exe-win32-20220515.zip</a></td>
+        <td class="filesize">69503</td>
+        <td class="filedate">15/05/2022 14:31:25 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=hexchat-2.16.patch'>hexchat-2.16.patch</a></td>
+        <td class="filesize">8147</td>
+        <td class="filedate">19/03/2022 16:52:05 +0000</td>
+        <td class="comment"></td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.3.0.tar.gz'>miniupnpd-2.3.0.tar.gz</a></td>
+        <td class="filesize">256069</td>
+        <td class="filedate">23/01/2022 00:23:32 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.3.0.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.5.20211105.tar.gz'>minissdpd-1.5.20211105.tar.gz</a></td>
+        <td class="filesize">38870</td>
+        <td class="filedate">04/11/2021 23:34:49 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td><a href="minissdpd-1.5.20211105.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.2.3.tar.gz'>miniupnpc-2.2.3.tar.gz</a></td>
+        <td class="filesize">101360</td>
+        <td class="filedate">28/09/2021 21:43:32 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.2.3.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.2.3.tar.gz'>miniupnpd-2.2.3.tar.gz</a></td>
+        <td class="filesize">254752</td>
+        <td class="filedate">21/08/2021 08:35:13 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.2.3.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.2.2.tar.gz'>miniupnpd-2.2.2.tar.gz</a></td>
+        <td class="filesize">250649</td>
+        <td class="filedate">13/05/2021 11:30:11 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.2.2.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.2.2.tar.gz'>miniupnpc-2.2.2.tar.gz</a></td>
+        <td class="filesize">100008</td>
+        <td class="filedate">02/03/2021 23:44:52 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.2.2.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.2.1.tar.gz'>miniupnpd-2.2.1.tar.gz</a></td>
+        <td class="filesize">250023</td>
+        <td class="filedate">20/12/2020 18:08:08 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.2.1.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.2.1.tar.gz'>miniupnpc-2.2.1.tar.gz</a></td>
+        <td class="filesize">99595</td>
+        <td class="filedate">20/12/2020 18:08:02 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.2.1.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.2.0.tar.gz'>miniupnpc-2.2.0.tar.gz</a></td>
+        <td class="filesize">98348</td>
+        <td class="filedate">09/11/2020 19:51:24 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.2.0.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.2.0.tar.gz'>miniupnpd-2.2.0.tar.gz</a></td>
+        <td class="filesize">249858</td>
+        <td class="filedate">31/10/2020 09:20:59 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.2.0.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.2.0-RC3.tar.gz'>miniupnpd-2.2.0-RC3.tar.gz</a></td>
+        <td class="filesize">249879</td>
+        <td class="filedate">30/10/2020 21:49:49 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.2.0-RC3.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.1.20201016.tar.gz'>miniupnpc-2.1.20201016.tar.gz</a></td>
+        <td class="filesize">97682</td>
+        <td class="filedate">15/10/2020 22:31:09 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td><a href="miniupnpc-2.1.20201016.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.2.0-RC2.tar.gz'>miniupnpd-2.2.0-RC2.tar.gz</a></td>
+        <td class="filesize">248756</td>
+        <td class="filedate">28/09/2020 21:57:22 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.2.0-RC2.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.1.20200928.tar.gz'>miniupnpc-2.1.20200928.tar.gz</a></td>
+        <td class="filesize">96508</td>
+        <td class="filedate">28/09/2020 21:56:09 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td><a href="miniupnpc-2.1.20200928.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.5.20200928.tar.gz'>minissdpd-1.5.20200928.tar.gz</a></td>
+        <td class="filesize">37860</td>
+        <td class="filedate">28/09/2020 21:55:40 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td><a href="minissdpd-1.5.20200928.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.2.0-RC1.tar.gz'>miniupnpd-2.2.0-RC1.tar.gz</a></td>
+        <td class="filesize">247772</td>
+        <td class="filedate">06/06/2020 18:34:50 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.2.0-RC1.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.2.0-RC0.tar.gz'>miniupnpd-2.2.0-RC0.tar.gz</a></td>
+        <td class="filesize">245507</td>
+        <td class="filedate">16/05/2020 18:03:17 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.2.0-RC0.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20200510.tar.gz'>miniupnpd-2.1.20200510.tar.gz</a></td>
+        <td class="filesize">245426</td>
+        <td class="filedate">10/05/2020 18:23:13 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20200510.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20200329.tar.gz'>miniupnpd-2.1.20200329.tar.gz</a></td>
+        <td class="filesize">243725</td>
+        <td class="filedate">29/03/2020 09:11:02 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20200329.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.1.20191224.tar.gz'>miniupnpc-2.1.20191224.tar.gz</a></td>
+        <td class="filesize">94740</td>
+        <td class="filedate">23/12/2019 23:37:32 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td><a href="miniupnpc-2.1.20191224.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20191006.tar.gz'>miniupnpd-2.1.20191006.tar.gz</a></td>
+        <td class="filesize">243255</td>
+        <td class="filedate">06/10/2019 21:02:31 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20191006.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20191005.tar.gz'>miniupnpd-2.1.20191005.tar.gz</a></td>
+        <td class="filesize">244100</td>
+        <td class="filedate">05/10/2019 21:33:08 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20191005.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20191003.tar.gz'>miniupnpd-2.1.20191003.tar.gz</a></td>
+        <td class="filesize">243287</td>
+        <td class="filedate">02/10/2019 22:23:51 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20191003.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20190924.tar.gz'>miniupnpd-2.1.20190924.tar.gz</a></td>
+        <td class="filesize">241008</td>
+        <td class="filedate">24/09/2019 11:58:15 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20190924.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20190902.tar.gz'>miniupnpd-2.1.20190902.tar.gz</a></td>
+        <td class="filesize">240742</td>
+        <td class="filedate">01/09/2019 23:03:03 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20190902.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20190824.tar.gz'>miniupnpd-2.1.20190824.tar.gz</a></td>
+        <td class="filesize">240490</td>
+        <td class="filedate">24/08/2019 09:21:52 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20190824.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.5.20190824.tar.gz'>minissdpd-1.5.20190824.tar.gz</a></td>
+        <td class="filesize">37300</td>
+        <td class="filedate">24/08/2019 09:17:32 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td><a href="minissdpd-1.5.20190824.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.1.20190824.tar.gz'>miniupnpc-2.1.20190824.tar.gz</a></td>
+        <td class="filesize">94564</td>
+        <td class="filedate">24/08/2019 09:12:50 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td><a href="miniupnpc-2.1.20190824.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20190630.tar.gz'>miniupnpd-2.1.20190630.tar.gz</a></td>
+        <td class="filesize">240466</td>
+        <td class="filedate">30/06/2019 20:27:38 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20190630.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20190625.tar.gz'>miniupnpd-2.1.20190625.tar.gz</a></td>
+        <td class="filesize">240120</td>
+        <td class="filedate">25/06/2019 21:33:49 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20190625.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.1.20190625.tar.gz'>miniupnpc-2.1.20190625.tar.gz</a></td>
+        <td class="filesize">94461</td>
+        <td class="filedate">25/06/2019 21:33:26 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td><a href="miniupnpc-2.1.20190625.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20190502.tar.gz'>miniupnpd-2.1.20190502.tar.gz</a></td>
+        <td class="filesize">236052</td>
+        <td class="filedate">02/05/2019 17:22:23 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20190502.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.1.20190408.tar.gz'>miniupnpc-2.1.20190408.tar.gz</a></td>
+        <td class="filesize">94216</td>
+        <td class="filedate">08/04/2019 12:50:21 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td><a href="miniupnpc-2.1.20190408.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20190408.tar.gz'>miniupnpd-2.1.20190408.tar.gz</a></td>
+        <td class="filesize">235989</td>
+        <td class="filedate">08/04/2019 12:50:01 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20190408.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.1.20190403.tar.gz'>miniupnpc-2.1.20190403.tar.gz</a></td>
+        <td class="filesize">94204</td>
+        <td class="filedate">03/04/2019 15:41:36 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td><a href="miniupnpc-2.1.20190403.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20190403.tar.gz'>miniupnpd-2.1.20190403.tar.gz</a></td>
+        <td class="filesize">235909</td>
+        <td class="filedate">03/04/2019 15:41:17 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20190403.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.5.20190210.tar.gz'>minissdpd-1.5.20190210.tar.gz</a></td>
+        <td class="filesize">37227</td>
+        <td class="filedate">10/02/2019 15:21:49 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td><a href="minissdpd-1.5.20190210.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.1.20190210.tar.gz'>miniupnpc-2.1.20190210.tar.gz</a></td>
+        <td class="filesize">94125</td>
+        <td class="filedate">10/02/2019 12:46:09 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td><a href="miniupnpc-2.1.20190210.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20190210.tar.gz'>miniupnpd-2.1.20190210.tar.gz</a></td>
+        <td class="filesize">235093</td>
+        <td class="filedate">10/02/2019 11:20:11 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20190210.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.20180706.tar.gz'>miniupnpd-2.1.20180706.tar.gz</a></td>
+        <td class="filesize">233675</td>
+        <td class="filedate">06/07/2018 12:44:24 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td><a href="miniupnpd-2.1.20180706.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.1.tar.gz'>miniupnpd-2.1.tar.gz</a></td>
+        <td class="filesize">225458</td>
+        <td class="filedate">08/05/2018 21:50:32 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.1.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.1.tar.gz'>miniupnpc-2.1.tar.gz</a></td>
+        <td class="filesize">91914</td>
+        <td class="filedate">07/05/2018 11:10:59 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td><a href="miniupnpc-2.1.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.0.20180503.tar.gz'>miniupnpd-2.0.20180503.tar.gz</a></td>
+        <td class="filesize">225454</td>
+        <td class="filedate">03/05/2018 08:33:10 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.0.20180503.tar.gz'>miniupnpc-2.0.20180503.tar.gz</a></td>
+        <td class="filesize">88207</td>
+        <td class="filedate">03/05/2018 08:31:22 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.0.20180422.tar.gz'>miniupnpd-2.0.20180422.tar.gz</a></td>
+        <td class="filesize">224942</td>
+        <td class="filedate">22/04/2018 19:48:54 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.0.20180412.tar.gz'>miniupnpd-2.0.20180412.tar.gz</a></td>
+        <td class="filesize">224831</td>
+        <td class="filedate">12/04/2018 08:16:25 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.0.20180410.tar.gz'>miniupnpd-2.0.20180410.tar.gz</a></td>
+        <td class="filesize">224736</td>
+        <td class="filedate">10/04/2018 07:58:28 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.0.20180410.tar.gz'>miniupnpc-2.0.20180410.tar.gz</a></td>
+        <td class="filesize">87363</td>
+        <td class="filedate">10/04/2018 07:52:55 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.0.20180406.tar.gz'>miniupnpc-2.0.20180406.tar.gz</a></td>
+        <td class="filesize">87374</td>
+        <td class="filedate">06/04/2018 10:55:21 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.5.20180223.tar.gz'>minissdpd-1.5.20180223.tar.gz</a></td>
+        <td class="filesize">36179</td>
+        <td class="filedate">23/02/2018 14:24:07 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.0.20180222.tar.gz'>miniupnpc-2.0.20180222.tar.gz</a></td>
+        <td class="filesize">87018</td>
+        <td class="filedate">22/02/2018 15:09:24 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.0.20180222.tar.gz'>miniupnpd-2.0.20180222.tar.gz</a></td>
+        <td class="filesize">223697</td>
+        <td class="filedate">22/02/2018 15:09:14 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.0.20180203.tar.gz'>miniupnpd-2.0.20180203.tar.gz</a></td>
+        <td class="filesize">223084</td>
+        <td class="filedate">03/02/2018 22:34:46 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.0.20180203.tar.gz'>miniupnpc-2.0.20180203.tar.gz</a></td>
+        <td class="filesize">86772</td>
+        <td class="filedate">03/02/2018 22:34:32 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.5.20180203.tar.gz'>minissdpd-1.5.20180203.tar.gz</a></td>
+        <td class="filesize">35848</td>
+        <td class="filedate">03/02/2018 22:33:08 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.0.20171212.tar.gz'>miniupnpc-2.0.20171212.tar.gz</a></td>
+        <td class="filesize">86607</td>
+        <td class="filedate">12/12/2017 12:03:38 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.0.20171212.tar.gz'>miniupnpd-2.0.20171212.tar.gz</a></td>
+        <td class="filesize">222617</td>
+        <td class="filedate">12/12/2017 12:03:32 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.0.20171102.tar.gz'>miniupnpc-2.0.20171102.tar.gz</a></td>
+        <td class="filesize">86363</td>
+        <td class="filedate">02/11/2017 17:58:34 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.0.20170509.tar.gz'>miniupnpc-2.0.20170509.tar.gz</a></td>
+        <td class="filesize">86055</td>
+        <td class="filedate">09/05/2017 10:14:56 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.0.20170421.tar.gz'>miniupnpc-2.0.20170421.tar.gz</a></td>
+        <td class="filesize">85984</td>
+        <td class="filedate">21/04/2017 12:02:26 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.0.20170421.tar.gz'>miniupnpd-2.0.20170421.tar.gz</a></td>
+        <td class="filesize">219191</td>
+        <td class="filedate">21/04/2017 12:02:06 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.0.20161216.tar.gz'>miniupnpd-2.0.20161216.tar.gz</a></td>
+        <td class="filesize">218119</td>
+        <td class="filedate">16/12/2016 09:34:08 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.0.20161216.tar.gz'>miniupnpc-2.0.20161216.tar.gz</a></td>
+        <td class="filesize">85780</td>
+        <td class="filedate">16/12/2016 09:34:03 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.5.20161216.tar.gz'>minissdpd-1.5.20161216.tar.gz</a></td>
+        <td class="filesize">35078</td>
+        <td class="filedate">16/12/2016 09:33:59 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-2.0.tar.gz'>miniupnpd-2.0.tar.gz</a></td>
+        <td class="filesize">217802</td>
+        <td class="filedate">19/04/2016 21:12:01 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td><a href="miniupnpd-2.0.tar.gz.sig">Signature</a></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-2.0.tar.gz'>miniupnpc-2.0.tar.gz</a></td>
+        <td class="filesize">85287</td>
+        <td class="filedate">19/04/2016 21:07:52 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.5.20160301.tar.gz'>minissdpd-1.5.20160301.tar.gz</a></td>
+        <td class="filesize">34827</td>
+        <td class="filedate">01/03/2016 18:08:23 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20160222.tar.gz'>miniupnpd-1.9.20160222.tar.gz</a></td>
+        <td class="filesize">217541</td>
+        <td class="filedate">22/02/2016 10:21:40 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20160216.tar.gz'>miniupnpd-1.9.20160216.tar.gz</a></td>
+        <td class="filesize">217007</td>
+        <td class="filedate">16/02/2016 12:41:44 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20160212.tar.gz'>miniupnpd-1.9.20160212.tar.gz</a></td>
+        <td class="filesize">215866</td>
+        <td class="filedate">12/02/2016 15:22:04 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20160209.tar.gz'>miniupnpd-1.9.20160209.tar.gz</a></td>
+        <td class="filesize">213416</td>
+        <td class="filedate">09/02/2016 09:47:03 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20160209.tar.gz'>miniupnpc-1.9.20160209.tar.gz</a></td>
+        <td class="filesize">85268</td>
+        <td class="filedate">09/02/2016 09:44:50 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.5.20160119.tar.gz'>minissdpd-1.5.20160119.tar.gz</a></td>
+        <td class="filesize">34711</td>
+        <td class="filedate">19/01/2016 13:39:51 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20160113.tar.gz'>miniupnpd-1.9.20160113.tar.gz</a></td>
+        <td class="filesize">211437</td>
+        <td class="filedate">13/01/2016 16:03:14 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.5.tar.gz'>minissdpd-1.5.tar.gz</a></td>
+        <td class="filesize">34404</td>
+        <td class="filedate">13/01/2016 15:26:53 +0000</td>
+        <td class="comment">MiniSSDPd release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20151212.tar.gz'>miniupnpd-1.9.20151212.tar.gz</a></td>
+        <td class="filesize">210912</td>
+        <td class="filedate">12/12/2015 10:06:07 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20151118.tar.gz'>miniupnpd-1.9.20151118.tar.gz</a></td>
+        <td class="filesize">210322</td>
+        <td class="filedate">18/11/2015 08:59:46 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20151026.tar.gz'>miniupnpc-1.9.20151026.tar.gz</a></td>
+        <td class="filesize">84208</td>
+        <td class="filedate">26/10/2015 17:07:34 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20151008.tar.gz'>miniupnpc-1.9.20151008.tar.gz</a></td>
+        <td class="filesize">83538</td>
+        <td class="filedate">08/10/2015 16:22:28 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20150922.tar.gz'>miniupnpd-1.9.20150922.tar.gz</a></td>
+        <td class="filesize">208700</td>
+        <td class="filedate">22/09/2015 10:21:50 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20150918.zip'>upnpc-exe-win32-20150918.zip</a></td>
+        <td class="filesize">100004</td>
+        <td class="filedate">18/09/2015 12:50:51 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20150917.tar.gz'>miniupnpc-1.9.20150917.tar.gz</a></td>
+        <td class="filesize">82609</td>
+        <td class="filedate">17/09/2015 14:09:14 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20150824.zip'>upnpc-exe-win32-20150824.zip</a></td>
+        <td class="filesize">99520</td>
+        <td class="filedate">24/08/2015 15:25:18 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.4.tar.gz'>minissdpd-1.4.tar.gz</a></td>
+        <td class="filesize">32017</td>
+        <td class="filedate">06/08/2015 13:38:37 +0000</td>
+        <td class="comment">MiniSSDPd release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20150730.tar.gz'>miniupnpc-1.9.20150730.tar.gz</a></td>
+        <td class="filesize">81431</td>
+        <td class="filedate">29/07/2015 22:10:00 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20150721.tar.gz'>miniupnpd-1.9.20150721.tar.gz</a></td>
+        <td class="filesize">207562</td>
+        <td class="filedate">21/07/2015 13:35:51 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20150721.tar.gz'>miniupnpc-1.9.20150721.tar.gz</a></td>
+        <td class="filesize">80521</td>
+        <td class="filedate">21/07/2015 13:27:00 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20150609.tar.gz'>libnatpmp-20150609.tar.gz</a></td>
+        <td class="filesize">24392</td>
+        <td class="filedate">09/06/2015 15:40:28 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20150609.tar.gz'>miniupnpc-1.9.20150609.tar.gz</a></td>
+        <td class="filesize">79311</td>
+        <td class="filedate">09/06/2015 15:39:48 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20150609.tar.gz'>miniupnpd-1.9.20150609.tar.gz</a></td>
+        <td class="filesize">207088</td>
+        <td class="filedate">09/06/2015 15:39:36 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.3.20150527.tar.gz'>minissdpd-1.3.20150527.tar.gz</a></td>
+        <td class="filesize">31025</td>
+        <td class="filedate">27/05/2015 09:17:15 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20150522.tar.gz'>miniupnpc-1.9.20150522.tar.gz</a></td>
+        <td class="filesize">79080</td>
+        <td class="filedate">22/05/2015 11:02:27 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.3.20150522.tar.gz'>minissdpd-1.3.20150522.tar.gz</a></td>
+        <td class="filesize">30334</td>
+        <td class="filedate">22/05/2015 11:02:04 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20150430.tar.gz'>miniupnpd-1.9.20150430.tar.gz</a></td>
+        <td class="filesize">205930</td>
+        <td class="filedate">30/04/2015 09:09:27 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20150430.tar.gz'>miniupnpc-1.9.20150430.tar.gz</a></td>
+        <td class="filesize">78459</td>
+        <td class="filedate">30/04/2015 08:39:31 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20150427.tar.gz'>miniupnpc-1.9.20150427.tar.gz</a></td>
+        <td class="filesize">78424</td>
+        <td class="filedate">27/04/2015 16:08:42 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20150427.tar.gz'>miniupnpd-1.9.20150427.tar.gz</a></td>
+        <td class="filesize">191157</td>
+        <td class="filedate">27/04/2015 16:08:27 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20150307.tar.gz'>miniupnpd-1.9.20150307.tar.gz</a></td>
+        <td class="filesize">190913</td>
+        <td class="filedate">07/03/2015 16:11:51 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20150206.tar.gz'>miniupnpc-1.9.20150206.tar.gz</a></td>
+        <td class="filesize">76864</td>
+        <td class="filedate">06/02/2015 14:38:00 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20141209.tar.gz'>miniupnpd-1.9.20141209.tar.gz</a></td>
+        <td class="filesize">193183</td>
+        <td class="filedate">09/12/2014 09:58:34 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.3.tar.gz'>minissdpd-1.3.tar.gz</a></td>
+        <td class="filesize">30326</td>
+        <td class="filedate">09/12/2014 09:57:30 +0000</td>
+        <td class="comment">MiniSSDPd release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.2.20141204.tar.gz'>minissdpd-1.2.20141204.tar.gz</a></td>
+        <td class="filesize">26978</td>
+        <td class="filedate">04/12/2014 10:55:26 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20141204.tar.gz'>miniupnpd-1.9.20141204.tar.gz</a></td>
+        <td class="filesize">192597</td>
+        <td class="filedate">04/12/2014 10:55:03 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.2.20141128.tar.gz'>minissdpd-1.2.20141128.tar.gz</a></td>
+        <td class="filesize">26795</td>
+        <td class="filedate">28/11/2014 16:33:10 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20141128.tar.gz'>miniupnpd-1.9.20141128.tar.gz</a></td>
+        <td class="filesize">192558</td>
+        <td class="filedate">28/11/2014 13:31:36 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20141128.tar.gz'>miniupnpc-1.9.20141128.tar.gz</a></td>
+        <td class="filesize">76541</td>
+        <td class="filedate">28/11/2014 13:31:15 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20141117.tar.gz'>miniupnpc-1.9.20141117.tar.gz</a></td>
+        <td class="filesize">73865</td>
+        <td class="filedate">17/11/2014 09:51:36 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20141113.tar.gz'>miniupnpc-1.9.20141113.tar.gz</a></td>
+        <td class="filesize">72857</td>
+        <td class="filedate">13/11/2014 10:36:44 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.2.20141108.tar.gz'>minissdpd-1.2.20141108.tar.gz</a></td>
+        <td class="filesize">22001</td>
+        <td class="filedate">08/11/2014 13:55:41 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20141108.tar.gz'>miniupnpc-1.9.20141108.tar.gz</a></td>
+        <td class="filesize">72781</td>
+        <td class="filedate">08/11/2014 13:53:48 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.20141108.tar.gz'>miniupnpd-1.9.20141108.tar.gz</a></td>
+        <td class="filesize">192413</td>
+        <td class="filedate">08/11/2014 13:53:38 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.9.tar.gz'>miniupnpd-1.9.tar.gz</a></td>
+        <td class="filesize">192183</td>
+        <td class="filedate">27/10/2014 16:45:34 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20141027.tar.gz'>miniupnpc-1.9.20141027.tar.gz</a></td>
+        <td class="filesize">76763</td>
+        <td class="filedate">27/10/2014 16:45:25 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20141022.tar.gz'>miniupnpd-1.8.20141022.tar.gz</a></td>
+        <td class="filesize">191630</td>
+        <td class="filedate">22/10/2014 09:17:41 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20141021.tar.gz'>miniupnpd-1.8.20141021.tar.gz</a></td>
+        <td class="filesize">191270</td>
+        <td class="filedate">21/10/2014 14:18:58 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20140911.tar.gz'>miniupnpc-1.9.20140911.tar.gz</a></td>
+        <td class="filesize">76855</td>
+        <td class="filedate">11/09/2014 14:15:23 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.2.20140906.tar.gz'>minissdpd-1.2.20140906.tar.gz</a></td>
+        <td class="filesize">21956</td>
+        <td class="filedate">06/09/2014 08:34:10 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20140906.tar.gz'>miniupnpd-1.8.20140906.tar.gz</a></td>
+        <td class="filesize">191183</td>
+        <td class="filedate">06/09/2014 08:34:02 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20140906.tar.gz'>miniupnpc-1.9.20140906.tar.gz</a></td>
+        <td class="filesize">76791</td>
+        <td class="filedate">06/09/2014 08:33:45 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20140701.tar.gz'>miniupnpc-1.9.20140701.tar.gz</a></td>
+        <td class="filesize">76735</td>
+        <td class="filedate">01/07/2014 13:06:51 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20140610.tar.gz'>miniupnpc-1.9.20140610.tar.gz</a></td>
+        <td class="filesize">76674</td>
+        <td class="filedate">10/06/2014 10:28:27 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.2.20140610.tar.gz'>minissdpd-1.2.20140610.tar.gz</a></td>
+        <td class="filesize">21909</td>
+        <td class="filedate">10/06/2014 10:03:29 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20140523.tar.gz'>miniupnpd-1.8.20140523.tar.gz</a></td>
+        <td class="filesize">190936</td>
+        <td class="filedate">23/05/2014 15:48:03 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20140422.zip'>upnpc-exe-win32-20140422.zip</a></td>
+        <td class="filesize">97505</td>
+        <td class="filedate">22/04/2014 10:10:07 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20140422.tar.gz'>miniupnpd-1.8.20140422.tar.gz</a></td>
+        <td class="filesize">187225</td>
+        <td class="filedate">22/04/2014 08:58:56 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20140401.tar.gz'>miniupnpd-1.8.20140401.tar.gz</a></td>
+        <td class="filesize">183131</td>
+        <td class="filedate">01/04/2014 10:07:20 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.20140401.tar.gz'>miniupnpc-1.9.20140401.tar.gz</a></td>
+        <td class="filesize">74703</td>
+        <td class="filedate">01/04/2014 09:49:46 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20140401.tar.gz'>libnatpmp-20140401.tar.gz</a></td>
+        <td class="filesize">23302</td>
+        <td class="filedate">01/04/2014 09:49:44 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20140313.tar.gz'>miniupnpd-1.8.20140313.tar.gz</a></td>
+        <td class="filesize">177120</td>
+        <td class="filedate">13/03/2014 10:39:11 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20140310.tar.gz'>miniupnpd-1.8.20140310.tar.gz</a></td>
+        <td class="filesize">176585</td>
+        <td class="filedate">09/03/2014 23:16:49 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20140225.tar.gz'>miniupnpd-1.8.20140225.tar.gz</a></td>
+        <td class="filesize">175183</td>
+        <td class="filedate">25/02/2014 11:01:29 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20140203.tar.gz'>miniupnpd-1.8.20140203.tar.gz</a></td>
+        <td class="filesize">170112</td>
+        <td class="filedate">03/02/2014 09:56:05 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.9.tar.gz'>miniupnpc-1.9.tar.gz</a></td>
+        <td class="filesize">74230</td>
+        <td class="filedate">31/01/2014 13:57:40 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20140127.tar.gz'>miniupnpd-1.8.20140127.tar.gz</a></td>
+        <td class="filesize">170467</td>
+        <td class="filedate">27/01/2014 11:25:34 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20140117.zip'>upnpc-exe-win32-20140117.zip</a></td>
+        <td class="filesize">97270</td>
+        <td class="filedate">17/01/2014 11:37:53 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20131216.tar.gz'>miniupnpd-1.8.20131216.tar.gz</a></td>
+        <td class="filesize">170277</td>
+        <td class="filedate">16/12/2013 16:15:40 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20131213.tar.gz'>miniupnpd-1.8.20131213.tar.gz</a></td>
+        <td class="filesize">169753</td>
+        <td class="filedate">13/12/2013 16:18:10 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.8.20131209.tar.gz'>miniupnpc-1.8.20131209.tar.gz</a></td>
+        <td class="filesize">73900</td>
+        <td class="filedate">09/12/2013 20:52:54 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20131126.tar.gz'>libnatpmp-20131126.tar.gz</a></td>
+        <td class="filesize">22972</td>
+        <td class="filedate">26/11/2013 08:51:36 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.8.20131007.tar.gz'>miniupnpc-1.8.20131007.tar.gz</a></td>
+        <td class="filesize">73750</td>
+        <td class="filedate">07/10/2013 10:10:25 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20130911.tar.gz'>libnatpmp-20130911.tar.gz</a></td>
+        <td class="filesize">18744</td>
+        <td class="filedate">11/09/2013 07:35:51 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20130910.tar.gz'>libnatpmp-20130910.tar.gz</a></td>
+        <td class="filesize">18734</td>
+        <td class="filedate">10/09/2013 20:15:34 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.2.20130907.tar.gz'>minissdpd-1.2.20130907.tar.gz</a></td>
+        <td class="filesize">20237</td>
+        <td class="filedate">07/09/2013 06:46:31 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.2.20130819.tar.gz'>minissdpd-1.2.20130819.tar.gz</a></td>
+        <td class="filesize">20772</td>
+        <td class="filedate">19/08/2013 16:50:29 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.8.20130801.tar.gz'>miniupnpc-1.8.20130801.tar.gz</a></td>
+        <td class="filesize">73426</td>
+        <td class="filedate">01/08/2013 21:38:05 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20130730.tar.gz'>miniupnpd-1.8.20130730.tar.gz</a></td>
+        <td class="filesize">149904</td>
+        <td class="filedate">30/07/2013 11:37:48 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20130607.tar.gz'>miniupnpd-1.8.20130607.tar.gz</a></td>
+        <td class="filesize">149521</td>
+        <td class="filedate">07/06/2013 08:46:17 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20130521.tar.gz'>miniupnpd-1.8.20130521.tar.gz</a></td>
+        <td class="filesize">149276</td>
+        <td class="filedate">21/05/2013 09:01:33 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20130503.tar.gz'>miniupnpd-1.8.20130503.tar.gz</a></td>
+        <td class="filesize">148420</td>
+        <td class="filedate">03/05/2013 19:27:16 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.8.20130503.tar.gz'>miniupnpc-1.8.20130503.tar.gz</a></td>
+        <td class="filesize">71858</td>
+        <td class="filedate">03/05/2013 19:27:07 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20130426.tar.gz'>miniupnpd-1.8.20130426.tar.gz</a></td>
+        <td class="filesize">147890</td>
+        <td class="filedate">26/04/2013 16:57:20 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.8.20130211.tar.gz'>miniupnpc-1.8.20130211.tar.gz</a></td>
+        <td class="filesize">70723</td>
+        <td class="filedate">11/02/2013 10:32:44 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.20130207.tar.gz'>miniupnpd-1.8.20130207.tar.gz</a></td>
+        <td class="filesize">147325</td>
+        <td class="filedate">07/02/2013 12:29:32 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.8.tar.gz'>miniupnpc-1.8.tar.gz</a></td>
+        <td class="filesize">70624</td>
+        <td class="filedate">06/02/2013 14:31:06 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.8.tar.gz'>miniupnpd-1.8.tar.gz</a></td>
+        <td class="filesize">146679</td>
+        <td class="filedate">06/02/2013 14:30:59 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20121009.zip'>upnpc-exe-win32-20121009.zip</a></td>
+        <td class="filesize">96513</td>
+        <td class="filedate">09/10/2012 17:54:12 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.7.20121005.tar.gz'>miniupnpd-1.7.20121005.tar.gz</a></td>
+        <td class="filesize">144393</td>
+        <td class="filedate">04/10/2012 22:39:05 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.7.20120830.tar.gz'>miniupnpc-1.7.20120830.tar.gz</a></td>
+        <td class="filesize">70074</td>
+        <td class="filedate">30/08/2012 08:41:51 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.7.20120824.tar.gz'>miniupnpd-1.7.20120824.tar.gz</a></td>
+        <td class="filesize">141960</td>
+        <td class="filedate">24/08/2012 18:15:01 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20120821.tar.gz'>libnatpmp-20120821.tar.gz</a></td>
+        <td class="filesize">17832</td>
+        <td class="filedate">21/08/2012 17:24:46 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.7.20120714.tar.gz'>miniupnpc-1.7.20120714.tar.gz</a></td>
+        <td class="filesize">69570</td>
+        <td class="filedate">14/07/2012 14:40:47 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.7.20120711.tar.gz'>miniupnpc-1.7.20120711.tar.gz</a></td>
+        <td class="filesize">69580</td>
+        <td class="filedate">10/07/2012 22:27:05 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.7.20120711.tar.gz'>miniupnpd-1.7.20120711.tar.gz</a></td>
+        <td class="filesize">141380</td>
+        <td class="filedate">10/07/2012 22:26:58 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.7.tar.gz'>miniupnpd-1.7.tar.gz</a></td>
+        <td class="filesize">138047</td>
+        <td class="filedate">27/05/2012 23:13:30 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.7.tar.gz'>miniupnpc-1.7.tar.gz</a></td>
+        <td class="filesize">68327</td>
+        <td class="filedate">24/05/2012 18:17:48 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.2.tar.gz'>minissdpd-1.2.tar.gz</a></td>
+        <td class="filesize">19874</td>
+        <td class="filedate">24/05/2012 18:06:24 +0000</td>
+        <td class="comment">MiniSSDPd release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120509.tar.gz'>miniupnpd-1.6.20120509.tar.gz</a></td>
+        <td class="filesize">137147</td>
+        <td class="filedate">09/05/2012 10:45:44 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.6.20120509.tar.gz'>miniupnpc-1.6.20120509.tar.gz</a></td>
+        <td class="filesize">68205</td>
+        <td class="filedate">09/05/2012 10:45:41 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.1.20120509.tar.gz'>minissdpd-1.1.20120509.tar.gz</a></td>
+        <td class="filesize">18123</td>
+        <td class="filedate">09/05/2012 10:45:39 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120502.tar.gz'>miniupnpd-1.6.20120502.tar.gz</a></td>
+        <td class="filesize">136688</td>
+        <td class="filedate">01/05/2012 22:51:18 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.6.20120502.tar.gz'>miniupnpc-1.6.20120502.tar.gz</a></td>
+        <td class="filesize">68170</td>
+        <td class="filedate">01/05/2012 22:51:11 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120426.tar.gz'>miniupnpd-1.6.20120426.tar.gz</a></td>
+        <td class="filesize">134764</td>
+        <td class="filedate">26/04/2012 16:24:29 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120424.tar.gz'>miniupnpd-1.6.20120424.tar.gz</a></td>
+        <td class="filesize">132522</td>
+        <td class="filedate">23/04/2012 22:43:17 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.6.20120424.tar.gz'>miniupnpc-1.6.20120424.tar.gz</a></td>
+        <td class="filesize">68067</td>
+        <td class="filedate">23/04/2012 22:43:10 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120420.tar.gz'>miniupnpd-1.6.20120420.tar.gz</a></td>
+        <td class="filesize">131972</td>
+        <td class="filedate">20/04/2012 14:58:57 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.6.20120420.tar.gz'>miniupnpc-1.6.20120420.tar.gz</a></td>
+        <td class="filesize">68068</td>
+        <td class="filedate">20/04/2012 14:58:39 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120419.tar.gz'>miniupnpd-1.6.20120419.tar.gz</a></td>
+        <td class="filesize">131088</td>
+        <td class="filedate">18/04/2012 23:41:36 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120418.tar.gz'>miniupnpd-1.6.20120418.tar.gz</a></td>
+        <td class="filesize">130879</td>
+        <td class="filedate">18/04/2012 21:01:10 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.1.20120410.tar.gz'>minissdpd-1.1.20120410.tar.gz</a></td>
+        <td class="filesize">18059</td>
+        <td class="filedate">09/04/2012 22:45:38 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.6.20120410.tar.gz'>miniupnpc-1.6.20120410.tar.gz</a></td>
+        <td class="filesize">67934</td>
+        <td class="filedate">09/04/2012 22:45:10 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120406.tar.gz'>miniupnpd-1.6.20120406.tar.gz</a></td>
+        <td class="filesize">128992</td>
+        <td class="filedate">06/04/2012 17:52:57 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.6.20120320.tar.gz'>miniupnpc-1.6.20120320.tar.gz</a></td>
+        <td class="filesize">67374</td>
+        <td class="filedate">20/03/2012 16:55:48 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120320.tar.gz'>miniupnpd-1.6.20120320.tar.gz</a></td>
+        <td class="filesize">127968</td>
+        <td class="filedate">20/03/2012 16:46:07 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120305.tar.gz'>miniupnpd-1.6.20120305.tar.gz</a></td>
+        <td class="filesize">126985</td>
+        <td class="filedate">05/03/2012 20:42:01 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120207.tar.gz'>miniupnpd-1.6.20120207.tar.gz</a></td>
+        <td class="filesize">127425</td>
+        <td class="filedate">07/02/2012 10:21:16 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120203.tar.gz'>miniupnpd-1.6.20120203.tar.gz</a></td>
+        <td class="filesize">126599</td>
+        <td class="filedate">03/02/2012 15:14:13 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.6.20120125.tar.gz'>miniupnpc-1.6.20120125.tar.gz</a></td>
+        <td class="filesize">67354</td>
+        <td class="filedate">25/01/2012 21:12:28 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.6.20120121.tar.gz'>miniupnpc-1.6.20120121.tar.gz</a></td>
+        <td class="filesize">67347</td>
+        <td class="filedate">21/01/2012 14:07:41 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20120121.tar.gz'>miniupnpd-1.6.20120121.tar.gz</a></td>
+        <td class="filesize">126021</td>
+        <td class="filedate">21/01/2012 14:07:33 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.1.20120121.tar.gz'>minissdpd-1.1.20120121.tar.gz</a></td>
+        <td class="filesize">17762</td>
+        <td class="filedate">21/01/2012 14:07:16 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20120121.zip'>upnpc-exe-win32-20120121.zip</a></td>
+        <td class="filesize">94575</td>
+        <td class="filedate">21/01/2012 13:59:11 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20111212.zip'>upnpc-exe-win32-20111212.zip</a></td>
+        <td class="filesize">94507</td>
+        <td class="filedate">12/12/2011 12:33:48 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20111118.tar.gz'>miniupnpd-1.6.20111118.tar.gz</a></td>
+        <td class="filesize">125683</td>
+        <td class="filedate">18/11/2011 11:26:12 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.1.20111007.tar.gz'>minissdpd-1.1.20111007.tar.gz</a></td>
+        <td class="filesize">17611</td>
+        <td class="filedate">07/10/2011 09:47:51 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=xchat-upnp20110811.patch'>xchat-upnp20110811.patch</a></td>
+        <td class="filesize">10329</td>
+        <td class="filedate">11/08/2011 15:18:25 +0000</td>
+        <td class="comment">Patch to add UPnP capabilities to xchat</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=xchat-upnp20110811-2.8.8.patch'>xchat-upnp20110811-2.8.8.patch</a></td>
+        <td class="filesize">11529</td>
+        <td class="filedate">11/08/2011 15:18:23 +0000</td>
+        <td class="comment">Patch to add UPnP capabilities to xchat</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20110808.tar.gz'>libnatpmp-20110808.tar.gz</a></td>
+        <td class="filesize">17762</td>
+        <td class="filedate">08/08/2011 21:21:34 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20110730.tar.gz'>libnatpmp-20110730.tar.gz</a></td>
+        <td class="filesize">17687</td>
+        <td class="filedate">30/07/2011 13:19:31 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.1.tar.gz'>minissdpd-1.1.tar.gz</a></td>
+        <td class="filesize">17481</td>
+        <td class="filedate">30/07/2011 13:17:30 +0000</td>
+        <td class="comment">MiniSSDPd release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.20110730.tar.gz'>miniupnpd-1.6.20110730.tar.gz</a></td>
+        <td class="filesize">125583</td>
+        <td class="filedate">30/07/2011 13:17:09 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.0.20110729.tar.gz'>minissdpd-1.0.20110729.tar.gz</a></td>
+        <td class="filesize">15898</td>
+        <td class="filedate">29/07/2011 08:47:26 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.6.tar.gz'>miniupnpc-1.6.tar.gz</a></td>
+        <td class="filesize">66454</td>
+        <td class="filedate">25/07/2011 18:03:09 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.6.tar.gz'>miniupnpd-1.6.tar.gz</a></td>
+        <td class="filesize">124917</td>
+        <td class="filedate">25/07/2011 16:37:57 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minidlna_1.0.21.minissdp1.patch'>minidlna_1.0.21.minissdp1.patch</a></td>
+        <td class="filesize">7598</td>
+        <td class="filedate">25/07/2011 14:57:50 +0000</td>
+        <td class="comment">Patch for MiniDLNA to use miniSSDPD</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20110715.tar.gz'>libnatpmp-20110715.tar.gz</a></td>
+        <td class="filesize">17943</td>
+        <td class="filedate">15/07/2011 08:31:40 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110715.tar.gz'>miniupnpd-1.5.20110715.tar.gz</a></td>
+        <td class="filesize">124519</td>
+        <td class="filedate">15/07/2011 07:55:17 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20110714.zip'>upnpc-exe-win32-20110714.zip</a></td>
+        <td class="filesize">94236</td>
+        <td class="filedate">13/07/2011 23:16:01 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110623.tar.gz'>miniupnpd-1.5.20110623.tar.gz</a></td>
+        <td class="filesize">123529</td>
+        <td class="filedate">22/06/2011 22:29:15 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110620.tar.gz'>miniupnpd-1.5.20110620.tar.gz</a></td>
+        <td class="filesize">123221</td>
+        <td class="filedate">20/06/2011 14:11:11 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110618.tar.gz'>miniupnpd-1.5.20110618.tar.gz</a></td>
+        <td class="filesize">123176</td>
+        <td class="filedate">17/06/2011 23:29:18 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.5.20110618.tar.gz'>miniupnpc-1.5.20110618.tar.gz</a></td>
+        <td class="filesize">66401</td>
+        <td class="filedate">17/06/2011 23:29:17 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20110618.tar.gz'>libnatpmp-20110618.tar.gz</a></td>
+        <td class="filesize">17901</td>
+        <td class="filedate">17/06/2011 23:29:16 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.0.20110618.tar.gz'>minissdpd-1.0.20110618.tar.gz</a></td>
+        <td class="filesize">15193</td>
+        <td class="filedate">17/06/2011 23:29:16 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename" colspan="2"><a href='download.php?file=minidlna_cvs20110529_minissdp1.patch'>minidlna_cvs20110529_minissdp1.patch</a></td>
+        <td class="filedate">29/05/2011 21:19:09 +0000</td>
+        <td class="comment">Patch for MiniDLNA to use miniSSDPD</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110528.tar.gz'>miniupnpd-1.5.20110528.tar.gz</a></td>
+        <td class="filesize">121985</td>
+        <td class="filedate">28/05/2011 09:39:04 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minidlna_1.0.19_minissdp1.patch'>minidlna_1.0.19_minissdp1.patch</a></td>
+        <td class="filesize">9080</td>
+        <td class="filedate">27/05/2011 09:55:04 +0000</td>
+        <td class="comment">Patch for MiniDLNA to use miniSSDPD</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110527.tar.gz'>miniupnpd-1.5.20110527.tar.gz</a></td>
+        <td class="filesize">120896</td>
+        <td class="filedate">27/05/2011 08:28:35 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.5.20110527.tar.gz'>miniupnpc-1.5.20110527.tar.gz</a></td>
+        <td class="filesize">66279</td>
+        <td class="filedate">27/05/2011 08:28:34 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20110527.tar.gz'>libnatpmp-20110527.tar.gz</a></td>
+        <td class="filesize">17627</td>
+        <td class="filedate">27/05/2011 08:28:33 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.0.20110523.tar.gz'>minissdpd-1.0.20110523.tar.gz</a></td>
+        <td class="filesize">15024</td>
+        <td class="filedate">23/05/2011 12:55:31 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110520.tar.gz'>miniupnpd-1.5.20110520.tar.gz</a></td>
+        <td class="filesize">119227</td>
+        <td class="filedate">20/05/2011 18:00:41 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110519.tar.gz'>miniupnpd-1.5.20110519.tar.gz</a></td>
+        <td class="filesize">114735</td>
+        <td class="filedate">18/05/2011 22:29:06 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110516.tar.gz'>miniupnpd-1.5.20110516.tar.gz</a></td>
+        <td class="filesize">113348</td>
+        <td class="filedate">16/05/2011 09:32:51 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110515.tar.gz'>miniupnpd-1.5.20110515.tar.gz</a></td>
+        <td class="filesize">113135</td>
+        <td class="filedate">15/05/2011 21:51:29 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.5.20110515.tar.gz'>miniupnpc-1.5.20110515.tar.gz</a></td>
+        <td class="filesize">66112</td>
+        <td class="filedate">15/05/2011 21:51:28 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110513.tar.gz'>miniupnpd-1.5.20110513.tar.gz</a></td>
+        <td class="filesize">111029</td>
+        <td class="filedate">13/05/2011 14:03:12 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.5.20110506.tar.gz'>miniupnpc-1.5.20110506.tar.gz</a></td>
+        <td class="filesize">65536</td>
+        <td class="filedate">06/05/2011 16:35:38 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4-v6.20100505.zip'>miniupnpc-1.4-v6.20100505.zip</a></td>
+        <td class="filesize">91833</td>
+        <td class="filedate">18/04/2011 20:14:11 +0000</td>
+        <td class="comment"></td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.4-v6.20100823.zip'>miniupnpd-1.4-v6.20100823.zip</a></td>
+        <td class="filesize">222235</td>
+        <td class="filedate">18/04/2011 20:14:07 +0000</td>
+        <td class="comment"></td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.5.20110418.tar.gz'>miniupnpc-1.5.20110418.tar.gz</a></td>
+        <td class="filesize">61820</td>
+        <td class="filedate">18/04/2011 20:09:22 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20110418.zip'>upnpc-exe-win32-20110418.zip</a></td>
+        <td class="filesize">94183</td>
+        <td class="filedate">18/04/2011 17:53:26 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.5.20110314.tar.gz'>miniupnpc-1.5.20110314.tar.gz</a></td>
+        <td class="filesize">57210</td>
+        <td class="filedate">14/03/2011 14:27:29 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110309.tar.gz'>miniupnpd-1.5.20110309.tar.gz</a></td>
+        <td class="filesize">100073</td>
+        <td class="filedate">09/03/2011 15:36:12 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110302.tar.gz'>miniupnpd-1.5.20110302.tar.gz</a></td>
+        <td class="filesize">100756</td>
+        <td class="filedate">02/03/2011 16:17:44 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110221.tar.gz'>miniupnpd-1.5.20110221.tar.gz</a></td>
+        <td class="filesize">100092</td>
+        <td class="filedate">20/02/2011 23:48:17 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20110215.zip'>upnpc-exe-win32-20110215.zip</a></td>
+        <td class="filesize">55409</td>
+        <td class="filedate">15/02/2011 23:05:00 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.5.20110215.tar.gz'>miniupnpc-1.5.20110215.tar.gz</a></td>
+        <td class="filesize">54880</td>
+        <td class="filedate">15/02/2011 11:16:04 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110214.tar.gz'>miniupnpd-1.5.20110214.tar.gz</a></td>
+        <td class="filesize">99629</td>
+        <td class="filedate">14/02/2011 18:00:43 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minidlna_1.0.18_minissdp1.patch'>minidlna_1.0.18_minissdp1.patch</a></td>
+        <td class="filesize">9747</td>
+        <td class="filedate">02/02/2011 15:12:19 +0000</td>
+        <td class="comment">Patch for MiniDLNA to use miniSSDPD</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.20110127.tar.gz'>miniupnpd-1.5.20110127.tar.gz</a></td>
+        <td class="filesize">97421</td>
+        <td class="filedate">27/01/2011 17:51:25 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.5.tar.gz'>miniupnpd-1.5.tar.gz</a></td>
+        <td class="filesize">98993</td>
+        <td class="filedate">04/01/2011 09:45:10 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.5.tar.gz'>miniupnpc-1.5.tar.gz</a></td>
+        <td class="filesize">53309</td>
+        <td class="filedate">04/01/2011 09:45:06 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20110103.tar.gz'>libnatpmp-20110103.tar.gz</a></td>
+        <td class="filesize">17529</td>
+        <td class="filedate">03/01/2011 17:33:16 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20101221.tar.gz'>miniupnpc-1.4.20101221.tar.gz</a></td>
+        <td class="filesize">52342</td>
+        <td class="filedate">21/12/2010 16:15:38 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20101213.zip'>upnpc-exe-win32-20101213.zip</a></td>
+        <td class="filesize">52359</td>
+        <td class="filedate">12/12/2010 23:44:01 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20101211.tar.gz'>libnatpmp-20101211.tar.gz</a></td>
+        <td class="filesize">17324</td>
+        <td class="filedate">11/12/2010 17:20:36 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20101209.tar.gz'>miniupnpc-1.4.20101209.tar.gz</a></td>
+        <td class="filesize">51900</td>
+        <td class="filedate">09/12/2010 16:17:30 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.4.20100921.tar.gz'>miniupnpd-1.4.20100921.tar.gz</a></td>
+        <td class="filesize">95483</td>
+        <td class="filedate">21/09/2010 15:50:00 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20100825.zip'>upnpc-exe-win32-20100825.zip</a></td>
+        <td class="filesize">50636</td>
+        <td class="filedate">25/08/2010 08:42:59 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20100609.tar.gz'>miniupnpc-1.4.20100609.tar.gz</a></td>
+        <td class="filesize">50390</td>
+        <td class="filedate">09/06/2010 11:03:11 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20100513.zip'>upnpc-exe-win32-20100513.zip</a></td>
+        <td class="filesize">50950</td>
+        <td class="filedate">13/05/2010 16:54:33 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.4.20100511.tar.gz'>miniupnpd-1.4.20100511.tar.gz</a></td>
+        <td class="filesize">93281</td>
+        <td class="filedate">11/05/2010 16:22:33 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20100418.zip'>upnpc-exe-win32-20100418.zip</a></td>
+        <td class="filesize">40758</td>
+        <td class="filedate">17/04/2010 23:00:37 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20100418.tar.gz'>miniupnpc-1.4.20100418.tar.gz</a></td>
+        <td class="filesize">50245</td>
+        <td class="filedate">17/04/2010 22:18:31 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20100412.tar.gz'>miniupnpc-1.4.20100412.tar.gz</a></td>
+        <td class="filesize">50145</td>
+        <td class="filedate">12/04/2010 20:42:53 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20100407.tar.gz'>miniupnpc-1.4.20100407.tar.gz</a></td>
+        <td class="filesize">49756</td>
+        <td class="filedate">07/04/2010 10:05:08 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20100405.tar.gz'>miniupnpc-1.4.20100405.tar.gz</a></td>
+        <td class="filesize">49549</td>
+        <td class="filedate">05/04/2010 14:34:38 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.4.20100308.tar.gz'>miniupnpd-1.4.20100308.tar.gz</a></td>
+        <td class="filesize">92889</td>
+        <td class="filedate">08/03/2010 17:18:00 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20100202.tar.gz'>libnatpmp-20100202.tar.gz</a></td>
+        <td class="filesize">17231</td>
+        <td class="filedate">02/02/2010 18:41:13 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20100202.tar.gz'>miniupnpc-1.4.20100202.tar.gz</a></td>
+        <td class="filesize">46710</td>
+        <td class="filedate">02/02/2010 18:41:13 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20100106.tar.gz'>miniupnpc-1.4.20100106.tar.gz</a></td>
+        <td class="filesize">46659</td>
+        <td class="filedate">06/01/2010 10:08:21 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.4.20091222.tar.gz'>miniupnpd-1.4.20091222.tar.gz</a></td>
+        <td class="filesize">90993</td>
+        <td class="filedate">22/12/2009 17:23:48 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20091219.tar.gz'>libnatpmp-20091219.tar.gz</a></td>
+        <td class="filesize">16839</td>
+        <td class="filedate">19/12/2009 14:35:22 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20091213.tar.gz'>miniupnpc-1.4.20091213.tar.gz</a></td>
+        <td class="filesize">46510</td>
+        <td class="filedate">12/12/2009 23:05:40 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20091211.tar.gz'>miniupnpc-1.4.20091211.tar.gz</a></td>
+        <td class="filesize">45852</td>
+        <td class="filedate">11/12/2009 16:43:01 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20091210.zip'>upnpc-exe-win32-20091210.zip</a></td>
+        <td class="filesize">38666</td>
+        <td class="filedate">10/12/2009 18:50:27 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20091208.tar.gz'>miniupnpc-1.4.20091208.tar.gz</a></td>
+        <td class="filesize">43392</td>
+        <td class="filedate">08/12/2009 10:58:26 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.20091203.tar.gz'>miniupnpc-1.4.20091203.tar.gz</a></td>
+        <td class="filesize">42040</td>
+        <td class="filedate">03/12/2009 13:56:28 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.4.20091106.tar.gz'>miniupnpd-1.4.20091106.tar.gz</a></td>
+        <td class="filesize">90787</td>
+        <td class="filedate">06/11/2009 21:18:50 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.4.tar.gz'>miniupnpd-1.4.tar.gz</a></td>
+        <td class="filesize">90071</td>
+        <td class="filedate">30/10/2009 09:20:05 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.4.tar.gz'>miniupnpc-1.4.tar.gz</a></td>
+        <td class="filesize">41790</td>
+        <td class="filedate">30/10/2009 09:20:04 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20091016.tar.gz'>miniupnpc-20091016.tar.gz</a></td>
+        <td class="filesize">41792</td>
+        <td class="filedate">16/10/2009 09:04:35 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20091010.tar.gz'>miniupnpd-20091010.tar.gz</a></td>
+        <td class="filesize">90043</td>
+        <td class="filedate">10/10/2009 19:21:30 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20091010.tar.gz'>miniupnpc-20091010.tar.gz</a></td>
+        <td class="filesize">41671</td>
+        <td class="filedate">10/10/2009 19:21:28 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20090921.tar.gz'>miniupnpd-20090921.tar.gz</a></td>
+        <td class="filesize">89476</td>
+        <td class="filedate">21/09/2009 13:00:04 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20090921.tar.gz'>miniupnpc-20090921.tar.gz</a></td>
+        <td class="filesize">41630</td>
+        <td class="filedate">21/09/2009 13:00:03 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20090904.tar.gz'>miniupnpd-20090904.tar.gz</a></td>
+        <td class="filesize">89344</td>
+        <td class="filedate">04/09/2009 16:24:26 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20090820.tar.gz'>miniupnpd-20090820.tar.gz</a></td>
+        <td class="filesize">89149</td>
+        <td class="filedate">20/08/2009 09:35:58 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20090807.tar.gz'>miniupnpc-20090807.tar.gz</a></td>
+        <td class="filesize">41288</td>
+        <td class="filedate">07/08/2009 14:46:11 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20090729.tar.gz'>miniupnpc-20090729.tar.gz</a></td>
+        <td class="filesize">40588</td>
+        <td class="filedate">29/07/2009 08:47:43 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=xchat-upnp20061022.patch'>xchat-upnp20061022.patch</a></td>
+        <td class="filesize">10258</td>
+        <td class="filedate">17/07/2009 15:49:46 +0000</td>
+        <td class="comment">Patch to add UPnP capabilities to xchat</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20090713.tar.gz'>miniupnpc-20090713.tar.gz</a></td>
+        <td class="filesize">40206</td>
+        <td class="filedate">13/07/2009 08:53:49 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20090713.tar.gz'>libnatpmp-20090713.tar.gz</a></td>
+        <td class="filesize">14262</td>
+        <td class="filedate">13/07/2009 08:53:49 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20090605.tar.gz'>miniupnpd-20090605.tar.gz</a></td>
+        <td class="filesize">83774</td>
+        <td class="filedate">04/06/2009 23:32:20 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20090605.tar.gz'>miniupnpc-20090605.tar.gz</a></td>
+        <td class="filesize">40077</td>
+        <td class="filedate">04/06/2009 23:32:16 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20090605.tar.gz'>libnatpmp-20090605.tar.gz</a></td>
+        <td class="filesize">13817</td>
+        <td class="filedate">04/06/2009 23:32:15 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20090516.tar.gz'>miniupnpd-20090516.tar.gz</a></td>
+        <td class="filesize">83689</td>
+        <td class="filedate">16/05/2009 08:47:31 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.3.tar.gz'>miniupnpc-1.3.tar.gz</a></td>
+        <td class="filesize">40058</td>
+        <td class="filedate">17/04/2009 21:27:55 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.3.tar.gz'>miniupnpd-1.3.tar.gz</a></td>
+        <td class="filesize">83464</td>
+        <td class="filedate">17/04/2009 20:11:21 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20090310.tar.gz'>libnatpmp-20090310.tar.gz</a></td>
+        <td class="filesize">11847</td>
+        <td class="filedate">10/03/2009 10:19:45 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20090214.tar.gz'>miniupnpd-20090214.tar.gz</a></td>
+        <td class="filesize">82921</td>
+        <td class="filedate">14/02/2009 11:27:03 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20090213.tar.gz'>miniupnpd-20090213.tar.gz</a></td>
+        <td class="filesize">82594</td>
+        <td class="filedate">13/02/2009 19:48:01 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20090129.tar.gz'>libnatpmp-20090129.tar.gz</a></td>
+        <td class="filesize">11748</td>
+        <td class="filedate">29/01/2009 21:50:31 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20090129.tar.gz'>miniupnpc-20090129.tar.gz</a></td>
+        <td class="filesize">39976</td>
+        <td class="filedate">29/01/2009 21:50:30 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20090129.tar.gz'>miniupnpd-20090129.tar.gz</a></td>
+        <td class="filesize">82487</td>
+        <td class="filedate">29/01/2009 21:50:27 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20081009.tar.gz'>miniupnpd-20081009.tar.gz</a></td>
+        <td class="filesize">81732</td>
+        <td class="filedate">09/10/2008 12:53:02 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.0.tar.gz'>minissdpd-1.0.tar.gz</a></td>
+        <td class="filesize">12996</td>
+        <td class="filedate">07/10/2008 14:03:49 +0000</td>
+        <td class="comment">MiniSSDPd release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.2.tar.gz'>miniupnpc-1.2.tar.gz</a></td>
+        <td class="filesize">38787</td>
+        <td class="filedate">07/10/2008 14:03:47 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.2.tar.gz'>miniupnpd-1.2.tar.gz</a></td>
+        <td class="filesize">81025</td>
+        <td class="filedate">07/10/2008 14:03:45 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20081006.tar.gz'>miniupnpd-20081006.tar.gz</a></td>
+        <td class="filesize">80510</td>
+        <td class="filedate">06/10/2008 15:50:34 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-20081006.tar.gz'>minissdpd-20081006.tar.gz</a></td>
+        <td class="filesize">12230</td>
+        <td class="filedate">06/10/2008 15:50:33 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20081006.tar.gz'>libnatpmp-20081006.tar.gz</a></td>
+        <td class="filesize">11710</td>
+        <td class="filedate">06/10/2008 15:50:31 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename" colspan="2"><a href='download.php?file=mediatomb_minissdp-20081006.patch'>mediatomb_minissdp-20081006.patch</a></td>
+        <td class="filedate">06/10/2008 15:48:18 +0000</td>
+        <td class="comment"></td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20081002.tar.gz'>miniupnpc-20081002.tar.gz</a></td>
+        <td class="filesize">38291</td>
+        <td class="filedate">02/10/2008 09:20:18 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20081001.tar.gz'>miniupnpd-20081001.tar.gz</a></td>
+        <td class="filesize">79696</td>
+        <td class="filedate">01/10/2008 13:11:20 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20080925.zip'>upnpc-exe-win32-20080925.zip</a></td>
+        <td class="filesize">36602</td>
+        <td class="filedate">25/09/2008 06:59:33 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20080710.tar.gz'>miniupnpd-20080710.tar.gz</a></td>
+        <td class="filesize">78898</td>
+        <td class="filedate">10/07/2008 09:38:41 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20080707.tar.gz'>libnatpmp-20080707.tar.gz</a></td>
+        <td class="filesize">11679</td>
+        <td class="filedate">06/07/2008 22:05:23 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.1.tar.gz'>miniupnpc-1.1.tar.gz</a></td>
+        <td class="filesize">38235</td>
+        <td class="filedate">04/07/2008 16:45:24 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20080703.tar.gz'>miniupnpc-20080703.tar.gz</a></td>
+        <td class="filesize">38204</td>
+        <td class="filedate">03/07/2008 15:47:37 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20080703.tar.gz'>libnatpmp-20080703.tar.gz</a></td>
+        <td class="filesize">11570</td>
+        <td class="filedate">03/07/2008 15:47:25 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20080703.zip'>upnpc-exe-win32-20080703.zip</a></td>
+        <td class="filesize">36137</td>
+        <td class="filedate">02/07/2008 23:35:14 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20080702.tar.gz'>libnatpmp-20080702.tar.gz</a></td>
+        <td class="filesize">8873</td>
+        <td class="filedate">02/07/2008 17:32:35 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20080630.tar.gz'>libnatpmp-20080630.tar.gz</a></td>
+        <td class="filesize">8864</td>
+        <td class="filedate">30/06/2008 14:20:16 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20080529.tar.gz'>libnatpmp-20080529.tar.gz</a></td>
+        <td class="filesize">7397</td>
+        <td class="filedate">29/05/2008 09:06:25 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20080514.zip'>upnpc-exe-win32-20080514.zip</a></td>
+        <td class="filesize">14227</td>
+        <td class="filedate">14/05/2008 20:23:19 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20080428.tar.gz'>libnatpmp-20080428.tar.gz</a></td>
+        <td class="filesize">7295</td>
+        <td class="filedate">28/04/2008 03:09:14 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20080427.tar.gz'>miniupnpd-20080427.tar.gz</a></td>
+        <td class="filesize">78765</td>
+        <td class="filedate">27/04/2008 18:16:36 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20080427.tar.gz'>miniupnpc-20080427.tar.gz</a></td>
+        <td class="filesize">37610</td>
+        <td class="filedate">27/04/2008 18:16:35 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.1.tar.gz'>miniupnpd-1.1.tar.gz</a></td>
+        <td class="filesize">78594</td>
+        <td class="filedate">25/04/2008 17:38:05 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20080423.tar.gz'>miniupnpc-20080423.tar.gz</a></td>
+        <td class="filesize">36818</td>
+        <td class="filedate">23/04/2008 11:57:36 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20080308.tar.gz'>miniupnpd-20080308.tar.gz</a></td>
+        <td class="filesize">75679</td>
+        <td class="filedate">08/03/2008 11:13:29 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20080303.tar.gz'>miniupnpd-20080303.tar.gz</a></td>
+        <td class="filesize">74202</td>
+        <td class="filedate">03/03/2008 01:43:16 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20080224.tar.gz'>miniupnpd-20080224.tar.gz</a></td>
+        <td class="filesize">72773</td>
+        <td class="filedate">24/02/2008 11:23:17 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.0.tar.gz'>miniupnpc-1.0.tar.gz</a></td>
+        <td class="filesize">36223</td>
+        <td class="filedate">21/02/2008 13:26:46 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20080221.tar.gz'>miniupnpd-20080221.tar.gz</a></td>
+        <td class="filesize">70823</td>
+        <td class="filedate">21/02/2008 10:23:46 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20080217.tar.gz'>miniupnpc-20080217.tar.gz</a></td>
+        <td class="filesize">35243</td>
+        <td class="filedate">16/02/2008 23:47:59 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20080207.tar.gz'>miniupnpd-20080207.tar.gz</a></td>
+        <td class="filesize">70647</td>
+        <td class="filedate">07/02/2008 21:21:00 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20080203.tar.gz'>miniupnpc-20080203.tar.gz</a></td>
+        <td class="filesize">34921</td>
+        <td class="filedate">03/02/2008 22:28:11 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0.tar.gz'>miniupnpd-1.0.tar.gz</a></td>
+        <td class="filesize">69427</td>
+        <td class="filedate">27/01/2008 22:41:25 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20080118.zip'>upnpc-exe-win32-20080118.zip</a></td>
+        <td class="filesize">13582</td>
+        <td class="filedate">18/01/2008 11:42:16 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC13.tar.gz'>miniupnpd-1.0-RC13.tar.gz</a></td>
+        <td class="filesize">67892</td>
+        <td class="filedate">03/01/2008 16:50:21 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.0-RC13.tar.gz'>miniupnpc-1.0-RC13.tar.gz</a></td>
+        <td class="filesize">34820</td>
+        <td class="filedate">03/01/2008 16:50:20 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20071220.tar.gz'>miniupnpd-20071220.tar.gz</a></td>
+        <td class="filesize">67211</td>
+        <td class="filedate">20/12/2007 12:08:34 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20071219.tar.gz'>miniupnpc-20071219.tar.gz</a></td>
+        <td class="filesize">34290</td>
+        <td class="filedate">19/12/2007 18:31:47 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.0-RC12.tar.gz'>minissdpd-1.0-RC12.tar.gz</a></td>
+        <td class="filesize">9956</td>
+        <td class="filedate">19/12/2007 18:30:12 +0000</td>
+        <td class="comment">MiniSSDPd release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC12.tar.gz'>miniupnpd-1.0-RC12.tar.gz</a></td>
+        <td class="filesize">66911</td>
+        <td class="filedate">14/12/2007 17:39:20 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.0-RC12.tar.gz'>miniupnpc-1.0-RC12.tar.gz</a></td>
+        <td class="filesize">32543</td>
+        <td class="filedate">14/12/2007 17:39:19 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20071213.tar.gz'>miniupnpc-20071213.tar.gz</a></td>
+        <td class="filesize">32541</td>
+        <td class="filedate">13/12/2007 17:09:51 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20071213.tar.gz'>miniupnpd-20071213.tar.gz</a></td>
+        <td class="filesize">66826</td>
+        <td class="filedate">13/12/2007 16:42:50 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20071213.tar.gz'>libnatpmp-20071213.tar.gz</a></td>
+        <td class="filesize">5997</td>
+        <td class="filedate">13/12/2007 14:56:30 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=libnatpmp-20071202.tar.gz'>libnatpmp-20071202.tar.gz</a></td>
+        <td class="filesize">5664</td>
+        <td class="filedate">02/12/2007 00:15:28 +0000</td>
+        <td class="comment">libnatpmp source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20071103.tar.gz'>miniupnpd-20071103.tar.gz</a></td>
+        <td class="filesize">65740</td>
+        <td class="filedate">02/11/2007 23:58:38 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20071102.tar.gz'>miniupnpd-20071102.tar.gz</a></td>
+        <td class="filesize">65733</td>
+        <td class="filedate">02/11/2007 23:05:44 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20071103.tar.gz'>miniupnpc-20071103.tar.gz</a></td>
+        <td class="filesize">32239</td>
+        <td class="filedate">02/11/2007 23:05:34 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC11.tar.gz'>miniupnpd-1.0-RC11.tar.gz</a></td>
+        <td class="filesize">64828</td>
+        <td class="filedate">25/10/2007 13:27:18 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.0-RC11.tar.gz'>miniupnpc-1.0-RC11.tar.gz</a></td>
+        <td class="filesize">32161</td>
+        <td class="filedate">25/10/2007 13:27:17 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20071025.zip'>upnpc-exe-win32-20071025.zip</a></td>
+        <td class="filesize">12809</td>
+        <td class="filedate">24/10/2007 23:15:55 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC10.tar.gz'>miniupnpd-1.0-RC10.tar.gz</a></td>
+        <td class="filesize">62674</td>
+        <td class="filedate">12/10/2007 08:38:33 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.0-RC10.tar.gz'>miniupnpc-1.0-RC10.tar.gz</a></td>
+        <td class="filesize">31962</td>
+        <td class="filedate">12/10/2007 08:38:31 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.0-RC10.tar.gz'>minissdpd-1.0-RC10.tar.gz</a></td>
+        <td class="filesize">9517</td>
+        <td class="filedate">12/10/2007 08:38:30 +0000</td>
+        <td class="comment">MiniSSDPd release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20071003.tar.gz'>miniupnpc-20071003.tar.gz</a></td>
+        <td class="filesize">31199</td>
+        <td class="filedate">03/10/2007 15:30:13 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20071001.zip'>upnpc-exe-win32-20071001.zip</a></td>
+        <td class="filesize">12604</td>
+        <td class="filedate">01/10/2007 17:09:22 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC9.tar.gz'>miniupnpd-1.0-RC9.tar.gz</a></td>
+        <td class="filesize">54778</td>
+        <td class="filedate">27/09/2007 19:38:36 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-1.0-RC9.tar.gz'>minissdpd-1.0-RC9.tar.gz</a></td>
+        <td class="filesize">9163</td>
+        <td class="filedate">27/09/2007 17:00:03 +0000</td>
+        <td class="comment">MiniSSDPd release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.0-RC9.tar.gz'>miniupnpc-1.0-RC9.tar.gz</a></td>
+        <td class="filesize">30538</td>
+        <td class="filedate">27/09/2007 17:00:03 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070924.tar.gz'>miniupnpd-20070924.tar.gz</a></td>
+        <td class="filesize">52338</td>
+        <td class="filedate">24/09/2007 20:26:05 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070923.tar.gz'>miniupnpd-20070923.tar.gz</a></td>
+        <td class="filesize">51060</td>
+        <td class="filedate">23/09/2007 21:13:34 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20070923.tar.gz'>miniupnpc-20070923.tar.gz</a></td>
+        <td class="filesize">30246</td>
+        <td class="filedate">23/09/2007 21:13:33 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-20070923.tar.gz'>minissdpd-20070923.tar.gz</a></td>
+        <td class="filesize">8978</td>
+        <td class="filedate">23/09/2007 21:13:32 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20070902.tar.gz'>miniupnpc-20070902.tar.gz</a></td>
+        <td class="filesize">30205</td>
+        <td class="filedate">01/09/2007 23:47:23 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=minissdpd-20070902.tar.gz'>minissdpd-20070902.tar.gz</a></td>
+        <td class="filesize">6539</td>
+        <td class="filedate">01/09/2007 23:47:20 +0000</td>
+        <td class="comment">MiniSSDPd source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC8.tar.gz'>miniupnpd-1.0-RC8.tar.gz</a></td>
+        <td class="filesize">50952</td>
+        <td class="filedate">29/08/2007 10:56:09 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.0-RC8.tar.gz'>miniupnpc-1.0-RC8.tar.gz</a></td>
+        <td class="filesize">29312</td>
+        <td class="filedate">29/08/2007 10:56:08 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC7.tar.gz'>miniupnpd-1.0-RC7.tar.gz</a></td>
+        <td class="filesize">50613</td>
+        <td class="filedate">20/07/2007 00:15:45 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC6.tar.gz'>miniupnpd-1.0-RC6.tar.gz</a></td>
+        <td class="filesize">49986</td>
+        <td class="filedate">12/06/2007 17:12:07 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.0-RC6.tar.gz'>miniupnpc-1.0-RC6.tar.gz</a></td>
+        <td class="filesize">29032</td>
+        <td class="filedate">12/06/2007 17:12:06 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070607.tar.gz'>miniupnpd-20070607.tar.gz</a></td>
+        <td class="filesize">49768</td>
+        <td class="filedate">06/06/2007 23:12:00 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070605.tar.gz'>miniupnpd-20070605.tar.gz</a></td>
+        <td class="filesize">49710</td>
+        <td class="filedate">05/06/2007 21:01:53 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070521.tar.gz'>miniupnpd-20070521.tar.gz</a></td>
+        <td class="filesize">48374</td>
+        <td class="filedate">21/05/2007 13:07:43 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20070519.zip'>upnpc-exe-win32-20070519.zip</a></td>
+        <td class="filesize">10836</td>
+        <td class="filedate">19/05/2007 13:14:15 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20070515.tar.gz'>miniupnpc-20070515.tar.gz</a></td>
+        <td class="filesize">25802</td>
+        <td class="filedate">15/05/2007 18:15:25 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC5.tar.gz'>miniupnpd-1.0-RC5.tar.gz</a></td>
+        <td class="filesize">48064</td>
+        <td class="filedate">10/05/2007 20:22:48 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.0-RC5.tar.gz'>miniupnpc-1.0-RC5.tar.gz</a></td>
+        <td class="filesize">25242</td>
+        <td class="filedate">10/05/2007 20:22:46 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070412.tar.gz'>miniupnpd-20070412.tar.gz</a></td>
+        <td class="filesize">47807</td>
+        <td class="filedate">12/04/2007 20:21:48 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC4.tar.gz'>miniupnpd-1.0-RC4.tar.gz</a></td>
+        <td class="filesize">47687</td>
+        <td class="filedate">17/03/2007 11:43:13 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.0-RC4.tar.gz'>miniupnpc-1.0-RC4.tar.gz</a></td>
+        <td class="filesize">25085</td>
+        <td class="filedate">17/03/2007 11:43:10 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070311.tar.gz'>miniupnpd-20070311.tar.gz</a></td>
+        <td class="filesize">47599</td>
+        <td class="filedate">11/03/2007 00:25:26 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070208.tar.gz'>miniupnpd-20070208.tar.gz</a></td>
+        <td class="filesize">45084</td>
+        <td class="filedate">07/02/2007 23:04:06 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC3.tar.gz'>miniupnpd-1.0-RC3.tar.gz</a></td>
+        <td class="filesize">44683</td>
+        <td class="filedate">30/01/2007 23:00:44 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.0-RC3.tar.gz'>miniupnpc-1.0-RC3.tar.gz</a></td>
+        <td class="filesize">25055</td>
+        <td class="filedate">30/01/2007 23:00:42 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070130.tar.gz'>miniupnpd-20070130.tar.gz</a></td>
+        <td class="filesize">43735</td>
+        <td class="filedate">29/01/2007 23:26:16 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20070130.tar.gz'>miniupnpc-20070130.tar.gz</a></td>
+        <td class="filesize">24466</td>
+        <td class="filedate">29/01/2007 23:26:13 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070127.tar.gz'>miniupnpd-20070127.tar.gz</a></td>
+        <td class="filesize">42643</td>
+        <td class="filedate">27/01/2007 16:02:35 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20070127.tar.gz'>miniupnpc-20070127.tar.gz</a></td>
+        <td class="filesize">24241</td>
+        <td class="filedate">27/01/2007 16:02:33 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC2.tar.gz'>miniupnpd-1.0-RC2.tar.gz</a></td>
+        <td class="filesize">40424</td>
+        <td class="filedate">17/01/2007 16:13:05 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070112.tar.gz'>miniupnpd-20070112.tar.gz</a></td>
+        <td class="filesize">40708</td>
+        <td class="filedate">12/01/2007 13:40:50 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070111.tar.gz'>miniupnpd-20070111.tar.gz</a></td>
+        <td class="filesize">40651</td>
+        <td class="filedate">11/01/2007 18:50:21 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070108.tar.gz'>miniupnpd-20070108.tar.gz</a></td>
+        <td class="filesize">40025</td>
+        <td class="filedate">08/01/2007 10:02:14 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20070103.tar.gz'>miniupnpd-20070103.tar.gz</a></td>
+        <td class="filesize">40065</td>
+        <td class="filedate">03/01/2007 14:39:11 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-20061214.tar.gz'>miniupnpc-20061214.tar.gz</a></td>
+        <td class="filesize">24106</td>
+        <td class="filedate">14/12/2006 15:43:54 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-20061214.tar.gz'>miniupnpd-20061214.tar.gz</a></td>
+        <td class="filesize">39750</td>
+        <td class="filedate">14/12/2006 13:44:51 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd-1.0-RC1.tar.gz'>miniupnpd-1.0-RC1.tar.gz</a></td>
+        <td class="filesize">39572</td>
+        <td class="filedate">07/12/2006 10:55:31 +0000</td>
+        <td class="comment">MiniUPnP daemon release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-1.0-RC1.tar.gz'>miniupnpc-1.0-RC1.tar.gz</a></td>
+        <td class="filesize">23582</td>
+        <td class="filedate">07/12/2006 10:55:30 +0000</td>
+        <td class="comment">MiniUPnP client release source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20061201.zip'>upnpc-exe-win32-20061201.zip</a></td>
+        <td class="filesize">10378</td>
+        <td class="filedate">01/12/2006 00:33:08 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061130.tar.gz'>miniupnpd20061130.tar.gz</a></td>
+        <td class="filesize">37184</td>
+        <td class="filedate">30/11/2006 12:25:25 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061129.tar.gz'>miniupnpd20061129.tar.gz</a></td>
+        <td class="filesize">36045</td>
+        <td class="filedate">29/11/2006 00:10:49 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061127.tar.gz'>miniupnpd20061127.tar.gz</a></td>
+        <td class="filesize">34155</td>
+        <td class="filedate">26/11/2006 23:15:28 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20061123.tar.gz'>miniupnpc20061123.tar.gz</a></td>
+        <td class="filesize">21004</td>
+        <td class="filedate">23/11/2006 22:41:46 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename" colspan="2"><a href='download.php?file=miniupnpd-bin-openwrt20061123.tar.gz'>miniupnpd-bin-openwrt20061123.tar.gz</a></td>
+        <td class="filedate">23/11/2006 22:41:44 +0000</td>
+        <td class="comment">Precompiled binaries for openwrt</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061123.tar.gz'>miniupnpd20061123.tar.gz</a></td>
+        <td class="filesize">33809</td>
+        <td class="filedate">23/11/2006 22:28:29 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20061119.tar.gz'>miniupnpc20061119.tar.gz</a></td>
+        <td class="filesize">20897</td>
+        <td class="filedate">19/11/2006 22:50:37 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061119.tar.gz'>miniupnpd20061119.tar.gz</a></td>
+        <td class="filesize">32580</td>
+        <td class="filedate">19/11/2006 22:50:36 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061117.tar.gz'>miniupnpd20061117.tar.gz</a></td>
+        <td class="filesize">32646</td>
+        <td class="filedate">17/11/2006 13:29:33 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20061112.zip'>upnpc-exe-win32-20061112.zip</a></td>
+        <td class="filesize">10262</td>
+        <td class="filedate">12/11/2006 22:41:25 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061112.tar.gz'>miniupnpd20061112.tar.gz</a></td>
+        <td class="filesize">32023</td>
+        <td class="filedate">12/11/2006 21:30:32 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20061112.tar.gz'>miniupnpc20061112.tar.gz</a></td>
+        <td class="filesize">21047</td>
+        <td class="filedate">12/11/2006 21:30:31 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061110.tar.gz'>miniupnpd20061110.tar.gz</a></td>
+        <td class="filesize">27926</td>
+        <td class="filedate">09/11/2006 23:35:02 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20061110.tar.gz'>miniupnpc20061110.tar.gz</a></td>
+        <td class="filesize">21009</td>
+        <td class="filedate">09/11/2006 23:32:19 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20061101.zip'>upnpc-exe-win32-20061101.zip</a></td>
+        <td class="filesize">10089</td>
+        <td class="filedate">08/11/2006 20:35:09 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20061020.zip'>upnpc-exe-win32-20061020.zip</a></td>
+        <td class="filesize">9183</td>
+        <td class="filedate">08/11/2006 20:35:08 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20060909.zip'>upnpc-exe-win32-20060909.zip</a></td>
+        <td class="filesize">9994</td>
+        <td class="filedate">08/11/2006 20:35:07 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20060801.zip'>upnpc-exe-win32-20060801.zip</a></td>
+        <td class="filesize">10002</td>
+        <td class="filedate">08/11/2006 20:35:06 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20060711.zip'>upnpc-exe-win32-20060711.zip</a></td>
+        <td class="filesize">13733</td>
+        <td class="filedate">08/11/2006 20:35:05 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20060709.zip'>upnpc-exe-win32-20060709.zip</a></td>
+        <td class="filesize">13713</td>
+        <td class="filedate">08/11/2006 20:35:04 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=upnpc-exe-win32-20060704.zip'>upnpc-exe-win32-20060704.zip</a></td>
+        <td class="filesize">13297</td>
+        <td class="filedate">08/11/2006 20:35:03 +0000</td>
+        <td class="comment">Windows executable</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20061107.tar.gz'>miniupnpc20061107.tar.gz</a></td>
+        <td class="filesize">20708</td>
+        <td class="filedate">06/11/2006 23:36:57 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061107.tar.gz'>miniupnpd20061107.tar.gz</a></td>
+        <td class="filesize">26992</td>
+        <td class="filedate">06/11/2006 23:35:06 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20061106.tar.gz'>miniupnpc20061106.tar.gz</a></td>
+        <td class="filesize">20575</td>
+        <td class="filedate">06/11/2006 17:02:15 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061106.tar.gz'>miniupnpd20061106.tar.gz</a></td>
+        <td class="filesize">26597</td>
+        <td class="filedate">06/11/2006 15:39:10 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20061101.tar.gz'>miniupnpc20061101.tar.gz</a></td>
+        <td class="filesize">20395</td>
+        <td class="filedate">04/11/2006 18:16:15 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20061031.tar.gz'>miniupnpc20061031.tar.gz</a></td>
+        <td class="filesize">20396</td>
+        <td class="filedate">04/11/2006 18:16:13 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20061023.tar.gz'>miniupnpc20061023.tar.gz</a></td>
+        <td class="filesize">20109</td>
+        <td class="filedate">04/11/2006 18:16:12 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20061020.tar.gz'>miniupnpc20061020.tar.gz</a></td>
+        <td class="filesize">19739</td>
+        <td class="filedate">04/11/2006 18:16:10 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20060909.tar.gz'>miniupnpc20060909.tar.gz</a></td>
+        <td class="filesize">19285</td>
+        <td class="filedate">04/11/2006 18:16:09 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20060731.tar.gz'>miniupnpc20060731.tar.gz</a></td>
+        <td class="filesize">19032</td>
+        <td class="filedate">04/11/2006 18:16:07 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20060711.tar.gz'>miniupnpc20060711.tar.gz</a></td>
+        <td class="filesize">19151</td>
+        <td class="filedate">04/11/2006 18:16:06 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20060709.tar.gz'>miniupnpc20060709.tar.gz</a></td>
+        <td class="filesize">19080</td>
+        <td class="filedate">04/11/2006 18:16:04 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc20060703.tar.gz'>miniupnpc20060703.tar.gz</a></td>
+        <td class="filesize">17906</td>
+        <td class="filedate">04/11/2006 18:16:03 +0000</td>
+        <td class="comment">MiniUPnP client source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpc-new20060630.tar.gz'>miniupnpc-new20060630.tar.gz</a></td>
+        <td class="filesize">14840</td>
+        <td class="filedate">04/11/2006 18:16:01 +0000</td>
+        <td class="comment">Jo&atilde;o Paulo Barraca version of the upnp client</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061029.tar.gz'>miniupnpd20061029.tar.gz</a></td>
+        <td class="filesize">24197</td>
+        <td class="filedate">03/11/2006 13:40:30 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061027.tar.gz'>miniupnpd20061027.tar.gz</a></td>
+        <td class="filesize">23904</td>
+        <td class="filedate">03/11/2006 13:40:29 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061028.tar.gz'>miniupnpd20061028.tar.gz</a></td>
+        <td class="filesize">24383</td>
+        <td class="filedate">03/11/2006 13:40:29 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061018.tar.gz'>miniupnpd20061018.tar.gz</a></td>
+        <td class="filesize">23051</td>
+        <td class="filedate">03/11/2006 13:40:28 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20061023.tar.gz'>miniupnpd20061023.tar.gz</a></td>
+        <td class="filesize">23478</td>
+        <td class="filedate">03/11/2006 13:40:28 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20060930.tar.gz'>miniupnpd20060930.tar.gz</a></td>
+        <td class="filesize">22832</td>
+        <td class="filedate">03/11/2006 13:40:28 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20060924.tar.gz'>miniupnpd20060924.tar.gz</a></td>
+        <td class="filesize">22038</td>
+        <td class="filedate">03/11/2006 13:40:27 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20060919.tar.gz'>miniupnpd20060919.tar.gz</a></td>
+        <td class="filesize">21566</td>
+        <td class="filedate">03/11/2006 13:40:27 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20060729.tar.gz'>miniupnpd20060729.tar.gz</a></td>
+        <td class="filesize">19202</td>
+        <td class="filedate">03/11/2006 13:40:26 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+<tr>
+        <td class="filename"><a href='download.php?file=miniupnpd20060909.tar.gz'>miniupnpd20060909.tar.gz</a></td>
+        <td class="filesize">19952</td>
+        <td class="filedate">03/11/2006 13:40:26 +0000</td>
+        <td class="comment">MiniUPnP daemon source code</td>
+        <td></td>
+</tr>
+</table>
+
+<p><a href="..">Home</a></p>
+<p>Contact: miniupnp _AT_ free _DOT_ fr</p>
+<p align="center">
+<a href="https://validator.w3.org/check?uri=referer"><img src="https://www.w3.org/Icons/valid-xhtml10" alt="Valid XHTML 1.0 Transitional" height="31" width="88" /></a>
+<a href="https://jigsaw.w3.org/css-validator/check/referer"><img style="border:0;width:88px;height:31px" src="https://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!" /></a>
+<!--
+  <a href="https://freshmeat.net/projects/miniupnp"><img src="https://s3.amazonaws.com/entp-tender-production/assets/bc5be96f147ec8db3c10fc017f1f53889904ef5b/fm_logo_white_150_normal.png" border="0" alt="freshmeat.net" /></a>
+-->
+<!-- https://futuresimple.github.com/images/github_logo.png -->
+<!-- <a href="https://github.com/miniupnp/miniupnp"><img src="https://assets-cdn.github.com/images/modules/logos_page/GitHub-Logo.png" alt="github.com" height="31" /></a> -->
+<a href="https://github.com/miniupnp/miniupnp"><img style="position: absolute; top: 0; left: 0; border: 0;" src="https://github.blog/wp-content/uploads/2008/12/forkme_left_green_007200.png" alt="Fork me on GitHub" /></a>
+</p>
+
+<script type="text/javascript">
+var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+</script>
+<script type="text/javascript">
+try {
+  var ua = 'UA-10295521';
+  if(window.location.hostname == 'miniupnp.free.fr')
+    ua += '-1';
+  else if(window.location.hostname == 'miniupnp.tuxfamily.org')
+    ua += '-2';
+  else ua = '';
+  if(ua != '') {
+    var pageTracker = _gat._getTracker(ua);
+    pageTracker._trackPageview();
+  }
+} catch(err) {}</script>
+</body>
+</html>
+
diff --git a/bitbake/lib/bb/tests/fetch.py b/bitbake/lib/bb/tests/fetch.py
index 7b2dac7b86..077472b8b3 100644
--- a/bitbake/lib/bb/tests/fetch.py
+++ b/bitbake/lib/bb/tests/fetch.py
@@ -6,21 +6,59 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import contextlib
+import shutil
 import unittest
+import unittest.mock
+import urllib.parse
 import hashlib
 import tempfile
 import collections
 import os
+import signal
+import tarfile
 from bb.fetch2 import URI
 from bb.fetch2 import FetchMethod
 import bb
+import bb.utils
 from bb.tests.support.httpserver import HTTPService
 
 def skipIfNoNetwork():
     if os.environ.get("BB_SKIP_NETTESTS") == "yes":
-        return unittest.skip("Network tests being skipped")
+        return unittest.skip("network test")
     return lambda f: f
 
+
+@contextlib.contextmanager
+def hide_directory(directory):
+    """Hide the given directory and restore it after the context is left"""
+    temp_name = directory + ".bak"
+    os.rename(directory, temp_name)
+    try:
+        yield
+    finally:
+        os.rename(temp_name, directory)
+
+
+class TestTimeout(Exception):
+    # Indicate to pytest that this is not a test suite
+    __test__ = False
+
+class Timeout():
+
+    def __init__(self, seconds):
+        self.seconds = seconds
+
+    def handle_timeout(self, signum, frame):
+        raise TestTimeout("Test failed: timeout reached")
+
+    def __enter__(self):
+        signal.signal(signal.SIGALRM, self.handle_timeout)
+        signal.alarm(self.seconds)
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        signal.alarm(0)
+
 class URITest(unittest.TestCase):
     test_uris = {
         "http://www.google.com/index.html" : {
@@ -286,6 +324,36 @@ class URITest(unittest.TestCase):
             'params': {"someparam" : "1"},
             'query': {},
             'relative': True
+        },
+        "https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip": {
+            'uri': 'https://www.innodisk.com/Download_file?9BE0BF6657;downloadfilename=EGPL-T101.zip',
+            'scheme': 'https',
+            'hostname': 'www.innodisk.com',
+            'port': None,
+            'hostport': 'www.innodisk.com',
+            'path': '/Download_file',
+            'userinfo': '',
+            'userinfo': '',
+            'username': '',
+            'password': '',
+            'params': {"downloadfilename" : "EGPL-T101.zip"},
+            'query': {"9BE0BF6657": None},
+            'relative': False
+        },
+        "file://example@.service": {
+            'uri': 'file:example%40.service',
+            'scheme': 'file',
+            'hostname': '',
+            'port': None,
+            'hostport': '',
+            'path': 'example@.service',
+            'userinfo': '',
+            'userinfo': '',
+            'username': '',
+            'password': '',
+            'params': {},
+            'query': {},
+            'relative': True
         }
 
     }
@@ -376,7 +444,7 @@ class FetcherTest(unittest.TestCase):
     def setUp(self):
         self.origdir = os.getcwd()
         self.d = bb.data.init()
-        self.tempdir = tempfile.mkdtemp()
+        self.tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-")
         self.dldir = os.path.join(self.tempdir, "download")
         os.mkdir(self.dldir)
         self.d.setVar("DL_DIR", self.dldir)
@@ -390,63 +458,104 @@ class FetcherTest(unittest.TestCase):
         if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
             print("Not cleaning up %s. Please remove manually." % self.tempdir)
         else:
+            bb.process.run('chmod u+rw -R %s' % self.tempdir)
             bb.utils.prunedir(self.tempdir)
 
+    def git(self, cmd, cwd=None):
+        if isinstance(cmd, str):
+            cmd = 'git -c safe.bareRepository=all ' + cmd
+        else:
+            cmd = ['git', '-c', 'safe.bareRepository=all'] + cmd
+        if cwd is None:
+            cwd = self.gitdir
+        return bb.process.run(cmd, cwd=cwd)[0]
+
+    def git_init(self, cwd=None):
+        self.git('init', cwd=cwd)
+        # Explicitly set initial branch to master as
+        # a common setup is to use other default
+        # branch than master.
+        self.git(['checkout', '-b', 'master'], cwd=cwd)
+
+        try:
+            self.git(['config', 'user.email'], cwd=cwd)
+        except bb.process.ExecutionError:
+            self.git(['config', 'user.email', 'you@example.com'], cwd=cwd)
+
+        try:
+            self.git(['config', 'user.name'], cwd=cwd)
+        except bb.process.ExecutionError:
+            self.git(['config', 'user.name', 'Your Name'], cwd=cwd)
+
 class MirrorUriTest(FetcherTest):
 
     replaceuris = {
-        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "http://somewhere.org/somedir/") 
+        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/.*", "http://somewhere.org/somedir/")
             : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
-        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 
-            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
-        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http") 
-            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
-        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http") 
-            : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
-        ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
-            : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890",
-        ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache") 
+        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
+            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/somedir/\\2;protocol=http")
+            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/([^/]+/)*([^/]*)", "git://somewhere.org/\\2;protocol=http")
+            : "git://somewhere.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+        ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake", "git://git.openembedded.org/bitbake")
+            : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master",
+        ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache")
             : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
-        ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/") 
+        ("file://sstate-xyz.tgz", "file://.*", "file:///somewhere/1234/sstate-cache/")
             : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
-        ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3") 
+        ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/somedir3")
             : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
-        ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz") 
+        ("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz")
             : "http://somewhere2.org/somedir3/somefile_1.2.3.tar.gz",
         ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://www.apache.org/dist", "http://archive.apache.org/dist")
             : "http://archive.apache.org/dist/subversion/subversion-1.7.1.tar.bz2",
         ("http://www.apache.org/dist/subversion/subversion-1.7.1.tar.bz2", "http://.*/.*", "file:///somepath/downloads/")
             : "file:///somepath/downloads/subversion-1.7.1.tar.bz2",
-        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 
-            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
-        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http") 
-            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
-        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http") 
-            : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;protocol=http", 
+        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
+            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/.*", "git://somewhere.org/somedir/BASENAME;protocol=http")
+            : "git://somewhere.org/somedir/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+        ("git://git.invalid.infradead.org/foo/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", "git://.*/.*", "git://somewhere.org/somedir/MIRRORNAME;protocol=http")
+            : "git://somewhere.org/somedir/git.invalid.infradead.org.foo.mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
         ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org")
             : "http://somewhere2.org/somefile_1.2.3.tar.gz",
         ("http://somewhere.org/somedir1/somedir2/somefile_1.2.3.tar.gz", "http://.*/.*", "http://somewhere2.org/")
             : "http://somewhere2.org/somefile_1.2.3.tar.gz",
         ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://git.openembedded.org/bitbake;protocol=http")
             : "git://git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+        ("git://user1@someserver.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master", "git://someserver.org/bitbake;branch=master", "git://user2@git.openembedded.org/bitbake;protocol=http")
+            : "git://user2@git.openembedded.org/bitbake;tag=1234567890123456789012345678901234567890;branch=master;protocol=http",
+        ("git://someserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=git;branch=master", "git://someserver.org/bitbake", "git://someotherserver.org/bitbake;protocol=https")
+            : "git://someotherserver.org/bitbake;tag=1234567890123456789012345678901234567890;protocol=https;branch=master",
+        ("gitsm://git.qemu.org/git/seabios.git/;protocol=https;name=roms/seabios;subpath=roms/seabios;bareclone=1;nobranch=1;rev=1234567890123456789012345678901234567890", "gitsm://.*/.*", "http://petalinux.xilinx.com/sswreleases/rel-v${XILINX_VER_MAIN}/downloads") : "http://petalinux.xilinx.com/sswreleases/rel-v%24%7BXILINX_VER_MAIN%7D/downloads/git2_git.qemu.org.git.seabios.git..tar.gz",
+        ("https://somewhere.org/example/1.0.0/example;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/PATH")
+            : "file:///mirror/example/1.0.0/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz",
+        ("https://somewhere.org/example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz", "https://.*/.*", "file:///mirror/some-example-1.0.0.tgz")
+            : "file:///mirror/some-example-1.0.0.tgz;downloadfilename=some-example-1.0.0.tgz",
+        ("git://git.invalid.infradead.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", r"git://(?!internal\.git\.server).*/.*", "http://somewhere.org/somedir/")
+            : "http://somewhere.org/somedir/git2_git.invalid.infradead.org.mtd-utils.git.tar.gz",
+        ("git://internal.git.server.org/mtd-utils.git;tag=1234567890123456789012345678901234567890;branch=master", r"git://(?!internal\.git\.server).*/.*", "http://somewhere.org/somedir/")
+            : None,
 
         #Renaming files doesn't work
         #("http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere.org/somedir1/somefile_1.2.3.tar.gz", "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz") : "http://somewhere2.org/somedir3/somefile_2.3.4.tar.gz"
         #("file://sstate-xyz.tgz", "file://.*/.*", "file:///somewhere/1234/sstate-cache") : "file:///somewhere/1234/sstate-cache/sstate-xyz.tgz",
     }
 
-    mirrorvar = "http://.*/.* file:///somepath/downloads/ \n" \
-                "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n" \
-                "https://.*/.* file:///someotherpath/downloads/ \n" \
-                "http://.*/.* file:///someotherpath/downloads/ \n"
+    mirrorvar = "http://.*/.* file:///somepath/downloads/ " \
+                "git://someserver.org/bitbake git://git.openembedded.org/bitbake " \
+                "https?://.*/.* file:///someotherpath/downloads/ " \
+                "svn://svn.server1.com/ svn://svn.server2.com/"
 
     def test_urireplace(self):
+        self.d.setVar("FILESPATH", ".")
         for k, v in self.replaceuris.items():
             ud = bb.fetch.FetchData(k[0], self.d)
             ud.setup_localpath(self.d)
             mirrors = bb.fetch2.mirror_from_string("%s %s" % (k[1], k[2]))
             newuris, uds = bb.fetch2.build_mirroruris(ud, mirrors, self.d)
-            self.assertEqual([v], newuris)
+            self.assertEqual([v] if v else [], newuris)
 
     def test_urilist1(self):
         fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
@@ -461,10 +570,17 @@ class MirrorUriTest(FetcherTest):
         uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
         self.assertEqual(uris, ['file:///someotherpath/downloads/bitbake-1.0.tar.gz'])
 
+    def test_urilistsvn(self):
+        # Catch svn:// -> svn:// bug
+        fetcher = bb.fetch.FetchData("svn://svn.server1.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2", self.d)
+        mirrors = bb.fetch2.mirror_from_string(self.mirrorvar)
+        uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
+        self.assertEqual(uris, ['svn://svn.server2.com/isource/svnroot/reponame/tags/tagname;module=path_in_tagnamefolder;protocol=https;rev=2'])
+
     def test_mirror_of_mirror(self):
         # Test if mirror of a mirror works
-        mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/ \n"
-        mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/ \n"
+        mirrorvar = self.mirrorvar + " http://.*/.* http://otherdownloads.yoctoproject.org/downloads/"
+        mirrorvar = mirrorvar + " http://otherdownloads.yoctoproject.org/.* http://downloads2.yoctoproject.org/downloads/"
         fetcher = bb.fetch.FetchData("http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
         mirrors = bb.fetch2.mirror_from_string(mirrorvar)
         uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
@@ -473,30 +589,30 @@ class MirrorUriTest(FetcherTest):
                                 'http://otherdownloads.yoctoproject.org/downloads/bitbake-1.0.tar.gz',
                                 'http://downloads2.yoctoproject.org/downloads/bitbake-1.0.tar.gz'])
 
-    recmirrorvar = "https://.*/[^/]*    http://AAAA/A/A/A/ \n" \
-                   "https://.*/[^/]*    https://BBBB/B/B/B/ \n"
+    recmirrorvar = "https://.*/[^/]*    http://aaaa/A/A/A/ " \
+                   "https://.*/[^/]*    https://bbbb/B/B/B/"
 
     def test_recursive(self):
         fetcher = bb.fetch.FetchData("https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", self.d)
         mirrors = bb.fetch2.mirror_from_string(self.recmirrorvar)
         uris, uds = bb.fetch2.build_mirroruris(fetcher, mirrors, self.d)
-        self.assertEqual(uris, ['http://AAAA/A/A/A/bitbake/bitbake-1.0.tar.gz',
-                                'https://BBBB/B/B/B/bitbake/bitbake-1.0.tar.gz',
-                                'http://AAAA/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
+        self.assertEqual(uris, ['http://aaaa/A/A/A/bitbake/bitbake-1.0.tar.gz',
+                                'https://bbbb/B/B/B/bitbake/bitbake-1.0.tar.gz',
+                                'http://aaaa/A/A/A/B/B/bitbake/bitbake-1.0.tar.gz'])
 
 
 class GitDownloadDirectoryNamingTest(FetcherTest):
     def setUp(self):
         super(GitDownloadDirectoryNamingTest, self).setUp()
-        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
         self.recipe_dir = "git.openembedded.org.bitbake"
-        self.mirror_url = "git://github.com/openembedded/bitbake.git"
+        self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
         self.mirror_dir = "github.com.openembedded.bitbake.git"
 
         self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
 
     def setup_mirror_rewrite(self):
-        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
 
     @skipIfNoNetwork()
     def test_that_directory_is_named_after_recipe_url_when_no_mirroring_is_used(self):
@@ -536,16 +652,16 @@ class GitDownloadDirectoryNamingTest(FetcherTest):
 class TarballNamingTest(FetcherTest):
     def setUp(self):
         super(TarballNamingTest, self).setUp()
-        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
         self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
-        self.mirror_url = "git://github.com/openembedded/bitbake.git"
+        self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
         self.mirror_tarball = "git2_github.com.openembedded.bitbake.git.tar.gz"
 
         self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
         self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
 
     def setup_mirror_rewrite(self):
-        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
 
     @skipIfNoNetwork()
     def test_that_the_recipe_tarball_is_created_when_no_mirroring_is_used(self):
@@ -570,9 +686,9 @@ class TarballNamingTest(FetcherTest):
 class GitShallowTarballNamingTest(FetcherTest):
     def setUp(self):
         super(GitShallowTarballNamingTest, self).setUp()
-        self.recipe_url = "git://git.openembedded.org/bitbake"
+        self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
         self.recipe_tarball = "gitshallow_git.openembedded.org.bitbake_82ea737-1_master.tar.gz"
-        self.mirror_url = "git://github.com/openembedded/bitbake.git"
+        self.mirror_url = "git://github.com/openembedded/bitbake.git;protocol=https;branch=master"
         self.mirror_tarball = "gitshallow_github.com.openembedded.bitbake.git_82ea737-1_master.tar.gz"
 
         self.d.setVar('BB_GIT_SHALLOW', '1')
@@ -580,7 +696,7 @@ class GitShallowTarballNamingTest(FetcherTest):
         self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
 
     def setup_mirror_rewrite(self):
-        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url + " \n")
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + self.mirror_url)
 
     @skipIfNoNetwork()
     def test_that_the_tarball_is_named_after_recipe_url_when_no_mirroring_is_used(self):
@@ -602,6 +718,39 @@ class GitShallowTarballNamingTest(FetcherTest):
         self.assertIn(self.mirror_tarball, dir)
 
 
+class CleanTarballTest(FetcherTest):
+    def setUp(self):
+        super(CleanTarballTest, self).setUp()
+        self.recipe_url = "git://git.openembedded.org/bitbake;protocol=https;branch=master"
+        self.recipe_tarball = "git2_git.openembedded.org.bitbake.tar.gz"
+
+        self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '1')
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+
+    @skipIfNoNetwork()
+    def test_that_the_tarball_contents_does_not_leak_info(self):
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+
+        fetcher.unpack(self.unpackdir)
+        mtime = bb.process.run('git log --all -1 --format=%ct',
+                cwd=os.path.join(self.unpackdir, 'git'))
+        self.assertEqual(len(mtime), 2)
+        mtime = int(mtime[0])
+
+        archive = tarfile.open(os.path.join(self.dldir, self.recipe_tarball))
+        self.assertNotEqual(len(archive.members), 0)
+        for member in archive.members:
+            if member.name == ".":
+                continue
+            self.assertEqual(member.uname, 'oe', "user name for %s differs" % member.name)
+            self.assertEqual(member.uid, 0, "uid for %s differs" % member.name)
+            self.assertEqual(member.gname, 'oe', "group name for %s differs" % member.name)
+            self.assertEqual(member.gid, 0, "gid for %s differs" % member.name)
+            self.assertEqual(member.mtime, mtime, "mtime for %s differs" % member.name)
+
+
 class FetcherLocalTest(FetcherTest):
     def setUp(self):
         def touch(fn):
@@ -613,12 +762,16 @@ class FetcherLocalTest(FetcherTest):
         os.makedirs(self.localsrcdir)
         touch(os.path.join(self.localsrcdir, 'a'))
         touch(os.path.join(self.localsrcdir, 'b'))
+        touch(os.path.join(self.localsrcdir, 'c@d'))
         os.makedirs(os.path.join(self.localsrcdir, 'dir'))
         touch(os.path.join(self.localsrcdir, 'dir', 'c'))
         touch(os.path.join(self.localsrcdir, 'dir', 'd'))
         os.makedirs(os.path.join(self.localsrcdir, 'dir', 'subdir'))
         touch(os.path.join(self.localsrcdir, 'dir', 'subdir', 'e'))
         touch(os.path.join(self.localsrcdir, r'backslash\x2dsystemd-unit.device'))
+        bb.process.run('tar cf archive.tar -C dir .', cwd=self.localsrcdir)
+        bb.process.run('tar czf archive.tar.gz -C dir .', cwd=self.localsrcdir)
+        bb.process.run('tar cjf archive.tar.bz2 -C dir .', cwd=self.localsrcdir)
         self.d.setVar("FILESPATH", self.localsrcdir)
 
     def fetchUnpack(self, uris):
@@ -632,10 +785,19 @@ class FetcherLocalTest(FetcherTest):
         flst.sort()
         return flst
 
+    def test_local_checksum_fails_no_file(self):
+        self.d.setVar("SRC_URI", "file://404")
+        with self.assertRaises(bb.BBHandledException):
+            bb.fetch.get_checksum_file_list(self.d)
+
     def test_local(self):
         tree = self.fetchUnpack(['file://a', 'file://dir/c'])
         self.assertEqual(tree, ['a', 'dir/c'])
 
+    def test_local_at(self):
+        tree = self.fetchUnpack(['file://c@d'])
+        self.assertEqual(tree, ['c@d'])
+
     def test_local_backslash(self):
         tree = self.fetchUnpack([r'file://backslash\x2dsystemd-unit.device'])
         self.assertEqual(tree, [r'backslash\x2dsystemd-unit.device'])
@@ -673,57 +835,58 @@ class FetcherLocalTest(FetcherTest):
         with self.assertRaises(bb.fetch2.UnpackError):
             self.fetchUnpack(['file://a;subdir=/bin/sh'])
 
-    def test_local_gitfetch_usehead(self):
+    def test_local_striplevel(self):
+        tree = self.fetchUnpack(['file://archive.tar;subdir=bar;striplevel=1'])
+        self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
+
+    def test_local_striplevel_gzip(self):
+        tree = self.fetchUnpack(['file://archive.tar.gz;subdir=bar;striplevel=1'])
+        self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
+
+    def test_local_striplevel_bzip2(self):
+        tree = self.fetchUnpack(['file://archive.tar.bz2;subdir=bar;striplevel=1'])
+        self.assertEqual(tree, ['bar/c', 'bar/d', 'bar/subdir/e'])
+
+    def dummyGitTest(self, suffix):
         # Create dummy local Git repo
         src_dir = tempfile.mkdtemp(dir=self.tempdir,
                                    prefix='gitfetch_localusehead_')
-        src_dir = os.path.abspath(src_dir)
-        bb.process.run("git init", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit'",
-                       cwd=src_dir)
+        self.gitdir = os.path.abspath(src_dir)
+        self.git_init()
+        self.git(['commit', '--allow-empty', '-m', 'Dummy commit'])
         # Use other branch than master
-        bb.process.run("git checkout -b my-devel", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
-                       cwd=src_dir)
-        stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
-        orig_rev = stdout[0].strip()
+        self.git(['checkout', '-b', 'my-devel'])
+        self.git(['commit', '--allow-empty', '-m', 'Dummy commit 2'])
+        orig_rev = self.git(['rev-parse', 'HEAD']).strip()
 
         # Fetch and check revision
         self.d.setVar("SRCREV", "AUTOINC")
-        url = "git://" + src_dir + ";protocol=file;usehead=1"
+        self.d.setVar("__BBSRCREV_SEEN", "1")
+        url = "git://" + self.gitdir + ";branch=master;protocol=file;" + suffix
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         fetcher.unpack(self.unpackdir)
-        stdout = bb.process.run("git rev-parse HEAD",
-                                cwd=os.path.join(self.unpackdir, 'git'))
-        unpack_rev = stdout[0].strip()
+        unpack_rev = self.git(['rev-parse', 'HEAD'],
+                              cwd=os.path.join(self.unpackdir, 'git')).strip()
         self.assertEqual(orig_rev, unpack_rev)
 
+    def test_local_gitfetch_usehead(self):
+        self.dummyGitTest("usehead=1")
+
     def test_local_gitfetch_usehead_withname(self):
-        # Create dummy local Git repo
-        src_dir = tempfile.mkdtemp(dir=self.tempdir,
-                                   prefix='gitfetch_localusehead_')
-        src_dir = os.path.abspath(src_dir)
-        bb.process.run("git init", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit'",
-                       cwd=src_dir)
-        # Use other branch than master
-        bb.process.run("git checkout -b my-devel", cwd=src_dir)
-        bb.process.run("git commit --allow-empty -m'Dummy commit 2'",
-                       cwd=src_dir)
-        stdout = bb.process.run("git rev-parse HEAD", cwd=src_dir)
-        orig_rev = stdout[0].strip()
+        self.dummyGitTest("usehead=1;name=newName")
 
-        # Fetch and check revision
-        self.d.setVar("SRCREV", "AUTOINC")
-        url = "git://" + src_dir + ";protocol=file;usehead=1;name=newName"
-        fetcher = bb.fetch.Fetch([url], self.d)
-        fetcher.download()
-        fetcher.unpack(self.unpackdir)
-        stdout = bb.process.run("git rev-parse HEAD",
-                                cwd=os.path.join(self.unpackdir, 'git'))
-        unpack_rev = stdout[0].strip()
-        self.assertEqual(orig_rev, unpack_rev)
+    def test_local_gitfetch_shared(self):
+        self.dummyGitTest("usehead=1;name=sharedName")
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertTrue(os.path.exists(alt))
+
+    def test_local_gitfetch_noshared(self):
+        self.d.setVar('BB_GIT_NOSHARED', '1')
+        self.unpackdir += '_noshared'
+        self.dummyGitTest("usehead=1;name=noSharedName")
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertFalse(os.path.exists(alt))
 
 class FetcherNoNetworkTest(FetcherTest):
     def setUp(self):
@@ -831,12 +994,12 @@ class FetcherNoNetworkTest(FetcherTest):
 class FetcherNetworkTest(FetcherTest):
     @skipIfNoNetwork()
     def test_fetch(self):
-        fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
+        fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
         fetcher.download()
         self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
         self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.1.tar.gz"), 57892)
         self.d.setVar("BB_NO_NETWORK", "1")
-        fetcher = bb.fetch.Fetch(["http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "http://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
+        fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz", "https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz"], self.d)
         fetcher.download()
         fetcher.unpack(self.unpackdir)
         self.assertEqual(len(os.listdir(self.unpackdir + "/bitbake-1.0/")), 9)
@@ -844,21 +1007,22 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_fetch_mirror(self):
-        self.d.setVar("MIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
+        self.d.setVar("MIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
         fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
         fetcher.download()
         self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
 
     @skipIfNoNetwork()
     def test_fetch_mirror_of_mirror(self):
-        self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ \n http://invalid2.yoctoproject.org/.* http://downloads.yoctoproject.org/releases/bitbake")
+        self.d.setVar("MIRRORS", "http://.*/.* http://invalid2.yoctoproject.org/ http://invalid2.yoctoproject.org/.* https://downloads.yoctoproject.org/releases/bitbake")
         fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
         fetcher.download()
         self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
 
     @skipIfNoNetwork()
     def test_fetch_file_mirror_of_mirror(self):
-        self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ \n file:///some1where/.* file://some2where/ \n file://some2where/.* http://downloads.yoctoproject.org/releases/bitbake")
+        self.d.setVar("FILESPATH", ".")
+        self.d.setVar("MIRRORS", "http://.*/.* file:///some1where/ file:///some1where/.* file://some2where/ file://some2where/.* https://downloads.yoctoproject.org/releases/bitbake")
         fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
         os.mkdir(self.dldir + "/some2where")
         fetcher.download()
@@ -866,16 +1030,46 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_fetch_premirror(self):
-        self.d.setVar("PREMIRRORS", "http://.*/.* http://downloads.yoctoproject.org/releases/bitbake")
+        self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
         fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz"], self.d)
         fetcher.download()
         self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
 
     @skipIfNoNetwork()
+    def test_fetch_specify_downloadfilename(self):
+        fetcher = bb.fetch.Fetch(["https://downloads.yoctoproject.org/releases/bitbake/bitbake-1.0.tar.gz;downloadfilename=bitbake-v1.0.0.tar.gz"], self.d)
+        fetcher.download()
+        self.assertEqual(os.path.getsize(self.dldir + "/bitbake-v1.0.0.tar.gz"), 57749)
+
+    @skipIfNoNetwork()
+    def test_fetch_premirror_specify_downloadfilename_regex_uri(self):
+        self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake/")
+        fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
+        fetcher.download()
+        self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
+
+    @skipIfNoNetwork()
+    # BZ13039
+    def test_fetch_premirror_specify_downloadfilename_specific_uri(self):
+        self.d.setVar("PREMIRRORS", "http://invalid.yoctoproject.org/releases/bitbake https://downloads.yoctoproject.org/releases/bitbake")
+        fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/1.0.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
+        fetcher.download()
+        self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
+
+    @skipIfNoNetwork()
+    def test_fetch_premirror_use_downloadfilename_to_fetch(self):
+        # Ensure downloadfilename is used when fetching from premirror.
+        self.d.setVar("PREMIRRORS", "http://.*/.* https://downloads.yoctoproject.org/releases/bitbake")
+        fetcher = bb.fetch.Fetch(["http://invalid.yoctoproject.org/releases/bitbake/bitbake-1.1.tar.gz;downloadfilename=bitbake-1.0.tar.gz"], self.d)
+        fetcher.download()
+        self.assertEqual(os.path.getsize(self.dldir + "/bitbake-1.0.tar.gz"), 57749)
+
+    @skipIfNoNetwork()
     def gitfetcher(self, url1, url2):
         def checkrevision(self, fetcher):
             fetcher.unpack(self.unpackdir)
-            revision = bb.process.run("git rev-parse HEAD", shell=True, cwd=self.unpackdir + "/git")[0].strip()
+            revision = self.git(['rev-parse', 'HEAD'],
+                                cwd=os.path.join(self.unpackdir, 'git')).strip()
             self.assertEqual(revision, "270a05b0b4ba0959fe0624d2a4885d7b70426da5")
 
         self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
@@ -893,25 +1087,19 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_gitfetch(self):
-        url1 = url2 = "git://git.openembedded.org/bitbake"
+        url1 = url2 = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
         self.gitfetcher(url1, url2)
 
     @skipIfNoNetwork()
     def test_gitfetch_goodsrcrev(self):
         # SRCREV is set but matches rev= parameter
-        url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
+        url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https"
         self.gitfetcher(url1, url2)
 
     @skipIfNoNetwork()
     def test_gitfetch_badsrcrev(self):
         # SRCREV is set but does not match rev= parameter
-        url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5"
-        self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
-
-    @skipIfNoNetwork()
-    def test_gitfetch_tagandrev(self):
-        # SRCREV is set but does not match rev= parameter
-        url1 = url2 = "git://git.openembedded.org/bitbake;rev=270a05b0b4ba0959fe0624d2a4885d7b70426da5;tag=270a05b0b4ba0959fe0624d2a4885d7b70426da5"
+        url1 = url2 = "git://git.openembedded.org/bitbake;rev=dead05b0b4ba0959fe0624d2a4885d7b70426da5;branch=master;protocol=https"
         self.assertRaises(bb.fetch.FetchError, self.gitfetcher, url1, url2)
 
     @skipIfNoNetwork()
@@ -920,7 +1108,7 @@ class FetcherNetworkTest(FetcherTest):
         # `usehead=1' and instead fetch the specified SRCREV. See
         # test_local_gitfetch_usehead() for a positive use of the usehead
         # feature.
-        url = "git://git.openembedded.org/bitbake;usehead=1"
+        url = "git://git.openembedded.org/bitbake;usehead=1;branch=master;protocol=https"
         self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
 
     @skipIfNoNetwork()
@@ -929,38 +1117,38 @@ class FetcherNetworkTest(FetcherTest):
         # `usehead=1' and instead fetch the specified SRCREV. See
         # test_local_gitfetch_usehead() for a positive use of the usehead
         # feature.
-        url = "git://git.openembedded.org/bitbake;usehead=1;name=newName"
+        url = "git://git.openembedded.org/bitbake;usehead=1;name=newName;branch=master;protocol=https"
         self.assertRaises(bb.fetch.ParameterError, self.gitfetcher, url, url)
 
     @skipIfNoNetwork()
     def test_gitfetch_finds_local_tarball_for_mirrored_url_when_previous_downloaded_by_the_recipe_url(self):
-        recipeurl = "git://git.openembedded.org/bitbake"
-        mirrorurl = "git://someserver.org/bitbake"
-        self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
+        recipeurl = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
+        mirrorurl = "git://someserver.org/bitbake;branch=master;protocol=https"
+        self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake")
         self.gitfetcher(recipeurl, mirrorurl)
 
     @skipIfNoNetwork()
     def test_gitfetch_finds_local_tarball_when_previous_downloaded_from_a_premirror(self):
-        recipeurl = "git://someserver.org/bitbake"
-        self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake \n")
+        recipeurl = "git://someserver.org/bitbake;branch=master;protocol=https"
+        self.d.setVar("PREMIRRORS", "git://someserver.org/bitbake git://git.openembedded.org/bitbake")
         self.gitfetcher(recipeurl, recipeurl)
 
     @skipIfNoNetwork()
     def test_gitfetch_finds_local_repository_when_premirror_rewrites_the_recipe_url(self):
-        realurl = "git://git.openembedded.org/bitbake"
-        recipeurl = "git://someserver.org/bitbake"
+        realurl = "https://git.openembedded.org/bitbake"
+        recipeurl = "git://someserver.org/bitbake;protocol=https;branch=master"
         self.sourcedir = self.unpackdir.replace("unpacked", "sourcemirror.git")
         os.chdir(self.tempdir)
-        bb.process.run("git clone %s %s 2> /dev/null" % (realurl, self.sourcedir), shell=True)
-        self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file \n" % (recipeurl, self.sourcedir))
+        self.git(['clone', realurl, self.sourcedir], cwd=self.tempdir)
+        self.d.setVar("PREMIRRORS", "%s git://%s;protocol=file" % (recipeurl, self.sourcedir))
         self.gitfetcher(recipeurl, recipeurl)
 
     @skipIfNoNetwork()
     def test_git_submodule(self):
         # URL with ssh submodules
-        url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7"
+        url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=ssh-gitsm-tests;rev=049da4a6cb198d7c0302e9e8b243a1443cb809a7;branch=master;protocol=https"
         # Original URL (comment this if you have ssh access to git.yoctoproject.org)
-        url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee"
+        url = "gitsm://git.yoctoproject.org/git-submodule-test;branch=master;rev=a2885dd7d25380d23627e7544b7bbb55014b16ee;branch=master;protocol=https"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -977,10 +1165,29 @@ class FetcherNetworkTest(FetcherTest):
             self.assertTrue(os.path.exists(os.path.join(repo_path, 'bitbake-gitsm-test1', 'bitbake')), msg='submodule of submodule missing')
 
     @skipIfNoNetwork()
+    def test_git_submodule_restricted_network_premirrors(self):
+        # this test is to ensure that premirrors will be tried in restricted network
+        # that is, BB_ALLOWED_NETWORKS does not contain the domain the url uses
+        url = "gitsm://github.com/grpc/grpc.git;protocol=https;name=grpc;branch=v1.60.x;rev=0ef13a7555dbaadd4633399242524129eef5e231"
+        # create a download directory to be used as premirror later
+        tempdir = tempfile.mkdtemp(prefix="bitbake-fetch-")
+        dl_premirror = os.path.join(tempdir, "download-premirror")
+        os.mkdir(dl_premirror)
+        self.d.setVar("DL_DIR", dl_premirror)
+        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher.download()
+        # now use the premirror in restricted network
+        self.d.setVar("DL_DIR", self.dldir)
+        self.d.setVar("PREMIRRORS", "gitsm://.*/.* gitsm://%s/git2/MIRRORNAME;protocol=file" % dl_premirror)
+        self.d.setVar("BB_ALLOWED_NETWORKS", "*.some.domain")
+        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher.download()
+
+    @skipIfNoNetwork()
     def test_git_submodule_dbus_broker(self):
         # The following external repositories have show failures in fetch and unpack operations
         # We want to avoid regressions!
-        url = "gitsm://github.com/bus1/dbus-broker;protocol=git;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
+        url = "gitsm://github.com/bus1/dbus-broker;protocol=https;rev=fc874afa0992d0c75ec25acb43d344679f0ee7d2;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -996,7 +1203,7 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_git_submodule_CLI11(self):
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=bd4dc911847d0cde7a6b41dfa626a85aab213baf;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -1011,12 +1218,12 @@ class FetcherNetworkTest(FetcherTest):
     @skipIfNoNetwork()
     def test_git_submodule_update_CLI11(self):
         """ Prevent regression on update detection not finding missing submodule, or modules without needed commits """
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=cf6a99fa69aaefe477cc52e3ef4a7d2d7fa40714;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
 
         # CLI11 that pulls in a newer nlohmann-json
-        url = "gitsm://github.com/CLIUtils/CLI11;protocol=git;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca"
+        url = "gitsm://github.com/CLIUtils/CLI11;protocol=https;rev=49ac989a9527ee9bb496de9ded7b4872c2e0e5ca;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -1030,7 +1237,7 @@ class FetcherNetworkTest(FetcherTest):
 
     @skipIfNoNetwork()
     def test_git_submodule_aktualizr(self):
-        url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=git;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
+        url = "gitsm://github.com/advancedtelematic/aktualizr;branch=master;protocol=https;rev=d00d1a04cc2366d1a5f143b84b9f507f8bd32c44"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -1050,7 +1257,7 @@ class FetcherNetworkTest(FetcherTest):
         """ Prevent regression on deeply nested submodules not being checked out properly, even though they were fetched. """
 
         # This repository also has submodules where the module (name), path and url do not align
-        url = "gitsm://github.com/azure/iotedge.git;protocol=git;rev=d76e0316c6f324345d77c48a83ce836d09392699"
+        url = "gitsm://github.com/azure/iotedge.git;protocol=https;rev=d76e0316c6f324345d77c48a83ce836d09392699;branch=main"
         fetcher = bb.fetch.Fetch([url], self.d)
         fetcher.download()
         # Previous cwd has been deleted
@@ -1073,9 +1280,17 @@ class FetcherNetworkTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/ctest/README.md')), msg='Missing submodule checkout')
         self.assertTrue(os.path.exists(os.path.join(repo_path, 'edgelet/hsm-sys/azure-iot-hsm-c/deps/utpm/deps/c-utility/testtools/umock-c/deps/testrunner/readme.md')), msg='Missing submodule checkout')
 
+    @skipIfNoNetwork()
+    def test_git_submodule_reference_to_parent(self):
+        self.recipe_url = "gitsm://github.com/gflags/gflags.git;protocol=https;branch=master"
+        self.d.setVar("SRCREV", "14e1138441bbbb584160cb1c0a0426ec1bac35f1")
+        with Timeout(60):
+            fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+            with self.assertRaises(bb.fetch2.FetchError):
+                fetcher.download()
+
 class SVNTest(FetcherTest):
     def skipIfNoSvn():
-        import shutil
         if not shutil.which("svn"):
             return unittest.skip("svn not installed,  tests being skipped")
 
@@ -1107,8 +1322,9 @@ class SVNTest(FetcherTest):
                        cwd=repo_dir)
 
         bb.process.run("svn co %s svnfetch_co" % self.repo_url, cwd=self.tempdir)
-        # Github will emulate SVN.  Use this to check if we're downloding...
-        bb.process.run("svn propset svn:externals 'bitbake svn://vcs.pcre.org/pcre2/code' .",
+        # Github won't emulate SVN anymore (see https://github.blog/2023-01-20-sunsetting-subversion-support/)
+        # Use still accessible svn repo (only trunk to avoid longer downloads)
+        bb.process.run("svn propset svn:externals 'bitbake https://svn.apache.org/repos/asf/serf/trunk' .",
                        cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
         bb.process.run("svn commit --non-interactive -m 'Add external'",
                        cwd=os.path.join(self.tempdir, 'svnfetch_co', 'trunk'))
@@ -1136,8 +1352,8 @@ class SVNTest(FetcherTest):
 
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
-        self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should NOT exist")
-        self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should NOT exit")
+        self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should NOT exist")
+        self.assertFalse(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should NOT exit")
 
     @skipIfNoSvn()
     def test_external_svn(self):
@@ -1150,66 +1366,71 @@ class SVNTest(FetcherTest):
 
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk')), msg="Missing trunk")
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk', 'README.md')), msg="Missing contents")
-        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk')), msg="External dir should exist")
-        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/trunk', 'README')), msg="External README should exit")
+        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols')), msg="External dir should exist")
+        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'trunk/bitbake/protocols', 'fcgi_buckets.h')), msg="External fcgi_buckets.h should exit")
 
 class TrustedNetworksTest(FetcherTest):
     def test_trusted_network(self):
         # Ensure trusted_network returns False when the host IS in the list.
-        url = "git://Someserver.org/foo;rev=1"
+        url = "git://Someserver.org/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org someserver.org server2.org server3.org")
         self.assertTrue(bb.fetch.trusted_network(self.d, url))
 
     def test_wild_trusted_network(self):
         # Ensure trusted_network returns true when the *.host IS in the list.
-        url = "git://Someserver.org/foo;rev=1"
+        url = "git://Someserver.org/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
         self.assertTrue(bb.fetch.trusted_network(self.d, url))
 
     def test_prefix_wild_trusted_network(self):
         # Ensure trusted_network returns true when the prefix matches *.host.
-        url = "git://git.Someserver.org/foo;rev=1"
+        url = "git://git.Someserver.org/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
         self.assertTrue(bb.fetch.trusted_network(self.d, url))
 
     def test_two_prefix_wild_trusted_network(self):
         # Ensure trusted_network returns true when the prefix matches *.host.
-        url = "git://something.git.Someserver.org/foo;rev=1"
+        url = "git://something.git.Someserver.org/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org *.someserver.org server2.org server3.org")
         self.assertTrue(bb.fetch.trusted_network(self.d, url))
 
     def test_port_trusted_network(self):
         # Ensure trusted_network returns True, even if the url specifies a port.
-        url = "git://someserver.org:8080/foo;rev=1"
+        url = "git://someserver.org:8080/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "someserver.org")
         self.assertTrue(bb.fetch.trusted_network(self.d, url))
 
     def test_untrusted_network(self):
         # Ensure trusted_network returns False when the host is NOT in the list.
-        url = "git://someserver.org/foo;rev=1"
+        url = "git://someserver.org/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
         self.assertFalse(bb.fetch.trusted_network(self.d, url))
 
     def test_wild_untrusted_network(self):
         # Ensure trusted_network returns False when the host is NOT in the list.
-        url = "git://*.someserver.org/foo;rev=1"
+        url = "git://*.someserver.org/foo;rev=1;branch=master"
         self.d.setVar("BB_ALLOWED_NETWORKS", "server1.org server2.org server3.org")
         self.assertFalse(bb.fetch.trusted_network(self.d, url))
 
 class URLHandle(unittest.TestCase):
-
+    # Quote password as per RFC3986
+    password = urllib.parse.quote(r"!#$%^&*()-_={}[]\|:?,.<>~`", r"!$&'/()*+,;=")
     datatable = {
        "http://www.google.com/index.html" : ('http', 'www.google.com', '/index.html', '', '', {}),
        "cvs://anoncvs@cvs.handhelds.org/cvs;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', '', {'module': 'familiar/dist/ipkg'}),
        "cvs://anoncvs:anonymous@cvs.handhelds.org/cvs;tag=V0-99-81;module=familiar/dist/ipkg" : ('cvs', 'cvs.handhelds.org', '/cvs', 'anoncvs', 'anonymous', collections.OrderedDict([('tag', 'V0-99-81'), ('module', 'familiar/dist/ipkg')])),
-       "git://git.openembedded.org/bitbake;branch=@foo" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo'}),
+       "git://git.openembedded.org/bitbake;branch=@foo;protocol=https" : ('git', 'git.openembedded.org', '/bitbake', '', '', {'branch': '@foo', 'protocol' : 'https'}),
        "file://somelocation;someparam=1": ('file', '', 'somelocation', '', '', {'someparam': '1'}),
+       "file://example@.service": ('file', '', 'example@.service', '', '', {}),
+       "https://somesite.com/somerepo.git;user=anyUser:idtoken=1234" : ('https', 'somesite.com', '/somerepo.git', '', '', {'user': 'anyUser:idtoken=1234'}),
+       'git://s.o-me_ONE:%s@git.openembedded.org/bitbake;branch=main;protocol=https' % password: ('git', 'git.openembedded.org', '/bitbake', 's.o-me_ONE', password, {'branch': 'main', 'protocol' : 'https'}),
     }
     # we require a pathname to encodeurl but users can still pass such urls to 
     # decodeurl and we need to handle them
     decodedata = datatable.copy()
     decodedata.update({
        "http://somesite.net;someparam=1": ('http', 'somesite.net', '/', '', '', {'someparam': '1'}),
+       "npmsw://some.registry.url;package=@pkg;version=latest": ('npmsw', 'some.registry.url', '/', '', '', {'package': '@pkg', 'version': 'latest'}),
     })
 
     def test_decodeurl(self):
@@ -1220,138 +1441,179 @@ class URLHandle(unittest.TestCase):
     def test_encodeurl(self):
         for k, v in self.datatable.items():
             result = bb.fetch.encodeurl(v)
+            if result.startswith("file:"):
+                result = urllib.parse.unquote(result)
             self.assertEqual(result, k)
 
 class FetchLatestVersionTest(FetcherTest):
 
     test_git_uris = {
         # version pattern "X.Y.Z"
-        ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4", "9b1db6b8060bd00b121a692f942404a24ae2960f", "")
+        ("mx-1.0", "git://github.com/clutter-project/mx.git;branch=mx-1.4;protocol=https", "9b1db6b8060bd00b121a692f942404a24ae2960f", "", "")
             : "1.99.4",
         # version pattern "vX.Y"
         # mirror of git.infradead.org since network issues interfered with testing
-        ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "")
+        ("mtd-utils", "git://git.yoctoproject.org/mtd-utils.git;branch=master;protocol=https", "ca39eb1d98e736109c64ff9c1aa2a6ecca222d8f", "", "")
             : "1.5.0",
         # version pattern "pkg_name-X.Y"
         # mirror of git://anongit.freedesktop.org/git/xorg/proto/presentproto since network issues interfered with testing
-        ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "")
+        ("presentproto", "git://git.yoctoproject.org/bbfetchtests-presentproto;branch=master;protocol=https", "24f3a56e541b0a9e6c6ee76081f441221a120ef9", "", "")
             : "1.0",
         # version pattern "pkg_name-vX.Y.Z"
-        ("dtc", "git://git.qemu.org/dtc.git", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "")
+        ("dtc", "git://git.yoctoproject.org/bbfetchtests-dtc.git;branch=master;protocol=https", "65cc4d2748a2c2e6f27f1cf39e07a5dbabd80ebf", "", "")
             : "1.4.0",
         # combination version pattern
-        ("sysprof", "git://gitlab.gnome.org/GNOME/sysprof.git;protocol=https", "cd44ee6644c3641507fb53b8a2a69137f2971219", "")
+        ("sysprof", "git://git.yoctoproject.org/sysprof.git;protocol=https;branch=master", "cd44ee6644c3641507fb53b8a2a69137f2971219", "", "")
             : "1.2.0",
-        ("u-boot-mkimage", "git://git.denx.de/u-boot.git;branch=master;protocol=git", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "")
+        ("u-boot-mkimage", "git://source.denx.de/u-boot/u-boot.git;branch=master;protocol=https", "62c175fbb8a0f9a926c88294ea9f7e88eb898f6c", "", "")
             : "2014.01",
         # version pattern "yyyymmdd"
-        ("mobile-broadband-provider-info", "git://gitlab.gnome.org/GNOME/mobile-broadband-provider-info.git;protocol=https", "4ed19e11c2975105b71b956440acdb25d46a347d", "")
+        ("mobile-broadband-provider-info", "git://git.yoctoproject.org/mobile-broadband-provider-info.git;protocol=https;branch=master", "4ed19e11c2975105b71b956440acdb25d46a347d", "", "")
             : "20120614",
         # packages with a valid UPSTREAM_CHECK_GITTAGREGEX
                 # mirror of git://anongit.freedesktop.org/xorg/driver/xf86-video-omap since network issues interfered with testing
-        ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap", "ae0394e687f1a77e966cf72f895da91840dffb8f", "(?P<pver>(\d+\.(\d\.?)*))")
+        ("xf86-video-omap", "git://git.yoctoproject.org/bbfetchtests-xf86-video-omap;branch=master;protocol=https", "ae0394e687f1a77e966cf72f895da91840dffb8f", r"(?P<pver>(\d+\.(\d\.?)*))", "")
             : "0.4.3",
-        ("build-appliance-image", "git://git.yoctoproject.org/poky", "b37dd451a52622d5b570183a81583cc34c2ff555", "(?P<pver>(([0-9][\.|_]?)+[0-9]))")
+        ("build-appliance-image", "git://git.yoctoproject.org/poky;branch=master;protocol=https", "b37dd451a52622d5b570183a81583cc34c2ff555", r"(?P<pver>(([0-9][\.|_]?)+[0-9]))", "")
             : "11.0.0",
-        ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot", "cd437ecbd8986c894442f8fce1e0061e20f04dee", "chkconfig\-(?P<pver>((\d+[\.\-_]*)+))")
+        ("chkconfig-alternatives-native", "git://github.com/kergoth/chkconfig;branch=sysroot;protocol=https", "cd437ecbd8986c894442f8fce1e0061e20f04dee", r"chkconfig\-(?P<pver>((\d+[\.\-_]*)+))", "")
             : "1.3.59",
-        ("remake", "git://github.com/rocky/remake.git", "f05508e521987c8494c92d9c2871aec46307d51d", "(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))")
+        ("remake", "git://github.com/rocky/remake.git;protocol=https;branch=master", "f05508e521987c8494c92d9c2871aec46307d51d", r"(?P<pver>(\d+\.(\d+\.)*\d*(\+dbg\d+(\.\d+)*)*))", "")
             : "3.82+dbg0.9",
+        ("sysdig", "git://github.com/draios/sysdig.git;branch=dev;protocol=https", "4fb6288275f567f63515df0ff0a6518043ecfa9b", r"^(?P<pver>\d+(\.\d+)+)", "10.0.0")
+            : "0.28.0",
     }
 
+    WgetTestData = collections.namedtuple("WgetTestData", ["pn", "path", "pv", "check_uri", "check_regex"], defaults=[None, None, None])
     test_wget_uris = {
         #
         # packages with versions inside directory name
         #
         # http://kernel.org/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2
-        ("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2", "", "")
+        WgetTestData("util-linux", "/pub/linux/utils/util-linux/v2.23/util-linux-2.24.2.tar.bz2")
             : "2.24.2",
         # http://www.abisource.com/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz
-        ("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz", "", "")
+        WgetTestData("enchant", "/downloads/enchant/1.6.0/enchant-1.6.0.tar.gz")
             : "1.6.0",
         # http://www.cmake.org/files/v2.8/cmake-2.8.12.1.tar.gz
-        ("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz", "", "")
+        WgetTestData("cmake", "/files/v2.8/cmake-2.8.12.1.tar.gz")
             : "2.8.12.1",
+        # https://download.gnome.org/sources/libxml2/2.9/libxml2-2.9.14.tar.xz
+        WgetTestData("libxml2", "/software/libxml2/2.9/libxml2-2.9.14.tar.xz")
+            : "2.10.3",
         #
         # packages with versions only in current directory
         #
-        # http://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2
-        ("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2", "", "")
+        # https://downloads.yoctoproject.org/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2
+        WgetTestData("eglic", "/releases/eglibc/eglibc-2.18-svnr23787.tar.bz2")
             : "2.19",
-        # http://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2
-        ("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2", "", "")
+        # https://downloads.yoctoproject.org/releases/gnu-config/gnu-config-20120814.tar.bz2
+        WgetTestData("gnu-config", "/releases/gnu-config/gnu-config-20120814.tar.bz2")
             : "20120814",
         #
         # packages with "99" in the name of possible version
         #
         # http://freedesktop.org/software/pulseaudio/releases/pulseaudio-4.0.tar.xz
-        ("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz", "", "")
+        WgetTestData("pulseaudio", "/software/pulseaudio/releases/pulseaudio-4.0.tar.xz")
             : "5.0",
         # http://xorg.freedesktop.org/releases/individual/xserver/xorg-server-1.15.1.tar.bz2
-        ("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2", "", "")
+        WgetTestData("xserver-xorg", "/releases/individual/xserver/xorg-server-1.15.1.tar.bz2")
             : "1.15.1",
         #
         # packages with valid UPSTREAM_CHECK_URI and UPSTREAM_CHECK_REGEX
         #
         # http://www.cups.org/software/1.7.2/cups-1.7.2-source.tar.bz2
         # https://github.com/apple/cups/releases
-        ("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", "/apple/cups/releases", "(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
+        WgetTestData("cups", "/software/1.7.2/cups-1.7.2-source.tar.bz2", check_uri="/apple/cups/releases", check_regex=r"(?P<name>cups\-)(?P<pver>((\d+[\.\-_]*)+))\-source\.tar\.gz")
             : "2.0.0",
         # http://download.oracle.com/berkeley-db/db-5.3.21.tar.gz
         # http://ftp.debian.org/debian/pool/main/d/db5.3/
-        ("db", "/berkeley-db/db-5.3.21.tar.gz", "/debian/pool/main/d/db5.3/", "(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz")
+        WgetTestData("db", "/berkeley-db/db-5.3.21.tar.gz", check_uri="/debian/pool/main/d/db5.3/", check_regex=r"(?P<name>db5\.3_)(?P<pver>\d+(\.\d+)+).+\.orig\.tar\.xz")
             : "5.3.10",
+        #
+        # packages where the tarball compression changed in the new version
+        #
+        # http://ftp.debian.org/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz
+        WgetTestData("minicom", "/debian/pool/main/m/minicom/minicom_2.7.1.orig.tar.gz")
+            : "2.8",
+
+        #
+        # packages where the path doesn't actually contain the filename, so downloadfilename should be respected
+        #
+        WgetTestData("miniupnpd", "/software/miniupnp/download.php?file=miniupnpd_2.1.20191006.tar.gz;downloadfilename=miniupnpd_2.1.20191006.tar.gz", pv="2.1.20191006", check_uri="/software/miniupnp/download.php", check_regex=r"miniupnpd-(?P<pver>\d+(\.\d+)+)\.tar")
+            : "2.3.7",
     }
 
+    test_crate_uris = {
+        # basic example; version pattern "A.B.C+cargo-D.E.F"
+        ("cargo-c", "crate://crates.io/cargo-c/0.9.18+cargo-0.69")
+            : "0.9.29"
+   }
+
     @skipIfNoNetwork()
     def test_git_latest_versionstring(self):
         for k, v in self.test_git_uris.items():
-            self.d.setVar("PN", k[0])
-            self.d.setVar("SRCREV", k[2])
-            self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3])
-            ud = bb.fetch2.FetchData(k[1], self.d)
-            pupver= ud.method.latest_versionstring(ud, self.d)
-            verstring = pupver[0]
-            self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
-            r = bb.utils.vercmp_string(v, verstring)
-            self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
+            with self.subTest(pn=k[0]):
+                self.d.setVar("PN", k[0])
+                self.d.setVar("SRCREV", k[2])
+                self.d.setVar("UPSTREAM_CHECK_GITTAGREGEX", k[3])
+                ud = bb.fetch2.FetchData(k[1], self.d)
+                pupver= ud.method.latest_versionstring(ud, self.d)
+                verstring = pupver[0]
+                self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
+                r = bb.utils.vercmp_string(v, verstring)
+                self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
+                if k[4]:
+                    r = bb.utils.vercmp_string(verstring, k[4])
+                    self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], verstring, k[4]))
 
     def test_wget_latest_versionstring(self):
         testdata = os.path.dirname(os.path.abspath(__file__)) + "/fetch-testdata"
-        server = HTTPService(testdata)
+        server = HTTPService(testdata, host="127.0.0.1")
         server.start()
         port = server.port
         try:
-            for k, v in self.test_wget_uris.items():
+            for data, v in self.test_wget_uris.items():
+                with self.subTest(pn=data.pn):
+                    self.d.setVar("PN", data.pn)
+                    self.d.setVar("PV", data.pv)
+                    if data.check_uri:
+                        checkuri = "http://127.0.0.1:%s/%s" % (port, data.check_uri)
+                        self.d.setVar("UPSTREAM_CHECK_URI", checkuri)
+                    if data.check_regex:
+                        self.d.setVar("UPSTREAM_CHECK_REGEX", data.check_regex)
+
+                    url = "http://127.0.0.1:%s/%s" % (port, data.path)
+                    ud = bb.fetch2.FetchData(url, self.d)
+                    pupver = ud.method.latest_versionstring(ud, self.d)
+                    verstring = pupver[0]
+                    self.assertTrue(verstring, msg="Could not find upstream version for %s" % data.pn)
+                    r = bb.utils.vercmp_string(v, verstring)
+                    self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (data.pn, v, verstring))
+        finally:
+            server.stop()
+
+    @skipIfNoNetwork()
+    def test_crate_latest_versionstring(self):
+        for k, v in self.test_crate_uris.items():
+            with self.subTest(pn=k[0]):
                 self.d.setVar("PN", k[0])
-                checkuri = ""
-                if k[2]:
-                    checkuri = "http://localhost:%s/" % port + k[2]
-                self.d.setVar("UPSTREAM_CHECK_URI", checkuri)
-                self.d.setVar("UPSTREAM_CHECK_REGEX", k[3])
-                url = "http://localhost:%s/" % port + k[1]
-                ud = bb.fetch2.FetchData(url, self.d)
+                ud = bb.fetch2.FetchData(k[1], self.d)
                 pupver = ud.method.latest_versionstring(ud, self.d)
                 verstring = pupver[0]
                 self.assertTrue(verstring, msg="Could not find upstream version for %s" % k[0])
                 r = bb.utils.vercmp_string(v, verstring)
                 self.assertTrue(r == -1 or r == 0, msg="Package %s, version: %s <= %s" % (k[0], v, verstring))
-        finally:
-            server.stop()
-
 
 class FetchCheckStatusTest(FetcherTest):
-    test_wget_uris = ["http://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
-                      "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
-                      "http://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
+    test_wget_uris = ["https://downloads.yoctoproject.org/releases/sato/sato-engine-0.1.tar.gz",
+                      "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.2.tar.gz",
+                      "https://downloads.yoctoproject.org/releases/sato/sato-engine-0.3.tar.gz",
                       "https://yoctoproject.org/",
-                      "https://yoctoproject.org/documentation",
-                      "http://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
-                      "http://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
+                      "https://docs.yoctoproject.org",
+                      "https://downloads.yoctoproject.org/releases/opkg/opkg-0.1.7.tar.gz",
+                      "https://downloads.yoctoproject.org/releases/opkg/opkg-0.3.0.tar.gz",
                       "ftp://sourceware.org/pub/libffi/libffi-1.20.tar.gz",
-                      "http://ftp.gnu.org/gnu/autoconf/autoconf-2.60.tar.gz",
-                      "https://ftp.gnu.org/gnu/chess/gnuchess-5.08.tar.gz",
-                      "https://ftp.gnu.org/gnu/gmp/gmp-4.0.tar.gz",
                       # GitHub releases are hosted on Amazon S3, which doesn't support HEAD
                       "https://github.com/kergoth/tslib/releases/download/1.1/tslib-1.1.tar.xz"
                       ]
@@ -1389,7 +1651,7 @@ class GitMakeShallowTest(FetcherTest):
         FetcherTest.setUp(self)
         self.gitdir = os.path.join(self.tempdir, 'gitshallow')
         bb.utils.mkdirhier(self.gitdir)
-        bb.process.run('git init', cwd=self.gitdir)
+        self.git_init()
 
     def assertRefs(self, expected_refs):
         actual_refs = self.git(['for-each-ref', '--format=%(refname)']).splitlines()
@@ -1403,13 +1665,6 @@ class GitMakeShallowTest(FetcherTest):
         actual_count = len(revs.splitlines())
         self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
 
-    def git(self, cmd):
-        if isinstance(cmd, str):
-            cmd = 'git ' + cmd
-        else:
-            cmd = ['git'] + cmd
-        return bb.process.run(cmd, cwd=self.gitdir)[0]
-
     def make_shallow(self, args=None):
         if args is None:
             args = ['HEAD']
@@ -1512,13 +1767,13 @@ class GitShallowTest(FetcherTest):
         self.srcdir = os.path.join(self.tempdir, 'gitsource')
 
         bb.utils.mkdirhier(self.srcdir)
-        self.git('init', cwd=self.srcdir)
+        self.git_init(cwd=self.srcdir)
         self.d.setVar('WORKDIR', self.tempdir)
         self.d.setVar('S', self.gitdir)
         self.d.delVar('PREMIRRORS')
         self.d.delVar('MIRRORS')
 
-        uri = 'git://%s;protocol=file;subdir=${S}' % self.srcdir
+        uri = 'git://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
         self.d.setVar('SRC_URI', uri)
         self.d.setVar('SRCREV', '${AUTOREV}')
         self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
@@ -1526,11 +1781,14 @@ class GitShallowTest(FetcherTest):
         self.d.setVar('BB_GIT_SHALLOW', '1')
         self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
         self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
+        self.d.setVar("__BBSRCREV_SEEN", "1")
 
     def assertRefs(self, expected_refs, cwd=None):
         if cwd is None:
             cwd = self.gitdir
         actual_refs = self.git(['for-each-ref', '--format=%(refname)'], cwd=cwd).splitlines()
+        # Resolve references into the same format as the comparision (needed by git 2.48 onwards)
+        actual_refs = self.git(['rev-parse', '--symbolic-full-name'] + actual_refs, cwd=cwd).splitlines()
         full_expected = self.git(['rev-parse', '--symbolic-full-name'] + expected_refs, cwd=cwd).splitlines()
         self.assertEqual(sorted(set(full_expected)), sorted(set(actual_refs)))
 
@@ -1543,15 +1801,6 @@ class GitShallowTest(FetcherTest):
         actual_count = len(revs.splitlines())
         self.assertEqual(expected_count, actual_count, msg='Object count `%d` is not the expected `%d`' % (actual_count, expected_count))
 
-    def git(self, cmd, cwd=None):
-        if isinstance(cmd, str):
-            cmd = 'git ' + cmd
-        else:
-            cmd = ['git'] + cmd
-        if cwd is None:
-            cwd = self.gitdir
-        return bb.process.run(cmd, cwd=cwd)[0]
-
     def add_empty_file(self, path, cwd=None, msg=None):
         if msg is None:
             msg = path
@@ -1586,7 +1835,6 @@ class GitShallowTest(FetcherTest):
     def fetch_shallow(self, uri=None, disabled=False, keepclone=False):
         """Fetch a uri, generating a shallow tarball, then unpack using it"""
         fetcher, ud = self.fetch_and_unpack(uri)
-        assert os.path.exists(ud.clonedir), 'Git clone in DLDIR (%s) does not exist for uri %s' % (ud.clonedir, uri)
 
         # Confirm that the unpacked repo is unshallow
         if not disabled:
@@ -1594,8 +1842,10 @@ class GitShallowTest(FetcherTest):
 
         # fetch and unpack, from the shallow tarball
         bb.utils.remove(self.gitdir, recurse=True)
-        bb.utils.remove(ud.clonedir, recurse=True)
-        bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
+        if os.path.exists(ud.clonedir):
+            bb.process.run('chmod u+w -R "%s"' % ud.clonedir)
+            bb.utils.remove(ud.clonedir, recurse=True)
+            bb.utils.remove(ud.clonedir.replace('gitsource', 'gitsubmodule'), recurse=True)
 
         # confirm that the unpacked repo is used when no git clone or git
         # mirror tarball is available
@@ -1678,7 +1928,12 @@ class GitShallowTest(FetcherTest):
         self.add_empty_file('c')
         self.assertRevCount(3, cwd=self.srcdir)
 
+        # Clone without tarball
+        self.d.setVar('BB_GIT_SHALLOW', '0')
+        fetcher, ud = self.fetch()
+
         # Clone and generate mirror tarball
+        self.d.setVar('BB_GIT_SHALLOW', '1')
         fetcher, ud = self.fetch()
 
         # Ensure we have a current mirror tarball, but an out of date clone
@@ -1690,6 +1945,7 @@ class GitShallowTest(FetcherTest):
         fetcher, ud = self.fetch()
         fetcher.unpack(self.d.getVar('WORKDIR'))
         self.assertRevCount(1)
+        assert os.path.exists(os.path.join(self.d.getVar('WORKDIR'), 'git', 'c'))
 
     def test_shallow_single_branch_no_merge(self):
         self.add_empty_file('a')
@@ -1745,7 +2001,7 @@ class GitShallowTest(FetcherTest):
 
         smdir = os.path.join(self.tempdir, 'gitsubmodule')
         bb.utils.mkdirhier(smdir)
-        self.git('init', cwd=smdir)
+        self.git_init(cwd=smdir)
         # Make this look like it was cloned from a remote...
         self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
         self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1753,11 +2009,11 @@ class GitShallowTest(FetcherTest):
         self.add_empty_file('bsub', cwd=smdir)
 
         self.git('submodule init', cwd=self.srcdir)
-        self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
+        self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir)
         self.git('submodule update', cwd=self.srcdir)
         self.git('commit -m submodule -a', cwd=self.srcdir)
 
-        uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
+        uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
         fetcher, ud = self.fetch_shallow(uri)
 
         # Verify the main repository is shallow
@@ -1775,7 +2031,7 @@ class GitShallowTest(FetcherTest):
 
         smdir = os.path.join(self.tempdir, 'gitsubmodule')
         bb.utils.mkdirhier(smdir)
-        self.git('init', cwd=smdir)
+        self.git_init(cwd=smdir)
         # Make this look like it was cloned from a remote...
         self.git('config --add remote.origin.url "%s"' % smdir, cwd=smdir)
         self.git('config --add remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"', cwd=smdir)
@@ -1783,19 +2039,19 @@ class GitShallowTest(FetcherTest):
         self.add_empty_file('bsub', cwd=smdir)
 
         self.git('submodule init', cwd=self.srcdir)
-        self.git('submodule add file://%s' % smdir, cwd=self.srcdir)
+        self.git('-c protocol.file.allow=always submodule add file://%s' % smdir, cwd=self.srcdir)
         self.git('submodule update', cwd=self.srcdir)
         self.git('commit -m submodule -a', cwd=self.srcdir)
 
-        uri = 'gitsm://%s;protocol=file;subdir=${S}' % self.srcdir
+        uri = 'gitsm://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
 
         # Fetch once to generate the shallow tarball
         fetcher, ud = self.fetch(uri)
 
         # Set up the mirror
         mirrordir = os.path.join(self.tempdir, 'mirror')
-        os.rename(self.dldir, mirrordir)
-        self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/\n' % mirrordir)
+        bb.utils.rename(self.dldir, mirrordir)
+        self.d.setVar('PREMIRRORS', 'gitsm://.*/.* file://%s/' % mirrordir)
 
         # Fetch from the mirror
         bb.utils.remove(self.dldir, recurse=True)
@@ -1818,80 +2074,27 @@ class GitShallowTest(FetcherTest):
             self.git('annex init', cwd=self.srcdir)
             open(os.path.join(self.srcdir, 'c'), 'w').close()
             self.git('annex add c', cwd=self.srcdir)
-            self.git('commit -m annex-c -a', cwd=self.srcdir)
-            bb.process.run('chmod u+w -R %s' % os.path.join(self.srcdir, '.git', 'annex'))
+            self.git('commit --author "Foo Bar <foo@bar>" -m annex-c -a', cwd=self.srcdir)
+            bb.process.run('chmod u+w -R %s' % self.srcdir)
 
-            uri = 'gitannex://%s;protocol=file;subdir=${S}' % self.srcdir
+            uri = 'gitannex://%s;protocol=file;subdir=${S};branch=master' % self.srcdir
             fetcher, ud = self.fetch_shallow(uri)
 
             self.assertRevCount(1)
             assert './.git/annex/' in bb.process.run('tar -tzf %s' % os.path.join(self.dldir, ud.mirrortarballs[0]))[0]
             assert os.path.exists(os.path.join(self.gitdir, 'c'))
 
-    def test_shallow_multi_one_uri(self):
-        # Create initial git repo
-        self.add_empty_file('a')
-        self.add_empty_file('b')
-        self.git('checkout -b a_branch', cwd=self.srcdir)
-        self.add_empty_file('c')
-        self.add_empty_file('d')
-        self.git('checkout master', cwd=self.srcdir)
-        self.git('tag v0.0 a_branch', cwd=self.srcdir)
-        self.add_empty_file('e')
-        self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
-        self.add_empty_file('f')
-        self.assertRevCount(7, cwd=self.srcdir)
-
-        uri = self.d.getVar('SRC_URI').split()[0]
-        uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
-
-        self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
-        self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
-        self.d.setVar('SRCREV_master', '${AUTOREV}')
-        self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
-
-        self.fetch_shallow(uri)
-
-        self.assertRevCount(5)
-        self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
-
-    def test_shallow_multi_one_uri_depths(self):
-        # Create initial git repo
-        self.add_empty_file('a')
-        self.add_empty_file('b')
-        self.git('checkout -b a_branch', cwd=self.srcdir)
-        self.add_empty_file('c')
-        self.add_empty_file('d')
-        self.git('checkout master', cwd=self.srcdir)
-        self.add_empty_file('e')
-        self.git('merge --no-ff --no-edit a_branch', cwd=self.srcdir)
-        self.add_empty_file('f')
-        self.assertRevCount(7, cwd=self.srcdir)
-
-        uri = self.d.getVar('SRC_URI').split()[0]
-        uri = '%s;branch=master,a_branch;name=master,a_branch' % uri
-
-        self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
-        self.d.setVar('BB_GIT_SHALLOW_DEPTH_master', '3')
-        self.d.setVar('BB_GIT_SHALLOW_DEPTH_a_branch', '1')
-        self.d.setVar('SRCREV_master', '${AUTOREV}')
-        self.d.setVar('SRCREV_a_branch', '${AUTOREV}')
-
-        self.fetch_shallow(uri)
-
-        self.assertRevCount(4, ['--all'])
-        self.assertRefs(['master', 'origin/master', 'origin/a_branch'])
-
     def test_shallow_clone_preferred_over_shallow(self):
         self.add_empty_file('a')
         self.add_empty_file('b')
 
         # Fetch once to generate the shallow tarball
+        self.d.setVar('BB_GIT_SHALLOW', '0')
         fetcher, ud = self.fetch()
-        assert os.path.exists(os.path.join(self.dldir, ud.mirrortarballs[0]))
 
         # Fetch and unpack with both the clonedir and shallow tarball available
         bb.utils.remove(self.gitdir, recurse=True)
+        self.d.setVar('BB_GIT_SHALLOW', '1')
         fetcher, ud = self.fetch_and_unpack()
 
         # The unpacked tree should *not* be shallow
@@ -1910,9 +2113,9 @@ class GitShallowTest(FetcherTest):
         # Set up the mirror
         mirrordir = os.path.join(self.tempdir, 'mirror')
         bb.utils.mkdirhier(mirrordir)
-        self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/\n' % mirrordir)
+        self.d.setVar('PREMIRRORS', 'git://.*/.* file://%s/' % mirrordir)
 
-        os.rename(os.path.join(self.dldir, mirrortarball),
+        bb.utils.rename(os.path.join(self.dldir, mirrortarball),
                   os.path.join(mirrordir, mirrortarball))
 
         # Fetch from the mirror
@@ -1999,7 +2202,7 @@ class GitShallowTest(FetcherTest):
 
         self.fetch_shallow()
 
-        self.assertRevCount(5)
+        self.assertRevCount(2)
 
     def test_shallow_invalid_revs(self):
         self.add_empty_file('a')
@@ -2018,7 +2221,10 @@ class GitShallowTest(FetcherTest):
         self.git('tag v0.0 master', cwd=self.srcdir)
         self.d.setVar('BB_GIT_SHALLOW_DEPTH', '0')
         self.d.setVar('BB_GIT_SHALLOW_REVS', 'v0.0')
-        self.fetch_shallow()
+
+        with self.assertRaises(bb.fetch2.FetchError), self.assertLogs("BitBake.Fetcher", level="ERROR") as cm:
+            self.fetch_shallow()
+        self.assertIn("fatal: no commits selected for shallow requests", cm.output[0])
 
     def test_shallow_fetch_missing_revs_fails(self):
         self.add_empty_file('a')
@@ -2032,8 +2238,35 @@ class GitShallowTest(FetcherTest):
         self.assertIn("Unable to find revision v0.0 even from upstream", cm.output[0])
 
     @skipIfNoNetwork()
+    def test_git_shallow_fetch_premirrors(self):
+        url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
+
+        # Create a separate premirror directory within tempdir
+        premirror = os.path.join(self.tempdir, "premirror")
+        os.mkdir(premirror)
+
+        # Fetch a non-shallow clone into the premirror subdir
+        self.d.setVar('BB_GIT_SHALLOW', '0')
+        self.d.setVar("DL_DIR", premirror)
+        fetcher, ud = self.fetch(url)
+
+        # Fetch a shallow clone from the premirror subdir with unpacking
+        # using the original recipe URL and the premirror mapping
+        self.d.setVar('BB_GIT_SHALLOW', '1')
+        self.d.setVar("DL_DIR", self.dldir)
+        self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
+        self.d.setVar('BB_NO_NETWORK', '1')
+        self.d.setVar('BB_GENERATE_MIRROR_TARBALLS', '0')
+        self.d.setVar("PREMIRRORS", "git://.*/.* git://{0};protocol=file".format(premirror + "/git2/" + ud.host + ud.path.replace("/", ".")))
+        fetcher = self.fetch_and_unpack(url)
+
+        # Verify that the unpacked sources are shallow clones
+        self.assertRevCount(1)
+        assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
+
+    @skipIfNoNetwork()
     def test_bitbake(self):
-        self.git('remote add --mirror=fetch origin git://github.com/openembedded/bitbake', cwd=self.srcdir)
+        self.git('remote add --mirror=fetch origin https://github.com/openembedded/bitbake', cwd=self.srcdir)
         self.git('config core.bare true', cwd=self.srcdir)
         self.git('fetch', cwd=self.srcdir)
 
@@ -2049,7 +2282,7 @@ class GitShallowTest(FetcherTest):
         revs = len(self.git('rev-list master').splitlines())
         self.assertNotEqual(orig_revs, revs)
         self.assertRefs(['master', 'origin/master'])
-        self.assertRevCount(orig_revs - 1758)
+        self.assertRevCount(orig_revs - 1760)
 
     def test_that_unpack_throws_an_error_when_the_git_clone_nor_shallow_tarball_exist(self):
         self.add_empty_file('a')
@@ -2063,27 +2296,43 @@ class GitShallowTest(FetcherTest):
         self.assertIn("No up to date source found", context.exception.msg)
         self.assertIn("clone directory not available or not up to date", context.exception.msg)
 
-    @skipIfNoNetwork()
-    def test_that_unpack_does_work_when_using_git_shallow_tarball_but_tarball_is_not_available(self):
-        self.d.setVar('SRCREV', 'e5939ff608b95cdd4d0ab0e1935781ab9a276ac0')
-        self.d.setVar('BB_GIT_SHALLOW', '1')
-        self.d.setVar('BB_GENERATE_SHALLOW_TARBALLS', '1')
-        fetcher = bb.fetch.Fetch(["git://git.yoctoproject.org/fstests"], self.d)
-        fetcher.download()
+    def test_shallow_check_is_shallow(self):
+        self.add_empty_file('a')
+        self.add_empty_file('b')
 
-        bb.utils.remove(self.dldir + "/*.tar.gz")
-        fetcher.unpack(self.unpackdir)
+        # Fetch and unpack without the clonedir and *only* shallow tarball available
+        bb.utils.remove(self.gitdir, recurse=True)
+        fetcher, ud = self.fetch_and_unpack()
 
-        dir = os.listdir(self.unpackdir + "/git/")
-        self.assertIn("fstests.doap", dir)
+        # The unpacked tree *should* be shallow
+        self.assertRevCount(1)
+        assert os.path.exists(os.path.join(self.gitdir, '.git', 'shallow'))
+
+    def test_shallow_succeeds_with_tag_containing_slash(self):
+        self.add_empty_file('a')
+        self.add_empty_file('b')
+        self.git('tag t1/t2/t3', cwd=self.srcdir)
+        self.assertRevCount(2, cwd=self.srcdir)
+
+        srcrev = self.git('rev-parse HEAD', cwd=self.srcdir).strip()
+        self.d.setVar('SRCREV', srcrev)
+        uri = self.d.getVar('SRC_URI').split()[0]
+        uri = '%s;tag=t1/t2/t3' % uri
+        self.fetch_shallow(uri)
+        self.assertRevCount(1)
 
 class GitLfsTest(FetcherTest):
+    def skipIfNoGitLFS():
+        if not shutil.which('git-lfs'):
+            return unittest.skip('git-lfs not installed')
+        return lambda f: f
+
     def setUp(self):
         FetcherTest.setUp(self)
 
         self.gitdir = os.path.join(self.tempdir, 'git')
         self.srcdir = os.path.join(self.tempdir, 'gitsource')
-        
+
         self.d.setVar('WORKDIR', self.tempdir)
         self.d.setVar('S', self.gitdir)
         self.d.delVar('PREMIRRORS')
@@ -2091,22 +2340,24 @@ class GitLfsTest(FetcherTest):
 
         self.d.setVar('SRCREV', '${AUTOREV}')
         self.d.setVar('AUTOREV', '${@bb.fetch2.get_autorev(d)}')
+        self.d.setVar("__BBSRCREV_SEEN", "1")
 
         bb.utils.mkdirhier(self.srcdir)
-        self.git('init', cwd=self.srcdir)
-        with open(os.path.join(self.srcdir, '.gitattributes'), 'wt') as attrs:
-            attrs.write('*.mp3 filter=lfs -text')
-        self.git(['add', '.gitattributes'], cwd=self.srcdir)
-        self.git(['commit', '-m', "attributes", '.gitattributes'], cwd=self.srcdir)
+        self.git_init(cwd=self.srcdir)
+        self.commit_file('.gitattributes', '*.mp3 filter=lfs -text')
 
-    def git(self, cmd, cwd=None):
-        if isinstance(cmd, str):
-            cmd = 'git ' + cmd
-        else:
-            cmd = ['git'] + cmd
-        if cwd is None:
-            cwd = self.gitdir
-        return bb.process.run(cmd, cwd=cwd)[0]
+    def commit(self, *, cwd=None):
+        cwd = cwd or self.srcdir
+        self.git(["commit", "-m", "Change"], cwd=cwd)
+        return self.git(["rev-parse", "HEAD"], cwd=cwd).strip()
+
+    def commit_file(self, filename, content, *, cwd=None):
+        cwd = cwd or self.srcdir
+
+        with open(os.path.join(cwd, filename), "w") as f:
+            f.write(content)
+        self.git(["add", filename], cwd=cwd)
+        return self.commit(cwd=cwd)
 
     def fetch(self, uri=None, download=True):
         uris = self.d.getVar('SRC_URI').split()
@@ -2119,65 +2370,259 @@ class GitLfsTest(FetcherTest):
         ud = fetcher.ud[uri]
         return fetcher, ud
 
+    def get_real_git_lfs_file(self):
+        self.d.setVar('PATH', os.environ.get('PATH'))
+        fetcher, ud = self.fetch()
+        fetcher.unpack(self.d.getVar('WORKDIR'))
+        unpacked_lfs_file = os.path.join(self.d.getVar('WORKDIR'), 'git', "Cat_poster_1.jpg")
+        return unpacked_lfs_file
+
+    @skipIfNoGitLFS()
+    def test_gitsm_lfs(self):
+        """Test that the gitsm fetcher caches objects stored via LFS"""
+        self.git(["lfs", "install", "--local"], cwd=self.srcdir)
+
+        def fetch_and_verify(revision, filename, content):
+            self.d.setVar('SRCREV', revision)
+            fetcher, ud = self.fetch()
+
+            with hide_directory(submoduledir), hide_directory(self.srcdir):
+                workdir = self.d.getVar('WORKDIR')
+                fetcher.unpack(workdir)
+
+                with open(os.path.join(workdir, "git", filename)) as f:
+                    self.assertEqual(f.read(), content)
+
+        # Create the git repository that will later be used as a submodule
+        submoduledir = self.tempdir + "/submodule"
+        bb.utils.mkdirhier(submoduledir)
+        self.git_init(submoduledir)
+        self.git(["lfs", "install", "--local"], cwd=submoduledir)
+        self.commit_file('.gitattributes', '*.mp3 filter=lfs -text', cwd=submoduledir)
+
+        submodule_commit_1 = self.commit_file("a.mp3", "submodule version 1", cwd=submoduledir)
+        _ = self.commit_file("a.mp3", "submodule version 2", cwd=submoduledir)
+
+        # Add the submodule to the repository at its current HEAD revision
+        self.git(["-c", "protocol.file.allow=always", "submodule", "add", submoduledir, "submodule"],
+                 cwd=self.srcdir)
+        base_commit_1 = self.commit()
+
+        # Let the submodule point at a different revision
+        self.git(["checkout", submodule_commit_1], self.srcdir + "/submodule")
+        self.git(["add", "submodule"], cwd=self.srcdir)
+        base_commit_2 = self.commit()
+
+        # Add a LFS file to the repository
+        base_commit_3 = self.commit_file("a.mp3", "version 1")
+        # Update the added LFS file
+        base_commit_4 = self.commit_file("a.mp3", "version 2")
+
+        self.d.setVar('SRC_URI', "gitsm://%s;protocol=file;lfs=1;branch=master" % self.srcdir)
+
+        # Verify that LFS objects referenced from submodules are fetched and checked out
+        fetch_and_verify(base_commit_1, "submodule/a.mp3", "submodule version 2")
+        # Verify that the repository inside the download cache of a submodile is extended with any
+        # additional LFS objects needed when checking out a different revision.
+        fetch_and_verify(base_commit_2, "submodule/a.mp3", "submodule version 1")
+        # Verify that LFS objects referenced from the base repository are fetched and checked out
+        fetch_and_verify(base_commit_3, "a.mp3", "version 1")
+        # Verify that the cached repository is extended with any additional LFS objects required
+        # when checking out a different revision.
+        fetch_and_verify(base_commit_4, "a.mp3", "version 2")
+
+    @skipIfNoGitLFS()
+    def test_gitsm_lfs_disabled(self):
+        """Test that the gitsm fetcher does not use LFS when explicitly disabled"""
+        self.git(["lfs", "install", "--local"], cwd=self.srcdir)
+
+        def fetch_and_verify(revision, filename, content):
+            self.d.setVar('SRCREV', revision)
+            fetcher, ud = self.fetch()
+
+            with hide_directory(submoduledir), hide_directory(self.srcdir):
+                workdir = self.d.getVar('WORKDIR')
+                fetcher.unpack(workdir)
+
+                with open(os.path.join(workdir, "git", filename)) as f:
+                    # Assume that LFS did not perform smudging when the expected content is
+                    # missing.
+                    self.assertNotEqual(f.read(), content)
+
+        # Create the git repository that will later be used as a submodule
+        submoduledir = self.tempdir + "/submodule"
+        bb.utils.mkdirhier(submoduledir)
+        self.git_init(submoduledir)
+        self.git(["lfs", "install", "--local"], cwd=submoduledir)
+        self.commit_file('.gitattributes', '*.mp3 filter=lfs -text', cwd=submoduledir)
+
+        submodule_commit_1 = self.commit_file("a.mp3", "submodule version 1", cwd=submoduledir)
+
+        # Add the submodule to the repository at its current HEAD revision
+        self.git(["-c", "protocol.file.allow=always", "submodule", "add", submoduledir, "submodule"],
+                 cwd=self.srcdir)
+        base_commit_1 = self.commit()
+
+        # Add a LFS file to the repository
+        base_commit_2 = self.commit_file("a.mp3", "version 1")
+
+        self.d.setVar('SRC_URI', "gitsm://%s;protocol=file;lfs=1;branch=master;lfs=0" % self.srcdir)
+
+        # Verify that LFS objects referenced from submodules are not fetched nor checked out
+        fetch_and_verify(base_commit_1, "submodule/a.mp3", "submodule version 1")
+        # Verify that the LFS objects referenced from the base repository are not fetched nor
+        # checked out
+        fetch_and_verify(base_commit_2, "a.mp3", "version 1")
+
+    @skipIfNoGitLFS()
+    def test_fetch_lfs_on_srcrev_change(self):
+        """Test if fetch downloads missing LFS objects when a different revision within an existing repository is requested"""
+        self.git(["lfs", "install", "--local"], cwd=self.srcdir)
+
+        def fetch_and_verify(revision, filename, content):
+            self.d.setVar('SRCREV', revision)
+            fetcher, ud = self.fetch()
+
+            with hide_directory(self.srcdir):
+                workdir = self.d.getVar('WORKDIR')
+                fetcher.unpack(workdir)
+
+                with open(os.path.join(workdir, "git", filename)) as f:
+                    self.assertEqual(f.read(), content)
+
+        commit_1 = self.commit_file("a.mp3", "version 1")
+        commit_2 = self.commit_file("a.mp3", "version 2")
+
+        self.d.setVar('SRC_URI', "git://%s;protocol=file;lfs=1;branch=master" % self.srcdir)
+
+        # Seed the local download folder by fetching the latest commit and verifying that the LFS contents are
+        # available even when the upstream repository disappears.
+        fetch_and_verify(commit_2, "a.mp3", "version 2")
+        # Verify that even when an older revision is fetched, the needed LFS objects are fetched into the download
+        # folder.
+        fetch_and_verify(commit_1, "a.mp3", "version 1")
+
+    @skipIfNoGitLFS()
+    @skipIfNoNetwork()
+    def test_real_git_lfs_repo_succeeds_without_lfs_param(self):
+        self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master")
+        f = self.get_real_git_lfs_file()
+        self.assertTrue(os.path.exists(f))
+        self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f))
+
+    @skipIfNoGitLFS()
+    @skipIfNoNetwork()
+    def test_real_git_lfs_repo_succeeds(self):
+        self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=1")
+        f = self.get_real_git_lfs_file()
+        self.assertTrue(os.path.exists(f))
+        self.assertEqual("c0baab607a97839c9a328b4310713307", bb.utils.md5_file(f))
+
+    @skipIfNoGitLFS()
+    @skipIfNoNetwork()
+    def test_real_git_lfs_repo_skips(self):
+        self.d.setVar('SRC_URI', "git://gitlab.com/gitlab-examples/lfs.git;protocol=https;branch=master;lfs=0")
+        f = self.get_real_git_lfs_file()
+        # This is the actual non-smudged placeholder file on the repo if git-lfs does not run
+        lfs_file = (
+                   'version https://git-lfs.github.com/spec/v1\n'
+                   'oid sha256:34be66b1a39a1955b46a12588df9d5f6fc1da790e05cf01f3c7422f4bbbdc26b\n'
+                   'size 11423554\n'
+        )
+
+        with open(f) as fh:
+            self.assertEqual(lfs_file, fh.read())
+
+    @skipIfNoGitLFS()
     def test_lfs_enabled(self):
-        import shutil
+        uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
+        self.d.setVar('SRC_URI', uri)
+
+        # With git-lfs installed, test that we can fetch and unpack
+        fetcher, ud = self.fetch()
+        shutil.rmtree(self.gitdir, ignore_errors=True)
+        fetcher.unpack(self.d.getVar('WORKDIR'))
 
-        uri = 'git://%s;protocol=file;subdir=${S};lfs=1' % self.srcdir
+    @skipIfNoGitLFS()
+    def test_lfs_disabled(self):
+        uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir
         self.d.setVar('SRC_URI', uri)
 
-        # Careful: suppress initial attempt at downloading until
-        # we know whether git-lfs is installed.
-        fetcher, ud = self.fetch(uri=None, download=False)
-        self.assertIsNotNone(ud.method._find_git_lfs)
+        # Verify that the fetcher can survive even if the source
+        # repository has Git LFS usage configured.
+        fetcher, ud = self.fetch()
+        fetcher.unpack(self.d.getVar('WORKDIR'))
 
-        # If git-lfs can be found, the unpack should be successful. Only
-        # attempt this with the real live copy of git-lfs installed.
-        if ud.method._find_git_lfs(self.d):
-            fetcher.download()
-            shutil.rmtree(self.gitdir, ignore_errors=True)
-            fetcher.unpack(self.d.getVar('WORKDIR'))
+    @skipIfNoGitLFS()
+    def test_lfs_enabled_not_installed_during_unpack(self):
+        uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
+        self.d.setVar('SRC_URI', uri)
 
+        # Careful: suppress initial attempt at downloading
+        fetcher, ud = self.fetch(uri=None, download=False)
+
+        fetcher.download()
         # If git-lfs cannot be found, the unpack should throw an error
         with self.assertRaises(bb.fetch2.FetchError):
+            with unittest.mock.patch("shutil.which", return_value=None):
+                shutil.rmtree(self.gitdir, ignore_errors=True)
+                fetcher.unpack(self.d.getVar('WORKDIR'))
+
+    def test_lfs_enabled_not_installed(self):
+        uri = 'git://%s;protocol=file;lfs=1;branch=master' % self.srcdir
+        self.d.setVar('SRC_URI', uri)
+
+        # Careful: suppress initial attempt at downloading
+        fetcher, ud = self.fetch(uri=None, download=False)
+
+        # If git-lfs cannot be found, the download should throw an error
+        with unittest.mock.patch("shutil.which", return_value=None):
+            with self.assertRaises(bb.fetch2.FetchError):
+                fetcher.download()
+
+    def test_lfs_disabled_not_installed(self):
+        uri = 'git://%s;protocol=file;lfs=0;branch=master' % self.srcdir
+        self.d.setVar('SRC_URI', uri)
+
+        # Careful: suppress initial attempt at downloading
+        fetcher, ud = self.fetch(uri=None, download=False)
+
+        # Even if git-lfs cannot be found, the download / unpack should be successful
+        with unittest.mock.patch("shutil.which", return_value=None):
             fetcher.download()
-            ud.method._find_git_lfs = lambda d: False
             shutil.rmtree(self.gitdir, ignore_errors=True)
             fetcher.unpack(self.d.getVar('WORKDIR'))
 
-    def test_lfs_disabled(self):
-        import shutil
+    def test_lfs_enabled_not_installed_but_not_needed(self):
+        srcdir = os.path.join(self.tempdir, "emptygit")
+        bb.utils.mkdirhier(srcdir)
+        self.git_init(srcdir)
+        self.commit_file("test", "test content", cwd=srcdir)
 
-        uri = 'git://%s;protocol=file;subdir=${S};lfs=0' % self.srcdir
+        uri = 'git://%s;protocol=file;lfs=1;branch=master' % srcdir
         self.d.setVar('SRC_URI', uri)
 
-        # In contrast to test_lfs_enabled(), allow the implicit download
-        # done by self.fetch() to occur here. The point of this test case
-        # is to verify that the fetcher can survive even if the source
-        # repository has Git LFS usage configured.
-        fetcher, ud = self.fetch()
-        self.assertIsNotNone(ud.method._find_git_lfs)
-
-        # If git-lfs can be found, the unpack should be successful. A
-        # live copy of git-lfs is not required for this case, so
-        # unconditionally forge its presence.
-        ud.method._find_git_lfs = lambda d: True
-        shutil.rmtree(self.gitdir, ignore_errors=True)
-        fetcher.unpack(self.d.getVar('WORKDIR'))
+        # Careful: suppress initial attempt at downloading
+        fetcher, ud = self.fetch(uri=None, download=False)
 
-        # If git-lfs cannot be found, the unpack should be successful
-        ud.method._find_git_lfs = lambda d: False
-        shutil.rmtree(self.gitdir, ignore_errors=True)
-        fetcher.unpack(self.d.getVar('WORKDIR'))
+        # It shouldnt't matter that git-lfs cannot be found as the repository configuration does not
+        # specify any LFS filters.
+        with unittest.mock.patch("shutil.which", return_value=None):
+            fetcher.download()
+            shutil.rmtree(self.gitdir, ignore_errors=True)
+            fetcher.unpack(self.d.getVar('WORKDIR'))
 
 class GitURLWithSpacesTest(FetcherTest):
     test_git_urls = {
-        "git://tfs-example.org:22/tfs/example%20path/example.git" : {
-            'url': 'git://tfs-example.org:22/tfs/example%20path/example.git',
+        "git://tfs-example.org:22/tfs/example%20path/example.git;branch=master" : {
+            'url': 'git://tfs-example.org:22/tfs/example%20path/example.git;branch=master',
+            'repo_url': 'git://tfs-example.org:22/tfs/example%20path/example.git',
             'gitsrcname': 'tfs-example.org.22.tfs.example_path.example.git',
             'path': '/tfs/example path/example.git'
         },
-        "git://tfs-example.org:22/tfs/example%20path/example%20repo.git" : {
-            'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git',
+        "git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master" : {
+            'url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git;branch=master',
+            'repo_url': 'git://tfs-example.org:22/tfs/example%20path/example%20repo.git',
             'gitsrcname': 'tfs-example.org.22.tfs.example_path.example_repo.git',
             'path': '/tfs/example path/example repo.git'
         }
@@ -2200,19 +2645,137 @@ class GitURLWithSpacesTest(FetcherTest):
             self.assertEqual(ud.lockfile, os.path.join(self.dldir, "git2", ref['gitsrcname'] + '.lock'))
             self.assertEqual(ud.clonedir, os.path.join(self.dldir, "git2", ref['gitsrcname']))
             self.assertEqual(ud.fullmirror, os.path.join(self.dldir, "git2_" + ref['gitsrcname'] + '.tar.gz'))
+            self.assertEqual(ud.method._get_repo_url(ud), ref['repo_url'])
+
+class CrateTest(FetcherTest):
+    @skipIfNoNetwork()
+    def test_crate_url(self):
+
+        uri = "crate://crates.io/glob/0.2.11"
+        self.d.setVar('SRC_URI', uri)
+
+        uris = self.d.getVar('SRC_URI').split()
+        d = self.d
+
+        fetcher = bb.fetch2.Fetch(uris, self.d)
+        ud = fetcher.ud[fetcher.urls[0]]
+
+        self.assertIn("name", ud.parm)
+        self.assertEqual(ud.parm["name"], "glob-0.2.11")
+        self.assertIn("downloadfilename", ud.parm)
+        self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
+
+        fetcher.download()
+        fetcher.unpack(self.tempdir)
+        self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
+        self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done'])
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json"))
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs"))
+
+    @skipIfNoNetwork()
+    def test_crate_url_matching_recipe(self):
+
+        self.d.setVar('BP', 'glob-0.2.11')
+
+        uri = "crate://crates.io/glob/0.2.11"
+        self.d.setVar('SRC_URI', uri)
+
+        uris = self.d.getVar('SRC_URI').split()
+        d = self.d
+
+        fetcher = bb.fetch2.Fetch(uris, self.d)
+        ud = fetcher.ud[fetcher.urls[0]]
+
+        self.assertIn("name", ud.parm)
+        self.assertEqual(ud.parm["name"], "glob-0.2.11")
+        self.assertIn("downloadfilename", ud.parm)
+        self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
+
+        fetcher.download()
+        fetcher.unpack(self.tempdir)
+        self.assertEqual(sorted(os.listdir(self.tempdir)), ['download', 'glob-0.2.11', 'unpacked'])
+        self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done'])
+        self.assertTrue(os.path.exists(self.tempdir + "/glob-0.2.11/src/lib.rs"))
+
+    @skipIfNoNetwork()
+    def test_crate_url_params(self):
+
+        uri = "crate://crates.io/aho-corasick/0.7.20;name=aho-corasick-renamed"
+        self.d.setVar('SRC_URI', uri)
+
+        uris = self.d.getVar('SRC_URI').split()
+        d = self.d
+
+        fetcher = bb.fetch2.Fetch(uris, self.d)
+        ud = fetcher.ud[fetcher.urls[0]]
+
+        self.assertIn("name", ud.parm)
+        self.assertEqual(ud.parm["name"], "aho-corasick-renamed")
+        self.assertIn("downloadfilename", ud.parm)
+        self.assertEqual(ud.parm["downloadfilename"], "aho-corasick-0.7.20.crate")
+
+        fetcher.download()
+        fetcher.unpack(self.tempdir)
+        self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
+        self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['aho-corasick-0.7.20.crate', 'aho-corasick-0.7.20.crate.done'])
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/.cargo-checksum.json"))
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/aho-corasick-0.7.20/src/lib.rs"))
+
+    @skipIfNoNetwork()
+    def test_crate_url_multi(self):
+
+        uri = "crate://crates.io/glob/0.2.11 crate://crates.io/time/0.1.35"
+        self.d.setVar('SRC_URI', uri)
+
+        uris = self.d.getVar('SRC_URI').split()
+        d = self.d
+
+        fetcher = bb.fetch2.Fetch(uris, self.d)
+        ud = fetcher.ud[fetcher.urls[0]]
+
+        self.assertIn("name", ud.parm)
+        self.assertEqual(ud.parm["name"], "glob-0.2.11")
+        self.assertIn("downloadfilename", ud.parm)
+        self.assertEqual(ud.parm["downloadfilename"], "glob-0.2.11.crate")
+
+        ud = fetcher.ud[fetcher.urls[1]]
+        self.assertIn("name", ud.parm)
+        self.assertEqual(ud.parm["name"], "time-0.1.35")
+        self.assertIn("downloadfilename", ud.parm)
+        self.assertEqual(ud.parm["downloadfilename"], "time-0.1.35.crate")
+
+        fetcher.download()
+        fetcher.unpack(self.tempdir)
+        self.assertEqual(sorted(os.listdir(self.tempdir)), ['cargo_home', 'download' , 'unpacked'])
+        self.assertEqual(sorted(os.listdir(self.tempdir + "/download")), ['glob-0.2.11.crate', 'glob-0.2.11.crate.done', 'time-0.1.35.crate', 'time-0.1.35.crate.done'])
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/.cargo-checksum.json"))
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/glob-0.2.11/src/lib.rs"))
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/.cargo-checksum.json"))
+        self.assertTrue(os.path.exists(self.tempdir + "/cargo_home/bitbake/time-0.1.35/src/lib.rs"))
+
+    @skipIfNoNetwork()
+    def test_crate_incorrect_cksum(self):
+        uri = "crate://crates.io/aho-corasick/0.7.20"
+        self.d.setVar('SRC_URI', uri)
+        self.d.setVarFlag("SRC_URI", "aho-corasick-0.7.20.sha256sum", hashlib.sha256("Invalid".encode("utf-8")).hexdigest())
+
+        uris = self.d.getVar('SRC_URI').split()
+
+        fetcher = bb.fetch2.Fetch(uris, self.d)
+        with self.assertRaisesRegex(bb.fetch2.FetchError, "Fetcher failure for URL"):
+            fetcher.download()
 
 class NPMTest(FetcherTest):
     def skipIfNoNpm():
-        import shutil
         if not shutil.which('npm'):
-            return unittest.skip('npm not installed, tests being skipped')
+            return unittest.skip('npm not installed')
         return lambda f: f
 
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm(self):
-        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
-        fetcher = bb.fetch.Fetch([url], self.d)
+        urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
+        fetcher = bb.fetch.Fetch(urls, self.d)
         ud = fetcher.ud[fetcher.urls[0]]
         fetcher.download()
         self.assertTrue(os.path.exists(ud.localpath))
@@ -2225,9 +2788,9 @@ class NPMTest(FetcherTest):
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_bad_checksum(self):
-        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
+        urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
         # Fetch once to get a tarball
-        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher = bb.fetch.Fetch(urls, self.d)
         ud = fetcher.ud[fetcher.urls[0]]
         fetcher.download()
         self.assertTrue(os.path.exists(ud.localpath))
@@ -2244,17 +2807,48 @@ class NPMTest(FetcherTest):
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_premirrors(self):
-        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
+        urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
         # Fetch once to get a tarball
-        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher = bb.fetch.Fetch(urls, self.d)
+        ud = fetcher.ud[fetcher.urls[0]]
+        fetcher.download()
+        self.assertTrue(os.path.exists(ud.localpath))
+
+        # Setup the mirror by renaming the download directory
+        mirrordir = os.path.join(self.tempdir, 'mirror')
+        bb.utils.rename(self.dldir, mirrordir)
+        os.mkdir(self.dldir)
+
+        # Configure the premirror to be used
+        self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/npm2' % mirrordir)
+        self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
+
+        # Fetch again
+        self.assertFalse(os.path.exists(ud.localpath))
+        # The npm fetcher doesn't handle that the .resolved file disappears
+        # while the fetcher object exists, which it does when we rename the
+        # download directory to "mirror" above. Thus we need a new fetcher to go
+        # with the now empty download directory.
+        fetcher = bb.fetch.Fetch(urls, self.d)
+        ud = fetcher.ud[fetcher.urls[0]]
+        fetcher.download()
+        self.assertTrue(os.path.exists(ud.localpath))
+
+    @skipIfNoNpm()
+    @skipIfNoNetwork()
+    def test_npm_premirrors_with_specified_filename(self):
+        urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
+        # Fetch once to get a tarball
+        fetcher = bb.fetch.Fetch(urls, self.d)
         ud = fetcher.ud[fetcher.urls[0]]
         fetcher.download()
         self.assertTrue(os.path.exists(ud.localpath))
         # Setup the mirror
         mirrordir = os.path.join(self.tempdir, 'mirror')
         bb.utils.mkdirhier(mirrordir)
-        os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
-        self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
+        mirrorfilename = os.path.join(mirrordir, os.path.basename(ud.localpath))
+        os.replace(ud.localpath, mirrorfilename)
+        self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s' % mirrorfilename)
         self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
         # Fetch again
         self.assertFalse(os.path.exists(ud.localpath))
@@ -2265,8 +2859,8 @@ class NPMTest(FetcherTest):
     @skipIfNoNetwork()
     def test_npm_mirrors(self):
         # Fetch once to get a tarball
-        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
-        fetcher = bb.fetch.Fetch([url], self.d)
+        urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
+        fetcher = bb.fetch.Fetch(urls, self.d)
         ud = fetcher.ud[fetcher.urls[0]]
         fetcher.download()
         self.assertTrue(os.path.exists(ud.localpath))
@@ -2274,7 +2868,7 @@ class NPMTest(FetcherTest):
         mirrordir = os.path.join(self.tempdir, 'mirror')
         bb.utils.mkdirhier(mirrordir)
         os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
-        self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
+        self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
         # Update the resolved url to an invalid url
         with open(ud.resolvefile, 'r') as f:
             url = f.read()
@@ -2290,27 +2884,27 @@ class NPMTest(FetcherTest):
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_destsuffix_downloadfilename(self):
-        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz'
-        fetcher = bb.fetch.Fetch([url], self.d)
+        urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0;destsuffix=foo/bar;downloadfilename=foo-bar.tgz']
+        fetcher = bb.fetch.Fetch(urls, self.d)
         fetcher.download()
-        self.assertTrue(os.path.exists(os.path.join(self.dldir, 'foo-bar.tgz')))
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'foo-bar.tgz')))
         fetcher.unpack(self.unpackdir)
         unpackdir = os.path.join(self.unpackdir, 'foo', 'bar')
         self.assertTrue(os.path.exists(os.path.join(unpackdir, 'package.json')))
 
     def test_npm_no_network_no_tarball(self):
-        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
+        urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
         self.d.setVar('BB_NO_NETWORK', '1')
-        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher = bb.fetch.Fetch(urls, self.d)
         with self.assertRaises(bb.fetch2.NetworkAccess):
             fetcher.download()
 
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_no_network_with_tarball(self):
-        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
+        urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
         # Fetch once to get a tarball
-        fetcher = bb.fetch.Fetch([url], self.d)
+        fetcher = bb.fetch.Fetch(urls, self.d)
         fetcher.download()
         # Disable network access
         self.d.setVar('BB_NO_NETWORK', '1')
@@ -2323,8 +2917,8 @@ class NPMTest(FetcherTest):
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_registry_alternate(self):
-        url = 'npm://registry.freajs.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
-        fetcher = bb.fetch.Fetch([url], self.d)
+        urls = ['npm://skimdb.npmjs.com;package=@savoirfairelinux/node-server-example;version=1.0.0']
+        fetcher = bb.fetch.Fetch(urls, self.d)
         fetcher.download()
         fetcher.unpack(self.unpackdir)
         unpackdir = os.path.join(self.unpackdir, 'npm')
@@ -2333,8 +2927,8 @@ class NPMTest(FetcherTest):
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_version_latest(self):
-        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest'
-        fetcher = bb.fetch.Fetch([url], self.d)
+        url = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=latest']
+        fetcher = bb.fetch.Fetch(url, self.d)
         fetcher.download()
         fetcher.unpack(self.unpackdir)
         unpackdir = os.path.join(self.unpackdir, 'npm')
@@ -2343,46 +2937,46 @@ class NPMTest(FetcherTest):
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_registry_invalid(self):
-        url = 'npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0'
-        fetcher = bb.fetch.Fetch([url], self.d)
+        urls = ['npm://registry.invalid.org;package=@savoirfairelinux/node-server-example;version=1.0.0']
+        fetcher = bb.fetch.Fetch(urls, self.d)
         with self.assertRaises(bb.fetch2.FetchError):
             fetcher.download()
 
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_package_invalid(self):
-        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0'
-        fetcher = bb.fetch.Fetch([url], self.d)
+        urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/invalid;version=1.0.0']
+        fetcher = bb.fetch.Fetch(urls, self.d)
         with self.assertRaises(bb.fetch2.FetchError):
             fetcher.download()
 
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_version_invalid(self):
-        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid'
+        urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example;version=invalid']
         with self.assertRaises(bb.fetch2.ParameterError):
-            fetcher = bb.fetch.Fetch([url], self.d)
+            fetcher = bb.fetch.Fetch(urls, self.d)
 
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_registry_none(self):
-        url = 'npm://;package=@savoirfairelinux/node-server-example;version=1.0.0'
+        urls = ['npm://;package=@savoirfairelinux/node-server-example;version=1.0.0']
         with self.assertRaises(bb.fetch2.MalformedUrl):
-            fetcher = bb.fetch.Fetch([url], self.d)
+            fetcher = bb.fetch.Fetch(urls, self.d)
 
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_package_none(self):
-        url = 'npm://registry.npmjs.org;version=1.0.0'
+        urls = ['npm://registry.npmjs.org;version=1.0.0']
         with self.assertRaises(bb.fetch2.MissingParameterError):
-            fetcher = bb.fetch.Fetch([url], self.d)
+            fetcher = bb.fetch.Fetch(urls, self.d)
 
     @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npm_version_none(self):
-        url = 'npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example'
+        urls = ['npm://registry.npmjs.org;package=@savoirfairelinux/node-server-example']
         with self.assertRaises(bb.fetch2.MissingParameterError):
-            fetcher = bb.fetch.Fetch([url], self.d)
+            fetcher = bb.fetch.Fetch(urls, self.d)
 
     def create_shrinkwrap_file(self, data):
         import json
@@ -2391,32 +2985,30 @@ class NPMTest(FetcherTest):
         bb.utils.mkdirhier(datadir)
         with open(swfile, 'w') as f:
             json.dump(data, f)
-        # Also configure the S directory
-        self.sdir = os.path.join(self.unpackdir, 'S')
-        self.d.setVar('S', self.sdir)
         return swfile
 
-    @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npmsw(self):
         swfile = self.create_shrinkwrap_file({
-            'dependencies': {
-                'array-flatten': {
+            'packages': {
+                'node_modules/array-flatten': {
                     'version': '1.1.1',
                     'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
                     'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=',
                     'dependencies': {
-                        'content-type': {
-                            'version': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
-                            'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
-                            'dependencies': {
-                                'cookie': {
-                                    'version': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09',
-                                    'from': 'git+https://github.com/jshttp/cookie.git'
-                                }
-                            }
-                        }
+                        'content-type': "1.0.4"
                     }
+                },
+                'node_modules/array-flatten/node_modules/content-type': {
+                    'version': '1.0.4',
+                    'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
+                    'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
+                    'dependencies': {
+                        'cookie': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09'
+                    }
+                },
+                'node_modules/array-flatten/node_modules/content-type/node_modules/cookie': {
+                    'resolved': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09'
                 }
             }
         })
@@ -2426,22 +3018,34 @@ class NPMTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
         self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
         fetcher.unpack(self.unpackdir)
-        self.assertTrue(os.path.exists(os.path.join(self.sdir, 'npm-shrinkwrap.json')))
-        self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json')))
-        self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json')))
-        self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json')))
+        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm-shrinkwrap.json')))
+        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'package.json')))
+        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'package.json')))
+        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'node_modules', 'content-type', 'node_modules', 'cookie', 'package.json')))
+
+    @skipIfNoNetwork()
+    def test_npmsw_git(self):
+        swfile = self.create_shrinkwrap_file({
+            'packages': {
+                'node_modules/cookie': {
+                    'resolved': 'git+https://github.com/jshttp/cookie.git#aec1177c7da67e3b3273df96cf476824dbc9ae09'
+                }
+            }
+        })
+        fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
+        fetcher.download()
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, 'git2', 'github.com.jshttp.cookie.git')))
 
-    @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npmsw_dev(self):
         swfile = self.create_shrinkwrap_file({
-            'dependencies': {
-                'array-flatten': {
+            'packages': {
+                'node_modules/array-flatten': {
                     'version': '1.1.1',
                     'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
                     'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
                 },
-                'content-type': {
+                'node_modules/content-type': {
                     'version': '1.0.4',
                     'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
                     'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
@@ -2460,12 +3064,11 @@ class NPMTest(FetcherTest):
         self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
         self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
 
-    @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npmsw_destsuffix(self):
         swfile = self.create_shrinkwrap_file({
-            'dependencies': {
-                'array-flatten': {
+            'packages': {
+                'node_modules/array-flatten': {
                     'version': '1.1.1',
                     'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
                     'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
@@ -2479,8 +3082,8 @@ class NPMTest(FetcherTest):
 
     def test_npmsw_no_network_no_tarball(self):
         swfile = self.create_shrinkwrap_file({
-            'dependencies': {
-                'array-flatten': {
+            'packages': {
+                'node_modules/array-flatten': {
                     'version': '1.1.1',
                     'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
                     'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
@@ -2502,8 +3105,8 @@ class NPMTest(FetcherTest):
         self.d.setVar('BB_NO_NETWORK', '1')
         # Fetch again
         swfile = self.create_shrinkwrap_file({
-            'dependencies': {
-                'array-flatten': {
+            'packages': {
+                'node_modules/array-flatten': {
                     'version': '1.1.1',
                     'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
                     'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
@@ -2513,15 +3116,14 @@ class NPMTest(FetcherTest):
         fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
         fetcher.download()
         fetcher.unpack(self.unpackdir)
-        self.assertTrue(os.path.exists(os.path.join(self.sdir, 'node_modules', 'array-flatten', 'package.json')))
+        self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'node_modules', 'array-flatten', 'package.json')))
 
-    @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npmsw_npm_reusability(self):
         # Fetch once with npmsw
         swfile = self.create_shrinkwrap_file({
-            'dependencies': {
-                'array-flatten': {
+            'packages': {
+                'node_modules/array-flatten': {
                     'version': '1.1.1',
                     'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
                     'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
@@ -2538,13 +3140,12 @@ class NPMTest(FetcherTest):
         fetcher.unpack(self.unpackdir)
         self.assertTrue(os.path.exists(os.path.join(self.unpackdir, 'npm', 'package.json')))
 
-    @skipIfNoNpm()
     @skipIfNoNetwork()
     def test_npmsw_bad_checksum(self):
         # Try to fetch with bad checksum
         swfile = self.create_shrinkwrap_file({
-            'dependencies': {
-                'array-flatten': {
+            'packages': {
+                'node_modules/array-flatten': {
                     'version': '1.1.1',
                     'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
                     'integrity': 'sha1-gfNEp2hqgLTFKT6P3AsBYMgsBqg='
@@ -2556,8 +3157,8 @@ class NPMTest(FetcherTest):
             fetcher.download()
         # Fetch correctly to get a tarball
         swfile = self.create_shrinkwrap_file({
-            'dependencies': {
-                'array-flatten': {
+            'packages': {
+                'node_modules/array-flatten': {
                     'version': '1.1.1',
                     'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
                     'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
@@ -2590,13 +3191,13 @@ class NPMTest(FetcherTest):
         mirrordir = os.path.join(self.tempdir, 'mirror')
         bb.utils.mkdirhier(mirrordir)
         os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
-        self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
+        self.d.setVar('PREMIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
         self.d.setVar('BB_FETCH_PREMIRRORONLY', '1')
         # Fetch again
         self.assertFalse(os.path.exists(ud.localpath))
         swfile = self.create_shrinkwrap_file({
-            'dependencies': {
-                'array-flatten': {
+            'packages': {
+                'node_modules/array-flatten': {
                     'version': '1.1.1',
                     'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
                     'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
@@ -2619,12 +3220,12 @@ class NPMTest(FetcherTest):
         mirrordir = os.path.join(self.tempdir, 'mirror')
         bb.utils.mkdirhier(mirrordir)
         os.replace(ud.localpath, os.path.join(mirrordir, os.path.basename(ud.localpath)))
-        self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/\n' % mirrordir)
+        self.d.setVar('MIRRORS', 'https?$://.*/.* file://%s/' % mirrordir)
         # Fetch again with invalid url
         self.assertFalse(os.path.exists(ud.localpath))
         swfile = self.create_shrinkwrap_file({
-            'dependencies': {
-                'array-flatten': {
+            'packages': {
+                'node_modules/array-flatten': {
                     'version': '1.1.1',
                     'resolved': 'https://invalid',
                     'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
@@ -2634,3 +3235,521 @@ class NPMTest(FetcherTest):
         fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
         fetcher.download()
         self.assertTrue(os.path.exists(ud.localpath))
+
+    @skipIfNoNetwork()
+    def test_npmsw_bundled(self):
+        swfile = self.create_shrinkwrap_file({
+            'packages': {
+                'node_modules/array-flatten': {
+                    'version': '1.1.1',
+                    'resolved': 'https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz',
+                    'integrity': 'sha1-ml9pkFGx5wczKPKgCJaLZOopVdI='
+                },
+                'node_modules/content-type': {
+                    'version': '1.0.4',
+                    'resolved': 'https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz',
+                    'integrity': 'sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==',
+                    'inBundle': True
+                }
+            }
+        })
+        fetcher = bb.fetch.Fetch(['npmsw://' + swfile], self.d)
+        fetcher.download()
+        self.assertTrue(os.path.exists(os.path.join(self.dldir, 'npm2', 'array-flatten-1.1.1.tgz')))
+        self.assertFalse(os.path.exists(os.path.join(self.dldir, 'npm2', 'content-type-1.0.4.tgz')))
+
+class GitSharedTest(FetcherTest):
+    def setUp(self):
+        super(GitSharedTest, self).setUp()
+        self.recipe_url = "git://git.openembedded.org/bitbake;branch=master;protocol=https"
+        self.d.setVar('SRCREV', '82ea737a0b42a8b53e11c9cde141e9e9c0bd8c40')
+        self.d.setVar("__BBSRCREV_SEEN", "1")
+
+    @skipIfNoNetwork()
+    def test_shared_unpack(self):
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertTrue(os.path.exists(alt))
+
+    @skipIfNoNetwork()
+    def test_noshared_unpack(self):
+        self.d.setVar('BB_GIT_NOSHARED', '1')
+        self.unpackdir += '_noshared'
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        alt = os.path.join(self.unpackdir, 'git/.git/objects/info/alternates')
+        self.assertFalse(os.path.exists(alt))
+
+class GitTagVerificationTests(FetcherTest):
+
+    @skipIfNoNetwork()
+    def test_tag_rev_match(self):
+        # Test a url with rev= and tag= set works
+        fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=aa0e540fc31a1c26839efd2c7785a751ce24ebfb;tag=2.8.7"], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+
+    def test_annotated_tag_rev_match(self):
+        # Test a url with rev= and tag= set works
+        # rev is the annotated tag revision in this case
+        fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=6d363159e4b7dc566fc40d069b2615e61774a7d8;tag=2.8.7"], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+
+    @skipIfNoNetwork()
+    def test_tag_rev_match2(self):
+        # Test a url with SRCREV and tag= set works
+        self.d.setVar('SRCREV', 'aa0e540fc31a1c26839efd2c7785a751ce24ebfb')
+        fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;tag=2.8.7"], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+
+    @skipIfNoNetwork()
+    def test_tag_rev_match3(self):
+        # Test a url with SRCREV, rev= and tag= set works
+        self.d.setVar('SRCREV', 'aa0e540fc31a1c26839efd2c7785a751ce24ebfb')
+        fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=aa0e540fc31a1c26839efd2c7785a751ce24ebfb;tag=2.8.7"], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+
+    @skipIfNoNetwork()
+    def test_tag_rev_match4(self):
+        # Test a url with SRCREV and rev= mismatching errors
+        self.d.setVar('SRCREV', 'bade540fc31a1c26839efd2c7785a751ce24ebfb')
+        with self.assertRaises(bb.fetch2.FetchError):
+            fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=aa0e540fc31a1c26839efd2c7785a751ce24ebfb;tag=2.8.7"], self.d)
+
+    @skipIfNoNetwork()
+    def test_tag_rev_match5(self):
+        # Test a url with SRCREV, rev= and tag= set works when using shallow clones
+        self.d.setVar('BB_GIT_SHALLOW', '1')
+        self.d.setVar('SRCREV', 'aa0e540fc31a1c26839efd2c7785a751ce24ebfb')
+        fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=aa0e540fc31a1c26839efd2c7785a751ce24ebfb;tag=2.8.7"], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+
+    @skipIfNoNetwork()
+    def test_tag_rev_match6(self):
+        # Test a url with SRCREV, rev= and a mismatched tag= when using shallow clones
+        self.d.setVar('BB_GIT_SHALLOW', '1')
+        fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=aa0e540fc31a1c26839efd2c7785a751ce24ebfb;tag=2.8.6"], self.d)
+        fetcher.download()
+        with self.assertRaises(bb.fetch2.FetchError):
+            fetcher.unpack(self.unpackdir)
+
+    @skipIfNoNetwork()
+    def test_tag_rev_match7(self):
+        # Test a url with SRCREV, rev= and a mismatched tag=
+        self.d.setVar('SRCREV', 'aa0e540fc31a1c26839efd2c7785a751ce24ebfb')
+        fetcher = bb.fetch.Fetch(["git://git.openembedded.org/bitbake;branch=2.8;protocol=https;rev=aa0e540fc31a1c26839efd2c7785a751ce24ebfb;tag=2.8.6"], self.d)
+        fetcher.download()
+        with self.assertRaises(bb.fetch2.FetchError):
+            fetcher.unpack(self.unpackdir)
+
+
+class FetchPremirroronlyLocalTest(FetcherTest):
+
+    def setUp(self):
+        super(FetchPremirroronlyLocalTest, self).setUp()
+        self.mirrordir = os.path.join(self.tempdir, "mirrors")
+        os.mkdir(self.mirrordir)
+        self.reponame = "bitbake"
+        self.gitdir = os.path.join(self.tempdir, "git", self.reponame)
+        self.recipe_url = "git://git.fake.repo/bitbake;branch=master;protocol=https"
+        self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
+        self.d.setVar("BB_NO_NETWORK", "1")
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
+        self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
+        self.mirrorfile = os.path.join(self.mirrordir, self.mirrorname)
+        self.testfilename = "bitbake-fetch.test"
+
+    def make_git_repo(self):
+        recipeurl = "git:/git.fake.repo/bitbake"
+        os.makedirs(self.gitdir)
+        self.git_init(cwd=self.gitdir)
+        for i in range(0):
+            self.git_new_commit()
+        bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd =  self.gitdir)
+
+    def git_new_commit(self):
+        import random
+        os.unlink(os.path.join(self.mirrordir, self.mirrorname))
+        branch = self.git("branch --show-current", self.gitdir).split()
+        with open(os.path.join(self.gitdir, self.testfilename), "w") as testfile:
+            testfile.write("File {} from branch {}; Useless random data {}".format(self.testfilename, branch, random.random()))
+        self.git("add {}".format(self.testfilename), self.gitdir)
+        self.git("commit -a -m \"This random commit {} in branch {}. I'm useless.\"".format(random.random(), branch), self.gitdir)
+        bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd =  self.gitdir)
+        return self.git("rev-parse HEAD", self.gitdir).strip()
+
+    def git_new_branch(self, name):
+        self.git_new_commit()
+        head = self.git("rev-parse HEAD", self.gitdir).strip()
+        self.git("checkout -b {}".format(name), self.gitdir)
+        newrev = self.git_new_commit()
+        self.git("checkout {}".format(head), self.gitdir)
+        return newrev
+
+    def test_mirror_multiple_fetches(self):
+        self.make_git_repo()
+        self.d.setVar("SRCREV", self.git_new_commit())
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        ## New commit in premirror. it's not in the download_dir
+        self.d.setVar("SRCREV", self.git_new_commit())
+        fetcher2 = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher2.download()
+        fetcher2.unpack(self.unpackdir)
+        ## New commit in premirror. it's not in the download_dir
+        self.d.setVar("SRCREV", self.git_new_commit())
+        fetcher3 = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher3.download()
+        fetcher3.unpack(self.unpackdir)
+
+
+    def test_mirror_commit_nonexistent(self):
+        self.make_git_repo()
+        self.d.setVar("SRCREV", "0"*40)
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+
+    def test_mirror_commit_exists(self):
+        self.make_git_repo()
+        self.d.setVar("SRCREV", self.git_new_commit())
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+
+    def test_mirror_tarball_nonexistent(self):
+        self.d.setVar("SRCREV", "0"*40)
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+
+
+class FetchPremirroronlyNetworkTest(FetcherTest):
+
+    def setUp(self):
+        super(FetchPremirroronlyNetworkTest, self).setUp()
+        self.mirrordir = os.path.join(self.tempdir, "mirrors")
+        os.mkdir(self.mirrordir)
+        self.reponame = "fstests"
+        self.clonedir = os.path.join(self.tempdir, "git")
+        self.gitdir = os.path.join(self.tempdir, "git", "{}.git".format(self.reponame))
+        self.recipe_url = "git://git.yoctoproject.org/fstests;protocol=https;branch=master"
+        self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
+        self.d.setVar("BB_NO_NETWORK", "0")
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
+
+    def make_git_repo(self):
+        self.mirrorname = "git2_git.yoctoproject.org.fstests.tar.gz"
+        os.makedirs(self.clonedir)
+        self.git("clone --bare {}".format(self.recipe_url), self.clonedir)
+        self.git("update-ref HEAD 15413486df1f5a5b5af699b6f3ba5f0984e52a9f", self.gitdir)
+        bb.process.run('tar -czvf {} .'.format(os.path.join(self.mirrordir, self.mirrorname)), cwd =  self.gitdir)
+        shutil.rmtree(self.clonedir)
+
+    @skipIfNoNetwork()
+    def test_mirror_tarball_updated(self):
+        self.make_git_repo()
+        ## Upstream commit is in the mirror
+        self.d.setVar("SRCREV", "15413486df1f5a5b5af699b6f3ba5f0984e52a9f")
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+
+    @skipIfNoNetwork()
+    def test_mirror_tarball_outdated(self):
+        self.make_git_repo()
+        ## Upstream commit not in the mirror
+        self.d.setVar("SRCREV", "49d65d53c2bf558ae6e9185af0f3af7b79d255ec")
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        with self.assertRaises(bb.fetch2.NetworkAccess):
+            fetcher.download()
+
+class FetchPremirroronlyMercurialTest(FetcherTest):
+    """ Test for premirrors with mercurial repos
+        the test covers also basic hg:// clone (see fetch_and_create_tarball
+    """
+    def skipIfNoHg():
+        if not shutil.which('hg'):
+            return unittest.skip('Mercurial not installed')
+        return lambda f: f
+
+    def setUp(self):
+        super(FetchPremirroronlyMercurialTest, self).setUp()
+        self.mirrordir = os.path.join(self.tempdir, "mirrors")
+        os.mkdir(self.mirrordir)
+        self.reponame = "libgnt"
+        self.clonedir = os.path.join(self.tempdir, "hg")
+        self.recipe_url = "hg://keep.imfreedom.org/libgnt;module=libgnt"
+        self.d.setVar("SRCREV", "53e8b422faaf")
+        self.mirrorname = "hg_libgnt_keep.imfreedom.org_.libgnt.tar.gz"
+
+    def fetch_and_create_tarball(self):
+        """
+        Ask bitbake to download repo and prepare mirror tarball for us
+        """
+        self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "1")
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+        mirrorfile = os.path.join(self.d.getVar("DL_DIR"), self.mirrorname)
+        self.assertTrue(os.path.exists(mirrorfile), "Mirror tarball {} has not been created".format(mirrorfile))
+        ## moving tarball to mirror directory
+        os.rename(mirrorfile, os.path.join(self.mirrordir, self.mirrorname))
+        self.d.setVar("BB_GENERATE_MIRROR_TARBALLS", "0")
+
+
+    @skipIfNoNetwork()
+    @skipIfNoHg()
+    def test_premirror_mercurial(self):
+        self.fetch_and_create_tarball()
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
+        self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
+        self.d.setVar("BB_NO_NETWORK", "1")
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        fetcher.download()
+
+class FetchPremirroronlyBrokenTarball(FetcherTest):
+
+    def setUp(self):
+        super(FetchPremirroronlyBrokenTarball, self).setUp()
+        self.mirrordir = os.path.join(self.tempdir, "mirrors")
+        os.mkdir(self.mirrordir)
+        self.reponame = "bitbake"
+        self.gitdir = os.path.join(self.tempdir, "git", self.reponame)
+        self.recipe_url = "git://git.fake.repo/bitbake;protocol=https;branch=master"
+        self.d.setVar("BB_FETCH_PREMIRRORONLY", "1")
+        self.d.setVar("BB_NO_NETWORK", "1")
+        self.d.setVar("PREMIRRORS", self.recipe_url + " " + "file://{}".format(self.mirrordir) + " \n")
+        self.mirrorname = "git2_git.fake.repo.bitbake.tar.gz"
+        with open(os.path.join(self.mirrordir, self.mirrorname), 'w') as targz:
+            targz.write("This is not tar.gz file!")
+
+    def test_mirror_broken_download(self):
+        self.d.setVar("SRCREV", "0"*40)
+        fetcher = bb.fetch.Fetch([self.recipe_url], self.d)
+        with self.assertRaises(bb.fetch2.FetchError), self.assertLogs() as logs:
+            fetcher.download()
+        output = "".join(logs.output)
+        self.assertFalse(" not a git repository (or any parent up to mount point /)" in output)
+
+class GoModTest(FetcherTest):
+
+    @skipIfNoNetwork()
+    def test_gomod_url(self):
+        urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;'
+                'sha256sum=9bb69aea32f1d59711701f9562d66432c9c0374205e5009d1d1a62f03fb4fdad']
+
+        fetcher = bb.fetch2.Fetch(urls, self.d)
+        ud = fetcher.ud[urls[0]]
+        self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.zip')
+        self.assertEqual(ud.parm['downloadfilename'], 'github.com.Azure.azure-sdk-for-go.sdk.storage.azblob@v1.0.0.zip')
+        self.assertEqual(ud.parm['name'], 'github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0')
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip')))
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
+        self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')),
+                         '7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873')
+
+    @skipIfNoNetwork()
+    def test_gomod_url_go_mod_only(self):
+        urls = ['gomod://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;mod=1;'
+                'sha256sum=7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873']
+
+        fetcher = bb.fetch2.Fetch(urls, self.d)
+        ud = fetcher.ud[urls[0]]
+        self.assertEqual(ud.url, 'https://proxy.golang.org/github.com/%21azure/azure-sdk-for-go/sdk/storage/azblob/%40v/v1.0.0.mod')
+        self.assertEqual(ud.parm['downloadfilename'], 'github.com.Azure.azure-sdk-for-go.sdk.storage.azblob@v1.0.0.mod')
+        self.assertEqual(ud.parm['name'], 'github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0')
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
+
+    @skipIfNoNetwork()
+    def test_gomod_url_sha256sum_varflag(self):
+        urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0']
+        self.d.setVarFlag('SRC_URI', 'gopkg.in/ini.v1@v1.67.0.sha256sum', 'bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6')
+
+        fetcher = bb.fetch2.Fetch(urls, self.d)
+        ud = fetcher.ud[urls[0]]
+        self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip')
+        self.assertEqual(ud.parm['downloadfilename'], 'gopkg.in.ini.v1@v1.67.0.zip')
+        self.assertEqual(ud.parm['name'], 'gopkg.in/ini.v1@v1.67.0')
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
+        self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')),
+                         '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1')
+
+    @skipIfNoNetwork()
+    def test_gomod_url_no_go_mod_in_module(self):
+        urls = ['gomod://gopkg.in/ini.v1;version=v1.67.0;'
+                'sha256sum=bd845dfc762a87a56e5a32a07770dc83e86976db7705d7f89c5dbafdc60b06c6']
+
+        fetcher = bb.fetch2.Fetch(urls, self.d)
+        ud = fetcher.ud[urls[0]]
+        self.assertEqual(ud.url, 'https://proxy.golang.org/gopkg.in/ini.v1/%40v/v1.67.0.zip')
+        self.assertEqual(ud.parm['downloadfilename'], 'gopkg.in.ini.v1@v1.67.0.zip')
+        self.assertEqual(ud.parm['name'], 'gopkg.in/ini.v1@v1.67.0')
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
+        self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')),
+                         '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1')
+
+    @skipIfNoNetwork()
+    def test_gomod_url_host_only(self):
+        urls = ['gomod://go.opencensus.io;version=v0.24.0;'
+                'sha256sum=203a767d7f8e7c1ebe5588220ad168d1e15b14ae70a636de7ca9a4a88a7e0d0c']
+
+        fetcher = bb.fetch2.Fetch(urls, self.d)
+        ud = fetcher.ud[urls[0]]
+        self.assertEqual(ud.url, 'https://proxy.golang.org/go.opencensus.io/%40v/v0.24.0.zip')
+        self.assertEqual(ud.parm['downloadfilename'], 'go.opencensus.io@v0.24.0.zip')
+        self.assertEqual(ud.parm['name'], 'go.opencensus.io@v0.24.0')
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.zip')))
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')))
+        self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')),
+                         '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96')
+
+class GoModGitTest(FetcherTest):
+
+    @skipIfNoNetwork()
+    def test_gomodgit_url_repo(self):
+        urls = ['gomodgit://golang.org/x/net;version=v0.9.0;'
+                'repo=go.googlesource.com/net;'
+                'srcrev=694cff8668bac64e0864b552bffc280cd27f21b1']
+
+        fetcher = bb.fetch2.Fetch(urls, self.d)
+        ud = fetcher.ud[urls[0]]
+        self.assertEqual(ud.host, 'go.googlesource.com')
+        self.assertEqual(ud.path, '/net')
+        self.assertEqual(ud.name, 'golang.org/x/net@v0.9.0')
+        self.assertEqual(self.d.getVar('SRCREV_golang.org/x/net@v0.9.0'), '694cff8668bac64e0864b552bffc280cd27f21b1')
+
+        fetcher.download()
+        self.assertTrue(os.path.exists(ud.localpath))
+
+        fetcher.unpack(self.unpackdir)
+        vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+        self.assertTrue(os.path.exists(os.path.join(vcsdir, 'ed42bd05533fd84ae290a5d33ebd3695a0a2b06131beebd5450825bee8603aca')))
+        downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.zip')))
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod')))
+        self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'golang.org/x/net/@v/v0.9.0.mod')),
+                         'c5d6851ede50ec1c001afb763040194b68961bf06997e2605e8bf06dcd2aeb2e')
+
+    @skipIfNoNetwork()
+    def test_gomodgit_url_subdir(self):
+        urls = ['gomodgit://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob;version=v1.0.0;'
+                'repo=github.com/Azure/azure-sdk-for-go;subdir=sdk/storage/azblob;'
+                'srcrev=ec928e0ed34db682b3f783d3739d1c538142e0c3']
+
+        fetcher = bb.fetch2.Fetch(urls, self.d)
+        ud = fetcher.ud[urls[0]]
+        self.assertEqual(ud.host, 'github.com')
+        self.assertEqual(ud.path, '/Azure/azure-sdk-for-go')
+        self.assertEqual(ud.parm['subpath'], 'sdk/storage/azblob')
+        self.assertEqual(ud.name, 'github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0')
+        self.assertEqual(self.d.getVar('SRCREV_github.com/Azure/azure-sdk-for-go/sdk/storage/azblob@v1.0.0'), 'ec928e0ed34db682b3f783d3739d1c538142e0c3')
+
+        fetcher.download()
+        self.assertTrue(os.path.exists(ud.localpath))
+
+        fetcher.unpack(self.unpackdir)
+        vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+        self.assertTrue(os.path.exists(os.path.join(vcsdir, 'd31d6145676ed3066ce573a8198f326dea5be45a43b3d8f41ce7787fd71d66b3')))
+        downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.zip')))
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')))
+        self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'github.com/!azure/azure-sdk-for-go/sdk/storage/azblob/@v/v1.0.0.mod')),
+                         '7873b8544842329b4f385a3aa6cf82cc2bc8defb41a04fa5291c35fd5900e873')
+
+    @skipIfNoNetwork()
+    def test_gomodgit_url_srcrev_var(self):
+        urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0']
+        self.d.setVar('SRCREV_gopkg.in/ini.v1@v1.67.0', 'b2f570e5b5b844226bbefe6fb521d891f529a951')
+
+        fetcher = bb.fetch2.Fetch(urls, self.d)
+        ud = fetcher.ud[urls[0]]
+        self.assertEqual(ud.host, 'gopkg.in')
+        self.assertEqual(ud.path, '/ini.v1')
+        self.assertEqual(ud.name, 'gopkg.in/ini.v1@v1.67.0')
+        self.assertEqual(ud.parm['srcrev'], 'b2f570e5b5b844226bbefe6fb521d891f529a951')
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+        self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3')))
+        downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
+        self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')),
+                         '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1')
+
+    @skipIfNoNetwork()
+    def test_gomodgit_url_no_go_mod_in_module(self):
+        urls = ['gomodgit://gopkg.in/ini.v1;version=v1.67.0;'
+                'srcrev=b2f570e5b5b844226bbefe6fb521d891f529a951']
+
+        fetcher = bb.fetch2.Fetch(urls, self.d)
+        ud = fetcher.ud[urls[0]]
+        self.assertEqual(ud.host, 'gopkg.in')
+        self.assertEqual(ud.path, '/ini.v1')
+        self.assertEqual(ud.name, 'gopkg.in/ini.v1@v1.67.0')
+        self.assertEqual(self.d.getVar('SRCREV_gopkg.in/ini.v1@v1.67.0'), 'b2f570e5b5b844226bbefe6fb521d891f529a951')
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+        self.assertTrue(os.path.exists(os.path.join(vcsdir, 'b7879a4be9ba8598851b8278b14c4f71a8316be64913298d1639cce6bde59bc3')))
+        downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.zip')))
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')))
+        self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'gopkg.in/ini.v1/@v/v1.67.0.mod')),
+                         '13aedd85db8e555104108e0e613bb7e4d1242af7f27c15423dd9ab63b60b72a1')
+
+    @skipIfNoNetwork()
+    def test_gomodgit_url_host_only(self):
+        urls = ['gomodgit://go.opencensus.io;version=v0.24.0;'
+                'repo=github.com/census-instrumentation/opencensus-go;'
+                'srcrev=b1a01ee95db0e690d91d7193d037447816fae4c5']
+
+        fetcher = bb.fetch2.Fetch(urls, self.d)
+        ud = fetcher.ud[urls[0]]
+        self.assertEqual(ud.host, 'github.com')
+        self.assertEqual(ud.path, '/census-instrumentation/opencensus-go')
+        self.assertEqual(ud.name, 'go.opencensus.io@v0.24.0')
+        self.assertEqual(self.d.getVar('SRCREV_go.opencensus.io@v0.24.0'), 'b1a01ee95db0e690d91d7193d037447816fae4c5')
+
+        fetcher.download()
+        fetcher.unpack(self.unpackdir)
+        vcsdir = os.path.join(self.unpackdir, 'pkg/mod/cache/vcs')
+        self.assertTrue(os.path.exists(os.path.join(vcsdir, 'aae3ac7b2122ed3345654e6327855e9682f4a5350d63e93dbcfc51c4419df0e1')))
+        downloaddir = os.path.join(self.unpackdir, 'pkg/mod/cache/download')
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.zip')))
+        self.assertTrue(os.path.exists(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')))
+        self.assertEqual(bb.utils.sha256_file(os.path.join(downloaddir, 'go.opencensus.io/@v/v0.24.0.mod')),
+                         '0dc9ccc660ad21cebaffd548f2cc6efa27891c68b4fbc1f8a3893b00f1acec96')
diff --git a/bitbake/lib/bb/tests/parse.py b/bitbake/lib/bb/tests/parse.py
index 9e21e18425..e3cba67ad4 100644
--- a/bitbake/lib/bb/tests/parse.py
+++ b/bitbake/lib/bb/tests/parse.py
@@ -75,6 +75,59 @@ unset B[flag]
         self.assertEqual(d.getVarFlag("A","flag"), None)
         self.assertEqual(d.getVar("B"), "2")
 
+    defaulttest = """
+A = "set value"
+A ??= "default value"
+
+A[flag_set_vs_question] = "set flag"
+A[flag_set_vs_question] ?= "question flag"
+
+A[flag_set_vs_default] = "set flag"
+A[flag_set_vs_default] ??= "default flag"
+
+A[flag_question] ?= "question flag"
+
+A[flag_default] ??= "default flag"
+
+A[flag_question_vs_default] ?= "question flag"
+A[flag_question_vs_default] ??= "default flag"
+
+A[flag_default_vs_question] ??= "default flag"
+A[flag_default_vs_question] ?= "question flag"
+
+A[flag_set_question_default] = "set flag"
+A[flag_set_question_default] ?= "question flag"
+A[flag_set_question_default] ??= "default flag"
+
+A[flag_set_default_question] = "set flag"
+A[flag_set_default_question] ??= "default flag"
+A[flag_set_default_question] ?= "question flag"
+
+A[flag_set_twice] = "set flag first"
+A[flag_set_twice] = "set flag second"
+
+A[flag_question_twice] ?= "question flag first"
+A[flag_question_twice] ?= "question flag second"
+
+A[flag_default_twice] ??= "default flag first"
+A[flag_default_twice] ??= "default flag second"
+"""
+    def test_parse_defaulttest(self):
+        f = self.parsehelper(self.defaulttest)
+        d = bb.parse.handle(f.name, self.d)['']
+        self.assertEqual(d.getVar("A"), "set value")
+        self.assertEqual(d.getVarFlag("A","flag_set_vs_question"), "set flag")
+        self.assertEqual(d.getVarFlag("A","flag_set_vs_default"), "set flag")
+        self.assertEqual(d.getVarFlag("A","flag_question"), "question flag")
+        self.assertEqual(d.getVarFlag("A","flag_default"), "default flag")
+        self.assertEqual(d.getVarFlag("A","flag_question_vs_default"), "question flag")
+        self.assertEqual(d.getVarFlag("A","flag_default_vs_question"), "question flag")
+        self.assertEqual(d.getVarFlag("A","flag_set_question_default"), "set flag")
+        self.assertEqual(d.getVarFlag("A","flag_set_default_question"), "set flag")
+        self.assertEqual(d.getVarFlag("A","flag_set_twice"), "set flag second")
+        self.assertEqual(d.getVarFlag("A","flag_question_twice"), "question flag first")
+        self.assertEqual(d.getVarFlag("A","flag_default_twice"), "default flag second")
+
     exporttest = """
 A = "a"
 export B = "b"
@@ -98,8 +151,8 @@ exportD = "d"
 
 
     overridetest = """
-RRECOMMENDS_${PN} = "a"
-RRECOMMENDS_${PN}_libc = "b"
+RRECOMMENDS:${PN} = "a"
+RRECOMMENDS:${PN}:libc = "b"
 OVERRIDES = "libc:${PN}"
 PN = "gtk+"
 """
@@ -110,16 +163,16 @@ PN = "gtk+"
         self.assertEqual(d.getVar("RRECOMMENDS"), "b")
         bb.data.expandKeys(d)
         self.assertEqual(d.getVar("RRECOMMENDS"), "b")
-        d.setVar("RRECOMMENDS_gtk+", "c")
+        d.setVar("RRECOMMENDS:gtk+", "c")
         self.assertEqual(d.getVar("RRECOMMENDS"), "c")
 
     overridetest2 = """
 EXTRA_OECONF = ""
-EXTRA_OECONF_class-target = "b"
-EXTRA_OECONF_append = " c"
+EXTRA_OECONF:class-target = "b"
+EXTRA_OECONF:append = " c"
 """
 
-    def test_parse_overrides(self):
+    def test_parse_overrides2(self):
         f = self.parsehelper(self.overridetest2)
         d = bb.parse.handle(f.name, self.d)['']
         d.appendVar("EXTRA_OECONF", " d")
@@ -128,7 +181,7 @@ EXTRA_OECONF_append = " c"
 
     overridetest3 = """
 DESCRIPTION = "A"
-DESCRIPTION_${PN}-dev = "${DESCRIPTION} B"
+DESCRIPTION:${PN}-dev = "${DESCRIPTION} B"
 PN = "bc"
 """
 
@@ -136,15 +189,15 @@ PN = "bc"
         f = self.parsehelper(self.overridetest3)
         d = bb.parse.handle(f.name, self.d)['']
         bb.data.expandKeys(d)
-        self.assertEqual(d.getVar("DESCRIPTION_bc-dev"), "A B")
+        self.assertEqual(d.getVar("DESCRIPTION:bc-dev"), "A B")
         d.setVar("DESCRIPTION", "E")
-        d.setVar("DESCRIPTION_bc-dev", "C D")
+        d.setVar("DESCRIPTION:bc-dev", "C D")
         d.setVar("OVERRIDES", "bc-dev")
         self.assertEqual(d.getVar("DESCRIPTION"), "C D")
 
 
     classextend = """
-VAR_var_override1 = "B"
+VAR_var:override1 = "B"
 EXTRA = ":override1"
 OVERRIDES = "nothing${EXTRA}"
 
@@ -164,6 +217,7 @@ python () {
     # become unset/disappear.
     #
     def test_parse_classextend_contamination(self):
+        self.d.setVar("__bbclasstype", "recipe")
         cls = self.parsehelper(self.classextend_bbclass, suffix=".bbclass")
         #clsname = os.path.basename(cls.name).replace(".bbclass", "")
         self.classextend = self.classextend.replace("###CLASS###", cls.name)
@@ -176,7 +230,19 @@ python () {
 
     addtask_deltask = """
 addtask do_patch after do_foo after do_unpack before do_configure before do_compile
-addtask do_fetch do_patch
+addtask do_fetch2 do_patch2
+
+addtask do_myplaintask
+addtask do_myplaintask2
+deltask do_myplaintask2
+addtask do_mytask# comment
+addtask do_mytask2 # comment2
+addtask do_mytask3
+deltask do_mytask3# comment
+deltask do_mytask4 # comment2
+
+# Ensure a missing task prefix on after works
+addtask do_mytask5 after mytask
 
 MYVAR = "do_patch"
 EMPTYVAR = ""
@@ -184,13 +250,216 @@ deltask do_fetch ${MYVAR} ${EMPTYVAR}
 deltask ${EMPTYVAR}
 """
     def test_parse_addtask_deltask(self):
-        import sys
+
         f = self.parsehelper(self.addtask_deltask)
         d = bb.parse.handle(f.name, self.d)['']
 
-        stdout = sys.stdout.getvalue()
-        self.assertTrue("addtask contained multiple 'before' keywords" in stdout)
-        self.assertTrue("addtask contained multiple 'after' keywords" in stdout)
-        self.assertTrue('addtask ignored: " do_patch"' in stdout)
-        #self.assertTrue('dependent task do_foo for do_patch does not exist' in stdout)
+        self.assertSequenceEqual(['do_fetch2', 'do_patch2', 'do_myplaintask', 'do_mytask', 'do_mytask2', 'do_mytask5'], bb.build.listtasks(d))
+        self.assertEqual(['do_mytask'], d.getVarFlag("do_mytask5", "deps"))
+
+    broken_multiline_comment = """
+# First line of comment \\
+# Second line of comment \\
+
+"""
+    def test_parse_broken_multiline_comment(self):
+        f = self.parsehelper(self.broken_multiline_comment)
+        with self.assertRaises(bb.BBHandledException):
+            d = bb.parse.handle(f.name, self.d)['']
+
+
+    comment_in_var = """
+VAR = " \\
+    SOMEVAL \\
+#   some comment \\
+    SOMEOTHERVAL \\
+"
+"""
+    def test_parse_comment_in_var(self):
+        f = self.parsehelper(self.comment_in_var)
+        with self.assertRaises(bb.BBHandledException):
+            d = bb.parse.handle(f.name, self.d)['']
+
+
+    at_sign_in_var_flag = """
+A[flag@.service] = "nonet"
+B[flag@.target] = "ntb"
+C[f] = "flag"
 
+unset A[flag@.service]
+"""
+    def test_parse_at_sign_in_var_flag(self):
+        f = self.parsehelper(self.at_sign_in_var_flag)
+        d = bb.parse.handle(f.name, self.d)['']
+        self.assertEqual(d.getVar("A"), None)
+        self.assertEqual(d.getVar("B"), None)
+        self.assertEqual(d.getVarFlag("A","flag@.service"), None)
+        self.assertEqual(d.getVarFlag("B","flag@.target"), "ntb")
+        self.assertEqual(d.getVarFlag("C","f"), "flag")
+
+    def test_parse_invalid_at_sign_in_var_flag(self):
+        invalid_at_sign = self.at_sign_in_var_flag.replace("B[f", "B[@f")
+        f = self.parsehelper(invalid_at_sign)
+        with self.assertRaises(bb.parse.ParseError):
+            d = bb.parse.handle(f.name, self.d)['']
+
+    export_function_recipe = """
+inherit someclass
+"""
+
+    export_function_recipe2 = """
+inherit someclass
+
+do_compile () {
+    false
+}
+
+python do_compilepython () {
+    bb.note("Something else")
+}
+
+"""
+    export_function_class = """
+someclass_do_compile() {
+    true
+}
+
+python someclass_do_compilepython () {
+    bb.note("Something")
+}
+
+EXPORT_FUNCTIONS do_compile do_compilepython
+"""
+
+    export_function_class2 = """
+secondclass_do_compile() {
+    true
+}
+
+python secondclass_do_compilepython () {
+    bb.note("Something")
+}
+
+EXPORT_FUNCTIONS do_compile do_compilepython
+"""
+
+    def test_parse_export_functions(self):
+        def check_function_flags(d):
+            self.assertEqual(d.getVarFlag("do_compile", "func"), 1)
+            self.assertEqual(d.getVarFlag("do_compilepython", "func"), 1)
+            self.assertEqual(d.getVarFlag("do_compile", "python"), None)
+            self.assertEqual(d.getVarFlag("do_compilepython", "python"), "1")
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            self.d.setVar("__bbclasstype", "recipe")
+            recipename = tempdir + "/recipe.bb"
+            os.makedirs(tempdir + "/classes")
+            with open(tempdir + "/classes/someclass.bbclass", "w") as f:
+                f.write(self.export_function_class)
+                f.flush()
+            with open(tempdir + "/classes/secondclass.bbclass", "w") as f:
+                f.write(self.export_function_class2)
+                f.flush()
+
+            with open(recipename, "w") as f:
+                f.write(self.export_function_recipe)
+                f.flush()
+            os.chdir(tempdir)
+            d = bb.parse.handle(recipename, bb.data.createCopy(self.d))['']
+            self.assertIn("someclass_do_compile", d.getVar("do_compile"))
+            self.assertIn("someclass_do_compilepython", d.getVar("do_compilepython"))
+            check_function_flags(d)
+
+            recipename2 = tempdir + "/recipe2.bb"
+            with open(recipename2, "w") as f:
+                f.write(self.export_function_recipe2)
+                f.flush()
+
+            d = bb.parse.handle(recipename2, bb.data.createCopy(self.d))['']
+            self.assertNotIn("someclass_do_compile", d.getVar("do_compile"))
+            self.assertNotIn("someclass_do_compilepython", d.getVar("do_compilepython"))
+            self.assertIn("false", d.getVar("do_compile"))
+            self.assertIn("else", d.getVar("do_compilepython"))
+            check_function_flags(d)
+
+            with open(recipename, "a+") as f:
+                f.write("\ninherit secondclass\n")
+                f.flush()
+            with open(recipename2, "a+") as f:
+                f.write("\ninherit secondclass\n")
+                f.flush()
+
+            d = bb.parse.handle(recipename, bb.data.createCopy(self.d))['']
+            self.assertIn("secondclass_do_compile", d.getVar("do_compile"))
+            self.assertIn("secondclass_do_compilepython", d.getVar("do_compilepython"))
+            check_function_flags(d)
+
+            d = bb.parse.handle(recipename2, bb.data.createCopy(self.d))['']
+            self.assertNotIn("someclass_do_compile", d.getVar("do_compile"))
+            self.assertNotIn("someclass_do_compilepython", d.getVar("do_compilepython"))
+            self.assertIn("false", d.getVar("do_compile"))
+            self.assertIn("else", d.getVar("do_compilepython"))
+            check_function_flags(d)
+
+    export_function_unclosed_tab = """
+do_compile () {
+       bb.note("Something")
+\t}
+"""
+    export_function_unclosed_space = """
+do_compile () {
+       bb.note("Something")
+ }
+"""
+    export_function_residue = """
+do_compile () {
+       bb.note("Something")
+}
+
+include \\
+"""
+
+    def test_unclosed_functions(self):
+        def test_helper(content, expected_error):
+            with tempfile.TemporaryDirectory() as tempdir:
+                recipename = tempdir + "/recipe_unclosed.bb"
+                with open(recipename, "w") as f:
+                    f.write(content)
+                    f.flush()
+                os.chdir(tempdir)
+                with self.assertRaises(bb.parse.ParseError) as error:
+                    bb.parse.handle(recipename, bb.data.createCopy(self.d))
+                self.assertIn(expected_error, str(error.exception))
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            test_helper(self.export_function_unclosed_tab, "Unparsed lines from unclosed function")
+            test_helper(self.export_function_unclosed_space, "Unparsed lines from unclosed function")
+            test_helper(self.export_function_residue, "Unparsed lines")
+
+            recipename_closed = tempdir + "/recipe_closed.bb"
+            with open(recipename_closed, "w") as in_file:
+                lines = self.export_function_unclosed_tab.split("\n")
+                lines[3] = "}"
+                in_file.write("\n".join(lines))
+                in_file.flush()
+            bb.parse.handle(recipename_closed, bb.data.createCopy(self.d))
+
+    special_character_assignment = """
+A+="a"
+A+ = "b"
++ = "c"
+"""
+    ambigous_assignment = """
++= "d"
+"""
+    def test_parse_special_character_assignment(self):
+        f = self.parsehelper(self.special_character_assignment)
+        d = bb.parse.handle(f.name, self.d)['']
+        self.assertEqual(d.getVar("A"), " a")
+        self.assertEqual(d.getVar("A+"), "b")
+        self.assertEqual(d.getVar("+"), "c")
+
+        f = self.parsehelper(self.ambigous_assignment)
+        with self.assertRaises(bb.parse.ParseError) as error:
+            bb.parse.handle(f.name, self.d)
+        self.assertIn("Empty variable name in assignment", str(error.exception))
diff --git a/bitbake/lib/bb/tests/persist_data.py b/bitbake/lib/bb/tests/persist_data.py
deleted file mode 100644
index f641b5acbc..0000000000
--- a/bitbake/lib/bb/tests/persist_data.py
+++ /dev/null
@@ -1,129 +0,0 @@
-#
-# BitBake Test for lib/bb/persist_data/
-#
-# Copyright (C) 2018 Garmin Ltd.
-#
-# SPDX-License-Identifier: GPL-2.0-only
-#
-
-import unittest
-import bb.data
-import bb.persist_data
-import tempfile
-import threading
-
-class PersistDataTest(unittest.TestCase):
-    def _create_data(self):
-        return bb.persist_data.persist('TEST_PERSIST_DATA', self.d)
-
-    def setUp(self):
-        self.d = bb.data.init()
-        self.tempdir = tempfile.TemporaryDirectory()
-        self.d['PERSISTENT_DIR'] = self.tempdir.name
-        self.data = self._create_data()
-        self.items = {
-                'A1': '1',
-                'B1': '2',
-                'C2': '3'
-                }
-        self.stress_count = 10000
-        self.thread_count = 5
-
-        for k,v in self.items.items():
-            self.data[k] = v
-
-    def tearDown(self):
-        self.tempdir.cleanup()
-
-    def _iter_helper(self, seen, iterator):
-        with iter(iterator):
-            for v in iterator:
-                self.assertTrue(v in seen)
-                seen.remove(v)
-        self.assertEqual(len(seen), 0, '%s not seen' % seen)
-
-    def test_get(self):
-        for k, v in self.items.items():
-            self.assertEqual(self.data[k], v)
-
-        self.assertIsNone(self.data.get('D'))
-        with self.assertRaises(KeyError):
-            self.data['D']
-
-    def test_set(self):
-        for k, v in self.items.items():
-            self.data[k] += '-foo'
-
-        for k, v in self.items.items():
-            self.assertEqual(self.data[k], v + '-foo')
-
-    def test_delete(self):
-        self.data['D'] = '4'
-        self.assertEqual(self.data['D'], '4')
-        del self.data['D']
-        self.assertIsNone(self.data.get('D'))
-        with self.assertRaises(KeyError):
-            self.data['D']
-
-    def test_contains(self):
-        for k in self.items:
-            self.assertTrue(k in self.data)
-            self.assertTrue(self.data.has_key(k))
-        self.assertFalse('NotFound' in self.data)
-        self.assertFalse(self.data.has_key('NotFound'))
-
-    def test_len(self):
-        self.assertEqual(len(self.data), len(self.items))
-
-    def test_iter(self):
-        self._iter_helper(set(self.items.keys()), self.data)
-
-    def test_itervalues(self):
-        self._iter_helper(set(self.items.values()), self.data.itervalues())
-
-    def test_iteritems(self):
-        self._iter_helper(set(self.items.items()), self.data.iteritems())
-
-    def test_get_by_pattern(self):
-        self._iter_helper({'1', '2'}, self.data.get_by_pattern('_1'))
-
-    def _stress_read(self, data):
-        for i in range(self.stress_count):
-            for k in self.items:
-                data[k]
-
-    def _stress_write(self, data):
-        for i in range(self.stress_count):
-            for k, v in self.items.items():
-                data[k] = v + str(i)
-
-    def _validate_stress(self):
-        for k, v in self.items.items():
-            self.assertEqual(self.data[k], v + str(self.stress_count - 1))
-
-    def test_stress(self):
-        self._stress_read(self.data)
-        self._stress_write(self.data)
-        self._validate_stress()
-
-    def test_stress_threads(self):
-        def read_thread():
-            data = self._create_data()
-            self._stress_read(data)
-
-        def write_thread():
-            data = self._create_data()
-            self._stress_write(data)
-
-        threads = []
-        for i in range(self.thread_count):
-            threads.append(threading.Thread(target=read_thread))
-            threads.append(threading.Thread(target=write_thread))
-
-        for t in threads:
-            t.start()
-        self._stress_read(self.data)
-        for t in threads:
-            t.join()
-        self._validate_stress()
-
diff --git a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
index b57650d591..80b003b2b5 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
+++ b/bitbake/lib/bb/tests/runqueue-tests/classes/base.bbclass
@@ -9,7 +9,7 @@ def stamptask(d):
     with open(stampname, "a+") as f:
         f.write(d.getVar("BB_UNIHASH") + "\n")
 
-    if d.getVar("BB_CURRENT_MC") != "default":
+    if d.getVar("BB_CURRENT_MC") != "":
         thistask = d.expand("${BB_CURRENT_MC}:${PN}:${BB_CURRENTTASK}")
     if thistask in d.getVar("SLOWTASKS").split():
         bb.note("Slowing task %s" % thistask)
diff --git a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
index efebf001a9..05d7fd07dd 100644
--- a/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
+++ b/bitbake/lib/bb/tests/runqueue-tests/conf/bitbake.conf
@@ -12,6 +12,6 @@ STAMP = "${TMPDIR}/stamps/${PN}"
 T = "${TMPDIR}/workdir/${PN}/temp"
 BB_NUMBER_THREADS = "4"
 
-BB_HASHBASE_WHITELIST = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE"
+BB_BASEHASH_IGNORE_VARS = "BB_CURRENT_MC BB_HASHSERVE TMPDIR TOPDIR SLOWTASKS SSTATEVALID FILE BB_CURRENTTASK"
 
 include conf/multiconfig/${BB_CURRENT_MC}.conf
diff --git a/bitbake/lib/bb/tests/runqueue-tests/recipes/g1.bb b/bitbake/lib/bb/tests/runqueue-tests/recipes/g1.bb
new file mode 100644
index 0000000000..3c7dca0257
--- /dev/null
+++ b/bitbake/lib/bb/tests/runqueue-tests/recipes/g1.bb
@@ -0,0 +1,2 @@
+do_build[mcdepends] = "mc::mc-1:h1:do_invalid"
+
diff --git a/bitbake/lib/bb/tests/runqueue-tests/recipes/h1.bb b/bitbake/lib/bb/tests/runqueue-tests/recipes/h1.bb
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/bitbake/lib/bb/tests/runqueue-tests/recipes/h1.bb
diff --git a/bitbake/lib/bb/tests/runqueue.py b/bitbake/lib/bb/tests/runqueue.py
index 3d51779d6c..74f5ded2e6 100644
--- a/bitbake/lib/bb/tests/runqueue.py
+++ b/bitbake/lib/bb/tests/runqueue.py
@@ -26,20 +26,23 @@ class RunQueueTests(unittest.TestCase):
     a1_sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_package_write_rpm a1:do_populate_lic a1:do_populate_sysroot"
     b1_sstatevalid = "b1:do_package b1:do_package_qa b1:do_packagedata b1:do_package_write_ipk b1:do_package_write_rpm b1:do_populate_lic b1:do_populate_sysroot"
 
-    def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None, cleanup=False):
+    def run_bitbakecmd(self, cmd, builddir, sstatevalid="", slowtasks="", extraenv=None, cleanup=False, allowfailure=False):
         env = os.environ.copy()
         env["BBPATH"] = os.path.realpath(os.path.join(os.path.dirname(__file__), "runqueue-tests"))
-        env["BB_ENV_EXTRAWHITE"] = "SSTATEVALID SLOWTASKS"
+        env["BB_ENV_PASSTHROUGH_ADDITIONS"] = "SSTATEVALID SLOWTASKS TOPDIR"
         env["SSTATEVALID"] = sstatevalid
         env["SLOWTASKS"] = slowtasks
+        env["TOPDIR"] = builddir
         if extraenv:
             for k in extraenv:
                 env[k] = extraenv[k]
-                env["BB_ENV_EXTRAWHITE"] = env["BB_ENV_EXTRAWHITE"] + " " + k
+                env["BB_ENV_PASSTHROUGH_ADDITIONS"] = env["BB_ENV_PASSTHROUGH_ADDITIONS"] + " " + k
         try:
             output = subprocess.check_output(cmd, env=env, stderr=subprocess.STDOUT,universal_newlines=True, cwd=builddir)
             print(output)
         except subprocess.CalledProcessError as e:
+            if allowfailure:
+                return e.output
             self.fail("Command %s failed with %s" % (cmd, e.output))
         tasks = []
         tasklog = builddir + "/task.log"
@@ -58,6 +61,8 @@ class RunQueueTests(unittest.TestCase):
             expected = ['a1:' + x for x in self.alltasks]
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_single_setscenevalid(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
@@ -68,6 +73,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot', 'a1:build']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_intermediate_setscenevalid(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
@@ -77,6 +84,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot_setscene', 'a1:build']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_intermediate_notcovered(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
@@ -86,6 +95,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_all_setscenevalid(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
@@ -95,6 +106,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_no_settasks(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1", "-c", "patch"]
@@ -103,6 +116,8 @@ class RunQueueTests(unittest.TestCase):
             expected = ['a1:fetch', 'a1:unpack', 'a1:patch']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_mix_covered_notcovered(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1:do_patch", "a1:do_populate_sysroot"]
@@ -111,6 +126,7 @@ class RunQueueTests(unittest.TestCase):
             expected = ['a1:fetch', 'a1:unpack', 'a1:patch', 'a1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
 
     # Test targets with intermediate setscene tasks alongside a target with no intermediate setscene tasks
     def test_mixed_direct_tasks_setscene_tasks(self):
@@ -122,6 +138,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:package_qa_setscene', 'a1:build', 'a1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     # This test slows down the execution of do_package_setscene until after other real tasks have
     # started running which tests for a bug where tasks were being lost from the buildable list of real
     # tasks if they weren't in tasks_covered or tasks_notcovered
@@ -136,12 +154,14 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot', 'a1:build']
             self.assertEqual(set(tasks), set(expected))
 
-    def test_setscenewhitelist(self):
+            self.shutdown(tempdir)
+
+    def test_setscene_ignore_tasks(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "a1"]
             extraenv = {
                 "BB_SETSCENE_ENFORCE" : "1",
-                "BB_SETSCENE_ENFORCE_WHITELIST" : "a1:do_package_write_rpm a1:do_build"
+                "BB_SETSCENE_ENFORCE_IGNORE_TASKS" : "a1:do_package_write_rpm a1:do_build"
             }
             sstatevalid = "a1:do_package a1:do_package_qa a1:do_packagedata a1:do_package_write_ipk a1:do_populate_lic a1:do_populate_sysroot"
             tasks = self.run_bitbakecmd(cmd, tempdir, sstatevalid, extraenv=extraenv)
@@ -149,6 +169,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot_setscene', 'a1:package_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     # Tests for problems with dependencies between setscene tasks
     def test_no_setscenevalid_harddeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
@@ -162,6 +184,8 @@ class RunQueueTests(unittest.TestCase):
                         'd1:populate_sysroot', 'd1:build']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_no_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -172,6 +196,8 @@ class RunQueueTests(unittest.TestCase):
             expected.remove('a1:package_qa')
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_single_a1_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -182,6 +208,8 @@ class RunQueueTests(unittest.TestCase):
                         'a1:populate_sysroot'] + ['b1:' + x for x in self.alltasks]
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_single_b1_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -193,6 +221,8 @@ class RunQueueTests(unittest.TestCase):
             expected.remove('b1:package')
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_intermediate_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -203,6 +233,8 @@ class RunQueueTests(unittest.TestCase):
             expected.remove('b1:package')
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_all_setscenevalid_withdeps(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             cmd = ["bitbake", "b1"]
@@ -213,6 +245,8 @@ class RunQueueTests(unittest.TestCase):
                         'b1:packagedata_setscene', 'b1:package_qa_setscene', 'b1:populate_sysroot_setscene']
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_multiconfig_setscene_optimise(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
@@ -232,6 +266,8 @@ class RunQueueTests(unittest.TestCase):
                 expected.remove(x)
             self.assertEqual(set(tasks), set(expected))
 
+            self.shutdown(tempdir)
+
     def test_multiconfig_bbmask(self):
         # This test validates that multiconfigs can independently mask off
         # recipes they do not want with BBMASK. It works by having recipes
@@ -248,11 +284,13 @@ class RunQueueTests(unittest.TestCase):
             cmd = ["bitbake", "mc:mc-1:fails-mc2", "mc:mc_2:fails-mc1"]
             self.run_bitbakecmd(cmd, tempdir, "", extraenv=extraenv)
 
+            self.shutdown(tempdir)
+
     def test_multiconfig_mcdepends(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
                 "BBMULTICONFIG" : "mc-1 mc_2",
-                "BB_SIGNATURE_HANDLER" : "TestMulticonfigDepends",
+                "BB_SIGNATURE_HANDLER" : "basichash",
                 "EXTRA_BBFILES": "${COREBASE}/recipes/fails-mc/*.bb",
             }
             tasks = self.run_bitbakecmd(["bitbake", "mc:mc-1:f1"], tempdir, "", extraenv=extraenv, cleanup=True)
@@ -278,7 +316,15 @@ class RunQueueTests(unittest.TestCase):
                        ["mc_2:a1:%s" % t for t in rerun_tasks]
             self.assertEqual(set(tasks), set(expected))
 
-    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
+            # Check that a multiconfig that doesn't exist rasies a correct error message
+            error_output = self.run_bitbakecmd(["bitbake", "g1"], tempdir, "", extraenv=extraenv, cleanup=True, allowfailure=True)
+            self.assertIn("non-existent task", error_output)
+            # If the word 'Traceback' or 'KeyError' is in the output we've regressed
+            self.assertNotIn("Traceback", error_output)
+            self.assertNotIn("KeyError", error_output)
+
+            self.shutdown(tempdir)
+
     def test_hashserv_single(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
@@ -304,7 +350,6 @@ class RunQueueTests(unittest.TestCase):
 
             self.shutdown(tempdir)
 
-    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
     def test_hashserv_double(self):
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
             extraenv = {
@@ -329,7 +374,6 @@ class RunQueueTests(unittest.TestCase):
 
             self.shutdown(tempdir)
 
-    @unittest.skipIf(sys.version_info < (3, 5, 0), 'Python 3.5 or later required')
     def test_hashserv_multiple_setscene(self):
         # Runs e1:do_package_setscene twice
         with tempfile.TemporaryDirectory(prefix="runqueuetest") as tempdir:
@@ -361,7 +405,6 @@ class RunQueueTests(unittest.TestCase):
 
     def shutdown(self, tempdir):
         # Wait for the hashserve socket to disappear else we'll see races with the tempdir cleanup
-        while os.path.exists(tempdir + "/hashserve.sock"):
+        while (os.path.exists(tempdir + "/hashserve.sock") or os.path.exists(tempdir + "cache/hashserv.db-wal") or os.path.exists(tempdir + "/bitbake.lock")):
             time.sleep(0.5)
 
-
diff --git a/bitbake/lib/bb/tests/siggen.py b/bitbake/lib/bb/tests/siggen.py
index c21ab4e4fb..0dc67e6cc2 100644
--- a/bitbake/lib/bb/tests/siggen.py
+++ b/bitbake/lib/bb/tests/siggen.py
@@ -17,75 +17,12 @@ import bb.siggen
 
 class SiggenTest(unittest.TestCase):
 
-    def test_clean_basepath_simple_target_basepath(self):
-        basepath = '/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask'
+    def test_build_pnid(self):
+        tests = {
+            ('', 'helloworld', 'do_sometask') : 'helloworld:do_sometask',
+            ('XX', 'helloworld', 'do_sometask') : 'mc:XX:helloworld:do_sometask',
+        }
 
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
+        for t in tests:
+            self.assertEqual(bb.siggen.build_pnid(*t), tests[t])
 
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_basic_virtual_basepath(self):
-        basepath = 'virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_mc_basepath(self):
-        basepath = 'mc:somemachine:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:mc:somemachine'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_virtual_long_prefix_basepath(self):
-        basepath = 'virtual:something:A:B:C:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:A:B:C'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_mc_virtual_basepath(self):
-        basepath = 'mc:somemachine:virtual:something:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:mc:somemachine'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-    def test_clean_basepath_mc_virtual_long_prefix_basepath(self):
-        basepath = 'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask'
-        expected_cleaned = 'helloworld/helloworld_1.2.3.bb:do_sometask:virtual:something:C:B:A:mc:X'
-
-        actual_cleaned = bb.siggen.clean_basepath(basepath)
-
-        self.assertEqual(actual_cleaned, expected_cleaned)
-
-
-    # def test_clean_basepath_performance(self):
-    #     input_basepaths = [
-    #         'mc:X:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         'mc:X:virtual:something:C:B:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         'virtual:something:C:B:A:/different/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         'virtual:something:A:/full/path/to/poky/meta/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         '/this/is/most/common/input/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         '/and/should/be/tested/with/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #         '/more/weight/recipes-whatever/helloworld/helloworld_1.2.3.bb:do_sometask',
-    #     ]
-
-    #     time_start = time.time()
-
-    #     i = 2000000
-    #     while i >= 0:
-    #         for basepath in input_basepaths:
-    #             bb.siggen.clean_basepath(basepath)
-    #         i -= 1
-
-    #     elapsed = time.time() - time_start
-    #     print('{} ({}s)'.format(self.id(), round(elapsed, 3)))
-
-    #     self.assertTrue(False)
diff --git a/bitbake/lib/bb/tests/utils.py b/bitbake/lib/bb/tests/utils.py
index a7ff33db52..52b7bf85bf 100644
--- a/bitbake/lib/bb/tests/utils.py
+++ b/bitbake/lib/bb/tests/utils.py
@@ -130,6 +130,14 @@ class Checksum(unittest.TestCase):
             checksum = bb.utils.sha256_file(f.name)
             self.assertEqual(checksum, "fcfbae8bf6b721dbb9d2dc6a9334a58f2031a9a9b302999243f99da4d7f12d0f")
 
+    def test_goh1(self):
+        import hashlib
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(self.filler)
+            f.flush()
+            checksum = bb.utils.goh1_file(f.name)
+            self.assertEqual(checksum, "81191f04d4abf413e5badd234814e4202d9efa73e6f9437e9ddd6b8165b569ef")
+
 class EditMetadataFile(unittest.TestCase):
     _origfile = """
 # A comment
@@ -418,7 +426,7 @@ MULTILINE = "  stuff \\
                                ['MULTILINE'],
                                handle_var)
 
-        testvalue = re.sub('\s+', ' ', value_in_callback.strip())
+        testvalue = re.sub(r'\s+', ' ', value_in_callback.strip())
         self.assertEqual(expected_value, testvalue)
 
 class EditBbLayersConf(unittest.TestCase):
@@ -666,3 +674,32 @@ class GetReferencedVars(unittest.TestCase):
 
         layers = [{"SRC_URI"}, {"QT_GIT", "QT_MODULE", "QT_MODULE_BRANCH_PARAM", "QT_GIT_PROTOCOL"}, {"QT_GIT_PROJECT", "QT_MODULE_BRANCH", "BPN"}, {"PN", "SPECIAL_PKGSUFFIX"}]
         self.check_referenced("${SRC_URI}", layers)
+
+
+class EnvironmentTests(unittest.TestCase):
+    def test_environment(self):
+        os.environ["A"] = "this is A"
+        self.assertIn("A", os.environ)
+        self.assertEqual(os.environ["A"], "this is A")
+        self.assertNotIn("B", os.environ)
+
+        with bb.utils.environment(B="this is B"):
+            self.assertIn("A", os.environ)
+            self.assertEqual(os.environ["A"], "this is A")
+            self.assertIn("B", os.environ)
+            self.assertEqual(os.environ["B"], "this is B")
+
+        self.assertIn("A", os.environ)
+        self.assertEqual(os.environ["A"], "this is A")
+        self.assertNotIn("B", os.environ)
+
+class FilemodeTests(unittest.TestCase):
+    def test_filemode_convert(self):
+        self.assertEqual(0o775, bb.utils.to_filemode("0o775"))
+        self.assertEqual(0o775, bb.utils.to_filemode(0o775))
+        self.assertEqual(0o775, bb.utils.to_filemode("775"))
+        with self.assertRaises(ValueError):
+            bb.utils.to_filemode("xyz")
+        with self.assertRaises(ValueError):
+            bb.utils.to_filemode("999")
+
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py
index 763c329810..e7fbcbca0a 100644
--- a/bitbake/lib/bb/tinfoil.py
+++ b/bitbake/lib/bb/tinfoil.py
@@ -10,10 +10,12 @@
 import logging
 import os
 import sys
+import time
 import atexit
 import re
 from collections import OrderedDict, defaultdict
-from functools import partial
+from functools import partial, wraps
+from contextlib import contextmanager
 
 import bb.cache
 import bb.cooker
@@ -25,6 +27,135 @@ import bb.remotedata
 from bb.main import setup_bitbake, BitBakeConfigParameters
 import bb.fetch2
 
+def wait_for(f):
+    """
+    Wrap a function that makes an asynchronous tinfoil call using
+    self.run_command() and wait for events to say that the call has been
+    successful, or an error has occurred.
+    """
+    @wraps(f)
+    def wrapper(self, *args, handle_events=True, extra_events=None, event_callback=None, **kwargs):
+        if handle_events:
+            # A reasonable set of default events matching up with those we handle below
+            eventmask = [
+                        'bb.event.BuildStarted',
+                        'bb.event.BuildCompleted',
+                        'logging.LogRecord',
+                        'bb.event.NoProvider',
+                        'bb.command.CommandCompleted',
+                        'bb.command.CommandFailed',
+                        'bb.build.TaskStarted',
+                        'bb.build.TaskFailed',
+                        'bb.build.TaskSucceeded',
+                        'bb.build.TaskFailedSilent',
+                        'bb.build.TaskProgress',
+                        'bb.runqueue.runQueueTaskStarted',
+                        'bb.runqueue.sceneQueueTaskStarted',
+                        'bb.event.ProcessStarted',
+                        'bb.event.ProcessProgress',
+                        'bb.event.ProcessFinished',
+                        ]
+            if extra_events:
+                eventmask.extend(extra_events)
+            ret = self.set_event_mask(eventmask)
+
+        includelogs = self.config_data.getVar('BBINCLUDELOGS')
+        loglines = self.config_data.getVar('BBINCLUDELOGS_LINES')
+
+        # Call actual function
+        ret = f(self, *args, **kwargs)
+
+        if handle_events:
+            lastevent = time.time()
+            result = False
+            # Borrowed from knotty, instead somewhat hackily we use the helper
+            # as the object to store "shutdown" on
+            helper = bb.ui.uihelper.BBUIHelper()
+            helper.shutdown = 0
+            parseprogress = None
+            termfilter = bb.ui.knotty.TerminalFilter(helper, helper, self.logger.handlers, quiet=self.quiet)
+            try:
+                while True:
+                    try:
+                        event = self.wait_event(0.25)
+                        if event:
+                            lastevent = time.time()
+                            if event_callback and event_callback(event):
+                                continue
+                            if helper.eventHandler(event):
+                                if isinstance(event, bb.build.TaskFailedSilent):
+                                    self.logger.warning("Logfile for failed setscene task is %s" % event.logfile)
+                                elif isinstance(event, bb.build.TaskFailed):
+                                    bb.ui.knotty.print_event_log(event, includelogs, loglines, termfilter)
+                                continue
+                            if isinstance(event, bb.event.ProcessStarted):
+                                if self.quiet > 1:
+                                    continue
+                                parseprogress = bb.ui.knotty.new_progress(event.processname, event.total)
+                                parseprogress.start(False)
+                                continue
+                            if isinstance(event, bb.event.ProcessProgress):
+                                if self.quiet > 1:
+                                    continue
+                                if parseprogress:
+                                    parseprogress.update(event.progress)
+                                else:
+                                    bb.warn("Got ProcessProgress event for something that never started?")
+                                continue
+                            if isinstance(event, bb.event.ProcessFinished):
+                                if self.quiet > 1:
+                                    continue
+                                if parseprogress:
+                                    parseprogress.finish()
+                                parseprogress = None
+                                continue
+                            if isinstance(event, bb.command.CommandCompleted):
+                                result = True
+                                break
+                            if isinstance(event, (bb.command.CommandFailed, bb.command.CommandExit)):
+                                self.logger.error(str(event))
+                                result = False
+                                break
+                            if isinstance(event, logging.LogRecord):
+                                if event.taskpid == 0 or event.levelno > logging.INFO:
+                                    self.logger.handle(event)
+                                continue
+                            if isinstance(event, bb.event.NoProvider):
+                                self.logger.error(str(event))
+                                result = False
+                                break
+                        elif helper.shutdown > 1:
+                            break
+                        termfilter.updateFooter()
+                        if time.time() > (lastevent + (3*60)):
+                            if not self.run_command('ping', handle_events=False):
+                                print("\nUnable to ping server and no events, closing down...\n")
+                                return False
+                    except KeyboardInterrupt:
+                        termfilter.clearFooter()
+                        if helper.shutdown == 1:
+                            print("\nSecond Keyboard Interrupt, stopping...\n")
+                            ret = self.run_command("stateForceShutdown")
+                            if ret and ret[2]:
+                                self.logger.error("Unable to cleanly stop: %s" % ret[2])
+                        elif helper.shutdown == 0:
+                            print("\nKeyboard Interrupt, closing down...\n")
+                            interrupted = True
+                            ret = self.run_command("stateShutdown")
+                            if ret and ret[2]:
+                                self.logger.error("Unable to cleanly shutdown: %s" % ret[2])
+                        helper.shutdown = helper.shutdown + 1
+                termfilter.clearFooter()
+            finally:
+                termfilter.finish()
+            if helper.failed_tasks:
+                result = False
+            return result
+        else:
+            return ret
+
+    return wrapper
+
 
 # We need this in order to shut down the connection to the bitbake server,
 # otherwise the process will never properly exit
@@ -52,6 +183,10 @@ class TinfoilDataStoreConnectorVarHistory:
     def remoteCommand(self, cmd, *args, **kwargs):
         return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs)
 
+    def emit(self, var, oval, val, o, d):
+        ret = self.tinfoil.run_command('dataStoreConnectorVarHistCmdEmit', self.dsindex, var, oval, val, d.dsindex)
+        o.write(ret)
+
     def __getattr__(self, name):
         if not hasattr(bb.data_smart.VariableHistory, name):
             raise AttributeError("VariableHistory has no such method %s" % name)
@@ -183,11 +318,19 @@ class TinfoilCookerAdapter:
             self._cache[name] = attrvalue
             return attrvalue
 
+    class TinfoilSkiplistByMcAdapter:
+        def __init__(self, tinfoil):
+            self.tinfoil = tinfoil
+
+        def __getitem__(self, mc):
+            return self.tinfoil.get_skipped_recipes(mc)
+
     def __init__(self, tinfoil):
         self.tinfoil = tinfoil
         self.multiconfigs = [''] + (tinfoil.config_data.getVar('BBMULTICONFIG') or '').split()
         self.collections = {}
         self.recipecaches = {}
+        self.skiplist_by_mc = self.TinfoilSkiplistByMcAdapter(tinfoil)
         for mc in self.multiconfigs:
             self.collections[mc] = self.TinfoilCookerCollectionAdapter(tinfoil, mc)
             self.recipecaches[mc] = self.TinfoilRecipeCacheAdapter(tinfoil, mc)
@@ -196,8 +339,6 @@ class TinfoilCookerAdapter:
         # Grab these only when they are requested since they aren't always used
         if name in self._cache:
             return self._cache[name]
-        elif name == 'skiplist':
-            attrvalue = self.tinfoil.get_skipped_recipes()
         elif name == 'bbfile_config_priorities':
             ret = self.tinfoil.run_command('getLayerPriorities')
             bbfile_config_priorities = []
@@ -320,11 +461,11 @@ class Tinfoil:
         self.recipes_parsed = False
         self.quiet = 0
         self.oldhandlers = self.logger.handlers[:]
+        self.localhandlers = []
         if setup_logging:
             # This is the *client-side* logger, nothing to do with
             # logging messages from the server
             bb.msg.logger_create('BitBake', output)
-            self.localhandlers = []
             for handler in self.logger.handlers:
                 if handler not in self.oldhandlers:
                     self.localhandlers.append(handler)
@@ -440,11 +581,17 @@ class Tinfoil:
         to initialise Tinfoil and use it with config_only=True first and
         then conditionally call this function to parse recipes later.
         """
-        config_params = TinfoilConfigParameters(config_only=False)
+        config_params = TinfoilConfigParameters(config_only=False, quiet=self.quiet)
         self.run_actions(config_params)
         self.recipes_parsed = True
 
-    def run_command(self, command, *params):
+    def modified_files(self):
+        """
+        Notify the server it needs to revalidate it's caches since the client has modified files
+        """
+        self.run_command("revalidateCaches")
+
+    def run_command(self, command, *params, handle_events=True):
         """
         Run a command on the server (as implemented in bb.command).
         Note that there are two types of command - synchronous and
@@ -464,7 +611,7 @@ class Tinfoil:
         try:
             result = self.server_connection.connection.runCommand(commandline)
         finally:
-            while True:
+            while handle_events:
                 event = self.wait_event()
                 if not event:
                     break
@@ -489,7 +636,7 @@ class Tinfoil:
         Wait for an event from the server for the specified time.
         A timeout of 0 means don't wait if there are no events in the queue.
         Returns the next event in the queue or None if the timeout was
-        reached. Note that in order to recieve any events you will
+        reached. Note that in order to receive any events you will
         first need to set the internal event mask using set_event_mask()
         (otherwise whatever event mask the UI set up will be in effect).
         """
@@ -503,12 +650,12 @@ class Tinfoil:
         """
         return defaultdict(list, self.run_command('getOverlayedRecipes', mc))
 
-    def get_skipped_recipes(self):
+    def get_skipped_recipes(self, mc=''):
         """
         Find recipes which were skipped (i.e. SkipRecipe was raised
         during parsing).
         """
-        return OrderedDict(self.run_command('getSkippedRecipes'))
+        return OrderedDict(self.run_command('getSkippedRecipes', mc))
 
     def get_all_providers(self, mc=''):
         return defaultdict(list, self.run_command('allProviders', mc))
@@ -522,6 +669,7 @@ class Tinfoil:
     def get_runtime_providers(self, rdep):
         return self.run_command('getRuntimeProviders', rdep)
 
+    # TODO: teach this method about mc
     def get_recipe_file(self, pn):
         """
         Get the file name for the specified recipe/target. Raises
@@ -530,6 +678,7 @@ class Tinfoil:
         """
         best = self.find_best_provider(pn)
         if not best or (len(best) > 3 and not best[3]):
+            # TODO: pass down mc
             skiplist = self.get_skipped_recipes()
             taskdata = bb.taskdata.TaskData(None, skiplist=skiplist)
             skipreasons = taskdata.get_reasons(pn)
@@ -622,6 +771,29 @@ class Tinfoil:
         fn = self.get_recipe_file(pn)
         return self.parse_recipe_file(fn)
 
+    @contextmanager
+    def _data_tracked_if_enabled(self):
+        """
+        A context manager to enable data tracking for a code segment if data
+        tracking was enabled for this tinfoil instance.
+        """
+        if self.tracking:
+            # Enable history tracking just for the operation
+            self.run_command('enableDataTracking')
+
+        # Here goes the operation with the optional data tracking
+        yield
+
+        if self.tracking:
+            self.run_command('disableDataTracking')
+
+    def finalizeData(self):
+        """
+        Run anonymous functions and expand keys
+        """
+        with self._data_tracked_if_enabled():
+            return self._reconvert_type(self.run_command('finalizeData'), 'DataStoreConnectionHandle')
+
     def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
         """
         Parse the specified recipe file (with or without bbappends)
@@ -634,10 +806,7 @@ class Tinfoil:
             appendlist: optional list of bbappend files to apply, if you
                         want to filter them
         """
-        if self.tracking:
-            # Enable history tracking just for the parse operation
-            self.run_command('enableDataTracking')
-        try:
+        with self._data_tracked_if_enabled():
             if appends and appendlist == []:
                 appends = False
             if config_data:
@@ -649,9 +818,6 @@ class Tinfoil:
                 return self._reconvert_type(dscon, 'DataStoreConnectionHandle')
             else:
                 return None
-        finally:
-            if self.tracking:
-                self.run_command('disableDataTracking')
 
     def build_file(self, buildfile, task, internal=True):
         """
@@ -663,6 +829,10 @@ class Tinfoil:
         """
         return self.run_command('buildFile', buildfile, task, internal)
 
+    @wait_for
+    def build_file_sync(self, *args):
+        self.build_file(*args)
+
     def build_targets(self, targets, task=None, handle_events=True, extra_events=None, event_callback=None):
         """
         Builds the specified targets. This is equivalent to a normal invocation
@@ -725,6 +895,7 @@ class Tinfoil:
 
         ret = self.run_command('buildTargets', targets, task)
         if handle_events:
+            lastevent = time.time()
             result = False
             # Borrowed from knotty, instead somewhat hackily we use the helper
             # as the object to store "shutdown" on
@@ -737,6 +908,7 @@ class Tinfoil:
                     try:
                         event = self.wait_event(0.25)
                         if event:
+                            lastevent = time.time()
                             if event_callback and event_callback(event):
                                 continue
                             if helper.eventHandler(event):
@@ -757,7 +929,7 @@ class Tinfoil:
                                 if parseprogress:
                                     parseprogress.update(event.progress)
                                 else:
-                                    bb.warn("Got ProcessProgress event for someting that never started?")
+                                    bb.warn("Got ProcessProgress event for something that never started?")
                                 continue
                             if isinstance(event, bb.event.ProcessFinished):
                                 if self.quiet > 1:
@@ -769,7 +941,7 @@ class Tinfoil:
                             if isinstance(event, bb.command.CommandCompleted):
                                 result = True
                                 break
-                            if isinstance(event, bb.command.CommandFailed):
+                            if isinstance(event, (bb.command.CommandFailed, bb.command.CommandExit)):
                                 self.logger.error(str(event))
                                 result = False
                                 break
@@ -781,10 +953,13 @@ class Tinfoil:
                                 self.logger.error(str(event))
                                 result = False
                                 break
-
                         elif helper.shutdown > 1:
                             break
                         termfilter.updateFooter()
+                        if time.time() > (lastevent + (3*60)):
+                            if not self.run_command('ping', handle_events=False):
+                                print("\nUnable to ping server and no events, closing down...\n")
+                                return False
                     except KeyboardInterrupt:
                         termfilter.clearFooter()
                         if helper.shutdown == 1:
diff --git a/bitbake/lib/bb/ui/buildinfohelper.py b/bitbake/lib/bb/ui/buildinfohelper.py
index 43aa592842..4ee45d67a2 100644
--- a/bitbake/lib/bb/ui/buildinfohelper.py
+++ b/bitbake/lib/bb/ui/buildinfohelper.py
@@ -45,7 +45,7 @@ from pprint import pformat
 import logging
 from datetime import datetime, timedelta
 
-from django.db import transaction, connection
+from django.db import transaction
 
 
 # pylint: disable=invalid-name
@@ -227,6 +227,12 @@ class ORMWrapper(object):
         build.completed_on = timezone.now()
         build.outcome = outcome
         build.save()
+
+        # We force a sync point here to force the outcome status commit,
+        # which resolves a race condition with the build completion takedown
+        transaction.set_autocommit(True)
+        transaction.set_autocommit(False)
+
         signal_runbuilds()
 
     def update_target_set_license_manifest(self, target, license_manifest_path):
@@ -483,14 +489,14 @@ class ORMWrapper(object):
 
             # we already created the root directory, so ignore any
             # entry for it
-            if len(path) == 0:
+            if not path:
                 continue
 
             parent_path = "/".join(path.split("/")[:len(path.split("/")) - 1])
-            if len(parent_path) == 0:
+            if not parent_path:
                 parent_path = "/"
             parent_obj = self._cached_get(Target_File, target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY)
-            tf_obj = Target_File.objects.create(
+            Target_File.objects.create(
                         target = target_obj,
                         path = path,
                         size = size,
@@ -553,9 +559,12 @@ class ORMWrapper(object):
                 # we might have an invalid link; no way to detect this. just set it to None
                 filetarget_obj = None
 
-            parent_obj = Target_File.objects.get(target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY)
+            try:
+                parent_obj = Target_File.objects.get(target = target_obj, path = parent_path, inodetype = Target_File.ITYPE_DIRECTORY)
+            except Target_File.DoesNotExist:
+                parent_obj = None
 
-            tf_obj = Target_File.objects.create(
+            Target_File.objects.create(
                         target = target_obj,
                         path = path,
                         size = size,
@@ -571,7 +580,7 @@ class ORMWrapper(object):
         assert isinstance(build_obj, Build)
         assert isinstance(target_obj, Target)
 
-        errormsg = ""
+        errormsg = []
         for p in packagedict:
             # Search name swtiches round the installed name vs package name
             # by default installed name == package name
@@ -633,10 +642,10 @@ class ORMWrapper(object):
                         packagefile_objects.append(Package_File( package = packagedict[p]['object'],
                             path = targetpath,
                             size = targetfilesize))
-                    if len(packagefile_objects):
+                    if packagefile_objects:
                         Package_File.objects.bulk_create(packagefile_objects)
                 except KeyError as e:
-                    errormsg += "  stpi: Key error, package %s key %s \n" % ( p, e )
+                    errormsg.append("  stpi: Key error, package %s key %s \n" % (p, e))
 
             # save disk installed size
             packagedict[p]['object'].installed_size = packagedict[p]['size']
@@ -673,13 +682,13 @@ class ORMWrapper(object):
                     logger.warning("Could not add dependency to the package %s "
                                    "because %s is an unknown package", p, px)
 
-        if len(packagedeps_objs) > 0:
+        if packagedeps_objs:
             Package_Dependency.objects.bulk_create(packagedeps_objs)
         else:
             logger.info("No package dependencies created")
 
-        if len(errormsg) > 0:
-            logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", errormsg)
+        if errormsg:
+            logger.warning("buildinfohelper: target_package_info could not identify recipes: \n%s", "".join(errormsg))
 
     def save_target_image_file_information(self, target_obj, file_name, file_size):
         Target_Image_File.objects.create(target=target_obj,
@@ -767,7 +776,7 @@ class ORMWrapper(object):
             packagefile_objects.append(Package_File( package = bp_object,
                                         path = path,
                                         size = package_info['FILES_INFO'][path] ))
-        if len(packagefile_objects):
+        if packagefile_objects:
             Package_File.objects.bulk_create(packagefile_objects)
 
         def _po_byname(p):
@@ -809,7 +818,7 @@ class ORMWrapper(object):
                 packagedeps_objs.append(Package_Dependency(  package = bp_object,
                     depends_on = _po_byname(p), dep_type = Package_Dependency.TYPE_RCONFLICTS))
 
-        if len(packagedeps_objs) > 0:
+        if packagedeps_objs:
             Package_Dependency.objects.bulk_create(packagedeps_objs)
 
         return bp_object
@@ -826,7 +835,7 @@ class ORMWrapper(object):
                     desc = vardump[root_var]['doc']
             if desc is None:
                 desc = ''
-            if len(desc):
+            if desc:
                 HelpText.objects.get_or_create(build=build_obj,
                                                area=HelpText.VARIABLE,
                                                key=k, text=desc)
@@ -846,7 +855,7 @@ class ORMWrapper(object):
                                 file_name = vh['file'],
                                 line_number = vh['line'],
                                 operation = vh['op']))
-                if len(varhist_objects):
+                if varhist_objects:
                     VariableHistory.objects.bulk_create(varhist_objects)
 
 
@@ -893,9 +902,6 @@ class BuildInfoHelper(object):
         self.task_order = 0
         self.autocommit_step = 1
         self.server = server
-        # we use manual transactions if the database doesn't autocommit on us
-        if not connection.features.autocommits_when_autocommit_is_off:
-            transaction.set_autocommit(False)
         self.orm_wrapper = ORMWrapper()
         self.has_build_history = has_build_history
         self.tmp_dir = self.server.runCommand(["getVariable", "TMPDIR"])[0]
@@ -1059,27 +1065,6 @@ class BuildInfoHelper(object):
 
         return recipe_info
 
-    def _get_path_information(self, task_object):
-        self._ensure_build()
-
-        assert isinstance(task_object, Task)
-        build_stats_format = "{tmpdir}/buildstats/{buildname}/{package}/"
-        build_stats_path = []
-
-        for t in self.internal_state['targets']:
-            buildname = self.internal_state['build'].build_name
-            pe, pv = task_object.recipe.version.split(":",1)
-            if len(pe) > 0:
-                package = task_object.recipe.name + "-" + pe + "_" + pv
-            else:
-                package = task_object.recipe.name + "-" + pv
-
-            build_stats_path.append(build_stats_format.format(tmpdir=self.tmp_dir,
-                                                     buildname=buildname,
-                                                     package=package))
-
-        return build_stats_path
-
 
     ################################
     ## external available methods to store information
@@ -1313,12 +1298,11 @@ class BuildInfoHelper(object):
                 task_information['outcome'] = Task.OUTCOME_FAILED
                 del self.internal_state['taskdata'][identifier]
 
-        if not connection.features.autocommits_when_autocommit_is_off:
-            # we force a sync point here, to get the progress bar to show
-            if self.autocommit_step % 3 == 0:
-                transaction.set_autocommit(True)
-                transaction.set_autocommit(False)
-            self.autocommit_step += 1
+        # we force a sync point here, to get the progress bar to show
+        if self.autocommit_step % 3 == 0:
+            transaction.set_autocommit(True)
+            transaction.set_autocommit(False)
+        self.autocommit_step += 1
 
         self.orm_wrapper.get_update_task_object(task_information, True) # must exist
 
@@ -1404,7 +1388,7 @@ class BuildInfoHelper(object):
         assert 'pn' in event._depgraph
         assert 'tdepends' in event._depgraph
 
-        errormsg = ""
+        errormsg = []
 
         # save layer version priorities
         if 'layer-priorities' in event._depgraph.keys():
@@ -1496,7 +1480,7 @@ class BuildInfoHelper(object):
                 elif dep in self.internal_state['recipes']:
                     dependency = self.internal_state['recipes'][dep]
                 else:
-                    errormsg += "  stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep)
+                    errormsg.append("  stpd: KeyError saving recipe dependency for %s, %s \n" % (recipe, dep))
                     continue
                 recipe_dep = Recipe_Dependency(recipe=target,
                                                depends_on=dependency,
@@ -1537,8 +1521,8 @@ class BuildInfoHelper(object):
                 taskdeps_objects.append(Task_Dependency( task = target, depends_on = dep ))
         Task_Dependency.objects.bulk_create(taskdeps_objects)
 
-        if len(errormsg) > 0:
-            logger.warning("buildinfohelper: dependency info not identify recipes: \n%s", errormsg)
+        if errormsg:
+            logger.warning("buildinfohelper: dependency info not identify recipes: \n%s", "".join(errormsg))
 
 
     def store_build_package_information(self, event):
@@ -1618,7 +1602,7 @@ class BuildInfoHelper(object):
 
         if 'backlog' in self.internal_state:
             # if we have a backlog of events, do our best to save them here
-            if len(self.internal_state['backlog']):
+            if self.internal_state['backlog']:
                 tempevent = self.internal_state['backlog'].pop()
                 logger.debug("buildinfohelper: Saving stored event %s "
                              % tempevent)
@@ -1765,7 +1749,6 @@ class BuildInfoHelper(object):
 
         buildname = self.server.runCommand(['getVariable', 'BUILDNAME'])[0]
         machine = self.server.runCommand(['getVariable', 'MACHINE'])[0]
-        image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
 
         # location of the manifest files for this build;
         # note that this file is only produced if an image is produced
@@ -1786,6 +1769,18 @@ class BuildInfoHelper(object):
         # filter out anything which isn't an image target
         image_targets = [target for target in targets if target.is_image]
 
+        if len(image_targets) > 0:
+            #if there are image targets retrieve image_name
+            image_name = self.server.runCommand(['getVariable', 'IMAGE_NAME'])[0]
+            if not image_name:
+                #When build target is an image and image_name is not found as an environment variable
+                logger.info("IMAGE_NAME not found, extracting from bitbake command")
+                cmd = self.server.runCommand(['getVariable','BB_CMDLINE'])[0]
+                #filter out tokens that are command line options
+                cmd = [token for token in cmd if not token.startswith('-')]
+                image_name = cmd[1].split(':', 1)[0] # remove everything after : in image name
+                logger.info("IMAGE_NAME found as : %s " % image_name)
+
         for image_target in image_targets:
             # this is set to True if we find at least one file relating to
             # this target; if this remains False after the scan, we copy the
@@ -1990,8 +1985,6 @@ class BuildInfoHelper(object):
             # Do not skip command line build events
             self.store_log_event(tempevent,False)
 
-        if not connection.features.autocommits_when_autocommit_is_off:
-            transaction.set_autocommit(True)
 
         # unset the brbe; this is to prevent subsequent command-line builds
         # being incorrectly attached to the previous Toaster-triggered build;
diff --git a/bitbake/lib/bb/ui/eventreplay.py b/bitbake/lib/bb/ui/eventreplay.py
new file mode 100644
index 0000000000..d62ecbfa56
--- /dev/null
+++ b/bitbake/lib/bb/ui/eventreplay.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# This file re-uses code spread throughout other Bitbake source files.
+# As such, all other copyrights belong to their own right holders.
+#
+
+
+import os
+import sys
+import json
+import pickle
+import codecs
+
+
+class EventPlayer:
+    """Emulate a connection to a bitbake server."""
+
+    def __init__(self, eventfile, variables):
+        self.eventfile = eventfile
+        self.variables = variables
+        self.eventmask = []
+
+    def waitEvent(self, _timeout):
+        """Read event from the file."""
+        line = self.eventfile.readline().strip()
+        if not line:
+            return
+        try:
+            decodedline = json.loads(line)
+            if 'allvariables' in decodedline:
+                self.variables = decodedline['allvariables']
+                return
+            if not 'vars' in decodedline:
+                raise ValueError
+            event_str = decodedline['vars'].encode('utf-8')
+            event = pickle.loads(codecs.decode(event_str, 'base64'))
+            event_name = "%s.%s" % (event.__module__, event.__class__.__name__)
+            if event_name not in self.eventmask:
+                return
+            return event
+        except ValueError as err:
+            print("Failed loading ", line)
+            raise err
+
+    def runCommand(self, command_line):
+        """Emulate running a command on the server."""
+        name = command_line[0]
+
+        if name == "getVariable":
+            var_name = command_line[1]
+            variable = self.variables.get(var_name)
+            if variable:
+                return variable['v'], None
+            return None, "Missing variable %s" % var_name
+
+        elif name == "getAllKeysWithFlags":
+            dump = {}
+            flaglist = command_line[1]
+            for key, val in self.variables.items():
+                try:
+                    if not key.startswith("__"):
+                        dump[key] = {
+                            'v': val['v'],
+                            'history' : val['history'],
+                        }
+                        for flag in flaglist:
+                            dump[key][flag] = val[flag]
+                except Exception as err:
+                    print(err)
+            return (dump, None)
+
+        elif name == 'setEventMask':
+            self.eventmask = command_line[-1]
+            return True, None
+
+        else:
+            raise Exception("Command %s not implemented" % command_line[0])
+
+    def getEventHandle(self):
+        """
+        This method is called by toasterui.
+        The return value is passed to self.runCommand but not used there.
+        """
+        pass
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index 0efa614dfc..9a589a5c8e 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -21,10 +21,17 @@ import fcntl
 import struct
 import copy
 import atexit
+from itertools import groupby
 
 from bb.ui import uihelper
+import bb.build
+import bb.command
+import bb.cooker
+import bb.event
+import bb.runqueue
+import bb.utils
 
-featureSet = [bb.cooker.CookerFeatures.SEND_SANITYEVENTS]
+featureSet = [bb.cooker.CookerFeatures.SEND_SANITYEVENTS, bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
 
 logger = logging.getLogger("BitBake")
 interactive = sys.stdout.isatty()
@@ -102,7 +109,7 @@ def new_progress(msg, maxval):
         return NonInteractiveProgress(msg, maxval)
 
 def pluralise(singular, plural, qty):
-    if(qty == 1):
+    if qty == 1:
         return singular % qty
     else:
         return plural % qty
@@ -111,6 +118,7 @@ def pluralise(singular, plural, qty):
 class InteractConsoleLogFilter(logging.Filter):
     def __init__(self, tf):
         self.tf = tf
+        super().__init__()
 
     def filter(self, record):
         if record.levelno == bb.msg.BBLogFormatter.NOTE and (record.msg.startswith("Running") or record.msg.startswith("recipe ")):
@@ -178,7 +186,7 @@ class TerminalFilter(object):
             new[3] = new[3] & ~termios.ECHO
             termios.tcsetattr(fd, termios.TCSADRAIN, new)
             curses.setupterm()
-            if curses.tigetnum("colors") > 2:
+            if curses.tigetnum("colors") > 2 and os.environ.get('NO_COLOR', '') == '':
                 for h in handlers:
                     try:
                         h.formatter.enable_color()
@@ -227,7 +235,9 @@ class TerminalFilter(object):
 
     def keepAlive(self, t):
         if not self.cuu:
-            print("Bitbake still alive (%ds)" % t)
+            print("Bitbake still alive (no events for %ds). Active tasks:" % t)
+            for t in self.helper.running_tasks:
+                print(t)
             sys.stdout.flush()
 
     def updateFooter(self):
@@ -249,58 +259,68 @@ class TerminalFilter(object):
             return
         tasks = []
         for t in runningpids:
+            start_time = activetasks[t].get("starttime", None)
+            if start_time:
+                msg = "%s - %s (pid %s)" % (activetasks[t]["title"], self.elapsed(currenttime - start_time), activetasks[t]["pid"])
+            else:
+                msg = "%s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"])
             progress = activetasks[t].get("progress", None)
             if progress is not None:
                 pbar = activetasks[t].get("progressbar", None)
                 rate = activetasks[t].get("rate", None)
-                start_time = activetasks[t].get("starttime", None)
                 if not pbar or pbar.bouncing != (progress < 0):
                     if progress < 0:
-                        pbar = BBProgress("0: %s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]), 100, widgets=[' ', progressbar.BouncingSlider(), ''], extrapos=3, resize_handler=self.sigwinch_handle)
+                        pbar = BBProgress("0: %s" % msg, 100, widgets=[' ', progressbar.BouncingSlider(), ''], extrapos=3, resize_handler=self.sigwinch_handle)
                         pbar.bouncing = True
                     else:
-                        pbar = BBProgress("0: %s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]), 100, widgets=[' ', progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=5, resize_handler=self.sigwinch_handle)
+                        pbar = BBProgress("0: %s" % msg, 100, widgets=[' ', progressbar.Percentage(), ' ', progressbar.Bar(), ''], extrapos=5, resize_handler=self.sigwinch_handle)
                         pbar.bouncing = False
                     activetasks[t]["progressbar"] = pbar
-                tasks.append((pbar, progress, rate, start_time))
+                tasks.append((pbar, msg, progress, rate, start_time))
             else:
-                start_time = activetasks[t].get("starttime", None)
-                if start_time:
-                    tasks.append("%s - %s (pid %s)" % (activetasks[t]["title"], self.elapsed(currenttime - start_time), activetasks[t]["pid"]))
-                else:
-                    tasks.append("%s (pid %s)" % (activetasks[t]["title"], activetasks[t]["pid"]))
+                tasks.append(msg)
 
         if self.main.shutdown:
-            content = "Waiting for %s running tasks to finish:" % len(activetasks)
+            content = pluralise("Waiting for %s running task to finish",
+                                "Waiting for %s running tasks to finish", len(activetasks))
+            if not self.quiet:
+                content += ':'
             print(content)
         else:
+            scene_tasks = "%s of %s" % (self.helper.setscene_current, self.helper.setscene_total)
+            cur_tasks = "%s of %s" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
+
+            content = ''
+            if not self.quiet:
+                msg = "Setscene tasks: %s" % scene_tasks
+                content += msg + "\n"
+                print(msg)
+
             if self.quiet:
-                content = "Running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
+                msg = "Running tasks (%s, %s)" % (scene_tasks, cur_tasks)
             elif not len(activetasks):
-                content = "No currently running tasks (%s of %s)" % (self.helper.tasknumber_current, self.helper.tasknumber_total)
+                msg = "No currently running tasks (%s)" % cur_tasks
             else:
-                content = "Currently %2s running tasks (%s of %s)" % (len(activetasks), self.helper.tasknumber_current, self.helper.tasknumber_total)
+                msg = "Currently %2s running tasks (%s)" % (len(activetasks), cur_tasks)
             maxtask = self.helper.tasknumber_total
             if not self.main_progress or self.main_progress.maxval != maxtask:
                 widgets = [' ', progressbar.Percentage(), ' ', progressbar.Bar()]
                 self.main_progress = BBProgress("Running tasks", maxtask, widgets=widgets, resize_handler=self.sigwinch_handle)
                 self.main_progress.start(False)
-            self.main_progress.setmessage(content)
-            progress = self.helper.tasknumber_current - 1
-            if progress < 0:
-                progress = 0
-            content = self.main_progress.update(progress)
+            self.main_progress.setmessage(msg)
+            progress = max(0, self.helper.tasknumber_current - 1)
+            content += self.main_progress.update(progress)
             print('')
-        lines = 1 + int(len(content) / (self.columns + 1))
-        if self.quiet == 0:
-            for tasknum, task in enumerate(tasks[:(self.rows - 2)]):
+        lines = self.getlines(content)
+        if not self.quiet:
+            for tasknum, task in enumerate(tasks[:(self.rows - 1 - lines)]):
                 if isinstance(task, tuple):
-                    pbar, progress, rate, start_time = task
+                    pbar, msg, progress, rate, start_time = task
                     if not pbar.start_time:
                         pbar.start(False)
                         if start_time:
                             pbar.start_time = start_time
-                    pbar.setmessage('%s:%s' % (tasknum, pbar.msg.split(':', 1)[1]))
+                    pbar.setmessage('%s: %s' % (tasknum, msg))
                     pbar.setextra(rate)
                     if progress > -1:
                         content = pbar.update(progress)
@@ -310,11 +330,17 @@ class TerminalFilter(object):
                 else:
                     content = "%s: %s" % (tasknum, task)
                     print(content)
-                lines = lines + 1 + int(len(content) / (self.columns + 1))
+                lines = lines + self.getlines(content)
         self.footer_present = lines
         self.lastpids = runningpids[:]
         self.lastcount = self.helper.tasknumber_current
 
+    def getlines(self, content):
+        lines = 0
+        for line in content.split("\n"):
+            lines = lines + 1 + int(len(line) / (self.columns + 1))
+        return lines
+
     def finish(self):
         if self.stdinbackup:
             fd = sys.stdin.fileno()
@@ -327,7 +353,7 @@ def print_event_log(event, includelogs, loglines, termfilter):
         termfilter.clearFooter()
         bb.error("Logfile of failure stored in: %s" % logfile)
         if includelogs and not event.errprinted:
-            print("Log data follows:")
+            bb.plain("Log data follows:")
             f = open(logfile, "r")
             lines = []
             while True:
@@ -340,11 +366,11 @@ def print_event_log(event, includelogs, loglines, termfilter):
                     if len(lines) > int(loglines):
                         lines.pop(0)
                 else:
-                    print('| %s' % l)
+                    bb.plain('| %s' % l)
             f.close()
             if lines:
                 for line in lines:
-                    print(line)
+                    bb.plain(line)
 
 def _log_settings_from_server(server, observe_only):
     # Get values of variables which control our output
@@ -401,6 +427,11 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     except bb.BBHandledException:
         drain_events_errorhandling(eventHandler)
         return 1
+    except Exception as e:
+        # bitbake-server comms failure
+        early_logger = bb.msg.logger_create('bitbake', sys.stdout)
+        early_logger.fatal("Attempting to set server environment: %s", e)
+        return 1
 
     if params.options.quiet == 0:
         console_loglevel = loglevel
@@ -531,13 +562,30 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 }
             })
 
-        bb.utils.mkdirhier(os.path.dirname(consolelogfile))
-        loglink = os.path.join(os.path.dirname(consolelogfile), 'console-latest.log')
+        consolelogdirname = os.path.dirname(consolelogfile)
+        # `bb.utils.mkdirhier` has this check, but it reports failure using bb.fatal, which logs
+        # to the very logger we are trying to set up.
+        if '${' in str(consolelogdirname):
+            print(
+                "FATAL: Directory name {} contains unexpanded bitbake variable. This may cause build failures and WORKDIR pollution.".format(
+                    consolelogdirname))
+            if '${MACHINE}' in consolelogdirname:
+                print("HINT: It looks like you forgot to set MACHINE in local.conf.")
+
+        bb.utils.mkdirhier(consolelogdirname)
+        loglink = os.path.join(consolelogdirname, 'console-latest.log')
         bb.utils.remove(loglink)
         try:
-           os.symlink(os.path.basename(consolelogfile), loglink)
+            os.symlink(os.path.basename(consolelogfile), loglink)
         except OSError:
-           pass
+            pass
+
+    # Add the logging domains specified by the user on the command line
+    for (domainarg, iterator) in groupby(params.debug_domains):
+        dlevel = len(tuple(iterator))
+        l = logconfig["loggers"].setdefault("BitBake.%s" % domainarg, {})
+        l["level"] = logging.DEBUG - dlevel + 1
+        l.setdefault("handlers", []).extend(["BitBake.verbconsole"])
 
     conf = bb.msg.setLoggingConfig(logconfig, logconfigfile)
 
@@ -546,6 +594,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     else:
         log_exec_tty = False
 
+    should_print_hyperlinks = sys.stdout.isatty() and os.environ.get('NO_COLOR', '') == ''
+
     helper = uihelper.BBUIHelper()
 
     # Look for the specially designated handlers which need to be passed to the
@@ -559,7 +609,12 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         return
 
     llevel, debug_domains = bb.msg.constructLogOptions()
-    server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list])
+    try:
+        server.runCommand(["setEventMask", server.getEventHandle(), llevel, debug_domains, _evt_list])
+    except (BrokenPipeError, EOFError) as e:
+        # bitbake-server comms failure
+        logger.fatal("Attempting to set event mask: %s", e)
+        return 1
 
     # The logging_tree module is *extremely* helpful in debugging logging
     # domains. Uncomment here to dump the logging tree when bitbake starts
@@ -568,7 +623,11 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
     universe = False
     if not params.observe_only:
-        params.updateFromServer(server)
+        try:
+            params.updateFromServer(server)
+        except Exception as e:
+            logger.fatal("Fetching command line: %s", e)
+            return 1
         cmdline = params.parseActions()
         if not cmdline:
             print("Nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
@@ -579,7 +638,12 @@ def main(server, eventHandler, params, tf = TerminalFilter):
         if cmdline['action'][0] == "buildTargets" and "universe" in cmdline['action'][1]:
             universe = True
 
-        ret, error = server.runCommand(cmdline['action'])
+        try:
+            ret, error = server.runCommand(cmdline['action'])
+        except (BrokenPipeError, EOFError) as e:
+            # bitbake-server comms failure
+            logger.fatal("Command '{}' failed: %s".format(cmdline), e)
+            return 1
         if error:
             logger.error("Command '%s' failed: %s" % (cmdline, error))
             return 1
@@ -595,28 +659,42 @@ def main(server, eventHandler, params, tf = TerminalFilter):
     return_value = 0
     errors = 0
     warnings = 0
-    taskfailures = []
+    taskfailures = {}
 
-    printinterval = 5000
-    lastprint = time.time()
+    printintervaldelta = 10 * 60 # 10 minutes
+    printinterval = printintervaldelta
+    pinginterval = 1 * 60 # 1 minute
+    lastevent = lastprint = time.time()
 
     termfilter = tf(main, helper, console_handlers, params.options.quiet)
     atexit.register(termfilter.finish)
 
-    while True:
+    # shutdown levels
+    # 0 - normal operation
+    # 1 - no new task execution, let current running tasks finish
+    # 2 - interrupting currently executing tasks
+    # 3 - we're done, exit
+    while main.shutdown < 3:
         try:
             if (lastprint + printinterval) <= time.time():
                 termfilter.keepAlive(printinterval)
-                printinterval += 5000
+                printinterval += printintervaldelta
             event = eventHandler.waitEvent(0)
             if event is None:
-                if main.shutdown > 1:
-                    break
+                if (lastevent + pinginterval) <= time.time():
+                    ret, error = server.runCommand(["ping"])
+                    if error or not ret:
+                        termfilter.clearFooter()
+                        print("No reply after pinging server (%s, %s), exiting." % (str(error), str(ret)))
+                        return_value = 3
+                        main.shutdown = 3
+                    lastevent = time.time()
                 if not parseprogress:
                     termfilter.updateFooter()
                 event = eventHandler.waitEvent(0.25)
                 if event is None:
                     continue
+            lastevent = time.time()
             helper.eventHandler(event)
             if isinstance(event, bb.runqueue.runQueueExitWait):
                 if not main.shutdown:
@@ -638,8 +716,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             if isinstance(event, logging.LogRecord):
                 lastprint = time.time()
-                printinterval = 5000
-                if event.levelno >= bb.msg.BBLogFormatter.ERROR:
+                printinterval = printintervaldelta
+                if event.levelno >= bb.msg.BBLogFormatter.ERRORONCE:
                     errors = errors + 1
                     return_value = 1
                 elif event.levelno == bb.msg.BBLogFormatter.WARNING:
@@ -653,10 +731,10 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                         continue
 
                     # Prefix task messages with recipe/task
-                    if event.taskpid in helper.pidmap and event.levelno != bb.msg.BBLogFormatter.PLAIN:
+                    if event.taskpid in helper.pidmap and event.levelno not in [bb.msg.BBLogFormatter.PLAIN, bb.msg.BBLogFormatter.WARNONCE, bb.msg.BBLogFormatter.ERRORONCE]:
                         taskinfo = helper.running_tasks[helper.pidmap[event.taskpid]]
                         event.msg = taskinfo['title'] + ': ' + event.msg
-                if hasattr(event, 'fn'):
+                if hasattr(event, 'fn') and event.levelno not in [bb.msg.BBLogFormatter.WARNONCE, bb.msg.BBLogFormatter.ERRORONCE]:
                     event.msg = event.fn + ': ' + event.msg
                 logging.getLogger(event.name).handle(event)
                 continue
@@ -667,6 +745,8 @@ def main(server, eventHandler, params, tf = TerminalFilter):
             if isinstance(event, bb.build.TaskFailed):
                 return_value = 1
                 print_event_log(event, includelogs, loglines, termfilter)
+                k = "{}:{}".format(event._fn, event._task)
+                taskfailures[k] = event.logfile
             if isinstance(event, bb.build.TaskBase):
                 logger.info(event._message)
                 continue
@@ -721,15 +801,15 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 if event.error:
                     errors = errors + 1
                     logger.error(str(event))
-                main.shutdown = 2
+                main.shutdown = 3
                 continue
             if isinstance(event, bb.command.CommandExit):
                 if not return_value:
                     return_value = event.exitcode
-                main.shutdown = 2
+                main.shutdown = 3
                 continue
             if isinstance(event, (bb.command.CommandCompleted, bb.cooker.CookerExit)):
-                main.shutdown = 2
+                main.shutdown = 3
                 continue
             if isinstance(event, bb.event.MultipleProviders):
                 logger.info(str(event))
@@ -745,7 +825,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                 continue
 
             if isinstance(event, bb.runqueue.sceneQueueTaskStarted):
-                logger.info("Running setscene task %d of %d (%s)" % (event.stats.completed + event.stats.active + event.stats.failed + 1, event.stats.total, event.taskstring))
+                logger.info("Running setscene task %d of %d (%s)" % (event.stats.setscene_covered + event.stats.setscene_active + event.stats.setscene_notcovered + 1, event.stats.setscene_total, event.taskstring))
                 continue
 
             if isinstance(event, bb.runqueue.runQueueTaskStarted):
@@ -762,7 +842,7 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             if isinstance(event, bb.runqueue.runQueueTaskFailed):
                 return_value = 1
-                taskfailures.append(event.taskstring)
+                taskfailures.setdefault(event.taskstring)
                 logger.error(str(event))
                 continue
 
@@ -814,15 +894,26 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             logger.error("Unknown event: %s", event)
 
+        except (BrokenPipeError, EOFError) as e:
+            # bitbake-server comms failure, don't attempt further comms and exit
+            logger.fatal("Executing event: %s", e)
+            return_value = 1
+            errors = errors + 1
+            main.shutdown = 3
         except EnvironmentError as ioerror:
             termfilter.clearFooter()
             # ignore interrupted io
             if ioerror.args[0] == 4:
                 continue
             sys.stderr.write(str(ioerror))
-            if not params.observe_only:
-                _, error = server.runCommand(["stateForceShutdown"])
             main.shutdown = 2
+            if not params.observe_only:
+                try:
+                    _, error = server.runCommand(["stateForceShutdown"])
+                except (BrokenPipeError, EOFError) as e:
+                    # bitbake-server comms failure, don't attempt further comms and exit
+                    logger.fatal("Unable to force shutdown: %s", e)
+                    main.shutdown = 3
         except KeyboardInterrupt:
             termfilter.clearFooter()
             if params.observe_only:
@@ -831,9 +922,13 @@ def main(server, eventHandler, params, tf = TerminalFilter):
 
             def state_force_shutdown():
                 print("\nSecond Keyboard Interrupt, stopping...\n")
-                _, error = server.runCommand(["stateForceShutdown"])
-                if error:
-                    logger.error("Unable to cleanly stop: %s" % error)
+                try:
+                    _, error = server.runCommand(["stateForceShutdown"])
+                    if error:
+                        logger.error("Unable to cleanly stop: %s" % error)
+                except (BrokenPipeError, EOFError) as e:
+                    # bitbake-server comms failure
+                    logger.fatal("Unable to cleanly stop: %s", e)
 
             if not params.observe_only and main.shutdown == 1:
                 state_force_shutdown()
@@ -846,32 +941,49 @@ def main(server, eventHandler, params, tf = TerminalFilter):
                     _, error = server.runCommand(["stateShutdown"])
                     if error:
                         logger.error("Unable to cleanly shutdown: %s" % error)
+                except (BrokenPipeError, EOFError) as e:
+                    # bitbake-server comms failure
+                    logger.fatal("Unable to cleanly shutdown: %s", e)
                 except KeyboardInterrupt:
                     state_force_shutdown()
 
             main.shutdown = main.shutdown + 1
-            pass
         except Exception as e:
             import traceback
             sys.stderr.write(traceback.format_exc())
-            if not params.observe_only:
-                _, error = server.runCommand(["stateForceShutdown"])
             main.shutdown = 2
+            if not params.observe_only:
+                try:
+                    _, error = server.runCommand(["stateForceShutdown"])
+                except (BrokenPipeError, EOFError) as e:
+                    # bitbake-server comms failure, don't attempt further comms and exit
+                    logger.fatal("Unable to force shutdown: %s", e)
+                    main.shudown = 3
             return_value = 1
     try:
         termfilter.clearFooter()
         summary = ""
+        def format_hyperlink(url, link_text):
+            if should_print_hyperlinks:
+                start = f'\033]8;;{url}\033\\'
+                end = '\033]8;;\033\\'
+                return f'{start}{link_text}{end}'
+            return link_text
+
         if taskfailures:
             summary += pluralise("\nSummary: %s task failed:",
                                  "\nSummary: %s tasks failed:", len(taskfailures))
-            for failure in taskfailures:
+            for (failure, log_file) in taskfailures.items():
                 summary += "\n  %s" % failure
+                if log_file:
+                    hyperlink = format_hyperlink(f"file://{log_file}", log_file)
+                    summary += "\n    log: {}".format(hyperlink)
         if warnings:
-            summary += pluralise("\nSummary: There was %s WARNING message shown.",
-                                 "\nSummary: There were %s WARNING messages shown.", warnings)
+            summary += pluralise("\nSummary: There was %s WARNING message.",
+                                 "\nSummary: There were %s WARNING messages.", warnings)
         if return_value and errors:
-            summary += pluralise("\nSummary: There was %s ERROR message shown, returning a non-zero exit code.",
-                                 "\nSummary: There were %s ERROR messages shown, returning a non-zero exit code.", errors)
+            summary += pluralise("\nSummary: There was %s ERROR message, returning a non-zero exit code.",
+                                 "\nSummary: There were %s ERROR messages, returning a non-zero exit code.", errors)
         if summary and params.options.quiet == 0:
             print(summary)
 
diff --git a/bitbake/lib/bb/ui/ncurses.py b/bitbake/lib/bb/ui/ncurses.py
index cf1c876a51..18a706547a 100644
--- a/bitbake/lib/bb/ui/ncurses.py
+++ b/bitbake/lib/bb/ui/ncurses.py
@@ -227,6 +227,9 @@ class NCursesUI:
         shutdown = 0
 
         try:
+            if not params.observe_only:
+                params.updateToServer(server, os.environ.copy())
+
             params.updateFromServer(server)
             cmdline = params.parseActions()
             if not cmdline:
diff --git a/bitbake/lib/bb/ui/taskexp.py b/bitbake/lib/bb/ui/taskexp.py
index 2b246710ca..bedfd69b09 100644
--- a/bitbake/lib/bb/ui/taskexp.py
+++ b/bitbake/lib/bb/ui/taskexp.py
@@ -8,6 +8,7 @@
 #
 
 import sys
+import traceback
 
 try:
     import gi
@@ -176,7 +177,7 @@ class gtkthread(threading.Thread):
     quit = threading.Event()
     def __init__(self, shutdown):
         threading.Thread.__init__(self)
-        self.setDaemon(True)
+        self.daemon = True
         self.shutdown = shutdown
         if not Gtk.init_check()[0]:
             sys.stderr.write("Gtk+ init failed. Make sure DISPLAY variable is set.\n")
@@ -196,6 +197,7 @@ def main(server, eventHandler, params):
     gtkgui.start()
 
     try:
+        params.updateToServer(server, os.environ.copy())
         params.updateFromServer(server)
         cmdline = params.parseActions()
         if not cmdline:
@@ -218,6 +220,9 @@ def main(server, eventHandler, params):
     except client.Fault as x:
         print("XMLRPC Fault getting commandline:\n %s" % x)
         return
+    except Exception as e:
+        print("Exception in startup:\n %s" % traceback.format_exc())
+        return
 
     if gtkthread.quit.isSet():
         return
diff --git a/bitbake/lib/bb/ui/taskexp_ncurses.py b/bitbake/lib/bb/ui/taskexp_ncurses.py
new file mode 100755
index 0000000000..ea94a4987f
--- /dev/null
+++ b/bitbake/lib/bb/ui/taskexp_ncurses.py
@@ -0,0 +1,1511 @@
+#
+# BitBake Graphical ncurses-based Dependency Explorer
+#   * Based on the GTK implementation
+#   * Intended to run on any Linux host
+#
+# Copyright (C) 2007        Ross Burton
+# Copyright (C) 2007 - 2008 Richard Purdie
+# Copyright (C) 2022 - 2024 David Reyna
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+#
+# Execution example:
+#   $ bitbake -g -u taskexp_ncurses zlib acl
+#
+# Self-test example (executes a script of GUI actions):
+#   $ TASK_EXP_UNIT_TEST=1 bitbake -g -u taskexp_ncurses zlib acl
+#   ...
+#   $ echo $?
+#   0
+#   $ TASK_EXP_UNIT_TEST=1 bitbake -g -u taskexp_ncurses zlib acl foo
+#   ERROR: Nothing PROVIDES 'foo'. Close matches:
+#   ofono
+#   $ echo $?
+#   1
+#
+# Self-test with no terminal example (only tests dependency fetch from bitbake):
+#   $ TASK_EXP_UNIT_TEST_NOTERM=1 bitbake -g -u taskexp_ncurses quilt
+#   $ echo $?
+#   0
+#
+# Features:
+# * Ncurses is used for the presentation layer. Only the 'curses'
+#   library is used (none of the extension libraries), plus only
+#   one main screen is used (no sub-windows)
+# * Uses the 'generateDepTreeEvent' bitbake event to fetch the
+#   dynamic dependency data based on passed recipes
+# * Computes and provides reverse dependencies
+# * Supports task sorting on:
+#   (a) Task dependency order within each recipe
+#   (b) Pure alphabetical order
+#   (c) Provisions for third sort order (bitbake order?)
+# * The 'Filter' does a "*string*" wildcard filter on tasks in the
+#   main window, dynamically re-ordering and re-centering the content
+# * A 'Print' function exports the selected task or its whole recipe
+#   task set to the default file "taskdep.txt"
+# * Supports a progress bar for bitbake loads and file printing
+# * Line art for box drawing supported, ASCII art an alernative
+# * No horizontal scrolling support. Selected task's full name
+#   shown in bottom bar
+# * Dynamically catches terminals that are (or become) too small
+# * Exception to insure return to normal terminal on errors
+# * Debugging support, self test option
+#
+
+import sys
+import traceback
+import curses
+import re
+import time
+
+# Bitbake server support
+import threading
+from xmlrpc import client
+import bb
+import bb.event
+
+# Dependency indexes (depends_model)
+(TYPE_DEP, TYPE_RDEP) = (0, 1)
+DEPENDS_TYPE = 0
+DEPENDS_TASK = 1
+DEPENDS_DEPS = 2
+# Task indexes (task_list)
+TASK_NAME = 0
+TASK_PRIMARY = 1
+TASK_SORT_ALPHA = 2
+TASK_SORT_DEPS = 3
+TASK_SORT_BITBAKE = 4
+# Sort options (default is SORT_DEPS)
+SORT_ALPHA = 0
+SORT_DEPS = 1
+SORT_BITBAKE_ENABLE = False # NOTE: future sort
+SORT_BITBAKE = 2
+sort_model = SORT_DEPS
+# Print options
+PRINT_MODEL_1 = 0
+PRINT_MODEL_2 = 1
+print_model = PRINT_MODEL_2
+print_file_name = "taskdep_print.log"
+print_file_backup_name = "taskdep_print_backup.log"
+is_printed = False
+is_filter = False
+
+# Standard (and backup) key mappings
+CHAR_NUL = 0            # Used as self-test nop char
+CHAR_BS_H = 8           # Alternate backspace key
+CHAR_TAB = 9
+CHAR_RETURN = 10
+CHAR_ESCAPE = 27
+CHAR_UP = ord('{')      # Used as self-test ASCII char
+CHAR_DOWN = ord('}')    # Used as self-test ASCII char
+
+# Color_pair IDs
+CURSES_NORMAL = 0
+CURSES_HIGHLIGHT = 1
+CURSES_WARNING = 2
+
+
+#################################################
+### Debugging support
+###
+
+verbose = False
+
+# Debug: message display slow-step through display update issues
+def alert(msg,screen):
+    if msg:
+        screen.addstr(0, 10, '[%-4s]' % msg)
+        screen.refresh();
+        curses.napms(2000)
+    else:
+        if do_line_art:
+            for i in range(10, 24):
+                screen.addch(0, i, curses.ACS_HLINE)
+        else:
+            screen.addstr(0, 10, '-' * 14)
+        screen.refresh();
+
+# Debug: display edge conditions on frame movements
+def debug_frame(nbox_ojb):
+    if verbose:
+        nbox_ojb.screen.addstr(0, 50, '[I=%2d,O=%2d,S=%3s,H=%2d,M=%4d]' % (
+            nbox_ojb.cursor_index,
+            nbox_ojb.cursor_offset,
+            nbox_ojb.scroll_offset,
+            nbox_ojb.inside_height,
+            len(nbox_ojb.task_list),
+        ))
+        nbox_ojb.screen.refresh();
+
+#
+# Unit test (assumes that 'quilt-native' is always present)
+#
+
+unit_test = os.environ.get('TASK_EXP_UNIT_TEST')
+unit_test_cmnds=[
+    '# Default selected task in primary box',
+    'tst_selected=<TASK>.do_recipe_qa',
+    '# Default selected task in deps',
+    'tst_entry=<TAB>',
+    'tst_selected=',
+    '# Default selected task in rdeps',
+    'tst_entry=<TAB>',
+    'tst_selected=<TASK>.do_fetch',
+    "# Test 'select' back to primary box",
+    'tst_entry=<CR>',
+    '#tst_entry=<DOWN>',  # optional injected error
+    'tst_selected=<TASK>.do_fetch',
+    '# Check filter',
+    'tst_entry=/uilt-nativ/',
+    'tst_selected=quilt-native.do_recipe_qa',
+    '# Check print',
+    'tst_entry=p',
+    'tst_printed=quilt-native.do_fetch',
+    '#tst_printed=quilt-foo.do_nothing',  # optional injected error
+    '# Done!',
+    'tst_entry=q',
+]
+unit_test_idx=0
+unit_test_command_chars=''
+unit_test_results=[]
+def unit_test_action(active_package):
+    global unit_test_idx
+    global unit_test_command_chars
+    global unit_test_results
+    ret = CHAR_NUL
+    if unit_test_command_chars:
+        ch = unit_test_command_chars[0]
+        unit_test_command_chars = unit_test_command_chars[1:]
+        time.sleep(0.5)
+        ret = ord(ch)
+    else:
+        line = unit_test_cmnds[unit_test_idx]
+        unit_test_idx += 1
+        line = re.sub('#.*', '', line).strip()
+        line = line.replace('<TASK>',active_package.primary[0])
+        line = line.replace('<TAB>','\t').replace('<CR>','\n')
+        line = line.replace('<UP>','{').replace('<DOWN>','}')
+        if not line: line = 'nop=nop'
+        cmnd,value = line.split('=')
+        if cmnd == 'tst_entry':
+            unit_test_command_chars = value
+        elif cmnd == 'tst_selected':
+            active_selected = active_package.get_selected()
+            if active_selected != value:
+                unit_test_results.append("ERROR:SELFTEST:expected '%s' but got '%s' (NOTE:bitbake may have changed)" % (value,active_selected))
+                ret = ord('Q')
+            else:
+                unit_test_results.append("Pass:SELFTEST:found '%s'" % (value))
+        elif cmnd == 'tst_printed':
+            result = os.system('grep %s %s' % (value,print_file_name))
+            if result:
+                unit_test_results.append("ERROR:PRINTTEST:expected '%s' in '%s'" % (value,print_file_name))
+                ret = ord('Q')
+            else:
+                unit_test_results.append("Pass:PRINTTEST:found '%s'" % (value))
+    # Return the action (CHAR_NUL for no action til next round)
+    return(ret)
+
+# Unit test without an interative terminal (e.g. ptest)
+unit_test_noterm = os.environ.get('TASK_EXP_UNIT_TEST_NOTERM')
+
+
+#################################################
+### Window frame rendering
+###
+### By default, use the normal line art. Since
+###  these extended characters are not ASCII, one
+###  must use the ncursus API to render them
+### The alternate ASCII line art set is optionally
+###  available via the 'do_line_art' flag
+
+# By default, render frames using line art
+do_line_art = True
+
+# ASCII render set option
+CHAR_HBAR = '-'
+CHAR_VBAR = '|'
+CHAR_UL_CORNER = '/'
+CHAR_UR_CORNER = '\\'
+CHAR_LL_CORNER = '\\'
+CHAR_LR_CORNER = '/'
+
+# Box frame drawing with line-art
+def line_art_frame(box):
+    x = box.base_x
+    y = box.base_y
+    w = box.width
+    h = box.height + 1
+
+    if do_line_art:
+        for i in range(1, w - 1):
+            box.screen.addch(y, x + i, curses.ACS_HLINE, box.color)
+            box.screen.addch(y + h - 1, x + i, curses.ACS_HLINE, box.color)
+        body_line = "%s" % (' ' * (w - 2))
+        for i in range(1, h - 1):
+            box.screen.addch(y + i, x, curses.ACS_VLINE, box.color)
+            box.screen.addstr(y + i, x + 1, body_line, box.color)
+            box.screen.addch(y + i, x + w - 1, curses.ACS_VLINE, box.color)
+        box.screen.addch(y, x, curses.ACS_ULCORNER, box.color)
+        box.screen.addch(y, x + w - 1, curses.ACS_URCORNER, box.color)
+        box.screen.addch(y + h - 1, x, curses.ACS_LLCORNER, box.color)
+        box.screen.addch(y + h - 1, x + w - 1, curses.ACS_LRCORNER, box.color)
+    else:
+        top_line  = "%s%s%s" % (CHAR_UL_CORNER,CHAR_HBAR * (w - 2),CHAR_UR_CORNER)
+        body_line = "%s%s%s" % (CHAR_VBAR,' ' * (w - 2),CHAR_VBAR)
+        bot_line  = "%s%s%s" % (CHAR_UR_CORNER,CHAR_HBAR * (w - 2),CHAR_UL_CORNER)
+        tag_line  = "%s%s%s" % ('[',CHAR_HBAR * (w - 2),']')
+        # Top bar
+        box.screen.addstr(y, x, top_line)
+        # Middle frame
+        for i in range(1, (h - 1)):
+            box.screen.addstr(y+i, x, body_line)
+        # Bottom bar
+        box.screen.addstr(y + (h - 1), x, bot_line)
+
+# Connect the separate boxes
+def line_art_fixup(box):
+    if do_line_art:
+        box.screen.addch(box.base_y+2, box.base_x, curses.ACS_LTEE, box.color)
+        box.screen.addch(box.base_y+2, box.base_x+box.width-1, curses.ACS_RTEE, box.color)
+
+
+#################################################
+### Ncurses box object : box frame object to display
+### and manage a sub-window's display elements
+### using basic ncurses
+###
+### Supports:
+###   * Frame drawing, content (re)drawing
+###   * Content scrolling via ArrowUp, ArrowDn, PgUp, PgDN,
+###   * Highlighting for active selected item
+###   * Content sorting based on selected sort model
+###
+
+class NBox():
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        # Box description
+        self.screen = screen
+        self.label = label
+        self.primary = primary
+        self.color = curses.color_pair(CURSES_NORMAL) if screen else None
+        # Box boundaries
+        self.base_x = base_x
+        self.base_y = base_y
+        self.width = width
+        self.height = height
+        # Cursor/scroll management
+        self.cursor_enable = False
+        self.cursor_index = 0   # Absolute offset
+        self.cursor_offset = 0  # Frame centric offset
+        self.scroll_offset = 0  # Frame centric offset
+        # Box specific content
+        # Format of each entry is [package_name,is_primary_recipe,alpha_sort_key,deps_sort_key]
+        self.task_list = []
+
+    @property
+    def inside_width(self):
+        return(self.width-2)
+
+    @property
+    def inside_height(self):
+        return(self.height-2)
+
+    # Populate the box's content, include the sort mappings and is_primary flag
+    def task_list_append(self,task_name,dep):
+        task_sort_alpha = task_name
+        task_sort_deps = dep.get_dep_sort(task_name)
+        is_primary = False
+        for primary in self.primary:
+            if task_name.startswith(primary+'.'):
+                is_primary = True
+        if SORT_BITBAKE_ENABLE:
+            task_sort_bitbake = dep.get_bb_sort(task_name)
+            self.task_list.append([task_name,is_primary,task_sort_alpha,task_sort_deps,task_sort_bitbake])
+        else:
+            self.task_list.append([task_name,is_primary,task_sort_alpha,task_sort_deps])
+
+    def reset(self):
+        self.task_list = []
+        self.cursor_index = 0   # Absolute offset
+        self.cursor_offset = 0  # Frame centric offset
+        self.scroll_offset = 0  # Frame centric offset
+
+    # Sort the box's content based on the current sort model
+    def sort(self):
+        if SORT_ALPHA == sort_model:
+            self.task_list.sort(key = lambda x: x[TASK_SORT_ALPHA])
+        elif SORT_DEPS == sort_model:
+            self.task_list.sort(key = lambda x: x[TASK_SORT_DEPS])
+        elif SORT_BITBAKE == sort_model:
+            self.task_list.sort(key = lambda x: x[TASK_SORT_BITBAKE])
+
+    # The target package list (to hightlight), from the command line
+    def set_primary(self,primary):
+        self.primary = primary
+
+    # Draw the box's outside frame
+    def draw_frame(self):
+        line_art_frame(self)
+        # Title
+        self.screen.addstr(self.base_y,
+            (self.base_x + (self.width//2))-((len(self.label)+2)//2),
+            '['+self.label+']')
+        self.screen.refresh()
+
+    # Draw the box's inside text content
+    def redraw(self):
+        task_list_len = len(self.task_list)
+        # Middle frame
+        body_line = "%s" % (' ' * (self.inside_width-1) )
+        for i in range(0,self.inside_height+1):
+            if i < (task_list_len + self.scroll_offset):
+                str_ctl = "%%-%ss" % (self.width-3)
+                # Safety assert
+                if (i + self.scroll_offset) >= task_list_len:
+                    alert("REDRAW:%2d,%4d,%4d" % (i,self.scroll_offset,task_list_len),self.screen)
+                    break
+
+                task_obj = self.task_list[i + self.scroll_offset]
+                task = task_obj[TASK_NAME][:self.inside_width-1]
+                task_primary = task_obj[TASK_PRIMARY]
+
+                if task_primary:
+                    line = str_ctl % task[:self.inside_width-1]
+                    self.screen.addstr(self.base_y+1+i, self.base_x+2, line, curses.A_BOLD)
+                else:
+                    line = str_ctl % task[:self.inside_width-1]
+                    self.screen.addstr(self.base_y+1+i, self.base_x+2, line)
+            else:
+                line = "%s" % (' ' * (self.inside_width-1) )
+                self.screen.addstr(self.base_y+1+i, self.base_x+2, line)
+        self.screen.refresh()
+
+    # Show the current selected task over the bottom of the frame
+    def show_selected(self,selected_task):
+        if not selected_task:
+            selected_task = self.get_selected()
+        tag_line   = "%s%s%s" % ('[',CHAR_HBAR * (self.width-2),']')
+        self.screen.addstr(self.base_y + self.height, self.base_x, tag_line)
+        self.screen.addstr(self.base_y + self.height,
+            (self.base_x + (self.width//2))-((len(selected_task)+2)//2),
+            '['+selected_task+']')
+        self.screen.refresh()
+
+    # Load box with new table of content
+    def update_content(self,task_list):
+        self.task_list = task_list
+        if self.cursor_enable:
+            cursor_update(turn_on=False)
+        self.cursor_index = 0
+        self.cursor_offset = 0
+        self.scroll_offset = 0
+        self.redraw()
+        if self.cursor_enable:
+            cursor_update(turn_on=True)
+
+    # Manage the box's highlighted task and blinking cursor character
+    def cursor_on(self,is_on):
+        self.cursor_enable = is_on
+        self.cursor_update(is_on)
+
+    # High-light the current pointed package, normal for released packages
+    def cursor_update(self,turn_on=True):
+        str_ctl = "%%-%ss" % (self.inside_width-1)
+        try:
+            if len(self.task_list):
+                task_obj = self.task_list[self.cursor_index]
+                task = task_obj[TASK_NAME][:self.inside_width-1]
+                task_primary = task_obj[TASK_PRIMARY]
+                task_font = curses.A_BOLD if task_primary else 0
+            else:
+                task = ''
+                task_font = 0
+        except Exception as e:
+            alert("CURSOR_UPDATE:%s" % (e),self.screen)
+            return
+        if turn_on:
+            self.screen.addstr(self.base_y+1+self.cursor_offset,self.base_x+1,">", curses.color_pair(CURSES_HIGHLIGHT) | curses.A_BLINK)
+            self.screen.addstr(self.base_y+1+self.cursor_offset,self.base_x+2,str_ctl % task, curses.color_pair(CURSES_HIGHLIGHT) | task_font)
+        else:
+            self.screen.addstr(self.base_y+1+self.cursor_offset,self.base_x+1," ")
+            self.screen.addstr(self.base_y+1+self.cursor_offset,self.base_x+2,str_ctl % task, task_font)
+
+    # Down arrow
+    def line_down(self):
+        if len(self.task_list) <= (self.cursor_index+1):
+            return
+        self.cursor_update(turn_on=False)
+        self.cursor_index += 1
+        self.cursor_offset += 1
+        if self.cursor_offset > (self.inside_height):
+            self.cursor_offset -= 1
+            self.scroll_offset += 1
+            self.redraw()
+        self.cursor_update(turn_on=True)
+        debug_frame(self)
+
+    # Up arrow
+    def line_up(self):
+        if 0 > (self.cursor_index-1):
+            return
+        self.cursor_update(turn_on=False)
+        self.cursor_index -= 1
+        self.cursor_offset -= 1
+        if self.cursor_offset < 0:
+            self.cursor_offset += 1
+            self.scroll_offset -= 1
+            self.redraw()
+        self.cursor_update(turn_on=True)
+        debug_frame(self)
+
+    # Page down
+    def page_down(self):
+        max_task = len(self.task_list)-1
+        if max_task < self.inside_height:
+            return
+        self.cursor_update(turn_on=False)
+        self.cursor_index += 10
+        self.cursor_index = min(self.cursor_index,max_task)
+        self.cursor_offset = min(self.inside_height,self.cursor_index)
+        self.scroll_offset = self.cursor_index - self.cursor_offset
+        self.redraw()
+        self.cursor_update(turn_on=True)
+        debug_frame(self)
+
+    # Page up
+    def page_up(self):
+        max_task = len(self.task_list)-1
+        if max_task < self.inside_height:
+            return
+        self.cursor_update(turn_on=False)
+        self.cursor_index -= 10
+        self.cursor_index = max(self.cursor_index,0)
+        self.cursor_offset = max(0, self.inside_height - (max_task - self.cursor_index))
+        self.scroll_offset = self.cursor_index - self.cursor_offset
+        self.redraw()
+        self.cursor_update(turn_on=True)
+        debug_frame(self)
+
+    # Return the currently selected task name for this box
+    def get_selected(self):
+        if self.task_list:
+            return(self.task_list[self.cursor_index][TASK_NAME])
+        else:
+            return('')
+
+#################################################
+### The helper sub-windows
+###
+
+# Show persistent help at the top of the screen
+class HelpBarView(NBox):
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        super(HelpBarView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+
+    def show_help(self,show):
+        self.screen.addstr(self.base_y,self.base_x, "%s" % (' ' * self.inside_width))
+        if show:
+            help = "Help='?' Filter='/' NextBox=<Tab> Select=<Enter> Print='p','P' Quit='q'"
+            bar_size = self.inside_width - 5 - len(help)
+            self.screen.addstr(self.base_y,self.base_x+((self.inside_width-len(help))//2), help)
+        self.screen.refresh()
+
+# Pop up a detailed Help box
+class HelpBoxView(NBox):
+    def __init__(self, screen, label, primary, base_x, base_y, width, height, dep):
+        super(HelpBoxView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+        self.x_pos = 0
+        self.y_pos = 0
+        self.dep = dep
+
+    # Instantial the pop-up help box
+    def show_help(self,show):
+        self.x_pos = self.base_x + 4
+        self.y_pos = self.base_y + 2
+
+        def add_line(line):
+            if line:
+                self.screen.addstr(self.y_pos,self.x_pos,line)
+            self.y_pos += 1
+
+        # Gather some statisics
+        dep_count = 0
+        rdep_count = 0
+        for task_obj in self.dep.depends_model:
+            if TYPE_DEP == task_obj[DEPENDS_TYPE]:
+                dep_count += 1
+            elif TYPE_RDEP == task_obj[DEPENDS_TYPE]:
+                rdep_count += 1
+
+        self.draw_frame()
+        line_art_fixup(self.dep)
+        add_line("Quit                : 'q' ")
+        add_line("Filter task names   : '/'")
+        add_line("Tab to next box     : <Tab>")
+        add_line("Select a task       : <Enter>")
+        add_line("Print task's deps   : 'p'")
+        add_line("Print recipe's deps : 'P'")
+        add_line(" -> '%s'" % print_file_name)
+        add_line("Sort toggle         : 's'")
+        add_line(" %s Recipe inner-depends order" % ('->' if (SORT_DEPS == sort_model) else '- '))
+        add_line(" %s Alpha-numeric order" % ('->' if (SORT_ALPHA == sort_model) else '- '))
+        if SORT_BITBAKE_ENABLE:
+            add_line(" %s Bitbake order" % ('->' if (TASK_SORT_BITBAKE == sort_model) else '- '))
+        add_line("Alternate backspace : <CTRL-H>")
+        add_line("")
+        add_line("Primary recipes = %s"  % ','.join(self.primary))
+        add_line("Task  count     = %4d" % len(self.dep.pkg_model))
+        add_line("Deps  count     = %4d" % dep_count)
+        add_line("RDeps count     = %4d" % rdep_count)
+        add_line("")
+        self.screen.addstr(self.y_pos,self.x_pos+7,"<Press any key>", curses.color_pair(CURSES_HIGHLIGHT))
+        self.screen.refresh()
+        c = self.screen.getch()
+
+# Show a progress bar
+class ProgressView(NBox):
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        super(ProgressView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+
+    def progress(self,title,current,max):
+        if title:
+            self.label = title
+        else:
+            title = self.label
+        if max <=0: max = 10
+        bar_size = self.width - 7 - len(title)
+        bar_done = int( (float(current)/float(max)) * float(bar_size) )
+        self.screen.addstr(self.base_y,self.base_x, " %s:[%s%s]" % (title,'*' * bar_done,' ' * (bar_size-bar_done)))
+        self.screen.refresh()
+        return(current+1)
+
+    def clear(self):
+        self.screen.addstr(self.base_y,self.base_x, "%s" % (' ' * self.width))
+        self.screen.refresh()
+
+# Implement a task filter bar
+class FilterView(NBox):
+    SEARCH_NOP = 0
+    SEARCH_GO = 1
+    SEARCH_CANCEL = 2
+
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        super(FilterView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+        self.do_show = False
+        self.filter_str = ""
+
+    def clear(self,enable_show=True):
+        self.filter_str = ""
+
+    def show(self,enable_show=True):
+        self.do_show = enable_show
+        if self.do_show:
+            self.screen.addstr(self.base_y,self.base_x, "[ Filter: %-25s ]      '/'=cancel, format='abc'       " % self.filter_str[0:25])
+        else:
+            self.screen.addstr(self.base_y,self.base_x, "%s" % (' ' * self.width))
+        self.screen.refresh()
+
+    def show_prompt(self):
+        self.screen.addstr(self.base_y,self.base_x + 10 + len(self.filter_str), " ")
+        self.screen.addstr(self.base_y,self.base_x + 10 + len(self.filter_str), "")
+
+    # Keys specific to the filter box (start/stop filter keys are in the main loop)
+    def input(self,c,ch):
+        ret = self.SEARCH_GO
+        if c in (curses.KEY_BACKSPACE,CHAR_BS_H):
+            #  Backspace
+            if self.filter_str:
+                self.filter_str = self.filter_str[0:-1]
+                self.show()
+        elif ((ch >= 'a') and (ch <= 'z')) or ((ch >= 'A') and (ch <= 'Z')) or ((ch >= '0') and (ch <= '9')) or (ch in (' ','_','.','-')):
+            # The isalnum() acts strangly with keypad(True), so explicit bounds
+            self.filter_str += ch
+            self.show()
+        else:
+            ret = self.SEARCH_NOP
+        return(ret)
+
+
+#################################################
+### The primary dependency windows
+###
+
+# The main list of package tasks
+class PackageView(NBox):
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        super(PackageView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+
+    # Find and verticaly center a selected task (from filter or from dependent box)
+    # The 'task_filter_str' can be a full or a partial (filter) task name
+    def find(self,task_filter_str):
+        found = False
+        max = self.height-2
+        if not task_filter_str:
+            return(found)
+        for i,task_obj in enumerate(self.task_list):
+            task = task_obj[TASK_NAME]
+            if task.startswith(task_filter_str):
+                self.cursor_on(False)
+                self.cursor_index = i
+
+                # Position selected at vertical center
+                vcenter = self.inside_height // 2
+                if self.cursor_index <= vcenter:
+                    self.scroll_offset = 0
+                    self.cursor_offset = self.cursor_index
+                elif self.cursor_index >= (len(self.task_list) - vcenter - 1):
+                    self.cursor_offset = self.inside_height-1
+                    self.scroll_offset = self.cursor_index - self.cursor_offset
+                else:
+                    self.cursor_offset = vcenter
+                    self.scroll_offset = self.cursor_index - self.cursor_offset
+
+                self.redraw()
+                self.cursor_on(True)
+                found = True
+                break
+        return(found)
+
+# The view of dependent packages
+class PackageDepView(NBox):
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        super(PackageDepView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+
+# The view of reverse-dependent packages
+class PackageReverseDepView(NBox):
+    def __init__(self, screen, label, primary, base_x, base_y, width, height):
+        super(PackageReverseDepView, self).__init__(screen, label, primary, base_x, base_y, width, height)
+
+
+#################################################
+### DepExplorer : The parent frame and object
+###
+
+class DepExplorer(NBox):
+    def __init__(self,screen):
+        title = "Task Dependency Explorer"
+        super(DepExplorer, self).__init__(screen, 'Task Dependency Explorer','',0,0,80,23)
+
+        self.screen = screen
+        self.pkg_model = []
+        self.depends_model = []
+        self.dep_sort_map = {}
+        self.bb_sort_map = {}
+        self.filter_str = ''
+        self.filter_prev = 'deadbeef'
+
+        if self.screen:
+            self.help_bar_view = HelpBarView(screen, "Help",'',1,1,79,1)
+            self.help_box_view = HelpBoxView(screen, "Help",'',0,2,40,20,self)
+            self.progress_view = ProgressView(screen, "Progress",'',2,1,76,1)
+            self.filter_view = FilterView(screen, "Filter",'',2,1,76,1)
+            self.package_view = PackageView(screen, "Package",'alpha', 0,2,40,20)
+            self.dep_view = PackageDepView(screen, "Dependencies",'beta',40,2,40,10)
+            self.reverse_view = PackageReverseDepView(screen, "Dependent Tasks",'gamma',40,13,40,9)
+            self.draw_frames()
+
+    # Draw this main window's frame and all sub-windows
+    def draw_frames(self):
+        self.draw_frame()
+        self.package_view.draw_frame()
+        self.dep_view.draw_frame()
+        self.reverse_view.draw_frame()
+        if is_filter:
+            self.filter_view.show(True)
+            self.filter_view.show_prompt()
+        else:
+            self.help_bar_view.show_help(True)
+        self.package_view.redraw()
+        self.dep_view.redraw()
+        self.reverse_view.redraw()
+        self.show_selected(self.package_view.get_selected())
+        line_art_fixup(self)
+
+    # Parse the bitbake dependency event object
+    def parse(self, depgraph):
+        for task in depgraph["tdepends"]:
+            self.pkg_model.insert(0, task)
+            for depend in depgraph["tdepends"][task]:
+                self.depends_model.insert (0, (TYPE_DEP, task, depend))
+                self.depends_model.insert (0, (TYPE_RDEP, depend, task))
+        if self.screen:
+            self.dep_sort_prep()
+
+    # Prepare the dependency sort order keys
+    # This method creates sort keys per recipe tasks in
+    # the order of each recipe's internal dependecies
+    # Method:
+    #   Filter the tasks in dep order in dep_sort_map = {}
+    #   (a) Find a task that has no dependecies
+    #       Ignore non-recipe specific tasks
+    #   (b) Add it to the sort mapping dict with
+    #       key of "<task_group>_<order>"
+    #   (c) Remove it as a dependency from the other tasks
+    #   (d) Repeat till all tasks are mapped
+    # Use placeholders to insure each sub-dict is instantiated
+    def dep_sort_prep(self):
+        self.progress_view.progress('DepSort',0,4)
+        # Init the task base entries
+        self.progress_view.progress('DepSort',1,4)
+        dep_table = {}
+        bb_index = 0
+        for task in self.pkg_model:
+            # First define the incoming bitbake sort order
+            self.bb_sort_map[task] = "%04d" % (bb_index)
+            bb_index += 1
+            task_group = task[0:task.find('.')]
+            if task_group not in dep_table:
+                dep_table[task_group] = {}
+                dep_table[task_group]['-'] = {}         # Placeholder
+            if task not in dep_table[task_group]:
+                dep_table[task_group][task] = {}
+                dep_table[task_group][task]['-'] = {}   # Placeholder
+        # Add the task dependecy entries
+        self.progress_view.progress('DepSort',2,4)
+        for task_obj in self.depends_model:
+            if task_obj[DEPENDS_TYPE] != TYPE_DEP:
+                continue
+            task = task_obj[DEPENDS_TASK]
+            task_dep = task_obj[DEPENDS_DEPS]
+            task_group = task[0:task.find('.')]
+            # Only track depends within same group
+            if task_dep.startswith(task_group+'.'):
+                dep_table[task_group][task][task_dep] = 1
+        self.progress_view.progress('DepSort',3,4)
+        for task_group in dep_table:
+            dep_index = 0
+            # Whittle down the tasks of each group
+            this_pass = 1
+            do_loop = True
+            while (len(dep_table[task_group]) > 1) and do_loop:
+                this_pass += 1
+                is_change = False
+                delete_list = []
+                for task in dep_table[task_group]:
+                    if '-' == task:
+                        continue
+                    if 1 == len(dep_table[task_group][task]):
+                        is_change = True
+                        # No more deps, so collect this task...
+                        self.dep_sort_map[task] = "%s_%04d" % (task_group,dep_index)
+                        dep_index += 1
+                        # ... remove it from other lists as resolved ...
+                        for dep_task in dep_table[task_group]:
+                            if task in dep_table[task_group][dep_task]:
+                                del dep_table[task_group][dep_task][task]
+                        # ... and remove it from from the task group
+                        delete_list.append(task)
+                for task in delete_list:
+                    del dep_table[task_group][task]
+                if not is_change:
+                    alert("ERROR:DEP_SIEVE_NO_CHANGE:%s" % task_group,self.screen)
+                    do_loop = False
+                    continue
+        self.progress_view.progress('',4,4)
+        self.progress_view.clear()
+        self.help_bar_view.show_help(True)
+        if len(self.dep_sort_map) != len(self.pkg_model):
+            alert("ErrorDepSort:%d/%d" % (len(self.dep_sort_map),len(self.pkg_model)),self.screen)
+
+    # Look up a dep sort order key
+    def get_dep_sort(self,key):
+        if key in self.dep_sort_map:
+            return(self.dep_sort_map[key])
+        else:
+            return(key)
+
+    # Look up a bitbake sort order key
+    def get_bb_sort(self,key):
+        if key in self.bb_sort_map:
+            return(self.bb_sort_map[key])
+        else:
+            return(key)
+
+    # Find the selected package in the main frame, update the dependency frames content accordingly
+    def select(self, package_name, only_update_dependents=False):
+        if not package_name:
+            package_name = self.package_view.get_selected()
+        # alert("SELECT:%s:" % package_name,self.screen)
+
+        if self.filter_str != self.filter_prev:
+            self.package_view.cursor_on(False)
+            # Fill of the main package task list using new filter
+            self.package_view.task_list = []
+            for package in self.pkg_model:
+                if self.filter_str:
+                    if self.filter_str in package:
+                        self.package_view.task_list_append(package,self)
+                else:
+                    self.package_view.task_list_append(package,self)
+            self.package_view.sort()
+            self.filter_prev = self.filter_str
+
+            # Old position is lost, assert new position of previous task (if still filtered in)
+            self.package_view.cursor_index = 0
+            self.package_view.cursor_offset = 0
+            self.package_view.scroll_offset = 0
+            self.package_view.redraw()
+            self.package_view.cursor_on(True)
+
+        # Make sure the selected package is in view, with implicit redraw()
+        if (not only_update_dependents):
+            self.package_view.find(package_name)
+        # In case selected name change (i.e. filter removed previous)
+        package_name = self.package_view.get_selected()
+
+        # Filter the package's dependent list to the dependent view
+        self.dep_view.reset()
+        for package_def in self.depends_model:
+            if (package_def[DEPENDS_TYPE] == TYPE_DEP) and (package_def[DEPENDS_TASK] == package_name):
+                self.dep_view.task_list_append(package_def[DEPENDS_DEPS],self)
+        self.dep_view.sort()
+        self.dep_view.redraw()
+        # Filter the package's dependent list to the reverse dependent view
+        self.reverse_view.reset()
+        for package_def in self.depends_model:
+            if (package_def[DEPENDS_TYPE] == TYPE_RDEP) and (package_def[DEPENDS_TASK] == package_name):
+                self.reverse_view.task_list_append(package_def[DEPENDS_DEPS],self)
+        self.reverse_view.sort()
+        self.reverse_view.redraw()
+        self.show_selected(package_name)
+        self.screen.refresh()
+
+    # The print-to-file method
+    def print_deps(self,whole_group=False):
+        global is_printed
+        # Print the selected deptree(s) to a file
+        if not is_printed:
+            try:
+                # Move to backup any exiting file before first write
+                if os.path.isfile(print_file_name):
+                    os.system('mv -f %s %s' % (print_file_name,print_file_backup_name))
+            except Exception as e:
+                alert(e,self.screen)
+                alert('',self.screen)
+        print_list = []
+        selected_task = self.package_view.get_selected()
+        if not selected_task:
+            return
+        if not whole_group:
+            print_list.append(selected_task)
+        else:
+            # Use the presorted task_group order from 'package_view'
+            task_group = selected_task[0:selected_task.find('.')+1]
+            for task_obj in self.package_view.task_list:
+                task = task_obj[TASK_NAME]
+                if task.startswith(task_group):
+                    print_list.append(task)
+        with open(print_file_name, "a") as fd:
+            print_max = len(print_list)
+            print_count = 1
+            self.progress_view.progress('Write "%s"' % print_file_name,0,print_max)
+            for task in print_list:
+                print_count = self.progress_view.progress('',print_count,print_max)
+                self.select(task)
+                self.screen.refresh();
+                # Utilize the current print output model
+                if print_model == PRINT_MODEL_1:
+                    print("=== Dependendency Snapshot ===",file=fd)
+                    print(" = Package =",file=fd)
+                    print('   '+task,file=fd)
+                    # Fill in the matching dependencies
+                    print(" = Dependencies =",file=fd)
+                    for task_obj in self.dep_view.task_list:
+                        print('   '+ task_obj[TASK_NAME],file=fd)
+                    print(" = Dependent Tasks =",file=fd)
+                    for task_obj in self.reverse_view.task_list:
+                        print('   '+ task_obj[TASK_NAME],file=fd)
+                if print_model == PRINT_MODEL_2:
+                    print("=== Dependendency Snapshot ===",file=fd)
+                    dep_count = len(self.dep_view.task_list) - 1
+                    for i,task_obj in enumerate(self.dep_view.task_list):
+                        print('%s%s' % ("Dep    =" if (i==dep_count) else "        ",task_obj[TASK_NAME]),file=fd)
+                    if not self.dep_view.task_list:
+                        print('Dep    =',file=fd)
+                    print("Package=%s" % task,file=fd)
+                    for i,task_obj in enumerate(self.reverse_view.task_list):
+                        print('%s%s' % ("RDep   =" if (i==0) else "        ",task_obj[TASK_NAME]),file=fd)
+                    if not self.reverse_view.task_list:
+                        print('RDep   =',file=fd)
+                curses.napms(2000)
+                self.progress_view.clear()
+                self.help_bar_view.show_help(True)
+            print('',file=fd)
+        # Restore display to original selected task
+        self.select(selected_task)
+        is_printed = True
+
+#################################################
+### Load bitbake data
+###
+
+def bitbake_load(server, eventHandler, params, dep, curses_off, screen):
+    global bar_len_old
+    bar_len_old = 0
+
+    # Support no screen
+    def progress(msg,count,max):
+        global bar_len_old
+        if screen:
+            dep.progress_view.progress(msg,count,max)
+        else:
+            if msg:
+                if bar_len_old:
+                    bar_len_old = 0
+                    print("\n")
+                print(f"{msg}: ({count} of {max})")
+            else:
+                bar_len = int((count*40)/max)
+                if bar_len_old != bar_len:
+                    print(f"{'*' * (bar_len-bar_len_old)}",end='',flush=True)
+                bar_len_old = bar_len
+    def clear():
+        if screen:
+            dep.progress_view.clear()
+    def clear_curses(screen):
+        if screen:
+            curses_off(screen)
+
+    #
+    # Trigger bitbake "generateDepTreeEvent"
+    #
+
+    cmdline = ''
+    try:
+        params.updateToServer(server, os.environ.copy())
+        params.updateFromServer(server)
+        cmdline = params.parseActions()
+        if not cmdline:
+            clear_curses(screen)
+            print("ERROR: nothing to do.  Use 'bitbake world' to build everything, or run 'bitbake --help' for usage information.")
+            return 1,cmdline
+        if 'msg' in cmdline and cmdline['msg']:
+            clear_curses(screen)
+            print('ERROR: ' + cmdline['msg'])
+            return 1,cmdline
+        cmdline = cmdline['action']
+        if not cmdline or cmdline[0] != "generateDotGraph":
+            clear_curses(screen)
+            print("ERROR: This UI requires the -g option")
+            return 1,cmdline
+        ret, error = server.runCommand(["generateDepTreeEvent", cmdline[1], cmdline[2]])
+        if error:
+            clear_curses(screen)
+            print("ERROR: running command '%s': %s" % (cmdline, error))
+            return 1,cmdline
+        elif not ret:
+            clear_curses(screen)
+            print("ERROR: running command '%s': returned %s" % (cmdline, ret))
+            return 1,cmdline
+    except client.Fault as x:
+        clear_curses(screen)
+        print("ERROR: XMLRPC Fault getting commandline:\n %s" % x)
+        return 1,cmdline
+    except Exception as e:
+        clear_curses(screen)
+        print("ERROR: in startup:\n %s" % traceback.format_exc())
+        return 1,cmdline
+
+    #
+    # Receive data from bitbake
+    #
+
+    progress_total = 0
+    load_bitbake = True
+    quit = False
+    try:
+        while load_bitbake:
+            try:
+                event = eventHandler.waitEvent(0.25)
+                if quit:
+                    _, error = server.runCommand(["stateForceShutdown"])
+                    clear_curses(screen)
+                    if error:
+                        print('Unable to cleanly stop: %s' % error)
+                    break
+
+                if event is None:
+                    continue
+
+                if isinstance(event, bb.event.CacheLoadStarted):
+                    progress_total = event.total
+                    progress('Loading Cache',0,progress_total)
+                    continue
+
+                if isinstance(event, bb.event.CacheLoadProgress):
+                    x = event.current
+                    progress('',x,progress_total)
+                    continue
+
+                if isinstance(event, bb.event.CacheLoadCompleted):
+                    clear()
+                    progress('Bitbake... ',1,2)
+                    continue
+
+                if isinstance(event, bb.event.ParseStarted):
+                    progress_total = event.total
+                    progress('Processing recipes',0,progress_total)
+                    if progress_total == 0:
+                        continue
+
+                if isinstance(event, bb.event.ParseProgress):
+                    x = event.current
+                    progress('',x,progress_total)
+                    continue
+
+                if isinstance(event, bb.event.ParseCompleted):
+                    progress('Generating dependency tree',0,3)
+                    continue
+
+                if isinstance(event, bb.event.DepTreeGenerated):
+                    progress('Generating dependency tree',1,3)
+                    dep.parse(event._depgraph)
+                    progress('Generating dependency tree',2,3)
+
+                if isinstance(event, bb.command.CommandCompleted):
+                    load_bitbake = False
+                    progress('Generating dependency tree',3,3)
+                    clear()
+                    if screen:
+                        dep.help_bar_view.show_help(True)
+                    continue
+
+                if isinstance(event, bb.event.NoProvider):
+                    clear_curses(screen)
+                    print('ERROR: %s' % event)
+
+                    _, error = server.runCommand(["stateShutdown"])
+                    if error:
+                        print('ERROR: Unable to cleanly shutdown: %s' % error)
+                    return 1,cmdline
+
+                if isinstance(event, bb.command.CommandFailed):
+                    clear_curses(screen)
+                    print('ERROR: ' + str(event))
+                    return event.exitcode,cmdline
+
+                if isinstance(event, bb.command.CommandExit):
+                    clear_curses(screen)
+                    return event.exitcode,cmdline
+
+                if isinstance(event, bb.cooker.CookerExit):
+                    break
+
+                continue
+            except EnvironmentError as ioerror:
+                # ignore interrupted io
+                if ioerror.args[0] == 4:
+                    pass
+            except KeyboardInterrupt:
+                if shutdown == 2:
+                    clear_curses(screen)
+                    print("\nThird Keyboard Interrupt, exit.\n")
+                    break
+                if shutdown == 1:
+                    clear_curses(screen)
+                    print("\nSecond Keyboard Interrupt, stopping...\n")
+                    _, error = server.runCommand(["stateForceShutdown"])
+                    if error:
+                        print('Unable to cleanly stop: %s' % error)
+                if shutdown == 0:
+                    clear_curses(screen)
+                    print("\nKeyboard Interrupt, closing down...\n")
+                    _, error = server.runCommand(["stateShutdown"])
+                    if error:
+                        print('Unable to cleanly shutdown: %s' % error)
+                shutdown = shutdown + 1
+                pass
+    except Exception as e:
+        # Safe exit on error
+        clear_curses(screen)
+        print("Exception : %s" % e)
+        print("Exception in startup:\n %s" % traceback.format_exc())
+
+    return 0,cmdline
+
+#################################################
+### main
+###
+
+SCREEN_COL_MIN = 83
+SCREEN_ROW_MIN = 26
+
+def main(server, eventHandler, params):
+    global verbose
+    global sort_model
+    global print_model
+    global is_printed
+    global is_filter
+    global screen_too_small
+
+    shutdown = 0
+    screen_too_small = False
+    quit = False
+
+    # Unit test with no terminal?
+    if unit_test_noterm:
+        # Load bitbake, test that there is valid dependency data, then exit
+        screen = None
+        print("* UNIT TEST:START")
+        dep = DepExplorer(screen)
+        print("* UNIT TEST:BITBAKE FETCH")
+        ret,cmdline = bitbake_load(server, eventHandler, params, dep, None, screen)
+        if ret:
+            print("* UNIT TEST: BITBAKE FAILED")
+            return ret
+        # Test the acquired dependency data
+        quilt_native_deps = 0
+        quilt_native_rdeps = 0
+        quilt_deps = 0
+        quilt_rdeps = 0
+        for i,task_obj in enumerate(dep.depends_model):
+            if TYPE_DEP == task_obj[0]:
+                task = task_obj[1]
+                if task.startswith('quilt-native'):
+                    quilt_native_deps += 1
+                elif task.startswith('quilt'):
+                    quilt_deps += 1
+            elif TYPE_RDEP == task_obj[0]:
+                task = task_obj[1]
+                if task.startswith('quilt-native'):
+                    quilt_native_rdeps += 1
+                elif task.startswith('quilt'):
+                    quilt_rdeps += 1
+        # Print results
+        failed = False
+        if 0 < len(dep.depends_model):
+             print(f"Pass:Bitbake dependency count = {len(dep.depends_model)}")
+        else:
+            failed = True
+            print(f"FAIL:Bitbake dependency count = 0")
+        if quilt_native_deps:
+             print(f"Pass:Quilt-native depends count = {quilt_native_deps}")
+        else:
+            failed = True
+            print(f"FAIL:Quilt-native depends count = 0")
+        if quilt_native_rdeps:
+             print(f"Pass:Quilt-native rdepends count = {quilt_native_rdeps}")
+        else:
+            failed = True
+            print(f"FAIL:Quilt-native rdepends count = 0")
+        if quilt_deps:
+             print(f"Pass:Quilt depends count = {quilt_deps}")
+        else:
+            failed = True
+            print(f"FAIL:Quilt depends count = 0")
+        if quilt_rdeps:
+             print(f"Pass:Quilt rdepends count = {quilt_rdeps}")
+        else:
+            failed = True
+            print(f"FAIL:Quilt rdepends count = 0")
+        print("* UNIT TEST:STOP")
+        return failed
+
+    # Help method to dynamically test parent window too small
+    def check_screen_size(dep, active_package):
+        global screen_too_small
+        rows, cols = screen.getmaxyx()
+        if (rows >= SCREEN_ROW_MIN) and (cols >= SCREEN_COL_MIN):
+            if screen_too_small:
+                # Now big enough, remove error message and redraw screen
+                dep.draw_frames()
+                active_package.cursor_on(True)
+                screen_too_small = False
+            return True
+        # Test on App init
+        if not dep:
+            # Do not start this app if screen not big enough
+            curses.endwin()
+            print("")
+            print("ERROR(Taskexp_cli): Mininal screen size is %dx%d" % (SCREEN_COL_MIN,SCREEN_ROW_MIN))
+            print("Current screen is Cols=%s,Rows=%d" % (cols,rows))
+            return False
+        # First time window too small
+        if not screen_too_small:
+            active_package.cursor_on(False)
+            dep.screen.addstr(0,2,'[BIGGER WINDOW PLEASE]', curses.color_pair(CURSES_WARNING) | curses.A_BLINK)
+            screen_too_small = True
+        return False
+
+    # Helper method to turn off curses mode
+    def curses_off(screen):
+        if not screen: return
+        # Safe error exit
+        screen.keypad(False)
+        curses.echo()
+        curses.curs_set(1)
+        curses.endwin()
+
+        if unit_test_results:
+            print('\nUnit Test Results:')
+            for line in unit_test_results:
+                print(" %s" % line)
+
+    #
+    # Initialize the ncurse environment
+    #
+
+    screen = curses.initscr()
+    try:
+        if not check_screen_size(None, None):
+            exit(1)
+        try:
+            curses.start_color()
+            curses.use_default_colors();
+            curses.init_pair(0xFF, curses.COLOR_BLACK, curses.COLOR_WHITE);
+            curses.init_pair(CURSES_NORMAL, curses.COLOR_WHITE, curses.COLOR_BLACK)
+            curses.init_pair(CURSES_HIGHLIGHT, curses.COLOR_WHITE, curses.COLOR_BLUE)
+            curses.init_pair(CURSES_WARNING, curses.COLOR_WHITE, curses.COLOR_RED)
+        except:
+            curses.endwin()
+            print("")
+            print("ERROR(Taskexp_cli): Requires 256 colors. Please use this or the equivalent:")
+            print("  $ export TERM='xterm-256color'")
+            exit(1)
+
+        screen.keypad(True)
+        curses.noecho()
+        curses.curs_set(0)
+        screen.refresh();
+    except Exception as e:
+        # Safe error exit
+        curses_off(screen)
+        print("Exception : %s" % e)
+        print("Exception in startup:\n %s" % traceback.format_exc())
+        exit(1)
+
+    try:
+        #
+        # Instantiate the presentation layers
+        #
+
+        dep = DepExplorer(screen)
+
+        #
+        # Prepare bitbake
+        #
+
+        # Fetch bitbake dependecy data
+        ret,cmdline = bitbake_load(server, eventHandler, params, dep, curses_off, screen)
+        if ret: return ret
+
+        #
+        # Preset the views
+        #
+
+        # Cmdline example = ['generateDotGraph', ['acl', 'zlib'], 'build']
+        primary_packages = cmdline[1]
+        dep.package_view.set_primary(primary_packages)
+        dep.dep_view.set_primary(primary_packages)
+        dep.reverse_view.set_primary(primary_packages)
+        dep.help_box_view.set_primary(primary_packages)
+        dep.help_bar_view.show_help(True)
+        active_package = dep.package_view
+        active_package.cursor_on(True)
+        dep.select(primary_packages[0]+'.')
+        if unit_test:
+            alert('UNIT_TEST',screen)
+
+        # Help method to start/stop the filter feature
+        def filter_mode(new_filter_status):
+            global is_filter
+            if is_filter == new_filter_status:
+                # Ignore no changes
+                return
+            if not new_filter_status:
+                # Turn off
+                curses.curs_set(0)
+                #active_package.cursor_on(False)
+                active_package = dep.package_view
+                active_package.cursor_on(True)
+                is_filter = False
+                dep.help_bar_view.show_help(True)
+                dep.filter_str = ''
+                dep.select('')
+            else:
+                # Turn on
+                curses.curs_set(1)
+                dep.help_bar_view.show_help(False)
+                dep.filter_view.clear()
+                dep.filter_view.show(True)
+                dep.filter_view.show_prompt()
+                is_filter = True
+
+        #
+        # Main user loop
+        #
+
+        while not quit:
+            if is_filter:
+                dep.filter_view.show_prompt()
+            if unit_test:
+                c = unit_test_action(active_package)
+            else:
+                c = screen.getch()
+            ch = chr(c)
+
+            # Do not draw if window now too small
+            if not check_screen_size(dep,active_package):
+                continue
+
+            if verbose:
+                if c == CHAR_RETURN:
+                    screen.addstr(0, 4, "|%3d,CR |" % (c))
+                else:
+                    screen.addstr(0, 4, "|%3d,%3s|" % (c,chr(c)))
+
+            # pre-map alternate filter close keys
+            if is_filter and (c == CHAR_ESCAPE):
+                # Alternate exit from filter
+                ch = '/'
+                c = ord(ch)
+
+            # Filter and non-filter mode command keys
+            # https://docs.python.org/3/library/curses.html
+            if c in (curses.KEY_UP,CHAR_UP):
+                active_package.line_up()
+                if active_package == dep.package_view:
+                    dep.select('',only_update_dependents=True)
+            elif c in (curses.KEY_DOWN,CHAR_DOWN):
+                active_package.line_down()
+                if active_package == dep.package_view:
+                    dep.select('',only_update_dependents=True)
+            elif curses.KEY_PPAGE == c:
+                active_package.page_up()
+                if active_package == dep.package_view:
+                    dep.select('',only_update_dependents=True)
+            elif curses.KEY_NPAGE == c:
+                active_package.page_down()
+                if active_package == dep.package_view:
+                    dep.select('',only_update_dependents=True)
+            elif CHAR_TAB == c:
+                # Tab between boxes
+                active_package.cursor_on(False)
+                if active_package == dep.package_view:
+                    active_package = dep.dep_view
+                elif active_package == dep.dep_view:
+                    active_package = dep.reverse_view
+                else:
+                    active_package = dep.package_view
+                active_package.cursor_on(True)
+            elif curses.KEY_BTAB == c:
+                # Shift-Tab reverse between boxes
+                active_package.cursor_on(False)
+                if active_package == dep.package_view:
+                    active_package = dep.reverse_view
+                elif active_package == dep.reverse_view:
+                    active_package = dep.dep_view
+                else:
+                    active_package = dep.package_view
+                active_package.cursor_on(True)
+            elif (CHAR_RETURN == c):
+                # CR to select
+                selected = active_package.get_selected()
+                if selected:
+                    active_package.cursor_on(False)
+                    active_package = dep.package_view
+                    filter_mode(False)
+                    dep.select(selected)
+                else:
+                    filter_mode(False)
+                    dep.select(primary_packages[0]+'.')
+
+            elif '/' == ch: # Enter/exit dep.filter_view
+                if is_filter:
+                    filter_mode(False)
+                else:
+                    filter_mode(True)
+            elif is_filter:
+                # If in filter mode, re-direct all these other keys to the filter box
+                result = dep.filter_view.input(c,ch)
+                dep.filter_str = dep.filter_view.filter_str
+                dep.select('')
+
+            # Non-filter mode command keys
+            elif 'p' == ch:
+                dep.print_deps(whole_group=False)
+            elif 'P' == ch:
+                dep.print_deps(whole_group=True)
+            elif 'w' == ch:
+                # Toggle the print model
+                if print_model == PRINT_MODEL_1:
+                    print_model = PRINT_MODEL_2
+                else:
+                    print_model = PRINT_MODEL_1
+            elif 's' == ch:
+                # Toggle the sort model
+                if sort_model == SORT_DEPS:
+                    sort_model = SORT_ALPHA
+                elif sort_model == SORT_ALPHA:
+                    if SORT_BITBAKE_ENABLE:
+                        sort_model = TASK_SORT_BITBAKE
+                    else:
+                        sort_model = SORT_DEPS
+                else:
+                    sort_model = SORT_DEPS
+                active_package.cursor_on(False)
+                current_task = active_package.get_selected()
+                dep.package_view.sort()
+                dep.dep_view.sort()
+                dep.reverse_view.sort()
+                active_package = dep.package_view
+                active_package.cursor_on(True)
+                dep.select(current_task)
+                # Announce the new sort model
+                alert("SORT=%s" % ("ALPHA" if (sort_model == SORT_ALPHA) else "DEPS"),screen)
+                alert('',screen)
+
+            elif 'q' == ch:
+                quit = True
+            elif ch in ('h','?'):
+                dep.help_box_view.show_help(True)
+                dep.select(active_package.get_selected())
+
+            #
+            # Debugging commands
+            #
+
+            elif 'V' == ch:
+                verbose = not verbose
+                alert('Verbose=%s' % str(verbose),screen)
+                alert('',screen)
+            elif 'R' == ch:
+                screen.refresh()
+            elif 'B' == ch:
+                # Progress bar unit test
+                dep.progress_view.progress('Test',0,40)
+                curses.napms(1000)
+                dep.progress_view.progress('',10,40)
+                curses.napms(1000)
+                dep.progress_view.progress('',20,40)
+                curses.napms(1000)
+                dep.progress_view.progress('',30,40)
+                curses.napms(1000)
+                dep.progress_view.progress('',40,40)
+                curses.napms(1000)
+                dep.progress_view.clear()
+                dep.help_bar_view.show_help(True)
+            elif 'Q' == ch:
+                # Simulated error
+                curses_off(screen)
+                print('ERROR: simulated error exit')
+                return 1
+
+        # Safe exit
+        curses_off(screen)
+    except Exception as e:
+        # Safe exit on error
+        curses_off(screen)
+        print("Exception : %s" % e)
+        print("Exception in startup:\n %s" % traceback.format_exc())
+
+    # Reminder to pick up your printed results
+    if is_printed:
+        print("")
+        print("You have output ready!")
+        print("  * Your printed dependency file is: %s" % print_file_name)
+        print("  * Your previous results  saved in: %s" % print_file_backup_name)
+        print("")
diff --git a/bitbake/lib/bb/ui/teamcity.py b/bitbake/lib/bb/ui/teamcity.py
index fca46c2874..7eeaab8d63 100644
--- a/bitbake/lib/bb/ui/teamcity.py
+++ b/bitbake/lib/bb/ui/teamcity.py
@@ -30,7 +30,6 @@ import bb.build
 import bb.command
 import bb.cooker
 import bb.event
-import bb.exceptions
 import bb.runqueue
 from bb.ui import uihelper
 
@@ -102,10 +101,6 @@ class TeamcityLogFormatter(logging.Formatter):
         details = ""
         if hasattr(record, 'bb_exc_formatted'):
             details = ''.join(record.bb_exc_formatted)
-        elif hasattr(record, 'bb_exc_info'):
-            etype, value, tb = record.bb_exc_info
-            formatted = bb.exceptions.format_exception(etype, value, tb, limit=5)
-            details = ''.join(formatted)
 
         if record.levelno in [bb.msg.BBLogFormatter.ERROR, bb.msg.BBLogFormatter.CRITICAL]:
             # ERROR gets a separate errorDetails field
diff --git a/bitbake/lib/bb/ui/toasterui.py b/bitbake/lib/bb/ui/toasterui.py
index ec5bd4f105..6bd21f1844 100644
--- a/bitbake/lib/bb/ui/toasterui.py
+++ b/bitbake/lib/bb/ui/toasterui.py
@@ -385,7 +385,7 @@ def main(server, eventHandler, params):
                     main.shutdown = 1
 
                 logger.info("ToasterUI build done, brbe: %s", brbe)
-                continue
+                break
 
             if isinstance(event, (bb.command.CommandCompleted,
                                   bb.command.CommandFailed,
diff --git a/bitbake/lib/bb/ui/uievent.py b/bitbake/lib/bb/ui/uievent.py
index 8607d0523b..c2f830d530 100644
--- a/bitbake/lib/bb/ui/uievent.py
+++ b/bitbake/lib/bb/ui/uievent.py
@@ -44,7 +44,7 @@ class BBUIEventQueue:
         for count_tries in range(5):
             ret = self.BBServer.registerEventHandler(self.host, self.port)
 
-            if isinstance(ret, collections.Iterable):
+            if isinstance(ret, collections.abc.Iterable):
                 self.EventHandle, error = ret
             else:
                 self.EventHandle = ret
@@ -65,35 +65,27 @@ class BBUIEventQueue:
         self.server = server
 
         self.t = threading.Thread()
-        self.t.setDaemon(True)
+        self.t.daemon = True
         self.t.run = self.startCallbackHandler
         self.t.start()
 
     def getEvent(self):
-
-        self.eventQueueLock.acquire()
-
-        if len(self.eventQueue) == 0:
-            self.eventQueueLock.release()
-            return None
-
-        item = self.eventQueue.pop(0)
-
-        if len(self.eventQueue) == 0:
-            self.eventQueueNotify.clear()
-
-        self.eventQueueLock.release()
-        return item
+        with bb.utils.lock_timeout(self.eventQueueLock):
+            if not self.eventQueue:
+                return None
+            item = self.eventQueue.pop(0)
+            if not self.eventQueue:
+                self.eventQueueNotify.clear()
+            return item
 
     def waitEvent(self, delay):
         self.eventQueueNotify.wait(delay)
         return self.getEvent()
 
     def queue_event(self, event):
-        self.eventQueueLock.acquire()
-        self.eventQueue.append(event)
-        self.eventQueueNotify.set()
-        self.eventQueueLock.release()
+        with bb.utils.lock_timeout(self.eventQueueLock):
+            self.eventQueue.append(event)
+            self.eventQueueNotify.set()
 
     def send_event(self, event):
         self.queue_event(pickle.loads(event))
diff --git a/bitbake/lib/bb/ui/uihelper.py b/bitbake/lib/bb/ui/uihelper.py
index 48d808ae28..e6983bd559 100644
--- a/bitbake/lib/bb/ui/uihelper.py
+++ b/bitbake/lib/bb/ui/uihelper.py
@@ -31,7 +31,7 @@ class BBUIHelper:
 
         if isinstance(event, bb.build.TaskStarted):
             tid = event._fn + ":" + event._task
-            if event._mc != "default":
+            if event._mc != "":
                 self.running_tasks[tid] = { 'title' : "mc:%s:%s %s" % (event._mc, event._package, event._task), 'starttime' : time.time(), 'pid' : event.pid }
             else:
                 self.running_tasks[tid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time(), 'pid' : event.pid }
@@ -49,9 +49,11 @@ class BBUIHelper:
             tid = event._fn + ":" + event._task
             removetid(event.pid, tid)
             self.failed_tasks.append( { 'title' : "%s %s" % (event._package, event._task)})
-        elif isinstance(event, bb.runqueue.runQueueTaskStarted):
-            self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed + 1
+        elif isinstance(event, bb.runqueue.runQueueTaskStarted) or isinstance(event, bb.runqueue.sceneQueueTaskStarted):
+            self.tasknumber_current = event.stats.completed + event.stats.active + event.stats.failed
             self.tasknumber_total = event.stats.total
+            self.setscene_current = event.stats.setscene_active + event.stats.setscene_covered + event.stats.setscene_notcovered
+            self.setscene_total = event.stats.setscene_total
             self.needUpdate = True
         elif isinstance(event, bb.build.TaskProgress):
             if event.pid > 0 and event.pid in self.pidmap:
diff --git a/bitbake/lib/bb/utils.py b/bitbake/lib/bb/utils.py
index b282d09abf..1cc74ed546 100644
--- a/bitbake/lib/bb/utils.py
+++ b/bitbake/lib/bb/utils.py
@@ -11,24 +11,29 @@ import re, fcntl, os, string, stat, shutil, time
 import sys
 import errno
 import logging
-import bb
-import bb.msg
+import locale
 import multiprocessing
-import fcntl
 import importlib
-from importlib import machinery
+import importlib.machinery
+import importlib.util
 import itertools
 import subprocess
 import glob
 import fnmatch
 import traceback
-import errno
 import signal
 import collections
 import copy
+import ctypes
+import random
+import socket
+import struct
+import tempfile
 from subprocess import getstatusoutput
 from contextlib import contextmanager
 from ctypes import cdll
+import bb
+import bb.msg
 
 logger = logging.getLogger("BitBake.Util")
 python_extensions = importlib.machinery.all_suffixes()
@@ -43,7 +48,7 @@ def clean_context():
 
 def get_context():
     return _context
-    
+
 
 def set_context(ctx):
     _context = ctx
@@ -77,7 +82,16 @@ def explode_version(s):
     return r
 
 def split_version(s):
-    """Split a version string into its constituent parts (PE, PV, PR)"""
+    """Split a version string into its constituent parts (PE, PV, PR).
+
+    Arguments:
+
+    -  ``s``: version string. The format of the input string should be::
+
+          ${PE}:${PV}-${PR}
+
+    Returns a tuple ``(pe, pv, pr)``.
+    """
     s = s.strip(" <>=")
     e = 0
     if s.count(':'):
@@ -129,16 +143,30 @@ def vercmp(ta, tb):
     return r
 
 def vercmp_string(a, b):
-    """ Split version strings and compare them """
+    """ Split version strings using ``bb.utils.split_version()`` and compare
+    them with ``bb.utils.vercmp().``
+
+    Arguments:
+
+    -  ``a``: left version string operand.
+    -  ``b``: right version string operand.
+
+    Returns what ``bb.utils.vercmp()`` returns."""
     ta = split_version(a)
     tb = split_version(b)
     return vercmp(ta, tb)
 
 def vercmp_string_op(a, b, op):
     """
-    Compare two versions and check if the specified comparison operator matches the result of the comparison.
-    This function is fairly liberal about what operators it will accept since there are a variety of styles
-    depending on the context.
+    Takes the return value ``bb.utils.vercmp()`` and returns the operation
+    defined by ``op`` between the return value and 0.
+
+    Arguments:
+
+    -  ``a``: left version string operand.
+    -  ``b``: right version string operand.
+    -  ``op``: operator string. Can be one of ``=``, ``==``, ``<=``, ``>=``,
+       ``>``, ``>>``, ``<``, ``<<`` or ``!=``.
     """
     res = vercmp_string(a, b)
     if op in ('=', '=='):
@@ -158,9 +186,16 @@ def vercmp_string_op(a, b, op):
 
 def explode_deps(s):
     """
-    Take an RDEPENDS style string of format:
-    "DEPEND1 (optional version) DEPEND2 (optional version) ..."
-    and return a list of dependencies.
+    Takes an RDEPENDS style string of format::
+
+      DEPEND1 (optional version) DEPEND2 (optional version) ...
+
+    Arguments:
+
+    -  ``s``: input RDEPENDS style string
+
+    Returns a list of dependencies.
+
     Version information is ignored.
     """
     r = []
@@ -182,9 +217,17 @@ def explode_deps(s):
 
 def explode_dep_versions2(s, *, sort=True):
     """
-    Take an RDEPENDS style string of format:
-    "DEPEND1 (optional version) DEPEND2 (optional version) ..."
-    and return a dictionary of dependencies and versions.
+    Takes an RDEPENDS style string of format::
+
+       DEPEND1 (optional version) DEPEND2 (optional version) ...
+
+    Arguments:
+
+    -  ``s``: input RDEPENDS style string
+    -  ``*``: *Unused*.
+    -  ``sort``: whether to sort the output or not.
+
+    Returns a dictionary of dependencies and versions.
     """
     r = collections.OrderedDict()
     l = s.replace(",", "").split()
@@ -205,8 +248,8 @@ def explode_dep_versions2(s, *, sort=True):
             inversion = True
             # This list is based on behavior and supported comparisons from deb, opkg and rpm.
             #
-            # Even though =<, <<, ==, !=, =>, and >> may not be supported, 
-            # we list each possibly valid item. 
+            # Even though =<, <<, ==, !=, =>, and >> may not be supported,
+            # we list each possibly valid item.
             # The build system is responsible for validation of what it supports.
             if i.startswith(('<=', '=<', '<<', '==', '!=', '>=', '=>', '>>')):
                 lastcmp = i[0:2]
@@ -249,10 +292,17 @@ def explode_dep_versions2(s, *, sort=True):
 
 def explode_dep_versions(s):
     """
-    Take an RDEPENDS style string of format:
-    "DEPEND1 (optional version) DEPEND2 (optional version) ..."
-    skip null value and items appeared in dependancy string multiple times
-    and return a dictionary of dependencies and versions.
+    Take an RDEPENDS style string of format::
+
+      DEPEND1 (optional version) DEPEND2 (optional version) ...
+
+    Skips null values and items appeared in dependency string multiple times.
+
+    Arguments:
+
+    -  ``s``: input RDEPENDS style string
+
+    Returns a dictionary of dependencies and versions.
     """
     r = explode_dep_versions2(s)
     for d in r:
@@ -266,7 +316,17 @@ def explode_dep_versions(s):
 
 def join_deps(deps, commasep=True):
     """
-    Take the result from explode_dep_versions and generate a dependency string
+    Take a result from ``bb.utils.explode_dep_versions()`` and generate a
+    dependency string.
+
+    Arguments:
+
+    -  ``deps``: dictionary of dependencies and versions.
+    -  ``commasep``: makes the return value separated by commas if ``True``,
+       separated by spaces otherwise.
+
+    Returns a comma-separated (space-separated if ``comma-sep`` is ``False``)
+    string of dependencies and versions.
     """
     result = []
     for dep in deps:
@@ -340,7 +400,7 @@ def _print_exception(t, value, tb, realfile, text, context):
         exception = traceback.format_exception_only(t, value)
         error.append('Error executing a python function in %s:\n' % realfile)
 
-        # Strip 'us' from the stack (better_exec call) unless that was where the 
+        # Strip 'us' from the stack (better_exec call) unless that was where the
         # error came from
         if tb.tb_next is not None:
             tb = tb.tb_next
@@ -379,7 +439,7 @@ def _print_exception(t, value, tb, realfile, text, context):
 
         error.append("Exception: %s" % ''.join(exception))
 
-        # If the exception is from spwaning a task, let's be helpful and display
+        # If the exception is from spawning a task, let's be helpful and display
         # the output (which hopefully includes stderr).
         if isinstance(value, subprocess.CalledProcessError) and value.output:
             error.append("Subprocess output:")
@@ -400,7 +460,7 @@ def better_exec(code, context, text = None, realfile = "<code>", pythonexception
         code = better_compile(code, realfile, realfile)
     try:
         exec(code, get_context(), context)
-    except (bb.BBHandledException, bb.parse.SkipRecipe, bb.data_smart.ExpansionError):
+    except (bb.BBHandledException, bb.parse.SkipRecipe, bb.data_smart.ExpansionError, bb.process.ExecutionError):
         # Error already shown so passthrough, no need for traceback
         raise
     except Exception as e:
@@ -427,33 +487,56 @@ def better_eval(source, locals, extraglobals = None):
     return eval(source, ctx, locals)
 
 @contextmanager
-def fileslocked(files):
-    """Context manager for locking and unlocking file locks."""
+def fileslocked(files, *args, **kwargs):
+    """Context manager for locking and unlocking file locks. Uses
+    ``bb.utils.lockfile()`` and ``bb.utils.unlockfile()`` to lock and unlock
+    files.
+
+    No return value."""
     locks = []
     if files:
         for lockfile in files:
-            locks.append(bb.utils.lockfile(lockfile))
+            l = bb.utils.lockfile(lockfile, *args, **kwargs)
+            if l is not None:
+                locks.append(l)
 
     try:
         yield
     finally:
+        locks.reverse()
         for lock in locks:
             bb.utils.unlockfile(lock)
 
 def lockfile(name, shared=False, retry=True, block=False):
     """
-    Use the specified file as a lock file, return when the lock has
-    been acquired. Returns a variable to pass to unlockfile().
-    Parameters:
-        retry: True to re-try locking if it fails, False otherwise
-        block: True to block until the lock succeeds, False otherwise
+    Use the specified file (with filename ``name``) as a lock file, return when
+    the lock has been acquired. Returns a variable to pass to unlockfile().
+
+    Arguments:
+
+    -  ``shared``: sets the lock as a shared lock instead of an
+       exclusive lock.
+    -  ``retry``: ``True`` to re-try locking if it fails, ``False``
+       otherwise.
+    -  ``block``: ``True`` to block until the lock succeeds,
+       ``False`` otherwise.
+
     The retry and block parameters are kind of equivalent unless you
     consider the possibility of sending a signal to the process to break
     out - at which point you want block=True rather than retry=True.
+
+    Returns the locked file descriptor in case of success, ``None`` otherwise.
     """
+    basename = os.path.basename(name)
+    if len(basename) > 255:
+        root, ext = os.path.splitext(basename)
+        basename = root[:255 - len(ext)] + ext
+
     dirname = os.path.dirname(name)
     mkdirhier(dirname)
 
+    name = os.path.join(dirname, basename)
+
     if not os.access(dirname, os.W_OK):
         logger.error("Unable to acquire lock '%s', directory is not writable",
                      name)
@@ -487,7 +570,7 @@ def lockfile(name, shared=False, retry=True, block=False):
                     return lf
             lf.close()
         except OSError as e:
-            if e.errno == errno.EACCES:
+            if e.errno == errno.EACCES or e.errno == errno.ENAMETOOLONG:
                 logger.error("Unable to acquire lock '%s', %s",
                              e.strerror, name)
                 sys.exit(1)
@@ -501,7 +584,13 @@ def lockfile(name, shared=False, retry=True, block=False):
 
 def unlockfile(lf):
     """
-    Unlock a file locked using lockfile()
+    Unlock a file locked using ``bb.utils.lockfile()``.
+
+    Arguments:
+
+    -  ``lf``: the locked file descriptor.
+
+    No return value.
     """
     try:
         # If we had a shared lock, we need to promote to exclusive before
@@ -529,43 +618,97 @@ def _hasher(method, filename):
 
 def md5_file(filename):
     """
-    Return the hex string representation of the MD5 checksum of filename.
+    Arguments:
+
+    -  ``filename``: path to the input file.
+
+    Returns the hexadecimal string representation of the MD5 checksum of filename.
     """
     import hashlib
-    return _hasher(hashlib.md5(), filename)
+    try:
+        sig = hashlib.new('MD5', usedforsecurity=False)
+    except TypeError:
+        # Some configurations don't appear to support two arguments
+        sig = hashlib.new('MD5')
+    return _hasher(sig, filename)
 
 def sha256_file(filename):
     """
-    Return the hex string representation of the 256-bit SHA checksum of
+    Returns the hexadecimal representation of the 256-bit SHA checksum of
     filename.
+
+    Arguments:
+
+    -  ``filename``: path to the file.
     """
     import hashlib
     return _hasher(hashlib.sha256(), filename)
 
 def sha1_file(filename):
     """
-    Return the hex string representation of the SHA1 checksum of the filename
+    Returns the hexadecimal representation of the SHA1 checksum of the filename
+
+    Arguments:
+
+    -  ``filename``: path to the file.
     """
     import hashlib
     return _hasher(hashlib.sha1(), filename)
 
 def sha384_file(filename):
     """
-    Return the hex string representation of the SHA384 checksum of the filename
+    Returns the hexadecimal representation of the SHA384 checksum of the filename
+
+    Arguments:
+
+    -  ``filename``: path to the file.
     """
     import hashlib
     return _hasher(hashlib.sha384(), filename)
 
 def sha512_file(filename):
     """
-    Return the hex string representation of the SHA512 checksum of the filename
+    Returns the hexadecimal representation of the SHA512 checksum of the filename
+
+    Arguments:
+
+    -  ``filename``: path to the file.
     """
     import hashlib
     return _hasher(hashlib.sha512(), filename)
 
+def goh1_file(filename):
+    """
+    Returns the hexadecimal string representation of the Go mod h1 checksum of the
+    filename. The Go mod h1 checksum uses the Go dirhash package. The package
+    defines hashes over directory trees and is used by go mod for mod files and
+    zip archives.
+
+    Arguments:
+
+    -  ``filename``: path to the file.
+    """
+    import hashlib
+    import zipfile
+
+    lines = []
+    if zipfile.is_zipfile(filename):
+        with zipfile.ZipFile(filename) as archive:
+            for fn in sorted(archive.namelist()):
+                method = hashlib.sha256()
+                method.update(archive.read(fn))
+                hash = method.hexdigest()
+                lines.append("%s  %s\n" % (hash, fn))
+    else:
+        hash = _hasher(hashlib.sha256(), filename)
+        lines.append("%s  go.mod\n" % hash)
+    method = hashlib.sha256()
+    method.update("".join(lines).encode('utf-8'))
+    return method.hexdigest()
+
 def preserved_envvars_exported():
-    """Variables which are taken from the environment and placed in and exported
-    from the metadata"""
+    """Returns the list of variables which are taken from the environment and
+    placed in and exported from the metadata."""
     return [
         'BB_TASKHASH',
         'HOME',
@@ -579,19 +722,42 @@ def preserved_envvars_exported():
     ]
 
 def preserved_envvars():
-    """Variables which are taken from the environment and placed in the metadata"""
+    """Returns the list of variables which are taken from the environment and
+    placed in the metadata."""
     v = [
         'BBPATH',
         'BB_PRESERVE_ENV',
-        'BB_ENV_WHITELIST',
-        'BB_ENV_EXTRAWHITE',
+        'BB_ENV_PASSTHROUGH_ADDITIONS',
     ]
     return v + preserved_envvars_exported()
 
+def check_system_locale():
+    """Make sure the required system locale are available and configured.
+
+    No return value."""
+    default_locale = locale.getlocale(locale.LC_CTYPE)
+
+    try:
+        locale.setlocale(locale.LC_CTYPE, ("en_US", "UTF-8"))
+    except:
+        sys.exit("Please make sure locale 'en_US.UTF-8' is available on your system")
+    else:
+        locale.setlocale(locale.LC_CTYPE, default_locale)
+
+    if sys.getfilesystemencoding() != "utf-8":
+        sys.exit("Please use a locale setting which supports UTF-8 (such as LANG=en_US.UTF-8).\n"
+                 "Python can't change the filesystem locale after loading so we need a UTF-8 when Python starts or things won't work.")
+
 def filter_environment(good_vars):
     """
     Create a pristine environment for bitbake. This will remove variables that
     are not known and may influence the build in a negative way.
+
+    Arguments:
+
+    -  ``good_vars``: list of variable to exclude from the filtering.
+
+    No return value.
     """
 
     removed_vars = {}
@@ -615,27 +781,29 @@ def filter_environment(good_vars):
 
 def approved_variables():
     """
-    Determine and return the list of whitelisted variables which are approved
+    Determine and return the list of variables which are approved
     to remain in the environment.
     """
     if 'BB_PRESERVE_ENV' in os.environ:
         return os.environ.keys()
     approved = []
-    if 'BB_ENV_WHITELIST' in os.environ:
-        approved = os.environ['BB_ENV_WHITELIST'].split()
-        approved.extend(['BB_ENV_WHITELIST'])
+    if 'BB_ENV_PASSTHROUGH' in os.environ:
+        approved = os.environ['BB_ENV_PASSTHROUGH'].split()
+        approved.extend(['BB_ENV_PASSTHROUGH'])
     else:
         approved = preserved_envvars()
-    if 'BB_ENV_EXTRAWHITE' in os.environ:
-        approved.extend(os.environ['BB_ENV_EXTRAWHITE'].split())
-        if 'BB_ENV_EXTRAWHITE' not in approved:
-            approved.extend(['BB_ENV_EXTRAWHITE'])
+    if 'BB_ENV_PASSTHROUGH_ADDITIONS' in os.environ:
+        approved.extend(os.environ['BB_ENV_PASSTHROUGH_ADDITIONS'].split())
+        if 'BB_ENV_PASSTHROUGH_ADDITIONS' not in approved:
+            approved.extend(['BB_ENV_PASSTHROUGH_ADDITIONS'])
     return approved
 
 def clean_environment():
     """
     Clean up any spurious environment variables. This will remove any
     variables the user hasn't chosen to preserve.
+
+    No return value.
     """
     if 'BB_PRESERVE_ENV' not in os.environ:
         good_vars = approved_variables()
@@ -646,6 +814,8 @@ def clean_environment():
 def empty_environment():
     """
     Remove all variables from the environment.
+
+    No return value.
     """
     for s in list(os.environ.keys()):
         os.unsetenv(s)
@@ -654,6 +824,12 @@ def empty_environment():
 def build_environment(d):
     """
     Build an environment from all exported variables.
+
+    Arguments:
+
+    -  ``d``: the data store.
+
+    No return value.
     """
     import bb.data
     for var in bb.data.keys(d):
@@ -678,13 +854,23 @@ def _check_unsafe_delete_path(path):
     return False
 
 def remove(path, recurse=False, ionice=False):
-    """Equivalent to rm -f or rm -rf"""
+    """Equivalent to rm -f or rm -rf.
+
+    Arguments:
+
+    -  ``path``: path to file/directory to remove.
+    -  ``recurse``: deletes recursively if ``True``.
+    -  ``ionice``: prepends ``ionice -c 3`` to the ``rm`` command. See ``man
+       ionice``.
+
+    No return value.
+    """
     if not path:
         return
     if recurse:
         for name in glob.glob(path):
-            if _check_unsafe_delete_path(path):
-                raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % path)
+            if _check_unsafe_delete_path(name):
+                raise Exception('bb.utils.remove: called with dangerous path "%s" and recurse=True, refusing to delete!' % name)
         # shutil.rmtree(name) would be ideal but its too slow
         cmd = []
         if ionice:
@@ -699,7 +885,17 @@ def remove(path, recurse=False, ionice=False):
                 raise
 
 def prunedir(topdir, ionice=False):
-    """ Delete everything reachable from the directory named in 'topdir'. """
+    """
+    Delete everything reachable from the directory named in ``topdir``.
+
+    Arguments:
+
+    -  ``topdir``: directory path.
+    -  ``ionice``: prepends ``ionice -c 3`` to the ``rm`` command. See ``man
+       ionice``.
+
+    No return value.
+    """
     # CAUTION:  This is dangerous!
     if _check_unsafe_delete_path(topdir):
         raise Exception('bb.utils.prunedir: called with dangerous path "%s", refusing to delete!' % topdir)
@@ -710,9 +906,16 @@ def prunedir(topdir, ionice=False):
 # but thats possibly insane and suffixes is probably going to be small
 #
 def prune_suffix(var, suffixes, d):
-    """ 
-    See if var ends with any of the suffixes listed and
-    remove it if found 
+    """
+    Check if ``var`` ends with any of the suffixes listed in ``suffixes`` and
+    remove it if found.
+
+    Arguments:
+
+    -  ``var``: string to check for suffixes.
+    -  ``suffixes``: list of strings representing suffixes to check for.
+
+    Returns the string ``var`` without the suffix.
     """
     for suffix in suffixes:
         if suffix and var.endswith(suffix):
@@ -721,9 +924,16 @@ def prune_suffix(var, suffixes, d):
 
 def mkdirhier(directory):
     """Create a directory like 'mkdir -p', but does not complain if
-    directory already exists like os.makedirs
-    """
+    directory already exists like ``os.makedirs()``.
 
+    Arguments:
+
+    -  ``directory``: path to the directory.
+
+    No return value.
+    """
+    if '${' in str(directory):
+        bb.fatal("Directory name {} contains unexpanded bitbake variable. This may cause build failures and WORKDIR polution.".format(directory))
     try:
         os.makedirs(directory)
     except OSError as e:
@@ -731,10 +941,24 @@ def mkdirhier(directory):
             raise e
 
 def movefile(src, dest, newmtime = None, sstat = None):
-    """Moves a file from src to dest, preserving all permissions and
+    """Moves a file from ``src`` to ``dest``, preserving all permissions and
     attributes; mtime will be preserved even when moving across
-    filesystems.  Returns true on success and false on failure. Move is
+    filesystems.  Returns ``True`` on success and ``False`` on failure. Move is
     atomic.
+
+    Arguments:
+
+    -  ``src`` -- Source file.
+    -  ``dest`` -- Destination file.
+    -  ``newmtime`` -- new mtime to be passed as float seconds since the epoch.
+    -  ``sstat`` -- os.stat_result to use for the destination file.
+
+    Returns an ``os.stat_result`` of the destination file if the
+    source file is a symbolic link or the ``sstat`` argument represents a
+    symbolic link - in which case the destination file will also be created as
+    a symbolic link.
+
+    Otherwise, returns ``newmtime`` on success and ``False`` on failure.
     """
 
     #print "movefile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
@@ -742,7 +966,7 @@ def movefile(src, dest, newmtime = None, sstat = None):
         if not sstat:
             sstat = os.lstat(src)
     except Exception as e:
-        print("movefile: Stating source file failed...", e)
+        logger.warning("movefile: Stating source file failed...", e)
         return None
 
     destexists = 1
@@ -770,7 +994,7 @@ def movefile(src, dest, newmtime = None, sstat = None):
             os.unlink(src)
             return os.lstat(dest)
         except Exception as e:
-            print("movefile: failed to properly create symlink:", dest, "->", target, e)
+            logger.warning("movefile: failed to properly create symlink:", dest, "->", target, e)
             return None
 
     renamefailed = 1
@@ -782,12 +1006,12 @@ def movefile(src, dest, newmtime = None, sstat = None):
 
     if sstat[stat.ST_DEV] == dstat[stat.ST_DEV]:
         try:
-            os.rename(src, destpath)
+            bb.utils.rename(src, destpath)
             renamefailed = 0
         except Exception as e:
             if e.errno != errno.EXDEV:
                 # Some random error.
-                print("movefile: Failed to move", src, "to", dest, e)
+                logger.warning("movefile: Failed to move", src, "to", dest, e)
                 return None
             # Invalid cross-device-link 'bind' mounted or actually Cross-Device
 
@@ -796,16 +1020,16 @@ def movefile(src, dest, newmtime = None, sstat = None):
         if stat.S_ISREG(sstat[stat.ST_MODE]):
             try: # For safety copy then move it over.
                 shutil.copyfile(src, destpath + "#new")
-                os.rename(destpath + "#new", destpath)
+                bb.utils.rename(destpath + "#new", destpath)
                 didcopy = 1
             except Exception as e:
-                print('movefile: copy', src, '->', dest, 'failed.', e)
+                logger.warning('movefile: copy', src, '->', dest, 'failed.', e)
                 return None
         else:
             #we don't yet handle special, so we need to fall back to /bin/mv
             a = getstatusoutput("/bin/mv -f " + "'" + src + "' '" + dest + "'")
             if a[0] != 0:
-                print("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a)
+                logger.warning("movefile: Failed to move special file:" + src + "' to '" + dest + "'", a)
                 return None # failure
         try:
             if didcopy:
@@ -813,7 +1037,7 @@ def movefile(src, dest, newmtime = None, sstat = None):
                 os.chmod(destpath, stat.S_IMODE(sstat[stat.ST_MODE])) # Sticky is reset on chown
                 os.unlink(src)
         except Exception as e:
-            print("movefile: Failed to chown/chmod/unlink", dest, e)
+            logger.warning("movefile: Failed to chown/chmod/unlink", dest, e)
             return None
 
     if newmtime:
@@ -825,9 +1049,24 @@ def movefile(src, dest, newmtime = None, sstat = None):
 
 def copyfile(src, dest, newmtime = None, sstat = None):
     """
-    Copies a file from src to dest, preserving all permissions and
+    Copies a file from ``src`` to ``dest``, preserving all permissions and
     attributes; mtime will be preserved even when moving across
-    filesystems.  Returns true on success and false on failure.
+    filesystems.
+
+    Arguments:
+
+    -  ``src``: Source file.
+    -  ``dest``: Destination file.
+    -  ``newmtime``: new mtime to be passed as float seconds since the epoch.
+    -  ``sstat``: os.stat_result to use for the destination file.
+
+    Returns an ``os.stat_result`` of the destination file if the
+    source file is a symbolic link or the ``sstat`` argument represents a
+    symbolic link - in which case the destination file will also be created as
+    a symbolic link.
+
+    Otherwise, returns ``newmtime`` on success and ``False`` on failure.
+
     """
     #print "copyfile(" + src + "," + dest + "," + str(newmtime) + "," + str(sstat) + ")"
     try:
@@ -874,7 +1113,7 @@ def copyfile(src, dest, newmtime = None, sstat = None):
 
             # For safety copy then move it over.
             shutil.copyfile(src, dest + "#new")
-            os.rename(dest + "#new", dest)
+            bb.utils.rename(dest + "#new", dest)
         except Exception as e:
             logger.warning("copyfile: copy %s to %s failed (%s)" % (src, dest, e))
             return False
@@ -905,10 +1144,16 @@ def copyfile(src, dest, newmtime = None, sstat = None):
 
 def break_hardlinks(src, sstat = None):
     """
-    Ensures src is the only hardlink to this file.  Other hardlinks,
+    Ensures ``src`` is the only hardlink to this file.  Other hardlinks,
     if any, are not affected (other than in their st_nlink value, of
-    course).  Returns true on success and false on failure.
+    course).
+
+    Arguments:
+
+    -  ``src``: source file path.
+    -  ``sstat``: os.stat_result to use when checking if the file is a link.
 
+    Returns ``True`` on success and ``False`` on failure.
     """
     try:
         if not sstat:
@@ -922,11 +1167,24 @@ def break_hardlinks(src, sstat = None):
 
 def which(path, item, direction = 0, history = False, executable=False):
     """
-    Locate `item` in the list of paths `path` (colon separated string like $PATH).
-    If `direction` is non-zero then the list is reversed.
-    If `history` is True then the list of candidates also returned as result,history.
-    If `executable` is True then the candidate has to be an executable file,
-    otherwise the candidate simply has to exist.
+    Locate ``item`` in the list of paths ``path`` (colon separated string like
+    ``$PATH``).
+
+    Arguments:
+
+    -  ``path``: list of colon-separated paths.
+    -  ``item``: string to search for.
+    -  ``direction``: if non-zero then the list is reversed.
+    -  ``history``: if ``True`` then the list of candidates also returned as
+       ``result,history`` where ``history`` is the list of previous path
+       checked.
+    -  ``executable``: if ``True`` then the candidate defined by ``path`` has
+       to be an executable file, otherwise if ``False`` the candidate simply
+       has to exist.
+
+    Returns the item if found in the list of path, otherwise an empty string.
+    If ``history`` is ``True``, return the list of previous path checked in a
+    tuple with the found (or not found) item as ``(item, history)``.
     """
 
     if executable:
@@ -953,10 +1211,29 @@ def which(path, item, direction = 0, history = False, executable=False):
         return "", hist
     return ""
 
+def to_filemode(input):
+    """
+    Take a bitbake variable contents defining a file mode and return
+    the proper python representation of the number
+
+    Arguments:
+
+    -  ``input``: a string or number to convert, e.g. a bitbake variable
+       string, assumed to be an octal representation
+
+    Returns the python file mode as a number
+    """
+    # umask might come in as a number or text string..
+    if type(input) is int:
+        return input
+    return int(input, 8)
+
 @contextmanager
 def umask(new_mask):
     """
     Context manager to set the umask to a specific mask, and restore it afterwards.
+
+    No return value.
     """
     current_mask = os.umask(new_mask)
     try:
@@ -965,13 +1242,26 @@ def umask(new_mask):
         os.umask(current_mask)
 
 def to_boolean(string, default=None):
-    """ 
+    """
     Check input string and return boolean value True/False/None
-    depending upon the checks 
+    depending upon the checks.
+
+    Arguments:
+
+    -  ``string``: input string.
+    -  ``default``: default return value if the input ``string`` is ``None``,
+       ``0``, ``False`` or an empty string.
+
+    Returns ``True`` if the string is one of "y", "yes", "1", "true", ``False``
+    if the string is one of "n", "no", "0", or "false". Return ``default`` if
+    the input ``string`` is ``None``, ``0``, ``False`` or an empty string.
     """
     if not string:
         return default
 
+    if isinstance(string, int):
+        return string != 0
+
     normalized = string.lower()
     if normalized in ("y", "yes", "1", "true"):
         return True
@@ -985,18 +1275,17 @@ def contains(variable, checkvalues, truevalue, falsevalue, d):
 
     Arguments:
 
-    variable -- the variable name. This will be fetched and expanded (using
-    d.getVar(variable)) and then split into a set().
-
-    checkvalues -- if this is a string it is split on whitespace into a set(),
-    otherwise coerced directly into a set().
+    -  ``variable``: the variable name. This will be fetched and expanded (using
+       d.getVar(variable)) and then split into a set().
+    -  ``checkvalues``: if this is a string it is split on whitespace into a set(),
+       otherwise coerced directly into a set().
+    -  ``truevalue``: the value to return if checkvalues is a subset of variable.
+    -  ``falsevalue``: the value to return if variable is empty or if checkvalues is
+       not a subset of variable.
+    -  ``d``: the data store.
 
-    truevalue -- the value to return if checkvalues is a subset of variable.
-
-    falsevalue -- the value to return if variable is empty or if checkvalues is
-    not a subset of variable.
-
-    d -- the data store.
+    Returns ``True`` if the variable contains the values specified, ``False``
+    otherwise.
     """
 
     val = d.getVar(variable)
@@ -1016,18 +1305,17 @@ def contains_any(variable, checkvalues, truevalue, falsevalue, d):
 
     Arguments:
 
-    variable -- the variable name. This will be fetched and expanded (using
-    d.getVar(variable)) and then split into a set().
-
-    checkvalues -- if this is a string it is split on whitespace into a set(),
-    otherwise coerced directly into a set().
-
-    truevalue -- the value to return if checkvalues is a subset of variable.
+    -  ``variable``: the variable name. This will be fetched and expanded (using
+       d.getVar(variable)) and then split into a set().
+    -  ``checkvalues``: if this is a string it is split on whitespace into a set(),
+       otherwise coerced directly into a set().
+    -  ``truevalue``: the value to return if checkvalues is a subset of variable.
+    -  ``falsevalue``: the value to return if variable is empty or if checkvalues is
+       not a subset of variable.
+    -  ``d``: the data store.
 
-    falsevalue -- the value to return if variable is empty or if checkvalues is
-    not a subset of variable.
-
-    d -- the data store.
+    Returns ``True`` if the variable contains any of the values specified,
+    ``False`` otherwise.
     """
     val = d.getVar(variable)
     if not val:
@@ -1042,17 +1330,17 @@ def contains_any(variable, checkvalues, truevalue, falsevalue, d):
     return falsevalue
 
 def filter(variable, checkvalues, d):
-    """Return all words in the variable that are present in the checkvalues.
+    """Return all words in the variable that are present in the ``checkvalues``.
 
     Arguments:
 
-    variable -- the variable name. This will be fetched and expanded (using
-    d.getVar(variable)) and then split into a set().
-
-    checkvalues -- if this is a string it is split on whitespace into a set(),
-    otherwise coerced directly into a set().
+    -  ``variable``: the variable name. This will be fetched and expanded (using
+       d.getVar(variable)) and then split into a set().
+    -  ``checkvalues``: if this is a string it is split on whitespace into a set(),
+       otherwise coerced directly into a set().
+    -  ``d``: the data store.
 
-    d -- the data store.
+    Returns a list of string.
     """
 
     val = d.getVar(variable)
@@ -1068,8 +1356,27 @@ def filter(variable, checkvalues, d):
 
 def get_referenced_vars(start_expr, d):
     """
-    :return: names of vars referenced in start_expr (recursively), in quasi-BFS order (variables within the same level
-    are ordered arbitrarily)
+    Get the names of the variables referenced in a given expression.
+
+    Arguments:
+
+      -  ``start_expr``: the expression where to look for variables references.
+
+         For example::
+
+            ${VAR_A} string ${VAR_B}
+
+         Or::
+
+            ${@d.getVar('VAR')}
+
+         If a variables makes references to other variables, the latter are also
+         returned recursively.
+
+      -  ``d``: the data store.
+
+    Returns the names of vars referenced in ``start_expr`` (recursively), in
+    quasi-BFS order (variables within the same level are ordered arbitrarily).
     """
 
     seen = set()
@@ -1103,7 +1410,10 @@ def get_referenced_vars(start_expr, d):
 
 
 def cpu_count():
-    return multiprocessing.cpu_count()
+    try:
+        return len(os.sched_getaffinity(0))
+    except OSError:
+        return multiprocessing.cpu_count()
 
 def nonblockingfd(fd):
     fcntl.fcntl(fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)
@@ -1146,7 +1456,9 @@ def multiprocessingpool(*args, **kwargs):
     return multiprocessing.Pool(*args, **kwargs)
 
 def exec_flat_python_func(func, *args, **kwargs):
-    """Execute a flat python function (defined with def funcname(args):...)"""
+    """Execute a flat python function (defined with ``def funcname(args): ...``)
+
+    Returns the return value of the function."""
     # Prepare a small piece of python code which calls the requested function
     # To do this we need to prepare two things - a set of variables we can use to pass
     # the values of arguments into the calling function, and the list of arguments for
@@ -1172,48 +1484,57 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
     """Edit lines from a recipe or config file and modify one or more
     specified variable values set in the file using a specified callback
     function. Lines are expected to have trailing newlines.
-    Parameters:
-        meta_lines: lines from the file; can be a list or an iterable
-            (e.g. file pointer)
-        variables: a list of variable names to look for. Functions
-            may also be specified, but must be specified with '()' at
-            the end of the name. Note that the function doesn't have
-            any intrinsic understanding of _append, _prepend, _remove,
-            or overrides, so these are considered as part of the name.
-            These values go into a regular expression, so regular
-            expression syntax is allowed.
-        varfunc: callback function called for every variable matching
-            one of the entries in the variables parameter. The function
-            should take four arguments:
-                varname: name of variable matched
-                origvalue: current value in file
-                op: the operator (e.g. '+=')
-                newlines: list of lines up to this point. You can use
-                    this to prepend lines before this variable setting
-                    if you wish.
-            and should return a four-element tuple:
-                newvalue: new value to substitute in, or None to drop
-                    the variable setting entirely. (If the removal
-                    results in two consecutive blank lines, one of the
-                    blank lines will also be dropped).
-                newop: the operator to use - if you specify None here,
-                    the original operation will be used.
-                indent: number of spaces to indent multi-line entries,
-                    or -1 to indent up to the level of the assignment
-                    and opening quote, or a string to use as the indent.
-                minbreak: True to allow the first element of a
-                    multi-line value to continue on the same line as
-                    the assignment, False to indent before the first
-                    element.
-            To clarify, if you wish not to change the value, then you
-            would return like this: return origvalue, None, 0, True
-        match_overrides: True to match items with _overrides on the end,
-            False otherwise
+
+    Arguments:
+
+    -  ``meta_lines``: lines from the file; can be a list or an iterable
+       (e.g. file pointer)
+    -  ``variables``: a list of variable names to look for. Functions
+       may also be specified, but must be specified with ``()`` at
+       the end of the name. Note that the function doesn't have
+       any intrinsic understanding of ``:append``, ``:prepend``, ``:remove``,
+       or overrides, so these are considered as part of the name.
+       These values go into a regular expression, so regular
+       expression syntax is allowed.
+    -  ``varfunc``: callback function called for every variable matching
+       one of the entries in the variables parameter.
+
+       The function should take four arguments:
+
+       -  ``varname``: name of variable matched
+       -  ``origvalue``: current value in file
+       -  ``op``: the operator (e.g. ``+=``)
+       -  ``newlines``: list of lines up to this point. You can use
+          this to prepend lines before this variable setting
+          if you wish.
+
+       And should return a four-element tuple:
+
+       -  ``newvalue``: new value to substitute in, or ``None`` to drop
+          the variable setting entirely. (If the removal
+          results in two consecutive blank lines, one of the
+          blank lines will also be dropped).
+       -  ``newop``: the operator to use - if you specify ``None`` here,
+          the original operation will be used.
+       -  ``indent``: number of spaces to indent multi-line entries,
+          or ``-1`` to indent up to the level of the assignment
+          and opening quote, or a string to use as the indent.
+       -  ``minbreak``: ``True`` to allow the first element of a
+          multi-line value to continue on the same line as
+          the assignment, ``False`` to indent before the first
+          element.
+
+       To clarify, if you wish not to change the value, then you
+       would return like this::
+
+          return origvalue, None, 0, True
+    -  ``match_overrides``: True to match items with _overrides on the end,
+       False otherwise
+
     Returns a tuple:
-        updated:
-            True if changes were made, False otherwise.
-        newlines:
-            Lines after processing
+
+    -  ``updated``: ``True`` if changes were made, ``False`` otherwise.
+    -  ``newlines``: Lines after processing.
     """
 
     var_res = {}
@@ -1357,12 +1678,13 @@ def edit_metadata(meta_lines, variables, varfunc, match_overrides=False):
 
 
 def edit_metadata_file(meta_file, variables, varfunc):
-    """Edit a recipe or config file and modify one or more specified
+    """Edit a recipe or configuration file and modify one or more specified
     variable values set in the file using a specified callback function.
     The file is only written to if the value(s) actually change.
-    This is basically the file version of edit_metadata(), see that
+    This is basically the file version of ``bb.utils.edit_metadata()``, see that
     function's description for parameter/usage information.
-    Returns True if the file was written to, False otherwise.
+
+    Returns ``True`` if the file was written to, ``False`` otherwise.
     """
     with open(meta_file, 'r') as f:
         (updated, newlines) = edit_metadata(f, variables, varfunc)
@@ -1373,23 +1695,25 @@ def edit_metadata_file(meta_file, variables, varfunc):
 
 
 def edit_bblayers_conf(bblayers_conf, add, remove, edit_cb=None):
-    """Edit bblayers.conf, adding and/or removing layers
-    Parameters:
-        bblayers_conf: path to bblayers.conf file to edit
-        add: layer path (or list of layer paths) to add; None or empty
-            list to add nothing
-        remove: layer path (or list of layer paths) to remove; None or
-            empty list to remove nothing
-        edit_cb: optional callback function that will be called after
-            processing adds/removes once per existing entry.
+    """Edit ``bblayers.conf``, adding and/or removing layers.
+
+    Arguments:
+
+    -  ``bblayers_conf``: path to ``bblayers.conf`` file to edit
+    -  ``add``: layer path (or list of layer paths) to add; ``None`` or empty
+       list to add nothing
+    -  ``remove``: layer path (or list of layer paths) to remove; ``None`` or
+       empty list to remove nothing
+    -  ``edit_cb``: optional callback function that will be called
+       after processing adds/removes once per existing entry.
+
     Returns a tuple:
-        notadded: list of layers specified to be added but weren't
-            (because they were already in the list)
-        notremoved: list of layers that were specified to be removed
-            but weren't (because they weren't in the list)
-    """
 
-    import fnmatch
+    -  ``notadded``: list of layers specified to be added but weren't
+       (because they were already in the list)
+    -  ``notremoved``: list of layers that were specified to be removed
+       but weren't (because they weren't in the list)
+    """
 
     def remove_trailing_sep(pth):
         if pth and pth[-1] == os.sep:
@@ -1508,7 +1832,22 @@ def get_collection_res(d):
 
 
 def get_file_layer(filename, d, collection_res={}):
-    """Determine the collection (as defined by a layer's layer.conf file) containing the specified file"""
+    """Determine the collection (or layer name, as defined by a layer's
+    ``layer.conf`` file) containing the specified file.
+
+    Arguments:
+
+    -  ``filename``: the filename to look for.
+    -  ``d``: the data store.
+    -  ``collection_res``: dictionary with the layer names as keys and file
+       patterns to match as defined with the BBFILE_COLLECTIONS and
+       BBFILE_PATTERN variables respectively. The return value of
+       ``bb.utils.get_collection_res()`` is the default if this variable is
+       not specified.
+
+    Returns the layer name containing the file. If multiple layers contain the
+    file, the last matching layer name from collection_res is returned.
+    """
     if not collection_res:
         collection_res = get_collection_res(d)
 
@@ -1546,7 +1885,13 @@ class PrCtlError(Exception):
 
 def signal_on_parent_exit(signame):
     """
-    Trigger signame to be sent when the parent process dies
+    Trigger ``signame`` to be sent when the parent process dies.
+
+    Arguments:
+
+    -  ``signame``: name of the signal. See ``man signal``.
+
+    No return value.
     """
     signum = getattr(signal, signame)
     # http://linux.die.net/man/2/prctl
@@ -1581,7 +1926,7 @@ def ioprio_set(who, cls, value):
         bb.warn("Unable to set IO Prio for arch %s" % _unamearch)
 
 def set_process_name(name):
-    from ctypes import cdll, byref, create_string_buffer
+    from ctypes import byref, create_string_buffer
     # This is nice to have for debugging, not essential
     try:
         libc = cdll.LoadLibrary('libc.so.6')
@@ -1590,33 +1935,96 @@ def set_process_name(name):
     except:
         pass
 
-def export_proxies(d):
-    """ export common proxies variables from datastore to environment """
-    import os
+def enable_loopback_networking():
+    # From bits/ioctls.h
+    SIOCGIFFLAGS = 0x8913
+    SIOCSIFFLAGS = 0x8914
+    SIOCSIFADDR = 0x8916
+    SIOCSIFNETMASK = 0x891C
 
-    variables = ['http_proxy', 'HTTP_PROXY', 'https_proxy', 'HTTPS_PROXY',
-                    'ftp_proxy', 'FTP_PROXY', 'no_proxy', 'NO_PROXY',
-                    'GIT_PROXY_COMMAND']
-    exported = False
+    # if.h
+    IFF_UP = 0x1
+    IFF_RUNNING = 0x40
 
-    for v in variables:
-        if v in os.environ.keys():
-            exported = True
-        else:
-            v_proxy = d.getVar(v)
-            if v_proxy is not None:
-                os.environ[v] = v_proxy
-                exported = True
+    # bits/socket.h
+    AF_INET = 2
+
+    # char ifr_name[IFNAMSIZ=16]
+    ifr_name = struct.pack("@16s", b"lo")
+    def netdev_req(fd, req, data = b""):
+        # Pad and add interface name
+        data = ifr_name + data + (b'\x00' * (16 - len(data)))
+        # Return all data after interface name
+        return fcntl.ioctl(fd, req, data)[16:]
+
+    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_IP) as sock:
+        fd = sock.fileno()
+
+        # struct sockaddr_in ifr_addr { unsigned short family; uint16_t sin_port ; uint32_t in_addr; }
+        req = struct.pack("@H", AF_INET) + struct.pack("=H4B", 0, 127, 0, 0, 1)
+        netdev_req(fd, SIOCSIFADDR, req)
 
-    return exported
+        # short ifr_flags
+        flags = struct.unpack_from('@h', netdev_req(fd, SIOCGIFFLAGS))[0]
+        flags |= IFF_UP | IFF_RUNNING
+        netdev_req(fd, SIOCSIFFLAGS, struct.pack('@h', flags))
 
+        # struct sockaddr_in ifr_netmask
+        req = struct.pack("@H", AF_INET) + struct.pack("=H4B", 0, 255, 0, 0, 0)
+        netdev_req(fd, SIOCSIFNETMASK, req)
+
+def disable_network(uid=None, gid=None):
+    """
+    Disable networking in the current process if the kernel supports it, else
+    just return after logging to debug. To do this we need to create a new user
+    namespace, then map back to the original uid/gid.
+
+    Arguments:
+
+    -  ``uid``: original user id.
+    -  ``gid``: original user group id.
+
+    No return value.
+    """
+    libc = ctypes.CDLL('libc.so.6')
+
+    # From sched.h
+    # New user namespace
+    CLONE_NEWUSER = 0x10000000
+    # New network namespace
+    CLONE_NEWNET = 0x40000000
+
+    if uid is None:
+        uid = os.getuid()
+    if gid is None:
+        gid = os.getgid()
+
+    ret = libc.unshare(CLONE_NEWNET | CLONE_NEWUSER)
+    if ret != 0:
+        logger.debug("System doesn't support disabling network without admin privs")
+        return
+    with open("/proc/self/uid_map", "w") as f:
+        f.write("%s %s 1" % (uid, uid))
+    with open("/proc/self/setgroups", "w") as f:
+        f.write("deny")
+    with open("/proc/self/gid_map", "w") as f:
+        f.write("%s %s 1" % (gid, gid))
+
+def export_proxies(d):
+    from bb.fetch2 import get_fetcher_environment
+    """ export common proxies variables from datastore to environment """
+    newenv = get_fetcher_environment(d)
+    for v in newenv:
+        os.environ[v] = newenv[v]
 
 def load_plugins(logger, plugins, pluginpath):
     def load_plugin(name):
         logger.debug('Loading plugin %s' % name)
         spec = importlib.machinery.PathFinder.find_spec(name, path=[pluginpath] )
         if spec:
-            return spec.loader.load_module()
+            mod = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(mod)
+            return mod
 
     logger.debug('Loading plugins from %s...' % pluginpath)
 
@@ -1646,9 +2054,14 @@ class LogCatcher(logging.Handler):
 
 def is_semver(version):
     """
-        Is the version string following the semver semantic?
+    Arguments:
+
+    -  ``version``: the version string.
+
+    Returns ``True`` if the version string follow semantic versioning, ``False``
+    otherwise.
 
-        https://semver.org/spec/v2.0.0.html
+    See https://semver.org/spec/v2.0.0.html.
     """
     regex = re.compile(
     r"""
@@ -1669,3 +2082,150 @@ def is_semver(version):
         return False
 
     return True
+
+# Wrapper around os.rename which can handle cross device problems
+# e.g. from container filesystems
+def rename(src, dst):
+    try:
+        os.rename(src, dst)
+    except OSError as err:
+        if err.errno == 18:
+            # Invalid cross-device link error
+            shutil.move(src, dst)
+        else:
+            raise err
+
+@contextmanager
+def environment(**envvars):
+    """
+    Context manager to selectively update the environment with the specified mapping.
+
+    No return value.
+    """
+    backup = dict(os.environ)
+    try:
+        os.environ.update(envvars)
+        yield
+    finally:
+        for var in envvars:
+            if var in backup:
+                os.environ[var] = backup[var]
+            elif var in os.environ:
+                del os.environ[var]
+
+def is_local_uid(uid=''):
+    """
+    Check whether uid is a local one or not.
+    Can't use pwd module since it gets all UIDs, not local ones only.
+
+    Arguments:
+
+    -  ``uid``: user id. If not specified the user id is determined from
+       ``os.getuid()``.
+
+    Returns ``True`` is the user id is local, ``False`` otherwise.
+    """
+    if not uid:
+        uid = os.getuid()
+    with open('/etc/passwd', 'r') as f:
+        for line in f:
+            line_split = line.split(':')
+            if len(line_split) < 3:
+                continue
+            if str(uid) == line_split[2]:
+                return True
+    return False
+
+def mkstemp(suffix=None, prefix=None, dir=None, text=False):
+    """
+    Generates a unique temporary file, independent of time.
+
+    mkstemp() in glibc (at least) generates unique file names based on the
+    current system time. When combined with highly parallel builds, and
+    operating over NFS (e.g. shared sstate/downloads) this can result in
+    conflicts and race conditions.
+
+    This function adds additional entropy to the file name so that a collision
+    is independent of time and thus extremely unlikely.
+
+    Arguments:
+
+    -  ``suffix``: filename suffix.
+    -  ``prefix``: filename prefix.
+    -  ``dir``: directory where the file will be created.
+    -  ``text``: if ``True``, the file is opened in text mode.
+
+    Returns a tuple containing:
+
+    -  the file descriptor for the created file
+    -  the name of the file.
+    """
+    entropy = "".join(random.choices("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890", k=20))
+    if prefix:
+        prefix = prefix + entropy
+    else:
+        prefix = tempfile.gettempprefix() + entropy
+    return tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text)
+
+def path_is_descendant(descendant, ancestor):
+    """
+    Returns ``True`` if the path ``descendant`` is a descendant of ``ancestor``
+    (including being equivalent to ``ancestor`` itself). Otherwise returns
+    ``False``.
+
+    Correctly accounts for symlinks, bind mounts, etc. by using
+    ``os.path.samestat()`` to compare paths.
+
+    May raise any exception that ``os.stat()`` raises.
+
+    Arguments:
+
+    -  ``descendant``: path to check for being an ancestor.
+    -  ``ancestor``: path to the ancestor ``descendant`` will be checked
+       against.
+    """
+
+    ancestor_stat = os.stat(ancestor)
+
+    # Recurse up each directory component of the descendant to see if it is
+    # equivalent to the ancestor
+    check_dir = os.path.abspath(descendant).rstrip("/")
+    while check_dir:
+        check_stat = os.stat(check_dir)
+        if os.path.samestat(check_stat, ancestor_stat):
+            return True
+        check_dir = os.path.dirname(check_dir).rstrip("/")
+
+    return False
+
+# If we don't have a timeout of some kind and a process/thread exits badly (for example
+# OOM killed) and held a lock, we'd just hang in the lock futex forever. It is better
+# we exit at some point than hang. 5 minutes with no progress means we're probably deadlocked.
+# This function can still deadlock python since it can't signal the other threads to exit
+# (signals are handled in the main thread) and even os._exit() will wait on non-daemon threads
+# to exit.
+@contextmanager
+def lock_timeout(lock):
+    try:
+        s = signal.pthread_sigmask(signal.SIG_BLOCK, signal.valid_signals())
+        held = lock.acquire(timeout=5*60)
+        if not held:
+            bb.server.process.serverlog("Couldn't get the lock for 5 mins, timed out, exiting.\n%s" % traceback.format_stack())
+            os._exit(1)
+        yield held
+    finally:
+        lock.release()
+        signal.pthread_sigmask(signal.SIG_SETMASK, s)
+
+# A version of lock_timeout without the check that the lock was locked and a shorter timeout
+@contextmanager
+def lock_timeout_nocheck(lock):
+    l = False
+    try:
+        s = signal.pthread_sigmask(signal.SIG_BLOCK, signal.valid_signals())
+        l = lock.acquire(timeout=10)
+        yield l
+    finally:
+        if l:
+            lock.release()
+        signal.pthread_sigmask(signal.SIG_SETMASK, s)
diff --git a/bitbake/lib/bb/xattr.py b/bitbake/lib/bb/xattr.py
new file mode 100755
index 0000000000..7b634944a4
--- /dev/null
+++ b/bitbake/lib/bb/xattr.py
@@ -0,0 +1,126 @@
+#! /usr/bin/env python3
+#
+# Copyright 2023 by Garmin Ltd. or its subsidiaries
+#
+# SPDX-License-Identifier: MIT
+
+import sys
+import ctypes
+import os
+import errno
+
+libc = ctypes.CDLL("libc.so.6", use_errno=True)
+fsencoding = sys.getfilesystemencoding()
+
+
+libc.listxattr.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t]
+libc.llistxattr.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t]
+
+
+def listxattr(path, follow=True):
+    func = libc.listxattr if follow else libc.llistxattr
+
+    os_path = os.fsencode(path)
+
+    while True:
+        length = func(os_path, None, 0)
+
+        if length < 0:
+            err = ctypes.get_errno()
+            raise OSError(err, os.strerror(err), str(path))
+
+        if length == 0:
+            return []
+
+        arr = ctypes.create_string_buffer(length)
+
+        read_length = func(os_path, arr, length)
+        if read_length != length:
+            # Race!
+            continue
+
+        return [a.decode(fsencoding) for a in arr.raw.split(b"\x00") if a]
+
+
+libc.getxattr.argtypes = [
+    ctypes.c_char_p,
+    ctypes.c_char_p,
+    ctypes.c_char_p,
+    ctypes.c_size_t,
+]
+libc.lgetxattr.argtypes = [
+    ctypes.c_char_p,
+    ctypes.c_char_p,
+    ctypes.c_char_p,
+    ctypes.c_size_t,
+]
+
+
+def getxattr(path, name, follow=True):
+    func = libc.getxattr if follow else libc.lgetxattr
+
+    os_path = os.fsencode(path)
+    os_name = os.fsencode(name)
+
+    while True:
+        length = func(os_path, os_name, None, 0)
+
+        if length < 0:
+            err = ctypes.get_errno()
+            if err == errno.ENODATA:
+                return None
+            raise OSError(err, os.strerror(err), str(path))
+
+        if length == 0:
+            return ""
+
+        arr = ctypes.create_string_buffer(length)
+
+        read_length = func(os_path, os_name, arr, length)
+        if read_length != length:
+            # Race!
+            continue
+
+        return arr.raw
+
+
+def get_all_xattr(path, follow=True):
+    attrs = {}
+
+    names = listxattr(path, follow)
+
+    for name in names:
+        value = getxattr(path, name, follow)
+        if value is None:
+            # This can happen if a value is erased after listxattr is called,
+            # so ignore it
+            continue
+        attrs[name] = value
+
+    return attrs
+
+
+def main():
+    import argparse
+    from pathlib import Path
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", help="File Path", type=Path)
+
+    args = parser.parse_args()
+
+    attrs = get_all_xattr(args.path)
+
+    for name, value in attrs.items():
+        try:
+            value = value.decode(fsencoding)
+        except UnicodeDecodeError:
+            pass
+
+        print(f"{name} = {value}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/bitbake/lib/bblayers/__init__.py b/bitbake/lib/bblayers/__init__.py
index 4e7c09da04..78efd29750 100644
--- a/bitbake/lib/bblayers/__init__.py
+++ b/bitbake/lib/bblayers/__init__.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
diff --git a/bitbake/lib/bblayers/action.py b/bitbake/lib/bblayers/action.py
index f05f5d330f..a14f19948e 100644
--- a/bitbake/lib/bblayers/action.py
+++ b/bitbake/lib/bblayers/action.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -9,6 +11,7 @@ import shutil
 import sys
 import tempfile
 
+from bb.cookerdata import findTopdir
 import bb.utils
 
 from bblayers.common import LayerPlugin
@@ -35,7 +38,7 @@ class ActionPlugin(LayerPlugin):
                 sys.stderr.write("Specified layer directory %s doesn't contain a conf/layer.conf file\n" % layerdir)
                 return 1
 
-        bblayers_conf = os.path.join('conf', 'bblayers.conf')
+        bblayers_conf = os.path.join(findTopdir(),'conf', 'bblayers.conf')
         if not os.path.exists(bblayers_conf):
             sys.stderr.write("Unable to find bblayers.conf\n")
             return 1
@@ -48,12 +51,14 @@ class ActionPlugin(LayerPlugin):
         try:
             notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdirs, None)
             if not (args.force or notadded):
+                self.tinfoil.modified_files()
                 try:
                     self.tinfoil.run_command('parseConfiguration')
                 except (bb.tinfoil.TinfoilUIException, bb.BBHandledException):
                     # Restore the back up copy of bblayers.conf
                     shutil.copy2(backup, bblayers_conf)
-                    bb.fatal("Parse failure with the specified layer added, aborting.")
+                    self.tinfoil.modified_files()
+                    bb.fatal("Parse failure with the specified layer added, exiting.")
                 else:
                     for item in notadded:
                         sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item)
@@ -63,7 +68,7 @@ class ActionPlugin(LayerPlugin):
 
     def do_remove_layer(self, args):
         """Remove one or more layers from bblayers.conf."""
-        bblayers_conf = os.path.join('conf', 'bblayers.conf')
+        bblayers_conf = os.path.join(findTopdir() ,'conf', 'bblayers.conf')
         if not os.path.exists(bblayers_conf):
             sys.stderr.write("Unable to find bblayers.conf\n")
             return 1
@@ -78,6 +83,9 @@ class ActionPlugin(LayerPlugin):
                 layerdir = os.path.abspath(item)
             layerdirs.append(layerdir)
         (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdirs)
+        if args.force > 1:
+            return 0
+        self.tinfoil.modified_files()
         if notremoved:
             for item in notremoved:
                 sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
@@ -237,6 +245,9 @@ build results (as the layer priority order has effectively changed).
                             if not entry_found:
                                 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
 
+        self.tinfoil.modified_files()
+
+
     def get_file_layer(self, filename):
         layerdir = self.get_file_layerdir(filename)
         if layerdir:
diff --git a/bitbake/lib/bblayers/common.py b/bitbake/lib/bblayers/common.py
index 6c76ef3505..f7b9cee371 100644
--- a/bitbake/lib/bblayers/common.py
+++ b/bitbake/lib/bblayers/common.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
diff --git a/bitbake/lib/bblayers/layerindex.py b/bitbake/lib/bblayers/layerindex.py
index b2f27b21ee..ba91fac669 100644
--- a/bitbake/lib/bblayers/layerindex.py
+++ b/bitbake/lib/bblayers/layerindex.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -47,6 +49,31 @@ class LayerIndexPlugin(ActionPlugin):
             else:
                 logger.plain("Repository %s needs to be fetched" % url)
                 return subdir, layername, layerdir
+        elif os.path.exists(repodir) and branch:
+            """
+            If the repo is already cloned, ensure it is on the correct branch,
+            switching branches if necessary and possible.
+            """
+            base_cmd = ['git', '--git-dir=%s/.git' % repodir, '--work-tree=%s' % repodir]
+            cmd = base_cmd + ['branch']
+            completed_proc = subprocess.run(cmd, text=True, capture_output=True)
+            if completed_proc.returncode:
+                logger.error("Unable to validate repo %s (%s)" % (repodir, stderr))
+                return None, None, None
+            else:
+                if branch != completed_proc.stdout[2:-1]:
+                    cmd = base_cmd + ['status', '--short']
+                    completed_proc = subprocess.run(cmd, text=True, capture_output=True)
+                    if completed_proc.stdout.count('\n') != 0:
+                        logger.warning("There are uncommitted changes in repo %s" % repodir)
+                    cmd = base_cmd + ['checkout', branch]
+                    completed_proc = subprocess.run(cmd, text=True, capture_output=True)
+                    if completed_proc.returncode:
+                        # Could be due to original shallow clone on a different branch for example
+                        logger.error("Unable to automatically switch %s to desired branch '%s' (%s)"
+                                     % (repodir, branch, completed_proc.stderr))
+                        return None, None, None
+            return subdir, layername, layerdir
         elif os.path.exists(layerdir):
             return subdir, layername, layerdir
         else:
@@ -159,12 +186,17 @@ class LayerIndexPlugin(ActionPlugin):
                 logger.plain('  recommended by: %s' % ' '.join(recommendedby))
 
         if dependencies:
-            fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
-            if not fetchdir:
-                logger.error("Cannot get BBLAYERS_FETCH_DIR")
-                return 1
+            if args.fetchdir:
+                fetchdir = args.fetchdir
+            else:
+                fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
+                if not fetchdir:
+                    logger.error("Cannot get BBLAYERS_FETCH_DIR")
+                    return 1
+
             if not os.path.exists(fetchdir):
                 os.makedirs(fetchdir)
+
             addlayers = []
 
             for deplayerbranch in dependencies:
@@ -206,6 +238,8 @@ class LayerIndexPlugin(ActionPlugin):
 """
         args.show_only = True
         args.ignore = []
+        args.fetchdir = ""
+        args.shallow = True
         self.do_layerindex_fetch(args)
 
     def register_commands(self, sp):
@@ -214,6 +248,7 @@ class LayerIndexPlugin(ActionPlugin):
         parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch')
         parser_layerindex_fetch.add_argument('-s', '--shallow', help='do only shallow clones (--depth=1)', action='store_true')
         parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER')
+        parser_layerindex_fetch.add_argument('-f', '--fetchdir', help='directory to fetch the layer(s) into (will be created if it does not exist)')
         parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch')
 
         parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends, parserecipes=False)
diff --git a/bitbake/lib/bblayers/query.py b/bitbake/lib/bblayers/query.py
index f5e3c84747..eb7cb465b4 100644
--- a/bitbake/lib/bblayers/query.py
+++ b/bitbake/lib/bblayers/query.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -27,12 +29,12 @@ class QueryPlugin(LayerPlugin):
 
     def do_show_layers(self, args):
         """show current configured layers."""
-        logger.plain("%s  %s  %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
-        logger.plain('=' * 74)
+        logger.plain("%s  %s  %s" % ("layer".ljust(20), "path".ljust(70), "priority"))
+        logger.plain('=' * 104)
         for layer, _, regex, pri in self.tinfoil.cooker.bbfile_config_priorities:
             layerdir = self.bbfile_collections.get(layer, None)
-            layername = self.get_layer_name(layerdir)
-            logger.plain("%s  %s  %d" % (layername.ljust(20), layerdir.ljust(40), pri))
+            layername = layer
+            logger.plain("%s  %s  %s" % (layername.ljust(20), layerdir.ljust(70), pri))
 
     def version_str(self, pe, pv, pr = None):
         verstr = "%s" % pv
@@ -55,11 +57,12 @@ are overlayed will also be listed, with a " (skipped)" suffix.
         # Check for overlayed .bbclass files
         classes = collections.defaultdict(list)
         for layerdir in self.bblayers:
-            classdir = os.path.join(layerdir, 'classes')
-            if os.path.exists(classdir):
-                for classfile in os.listdir(classdir):
-                    if os.path.splitext(classfile)[1] == '.bbclass':
-                        classes[classfile].append(classdir)
+            for c in ["classes-global", "classes-recipe", "classes"]:
+                classdir = os.path.join(layerdir, c)
+                if os.path.exists(classdir):
+                    for classfile in os.listdir(classdir):
+                        if os.path.splitext(classfile)[1] == '.bbclass':
+                            classes[classfile].append(classdir)
 
         # Locating classes and other files is a bit more complicated than recipes -
         # layer priority is not a factor; instead BitBake uses the first matching
@@ -122,22 +125,27 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
         if inherits:
             bbpath = str(self.tinfoil.config_data.getVar('BBPATH'))
             for classname in inherits:
-                classfile = 'classes/%s.bbclass' % classname
-                if not bb.utils.which(bbpath, classfile, history=False):
-                    logger.error('No class named %s found in BBPATH', classfile)
+                found = False
+                for c in ["classes-global", "classes-recipe", "classes"]:
+                    cfile = c + '/%s.bbclass' % classname
+                    if bb.utils.which(bbpath, cfile, history=False):
+                        found = True
+                        break
+                if not found:
+                    logger.error('No class named %s found in BBPATH', classname)
                     sys.exit(1)
 
         pkg_pn = self.tinfoil.cooker.recipecaches[mc].pkg_pn
-        (latest_versions, preferred_versions) = self.tinfoil.find_providers(mc)
+        (latest_versions, preferred_versions, required_versions) = self.tinfoil.find_providers(mc)
         allproviders = self.tinfoil.get_all_providers(mc)
 
         # Ensure we list skipped recipes
         # We are largely guessing about PN, PV and the preferred version here,
         # but we have no choice since skipped recipes are not fully parsed
-        skiplist = list(self.tinfoil.cooker.skiplist.keys())
-        mcspec = 'mc:%s:' % mc
+        skiplist = list(self.tinfoil.cooker.skiplist_by_mc[mc].keys())
+
         if mc:
-            skiplist = [s[len(mcspec):] for s in skiplist if s.startswith(mcspec)]
+            skiplist = [s.removeprefix(f'mc:{mc}:') for s in skiplist]
 
         for fn in skiplist:
             recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
@@ -154,7 +162,7 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
         def print_item(f, pn, ver, layer, ispref):
             if not selected_layer or layer == selected_layer:
                 if not bare and f in skiplist:
-                    skipped = ' (skipped)'
+                    skipped = ' (skipped: %s)' % self.tinfoil.cooker.skiplist_by_mc[mc][f].skipreason
                 else:
                     skipped = ''
                 if show_filenames:
@@ -172,7 +180,7 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
                     logger.plain("  %s %s%s", layer.ljust(20), ver, skipped)
 
         global_inherit = (self.tinfoil.config_data.getVar('INHERIT') or "").split()
-        cls_re = re.compile('classes/')
+        cls_re = re.compile('classes.*/')
 
         preffiles = []
         show_unique_pn = []
@@ -274,7 +282,10 @@ Lists recipes with the bbappends that apply to them as subitems.
         else:
             logger.plain('=== Appended recipes ===')
 
-        pnlist = list(self.tinfoil.cooker_data.pkg_pn.keys())
+
+        cooker_data = self.tinfoil.cooker.recipecaches[args.mc]
+
+        pnlist = list(cooker_data.pkg_pn.keys())
         pnlist.sort()
         appends = False
         for pn in pnlist:
@@ -287,26 +298,28 @@ Lists recipes with the bbappends that apply to them as subitems.
                 if not found:
                     continue
 
-            if self.show_appends_for_pn(pn):
+            if self.show_appends_for_pn(pn, cooker_data, args.mc):
                 appends = True
 
-        if not args.pnspec and self.show_appends_for_skipped():
+        if not args.pnspec and self.show_appends_for_skipped(args.mc):
             appends = True
 
         if not appends:
             logger.plain('No append files found')
 
-    def show_appends_for_pn(self, pn):
-        filenames = self.tinfoil.cooker_data.pkg_pn[pn]
+    def show_appends_for_pn(self, pn, cooker_data, mc):
+        filenames = cooker_data.pkg_pn[pn]
+        if mc:
+            pn = "mc:%s:%s" % (mc, pn)
 
         best = self.tinfoil.find_best_provider(pn)
         best_filename = os.path.basename(best[3])
 
         return self.show_appends_output(filenames, best_filename)
 
-    def show_appends_for_skipped(self):
+    def show_appends_for_skipped(self, mc):
         filenames = [os.path.basename(f)
-                    for f in self.tinfoil.cooker.skiplist.keys()]
+                    for f in self.tinfoil.cooker.skiplist_by_mc[mc].keys()]
         return self.show_appends_output(filenames, None, " (skipped)")
 
     def show_appends_output(self, filenames, best_filename, name_suffix = ''):
@@ -405,7 +418,7 @@ NOTE: .bbappend files can impact the dependencies.
                     self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers)
 
             # The inherit class
-            cls_re = re.compile('classes/')
+            cls_re = re.compile('classes.*/')
             if f in self.tinfoil.cooker_data.inherits:
                 inherits = self.tinfoil.cooker_data.inherits[f]
                 for cls in inherits:
@@ -441,10 +454,10 @@ NOTE: .bbappend files can impact the dependencies.
                     line = fnfile.readline()
 
         # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass
-        conf_re = re.compile(".*/conf/machine/[^\/]*\.conf$")
-        inc_re = re.compile(".*\.inc$")
+        conf_re = re.compile(r".*/conf/machine/[^\/]*\.conf$")
+        inc_re = re.compile(r".*\.inc$")
         # The "inherit xxx" in .bbclass
-        bbclass_re = re.compile(".*\.bbclass$")
+        bbclass_re = re.compile(r".*\.bbclass$")
         for layerdir in self.bblayers:
             layername = self.get_layer_name(layerdir)
             for dirpath, dirnames, filenames in os.walk(layerdir):
@@ -522,6 +535,7 @@ NOTE: .bbappend files can impact the dependencies.
 
         parser_show_appends = self.add_command(sp, 'show-appends', self.do_show_appends)
         parser_show_appends.add_argument('pnspec', nargs='*', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)')
+        parser_show_appends.add_argument('--mc', help='use specified multiconfig', default='')
 
         parser_show_cross_depends = self.add_command(sp, 'show-cross-depends', self.do_show_cross_depends)
         parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true')
diff --git a/bitbake/lib/bs4/AUTHORS b/bitbake/lib/bs4/AUTHORS
new file mode 100644
index 0000000000..1f14fe07de
--- /dev/null
+++ b/bitbake/lib/bs4/AUTHORS
@@ -0,0 +1,49 @@
+Behold, mortal, the origins of Beautiful Soup...
+================================================
+
+Leonard Richardson is the primary maintainer.
+
+Aaron DeVore and Isaac Muse have made significant contributions to the
+code base.
+
+Mark Pilgrim provided the encoding detection code that forms the base
+of UnicodeDammit.
+
+Thomas Kluyver and Ezio Melotti finished the work of getting Beautiful
+Soup 4 working under Python 3.
+
+Simon Willison wrote soupselect, which was used to make Beautiful Soup
+support CSS selectors. Isaac Muse wrote SoupSieve, which made it
+possible to _remove_ the CSS selector code from Beautiful Soup.
+
+Sam Ruby helped with a lot of edge cases.
+
+Jonathan Ellis was awarded the prestigious Beau Potage D'Or for his
+work in solving the nestable tags conundrum.
+
+An incomplete list of people have contributed patches to Beautiful
+Soup:
+
+ Istvan Albert, Andrew Lin, Anthony Baxter, Oliver Beattie, Andrew
+Boyko, Tony Chang, Francisco Canas, "Delong", Zephyr Fang, Fuzzy,
+Roman Gaufman, Yoni Gilad, Richie Hindle, Toshihiro Kamiya, Peteris
+Krumins, Kent Johnson, Marek Kapolka, Andreas Kostyrka, Roel Kramer,
+Ben Last, Robert Leftwich, Stefaan Lippens, "liquider", Staffan
+Malmgren, Ksenia Marasanova, JP Moins, Adam Monsen, John Nagle, "Jon",
+Ed Oskiewicz, Martijn Peters, Greg Phillips, Giles Radford, Stefano
+Revera, Arthur Rudolph, Marko Samastur, James Salter, Jouni Seppänen,
+Alexander Schmolck, Tim Shirley, Geoffrey Sneddon, Ville Skyttä,
+"Vikas", Jens Svalgaard, Andy Theyers, Eric Weiser, Glyn Webster, John
+Wiseman, Paul Wright, Danny Yoo
+
+An incomplete list of people who made suggestions or found bugs or
+found ways to break Beautiful Soup:
+
+ Hanno Böck, Matteo Bertini, Chris Curvey, Simon Cusack, Bruce Eckel,
+ Matt Ernst, Michael Foord, Tom Harris, Bill de hOra, Donald Howes,
+ Matt Patterson, Scott Roberts, Steve Strassmann, Mike Williams,
+ warchild at redho dot com, Sami Kuisma, Carlos Rocha, Bob Hutchison,
+ Joren Mc, Michal Migurski, John Kleven, Tim Heaney, Tripp Lilley, Ed
+ Summers, Dennis Sutch, Chris Smith, Aaron Swartz, Stuart
+ Turner, Greg Edwards, Kevin J Kalupson, Nikos Kouremenos, Artur de
+ Sousa Rocha, Yichun Wei, Per Vognsen
diff --git a/bitbake/lib/bs4/AUTHORS.txt b/bitbake/lib/bs4/AUTHORS.txt
deleted file mode 100644
index 2ac8fcc8cc..0000000000
--- a/bitbake/lib/bs4/AUTHORS.txt
+++ /dev/null
@@ -1,43 +0,0 @@
-Behold, mortal, the origins of Beautiful Soup...
-================================================
-
-Leonard Richardson is the primary programmer.
-
-Aaron DeVore is awesome.
-
-Mark Pilgrim provided the encoding detection code that forms the base
-of UnicodeDammit.
-
-Thomas Kluyver and Ezio Melotti finished the work of getting Beautiful
-Soup 4 working under Python 3.
-
-Simon Willison wrote soupselect, which was used to make Beautiful Soup
-support CSS selectors.
-
-Sam Ruby helped with a lot of edge cases.
-
-Jonathan Ellis was awarded the prestigous Beau Potage D'Or for his
-work in solving the nestable tags conundrum.
-
-An incomplete list of people have contributed patches to Beautiful
-Soup:
-
- Istvan Albert, Andrew Lin, Anthony Baxter, Andrew Boyko, Tony Chang,
- Zephyr Fang, Fuzzy, Roman Gaufman, Yoni Gilad, Richie Hindle, Peteris
- Krumins, Kent Johnson, Ben Last, Robert Leftwich, Staffan Malmgren,
- Ksenia Marasanova, JP Moins, Adam Monsen, John Nagle, "Jon", Ed
- Oskiewicz, Greg Phillips, Giles Radford, Arthur Rudolph, Marko
- Samastur, Jouni Seppänen, Alexander Schmolck, Andy Theyers, Glyn
- Webster, Paul Wright, Danny Yoo
-
-An incomplete list of people who made suggestions or found bugs or
-found ways to break Beautiful Soup:
-
- Hanno Böck, Matteo Bertini, Chris Curvey, Simon Cusack, Bruce Eckel,
- Matt Ernst, Michael Foord, Tom Harris, Bill de hOra, Donald Howes,
- Matt Patterson, Scott Roberts, Steve Strassmann, Mike Williams,
- warchild at redho dot com, Sami Kuisma, Carlos Rocha, Bob Hutchison,
- Joren Mc, Michal Migurski, John Kleven, Tim Heaney, Tripp Lilley, Ed
- Summers, Dennis Sutch, Chris Smith, Aaron Sweep^W Swartz, Stuart
- Turner, Greg Edwards, Kevin J Kalupson, Nikos Kouremenos, Artur de
- Sousa Rocha, Yichun Wei, Per Vognsen
diff --git a/bitbake/lib/bs4/NEWS.txt b/bitbake/lib/bs4/CHANGELOG
index 88a60a2458..2701446a6d 100644
--- a/bitbake/lib/bs4/NEWS.txt
+++ b/bitbake/lib/bs4/CHANGELOG
@@ -1,3 +1,776 @@
+= 4.12.3 (20240117)
+
+* The Beautiful Soup documentation now has a Spanish translation, thanks
+  to Carlos Romero. Delong Wang's Chinese translation has been updated
+  to cover Beautiful Soup 4.12.0.
+
+* Fixed a regression such that if you set .hidden on a tag, the tag
+  becomes invisible but its contents are still visible. User manipulation
+  of .hidden is not a documented or supported feature, so don't do this,
+  but it wasn't too difficult to keep the old behavior working.
+
+* Fixed a case found by Mengyuhan where html.parser giving up on
+  markup would result in an AssertionError instead of a
+  ParserRejectedMarkup exception.
+
+* Added the correct stacklevel to instances of the XMLParsedAsHTMLWarning.
+  [bug=2034451]
+
+* Corrected the syntax of the license definition in pyproject.toml. Patch
+  by Louis Maddox. [bug=2032848]
+
+* Corrected a typo in a test that was causing test failures when run against
+  libxml2 2.12.1. [bug=2045481]
+
+= 4.12.2 (20230407)
+
+* Fixed an unhandled exception in BeautifulSoup.decode_contents
+  and methods that call it. [bug=2015545]
+
+= 4.12.1 (20230405)
+
+NOTE: the following things are likely to be dropped in the next
+feature release of Beautiful Soup:
+
+ Official support for Python 3.6.
+ Inclusion of unit tests and test data in the wheel file.
+ Two scripts: demonstrate_parser_differences.py and test-all-versions.
+
+Changes:
+
+* This version of Beautiful Soup replaces setup.py and setup.cfg
+  with pyproject.toml. Beautiful Soup now uses tox as its test backend
+  and hatch to do builds.
+
+* The main functional improvement in this version is a nonrecursive technique
+  for regenerating a tree. This technique is used to avoid situations where,
+  in previous versions, doing something to a very deeply nested tree
+  would overflow the Python interpreter stack:
+
+  1. Outputting a tree as a string, e.g. with
+     BeautifulSoup.encode() [bug=1471755]
+
+  2. Making copies of trees (copy.copy() and
+     copy.deepcopy() from the Python standard library). [bug=1709837]
+
+  3. Pickling a BeautifulSoup object. (Note that pickling a Tag
+     object can still cause an overflow.)
+
+* Making a copy of a BeautifulSoup object no longer parses the
+  document again, which should improve performance significantly.
+
+* When a BeautifulSoup object is unpickled, Beautiful Soup now
+  tries to associate an appropriate TreeBuilder object with it.
+
+* Tag.prettify() will now consistently end prettified markup with
+  a newline.
+
+* Added unit tests for fuzz test cases created by third
+  parties. Some of these tests are skipped since they point
+  to problems outside of Beautiful Soup, but this change
+  puts them all in one convenient place.
+
+* PageElement now implements the known_xml attribute. (This was technically
+  a bug, but it shouldn't be an issue in normal use.) [bug=2007895]
+
+* The demonstrate_parser_differences.py script was still written in
+  Python 2. I've converted it to Python 3, but since no one has
+  mentioned this over the years, it's a sign that no one uses this
+  script and it's not serving its purpose.
+
+= 4.12.0 (20230320)
+
+* Introduced the .css property, which centralizes all access to
+  the Soup Sieve API. This allows Beautiful Soup to give direct
+  access to as much of Soup Sieve that makes sense, without cluttering
+  the BeautifulSoup and Tag classes with a lot of new methods.
+
+  This does mean one addition to the BeautifulSoup and Tag classes
+  (the .css property itself), so this might be a breaking change if you
+  happen to use Beautiful Soup to parse XML that includes a tag called
+  <css>. In particular, code like this will stop working in 4.12.0:
+
+    soup.css['id']
+
+  Code like this will work just as before:
+
+    soup.find_one('css')['id']
+
+  The Soup Sieve methods supported through the .css property are
+  select(), select_one(), iselect(), closest(), match(), filter(),
+  escape(), and compile(). The BeautifulSoup and Tag classes still
+  support the select() and select_one() methods; they have not been
+  deprecated, but they have been demoted to convenience methods.
+
+  [bug=2003677]
+
+* When the html.parser parser decides it can't parse a document, Beautiful
+  Soup now consistently propagates this fact by raising a
+  ParserRejectedMarkup error. [bug=2007343]
+
+* Removed some error checking code from diagnose(), which is redundant with
+  similar (but more Pythonic) code in the BeautifulSoup constructor.
+  [bug=2007344]
+
+* Added intersphinx references to the documentation so that other
+  projects have a target to point to when they reference Beautiful
+  Soup classes. [bug=1453370]
+
+= 4.11.2 (20230131)
+
+* Fixed test failures caused by nondeterministic behavior of
+  UnicodeDammit's character detection, depending on the platform setup.
+  [bug=1973072]
+
+* Fixed another crash when overriding multi_valued_attributes and using the
+  html5lib parser. [bug=1948488]
+
+* The HTMLFormatter and XMLFormatter constructors no longer return a
+  value. [bug=1992693]
+
+* Tag.interesting_string_types is now propagated when a tag is
+  copied. [bug=1990400]
+
+* Warnings now do their best to provide an appropriate stacklevel,
+  improving the usefulness of the message. [bug=1978744]
+
+* Passing a Tag's .contents into PageElement.extend() now works the
+  same way as passing the Tag itself.
+
+* Soup Sieve tests will be skipped if the library is not installed.
+
+= 4.11.1 (20220408)
+
+This release was done to ensure that the unit tests are packaged along
+with the released source. There are no functionality changes in this
+release, but there are a few other packaging changes:
+
+* The Japanese and Korean translations of the documentation are included.
+* The changelog is now packaged as CHANGELOG, and the license file is
+  packaged as LICENSE. NEWS.txt and COPYING.txt are still present,
+  but may be removed in the future.
+* TODO.txt is no longer packaged, since a TODO is not relevant for released
+  code.
+
+= 4.11.0 (20220407)
+
+* Ported unit tests to use pytest.
+
+* Added special string classes, RubyParenthesisString and RubyTextString,
+  to make it possible to treat ruby text specially in get_text() calls.
+  [bug=1941980]
+
+* It's now possible to customize the way output is indented by
+  providing a value for the 'indent' argument to the Formatter
+  constructor. The 'indent' argument works very similarly to the
+  argument of the same name in the Python standard library's
+  json.dump() function. [bug=1955497]
+
+* If the charset-normalizer Python module
+  (https://pypi.org/project/charset-normalizer/) is installed, Beautiful
+  Soup will use it to detect the character sets of incoming documents.
+  This is also the module used by newer versions of the Requests library.
+  For the sake of backwards compatibility, chardet and cchardet both take
+  precedence if installed. [bug=1955346]
+
+* Added a workaround for an lxml bug
+  (https://bugs.launchpad.net/lxml/+bug/1948551) that causes
+  problems when parsing a Unicode string beginning with BYTE ORDER MARK.
+  [bug=1947768]
+
+* Issue a warning when an HTML parser is used to parse a document that
+  looks like XML but not XHTML. [bug=1939121]
+
+* Do a better job of keeping track of namespaces as an XML document is
+  parsed, so that CSS selectors that use namespaces will do the right
+  thing more often. [bug=1946243]
+
+* Some time ago, the misleadingly named "text" argument to find-type
+  methods was renamed to the more accurate "string." But this supposed
+  "renaming" didn't make it into important places like the method
+  signatures or the docstrings. That's corrected in this
+  version. "text" still works, but will give a DeprecationWarning.
+  [bug=1947038]
+
+* Fixed a crash when pickling a BeautifulSoup object that has no
+  tree builder. [bug=1934003]
+
+* Fixed a crash when overriding multi_valued_attributes and using the
+  html5lib parser. [bug=1948488]
+
+* Standardized the wording of the MarkupResemblesLocatorWarning
+  warnings to omit untrusted input and make the warnings less
+  judgmental about what you ought to be doing. [bug=1955450]
+
+* Removed support for the iconv_codec library, which doesn't seem
+  to exist anymore and was never put up on PyPI. (The closest
+  replacement on PyPI, iconv_codecs, is GPL-licensed, so we can't use
+  it--it's also quite old.)
+
+= 4.10.0 (20210907)
+
+* This is the first release of Beautiful Soup to only support Python
+  3. I dropped Python 2 support to maintain support for newer versions
+  (58 and up) of setuptools. See:
+  https://github.com/pypa/setuptools/issues/2769 [bug=1942919]
+
+* The behavior of methods like .get_text() and .strings now differs
+  depending on the type of tag. The change is visible with HTML tags
+  like <script>, <style>, and <template>. Starting in 4.9.0, methods
+  like get_text() returned no results on such tags, because the
+  contents of those tags are not considered 'text' within the document
+  as a whole.
+
+  But a user who calls script.get_text() is working from a different
+  definition of 'text' than a user who calls div.get_text()--otherwise
+  there would be no need to call script.get_text() at all. In 4.10.0,
+  the contents of (e.g.) a <script> tag are considered 'text' during a
+  get_text() call on the tag itself, but not considered 'text' during
+  a get_text() call on the tag's parent.
+
+  Because of this change, calling get_text() on each child of a tag
+  may now return a different result than calling get_text() on the tag
+  itself. That's because different tags now have different
+  understandings of what counts as 'text'. [bug=1906226] [bug=1868861]
+
+* NavigableString and its subclasses now implement the get_text()
+  method, as well as the properties .strings and
+  .stripped_strings. These methods will either return the string
+  itself, or nothing, so the only reason to use this is when iterating
+  over a list of mixed Tag and NavigableString objects. [bug=1904309]
+
+* The 'html5' formatter now treats attributes whose values are the
+  empty string as HTML boolean attributes. Previously (and in other
+  formatters), an attribute value must be set as None to be treated as
+  a boolean attribute. In a future release, I plan to also give this
+  behavior to the 'html' formatter. Patch by Isaac Muse. [bug=1915424]
+
+* The 'replace_with()' method now takes a variable number of arguments,
+  and can be used to replace a single element with a sequence of elements.
+  Patch by Bill Chandos. [rev=605]
+
+* Corrected output when the namespace prefix associated with a
+  namespaced attribute is the empty string, as opposed to
+  None. [bug=1915583]
+
+* Performance improvement when processing tags that speeds up overall
+  tree construction by 2%. Patch by Morotti. [bug=1899358]
+
+* Corrected the use of special string container classes in cases when a
+  single tag may contain strings with different containers; such as
+  the <template> tag, which may contain both TemplateString objects
+  and Comment objects. [bug=1913406]
+
+* The html.parser tree builder can now handle named entities
+  found in the HTML5 spec in much the same way that the html5lib
+  tree builder does. Note that the lxml HTML tree builder doesn't handle
+  named entities this way. [bug=1924908]
+
+* Added a second way to pass specify encodings to UnicodeDammit and
+  EncodingDetector, based on the order of precedence defined in the
+  HTML5 spec, starting at:
+  https://html.spec.whatwg.org/multipage/parsing.html#parsing-with-a-known-character-encoding
+
+  Encodings in 'known_definite_encodings' are tried first, then
+  byte-order-mark sniffing is run, then encodings in 'user_encodings'
+  are tried. The old argument, 'override_encodings', is now a
+  deprecated alias for 'known_definite_encodings'.
+
+  This changes the default behavior of the html.parser and lxml tree
+  builders, in a way that may slightly improve encoding
+  detection but will probably have no effect. [bug=1889014]
+
+* Improve the warning issued when a directory name (as opposed to
+  the name of a regular file) is passed as markup into the BeautifulSoup
+  constructor. [bug=1913628]
+
+= 4.9.3 (20201003)
+
+This is the final release of Beautiful Soup to support Python
+2. Beautiful Soup's official support for Python 2 ended on 01 January,
+2021. In the Launchpad Git repository, the final revision to support
+Python 2 was revision 70f546b1e689a70e2f103795efce6d261a3dadf7; it is
+tagged as "python2".
+
+* Implemented a significant performance optimization to the process of
+  searching the parse tree. Patch by Morotti. [bug=1898212]
+
+= 4.9.2 (20200926)
+
+* Fixed a bug that caused too many tags to be popped from the tag
+  stack during tree building, when encountering a closing tag that had
+  no matching opening tag. [bug=1880420]
+
+* Fixed a bug that inconsistently moved elements over when passing
+  a Tag, rather than a list, into Tag.extend(). [bug=1885710]
+
+* Specify the soupsieve dependency in a way that complies with
+  PEP 508. Patch by Mike Nerone. [bug=1893696]
+
+* Change the signatures for BeautifulSoup.insert_before and insert_after
+  (which are not implemented) to match PageElement.insert_before and
+  insert_after, quieting warnings in some IDEs. [bug=1897120]
+
+= 4.9.1 (20200517)
+
+* Added a keyword argument 'on_duplicate_attribute' to the
+  BeautifulSoupHTMLParser constructor (used by the html.parser tree
+  builder) which lets you customize the handling of markup that
+  contains the same attribute more than once, as in:
+  <a href="url1" href="url2"> [bug=1878209]
+
+* Added a distinct subclass, GuessedAtParserWarning, for the warning
+  issued when BeautifulSoup is instantiated without a parser being
+  specified. [bug=1873787]
+
+* Added a distinct subclass, MarkupResemblesLocatorWarning, for the
+  warning issued when BeautifulSoup is instantiated with 'markup' that
+  actually seems to be a URL or the path to a file on
+  disk. [bug=1873787]
+
+* The new NavigableString subclasses (Stylesheet, Script, and
+  TemplateString) can now be imported directly from the bs4 package.
+
+* If you encode a document with a Python-specific encoding like
+  'unicode_escape', that encoding is no longer mentioned in the final
+  XML or HTML document. Instead, encoding information is omitted or
+  left blank. [bug=1874955]
+
+* Fixed test failures when run against soupselect 2.0. Patch by Tomáš
+  Chvátal. [bug=1872279]
+
+= 4.9.0 (20200405)
+
+* Added PageElement.decomposed, a new property which lets you
+  check whether you've already called decompose() on a Tag or
+  NavigableString.
+
+* Embedded CSS and Javascript is now stored in distinct Stylesheet and
+  Script tags, which are ignored by methods like get_text() since most
+  people don't consider this sort of content to be 'text'. This
+  feature is not supported by the html5lib treebuilder. [bug=1868861]
+
+* Added a Russian translation by 'authoress' to the repository.
+
+* Fixed an unhandled exception when formatting a Tag that had been
+  decomposed.[bug=1857767]
+
+* Fixed a bug that happened when passing a Unicode filename containing
+  non-ASCII characters as markup into Beautiful Soup, on a system that
+  allows Unicode filenames. [bug=1866717]
+
+* Added a performance optimization to PageElement.extract(). Patch by
+  Arthur Darcet.
+
+= 4.8.2 (20191224)
+
+* Added Python docstrings to all public methods of the most commonly
+  used classes.
+
+* Added a Chinese translation by Deron Wang and a Brazilian Portuguese
+  translation by Cezar Peixeiro to the repository.
+
+* Fixed two deprecation warnings. Patches by Colin
+  Watson and Nicholas Neumann. [bug=1847592] [bug=1855301]
+
+* The html.parser tree builder now correctly handles DOCTYPEs that are
+  not uppercase. [bug=1848401]
+
+* PageElement.select() now returns a ResultSet rather than a regular
+  list, making it consistent with methods like find_all().
+
+= 4.8.1 (20191006)
+
+* When the html.parser or html5lib parsers are in use, Beautiful Soup
+  will, by default, record the position in the original document where
+  each tag was encountered. This includes line number (Tag.sourceline)
+  and position within a line (Tag.sourcepos).  Based on code by Chris
+  Mayo. [bug=1742921]
+
+* When instantiating a BeautifulSoup object, it's now possible to
+   provide a dictionary ('element_classes') of the classes you'd like to be
+   instantiated instead of Tag, NavigableString, etc.
+
+* Fixed the definition of the default XML namespace when using
+   lxml 4.4. Patch by Isaac Muse. [bug=1840141]
+
+* Fixed a crash when pretty-printing tags that were not created
+   during initial parsing. [bug=1838903]
+
+* Copying a Tag preserves information that was originally obtained from
+   the TreeBuilder used to build the original Tag. [bug=1838903]
+
+* Raise an explanatory exception when the underlying parser
+   completely rejects the incoming markup. [bug=1838877]
+
+* Avoid a crash when trying to detect the declared encoding of a
+   Unicode document. [bug=1838877]
+
+* Avoid a crash when unpickling certain parse trees generated
+   using html5lib on Python 3. [bug=1843545]
+
+= 4.8.0 (20190720, "One Small Soup")
+
+This release focuses on making it easier to customize Beautiful Soup's
+input mechanism (the TreeBuilder) and output mechanism (the Formatter).
+
+* You can customize the TreeBuilder object by passing keyword
+  arguments into the BeautifulSoup constructor. Those keyword
+  arguments will be passed along into the TreeBuilder constructor.
+
+  The main reason to do this right now is to change how which
+  attributes are treated as multi-valued attributes (the way 'class'
+  is treated by default). You can do this with the
+  'multi_valued_attributes' argument. [bug=1832978]
+
+* The role of Formatter objects has been greatly expanded. The Formatter
+  class now controls the following:
+
+  - The function to call to perform entity substitution. (This was
+    previously Formatter's only job.)
+  - Which tags should be treated as containing CDATA and have their
+    contents exempt from entity substitution.
+  - The order in which a tag's attributes are output. [bug=1812422]
+  - Whether or not to put a '/' inside a void element, e.g. '<br/>' vs '<br>'
+
+  All preexisting code should work as before.
+
+* Added a new method to the API, Tag.smooth(), which consolidates
+  multiple adjacent NavigableString elements. [bug=1697296]
+
+* &apos; (which is valid in XML, XHTML, and HTML 5, but not HTML 4) is always
+  recognized as a named entity and converted to a single quote. [bug=1818721]
+
+= 4.7.1 (20190106)
+
+* Fixed a significant performance problem introduced in 4.7.0. [bug=1810617]
+
+* Fixed an incorrectly raised exception when inserting a tag before or
+  after an identical tag. [bug=1810692]
+
+* Beautiful Soup will no longer try to keep track of namespaces that
+  are not defined with a prefix; this can confuse soupselect. [bug=1810680]
+
+* Tried even harder to avoid the deprecation warning originally fixed in
+   4.6.1. [bug=1778909]
+
+= 4.7.0 (20181231)
+
+* Beautiful Soup's CSS Selector implementation has been replaced by a
+  dependency on Isaac Muse's SoupSieve project (the soupsieve package
+  on PyPI). The good news is that SoupSieve has a much more robust and
+  complete implementation of CSS selectors, resolving a large number
+  of longstanding issues. The bad news is that from this point onward,
+  SoupSieve must be installed if you want to use the select() method.
+
+  You don't have to change anything lf you installed Beautiful Soup
+  through pip (SoupSieve will be automatically installed when you
+  upgrade Beautiful Soup) or if you don't use CSS selectors from
+  within Beautiful Soup.
+
+  SoupSieve documentation: https://facelessuser.github.io/soupsieve/
+
+* Added the PageElement.extend() method, which works like list.append().
+   [bug=1514970]
+
+* PageElement.insert_before() and insert_after() now take a variable
+   number of arguments. [bug=1514970]
+
+* Fix a number of problems with the tree builder that caused
+  trees that were superficially okay, but which fell apart when bits
+  were extracted. Patch by Isaac Muse. [bug=1782928,1809910]
+
+* Fixed a problem with the tree builder in which elements that
+  contained no content (such as empty comments and all-whitespace
+  elements) were not being treated as part of the tree. Patch by Isaac
+  Muse. [bug=1798699]
+
+* Fixed a problem with multi-valued attributes where the value
+  contained whitespace. Thanks to Jens Svalgaard for the
+  fix. [bug=1787453]
+
+* Clarified ambiguous license statements in the source code. Beautiful
+  Soup is released under the MIT license, and has been since 4.4.0.
+
+* This file has been renamed from NEWS.txt to CHANGELOG.
+
+= 4.6.3 (20180812)
+
+* Exactly the same as 4.6.2. Re-released to make the README file
+  render properly on PyPI.
+
+= 4.6.2 (20180812)
+
+* Fix an exception when a custom formatter was asked to format a void
+  element. [bug=1784408]
+
+= 4.6.1 (20180728)
+
+* Stop data loss when encountering an empty numeric entity, and
+  possibly in other cases.  Thanks to tos.kamiya for the fix. [bug=1698503]
+
+* Preserve XML namespaces introduced inside an XML document, not just
+   the ones introduced at the top level. [bug=1718787]
+
+* Added a new formatter, "html5", which represents void elements
+   as "<element>" rather than "<element/>".  [bug=1716272]
+
+* Fixed a problem where the html.parser tree builder interpreted
+  a string like "&foo " as the character entity "&foo;"  [bug=1728706]
+
+* Correctly handle invalid HTML numeric character entities like &#147;
+  which reference code points that are not Unicode code points. Note
+  that this is only fixed when Beautiful Soup is used with the
+  html.parser parser -- html5lib already worked and I couldn't fix it
+  with lxml.  [bug=1782933]
+
+* Improved the warning given when no parser is specified. [bug=1780571]
+
+* When markup contains duplicate elements, a select() call that
+  includes multiple match clauses will match all relevant
+  elements. [bug=1770596]
+
+* Fixed code that was causing deprecation warnings in recent Python 3
+  versions. Includes a patch from Ville Skyttä. [bug=1778909] [bug=1689496]
+
+* Fixed a Windows crash in diagnose() when checking whether a long
+  markup string is a filename. [bug=1737121]
+
+* Stopped HTMLParser from raising an exception in very rare cases of
+  bad markup. [bug=1708831]
+
+* Fixed a bug where find_all() was not working when asked to find a
+  tag with a namespaced name in an XML document that was parsed as
+  HTML. [bug=1723783]
+
+* You can get finer control over formatting by subclassing
+  bs4.element.Formatter and passing a Formatter instance into (e.g.)
+  encode(). [bug=1716272]
+
+* You can pass a dictionary of `attrs` into
+  BeautifulSoup.new_tag. This makes it possible to create a tag with
+  an attribute like 'name' that would otherwise be masked by another
+  argument of new_tag. [bug=1779276]
+
+* Clarified the deprecation warning when accessing tag.fooTag, to cover
+  the possibility that you might really have been looking for a tag
+  called 'fooTag'.
+
+= 4.6.0 (20170507) =
+
+* Added the `Tag.get_attribute_list` method, which acts like `Tag.get` for
+  getting the value of an attribute, but which always returns a list,
+  whether or not the attribute is a multi-value attribute. [bug=1678589]
+
+* It's now possible to use a tag's namespace prefix when searching,
+  e.g. soup.find('namespace:tag') [bug=1655332]
+
+* Improved the handling of empty-element tags like <br> when using the
+  html.parser parser. [bug=1676935]
+
+* HTML parsers treat all HTML4 and HTML5 empty element tags (aka void
+  element tags) correctly. [bug=1656909]
+
+* Namespace prefix is preserved when an XML tag is copied. Thanks
+  to Vikas for a patch and test. [bug=1685172]
+
+= 4.5.3 (20170102) =
+
+* Fixed foster parenting when html5lib is the tree builder. Thanks to
+  Geoffrey Sneddon for a patch and test.
+  
+* Fixed yet another problem that caused the html5lib tree builder to
+  create a disconnected parse tree. [bug=1629825]
+
+= 4.5.2 (20170102) =
+
+* Apart from the version number, this release is identical to
+  4.5.3. Due to user error, it could not be completely uploaded to
+  PyPI. Use 4.5.3 instead.
+
+= 4.5.1 (20160802) =
+
+* Fixed a crash when passing Unicode markup that contained a
+  processing instruction into the lxml HTML parser on Python
+  3. [bug=1608048]
+
+= 4.5.0 (20160719) =
+
+* Beautiful Soup is no longer compatible with Python 2.6. This
+  actually happened a few releases ago, but it's now official.
+
+* Beautiful Soup will now work with versions of html5lib greater than
+  0.99999999. [bug=1603299]
+
+* If a search against each individual value of a multi-valued
+  attribute fails, the search will be run one final time against the
+  complete attribute value considered as a single string. That is, if
+  a tag has class="foo bar" and neither "foo" nor "bar" matches, but
+  "foo bar" does, the tag is now considered a match.
+
+  This happened in previous versions, but only when the value being
+  searched for was a string. Now it also works when that value is
+  a regular expression, a list of strings, etc. [bug=1476868]
+
+* Fixed a bug that deranged the tree when a whitespace element was
+  reparented into a tag that contained an identical whitespace
+  element. [bug=1505351]
+
+* Added support for CSS selector values that contain quoted spaces,
+  such as tag[style="display: foo"]. [bug=1540588]
+
+* Corrected handling of XML processing instructions. [bug=1504393]
+
+* Corrected an encoding error that happened when a BeautifulSoup
+  object was copied. [bug=1554439]
+
+* The contents of <textarea> tags will no longer be modified when the
+  tree is prettified. [bug=1555829]
+
+* When a BeautifulSoup object is pickled but its tree builder cannot
+  be pickled, its .builder attribute is set to None instead of being
+  destroyed. This avoids a performance problem once the object is
+  unpickled. [bug=1523629]
+
+* Specify the file and line number when warning about a
+  BeautifulSoup object being instantiated without a parser being
+  specified. [bug=1574647]
+
+* The `limit` argument to `select()` now works correctly, though it's
+  not implemented very efficiently. [bug=1520530]
+
+* Fixed a Python 3 ByteWarning when a URL was passed in as though it
+  were markup. Thanks to James Salter for a patch and
+  test. [bug=1533762]
+
+* We don't run the check for a filename passed in as markup if the
+  'filename' contains a less-than character; the less-than character
+  indicates it's most likely a very small document. [bug=1577864]
+
+= 4.4.1 (20150928) =
+
+* Fixed a bug that deranged the tree when part of it was
+  removed. Thanks to Eric Weiser for the patch and John Wiseman for a
+  test. [bug=1481520]
+
+* Fixed a parse bug with the html5lib tree-builder. Thanks to Roel
+  Kramer for the patch. [bug=1483781]
+
+* Improved the implementation of CSS selector grouping. Thanks to
+  Orangain for the patch. [bug=1484543]
+
+* Fixed the test_detect_utf8 test so that it works when chardet is
+  installed. [bug=1471359]
+
+* Corrected the output of Declaration objects. [bug=1477847]
+
+
+= 4.4.0 (20150703) =
+
+Especially important changes:
+
+* Added a warning when you instantiate a BeautifulSoup object without
+  explicitly naming a parser. [bug=1398866]
+
+* __repr__ now returns an ASCII bytestring in Python 2, and a Unicode
+  string in Python 3, instead of a UTF8-encoded bytestring in both
+  versions. In Python 3, __str__ now returns a Unicode string instead
+  of a bytestring. [bug=1420131]
+
+* The `text` argument to the find_* methods is now called `string`,
+  which is more accurate. `text` still works, but `string` is the
+  argument described in the documentation. `text` may eventually
+  change its meaning, but not for a very long time. [bug=1366856]
+
+* Changed the way soup objects work under copy.copy(). Copying a
+  NavigableString or a Tag will give you a new NavigableString that's
+  equal to the old one but not connected to the parse tree. Patch by
+  Martijn Peters. [bug=1307490]
+
+* Started using a standard MIT license. [bug=1294662]
+
+* Added a Chinese translation of the documentation by Delong .w.
+
+New features:
+
+* Introduced the select_one() method, which uses a CSS selector but
+  only returns the first match, instead of a list of
+  matches. [bug=1349367]
+
+* You can now create a Tag object without specifying a
+  TreeBuilder. Patch by Martijn Pieters. [bug=1307471]
+
+* You can now create a NavigableString or a subclass just by invoking
+  the constructor. [bug=1294315]
+
+* Added an `exclude_encodings` argument to UnicodeDammit and to the
+  Beautiful Soup constructor, which lets you prohibit the detection of
+  an encoding that you know is wrong. [bug=1469408]
+
+* The select() method now supports selector grouping. Patch by
+  Francisco Canas [bug=1191917]
+
+Bug fixes:
+
+* Fixed yet another problem that caused the html5lib tree builder to
+  create a disconnected parse tree. [bug=1237763]
+
+* Force object_was_parsed() to keep the tree intact even when an element
+  from later in the document is moved into place. [bug=1430633]
+
+* Fixed yet another bug that caused a disconnected tree when html5lib
+  copied an element from one part of the tree to another. [bug=1270611]
+
+* Fixed a bug where Element.extract() could create an infinite loop in
+  the remaining tree.
+
+* The select() method can now find tags whose names contain
+  dashes. Patch by Francisco Canas. [bug=1276211]
+
+* The select() method can now find tags with attributes whose names
+  contain dashes. Patch by Marek Kapolka. [bug=1304007]
+
+* Improved the lxml tree builder's handling of processing
+  instructions. [bug=1294645]
+
+* Restored the helpful syntax error that happens when you try to
+  import the Python 2 edition of Beautiful Soup under Python
+  3. [bug=1213387]
+
+* In Python 3.4 and above, set the new convert_charrefs argument to
+  the html.parser constructor to avoid a warning and future
+  failures. Patch by Stefano Revera. [bug=1375721]
+
+* The warning when you pass in a filename or URL as markup will now be
+  displayed correctly even if the filename or URL is a Unicode
+  string. [bug=1268888]
+
+* If the initial <html> tag contains a CDATA list attribute such as
+  'class', the html5lib tree builder will now turn its value into a
+  list, as it would with any other tag. [bug=1296481]
+
+* Fixed an import error in Python 3.5 caused by the removal of the
+  HTMLParseError class. [bug=1420063]
+
+* Improved docstring for encode_contents() and
+  decode_contents(). [bug=1441543]
+
+* Fixed a crash in Unicode, Dammit's encoding detector when the name
+  of the encoding itself contained invalid bytes. [bug=1360913]
+
+* Improved the exception raised when you call .unwrap() or
+  .replace_with() on an element that's not attached to a tree.
+
+* Raise a NotImplementedError whenever an unsupported CSS pseudoclass
+  is used in select(). Previously some cases did not result in a
+  NotImplementedError.
+
+* It's now possible to pickle a BeautifulSoup object no matter which
+  tree builder was used to create it. However, the only tree builder
+  that survives the pickling process is the HTMLParserTreeBuilder
+  ('html.parser'). If you unpickle a BeautifulSoup object created with
+  some other tree builder, soup.builder will be None. [bug=1231545]
+
 = 4.3.2 (20131002) =
 
 * Fixed a bug in which short Unicode input was improperly encoded to
@@ -331,7 +1104,7 @@
 * Renamed Tag.nsprefix to Tag.prefix, for consistency with
   NamespacedAttribute.
 
-* Fixed a test failure that occured on Python 3.x when chardet was
+* Fixed a test failure that occurred on Python 3.x when chardet was
   installed.
 
 * Made prettify() return Unicode by default, so it will look nice on
@@ -365,7 +1138,7 @@
 
 * Restored compatibility with Python 2.6.
 
-* The install process no longer installs docs or auxillary text files.
+* The install process no longer installs docs or auxiliary text files.
 
 * It's now possible to deepcopy a BeautifulSoup object created with
   Python's built-in HTML parser.
@@ -604,7 +1377,7 @@ Added an import that makes BS work in Python 2.3.
 Fixed a UnicodeDecodeError when unpickling documents that contain
 non-ASCII characters.
 
-Fixed a TypeError that occured in some circumstances when a tag
+Fixed a TypeError that occurred in some circumstances when a tag
 contained no text.
 
 Jump through hoops to avoid the use of chardet, which can be extremely
diff --git a/bitbake/lib/bs4/COPYING.txt b/bitbake/lib/bs4/LICENSE
index d668d13f04..08e3a9cf8c 100644
--- a/bitbake/lib/bs4/COPYING.txt
+++ b/bitbake/lib/bs4/LICENSE
@@ -1,6 +1,6 @@
 Beautiful Soup is made available under the MIT license:
 
- Copyright (c) 2004-2012 Leonard Richardson
+ Copyright (c) Leonard Richardson
 
  Permission is hereby granted, free of charge, to any person obtaining
  a copy of this software and associated documentation files (the
@@ -20,7 +20,12 @@ Beautiful Soup is made available under the MIT license:
  BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE, DAMMIT.
+ SOFTWARE.
 
 Beautiful Soup incorporates code from the html5lib library, which is
-also made available under the MIT license.
+also made available under the MIT license. Copyright (c) James Graham
+and other contributors
+
+Beautiful Soup has an optional dependency on the soupsieve library,
+which is also made available under the MIT license. Copyright (c)
+Isaac Muse
diff --git a/bitbake/lib/bs4/__init__.py b/bitbake/lib/bs4/__init__.py
index e35725b86e..725203d94a 100644
--- a/bitbake/lib/bs4/__init__.py
+++ b/bitbake/lib/bs4/__init__.py
@@ -1,65 +1,99 @@
-"""Beautiful Soup
-Elixir and Tonic
-"The Screen-Scraper's Friend"
+"""Beautiful Soup Elixir and Tonic - "The Screen-Scraper's Friend".
+
 http://www.crummy.com/software/BeautifulSoup/
 
 Beautiful Soup uses a pluggable XML or HTML parser to parse a
 (possibly invalid) document into a tree representation. Beautiful Soup
-provides provides methods and Pythonic idioms that make it easy to
-navigate, search, and modify the parse tree.
+provides methods and Pythonic idioms that make it easy to navigate,
+search, and modify the parse tree.
 
-Beautiful Soup works with Python 2.6 and up. It works better if lxml
+Beautiful Soup works with Python 3.6 and up. It works better if lxml
 and/or html5lib is installed.
 
 For more than you ever wanted to know about Beautiful Soup, see the
-documentation:
-http://www.crummy.com/software/BeautifulSoup/bs4/doc/
+documentation: http://www.crummy.com/software/BeautifulSoup/bs4/doc/
 """
 
 __author__ = "Leonard Richardson (leonardr@segfault.org)"
-__version__ = "4.4.1"
-__copyright__ = "Copyright (c) 2004-2015 Leonard Richardson"
+__version__ = "4.12.3"
+__copyright__ = "Copyright (c) 2004-2024 Leonard Richardson"
+# Use of this source code is governed by the MIT license.
 __license__ = "MIT"
 
 __all__ = ['BeautifulSoup']
 
+from collections import Counter
 import os
 import re
+import sys
+import traceback
 import warnings
 
-from .builder import builder_registry, ParserRejectedMarkup
+# The very first thing we do is give a useful error if someone is
+# running this code under Python 2.
+if sys.version_info.major < 3:
+    raise ImportError('You are trying to use a Python 3-specific version of Beautiful Soup under Python 2. This will not work. The final version of Beautiful Soup to support Python 2 was 4.9.3.')
+
+from .builder import (
+    builder_registry,
+    ParserRejectedMarkup,
+    XMLParsedAsHTMLWarning,
+    HTMLParserTreeBuilder
+)
 from .dammit import UnicodeDammit
 from .element import (
     CData,
     Comment,
+    CSS,
     DEFAULT_OUTPUT_ENCODING,
     Declaration,
     Doctype,
     NavigableString,
     PageElement,
     ProcessingInstruction,
+    PYTHON_SPECIFIC_ENCODINGS,
     ResultSet,
+    Script,
+    Stylesheet,
     SoupStrainer,
     Tag,
+    TemplateString,
     )
 
-# The very first thing we do is give a useful error if someone is
-# running this code under Python 3 without converting it.
-'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.'!='You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
+# Define some custom warnings.
+class GuessedAtParserWarning(UserWarning):
+    """The warning issued when BeautifulSoup has to guess what parser to
+    use -- probably because no parser was specified in the constructor.
+    """
 
-class BeautifulSoup(Tag):
+class MarkupResemblesLocatorWarning(UserWarning):
+    """The warning issued when BeautifulSoup is given 'markup' that
+    actually looks like a resource locator -- a URL or a path to a file
+    on disk.
     """
-    This class defines the basic interface called by the tree builders.
 
-    These methods will be called by the parser:
-      reset()
-      feed(markup)
+   
+class BeautifulSoup(Tag):
+    """A data structure representing a parsed HTML or XML document.
+
+    Most of the methods you'll call on a BeautifulSoup object are inherited from
+    PageElement or Tag.
+
+    Internally, this class defines the basic interface called by the
+    tree builders when converting an HTML/XML document into a data
+    structure. The interface abstracts away the differences between
+    parsers. To write a new tree builder, you'll need to understand
+    these methods as a whole.
+
+    These methods will be called by the BeautifulSoup constructor:
+      * reset()
+      * feed(markup)
 
     The tree builder may call these methods from its feed() implementation:
-      handle_starttag(name, attrs) # See note about return value
-      handle_endtag(name)
-      handle_data(data) # Appends to the current data node
-      endData(containerClass=NavigableString) # Ends the current data node
+      * handle_starttag(name, attrs) # See note about return value
+      * handle_endtag(name)
+      * handle_data(data) # Appends to the current data node
+      * endData(containerClass) # Ends the current data node
 
     No matter how complicated the underlying parser is, you should be
     able to build a tree using 'start tag' events, 'end tag' events,
@@ -69,24 +103,77 @@ class BeautifulSoup(Tag):
     like HTML's <br> tag), call handle_starttag and then
     handle_endtag.
     """
+
+    # Since BeautifulSoup subclasses Tag, it's possible to treat it as
+    # a Tag with a .name. This name makes it clear the BeautifulSoup
+    # object isn't a real markup tag.
     ROOT_TAG_NAME = '[document]'
 
     # If the end-user gives no indication which tree builder they
     # want, look for one with these features.
     DEFAULT_BUILDER_FEATURES = ['html', 'fast']
 
+    # A string containing all ASCII whitespace characters, used in
+    # endData() to detect data chunks that seem 'empty'.
     ASCII_SPACES = '\x20\x0a\x09\x0c\x0d'
 
-    NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nTo get rid of this warning, change this:\n\n BeautifulSoup([your markup])\n\nto this:\n\n BeautifulSoup([your markup], \"%(parser)s\")\n"
-
+    NO_PARSER_SPECIFIED_WARNING = "No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system (\"%(parser)s\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n\nThe code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, pass the additional argument 'features=\"%(parser)s\"' to the BeautifulSoup constructor.\n"
+   
     def __init__(self, markup="", features=None, builder=None,
                  parse_only=None, from_encoding=None, exclude_encodings=None,
-                 **kwargs):
-        """The Soup object is initialized as the 'root tag', and the
-        provided markup (which can be a string or a file-like object)
-        is fed into the underlying parser."""
-
+                 element_classes=None, **kwargs):
+        """Constructor.
+
+        :param markup: A string or a file-like object representing
+         markup to be parsed.
+
+        :param features: Desirable features of the parser to be
+         used. This may be the name of a specific parser ("lxml",
+         "lxml-xml", "html.parser", or "html5lib") or it may be the
+         type of markup to be used ("html", "html5", "xml"). It's
+         recommended that you name a specific parser, so that
+         Beautiful Soup gives you the same results across platforms
+         and virtual environments.
+
+        :param builder: A TreeBuilder subclass to instantiate (or
+         instance to use) instead of looking one up based on
+         `features`. You only need to use this if you've implemented a
+         custom TreeBuilder.
+
+        :param parse_only: A SoupStrainer. Only parts of the document
+         matching the SoupStrainer will be considered. This is useful
+         when parsing part of a document that would otherwise be too
+         large to fit into memory.
+
+        :param from_encoding: A string indicating the encoding of the
+         document to be parsed. Pass this in if Beautiful Soup is
+         guessing wrongly about the document's encoding.
+
+        :param exclude_encodings: A list of strings indicating
+         encodings known to be wrong. Pass this in if you don't know
+         the document's encoding but you know Beautiful Soup's guess is
+         wrong.
+
+        :param element_classes: A dictionary mapping BeautifulSoup
+         classes like Tag and NavigableString, to other classes you'd
+         like to be instantiated instead as the parse tree is
+         built. This is useful for subclassing Tag or NavigableString
+         to modify default behavior.
+
+        :param kwargs: For backwards compatibility purposes, the
+         constructor accepts certain keyword arguments used in
+         Beautiful Soup 3. None of these arguments do anything in
+         Beautiful Soup 4; they will result in a warning and then be
+         ignored.
+         
+         Apart from this, any keyword arguments passed into the
+         BeautifulSoup constructor are propagated to the TreeBuilder
+         constructor. This makes it possible to configure a
+         TreeBuilder by passing in arguments, not just by saying which
+         one to use.
+        """
         if 'convertEntities' in kwargs:
+            del kwargs['convertEntities']
             warnings.warn(
                 "BS4 does not respect the convertEntities argument to the "
                 "BeautifulSoup constructor. Entities are always converted "
@@ -125,10 +212,10 @@ class BeautifulSoup(Tag):
             if old_name in kwargs:
                 warnings.warn(
                     'The "%s" argument to the BeautifulSoup constructor '
-                    'has been renamed to "%s."' % (old_name, new_name))
-                value = kwargs[old_name]
-                del kwargs[old_name]
-                return value
+                    'has been renamed to "%s."' % (old_name, new_name),
+                    DeprecationWarning, stacklevel=3
+                )
+                return kwargs.pop(old_name)
             return None
 
         parse_only = parse_only or deprecated_argument(
@@ -137,13 +224,23 @@ class BeautifulSoup(Tag):
         from_encoding = from_encoding or deprecated_argument(
             "fromEncoding", "from_encoding")
 
-        if len(kwargs) > 0:
-            arg = list(kwargs.keys()).pop()
-            raise TypeError(
-                "__init__() got an unexpected keyword argument '%s'" % arg)
-
-        if builder is None:
-            original_features = features
+        if from_encoding and isinstance(markup, str):
+            warnings.warn("You provided Unicode markup but also provided a value for from_encoding. Your from_encoding will be ignored.")
+            from_encoding = None
+
+        self.element_classes = element_classes or dict()
+
+        # We need this information to track whether or not the builder
+        # was specified well enough that we can omit the 'you need to
+        # specify a parser' warning.
+        original_builder = builder
+        original_features = features
+            
+        if isinstance(builder, type):
+            # A builder class was passed in; it needs to be instantiated.
+            builder_class = builder
+            builder = None
+        elif builder is None:
             if isinstance(features, str):
                 features = [features]
             if features is None or len(features) == 0:
@@ -154,85 +251,227 @@ class BeautifulSoup(Tag):
                     "Couldn't find a tree builder with the features you "
                     "requested: %s. Do you need to install a parser library?"
                     % ",".join(features))
-            builder = builder_class()
-            if not (original_features == builder.NAME or
-                    original_features in builder.ALTERNATE_NAMES):
+
+        # At this point either we have a TreeBuilder instance in
+        # builder, or we have a builder_class that we can instantiate
+        # with the remaining **kwargs.
+        if builder is None:
+            builder = builder_class(**kwargs)
+            if not original_builder and not (
+                    original_features == builder.NAME or
+                    original_features in builder.ALTERNATE_NAMES
+            ) and markup:
+                # The user did not tell us which TreeBuilder to use,
+                # and we had to guess. Issue a warning.
                 if builder.is_xml:
                     markup_type = "XML"
                 else:
                     markup_type = "HTML"
-                warnings.warn(self.NO_PARSER_SPECIFIED_WARNING % dict(
-                    parser=builder.NAME,
-                    markup_type=markup_type))
 
+                # This code adapted from warnings.py so that we get the same line
+                # of code as our warnings.warn() call gets, even if the answer is wrong
+                # (as it may be in a multithreading situation).
+                caller = None
+                try:
+                    caller = sys._getframe(1)
+                except ValueError:
+                    pass
+                if caller:
+                    globals = caller.f_globals
+                    line_number = caller.f_lineno
+                else:
+                    globals = sys.__dict__
+                    line_number= 1                    
+                filename = globals.get('__file__')
+                if filename:
+                    fnl = filename.lower()
+                    if fnl.endswith((".pyc", ".pyo")):
+                        filename = filename[:-1]
+                if filename:
+                    # If there is no filename at all, the user is most likely in a REPL,
+                    # and the warning is not necessary.
+                    values = dict(
+                        filename=filename,
+                        line_number=line_number,
+                        parser=builder.NAME,
+                        markup_type=markup_type
+                    )
+                    warnings.warn(
+                        self.NO_PARSER_SPECIFIED_WARNING % values,
+                        GuessedAtParserWarning, stacklevel=2
+                    )
+        else:
+            if kwargs:
+                warnings.warn("Keyword arguments to the BeautifulSoup constructor will be ignored. These would normally be passed into the TreeBuilder constructor, but a TreeBuilder instance was passed in as `builder`.")
+                    
         self.builder = builder
         self.is_xml = builder.is_xml
-        self.builder.soup = self
-
+        self.known_xml = self.is_xml
+        self._namespaces = dict()
         self.parse_only = parse_only
 
         if hasattr(markup, 'read'):        # It's a file-type object.
             markup = markup.read()
-        elif len(markup) <= 256:
-            # Print out warnings for a couple beginner problems
+        elif len(markup) <= 256 and (
+                (isinstance(markup, bytes) and not b'<' in markup)
+                or (isinstance(markup, str) and not '<' in markup)
+        ):
+            # Issue warnings for a couple beginner problems
             # involving passing non-markup to Beautiful Soup.
             # Beautiful Soup will still parse the input as markup,
-            # just in case that's what the user really wants.
-            if (isinstance(markup, str)
-                and not os.path.supports_unicode_filenames):
-                possible_filename = markup.encode("utf8")
-            else:
-                possible_filename = markup
-            is_file = False
-            try:
-                is_file = os.path.exists(possible_filename)
-            except Exception as e:
-                # This is almost certainly a problem involving
-                # characters not valid in filenames on this
-                # system. Just let it go.
-                pass
-            if is_file:
-                if isinstance(markup, str):
-                    markup = markup.encode("utf8")
-                warnings.warn(
-                    '"%s" looks like a filename, not markup. You should probably open this file and pass the filehandle into Beautiful Soup.' % markup)
-            if markup[:5] == "http:" or markup[:6] == "https:":
-                # TODO: This is ugly but I couldn't get it to work in
-                # Python 3 otherwise.
-                if ((isinstance(markup, bytes) and not b' ' in markup)
-                    or (isinstance(markup, str) and not ' ' in markup)):
-                    if isinstance(markup, str):
-                        markup = markup.encode("utf8")
-                    warnings.warn(
-                        '"%s" looks like a URL. Beautiful Soup is not an HTTP client. You should probably use an HTTP client to get the document behind the URL, and feed that document to Beautiful Soup.' % markup)
+            # since that is sometimes the intended behavior.
+            if not self._markup_is_url(markup):
+                self._markup_resembles_filename(markup)                
 
+        rejections = []
+        success = False
         for (self.markup, self.original_encoding, self.declared_html_encoding,
          self.contains_replacement_characters) in (
              self.builder.prepare_markup(
                  markup, from_encoding, exclude_encodings=exclude_encodings)):
             self.reset()
+            self.builder.initialize_soup(self)
             try:
                 self._feed()
+                success = True
                 break
-            except ParserRejectedMarkup:
+            except ParserRejectedMarkup as e:
+                rejections.append(e)
                 pass
 
+        if not success:
+            other_exceptions = [str(e) for e in rejections]
+            raise ParserRejectedMarkup(
+                "The markup you provided was rejected by the parser. Trying a different parser or a different encoding may help.\n\nOriginal exception(s) from parser:\n " + "\n ".join(other_exceptions)
+            )
+
         # Clear out the markup and remove the builder's circular
         # reference to this object.
         self.markup = None
         self.builder.soup = None
 
-    def __copy__(self):
-        return type(self)(self.encode(), builder=self.builder)
+    def _clone(self):
+        """Create a new BeautifulSoup object with the same TreeBuilder,
+        but not associated with any markup.
+
+        This is the first step of the deepcopy process.
+        """
+        clone = type(self)("", None, self.builder)
 
+        # Keep track of the encoding of the original document,
+        # since we won't be parsing it again.
+        clone.original_encoding = self.original_encoding
+        return clone
+        
     def __getstate__(self):
         # Frequently a tree builder can't be pickled.
         d = dict(self.__dict__)
-        if 'builder' in d and not self.builder.picklable:
-            del d['builder']
+        if 'builder' in d and d['builder'] is not None and not self.builder.picklable:
+            d['builder'] = type(self.builder)
+        # Store the contents as a Unicode string.
+        d['contents'] = []
+        d['markup'] = self.decode()
+
+        # If _most_recent_element is present, it's a Tag object left
+        # over from initial parse. It might not be picklable and we
+        # don't need it.
+        if '_most_recent_element' in d:
+            del d['_most_recent_element']
         return d
 
+    def __setstate__(self, state):
+        # If necessary, restore the TreeBuilder by looking it up.
+        self.__dict__ = state
+        if isinstance(self.builder, type):
+            self.builder = self.builder()
+        elif not self.builder:
+            # We don't know which builder was used to build this
+            # parse tree, so use a default we know is always available.
+            self.builder = HTMLParserTreeBuilder()
+        self.builder.soup = self
+        self.reset()
+        self._feed()
+        return state
+
+    
+    @classmethod
+    def _decode_markup(cls, markup):
+        """Ensure `markup` is bytes so it's safe to send into warnings.warn.
+
+        TODO: warnings.warn had this problem back in 2010 but it might not
+        anymore.
+        """
+        if isinstance(markup, bytes):
+            decoded = markup.decode('utf-8', 'replace')
+        else:
+            decoded = markup
+        return decoded
+
+    @classmethod
+    def _markup_is_url(cls, markup):
+        """Error-handling method to raise a warning if incoming markup looks
+        like a URL.
+
+        :param markup: A string.
+        :return: Whether or not the markup resembles a URL
+            closely enough to justify a warning.
+        """
+        if isinstance(markup, bytes):
+            space = b' '
+            cant_start_with = (b"http:", b"https:")
+        elif isinstance(markup, str):
+            space = ' '
+            cant_start_with = ("http:", "https:")
+        else:
+            return False
+
+        if any(markup.startswith(prefix) for prefix in cant_start_with):
+            if not space in markup:
+                warnings.warn(
+                    'The input looks more like a URL than markup. You may want to use'
+                    ' an HTTP client like requests to get the document behind'
+                    ' the URL, and feed that document to Beautiful Soup.',
+                    MarkupResemblesLocatorWarning,
+                    stacklevel=3
+                )
+                return True
+        return False
+
+    @classmethod
+    def _markup_resembles_filename(cls, markup):
+        """Error-handling method to raise a warning if incoming markup
+        resembles a filename.
+
+        :param markup: A bytestring or string.
+        :return: Whether or not the markup resembles a filename
+            closely enough to justify a warning.
+        """
+        path_characters = '/\\'
+        extensions = ['.html', '.htm', '.xml', '.xhtml', '.txt']
+        if isinstance(markup, bytes):
+            path_characters = path_characters.encode("utf8")
+            extensions = [x.encode('utf8') for x in extensions]
+        filelike = False
+        if any(x in markup for x in path_characters):
+            filelike = True
+        else:
+            lower = markup.lower()
+            if any(lower.endswith(ext) for ext in extensions):
+                filelike = True
+        if filelike:
+            warnings.warn(
+                'The input looks more like a filename than markup. You may'
+                ' want to open this file and pass the filehandle into'
+                ' Beautiful Soup.',
+                MarkupResemblesLocatorWarning, stacklevel=3
+            )
+            return True
+        return False
+    
     def _feed(self):
+        """Internal method that parses previously set markup, creating a large
+        number of Tag and NavigableString objects.
+        """
         # Convert the document to Unicode.
         self.builder.reset()
 
@@ -243,48 +482,111 @@ class BeautifulSoup(Tag):
             self.popTag()
 
     def reset(self):
+        """Reset this object to a state as though it had never parsed any
+        markup.
+        """
         Tag.__init__(self, self, self.builder, self.ROOT_TAG_NAME)
         self.hidden = 1
         self.builder.reset()
         self.current_data = []
         self.currentTag = None
         self.tagStack = []
+        self.open_tag_counter = Counter()
         self.preserve_whitespace_tag_stack = []
+        self.string_container_stack = []
+        self._most_recent_element = None
         self.pushTag(self)
 
-    def new_tag(self, name, namespace=None, nsprefix=None, **attrs):
-        """Create a new tag associated with this soup."""
-        return Tag(None, self.builder, name, namespace, nsprefix, attrs)
+    def new_tag(self, name, namespace=None, nsprefix=None, attrs={},
+                sourceline=None, sourcepos=None, **kwattrs):
+        """Create a new Tag associated with this BeautifulSoup object.
+
+        :param name: The name of the new Tag.
+        :param namespace: The URI of the new Tag's XML namespace, if any.
+        :param prefix: The prefix for the new Tag's XML namespace, if any.
+        :param attrs: A dictionary of this Tag's attribute values; can
+            be used instead of `kwattrs` for attributes like 'class'
+            that are reserved words in Python.
+        :param sourceline: The line number where this tag was
+            (purportedly) found in its source document.
+        :param sourcepos: The character position within `sourceline` where this
+            tag was (purportedly) found.
+        :param kwattrs: Keyword arguments for the new Tag's attribute values.
 
-    def new_string(self, s, subclass=NavigableString):
-        """Create a new NavigableString associated with this soup."""
-        return subclass(s)
+        """
+        kwattrs.update(attrs)
+        return self.element_classes.get(Tag, Tag)(
+            None, self.builder, name, namespace, nsprefix, kwattrs,
+            sourceline=sourceline, sourcepos=sourcepos
+        )
+
+    def string_container(self, base_class=None):
+        container = base_class or NavigableString
+        
+        # There may be a general override of NavigableString.
+        container = self.element_classes.get(
+            container, container
+        )
+
+        # On top of that, we may be inside a tag that needs a special
+        # container class.
+        if self.string_container_stack and container is NavigableString:
+            container = self.builder.string_containers.get(
+                self.string_container_stack[-1].name, container
+            )
+        return container
+        
+    def new_string(self, s, subclass=None):
+        """Create a new NavigableString associated with this BeautifulSoup
+        object.
+        """
+        container = self.string_container(subclass)
+        return container(s)
 
-    def insert_before(self, successor):
+    def insert_before(self, *args):
+        """This method is part of the PageElement API, but `BeautifulSoup` doesn't implement
+        it because there is nothing before or after it in the parse tree.
+        """
         raise NotImplementedError("BeautifulSoup objects don't support insert_before().")
 
-    def insert_after(self, successor):
+    def insert_after(self, *args):
+        """This method is part of the PageElement API, but `BeautifulSoup` doesn't implement
+        it because there is nothing before or after it in the parse tree.
+        """
         raise NotImplementedError("BeautifulSoup objects don't support insert_after().")
 
     def popTag(self):
+        """Internal method called by _popToTag when a tag is closed."""
         tag = self.tagStack.pop()
+        if tag.name in self.open_tag_counter:
+            self.open_tag_counter[tag.name] -= 1
         if self.preserve_whitespace_tag_stack and tag == self.preserve_whitespace_tag_stack[-1]:
             self.preserve_whitespace_tag_stack.pop()
-        #print "Pop", tag.name
+        if self.string_container_stack and tag == self.string_container_stack[-1]:
+            self.string_container_stack.pop()
+        #print("Pop", tag.name)
         if self.tagStack:
             self.currentTag = self.tagStack[-1]
         return self.currentTag
 
     def pushTag(self, tag):
-        #print "Push", tag.name
-        if self.currentTag:
+        """Internal method called by handle_starttag when a tag is opened."""
+        #print("Push", tag.name)
+        if self.currentTag is not None:
             self.currentTag.contents.append(tag)
         self.tagStack.append(tag)
         self.currentTag = self.tagStack[-1]
+        if tag.name != self.ROOT_TAG_NAME:
+            self.open_tag_counter[tag.name] += 1
         if tag.name in self.builder.preserve_whitespace_tags:
             self.preserve_whitespace_tag_stack.append(tag)
+        if tag.name in self.builder.string_containers:
+            self.string_container_stack.append(tag)
 
-    def endData(self, containerClass=NavigableString):
+    def endData(self, containerClass=None):
+        """Method called by the TreeBuilder when the end of a data segment
+        occurs.
+        """       
         if self.current_data:
             current_data = ''.join(self.current_data)
             # If whitespace is not preserved, and this string contains
@@ -311,61 +613,93 @@ class BeautifulSoup(Tag):
                     not self.parse_only.search(current_data)):
                 return
 
+            containerClass = self.string_container(containerClass)
             o = containerClass(current_data)
             self.object_was_parsed(o)
 
     def object_was_parsed(self, o, parent=None, most_recent_element=None):
-        """Add an object to the parse tree."""
-        parent = parent or self.currentTag
-        previous_element = most_recent_element or self._most_recent_element
+        """Method called by the TreeBuilder to integrate an object into the parse tree."""
+        if parent is None:
+            parent = self.currentTag
+        if most_recent_element is not None:
+            previous_element = most_recent_element
+        else:
+            previous_element = self._most_recent_element
 
         next_element = previous_sibling = next_sibling = None
         if isinstance(o, Tag):
             next_element = o.next_element
             next_sibling = o.next_sibling
             previous_sibling = o.previous_sibling
-            if not previous_element:
+            if previous_element is None:
                 previous_element = o.previous_element
 
+        fix = parent.next_element is not None
+
         o.setup(parent, previous_element, next_element, previous_sibling, next_sibling)
 
         self._most_recent_element = o
         parent.contents.append(o)
 
-        if parent.next_sibling:
-            # This node is being inserted into an element that has
-            # already been parsed. Deal with any dangling references.
-            index = parent.contents.index(o)
-            if index == 0:
-                previous_element = parent
-                previous_sibling = None
-            else:
-                previous_element = previous_sibling = parent.contents[index-1]
-            if index == len(parent.contents)-1:
-                next_element = parent.next_sibling
-                next_sibling = None
-            else:
-                next_element = next_sibling = parent.contents[index+1]
-
-            o.previous_element = previous_element
-            if previous_element:
-                previous_element.next_element = o
-            o.next_element = next_element
-            if next_element:
-                next_element.previous_element = o
-            o.next_sibling = next_sibling
-            if next_sibling:
-                next_sibling.previous_sibling = o
-            o.previous_sibling = previous_sibling
-            if previous_sibling:
-                previous_sibling.next_sibling = o
+        # Check if we are inserting into an already parsed node.
+        if fix:
+            self._linkage_fixer(parent)
+
+    def _linkage_fixer(self, el):
+        """Make sure linkage of this fragment is sound."""
+
+        first = el.contents[0]
+        child = el.contents[-1]
+        descendant = child
+
+        if child is first and el.parent is not None:
+            # Parent should be linked to first child
+            el.next_element = child
+            # We are no longer linked to whatever this element is
+            prev_el = child.previous_element
+            if prev_el is not None and prev_el is not el:
+                prev_el.next_element = None
+            # First child should be linked to the parent, and no previous siblings.
+            child.previous_element = el
+            child.previous_sibling = None
+
+        # We have no sibling as we've been appended as the last.
+        child.next_sibling = None
+
+        # This index is a tag, dig deeper for a "last descendant"
+        if isinstance(child, Tag) and child.contents:
+            descendant = child._last_descendant(False)
+
+        # As the final step, link last descendant. It should be linked
+        # to the parent's next sibling (if found), else walk up the chain
+        # and find a parent with a sibling. It should have no next sibling.
+        descendant.next_element = None
+        descendant.next_sibling = None
+        target = el
+        while True:
+            if target is None:
+                break
+            elif target.next_sibling is not None:
+                descendant.next_element = target.next_sibling
+                target.next_sibling.previous_element = child
+                break
+            target = target.parent
 
     def _popToTag(self, name, nsprefix=None, inclusivePop=True):
         """Pops the tag stack up to and including the most recent
-        instance of the given tag. If inclusivePop is false, pops the tag
-        stack up to but *not* including the most recent instqance of
-        the given tag."""
-        #print "Popping to %s" % name
+        instance of the given tag.
+
+        If there are no open tags with the given name, nothing will be
+        popped.
+
+        :param name: Pop up to the most recent tag with this name.
+        :param nsprefix: The namespace prefix that goes with `name`.
+        :param inclusivePop: It this is false, pops the tag stack up
+          to but *not* including the most recent instqance of the
+          given tag.
+
+        """
+        #print("Popping to %s" % name)
         if name == self.ROOT_TAG_NAME:
             # The BeautifulSoup object itself can never be popped.
             return
@@ -374,6 +708,8 @@ class BeautifulSoup(Tag):
 
         stack_size = len(self.tagStack)
         for i in range(stack_size - 1, 0, -1):
+            if not self.open_tag_counter.get(name):
+                break
             t = self.tagStack[i]
             if (name == t.name and nsprefix == t.prefix):
                 if inclusivePop:
@@ -383,16 +719,26 @@ class BeautifulSoup(Tag):
 
         return most_recently_popped
 
-    def handle_starttag(self, name, namespace, nsprefix, attrs):
-        """Push a start tag on to the stack.
-
-        If this method returns None, the tag was rejected by the
-        SoupStrainer. You should proceed as if the tag had not occured
+    def handle_starttag(self, name, namespace, nsprefix, attrs, sourceline=None,
+                        sourcepos=None, namespaces=None):
+        """Called by the tree builder when a new tag is encountered.
+
+        :param name: Name of the tag.
+        :param nsprefix: Namespace prefix for the tag.
+        :param attrs: A dictionary of attribute values.
+        :param sourceline: The line number where this tag was found in its
+            source document.
+        :param sourcepos: The character position within `sourceline` where this
+            tag was found.
+        :param namespaces: A dictionary of all namespace prefix mappings 
+            currently in scope in the document.
+
+        If this method returns None, the tag was rejected by an active
+        SoupStrainer. You should proceed as if the tag had not occurred
         in the document. For instance, if this was a self-closing tag,
         don't call handle_endtag.
         """
-
-        # print "Start tag %s: %s" % (name, attrs)
+        # print("Start tag %s: %s" % (name, attrs))
         self.endData()
 
         if (self.parse_only and len(self.tagStack) <= 1
@@ -400,34 +746,54 @@ class BeautifulSoup(Tag):
                  or not self.parse_only.search_tag(name, attrs))):
             return None
 
-        tag = Tag(self, self.builder, name, namespace, nsprefix, attrs,
-                  self.currentTag, self._most_recent_element)
+        tag = self.element_classes.get(Tag, Tag)(
+            self, self.builder, name, namespace, nsprefix, attrs,
+            self.currentTag, self._most_recent_element,
+            sourceline=sourceline, sourcepos=sourcepos,
+            namespaces=namespaces
+        )
         if tag is None:
             return tag
-        if self._most_recent_element:
+        if self._most_recent_element is not None:
             self._most_recent_element.next_element = tag
         self._most_recent_element = tag
         self.pushTag(tag)
         return tag
 
     def handle_endtag(self, name, nsprefix=None):
-        #print "End tag: " + name
+        """Called by the tree builder when an ending tag is encountered.
+
+        :param name: Name of the tag.
+        :param nsprefix: Namespace prefix for the tag.
+        """
+        #print("End tag: " + name)
         self.endData()
         self._popToTag(name, nsprefix)
-
+        
     def handle_data(self, data):
+        """Called by the tree builder when a chunk of textual data is encountered."""
         self.current_data.append(data)
-
+       
     def decode(self, pretty_print=False,
                eventual_encoding=DEFAULT_OUTPUT_ENCODING,
-               formatter="minimal"):
-        """Returns a string or Unicode representation of this document.
-        To get Unicode, pass None for encoding."""
-
+               formatter="minimal", iterator=None):
+        """Returns a string or Unicode representation of the parse tree
+            as an HTML or XML document.
+
+        :param pretty_print: If this is True, indentation will be used to
+            make the document more readable.
+        :param eventual_encoding: The encoding of the final document.
+            If this is None, the document will be a Unicode string.
+        """
         if self.is_xml:
             # Print the XML declaration
             encoding_part = ''
-            if eventual_encoding is not None:
+            if eventual_encoding in PYTHON_SPECIFIC_ENCODINGS:
+                # This is a special Python encoding; it can't actually
+                # go into an XML document because it means nothing
+                # outside of Python.
+                eventual_encoding = None
+            if eventual_encoding != None:
                 encoding_part = ' encoding="%s"' % eventual_encoding
             prefix = '<?xml version="1.0"%s?>\n' % encoding_part
         else:
@@ -437,9 +803,9 @@ class BeautifulSoup(Tag):
         else:
             indent_level = 0
         return prefix + super(BeautifulSoup, self).decode(
-            indent_level, eventual_encoding, formatter)
+            indent_level, eventual_encoding, formatter, iterator)
 
-# Alias to make it easier to type import: 'from bs4 import _soup'
+# Aliases to make it easier to get started quickly, e.g. 'from bs4 import _soup'
 _s = BeautifulSoup
 _soup = BeautifulSoup
 
@@ -450,19 +816,24 @@ class BeautifulStoneSoup(BeautifulSoup):
         kwargs['features'] = 'xml'
         warnings.warn(
             'The BeautifulStoneSoup class is deprecated. Instead of using '
-            'it, pass features="xml" into the BeautifulSoup constructor.')
+            'it, pass features="xml" into the BeautifulSoup constructor.',
+            DeprecationWarning, stacklevel=2
+        )
         super(BeautifulStoneSoup, self).__init__(*args, **kwargs)
 
 
 class StopParsing(Exception):
+    """Exception raised by a TreeBuilder if it's unable to continue parsing."""
     pass
 
 class FeatureNotFound(ValueError):
+    """Exception raised by the BeautifulSoup constructor if no parser with the
+    requested features is found.
+    """
     pass
 
 
-#By default, act as an HTML pretty-printer.
+#If this file is run as a script, act as an HTML pretty-printer.
 if __name__ == '__main__':
-    import sys
     soup = BeautifulSoup(sys.stdin)
-    print(soup.prettify())
+    print((soup.prettify()))
diff --git a/bitbake/lib/bs4/builder/__init__.py b/bitbake/lib/bs4/builder/__init__.py
index 6ccd4d23d6..ffb31fc25e 100644
--- a/bitbake/lib/bs4/builder/__init__.py
+++ b/bitbake/lib/bs4/builder/__init__.py
@@ -1,11 +1,21 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 from collections import defaultdict
 import itertools
+import re
+import warnings
 import sys
 from bs4.element import (
     CharsetMetaAttributeValue,
     ContentMetaAttributeValue,
-    whitespace_re
-    )
+    RubyParenthesisString,
+    RubyTextString,
+    Stylesheet,
+    Script,
+    TemplateString,
+    nonwhitespace_re
+)
 
 __all__ = [
     'HTMLTreeBuilder',
@@ -22,20 +32,41 @@ XML = 'xml'
 HTML = 'html'
 HTML_5 = 'html5'
 
+class XMLParsedAsHTMLWarning(UserWarning):
+    """The warning issued when an HTML parser is used to parse
+    XML that is not XHTML.
+    """
+    MESSAGE = """It looks like you're parsing an XML document using an HTML parser. If this really is an HTML document (maybe it's XHTML?), you can ignore or filter this warning. If it's XML, you should know that using an XML parser will be more reliable. To parse this document as XML, make sure you have the lxml package installed, and pass the keyword argument `features="xml"` into the BeautifulSoup constructor."""
+
 
 class TreeBuilderRegistry(object):
-
+    """A way of looking up TreeBuilder subclasses by their name or by desired
+    features.
+    """
+    
     def __init__(self):
         self.builders_for_feature = defaultdict(list)
         self.builders = []
 
     def register(self, treebuilder_class):
-        """Register a treebuilder based on its advertised features."""
+        """Register a treebuilder based on its advertised features.
+
+        :param treebuilder_class: A subclass of Treebuilder. its .features
+           attribute should list its features.
+        """
         for feature in treebuilder_class.features:
             self.builders_for_feature[feature].insert(0, treebuilder_class)
         self.builders.insert(0, treebuilder_class)
 
     def lookup(self, *features):
+        """Look up a TreeBuilder subclass with the desired features.
+
+        :param features: A list of features to look for. If none are
+            provided, the most recently registered TreeBuilder subclass
+            will be used.
+        :return: A TreeBuilder subclass, or None if there's no
+            registered subclass with all the requested features.
+        """
         if len(self.builders) == 0:
             # There are no builders at all.
             return None
@@ -78,7 +109,7 @@ class TreeBuilderRegistry(object):
 builder_registry = TreeBuilderRegistry()
 
 class TreeBuilder(object):
-    """Turn a document into a Beautiful Soup object tree."""
+    """Turn a textual document into a Beautiful Soup object tree."""
 
     NAME = "[Unknown tree builder]"
     ALTERNATE_NAMES = []
@@ -86,19 +117,89 @@ class TreeBuilder(object):
 
     is_xml = False
     picklable = False
-    preserve_whitespace_tags = set()
     empty_element_tags = None # A tag will be considered an empty-element
                               # tag when and only when it has no contents.
-
+    
     # A value for these tag/attribute combinations is a space- or
     # comma-separated list of CDATA, rather than a single CDATA.
-    cdata_list_attributes = {}
-
-
-    def __init__(self):
+    DEFAULT_CDATA_LIST_ATTRIBUTES = defaultdict(list)
+
+    # Whitespace should be preserved inside these tags.
+    DEFAULT_PRESERVE_WHITESPACE_TAGS = set()
+
+    # The textual contents of tags with these names should be
+    # instantiated with some class other than NavigableString.
+    DEFAULT_STRING_CONTAINERS = {}
+    
+    USE_DEFAULT = object()
+
+    # Most parsers don't keep track of line numbers.
+    TRACKS_LINE_NUMBERS = False
+    
+    def __init__(self, multi_valued_attributes=USE_DEFAULT,
+                 preserve_whitespace_tags=USE_DEFAULT,
+                 store_line_numbers=USE_DEFAULT,
+                 string_containers=USE_DEFAULT,
+    ):
+        """Constructor.
+
+        :param multi_valued_attributes: If this is set to None, the
+         TreeBuilder will not turn any values for attributes like
+         'class' into lists. Setting this to a dictionary will
+         customize this behavior; look at DEFAULT_CDATA_LIST_ATTRIBUTES
+         for an example.
+
+         Internally, these are called "CDATA list attributes", but that
+         probably doesn't make sense to an end-user, so the argument name
+         is `multi_valued_attributes`.
+
+        :param preserve_whitespace_tags: A list of tags to treat
+         the way <pre> tags are treated in HTML. Tags in this list
+         are immune from pretty-printing; their contents will always be
+         output as-is.
+
+        :param string_containers: A dictionary mapping tag names to
+        the classes that should be instantiated to contain the textual
+        contents of those tags. The default is to use NavigableString
+        for every tag, no matter what the name. You can override the
+        default by changing DEFAULT_STRING_CONTAINERS.
+
+        :param store_line_numbers: If the parser keeps track of the
+         line numbers and positions of the original markup, that
+         information will, by default, be stored in each corresponding
+         `Tag` object. You can turn this off by passing
+         store_line_numbers=False. If the parser you're using doesn't 
+         keep track of this information, then setting store_line_numbers=True
+         will do nothing.
+        """
         self.soup = None
-
+        if multi_valued_attributes is self.USE_DEFAULT:
+            multi_valued_attributes = self.DEFAULT_CDATA_LIST_ATTRIBUTES
+        self.cdata_list_attributes = multi_valued_attributes
+        if preserve_whitespace_tags is self.USE_DEFAULT:
+            preserve_whitespace_tags = self.DEFAULT_PRESERVE_WHITESPACE_TAGS
+        self.preserve_whitespace_tags = preserve_whitespace_tags
+        if store_line_numbers == self.USE_DEFAULT:
+            store_line_numbers = self.TRACKS_LINE_NUMBERS
+        self.store_line_numbers = store_line_numbers 
+        if string_containers == self.USE_DEFAULT:
+            string_containers = self.DEFAULT_STRING_CONTAINERS
+        self.string_containers = string_containers
+        
+    def initialize_soup(self, soup):
+        """The BeautifulSoup object has been initialized and is now
+        being associated with the TreeBuilder.
+
+        :param soup: A BeautifulSoup object.
+        """
+        self.soup = soup
+        
     def reset(self):
+        """Do any work necessary to reset the underlying parser
+        for a new document.
+
+        By default, this does nothing.
+        """
         pass
 
     def can_be_empty_element(self, tag_name):
@@ -110,24 +211,58 @@ class TreeBuilder(object):
         For instance: an HTMLBuilder does not consider a <p> tag to be
         an empty-element tag (it's not in
         HTMLBuilder.empty_element_tags). This means an empty <p> tag
-        will be presented as "<p></p>", not "<p />".
+        will be presented as "<p></p>", not "<p/>" or "<p>".
 
         The default implementation has no opinion about which tags are
         empty-element tags, so a tag will be presented as an
-        empty-element tag if and only if it has no contents.
-        "<foo></foo>" will become "<foo />", and "<foo>bar</foo>" will
+        empty-element tag if and only if it has no children.
+        "<foo></foo>" will become "<foo/>", and "<foo>bar</foo>" will
         be left alone.
+
+        :param tag_name: The name of a markup tag.
         """
         if self.empty_element_tags is None:
             return True
         return tag_name in self.empty_element_tags
-
+    
     def feed(self, markup):
+        """Run some incoming markup through some parsing process,
+        populating the `BeautifulSoup` object in self.soup.
+
+        This method is not implemented in TreeBuilder; it must be
+        implemented in subclasses.
+
+        :return: None.
+        """
         raise NotImplementedError()
 
     def prepare_markup(self, markup, user_specified_encoding=None,
-                       document_declared_encoding=None):
-        return markup, None, None, False
+                       document_declared_encoding=None, exclude_encodings=None):
+        """Run any preliminary steps necessary to make incoming markup
+        acceptable to the parser.
+
+        :param markup: Some markup -- probably a bytestring.
+        :param user_specified_encoding: The user asked to try this encoding.
+        :param document_declared_encoding: The markup itself claims to be
+            in this encoding. NOTE: This argument is not used by the
+            calling code and can probably be removed.
+        :param exclude_encodings: The user asked _not_ to try any of
+            these encodings.
+
+        :yield: A series of 4-tuples:
+         (markup, encoding, declared encoding,
+          has undergone character replacement)
+
+         Each 4-tuple represents a strategy for converting the
+         document to Unicode and parsing it. Each strategy will be tried 
+         in turn.
+
+         By default, the only strategy is to parse the markup
+         as-is. See `LXMLTreeBuilderForXML` and
+         `HTMLParserTreeBuilder` for implementations that take into
+         account the quirks of particular parsers.
+        """
+        yield markup, None, None, False
 
     def test_fragment_to_document(self, fragment):
         """Wrap an HTML fragment to make it look like a document.
@@ -139,16 +274,36 @@ class TreeBuilder(object):
         results against other HTML fragments.
 
         This method should not be used outside of tests.
+
+        :param fragment: A string -- fragment of HTML.
+        :return: A string -- a full HTML document.
         """
         return fragment
 
     def set_up_substitutions(self, tag):
+        """Set up any substitutions that will need to be performed on 
+        a `Tag` when it's output as a string.
+
+        By default, this does nothing. See `HTMLTreeBuilder` for a
+        case where this is used.
+
+        :param tag: A `Tag`
+        :return: Whether or not a substitution was performed.
+        """
         return False
 
     def _replace_cdata_list_attribute_values(self, tag_name, attrs):
-        """Replaces class="foo bar" with class=["foo", "bar"]
+        """When an attribute value is associated with a tag that can
+        have multiple values for that attribute, convert the string
+        value to a list of strings.
 
-        Modifies its input in place.
+        Basically, replaces class="foo bar" with class=["foo", "bar"]
+
+        NOTE: This method modifies its input in place.
+
+        :param tag_name: The name of a tag.
+        :param attrs: A dictionary containing the tag's attributes.
+           Any appropriate attribute values will be modified in place.
         """
         if not attrs:
             return attrs
@@ -163,7 +318,7 @@ class TreeBuilder(object):
                     # values. Split it into a list.
                     value = attrs[attr]
                     if isinstance(value, str):
-                        values = whitespace_re.split(value)
+                        values = nonwhitespace_re.findall(value)
                     else:
                         # html5lib sometimes calls setAttributes twice
                         # for the same tag when rearranging the parse
@@ -174,9 +329,13 @@ class TreeBuilder(object):
                         values = value
                     attrs[attr] = values
         return attrs
-
+    
 class SAXTreeBuilder(TreeBuilder):
-    """A Beautiful Soup treebuilder that listens for SAX events."""
+    """A Beautiful Soup treebuilder that listens for SAX events.
+
+    This is not currently used for anything, but it demonstrates
+    how a simple TreeBuilder would work.
+    """
 
     def feed(self, markup):
         raise NotImplementedError()
@@ -186,11 +345,11 @@ class SAXTreeBuilder(TreeBuilder):
 
     def startElement(self, name, attrs):
         attrs = dict((key[1], value) for key, value in list(attrs.items()))
-        #print "Start %s, %r" % (name, attrs)
+        #print("Start %s, %r" % (name, attrs))
         self.soup.handle_starttag(name, attrs)
 
     def endElement(self, name):
-        #print "End %s" % name
+        #print("End %s" % name)
         self.soup.handle_endtag(name)
 
     def startElementNS(self, nsTuple, nodeName, attrs):
@@ -227,10 +386,44 @@ class HTMLTreeBuilder(TreeBuilder):
     Such as which tags are empty-element tags.
     """
 
-    preserve_whitespace_tags = set(['pre', 'textarea'])
-    empty_element_tags = set(['br' , 'hr', 'input', 'img', 'meta',
-                              'spacer', 'link', 'frame', 'base'])
-
+    empty_element_tags = set([
+        # These are from HTML5.
+        'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', 'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr',
+        
+        # These are from earlier versions of HTML and are removed in HTML5.
+        'basefont', 'bgsound', 'command', 'frame', 'image', 'isindex', 'nextid', 'spacer'
+    ])
+
+    # The HTML standard defines these as block-level elements. Beautiful
+    # Soup does not treat these elements differently from other elements,
+    # but it may do so eventually, and this information is available if
+    # you need to use it.
+    block_elements = set(["address", "article", "aside", "blockquote", "canvas", "dd", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "noscript", "ol", "output", "p", "pre", "section", "table", "tfoot", "ul", "video"])
+
+    # These HTML tags need special treatment so they can be
+    # represented by a string class other than NavigableString.
+    #
+    # For some of these tags, it's because the HTML standard defines
+    # an unusual content model for them. I made this list by going
+    # through the HTML spec
+    # (https://html.spec.whatwg.org/#metadata-content) and looking for
+    # "metadata content" elements that can contain strings.
+    #
+    # The Ruby tags (<rt> and <rp>) are here despite being normal
+    # "phrasing content" tags, because the content they contain is
+    # qualitatively different from other text in the document, and it
+    # can be useful to be able to distinguish it.
+    #
+    # TODO: Arguably <noscript> could go here but it seems
+    # qualitatively different from the other tags.
+    DEFAULT_STRING_CONTAINERS = {
+        'rt' : RubyTextString,
+        'rp' : RubyParenthesisString,
+        'style': Stylesheet,
+        'script': Script,
+        'template': TemplateString,
+    }    
+    
     # The HTML standard defines these attributes as containing a
     # space-separated list of values, not a single value. That is,
     # class="foo bar" means that the 'class' attribute has two values,
@@ -238,7 +431,7 @@ class HTMLTreeBuilder(TreeBuilder):
     # encounter one of these attributes, we will parse its value into
     # a list of values if possible. Upon output, the list will be
     # converted back into a string.
-    cdata_list_attributes = {
+    DEFAULT_CDATA_LIST_ATTRIBUTES = {
         "*" : ['class', 'accesskey', 'dropzone'],
         "a" : ['rel', 'rev'],
         "link" :  ['rel', 'rev'],
@@ -255,7 +448,19 @@ class HTMLTreeBuilder(TreeBuilder):
         "output" : ["for"],
         }
 
+    DEFAULT_PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea'])
+
     def set_up_substitutions(self, tag):
+        """Replace the declared encoding in a <meta> tag with a placeholder,
+        to be substituted when the tag is output to a string.
+
+        An HTML document may come in to Beautiful Soup as one
+        encoding, but exit in a different encoding, and the <meta> tag
+        needs to be changed to reflect this.
+
+        :param tag: A `Tag`
+        :return: Whether or not a substitution was performed.
+        """
         # We are only interested in <meta> tags
         if tag.name != 'meta':
             return False
@@ -288,10 +493,107 @@ class HTMLTreeBuilder(TreeBuilder):
 
         return (meta_encoding is not None)
 
+class DetectsXMLParsedAsHTML(object):
+    """A mixin class for any class (a TreeBuilder, or some class used by a
+    TreeBuilder) that's in a position to detect whether an XML
+    document is being incorrectly parsed as HTML, and issue an
+    appropriate warning.
+
+    This requires being able to observe an incoming processing
+    instruction that might be an XML declaration, and also able to
+    observe tags as they're opened. If you can't do that for a given
+    TreeBuilder, there's a less reliable implementation based on
+    examining the raw markup.
+    """
+
+    # Regular expression for seeing if markup has an <html> tag.
+    LOOKS_LIKE_HTML = re.compile("<[^ +]html", re.I)
+    LOOKS_LIKE_HTML_B = re.compile(b"<[^ +]html", re.I)
+
+    XML_PREFIX = '<?xml'
+    XML_PREFIX_B = b'<?xml'
+    
+    @classmethod
+    def warn_if_markup_looks_like_xml(cls, markup, stacklevel=3):
+        """Perform a check on some markup to see if it looks like XML
+        that's not XHTML. If so, issue a warning.
+
+        This is much less reliable than doing the check while parsing,
+        but some of the tree builders can't do that.
+
+        :param stacklevel: The stacklevel of the code calling this
+        function.
+
+        :return: True if the markup looks like non-XHTML XML, False
+        otherwise.
+
+        """
+        if isinstance(markup, bytes):
+            prefix = cls.XML_PREFIX_B
+            looks_like_html = cls.LOOKS_LIKE_HTML_B
+        else:
+            prefix = cls.XML_PREFIX
+            looks_like_html = cls.LOOKS_LIKE_HTML
+        
+        if (markup is not None
+            and markup.startswith(prefix)
+            and not looks_like_html.search(markup[:500])
+        ):
+            cls._warn(stacklevel=stacklevel+2)
+            return True
+        return False
+
+    @classmethod
+    def _warn(cls, stacklevel=5):
+        """Issue a warning about XML being parsed as HTML."""
+        warnings.warn(
+            XMLParsedAsHTMLWarning.MESSAGE, XMLParsedAsHTMLWarning,
+            stacklevel=stacklevel
+        )
+        
+    def _initialize_xml_detector(self):
+        """Call this method before parsing a document."""
+        self._first_processing_instruction = None
+        self._root_tag = None
+       
+    def _document_might_be_xml(self, processing_instruction):
+        """Call this method when encountering an XML declaration, or a
+        "processing instruction" that might be an XML declaration.
+        """
+        if (self._first_processing_instruction is not None
+            or self._root_tag is not None):
+            # The document has already started. Don't bother checking
+            # anymore.
+            return
+
+        self._first_processing_instruction = processing_instruction
+
+        # We won't know until we encounter the first tag whether or
+        # not this is actually a problem.
+        
+    def _root_tag_encountered(self, name):
+        """Call this when you encounter the document's root tag.
+
+        This is where we actually check whether an XML document is
+        being incorrectly parsed as HTML, and issue the warning.
+        """
+        if self._root_tag is not None:
+            # This method was incorrectly called multiple times. Do
+            # nothing.
+            return
+
+        self._root_tag = name
+        if (name != 'html' and self._first_processing_instruction is not None
+            and self._first_processing_instruction.lower().startswith('xml ')):
+            # We encountered an XML declaration and then a tag other
+            # than 'html'. This is a reliable indicator that a
+            # non-XHTML document is being parsed as XML.
+            self._warn()
+
+    
 def register_treebuilders_from(module):
     """Copy TreeBuilders from the given module into this module."""
-    # I'm fairly sure this is not the best way to do this.
-    this_module = sys.modules['bs4.builder']
+    this_module = sys.modules[__name__]
     for name in module.__all__:
         obj = getattr(module, name)
 
@@ -302,12 +604,22 @@ def register_treebuilders_from(module):
             this_module.builder_registry.register(obj)
 
 class ParserRejectedMarkup(Exception):
-    pass
-
+    """An Exception to be raised when the underlying parser simply
+    refuses to parse the given markup.
+    """
+    def __init__(self, message_or_exception):
+        """Explain why the parser rejected the given markup, either
+        with a textual explanation or another exception.
+        """
+        if isinstance(message_or_exception, Exception):
+            e = message_or_exception
+            message_or_exception = "%s: %s" % (e.__class__.__name__, str(e))
+        super(ParserRejectedMarkup, self).__init__(message_or_exception)
+            
 # Builders are registered in reverse order of priority, so that custom
 # builder registrations will take precedence. In general, we want lxml
 # to take precedence over html5lib, because it's faster. And we only
-# want to use HTMLParser as a last result.
+# want to use HTMLParser as a last resort.
 from . import _htmlparser
 register_treebuilders_from(_htmlparser)
 try:
diff --git a/bitbake/lib/bs4/builder/_html5lib.py b/bitbake/lib/bs4/builder/_html5lib.py
index 9e9216ef9c..7c46a85118 100644
--- a/bitbake/lib/bs4/builder/_html5lib.py
+++ b/bitbake/lib/bs4/builder/_html5lib.py
@@ -1,9 +1,14 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 __all__ = [
     'HTML5TreeBuilder',
     ]
 
 import warnings
+import re
 from bs4.builder import (
+    DetectsXMLParsedAsHTML,
     PERMISSIVE,
     HTML,
     HTML_5,
@@ -11,17 +16,13 @@ from bs4.builder import (
     )
 from bs4.element import (
     NamespacedAttribute,
-    whitespace_re,
+    nonwhitespace_re,
 )
 import html5lib
-try:
-    # html5lib >= 0.99999999/1.0b9
-    from html5lib.treebuilders import base as treebuildersbase
-except ImportError:
-    # html5lib <= 0.9999999/1.0b8
-    from html5lib.treebuilders import _base as treebuildersbase
-from html5lib.constants import namespaces
-
+from html5lib.constants import (
+    namespaces,
+    prefixes,
+    )
 from bs4.element import (
     Comment,
     Doctype,
@@ -29,13 +30,37 @@ from bs4.element import (
     Tag,
     )
 
+try:
+    # Pre-0.99999999
+    from html5lib.treebuilders import _base as treebuilder_base
+    new_html5lib = False
+except ImportError as e:
+    # 0.99999999 and up
+    from html5lib.treebuilders import base as treebuilder_base
+    new_html5lib = True
+
 class HTML5TreeBuilder(HTMLTreeBuilder):
-    """Use html5lib to build a tree."""
+    """Use html5lib to build a tree.
+
+    Note that this TreeBuilder does not support some features common
+    to HTML TreeBuilders. Some of these features could theoretically
+    be implemented, but at the very least it's quite difficult,
+    because html5lib moves the parse tree around as it's being built.
+
+    * This TreeBuilder doesn't use different subclasses of NavigableString
+      based on the name of the tag in which the string was found.
+
+    * You can't use a SoupStrainer to parse only part of a document.
+    """
 
     NAME = "html5lib"
 
     features = [NAME, PERMISSIVE, HTML_5, HTML]
 
+    # html5lib can tell us which line number and position in the
+    # original file is the source of an element.
+    TRACKS_LINE_NUMBERS = True
+    
     def prepare_markup(self, markup, user_specified_encoding,
                        document_declared_encoding=None, exclude_encodings=None):
         # Store the user-specified encoding for use later on.
@@ -45,27 +70,56 @@ class HTML5TreeBuilder(HTMLTreeBuilder):
         # ATM because the html5lib TreeBuilder doesn't use
         # UnicodeDammit.
         if exclude_encodings:
-            warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
+            warnings.warn(
+                "You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.",
+                stacklevel=3
+            )
+
+        # html5lib only parses HTML, so if it's given XML that's worth
+        # noting.
+        DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(
+            markup, stacklevel=3
+        )
+
         yield (markup, None, None, False)
 
     # These methods are defined by Beautiful Soup.
     def feed(self, markup):
         if self.soup.parse_only is not None:
-            warnings.warn("You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.")
+            warnings.warn(
+                "You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.",
+                stacklevel=4
+            )
         parser = html5lib.HTMLParser(tree=self.create_treebuilder)
-        doc = parser.parse(markup, encoding=self.user_specified_encoding)
-
+        self.underlying_builder.parser = parser
+        extra_kwargs = dict()
+        if not isinstance(markup, str):
+            if new_html5lib:
+                extra_kwargs['override_encoding'] = self.user_specified_encoding
+            else:
+                extra_kwargs['encoding'] = self.user_specified_encoding
+        doc = parser.parse(markup, **extra_kwargs)
+        
         # Set the character encoding detected by the tokenizer.
         if isinstance(markup, str):
             # We need to special-case this because html5lib sets
             # charEncoding to UTF-8 if it gets Unicode input.
             doc.original_encoding = None
         else:
-            doc.original_encoding = parser.tokenizer.stream.charEncoding[0]
-
+            original_encoding = parser.tokenizer.stream.charEncoding[0]
+            if not isinstance(original_encoding, str):
+                # In 0.99999999 and up, the encoding is an html5lib
+                # Encoding object. We want to use a string for compatibility
+                # with other tree builders.
+                original_encoding = original_encoding.name
+            doc.original_encoding = original_encoding
+        self.underlying_builder.parser = None
+            
     def create_treebuilder(self, namespaceHTMLElements):
         self.underlying_builder = TreeBuilderForHtml5lib(
-            self.soup, namespaceHTMLElements)
+            namespaceHTMLElements, self.soup,
+            store_line_numbers=self.store_line_numbers
+        )
         return self.underlying_builder
 
     def test_fragment_to_document(self, fragment):
@@ -73,12 +127,30 @@ class HTML5TreeBuilder(HTMLTreeBuilder):
         return '<html><head></head><body>%s</body></html>' % fragment
 
 
-class TreeBuilderForHtml5lib(treebuildersbase.TreeBuilder):
-
-    def __init__(self, soup, namespaceHTMLElements):
-        self.soup = soup
+class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder):
+    
+    def __init__(self, namespaceHTMLElements, soup=None,
+                 store_line_numbers=True, **kwargs):
+        if soup:
+            self.soup = soup
+        else:
+            from bs4 import BeautifulSoup
+            # TODO: Why is the parser 'html.parser' here? To avoid an
+            # infinite loop?
+            self.soup = BeautifulSoup(
+                "", "html.parser", store_line_numbers=store_line_numbers,
+                **kwargs
+            )
+        # TODO: What are **kwargs exactly? Should they be passed in
+        # here in addition to/instead of being passed to the BeautifulSoup
+        # constructor?
         super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
 
+        # This will be set later to an html5lib.html5parser.HTMLParser
+        # object, which we can use to track the current line number.
+        self.parser = None
+        self.store_line_numbers = store_line_numbers
+        
     def documentClass(self):
         self.soup.reset()
         return Element(self.soup, self.soup, None)
@@ -92,14 +164,26 @@ class TreeBuilderForHtml5lib(treebuildersbase.TreeBuilder):
         self.soup.object_was_parsed(doctype)
 
     def elementClass(self, name, namespace):
-        tag = self.soup.new_tag(name, namespace)
+        kwargs = {}
+        if self.parser and self.store_line_numbers:
+            # This represents the point immediately after the end of the
+            # tag. We don't know when the tag started, but we do know
+            # where it ended -- the character just before this one.
+            sourceline, sourcepos = self.parser.tokenizer.stream.position()
+            kwargs['sourceline'] = sourceline
+            kwargs['sourcepos'] = sourcepos-1
+        tag = self.soup.new_tag(name, namespace, **kwargs)
+
         return Element(tag, self.soup, namespace)
 
     def commentClass(self, data):
         return TextNode(Comment(data), self.soup)
 
     def fragmentClass(self):
-        self.soup = BeautifulSoup("")
+        from bs4 import BeautifulSoup
+        # TODO: Why is the parser 'html.parser' here? To avoid an
+        # infinite loop?
+        self.soup = BeautifulSoup("", "html.parser")
         self.soup.name = "[document_fragment]"
         return Element(self.soup, self.soup, None)
 
@@ -111,7 +195,57 @@ class TreeBuilderForHtml5lib(treebuildersbase.TreeBuilder):
         return self.soup
 
     def getFragment(self):
-        return treebuildersbase.TreeBuilder.getFragment(self).element
+        return treebuilder_base.TreeBuilder.getFragment(self).element
+
+    def testSerializer(self, element):
+        from bs4 import BeautifulSoup
+        rv = []
+        doctype_re = re.compile(r'^(.*?)(?: PUBLIC "(.*?)"(?: "(.*?)")?| SYSTEM "(.*?)")?$')
+
+        def serializeElement(element, indent=0):
+            if isinstance(element, BeautifulSoup):
+                pass
+            if isinstance(element, Doctype):
+                m = doctype_re.match(element)
+                if m:
+                    name = m.group(1)
+                    if m.lastindex > 1:
+                        publicId = m.group(2) or ""
+                        systemId = m.group(3) or m.group(4) or ""
+                        rv.append("""|%s<!DOCTYPE %s "%s" "%s">""" %
+                                  (' ' * indent, name, publicId, systemId))
+                    else:
+                        rv.append("|%s<!DOCTYPE %s>" % (' ' * indent, name))
+                else:
+                    rv.append("|%s<!DOCTYPE >" % (' ' * indent,))
+            elif isinstance(element, Comment):
+                rv.append("|%s<!-- %s -->" % (' ' * indent, element))
+            elif isinstance(element, NavigableString):
+                rv.append("|%s\"%s\"" % (' ' * indent, element))
+            else:
+                if element.namespace:
+                    name = "%s %s" % (prefixes[element.namespace],
+                                      element.name)
+                else:
+                    name = element.name
+                rv.append("|%s<%s>" % (' ' * indent, name))
+                if element.attrs:
+                    attributes = []
+                    for name, value in list(element.attrs.items()):
+                        if isinstance(name, NamespacedAttribute):
+                            name = "%s %s" % (prefixes[name.namespace], name.name)
+                        if isinstance(value, list):
+                            value = " ".join(value)
+                        attributes.append((name, value))
+
+                    for name, value in sorted(attributes):
+                        rv.append('|%s%s="%s"' % (' ' * (indent + 2), name, value))
+                indent += 2
+                for child in element.children:
+                    serializeElement(child, indent)
+        serializeElement(element, 0)
+
+        return "\n".join(rv)
 
 class AttrList(object):
     def __init__(self, element):
@@ -122,14 +256,14 @@ class AttrList(object):
     def __setitem__(self, name, value):
         # If this attribute is a multi-valued attribute for this element,
         # turn its value into a list.
-        list_attr = HTML5TreeBuilder.cdata_list_attributes
-        if (name in list_attr['*']
+        list_attr = self.element.cdata_list_attributes or {}
+        if (name in list_attr.get('*', [])
             or (self.element.name in list_attr
-                and name in list_attr[self.element.name])):
+                and name in list_attr.get(self.element.name, []))):
             # A node that is being cloned may have already undergone
             # this procedure.
             if not isinstance(value, list):
-                value = whitespace_re.split(value)
+                value = nonwhitespace_re.findall(value)
         self.element[name] = value
     def items(self):
         return list(self.attrs.items())
@@ -143,9 +277,9 @@ class AttrList(object):
         return name in list(self.attrs.keys())
 
 
-class Element(treebuildersbase.Node):
+class Element(treebuilder_base.Node):
     def __init__(self, element, soup, namespace):
-        treebuildersbase.Node.__init__(self, element.name)
+        treebuilder_base.Node.__init__(self, element.name)
         self.element = element
         self.soup = soup
         self.namespace = namespace
@@ -164,13 +298,15 @@ class Element(treebuildersbase.Node):
             child = node
         elif node.element.__class__ == NavigableString:
             string_child = child = node.element
+            node.parent = self
         else:
             child = node.element
+            node.parent = self
 
         if not isinstance(child, str) and child.parent is not None:
             node.element.extract()
 
-        if (string_child and self.element.contents
+        if (string_child is not None and self.element.contents
             and self.element.contents[-1].__class__ == NavigableString):
             # We are appending a string onto another string.
             # TODO This has O(n^2) performance, for input like
@@ -203,12 +339,12 @@ class Element(treebuildersbase.Node):
                 most_recent_element=most_recent_element)
 
     def getAttributes(self):
+        if isinstance(self.element, Comment):
+            return {}
         return AttrList(self.element)
 
     def setAttributes(self, attributes):
-
         if attributes is not None and len(attributes) > 0:
-
             converted_attributes = []
             for name, value in list(attributes.items()):
                 if isinstance(name, tuple):
@@ -230,11 +366,11 @@ class Element(treebuildersbase.Node):
     attributes = property(getAttributes, setAttributes)
 
     def insertText(self, data, insertBefore=None):
+        text = TextNode(self.soup.new_string(data), self.soup)
         if insertBefore:
-            text = TextNode(self.soup.new_string(data), self.soup)
-            self.insertBefore(data, insertBefore)
+            self.insertBefore(text, insertBefore)
         else:
-            self.appendChild(data)
+            self.appendChild(text)
 
     def insertBefore(self, node, refNode):
         index = self.element.index(refNode.element)
@@ -253,9 +389,10 @@ class Element(treebuildersbase.Node):
 
     def reparentChildren(self, new_parent):
         """Move all of this tag's children into another tag."""
-        # print "MOVE", self.element.contents
-        # print "FROM", self.element
-        # print "TO", new_parent.element
+        # print("MOVE", self.element.contents)
+        # print("FROM", self.element)
+        # print("TO", new_parent.element)
+
         element = self.element
         new_parent_element = new_parent.element
         # Determine what this tag's next_element will be once all the children
@@ -274,29 +411,35 @@ class Element(treebuildersbase.Node):
             new_parents_last_descendant_next_element = new_parent_element.next_element
 
         to_append = element.contents
-        append_after = new_parent_element.contents
         if len(to_append) > 0:
             # Set the first child's previous_element and previous_sibling
             # to elements within the new parent
             first_child = to_append[0]
-            if new_parents_last_descendant:
+            if new_parents_last_descendant is not None:
                 first_child.previous_element = new_parents_last_descendant
             else:
                 first_child.previous_element = new_parent_element
             first_child.previous_sibling = new_parents_last_child
-            if new_parents_last_descendant:
+            if new_parents_last_descendant is not None:
                 new_parents_last_descendant.next_element = first_child
             else:
                 new_parent_element.next_element = first_child
-            if new_parents_last_child:
+            if new_parents_last_child is not None:
                 new_parents_last_child.next_sibling = first_child
 
-            # Fix the last child's next_element and next_sibling
-            last_child = to_append[-1]
-            last_child.next_element = new_parents_last_descendant_next_element
-            if new_parents_last_descendant_next_element:
-                new_parents_last_descendant_next_element.previous_element = last_child
-            last_child.next_sibling = None
+            # Find the very last element being moved. It is now the
+            # parent's last descendant. It has no .next_sibling and
+            # its .next_element is whatever the previous last
+            # descendant had.
+            last_childs_last_descendant = to_append[-1]._last_descendant(False, True)
+
+            last_childs_last_descendant.next_element = new_parents_last_descendant_next_element
+            if new_parents_last_descendant_next_element is not None:
+                # TODO: This code has no test coverage and I'm not sure
+                # how to get html5lib to go through this path, but it's
+                # just the other side of the previous line.
+                new_parents_last_descendant_next_element.previous_element = last_childs_last_descendant
+            last_childs_last_descendant.next_sibling = None
 
         for child in to_append:
             child.parent = new_parent_element
@@ -306,9 +449,9 @@ class Element(treebuildersbase.Node):
         element.contents = []
         element.next_element = final_next_element
 
-        # print "DONE WITH MOVE"
-        # print "FROM", self.element
-        # print "TO", new_parent_element
+        # print("DONE WITH MOVE")
+        # print("FROM", self.element)
+        # print("TO", new_parent_element)
 
     def cloneNode(self):
         tag = self.soup.new_tag(self.element.name, self.namespace)
@@ -321,7 +464,7 @@ class Element(treebuildersbase.Node):
         return self.element.contents
 
     def getNameTuple(self):
-        if self.namespace is None:
+        if self.namespace == None:
             return namespaces["html"], self.name
         else:
             return self.namespace, self.name
@@ -330,7 +473,7 @@ class Element(treebuildersbase.Node):
 
 class TextNode(Element):
     def __init__(self, element, soup):
-        treebuildersbase.Node.__init__(self, None)
+        treebuilder_base.Node.__init__(self, None)
         self.element = element
         self.soup = soup
 
diff --git a/bitbake/lib/bs4/builder/_htmlparser.py b/bitbake/lib/bs4/builder/_htmlparser.py
index bb0a63f2f3..3cc187f892 100644
--- a/bitbake/lib/bs4/builder/_htmlparser.py
+++ b/bitbake/lib/bs4/builder/_htmlparser.py
@@ -1,35 +1,18 @@
+# encoding: utf-8
 """Use the HTMLParser library to parse HTML files that aren't too bad."""
 
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 __all__ = [
     'HTMLParserTreeBuilder',
     ]
 
 from html.parser import HTMLParser
 
-try:
-    from html.parser import HTMLParseError
-except ImportError as e:
-    # HTMLParseError is removed in Python 3.5. Since it can never be
-    # thrown in 3.5, we can just define our own class as a placeholder.
-    class HTMLParseError(Exception):
-        pass
-
 import sys
 import warnings
 
-# Starting in Python 3.2, the HTMLParser constructor takes a 'strict'
-# argument, which we'd like to set to False. Unfortunately,
-# http://bugs.python.org/issue13273 makes strict=True a better bet
-# before Python 3.2.3.
-#
-# At the end of this file, we monkeypatch HTMLParser so that
-# strict=True works well on Python 3.2.2.
-major, minor, release = sys.version_info[:3]
-CONSTRUCTOR_TAKES_STRICT = major == 3 and minor == 2 and release >= 3
-CONSTRUCTOR_STRICT_IS_DEPRECATED = major == 3 and minor == 3
-CONSTRUCTOR_TAKES_CONVERT_CHARREFS = major == 3 and minor >= 4
-
-
 from bs4.element import (
     CData,
     Comment,
@@ -40,6 +23,8 @@ from bs4.element import (
 from bs4.dammit import EntitySubstitution, UnicodeDammit
 
 from bs4.builder import (
+    DetectsXMLParsedAsHTML,
+    ParserRejectedMarkup,
     HTML,
     HTMLTreeBuilder,
     STRICT,
@@ -48,8 +33,84 @@ from bs4.builder import (
 
 HTMLPARSER = 'html.parser'
 
-class BeautifulSoupHTMLParser(HTMLParser):
-    def handle_starttag(self, name, attrs):
+class BeautifulSoupHTMLParser(HTMLParser, DetectsXMLParsedAsHTML):
+    """A subclass of the Python standard library's HTMLParser class, which
+    listens for HTMLParser events and translates them into calls
+    to Beautiful Soup's tree construction API.
+    """
+
+    # Strategies for handling duplicate attributes
+    IGNORE = 'ignore'
+    REPLACE = 'replace'
+    
+    def __init__(self, *args, **kwargs):
+        """Constructor.
+
+        :param on_duplicate_attribute: A strategy for what to do if a
+            tag includes the same attribute more than once. Accepted
+            values are: REPLACE (replace earlier values with later
+            ones, the default), IGNORE (keep the earliest value
+            encountered), or a callable. A callable must take three
+            arguments: the dictionary of attributes already processed,
+            the name of the duplicate attribute, and the most recent value
+            encountered.           
+        """
+        self.on_duplicate_attribute = kwargs.pop(
+            'on_duplicate_attribute', self.REPLACE
+        )
+        HTMLParser.__init__(self, *args, **kwargs)
+
+        # Keep a list of empty-element tags that were encountered
+        # without an explicit closing tag. If we encounter a closing tag
+        # of this type, we'll associate it with one of those entries.
+        #
+        # This isn't a stack because we don't care about the
+        # order. It's a list of closing tags we've already handled and
+        # will ignore, assuming they ever show up.
+        self.already_closed_empty_element = []
+
+        self._initialize_xml_detector()
+
+    def error(self, message):
+        # NOTE: This method is required so long as Python 3.9 is
+        # supported. The corresponding code is removed from HTMLParser
+        # in 3.5, but not removed from ParserBase until 3.10.
+        # https://github.com/python/cpython/issues/76025
+        #
+        # The original implementation turned the error into a warning,
+        # but in every case I discovered, this made HTMLParser
+        # immediately crash with an error message that was less
+        # helpful than the warning. The new implementation makes it
+        # more clear that html.parser just can't parse this
+        # markup. The 3.10 implementation does the same, though it
+        # raises AssertionError rather than calling a method. (We
+        # catch this error and wrap it in a ParserRejectedMarkup.)
+        raise ParserRejectedMarkup(message)
+
+    def handle_startendtag(self, name, attrs):
+        """Handle an incoming empty-element tag.
+
+        This is only called when the markup looks like <tag/>.
+
+        :param name: Name of the tag.
+        :param attrs: Dictionary of the tag's attributes.
+        """
+        # is_startend() tells handle_starttag not to close the tag
+        # just because its name matches a known empty-element tag. We
+        # know that this is an empty-element tag and we want to call
+        # handle_endtag ourselves.
+        tag = self.handle_starttag(name, attrs, handle_empty_element=False)
+        self.handle_endtag(name)
+        
+    def handle_starttag(self, name, attrs, handle_empty_element=True):
+        """Handle an opening tag, e.g. '<tag>'
+
+        :param name: Name of the tag.
+        :param attrs: Dictionary of the tag's attributes.
+        :param handle_empty_element: True if this tag is known to be
+            an empty-element tag (i.e. there is not expected to be any
+            closing tag).
+        """
         # XXX namespace
         attr_dict = {}
         for key, value in attrs:
@@ -57,20 +118,78 @@ class BeautifulSoupHTMLParser(HTMLParser):
             # for consistency with the other tree builders.
             if value is None:
                 value = ''
-            attr_dict[key] = value
+            if key in attr_dict:
+                # A single attribute shows up multiple times in this
+                # tag. How to handle it depends on the
+                # on_duplicate_attribute setting.
+                on_dupe = self.on_duplicate_attribute
+                if on_dupe == self.IGNORE:
+                    pass
+                elif on_dupe in (None, self.REPLACE):
+                    attr_dict[key] = value
+                else:
+                    on_dupe(attr_dict, key, value)
+            else:
+                attr_dict[key] = value
             attrvalue = '""'
-        self.soup.handle_starttag(name, None, None, attr_dict)
-
-    def handle_endtag(self, name):
-        self.soup.handle_endtag(name)
-
+        #print("START", name)
+        sourceline, sourcepos = self.getpos()
+        tag = self.soup.handle_starttag(
+            name, None, None, attr_dict, sourceline=sourceline,
+            sourcepos=sourcepos
+        )
+        if tag and tag.is_empty_element and handle_empty_element:
+            # Unlike other parsers, html.parser doesn't send separate end tag
+            # events for empty-element tags. (It's handled in
+            # handle_startendtag, but only if the original markup looked like
+            # <tag/>.)
+            #
+            # So we need to call handle_endtag() ourselves. Since we
+            # know the start event is identical to the end event, we
+            # don't want handle_endtag() to cross off any previous end
+            # events for tags of this name.
+            self.handle_endtag(name, check_already_closed=False)
+
+            # But we might encounter an explicit closing tag for this tag
+            # later on. If so, we want to ignore it.
+            self.already_closed_empty_element.append(name)
+
+        if self._root_tag is None:
+            self._root_tag_encountered(name)
+            
+    def handle_endtag(self, name, check_already_closed=True):
+        """Handle a closing tag, e.g. '</tag>'
+        
+        :param name: A tag name.
+        :param check_already_closed: True if this tag is expected to
+           be the closing portion of an empty-element tag,
+           e.g. '<tag></tag>'.
+        """
+        #print("END", name)
+        if check_already_closed and name in self.already_closed_empty_element:
+            # This is a redundant end tag for an empty-element tag.
+            # We've already called handle_endtag() for it, so just
+            # check it off the list.
+            #print("ALREADY CLOSED", name)
+            self.already_closed_empty_element.remove(name)
+        else:
+            self.soup.handle_endtag(name)
+            
     def handle_data(self, data):
+        """Handle some textual data that shows up between tags."""
         self.soup.handle_data(data)
 
     def handle_charref(self, name):
-        # XXX workaround for a bug in HTMLParser. Remove this once
-        # it's fixed in all supported versions.
-        # http://bugs.python.org/issue13633
+        """Handle a numeric character reference by converting it to the
+        corresponding Unicode character and treating it as textual
+        data.
+
+        :param name: Character number, possibly in hexadecimal.
+        """
+        # TODO: This was originally a workaround for a bug in
+        # HTMLParser. (http://bugs.python.org/issue13633) The bug has
+        # been fixed, but removing this code still makes some
+        # Beautiful Soup tests fail. This needs investigation.
         if name.startswith('x'):
             real_name = int(name.lstrip('x'), 16)
         elif name.startswith('X'):
@@ -78,37 +197,71 @@ class BeautifulSoupHTMLParser(HTMLParser):
         else:
             real_name = int(name)
 
-        try:
-            data = chr(real_name)
-        except (ValueError, OverflowError) as e:
-            data = "\N{REPLACEMENT CHARACTER}"
-
+        data = None
+        if real_name < 256:
+            # HTML numeric entities are supposed to reference Unicode
+            # code points, but sometimes they reference code points in
+            # some other encoding (ahem, Windows-1252). E.g. &#147;
+            # instead of &#201; for LEFT DOUBLE QUOTATION MARK. This
+            # code tries to detect this situation and compensate.
+            for encoding in (self.soup.original_encoding, 'windows-1252'):
+                if not encoding:
+                    continue
+                try:
+                    data = bytearray([real_name]).decode(encoding)
+                except UnicodeDecodeError as e:
+                    pass
+        if not data:
+            try:
+                data = chr(real_name)
+            except (ValueError, OverflowError) as e:
+                pass
+        data = data or "\N{REPLACEMENT CHARACTER}"
         self.handle_data(data)
 
     def handle_entityref(self, name):
+        """Handle a named entity reference by converting it to the
+        corresponding Unicode character(s) and treating it as textual
+        data.
+
+        :param name: Name of the entity reference.
+        """
         character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
         if character is not None:
             data = character
         else:
-            data = "&%s;" % name
+            # If this were XML, it would be ambiguous whether "&foo"
+            # was an character entity reference with a missing
+            # semicolon or the literal string "&foo". Since this is
+            # HTML, we have a complete list of all character entity references,
+            # and this one wasn't found, so assume it's the literal string "&foo".
+            data = "&%s" % name
         self.handle_data(data)
 
     def handle_comment(self, data):
+        """Handle an HTML comment.
+
+        :param data: The text of the comment.
+        """
         self.soup.endData()
         self.soup.handle_data(data)
         self.soup.endData(Comment)
 
     def handle_decl(self, data):
+        """Handle a DOCTYPE declaration.
+
+        :param data: The text of the declaration.
+        """
         self.soup.endData()
-        if data.startswith("DOCTYPE "):
-            data = data[len("DOCTYPE "):]
-        elif data == 'DOCTYPE':
-            # i.e. "<!DOCTYPE>"
-            data = ''
+        data = data[len("DOCTYPE "):]
         self.soup.handle_data(data)
         self.soup.endData(Doctype)
 
     def unknown_decl(self, data):
+        """Handle a declaration of unknown type -- probably a CDATA block.
+
+        :param data: The text of the declaration.
+        """
         if data.upper().startswith('CDATA['):
             cls = CData
             data = data[len('CDATA['):]
@@ -119,144 +272,116 @@ class BeautifulSoupHTMLParser(HTMLParser):
         self.soup.endData(cls)
 
     def handle_pi(self, data):
+        """Handle a processing instruction.
+
+        :param data: The text of the instruction.
+        """
         self.soup.endData()
         self.soup.handle_data(data)
+        self._document_might_be_xml(data)
         self.soup.endData(ProcessingInstruction)
 
 
 class HTMLParserTreeBuilder(HTMLTreeBuilder):
-
+    """A Beautiful soup `TreeBuilder` that uses the `HTMLParser` parser,
+    found in the Python standard library.
+    """
     is_xml = False
     picklable = True
     NAME = HTMLPARSER
     features = [NAME, HTML, STRICT]
 
-    def __init__(self, *args, **kwargs):
-        if CONSTRUCTOR_TAKES_STRICT and not CONSTRUCTOR_STRICT_IS_DEPRECATED:
-            kwargs['strict'] = False
-        if CONSTRUCTOR_TAKES_CONVERT_CHARREFS:
-            kwargs['convert_charrefs'] = False
-        self.parser_args = (args, kwargs)
+    # The html.parser knows which line number and position in the
+    # original file is the source of an element.
+    TRACKS_LINE_NUMBERS = True
 
+    def __init__(self, parser_args=None, parser_kwargs=None, **kwargs):
+        """Constructor.
+
+        :param parser_args: Positional arguments to pass into 
+            the BeautifulSoupHTMLParser constructor, once it's
+            invoked.
+        :param parser_kwargs: Keyword arguments to pass into 
+            the BeautifulSoupHTMLParser constructor, once it's
+            invoked.
+        :param kwargs: Keyword arguments for the superclass constructor.
+        """
+        # Some keyword arguments will be pulled out of kwargs and placed
+        # into parser_kwargs.
+        extra_parser_kwargs = dict()
+        for arg in ('on_duplicate_attribute',):
+            if arg in kwargs:
+                value = kwargs.pop(arg)
+                extra_parser_kwargs[arg] = value
+        super(HTMLParserTreeBuilder, self).__init__(**kwargs)
+        parser_args = parser_args or []
+        parser_kwargs = parser_kwargs or {}
+        parser_kwargs.update(extra_parser_kwargs)
+        parser_kwargs['convert_charrefs'] = False
+        self.parser_args = (parser_args, parser_kwargs)
+        
     def prepare_markup(self, markup, user_specified_encoding=None,
                        document_declared_encoding=None, exclude_encodings=None):
-        """
-        :return: A 4-tuple (markup, original encoding, encoding
-        declared within markup, whether any characters had to be
-        replaced with REPLACEMENT CHARACTER).
+
+        """Run any preliminary steps necessary to make incoming markup
+        acceptable to the parser.
+
+        :param markup: Some markup -- probably a bytestring.
+        :param user_specified_encoding: The user asked to try this encoding.
+        :param document_declared_encoding: The markup itself claims to be
+            in this encoding.
+        :param exclude_encodings: The user asked _not_ to try any of
+            these encodings.
+
+        :yield: A series of 4-tuples:
+         (markup, encoding, declared encoding,
+          has undergone character replacement)
+
+         Each 4-tuple represents a strategy for converting the
+         document to Unicode and parsing it. Each strategy will be tried 
+         in turn.
         """
         if isinstance(markup, str):
+            # Parse Unicode as-is.
             yield (markup, None, None, False)
             return
 
+        # Ask UnicodeDammit to sniff the most likely encoding.
+
+        # This was provided by the end-user; treat it as a known
+        # definite encoding per the algorithm laid out in the HTML5
+        # spec.  (See the EncodingDetector class for details.)
+        known_definite_encodings = [user_specified_encoding]
+
+        # This was found in the document; treat it as a slightly lower-priority
+        # user encoding.
+        user_encodings = [document_declared_encoding]
+
         try_encodings = [user_specified_encoding, document_declared_encoding]
-        dammit = UnicodeDammit(markup, try_encodings, is_html=True,
-                               exclude_encodings=exclude_encodings)
+        dammit = UnicodeDammit(
+            markup,
+            known_definite_encodings=known_definite_encodings,
+            user_encodings=user_encodings,
+            is_html=True,
+            exclude_encodings=exclude_encodings
+        )
         yield (dammit.markup, dammit.original_encoding,
                dammit.declared_html_encoding,
                dammit.contains_replacement_characters)
 
     def feed(self, markup):
+        """Run some incoming markup through some parsing process,
+        populating the `BeautifulSoup` object in self.soup.
+        """
         args, kwargs = self.parser_args
         parser = BeautifulSoupHTMLParser(*args, **kwargs)
         parser.soup = self.soup
         try:
             parser.feed(markup)
-        except HTMLParseError as e:
-            warnings.warn(RuntimeWarning(
-                "Python's built-in HTMLParser cannot parse the given document. This is not a bug in Beautiful Soup. The best solution is to install an external parser (lxml or html5lib), and use Beautiful Soup with that parser. See http://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser for help."))
-            raise e
-
-# Patch 3.2 versions of HTMLParser earlier than 3.2.3 to use some
-# 3.2.3 code. This ensures they don't treat markup like <p></p> as a
-# string.
-#
-# XXX This code can be removed once most Python 3 users are on 3.2.3.
-if major == 3 and minor == 2 and not CONSTRUCTOR_TAKES_STRICT:
-    import re
-    attrfind_tolerant = re.compile(
-        r'\s*((?<=[\'"\s])[^\s/>][^\s/=>]*)(\s*=+\s*'
-        r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?')
-    HTMLParserTreeBuilder.attrfind_tolerant = attrfind_tolerant
-
-    locatestarttagend = re.compile(r"""
-  <[a-zA-Z][-.a-zA-Z0-9:_]*          # tag name
-  (?:\s+                             # whitespace before attribute name
-    (?:[a-zA-Z_][-.:a-zA-Z0-9_]*     # attribute name
-      (?:\s*=\s*                     # value indicator
-        (?:'[^']*'                   # LITA-enclosed value
-          |\"[^\"]*\"                # LIT-enclosed value
-          |[^'\">\s]+                # bare value
-         )
-       )?
-     )
-   )*
-  \s*                                # trailing whitespace
-""", re.VERBOSE)
-    BeautifulSoupHTMLParser.locatestarttagend = locatestarttagend
-
-    from html.parser import tagfind, attrfind
-
-    def parse_starttag(self, i):
-        self.__starttag_text = None
-        endpos = self.check_for_whole_start_tag(i)
-        if endpos < 0:
-            return endpos
-        rawdata = self.rawdata
-        self.__starttag_text = rawdata[i:endpos]
-
-        # Now parse the data between i+1 and j into a tag and attrs
-        attrs = []
-        match = tagfind.match(rawdata, i+1)
-        assert match, 'unexpected call to parse_starttag()'
-        k = match.end()
-        self.lasttag = tag = rawdata[i+1:k].lower()
-        while k < endpos:
-            if self.strict:
-                m = attrfind.match(rawdata, k)
-            else:
-                m = attrfind_tolerant.match(rawdata, k)
-            if not m:
-                break
-            attrname, rest, attrvalue = m.group(1, 2, 3)
-            if not rest:
-                attrvalue = None
-            elif attrvalue[:1] == '\'' == attrvalue[-1:] or \
-                 attrvalue[:1] == '"' == attrvalue[-1:]:
-                attrvalue = attrvalue[1:-1]
-            if attrvalue:
-                attrvalue = self.unescape(attrvalue)
-            attrs.append((attrname.lower(), attrvalue))
-            k = m.end()
-
-        end = rawdata[k:endpos].strip()
-        if end not in (">", "/>"):
-            lineno, offset = self.getpos()
-            if "\n" in self.__starttag_text:
-                lineno = lineno + self.__starttag_text.count("\n")
-                offset = len(self.__starttag_text) \
-                         - self.__starttag_text.rfind("\n")
-            else:
-                offset = offset + len(self.__starttag_text)
-            if self.strict:
-                self.error("junk characters in start tag: %r"
-                           % (rawdata[k:endpos][:20],))
-            self.handle_data(rawdata[i:endpos])
-            return endpos
-        if end.endswith('/>'):
-            # XHTML-style empty tag: <span attr="value" />
-            self.handle_startendtag(tag, attrs)
-        else:
-            self.handle_starttag(tag, attrs)
-            if tag in self.CDATA_CONTENT_ELEMENTS:
-                self.set_cdata_mode(tag)
-        return endpos
-
-    def set_cdata_mode(self, elem):
-        self.cdata_elem = elem.lower()
-        self.interesting = re.compile(r'</\s*%s\s*>' % self.cdata_elem, re.I)
-
-    BeautifulSoupHTMLParser.parse_starttag = parse_starttag
-    BeautifulSoupHTMLParser.set_cdata_mode = set_cdata_mode
-
-    CONSTRUCTOR_TAKES_STRICT = True
+            parser.close()
+        except AssertionError as e:
+            # html.parser raises AssertionError in rare cases to
+            # indicate a fatal problem with the markup, especially
+            # when there's an error in the doctype declaration.
+            raise ParserRejectedMarkup(e)
+        parser.already_closed_empty_element = []
diff --git a/bitbake/lib/bs4/builder/_lxml.py b/bitbake/lib/bs4/builder/_lxml.py
index 9c6c14ee65..4f7cf74681 100644
--- a/bitbake/lib/bs4/builder/_lxml.py
+++ b/bitbake/lib/bs4/builder/_lxml.py
@@ -1,19 +1,28 @@
+# Use of this source code is governed by the MIT license.
+__license__ = "MIT"
+
 __all__ = [
     'LXMLTreeBuilderForXML',
     'LXMLTreeBuilder',
     ]
 
+try:
+    from collections.abc import Callable # Python 3.6
+except ImportError as e:
+    from collections import Callable
+
 from io import BytesIO
 from io import StringIO
-import collections
 from lxml import etree
 from bs4.element import (
     Comment,
     Doctype,
     NamespacedAttribute,
     ProcessingInstruction,
+    XMLProcessingInstruction,
 )
 from bs4.builder import (
+    DetectsXMLParsedAsHTML,
     FAST,
     HTML,
     HTMLTreeBuilder,
@@ -25,10 +34,15 @@ from bs4.dammit import EncodingDetector
 
 LXML = 'lxml'
 
+def _invert(d):
+    "Invert a dictionary."
+    return dict((v,k) for k, v in list(d.items()))
+
 class LXMLTreeBuilderForXML(TreeBuilder):
     DEFAULT_PARSER_CLASS = etree.XMLParser
 
     is_xml = True
+    processing_instruction_class = XMLProcessingInstruction
 
     NAME = "lxml-xml"
     ALTERNATE_NAMES = ["xml"]
@@ -40,26 +54,79 @@ class LXMLTreeBuilderForXML(TreeBuilder):
 
     # This namespace mapping is specified in the XML Namespace
     # standard.
-    DEFAULT_NSMAPS = {'http://www.w3.org/XML/1998/namespace' : "xml"}
+    DEFAULT_NSMAPS = dict(xml='http://www.w3.org/XML/1998/namespace')
+
+    DEFAULT_NSMAPS_INVERTED = _invert(DEFAULT_NSMAPS)
+
+    # NOTE: If we parsed Element objects and looked at .sourceline,
+    # we'd be able to see the line numbers from the original document.
+    # But instead we build an XMLParser or HTMLParser object to serve
+    # as the target of parse messages, and those messages don't include
+    # line numbers.
+    # See: https://bugs.launchpad.net/lxml/+bug/1846906
+    
+    def initialize_soup(self, soup):
+        """Let the BeautifulSoup object know about the standard namespace
+        mapping.
+
+        :param soup: A `BeautifulSoup`.
+        """
+        super(LXMLTreeBuilderForXML, self).initialize_soup(soup)
+        self._register_namespaces(self.DEFAULT_NSMAPS)
+
+    def _register_namespaces(self, mapping):
+        """Let the BeautifulSoup object know about namespaces encountered
+        while parsing the document.
+
+        This might be useful later on when creating CSS selectors.
+
+        This will track (almost) all namespaces, even ones that were
+        only in scope for part of the document. If two namespaces have
+        the same prefix, only the first one encountered will be
+        tracked. Un-prefixed namespaces are not tracked.
 
+        :param mapping: A dictionary mapping namespace prefixes to URIs.
+        """
+        for key, value in list(mapping.items()):
+            # This is 'if key' and not 'if key is not None' because we
+            # don't track un-prefixed namespaces. Soupselect will
+            # treat an un-prefixed namespace as the default, which
+            # causes confusion in some cases.
+            if key and key not in self.soup._namespaces:
+                # Let the BeautifulSoup object know about a new namespace.
+                # If there are multiple namespaces defined with the same
+                # prefix, the first one in the document takes precedence.
+                self.soup._namespaces[key] = value
+                
     def default_parser(self, encoding):
-        # This can either return a parser object or a class, which
-        # will be instantiated with default arguments.
+        """Find the default parser for the given encoding.
+
+        :param encoding: A string.
+        :return: Either a parser object or a class, which
+          will be instantiated with default arguments.
+        """
         if self._default_parser is not None:
             return self._default_parser
         return etree.XMLParser(
             target=self, strip_cdata=False, recover=True, encoding=encoding)
 
     def parser_for(self, encoding):
+        """Instantiate an appropriate parser for the given encoding.
+
+        :param encoding: A string.
+        :return: A parser object such as an `etree.XMLParser`.
+        """
         # Use the default parser.
         parser = self.default_parser(encoding)
 
-        if isinstance(parser, collections.Callable):
+        if isinstance(parser, Callable):
             # Instantiate the parser with default arguments
-            parser = parser(target=self, strip_cdata=False, encoding=encoding)
+            parser = parser(
+                target=self, strip_cdata=False, recover=True, encoding=encoding
+            )
         return parser
 
-    def __init__(self, parser=None, empty_element_tags=None):
+    def __init__(self, parser=None, empty_element_tags=None, **kwargs):
         # TODO: Issue a warning if parser is present but not a
         # callable, since that means there's no way to create new
         # parsers for different encodings.
@@ -67,8 +134,10 @@ class LXMLTreeBuilderForXML(TreeBuilder):
         if empty_element_tags is not None:
             self.empty_element_tags = set(empty_element_tags)
         self.soup = None
-        self.nsmaps = [self.DEFAULT_NSMAPS]
-
+        self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
+        self.active_namespace_prefixes = [dict(self.DEFAULT_NSMAPS)]
+        super(LXMLTreeBuilderForXML, self).__init__(**kwargs)
+        
     def _getNsTag(self, tag):
         # Split the namespace URL out of a fully-qualified lxml tag
         # name. Copied from lxml's src/lxml/sax.py.
@@ -80,16 +149,51 @@ class LXMLTreeBuilderForXML(TreeBuilder):
     def prepare_markup(self, markup, user_specified_encoding=None,
                        exclude_encodings=None,
                        document_declared_encoding=None):
-        """
-        :yield: A series of 4-tuples.
+        """Run any preliminary steps necessary to make incoming markup
+        acceptable to the parser.
+
+        lxml really wants to get a bytestring and convert it to
+        Unicode itself. So instead of using UnicodeDammit to convert
+        the bytestring to Unicode using different encodings, this
+        implementation uses EncodingDetector to iterate over the
+        encodings, and tell lxml to try to parse the document as each
+        one in turn.
+
+        :param markup: Some markup -- hopefully a bytestring.
+        :param user_specified_encoding: The user asked to try this encoding.
+        :param document_declared_encoding: The markup itself claims to be
+            in this encoding.
+        :param exclude_encodings: The user asked _not_ to try any of
+            these encodings.
+
+        :yield: A series of 4-tuples:
          (markup, encoding, declared encoding,
           has undergone character replacement)
 
-        Each 4-tuple represents a strategy for parsing the document.
+         Each 4-tuple represents a strategy for converting the
+         document to Unicode and parsing it. Each strategy will be tried 
+         in turn.
         """
+        is_html = not self.is_xml
+        if is_html:
+            self.processing_instruction_class = ProcessingInstruction
+            # We're in HTML mode, so if we're given XML, that's worth
+            # noting.
+            DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(
+                markup, stacklevel=3
+            )
+        else:
+            self.processing_instruction_class = XMLProcessingInstruction
+
         if isinstance(markup, str):
             # We were given Unicode. Maybe lxml can parse Unicode on
             # this system?
+
+            # TODO: This is a workaround for
+            # https://bugs.launchpad.net/lxml/+bug/1948551.
+            # We can remove it once the upstream issue is fixed.
+            if len(markup) > 0 and markup[0] == u'\N{BYTE ORDER MARK}':
+                markup = markup[1:]
             yield markup, None, document_declared_encoding, False
 
         if isinstance(markup, str):
@@ -98,14 +202,19 @@ class LXMLTreeBuilderForXML(TreeBuilder):
             yield (markup.encode("utf8"), "utf8",
                    document_declared_encoding, False)
 
-        # Instead of using UnicodeDammit to convert the bytestring to
-        # Unicode using different encodings, use EncodingDetector to
-        # iterate over the encodings, and tell lxml to try to parse
-        # the document as each one in turn.
-        is_html = not self.is_xml
-        try_encodings = [user_specified_encoding, document_declared_encoding]
+        # This was provided by the end-user; treat it as a known
+        # definite encoding per the algorithm laid out in the HTML5
+        # spec.  (See the EncodingDetector class for details.)
+        known_definite_encodings = [user_specified_encoding]
+
+        # This was found in the document; treat it as a slightly lower-priority
+        # user encoding.
+        user_encodings = [document_declared_encoding]
         detector = EncodingDetector(
-            markup, try_encodings, is_html, exclude_encodings)
+            markup, known_definite_encodings=known_definite_encodings,
+            user_encodings=user_encodings, is_html=is_html,
+            exclude_encodings=exclude_encodings
+        )
         for encoding in detector.encodings:
             yield (detector.markup, encoding, document_declared_encoding, False)
 
@@ -128,25 +237,45 @@ class LXMLTreeBuilderForXML(TreeBuilder):
                     self.parser.feed(data)
             self.parser.close()
         except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
-            raise ParserRejectedMarkup(str(e))
+            raise ParserRejectedMarkup(e)
 
     def close(self):
-        self.nsmaps = [self.DEFAULT_NSMAPS]
+        self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
 
     def start(self, name, attrs, nsmap={}):
         # Make sure attrs is a mutable dict--lxml may send an immutable dictproxy.
         attrs = dict(attrs)
         nsprefix = None
         # Invert each namespace map as it comes in.
-        if len(self.nsmaps) > 1:
-            # There are no new namespaces for this tag, but
-            # non-default namespaces are in play, so we need a
-            # separate tag stack to know when they end.
-            self.nsmaps.append(None)
+        if len(nsmap) == 0 and len(self.nsmaps) > 1:
+                # There are no new namespaces for this tag, but
+                # non-default namespaces are in play, so we need a
+                # separate tag stack to know when they end.
+                self.nsmaps.append(None)
         elif len(nsmap) > 0:
             # A new namespace mapping has come into play.
-            inverted_nsmap = dict((value, key) for key, value in list(nsmap.items()))
-            self.nsmaps.append(inverted_nsmap)
+
+            # First, Let the BeautifulSoup object know about it.
+            self._register_namespaces(nsmap)
+
+            # Then, add it to our running list of inverted namespace
+            # mappings.
+            self.nsmaps.append(_invert(nsmap))
+
+            # The currently active namespace prefixes have
+            # changed. Calculate the new mapping so it can be stored
+            # with all Tag objects created while these prefixes are in
+            # scope.
+            current_mapping = dict(self.active_namespace_prefixes[-1])
+            current_mapping.update(nsmap)
+
+            # We should not track un-prefixed namespaces as we can only hold one
+            # and it will be recognized as the default namespace by soupsieve,
+            # which may be confusing in some situations.
+            if '' in current_mapping:
+                del current_mapping['']
+            self.active_namespace_prefixes.append(current_mapping)
+            
             # Also treat the namespace mapping as a set of attributes on the
             # tag, so we can recreate it later.
             attrs = attrs.copy()
@@ -171,8 +300,11 @@ class LXMLTreeBuilderForXML(TreeBuilder):
 
         namespace, name = self._getNsTag(name)
         nsprefix = self._prefix_for_namespace(namespace)
-        self.soup.handle_starttag(name, namespace, nsprefix, attrs)
-
+        self.soup.handle_starttag(
+            name, namespace, nsprefix, attrs,
+            namespaces=self.active_namespace_prefixes[-1]
+        )
+        
     def _prefix_for_namespace(self, namespace):
         """Find the currently active prefix for the given namespace."""
         if namespace is None:
@@ -196,13 +328,20 @@ class LXMLTreeBuilderForXML(TreeBuilder):
         if len(self.nsmaps) > 1:
             # This tag, or one of its parents, introduced a namespace
             # mapping, so pop it off the stack.
-            self.nsmaps.pop()
-
+            out_of_scope_nsmap = self.nsmaps.pop()
+
+            if out_of_scope_nsmap is not None:
+                # This tag introduced a namespace mapping which is no
+                # longer in scope. Recalculate the currently active
+                # namespace prefixes.
+                self.active_namespace_prefixes.pop()
+            
     def pi(self, target, data):
         self.soup.endData()
-        self.soup.handle_data(target + ' ' + data)
-        self.soup.endData(ProcessingInstruction)
-
+        data = target + ' ' + data
+        self.soup.handle_data(data)
+        self.soup.endData(self.processing_instruction_class)
+        
     def data(self, content):
         self.soup.handle_data(content)
 
@@ -229,6 +368,7 @@ class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
 
     features = ALTERNATE_NAMES + [NAME, HTML, FAST, PERMISSIVE]
     is_xml = False
+    processing_instruction_class = ProcessingInstruction
 
     def default_parser(self, encoding):
         return etree.HTMLParser
@@ -240,7 +380,7 @@ class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
             self.parser.feed(markup)
             self.parser.close()
         except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
-            raise ParserRejectedMarkup(str(e))
+            raise ParserRejectedMarkup(e)
 
 
     def test_fragment_to_document(self, fragment):
diff --git a/bitbake/lib/bs4/css.py b/bitbake/lib/bs4/css.py
new file mode 100644
index 0000000000..cd1fd2df88
--- /dev/null
+++ b/bitbake/lib/bs4/css.py
@@ -0,0 +1,274 @@
+"""Integration code for CSS selectors using Soup Sieve (pypi: soupsieve)."""
+
+# We don't use soupsieve
+soupsieve = None
+
+
+class CSS(object):
+    """A proxy object against the soupsieve library, to simplify its
+    CSS selector API.
+
+    Acquire this object through the .css attribute on the
+    BeautifulSoup object, or on the Tag you want to use as the
+    starting point for a CSS selector.
+
+    The main advantage of doing this is that the tag to be selected
+    against doesn't need to be explicitly specified in the function
+    calls, since it's already scoped to a tag.
+    """
+
+    def __init__(self, tag, api=soupsieve):
+        """Constructor.
+
+        You don't need to instantiate this class yourself; instead,
+        access the .css attribute on the BeautifulSoup object, or on
+        the Tag you want to use as the starting point for your CSS
+        selector.
+
+        :param tag: All CSS selectors will use this as their starting
+        point.
+
+        :param api: A plug-in replacement for the soupsieve module,
+        designed mainly for use in tests.
+        """
+        if api is None:
+            raise NotImplementedError(
+                "Cannot execute CSS selectors because the soupsieve package is not installed."
+            )
+        self.api = api
+        self.tag = tag
+
+    def escape(self, ident):
+        """Escape a CSS identifier.
+
+        This is a simple wrapper around soupselect.escape(). See the
+        documentation for that function for more information.
+        """
+        if soupsieve is None:
+            raise NotImplementedError(
+                "Cannot escape CSS identifiers because the soupsieve package is not installed."
+            )
+        return self.api.escape(ident)
+
+    def _ns(self, ns, select):
+        """Normalize a dictionary of namespaces."""
+        if not isinstance(select, self.api.SoupSieve) and ns is None:
+            # If the selector is a precompiled pattern, it already has
+            # a namespace context compiled in, which cannot be
+            # replaced.
+            ns = self.tag._namespaces
+        return ns
+
+    def _rs(self, results):
+        """Normalize a list of results to a Resultset.
+
+        A ResultSet is more consistent with the rest of Beautiful
+        Soup's API, and ResultSet.__getattr__ has a helpful error
+        message if you try to treat a list of results as a single
+        result (a common mistake).
+        """
+        # Import here to avoid circular import
+        from bs4.element import ResultSet
+        return ResultSet(None, results)
+
+    def compile(self, select, namespaces=None, flags=0, **kwargs):
+        """Pre-compile a selector and return the compiled object.
+
+        :param selector: A CSS selector.
+
+        :param namespaces: A dictionary mapping namespace prefixes
+           used in the CSS selector to namespace URIs. By default,
+           Beautiful Soup will use the prefixes it encountered while
+           parsing the document.
+
+        :param flags: Flags to be passed into Soup Sieve's
+            soupsieve.compile() method.
+
+        :param kwargs: Keyword arguments to be passed into SoupSieve's
+           soupsieve.compile() method.
+
+        :return: A precompiled selector object.
+        :rtype: soupsieve.SoupSieve
+        """
+        return self.api.compile(
+            select, self._ns(namespaces, select), flags, **kwargs
+        )
+
+    def select_one(self, select, namespaces=None, flags=0, **kwargs):
+        """Perform a CSS selection operation on the current Tag and return the
+        first result.
+
+        This uses the Soup Sieve library. For more information, see
+        that library's documentation for the soupsieve.select_one()
+        method.
+
+        :param selector: A CSS selector.
+
+        :param namespaces: A dictionary mapping namespace prefixes
+           used in the CSS selector to namespace URIs. By default,
+           Beautiful Soup will use the prefixes it encountered while
+           parsing the document.
+
+        :param flags: Flags to be passed into Soup Sieve's
+            soupsieve.select_one() method.
+
+        :param kwargs: Keyword arguments to be passed into SoupSieve's
+           soupsieve.select_one() method.
+
+        :return: A Tag, or None if the selector has no match.
+        :rtype: bs4.element.Tag
+
+        """
+        return self.api.select_one(
+            select, self.tag, self._ns(namespaces, select), flags, **kwargs
+        )
+
+    def select(self, select, namespaces=None, limit=0, flags=0, **kwargs):
+        """Perform a CSS selection operation on the current Tag.
+
+        This uses the Soup Sieve library. For more information, see
+        that library's documentation for the soupsieve.select()
+        method.
+
+        :param selector: A string containing a CSS selector.
+
+        :param namespaces: A dictionary mapping namespace prefixes
+            used in the CSS selector to namespace URIs. By default,
+            Beautiful Soup will pass in the prefixes it encountered while
+            parsing the document.
+
+        :param limit: After finding this number of results, stop looking.
+
+        :param flags: Flags to be passed into Soup Sieve's
+            soupsieve.select() method.
+
+        :param kwargs: Keyword arguments to be passed into SoupSieve's
+            soupsieve.select() method.
+
+        :return: A ResultSet of Tag objects.
+        :rtype: bs4.element.ResultSet
+
+        """
+        if limit is None:
+            limit = 0
+
+        return self._rs(
+            self.api.select(
+                select, self.tag, self._ns(namespaces, select), limit, flags,
+                **kwargs
+            )
+        )
+
+    def iselect(self, select, namespaces=None, limit=0, flags=0, **kwargs):
+        """Perform a CSS selection operation on the current Tag.
+
+        This uses the Soup Sieve library. For more information, see
+        that library's documentation for the soupsieve.iselect()
+        method. It is the same as select(), but it returns a generator
+        instead of a list.
+
+        :param selector: A string containing a CSS selector.
+
+        :param namespaces: A dictionary mapping namespace prefixes
+            used in the CSS selector to namespace URIs. By default,
+            Beautiful Soup will pass in the prefixes it encountered while
+            parsing the document.
+
+        :param limit: After finding this number of results, stop looking.
+
+        :param flags: Flags to be passed into Soup Sieve's
+            soupsieve.iselect() method.
+
+        :param kwargs: Keyword arguments to be passed into SoupSieve's
+            soupsieve.iselect() method.
+
+        :return: A generator
+        :rtype: types.GeneratorType
+        """
+        return self.api.iselect(
+            select, self.tag, self._ns(namespaces, select), limit, flags, **kwargs
+        )
+
+    def closest(self, select, namespaces=None, flags=0, **kwargs):
+        """Find the Tag closest to this one that matches the given selector.
+
+        This uses the Soup Sieve library. For more information, see
+        that library's documentation for the soupsieve.closest()
+        method.
+
+        :param selector: A string containing a CSS selector.
+
+        :param namespaces: A dictionary mapping namespace prefixes
+            used in the CSS selector to namespace URIs. By default,
+            Beautiful Soup will pass in the prefixes it encountered while
+            parsing the document.
+
+        :param flags: Flags to be passed into Soup Sieve's
+            soupsieve.closest() method.
+
+        :param kwargs: Keyword arguments to be passed into SoupSieve's
+            soupsieve.closest() method.
+
+        :return: A Tag, or None if there is no match.
+        :rtype: bs4.Tag
+
+        """
+        return self.api.closest(
+            select, self.tag, self._ns(namespaces, select), flags, **kwargs
+        )
+
+    def match(self, select, namespaces=None, flags=0, **kwargs):
+        """Check whether this Tag matches the given CSS selector.
+
+        This uses the Soup Sieve library. For more information, see
+        that library's documentation for the soupsieve.match()
+        method.
+
+        :param: a CSS selector.
+
+        :param namespaces: A dictionary mapping namespace prefixes
+            used in the CSS selector to namespace URIs. By default,
+            Beautiful Soup will pass in the prefixes it encountered while
+            parsing the document.
+
+        :param flags: Flags to be passed into Soup Sieve's
+            soupsieve.match() method.
+
+        :param kwargs: Keyword arguments to be passed into SoupSieve's
+            soupsieve.match() method.
+
+        :return: True if this Tag matches the selector; False otherwise.
+        :rtype: bool
+        """
+        return self.api.match(
+            select, self.tag, self._ns(namespaces, select), flags, **kwargs
+        )
+
+    def filter(self, select, namespaces=None, flags=0, **kwargs):
+        """Filter this Tag's direct children based on the given CSS selector.
+
+        This uses the Soup Sieve library. It works the same way as
+        passing this Tag into that library's soupsieve.filter()
+        method. More information, for more information see the
+        documentation for soupsieve.filter().
+
+        :param namespaces: A dictionary mapping namespace prefixes
+            used in the CSS selector to namespace URIs. By default,
+            Beautiful Soup will pass in the prefixes it encountered while
+            parsing the document.
+
+        :param flags: Flags to be passed into Soup Sieve's
+            soupsieve.filter() method.
+
+        :param kwargs: Keyword arguments to be passed into SoupSieve's
+            soupsieve.filter() method.
+
+        :return: A ResultSet of Tag objects.
+        :rtype: bs4.element.ResultSet
+
+        """
+        return self._rs(
+            self.api.filter(
+                select, self.tag, self._ns(namespaces, select), flags, **kwargs
+            )
+        )
diff --git a/bitbake/lib/bs4/dammit.py b/bitbake/lib/bs4/dammit.py
index 7ad9e0dd1e..692433c57a 100644
--- a/bitbake/lib/bs4/dammit.py
+++ b/bitbake/lib/bs4/dammit.py
@@ -6,61 +6,185 @@ necessary. It is heavily based on code from Mark Pilgrim's Universal
 Feed Parser. It works best on XML and HTML, but it does not rewrite the
 XML or HTML to reflect a new encoding; that's the tree builder's job.
 """
+# Use of this source code is governed by the MIT license.
 __license__ = "MIT"
 
-import codecs
 from html.entities import codepoint2name
+from collections import defaultdict
+import codecs
 import re
 import logging
-
-# Import a library to autodetect character encodings.
-chardet_type = None
+import string
+
+# Import a library to autodetect character encodings. We'll support
+# any of a number of libraries that all support the same API:
+#
+# * cchardet
+# * chardet
+# * charset-normalizer
+chardet_module = None
 try:
-    # First try the fast C implementation.
     #  PyPI package: cchardet
-    import cchardet
-    def chardet_dammit(s):
-        return cchardet.detect(s)['encoding']
+    import cchardet as chardet_module
 except ImportError:
     try:
-        # Fall back to the pure Python implementation
         #  Debian package: python-chardet
         #  PyPI package: chardet
-        import chardet
-        def chardet_dammit(s):
-            return chardet.detect(s)['encoding']
-        #import chardet.constants
-        #chardet.constants._debug = 1
+        import chardet as chardet_module
     except ImportError:
-        # No chardet available.
-        def chardet_dammit(s):
+        try:
+            # PyPI package: charset-normalizer
+            import charset_normalizer as chardet_module
+        except ImportError:
+            # No chardet available.
+            chardet_module = None
+
+if chardet_module:
+    def chardet_dammit(s):
+        if isinstance(s, str):
             return None
+        return chardet_module.detect(s)['encoding']
+else:
+    def chardet_dammit(s):
+        return None
 
-xml_encoding_re = re.compile(
-    r'^<\?.*encoding=[\'"](.*?)[\'"].*\?>'.encode(), re.I)
-html_meta_re = re.compile(
-    r'<\s*meta[^>]+charset\s*=\s*["\']?([^>]*?)[ /;\'">]'.encode(), re.I)
+# Build bytestring and Unicode versions of regular expressions for finding
+# a declared encoding inside an XML or HTML document.
+xml_encoding = '^\\s*<\\?.*encoding=[\'"](.*?)[\'"].*\\?>'
+html_meta = '<\\s*meta[^>]+charset\\s*=\\s*["\']?([^>]*?)[ /;\'">]'
+encoding_res = dict()
+encoding_res[bytes] = {
+    'html' : re.compile(html_meta.encode("ascii"), re.I),
+    'xml' : re.compile(xml_encoding.encode("ascii"), re.I),
+}
+encoding_res[str] = {
+    'html' : re.compile(html_meta, re.I),
+    'xml' : re.compile(xml_encoding, re.I)
+}
+
+from html.entities import html5
 
 class EntitySubstitution(object):
-
-    """Substitute XML or HTML entities for the corresponding characters."""
+    """The ability to substitute XML or HTML entities for certain characters."""
 
     def _populate_class_variables():
-        lookup = {}
-        reverse_lookup = {}
-        characters_for_re = []
+        """Initialize variables used by this class to manage the plethora of
+        HTML5 named entities.
+
+        This function returns a 3-tuple containing two dictionaries
+        and a regular expression:
+
+        unicode_to_name - A mapping of Unicode strings like "⦨" to
+        entity names like "angmsdaa". When a single Unicode string has
+        multiple entity names, we try to choose the most commonly-used
+        name.
+
+        name_to_unicode: A mapping of entity names like "angmsdaa" to 
+        Unicode strings like "⦨".
+
+        named_entity_re: A regular expression matching (almost) any
+        Unicode string that corresponds to an HTML5 named entity.
+        """
+        unicode_to_name = {}
+        name_to_unicode = {}
+
+        short_entities = set()
+        long_entities_by_first_character = defaultdict(set)
+        
+        for name_with_semicolon, character in sorted(html5.items()):
+            # "It is intentional, for legacy compatibility, that many
+            # code points have multiple character reference names. For
+            # example, some appear both with and without the trailing
+            # semicolon, or with different capitalizations."
+            # - https://html.spec.whatwg.org/multipage/named-characters.html#named-character-references
+            #
+            # The parsers are in charge of handling (or not) character
+            # references with no trailing semicolon, so we remove the
+            # semicolon whenever it appears.
+            if name_with_semicolon.endswith(';'):
+                name = name_with_semicolon[:-1]
+            else:
+                name = name_with_semicolon
+
+            # When parsing HTML, we want to recognize any known named
+            # entity and convert it to a sequence of Unicode
+            # characters.
+            if name not in name_to_unicode:
+                name_to_unicode[name] = character
+
+            # When _generating_ HTML, we want to recognize special
+            # character sequences that _could_ be converted to named
+            # entities.
+            unicode_to_name[character] = name
+
+            # We also need to build a regular expression that lets us
+            # _find_ those characters in output strings so we can
+            # replace them.
+            #
+            # This is tricky, for two reasons.
+
+            if (len(character) == 1 and ord(character) < 128
+                and character not in '<>&'):
+                # First, it would be annoying to turn single ASCII
+                # characters like | into named entities like
+                # &verbar;. The exceptions are <>&, which we _must_
+                # turn into named entities to produce valid HTML.
+                continue
+
+            if len(character) > 1 and all(ord(x) < 128 for x in character):
+                # We also do not want to turn _combinations_ of ASCII
+                # characters like 'fj' into named entities like '&fjlig;',
+                # though that's more debateable.
+                continue
+
+            # Second, some named entities have a Unicode value that's
+            # a subset of the Unicode value for some _other_ named
+            # entity.  As an example, \u2267' is &GreaterFullEqual;,
+            # but '\u2267\u0338' is &NotGreaterFullEqual;. Our regular
+            # expression needs to match the first two characters of
+            # "\u2267\u0338foo", but only the first character of
+            # "\u2267foo".
+            #
+            # In this step, we build two sets of characters that
+            # _eventually_ need to go into the regular expression. But
+            # we won't know exactly what the regular expression needs
+            # to look like until we've gone through the entire list of
+            # named entities.
+            if len(character) == 1:
+                short_entities.add(character)
+            else:
+                long_entities_by_first_character[character[0]].add(character)
+
+        # Now that we've been through the entire list of entities, we
+        # can create a regular expression that matches any of them.
+        particles = set()
+        for short in short_entities:
+            long_versions = long_entities_by_first_character[short]
+            if not long_versions:
+                particles.add(short)
+            else:
+                ignore = "".join([x[1] for x in long_versions])
+                # This finds, e.g. \u2267 but only if it is _not_
+                # followed by \u0338.
+                particles.add("%s(?![%s])" % (short, ignore))
+        
+        for long_entities in list(long_entities_by_first_character.values()):
+            for long_entity in long_entities:
+                particles.add(long_entity)
+
+        re_definition = "(%s)" % "|".join(particles)
+                
+        # If an entity shows up in both html5 and codepoint2name, it's
+        # likely that HTML5 gives it several different names, such as
+        # 'rsquo' and 'rsquor'. When converting Unicode characters to
+        # named entities, the codepoint2name name should take
+        # precedence where possible, since that's the more easily
+        # recognizable one.
         for codepoint, name in list(codepoint2name.items()):
             character = chr(codepoint)
-            if codepoint != 34:
-                # There's no point in turning the quotation mark into
-                # &quot;, unless it happens within an attribute value, which
-                # is handled elsewhere.
-                characters_for_re.append(character)
-                lookup[character] = name
-            # But we do want to turn &quot; into the quotation mark.
-            reverse_lookup[name] = character
-        re_definition = "[%s]" % "".join(characters_for_re)
-        return lookup, reverse_lookup, re.compile(re_definition)
+            unicode_to_name[character] = name
+
+        return unicode_to_name, name_to_unicode, re.compile(re_definition)
     (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
      CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()
 
@@ -72,21 +196,23 @@ class EntitySubstitution(object):
         ">": "gt",
         }
 
-    BARE_AMPERSAND_OR_BRACKET = re.compile(r"([<>]|"
-                                           r"&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)"
-                                           r")")
+    BARE_AMPERSAND_OR_BRACKET = re.compile("([<>]|"
+                                           "&(?!#\\d+;|#x[0-9a-fA-F]+;|\\w+;)"
+                                           ")")
 
-    AMPERSAND_OR_BRACKET = re.compile(r"([<>&])")
+    AMPERSAND_OR_BRACKET = re.compile("([<>&])")
 
     @classmethod
     def _substitute_html_entity(cls, matchobj):
+        """Used with a regular expression to substitute the
+        appropriate HTML entity for a special character string."""
         entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
         return "&%s;" % entity
 
     @classmethod
     def _substitute_xml_entity(cls, matchobj):
         """Used with a regular expression to substitute the
-        appropriate XML entity for an XML special character."""
+        appropriate XML entity for a special character string."""
         entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
         return "&%s;" % entity
 
@@ -181,6 +307,8 @@ class EntitySubstitution(object):
         containg a LATIN SMALL LETTER E WITH ACUTE, but replacing that
         character with "&eacute;" will make it more readable to some
         people.
+
+        :param s: A Unicode string.
         """
         return cls.CHARACTER_TO_HTML_ENTITY_RE.sub(
             cls._substitute_html_entity, s)
@@ -192,23 +320,65 @@ class EncodingDetector:
     Order of precedence:
 
     1. Encodings you specifically tell EncodingDetector to try first
-    (the override_encodings argument to the constructor).
+    (the known_definite_encodings argument to the constructor).
+
+    2. An encoding determined by sniffing the document's byte-order mark.
+
+    3. Encodings you specifically tell EncodingDetector to try if
+    byte-order mark sniffing fails (the user_encodings argument to the
+    constructor).
 
-    2. An encoding declared within the bytestring itself, either in an
+    4. An encoding declared within the bytestring itself, either in an
     XML declaration (if the bytestring is to be interpreted as an XML
     document), or in a <meta> tag (if the bytestring is to be
     interpreted as an HTML document.)
 
-    3. An encoding detected through textual analysis by chardet,
+    5. An encoding detected through textual analysis by chardet,
     cchardet, or a similar external library.
 
     4. UTF-8.
 
     5. Windows-1252.
+
     """
-    def __init__(self, markup, override_encodings=None, is_html=False,
-                 exclude_encodings=None):
-        self.override_encodings = override_encodings or []
+    def __init__(self, markup, known_definite_encodings=None,
+                 is_html=False, exclude_encodings=None,
+                 user_encodings=None, override_encodings=None):
+        """Constructor.
+
+        :param markup: Some markup in an unknown encoding.
+
+        :param known_definite_encodings: When determining the encoding
+            of `markup`, these encodings will be tried first, in
+            order. In HTML terms, this corresponds to the "known
+            definite encoding" step defined here:
+            https://html.spec.whatwg.org/multipage/parsing.html#parsing-with-a-known-character-encoding
+
+        :param user_encodings: These encodings will be tried after the
+            `known_definite_encodings` have been tried and failed, and
+            after an attempt to sniff the encoding by looking at a
+            byte order mark has failed. In HTML terms, this
+            corresponds to the step "user has explicitly instructed
+            the user agent to override the document's character
+            encoding", defined here:
+            https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
+
+        :param override_encodings: A deprecated alias for
+            known_definite_encodings. Any encodings here will be tried
+            immediately after the encodings in
+            known_definite_encodings.
+
+        :param is_html: If True, this markup is considered to be
+            HTML. Otherwise it's assumed to be XML.
+
+        :param exclude_encodings: These encodings will not be tried,
+            even if they otherwise would be.
+
+        """
+        self.known_definite_encodings = list(known_definite_encodings or [])
+        if override_encodings:
+            self.known_definite_encodings += override_encodings
+        self.user_encodings = user_encodings or []
         exclude_encodings = exclude_encodings or []
         self.exclude_encodings = set([x.lower() for x in exclude_encodings])
         self.chardet_encoding = None
@@ -219,6 +389,12 @@ class EncodingDetector:
         self.markup, self.sniffed_encoding = self.strip_byte_order_mark(markup)
 
     def _usable(self, encoding, tried):
+        """Should we even bother to try this encoding?
+
+        :param encoding: Name of an encoding.
+        :param tried: Encodings that have already been tried. This will be modified
+            as a side effect.
+        """
         if encoding is not None:
             encoding = encoding.lower()
             if encoding in self.exclude_encodings:
@@ -230,9 +406,14 @@ class EncodingDetector:
 
     @property
     def encodings(self):
-        """Yield a number of encodings that might work for this markup."""
+        """Yield a number of encodings that might work for this markup.
+
+        :yield: A sequence of strings.
+        """
         tried = set()
-        for e in self.override_encodings:
+
+        # First, try the known definite encodings
+        for e in self.known_definite_encodings:
             if self._usable(e, tried):
                 yield e
 
@@ -241,6 +422,12 @@ class EncodingDetector:
         if self._usable(self.sniffed_encoding, tried):
             yield self.sniffed_encoding
 
+        # Sniffing the byte-order mark did nothing; try the user
+        # encodings.
+        for e in self.user_encodings:
+            if self._usable(e, tried):
+                yield e
+            
         # Look within the document for an XML or HTML encoding
         # declaration.
         if self.declared_encoding is None:
@@ -263,7 +450,11 @@ class EncodingDetector:
 
     @classmethod
     def strip_byte_order_mark(cls, data):
-        """If a byte-order mark is present, strip it and return the encoding it implies."""
+        """If a byte-order mark is present, strip it and return the encoding it implies.
+
+        :param data: Some markup.
+        :return: A 2-tuple (modified data, implied encoding)
+        """
         encoding = None
         if isinstance(data, str):
             # Unicode data cannot have a byte-order mark.
@@ -295,21 +486,36 @@ class EncodingDetector:
 
         An HTML encoding is declared in a <meta> tag, hopefully near the
         beginning of the document.
+
+        :param markup: Some markup.
+        :param is_html: If True, this markup is considered to be HTML. Otherwise
+            it's assumed to be XML.
+        :param search_entire_document: Since an encoding is supposed to declared near the beginning
+            of the document, most of the time it's only necessary to search a few kilobytes of data.
+            Set this to True to force this method to search the entire document.
         """
         if search_entire_document:
             xml_endpos = html_endpos = len(markup)
         else:
             xml_endpos = 1024
             html_endpos = max(2048, int(len(markup) * 0.05))
-            
+
+        if isinstance(markup, bytes):
+            res = encoding_res[bytes]
+        else:
+            res = encoding_res[str]
+
+        xml_re = res['xml']
+        html_re = res['html']
         declared_encoding = None
-        declared_encoding_match = xml_encoding_re.search(markup, endpos=xml_endpos)
+        declared_encoding_match = xml_re.search(markup, endpos=xml_endpos)
         if not declared_encoding_match and is_html:
-            declared_encoding_match = html_meta_re.search(markup, endpos=html_endpos)
+            declared_encoding_match = html_re.search(markup, endpos=html_endpos)
         if declared_encoding_match is not None:
-            declared_encoding = declared_encoding_match.groups()[0].decode(
-                'ascii', 'replace')
+            declared_encoding = declared_encoding_match.groups()[0]
         if declared_encoding:
+            if isinstance(declared_encoding, bytes):
+                declared_encoding = declared_encoding.decode('ascii', 'replace')
             return declared_encoding.lower()
         return None
 
@@ -332,15 +538,53 @@ class UnicodeDammit:
         "iso-8859-2",
         ]
 
-    def __init__(self, markup, override_encodings=[],
-                 smart_quotes_to=None, is_html=False, exclude_encodings=[]):
+    def __init__(self, markup, known_definite_encodings=[],
+                 smart_quotes_to=None, is_html=False, exclude_encodings=[],
+                 user_encodings=None, override_encodings=None
+    ):
+        """Constructor.
+
+        :param markup: A bytestring representing markup in an unknown encoding.
+
+        :param known_definite_encodings: When determining the encoding
+            of `markup`, these encodings will be tried first, in
+            order. In HTML terms, this corresponds to the "known
+            definite encoding" step defined here:
+            https://html.spec.whatwg.org/multipage/parsing.html#parsing-with-a-known-character-encoding
+
+        :param user_encodings: These encodings will be tried after the
+            `known_definite_encodings` have been tried and failed, and
+            after an attempt to sniff the encoding by looking at a
+            byte order mark has failed. In HTML terms, this
+            corresponds to the step "user has explicitly instructed
+            the user agent to override the document's character
+            encoding", defined here:
+            https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
+
+        :param override_encodings: A deprecated alias for
+            known_definite_encodings. Any encodings here will be tried
+            immediately after the encodings in
+            known_definite_encodings.
+
+        :param smart_quotes_to: By default, Microsoft smart quotes will, like all other characters, be converted
+           to Unicode characters. Setting this to 'ascii' will convert them to ASCII quotes instead.
+           Setting it to 'xml' will convert them to XML entity references, and setting it to 'html'
+           will convert them to HTML entity references.
+        :param is_html: If True, this markup is considered to be HTML. Otherwise
+            it's assumed to be XML.
+        :param exclude_encodings: These encodings will not be considered, even
+            if the sniffing code thinks they might make sense.
+
+        """
         self.smart_quotes_to = smart_quotes_to
         self.tried_encodings = []
         self.contains_replacement_characters = False
         self.is_html = is_html
-
+        self.log = logging.getLogger(__name__)
         self.detector = EncodingDetector(
-            markup, override_encodings, is_html, exclude_encodings)
+            markup, known_definite_encodings, is_html, exclude_encodings,
+            user_encodings, override_encodings
+        )
 
         # Short-circuit if the data is in Unicode to begin with.
         if isinstance(markup, str) or markup == '':
@@ -368,9 +612,10 @@ class UnicodeDammit:
                 if encoding != "ascii":
                     u = self._convert_from(encoding, "replace")
                 if u is not None:
-                    logging.warning(
+                    self.log.warning(
                             "Some characters could not be decoded, and were "
-                            "replaced with REPLACEMENT CHARACTER.")
+                            "replaced with REPLACEMENT CHARACTER."
+                    )
                     self.contains_replacement_characters = True
                     break
 
@@ -399,6 +644,10 @@ class UnicodeDammit:
         return sub
 
     def _convert_from(self, proposed, errors="strict"):
+        """Attempt to convert the markup to the proposed encoding.
+
+        :param proposed: The name of a character encoding.
+        """
         proposed = self.find_codec(proposed)
         if not proposed or (proposed, errors) in self.tried_encodings:
             return None
@@ -413,30 +662,40 @@ class UnicodeDammit:
             markup = smart_quotes_compiled.sub(self._sub_ms_char, markup)
 
         try:
-            #print "Trying to convert document to %s (errors=%s)" % (
-            #    proposed, errors)
+            #print("Trying to convert document to %s (errors=%s)" % (
+            #    proposed, errors))
             u = self._to_unicode(markup, proposed, errors)
             self.markup = u
             self.original_encoding = proposed
         except Exception as e:
-            #print "That didn't work!"
-            #print e
+            #print("That didn't work!")
+            #print(e)
             return None
-        #print "Correct encoding: %s" % proposed
+        #print("Correct encoding: %s" % proposed)
         return self.markup
 
     def _to_unicode(self, data, encoding, errors="strict"):
-        '''Given a string and its encoding, decodes the string into Unicode.
-        %encoding is a string recognized by encodings.aliases'''
+        """Given a string and its encoding, decodes the string into Unicode.
+
+        :param encoding: The name of an encoding.
+        """
         return str(data, encoding, errors)
 
     @property
     def declared_html_encoding(self):
+        """If the markup is an HTML document, returns the encoding declared _within_
+        the document.
+        """
         if not self.is_html:
             return None
         return self.detector.declared_encoding
 
     def find_codec(self, charset):
+        """Convert the name of a character set to a codec name.
+
+        :param charset: The name of a character set.
+        :return: The name of a codec.
+        """
         value = (self._codec(self.CHARSET_ALIASES.get(charset, charset))
                or (charset and self._codec(charset.replace("-", "")))
                or (charset and self._codec(charset.replace("-", "_")))
@@ -726,7 +985,7 @@ class UnicodeDammit:
         0xde : b'\xc3\x9e',     # Þ
         0xdf : b'\xc3\x9f',     # ß
         0xe0 : b'\xc3\xa0',     # à
-        0xe1 : b'\xa1',     # á
+        0xe1 : b'\xa1',         # á
         0xe2 : b'\xc3\xa2',     # â
         0xe3 : b'\xc3\xa3',     # ã
         0xe4 : b'\xc3\xa4',     # ä
@@ -775,12 +1034,16 @@ class UnicodeDammit:
         Currently the only situation supported is Windows-1252 (or its
         subset ISO-8859-1), embedded in UTF-8.
 
-        The input must be a bytestring. If you've already converted
-        the document to Unicode, you're too late.
-
-        The output is a bytestring in which `embedded_encoding`
-        characters have been converted to their `main_encoding`
-        equivalents.
+        :param in_bytes: A bytestring that you suspect contains
+            characters from multiple encodings. Note that this _must_
+            be a bytestring. If you've already converted the document
+            to Unicode, you're too late.
+        :param main_encoding: The primary encoding of `in_bytes`.
+        :param embedded_encoding: The encoding that was used to embed characters
+            in the main document.
+        :return: A bytestring in which `embedded_encoding`
+          characters have been converted to their `main_encoding`
+          equivalents.
         """
         if embedded_encoding.replace('_', '-').lower() not in (
             'windows-1252', 'windows_1252'):
diff --git a/bitbake/lib/bs4/diagnose.py b/bitbake/lib/bs4/diagnose.py
index 083395fb46..4692795340 100644
--- a/bitbake/lib/bs4/diagnose.py
+++ b/bitbake/lib/bs4/diagnose.py
@@ -1,9 +1,10 @@
 """Diagnostic functions, mainly for use when doing tech support."""
 
+# Use of this source code is governed by the MIT license.
 __license__ = "MIT"
 
 import cProfile
-from io import StringIO
+from io import BytesIO
 from html.parser import HTMLParser
 import bs4
 from bs4 import BeautifulSoup, __version__
@@ -16,12 +17,15 @@ import tempfile
 import time
 import traceback
 import sys
-import cProfile
 
 def diagnose(data):
-    """Diagnostic suite for isolating common problems."""
-    print("Diagnostic running on Beautiful Soup %s" % __version__)
-    print("Python version %s" % sys.version)
+    """Diagnostic suite for isolating common problems.
+
+    :param data: A string containing markup that needs to be explained.
+    :return: None; diagnostics are printed to standard output.
+    """
+    print(("Diagnostic running on Beautiful Soup %s" % __version__))
+    print(("Python version %s" % sys.version))
 
     basic_parsers = ["html.parser", "html5lib", "lxml"]
     for name in basic_parsers:
@@ -35,61 +39,70 @@ def diagnose(data):
                 name))
 
     if 'lxml' in basic_parsers:
-        basic_parsers.append(["lxml", "xml"])
+        basic_parsers.append("lxml-xml")
         try:
             from lxml import etree
-            print("Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION)))
+            print(("Found lxml version %s" % ".".join(map(str,etree.LXML_VERSION))))
         except ImportError as e:
-            print (
+            print(
                 "lxml is not installed or couldn't be imported.")
 
 
     if 'html5lib' in basic_parsers:
         try:
             import html5lib
-            print("Found html5lib version %s" % html5lib.__version__)
+            print(("Found html5lib version %s" % html5lib.__version__))
         except ImportError as e:
-            print (
+            print(
                 "html5lib is not installed or couldn't be imported.")
 
     if hasattr(data, 'read'):
         data = data.read()
-    elif os.path.exists(data):
-        print('"%s" looks like a filename. Reading data from the file.' % data)
-        data = open(data).read()
-    elif data.startswith("http:") or data.startswith("https:"):
-        print('"%s" looks like a URL. Beautiful Soup is not an HTTP client.' % data)
-        print("You need to use some other library to get the document behind the URL, and feed that document to Beautiful Soup.")
-        return
-    print()
 
     for parser in basic_parsers:
-        print("Trying to parse your markup with %s" % parser)
+        print(("Trying to parse your markup with %s" % parser))
         success = False
         try:
-            soup = BeautifulSoup(data, parser)
+            soup = BeautifulSoup(data, features=parser)
             success = True
         except Exception as e:
-            print("%s could not parse the markup." % parser)
+            print(("%s could not parse the markup." % parser))
             traceback.print_exc()
         if success:
-            print("Here's what %s did with the markup:" % parser)
-            print(soup.prettify())
+            print(("Here's what %s did with the markup:" % parser))
+            print((soup.prettify()))
 
-        print("-" * 80)
+        print(("-" * 80))
 
 def lxml_trace(data, html=True, **kwargs):
     """Print out the lxml events that occur during parsing.
 
     This lets you see how lxml parses a document when no Beautiful
-    Soup code is running.
+    Soup code is running. You can use this to determine whether
+    an lxml-specific problem is in Beautiful Soup's lxml tree builders
+    or in lxml itself.
+
+    :param data: Some markup.
+    :param html: If True, markup will be parsed with lxml's HTML parser.
+       if False, lxml's XML parser will be used.
     """
     from lxml import etree
-    for event, element in etree.iterparse(StringIO(data), html=html, **kwargs):
+    recover = kwargs.pop('recover', True)
+    if isinstance(data, str):
+        data = data.encode("utf8")
+    reader = BytesIO(data)
+    for event, element in etree.iterparse(
+        reader, html=html, recover=recover, **kwargs
+    ):
         print(("%s, %4s, %s" % (event, element.tag, element.text)))
 
 class AnnouncingParser(HTMLParser):
-    """Announces HTMLParser parse events, without doing anything else."""
+    """Subclass of HTMLParser that announces parse events, without doing
+    anything else.
+
+    You can use this to get a picture of how html.parser sees a given
+    document. The easiest way to do this is to call `htmlparser_trace`.
+    """
 
     def _p(self, s):
         print(s)
@@ -126,6 +139,8 @@ def htmlparser_trace(data):
 
     This lets you see how HTMLParser parses a document when no
     Beautiful Soup code is running.
+
+    :param data: Some markup.
     """
     parser = AnnouncingParser()
     parser.feed(data)
@@ -168,9 +183,9 @@ def rdoc(num_elements=1000):
 
 def benchmark_parsers(num_elements=100000):
     """Very basic head-to-head performance benchmark."""
-    print("Comparative parser benchmark on Beautiful Soup %s" % __version__)
+    print(("Comparative parser benchmark on Beautiful Soup %s" % __version__))
     data = rdoc(num_elements)
-    print("Generated a large invalid HTML document (%d bytes)." % len(data))
+    print(("Generated a large invalid HTML document (%d bytes)." % len(data)))
     
     for parser in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
         success = False
@@ -180,26 +195,26 @@ def benchmark_parsers(num_elements=100000):
             b = time.time()
             success = True
         except Exception as e:
-            print("%s could not parse the markup." % parser)
+            print(("%s could not parse the markup." % parser))
             traceback.print_exc()
         if success:
-            print("BS4+%s parsed the markup in %.2fs." % (parser, b-a))
+            print(("BS4+%s parsed the markup in %.2fs." % (parser, b-a)))
 
     from lxml import etree
     a = time.time()
     etree.HTML(data)
     b = time.time()
-    print("Raw lxml parsed the markup in %.2fs." % (b-a))
+    print(("Raw lxml parsed the markup in %.2fs." % (b-a)))
 
     import html5lib
     parser = html5lib.HTMLParser()
     a = time.time()
     parser.parse(data)
     b = time.time()
-    print("Raw html5lib parsed the markup in %.2fs." % (b-a))
+    print(("Raw html5lib parsed the markup in %.2fs." % (b-a)))
 
 def profile(num_elements=100000, parser="lxml"):
-
+    """Use Python's profiler on a randomly generated document."""
     filehandle = tempfile.NamedTemporaryFile()
     filename = filehandle.name
 
@@ -212,5 +227,6 @@ def profile(num_elements=100000, parser="lxml"):
     stats.sort_stats("cumulative")
     stats.print_stats('_html5lib|bs4', 50)
 
+# If this file is run as a script, standard input is diagnosed.
 if __name__ == '__main__':
     diagnose(sys.stdin.read())
diff --git a/bitbake/lib/bs4/element.py b/bitbake/lib/bs4/element.py
index 68be42d138..0aefe734b2 100644
--- a/bitbake/lib/bs4/element.py
+++ b/bitbake/lib/bs4/element.py
@@ -1,14 +1,27 @@
+# Use of this source code is governed by the MIT license.
 __license__ = "MIT"
 
-import collections.abc
+try:
+    from collections.abc import Callable # Python 3.6
+except ImportError as e:
+    from collections import Callable
 import re
 import sys
 import warnings
-from bs4.dammit import EntitySubstitution
+
+from bs4.css import CSS
+from bs4.formatter import (
+    Formatter,
+    HTMLFormatter,
+    XMLFormatter,
+)
 
 DEFAULT_OUTPUT_ENCODING = "utf-8"
-PY3K = (sys.version_info[0] > 2)
 
+nonwhitespace_re = re.compile(r"\S+")
+
+# NOTE: This isn't used as of 4.7.0. I'm leaving it for a little bit on
+# the off chance someone imported it for their own use.
 whitespace_re = re.compile(r"\s+")
 
 def _alias(attr):
@@ -23,12 +36,49 @@ def _alias(attr):
     return alias
 
 
+# These encodings are recognized by Python (so PageElement.encode
+# could theoretically support them) but XML and HTML don't recognize
+# them (so they should not show up in an XML or HTML document as that
+# document's encoding).
+#
+# If an XML document is encoded in one of these encodings, no encoding
+# will be mentioned in the XML declaration. If an HTML document is
+# encoded in one of these encodings, and the HTML document has a
+# <meta> tag that mentions an encoding, the encoding will be given as
+# the empty string.
+#
+# Source:
+# https://docs.python.org/3/library/codecs.html#python-specific-encodings
+PYTHON_SPECIFIC_ENCODINGS = set([
+    "idna",
+    "mbcs",
+    "oem",
+    "palmos",
+    "punycode",
+    "raw_unicode_escape",
+    "undefined",
+    "unicode_escape",
+    "raw-unicode-escape",
+    "unicode-escape",
+    "string-escape",
+    "string_escape",
+])
+
+
 class NamespacedAttribute(str):
+    """A namespaced string (e.g. 'xml:lang') that remembers the namespace
+    ('xml') and the name ('lang') that were used to create it.
+    """
 
-    def __new__(cls, prefix, name, namespace=None):
-        if name is None:
+    def __new__(cls, prefix, name=None, namespace=None):
+        if not name:
+            # This is the default namespace. Its name "has no value"
+            # per https://www.w3.org/TR/xml-names/#defaulting
+            name = None
+
+        if not name:
             obj = str.__new__(cls, prefix)
-        elif prefix is None:
+        elif not prefix:
             # Not really namespaced.
             obj = str.__new__(cls, name)
         else:
@@ -54,6 +104,11 @@ class CharsetMetaAttributeValue(AttributeValueWithCharsetSubstitution):
         return obj
 
     def encode(self, encoding):
+        """When an HTML document is being encoded to a given encoding, the
+        value of a meta tag's 'charset' is the name of the encoding.
+        """
+        if encoding in PYTHON_SPECIFIC_ENCODINGS:
+            return ''
         return encoding
 
 
@@ -79,118 +134,44 @@ class ContentMetaAttributeValue(AttributeValueWithCharsetSubstitution):
         return obj
 
     def encode(self, encoding):
+        if encoding in PYTHON_SPECIFIC_ENCODINGS:
+            return ''
         def rewrite(match):
             return match.group(1) + encoding
         return self.CHARSET_RE.sub(rewrite, self.original_value)
 
-class HTMLAwareEntitySubstitution(EntitySubstitution):
-
-    """Entity substitution rules that are aware of some HTML quirks.
 
-    Specifically, the contents of <script> and <style> tags should not
-    undergo entity substitution.
+class PageElement(object):
+    """Contains the navigational information for some part of the page:
+    that is, its current location in the parse tree.
 
-    Incoming NavigableString objects are checked to see if they're the
-    direct children of a <script> or <style> tag.
+    NavigableString, Tag, etc. are all subclasses of PageElement.
     """
 
-    cdata_containing_tags = set(["script", "style"])
+    # In general, we can't tell just by looking at an element whether
+    # it's contained in an XML document or an HTML document. But for
+    # Tags (q.v.) we can store this information at parse time.
+    known_xml = None
 
-    preformatted_tags = set(["pre"])
-
-    @classmethod
-    def _substitute_if_appropriate(cls, ns, f):
-        if (isinstance(ns, NavigableString)
-            and ns.parent is not None
-            and ns.parent.name in cls.cdata_containing_tags):
-            # Do nothing.
-            return ns
-        # Substitute.
-        return f(ns)
+    def setup(self, parent=None, previous_element=None, next_element=None,
+              previous_sibling=None, next_sibling=None):
+        """Sets up the initial relations between this element and
+        other elements.
 
-    @classmethod
-    def substitute_html(cls, ns):
-        return cls._substitute_if_appropriate(
-            ns, EntitySubstitution.substitute_html)
+        :param parent: The parent of this element.
 
-    @classmethod
-    def substitute_xml(cls, ns):
-        return cls._substitute_if_appropriate(
-            ns, EntitySubstitution.substitute_xml)
+        :param previous_element: The element parsed immediately before
+            this one.
 
-class PageElement(object):
-    """Contains the navigational information for some part of the page
-    (either a tag or a piece of text)"""
-
-    # There are five possible values for the "formatter" argument passed in
-    # to methods like encode() and prettify():
-    #
-    # "html" - All Unicode characters with corresponding HTML entities
-    #   are converted to those entities on output.
-    # "minimal" - Bare ampersands and angle brackets are converted to
-    #   XML entities: &amp; &lt; &gt;
-    # None - The null formatter. Unicode characters are never
-    #   converted to entities.  This is not recommended, but it's
-    #   faster than "minimal".
-    # A function - This function will be called on every string that
-    #  needs to undergo entity substitution.
-    #
-
-    # In an HTML document, the default "html" and "minimal" functions
-    # will leave the contents of <script> and <style> tags alone. For
-    # an XML document, all tags will be given the same treatment.
-
-    HTML_FORMATTERS = {
-        "html" : HTMLAwareEntitySubstitution.substitute_html,
-        "minimal" : HTMLAwareEntitySubstitution.substitute_xml,
-        None : None
-        }
-
-    XML_FORMATTERS = {
-        "html" : EntitySubstitution.substitute_html,
-        "minimal" : EntitySubstitution.substitute_xml,
-        None : None
-        }
-
-    def format_string(self, s, formatter='minimal'):
-        """Format the given string using the given formatter."""
-        if not isinstance(formatter, collections.abc.Callable):
-            formatter = self._formatter_for_name(formatter)
-        if formatter is None:
-            output = s
-        else:
-            output = formatter(s)
-        return output
+        :param next_element: The element parsed immediately before
+            this one.
 
-    @property
-    def _is_xml(self):
-        """Is this element part of an XML tree or an HTML tree?
+        :param previous_sibling: The most recently encountered element
+            on the same level of the parse tree as this one.
 
-        This is used when mapping a formatter name ("minimal") to an
-        appropriate function (one that performs entity-substitution on
-        the contents of <script> and <style> tags, or not). It's
-        inefficient, but it should be called very rarely.
+        :param previous_sibling: The next element to be encountered
+            on the same level of the parse tree as this one.
         """
-        if self.parent is None:
-            # This is the top-level object. It should have .is_xml set
-            # from tree creation. If not, take a guess--BS is usually
-            # used on HTML markup.
-            return getattr(self, 'is_xml', False)
-        return self.parent._is_xml
-
-    def _formatter_for_name(self, name):
-        "Look up a formatter function based on its name and the tree."
-        if self._is_xml:
-            return self.XML_FORMATTERS.get(
-                name, EntitySubstitution.substitute_xml)
-        else:
-            return self.HTML_FORMATTERS.get(
-                name, HTMLAwareEntitySubstitution.substitute_xml)
-
-    def setup(self, parent=None, previous_element=None, next_element=None,
-              previous_sibling=None, next_sibling=None):
-        """Sets up the initial relations between this element and
-        other elements."""
         self.parent = parent
 
         self.previous_element = previous_element
@@ -198,48 +179,156 @@ class PageElement(object):
             self.previous_element.next_element = self
 
         self.next_element = next_element
-        if self.next_element:
+        if self.next_element is not None:
             self.next_element.previous_element = self
 
         self.next_sibling = next_sibling
-        if self.next_sibling:
+        if self.next_sibling is not None:
             self.next_sibling.previous_sibling = self
 
-        if (not previous_sibling
+        if (previous_sibling is None
             and self.parent is not None and self.parent.contents):
             previous_sibling = self.parent.contents[-1]
 
         self.previous_sibling = previous_sibling
-        if previous_sibling:
+        if previous_sibling is not None:
             self.previous_sibling.next_sibling = self
 
+    def format_string(self, s, formatter):
+        """Format the given string using the given formatter.
+
+        :param s: A string.
+        :param formatter: A Formatter object, or a string naming one of the standard formatters.
+        """
+        if formatter is None:
+            return s
+        if not isinstance(formatter, Formatter):
+            formatter = self.formatter_for_name(formatter)
+        output = formatter.substitute(s)
+        return output
+
+    def formatter_for_name(self, formatter):
+        """Look up or create a Formatter for the given identifier,
+        if necessary.
+
+        :param formatter: Can be a Formatter object (used as-is), a
+            function (used as the entity substitution hook for an
+            XMLFormatter or HTMLFormatter), or a string (used to look
+            up an XMLFormatter or HTMLFormatter in the appropriate
+            registry.
+        """
+        if isinstance(formatter, Formatter):
+            return formatter
+        if self._is_xml:
+            c = XMLFormatter
+        else:
+            c = HTMLFormatter
+        if isinstance(formatter, Callable):
+            return c(entity_substitution=formatter)
+        return c.REGISTRY[formatter]
+
+    @property
+    def _is_xml(self):
+        """Is this element part of an XML tree or an HTML tree?
+
+        This is used in formatter_for_name, when deciding whether an
+        XMLFormatter or HTMLFormatter is more appropriate. It can be
+        inefficient, but it should be called very rarely.
+        """
+        if self.known_xml is not None:
+            # Most of the time we will have determined this when the
+            # document is parsed.
+            return self.known_xml
+
+        # Otherwise, it's likely that this element was created by
+        # direct invocation of the constructor from within the user's
+        # Python code.
+        if self.parent is None:
+            # This is the top-level object. It should have .known_xml set
+            # from tree creation. If not, take a guess--BS is usually
+            # used on HTML markup.
+            return getattr(self, 'is_xml', False)
+        return self.parent._is_xml
+
     nextSibling = _alias("next_sibling")  # BS3
     previousSibling = _alias("previous_sibling")  # BS3
 
-    def replace_with(self, replace_with):
-        if not self.parent:
+    default = object()
+    def _all_strings(self, strip=False, types=default):
+        """Yield all strings of certain classes, possibly stripping them.
+
+        This is implemented differently in Tag and NavigableString.
+        """
+        raise NotImplementedError()
+
+    @property
+    def stripped_strings(self):
+        """Yield all strings in this PageElement, stripping them first.
+
+        :yield: A sequence of stripped strings.
+        """
+        for string in self._all_strings(True):
+            yield string
+
+    def get_text(self, separator="", strip=False,
+                 types=default):
+        """Get all child strings of this PageElement, concatenated using the
+        given separator.
+
+        :param separator: Strings will be concatenated using this separator.
+
+        :param strip: If True, strings will be stripped before being
+            concatenated.
+
+        :param types: A tuple of NavigableString subclasses. Any
+            strings of a subclass not found in this list will be
+            ignored. Although there are exceptions, the default
+            behavior in most cases is to consider only NavigableString
+            and CData objects. That means no comments, processing
+            instructions, etc.
+
+        :return: A string.
+        """
+        return separator.join([s for s in self._all_strings(
+                    strip, types=types)])
+    getText = get_text
+    text = property(get_text)
+
+    def replace_with(self, *args):
+        """Replace this PageElement with one or more PageElements, keeping the
+        rest of the tree the same.
+
+        :param args: One or more PageElements.
+        :return: `self`, no longer part of the tree.
+        """
+        if self.parent is None:
             raise ValueError(
-                "Cannot replace one element with another when the"
+                "Cannot replace one element with another when the "
                 "element to be replaced is not part of a tree.")
-        if replace_with is self:
+        if len(args) == 1 and args[0] is self:
             return
-        if replace_with is self.parent:
+        if any(x is self.parent for x in args):
             raise ValueError("Cannot replace a Tag with its parent.")
         old_parent = self.parent
         my_index = self.parent.index(self)
-        self.extract()
-        old_parent.insert(my_index, replace_with)
+        self.extract(_self_index=my_index)
+        for idx, replace_with in enumerate(args, start=my_index):
+            old_parent.insert(idx, replace_with)
         return self
     replaceWith = replace_with  # BS3
 
     def unwrap(self):
+        """Replace this PageElement with its contents.
+
+        :return: `self`, no longer part of the tree.
+        """
         my_parent = self.parent
-        if not self.parent:
+        if self.parent is None:
             raise ValueError(
                 "Cannot replace an element with its contents when that"
                 "element is not part of a tree.")
         my_index = self.parent.index(self)
-        self.extract()
+        self.extract(_self_index=my_index)
         for child in reversed(self.contents[:]):
             my_parent.insert(my_index, child)
         return self
@@ -247,14 +336,29 @@ class PageElement(object):
     replaceWithChildren = unwrap  # BS3
 
     def wrap(self, wrap_inside):
+        """Wrap this PageElement inside another one.
+
+        :param wrap_inside: A PageElement.
+        :return: `wrap_inside`, occupying the position in the tree that used
+           to be occupied by `self`, and with `self` inside it.
+        """
         me = self.replace_with(wrap_inside)
         wrap_inside.append(me)
         return wrap_inside
 
-    def extract(self):
-        """Destructively rips this element out of the tree."""
+    def extract(self, _self_index=None):
+        """Destructively rips this element out of the tree.
+
+        :param _self_index: The location of this element in its parent's
+           .contents, if known. Passing this in allows for a performance
+           optimization.
+
+        :return: `self`, no longer part of the tree.
+        """
         if self.parent is not None:
-            del self.parent.contents[self.parent.index(self)]
+            if _self_index is None:
+                _self_index = self.parent.index(self)
+            del self.parent.contents[_self_index]
 
         #Find the two elements that would be next to each other if
         #this element (and any children) hadn't been parsed. Connect
@@ -281,8 +385,13 @@ class PageElement(object):
         return self
 
     def _last_descendant(self, is_initialized=True, accept_self=True):
-        "Finds the last element beneath this object to be parsed."
-        if is_initialized and self.next_sibling:
+        """Finds the last element beneath this object to be parsed.
+
+        :param is_initialized: Has `setup` been called on this PageElement
+            yet?
+        :param accept_self: Is `self` an acceptable answer to the question?
+        """
+        if is_initialized and self.next_sibling is not None:
             last_child = self.next_sibling.previous_element
         else:
             last_child = self
@@ -295,6 +404,14 @@ class PageElement(object):
     _lastRecursiveChild = _last_descendant
 
     def insert(self, position, new_child):
+        """Insert a new PageElement in the list of this PageElement's children.
+
+        This works the same way as `list.insert`.
+
+        :param position: The numeric position that should be occupied
+           in `self.children` by the new PageElement.
+        :param new_child: A PageElement.
+        """
         if new_child is None:
             raise ValueError("Cannot insert None into a tag.")
         if new_child is self:
@@ -303,6 +420,14 @@ class PageElement(object):
             and not isinstance(new_child, NavigableString)):
             new_child = NavigableString(new_child)
 
+        from bs4 import BeautifulSoup
+        if isinstance(new_child, BeautifulSoup):
+            # We don't want to end up with a situation where one BeautifulSoup
+            # object contains another. Insert the children one at a time.
+            for subchild in list(new_child.contents):
+                self.insert(position, subchild)
+                position += 1
+            return
         position = min(position, len(self.contents))
         if hasattr(new_child, 'parent') and new_child.parent is not None:
             # We're 'inserting' an element that's already one
@@ -361,160 +486,326 @@ class PageElement(object):
         self.contents.insert(position, new_child)
 
     def append(self, tag):
-        """Appends the given tag to the contents of this tag."""
+        """Appends the given PageElement to the contents of this one.
+
+        :param tag: A PageElement.
+        """
         self.insert(len(self.contents), tag)
 
-    def insert_before(self, predecessor):
-        """Makes the given element the immediate predecessor of this one.
+    def extend(self, tags):
+        """Appends the given PageElements to this one's contents.
 
-        The two elements will have the same parent, and the given element
+        :param tags: A list of PageElements. If a single Tag is
+            provided instead, this PageElement's contents will be extended
+            with that Tag's contents.
+        """
+        if isinstance(tags, Tag):
+            tags = tags.contents
+        if isinstance(tags, list):
+            # Moving items around the tree may change their position in
+            # the original list. Make a list that won't change.
+            tags = list(tags)
+        for tag in tags:
+            self.append(tag)
+
+    def insert_before(self, *args):
+        """Makes the given element(s) the immediate predecessor of this one.
+
+        All the elements will have the same parent, and the given elements
         will be immediately before this one.
+
+        :param args: One or more PageElements.
         """
-        if self is predecessor:
-            raise ValueError("Can't insert an element before itself.")
         parent = self.parent
         if parent is None:
             raise ValueError(
                 "Element has no parent, so 'before' has no meaning.")
-        # Extract first so that the index won't be screwed up if they
-        # are siblings.
-        if isinstance(predecessor, PageElement):
-            predecessor.extract()
-        index = parent.index(self)
-        parent.insert(index, predecessor)
-
-    def insert_after(self, successor):
-        """Makes the given element the immediate successor of this one.
-
-        The two elements will have the same parent, and the given element
+        if any(x is self for x in args):
+                raise ValueError("Can't insert an element before itself.")
+        for predecessor in args:
+            # Extract first so that the index won't be screwed up if they
+            # are siblings.
+            if isinstance(predecessor, PageElement):
+                predecessor.extract()
+            index = parent.index(self)
+            parent.insert(index, predecessor)
+
+    def insert_after(self, *args):
+        """Makes the given element(s) the immediate successor of this one.
+
+        The elements will have the same parent, and the given elements
         will be immediately after this one.
+
+        :param args: One or more PageElements.
         """
-        if self is successor:
-            raise ValueError("Can't insert an element after itself.")
+        # Do all error checking before modifying the tree.
         parent = self.parent
         if parent is None:
             raise ValueError(
                 "Element has no parent, so 'after' has no meaning.")
-        # Extract first so that the index won't be screwed up if they
-        # are siblings.
-        if isinstance(successor, PageElement):
-            successor.extract()
-        index = parent.index(self)
-        parent.insert(index+1, successor)
-
-    def find_next(self, name=None, attrs={}, text=None, **kwargs):
-        """Returns the first item that matches the given criteria and
-        appears after this Tag in the document."""
-        return self._find_one(self.find_all_next, name, attrs, text, **kwargs)
+        if any(x is self for x in args):
+            raise ValueError("Can't insert an element after itself.")
+
+        offset = 0
+        for successor in args:
+            # Extract first so that the index won't be screwed up if they
+            # are siblings.
+            if isinstance(successor, PageElement):
+                successor.extract()
+            index = parent.index(self)
+            parent.insert(index+1+offset, successor)
+            offset += 1
+
+    def find_next(self, name=None, attrs={}, string=None, **kwargs):
+        """Find the first PageElement that matches the given criteria and
+        appears later in the document than this PageElement.
+
+        All find_* methods take a common set of arguments. See the online
+        documentation for detailed explanations.
+
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :param string: A filter for a NavigableString with specific text.
+        :kwargs: A dictionary of filters on attribute values.
+        :return: A PageElement.
+        :rtype: bs4.element.Tag | bs4.element.NavigableString
+        """
+        return self._find_one(self.find_all_next, name, attrs, string, **kwargs)
     findNext = find_next  # BS3
 
-    def find_all_next(self, name=None, attrs={}, text=None, limit=None,
+    def find_all_next(self, name=None, attrs={}, string=None, limit=None,
                     **kwargs):
-        """Returns all items that match the given criteria and appear
-        after this Tag in the document."""
-        return self._find_all(name, attrs, text, limit, self.next_elements,
-                             **kwargs)
+        """Find all PageElements that match the given criteria and appear
+        later in the document than this PageElement.
+
+        All find_* methods take a common set of arguments. See the online
+        documentation for detailed explanations.
+
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :param string: A filter for a NavigableString with specific text.
+        :param limit: Stop looking after finding this many results.
+        :kwargs: A dictionary of filters on attribute values.
+        :return: A ResultSet containing PageElements.
+        """
+        _stacklevel = kwargs.pop('_stacklevel', 2)
+        return self._find_all(name, attrs, string, limit, self.next_elements,
+                              _stacklevel=_stacklevel+1, **kwargs)
     findAllNext = find_all_next  # BS3
 
-    def find_next_sibling(self, name=None, attrs={}, text=None, **kwargs):
-        """Returns the closest sibling to this Tag that matches the
-        given criteria and appears after this Tag in the document."""
-        return self._find_one(self.find_next_siblings, name, attrs, text,
+    def find_next_sibling(self, name=None, attrs={}, string=None, **kwargs):
+        """Find the closest sibling to this PageElement that matches the
+        given criteria and appears later in the document.
+
+        All find_* methods take a common set of arguments. See the
+        online documentation for detailed explanations.
+
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :param string: A filter for a NavigableString with specific text.
+        :kwargs: A dictionary of filters on attribute values.
+        :return: A PageElement.
+        :rtype: bs4.element.Tag | bs4.element.NavigableString
+        """
+        return self._find_one(self.find_next_siblings, name, attrs, string,
                              **kwargs)
     findNextSibling = find_next_sibling  # BS3
 
-    def find_next_siblings(self, name=None, attrs={}, text=None, limit=None,
+    def find_next_siblings(self, name=None, attrs={}, string=None, limit=None,
                            **kwargs):
-        """Returns the siblings of this Tag that match the given
-        criteria and appear after this Tag in the document."""
-        return self._find_all(name, attrs, text, limit,
-                              self.next_siblings, **kwargs)
+        """Find all siblings of this PageElement that match the given criteria
+        and appear later in the document.
+
+        All find_* methods take a common set of arguments. See the online
+        documentation for detailed explanations.
+
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :param string: A filter for a NavigableString with specific text.
+        :param limit: Stop looking after finding this many results.
+        :kwargs: A dictionary of filters on attribute values.
+        :return: A ResultSet of PageElements.
+        :rtype: bs4.element.ResultSet
+        """
+        _stacklevel = kwargs.pop('_stacklevel', 2)
+        return self._find_all(
+            name, attrs, string, limit,
+            self.next_siblings, _stacklevel=_stacklevel+1, **kwargs
+        )
     findNextSiblings = find_next_siblings   # BS3
     fetchNextSiblings = find_next_siblings  # BS2
 
-    def find_previous(self, name=None, attrs={}, text=None, **kwargs):
-        """Returns the first item that matches the given criteria and
-        appears before this Tag in the document."""
+    def find_previous(self, name=None, attrs={}, string=None, **kwargs):
+        """Look backwards in the document from this PageElement and find the
+        first PageElement that matches the given criteria.
+
+        All find_* methods take a common set of arguments. See the online
+        documentation for detailed explanations.
+
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :param string: A filter for a NavigableString with specific text.
+        :kwargs: A dictionary of filters on attribute values.
+        :return: A PageElement.
+        :rtype: bs4.element.Tag | bs4.element.NavigableString
+        """
         return self._find_one(
-            self.find_all_previous, name, attrs, text, **kwargs)
+            self.find_all_previous, name, attrs, string, **kwargs)
     findPrevious = find_previous  # BS3
 
-    def find_all_previous(self, name=None, attrs={}, text=None, limit=None,
+    def find_all_previous(self, name=None, attrs={}, string=None, limit=None,
                         **kwargs):
-        """Returns all items that match the given criteria and appear
-        before this Tag in the document."""
-        return self._find_all(name, attrs, text, limit, self.previous_elements,
-                           **kwargs)
+        """Look backwards in the document from this PageElement and find all
+        PageElements that match the given criteria.
+
+        All find_* methods take a common set of arguments. See the online
+        documentation for detailed explanations.
+
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :param string: A filter for a NavigableString with specific text.
+        :param limit: Stop looking after finding this many results.
+        :kwargs: A dictionary of filters on attribute values.
+        :return: A ResultSet of PageElements.
+        :rtype: bs4.element.ResultSet
+        """
+        _stacklevel = kwargs.pop('_stacklevel', 2)
+        return self._find_all(
+            name, attrs, string, limit, self.previous_elements,
+            _stacklevel=_stacklevel+1, **kwargs
+        )
     findAllPrevious = find_all_previous  # BS3
     fetchPrevious = find_all_previous    # BS2
 
-    def find_previous_sibling(self, name=None, attrs={}, text=None, **kwargs):
-        """Returns the closest sibling to this Tag that matches the
-        given criteria and appears before this Tag in the document."""
-        return self._find_one(self.find_previous_siblings, name, attrs, text,
+    def find_previous_sibling(self, name=None, attrs={}, string=None, **kwargs):
+        """Returns the closest sibling to this PageElement that matches the
+        given criteria and appears earlier in the document.
+
+        All find_* methods take a common set of arguments. See the online
+        documentation for detailed explanations.
+
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :param string: A filter for a NavigableString with specific text.
+        :kwargs: A dictionary of filters on attribute values.
+        :return: A PageElement.
+        :rtype: bs4.element.Tag | bs4.element.NavigableString
+        """
+        return self._find_one(self.find_previous_siblings, name, attrs, string,
                              **kwargs)
     findPreviousSibling = find_previous_sibling  # BS3
 
-    def find_previous_siblings(self, name=None, attrs={}, text=None,
+    def find_previous_siblings(self, name=None, attrs={}, string=None,
                                limit=None, **kwargs):
-        """Returns the siblings of this Tag that match the given
-        criteria and appear before this Tag in the document."""
-        return self._find_all(name, attrs, text, limit,
-                              self.previous_siblings, **kwargs)
+        """Returns all siblings to this PageElement that match the
+        given criteria and appear earlier in the document.
+
+        All find_* methods take a common set of arguments. See the online
+        documentation for detailed explanations.
+
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :param string: A filter for a NavigableString with specific text.
+        :param limit: Stop looking after finding this many results.
+        :kwargs: A dictionary of filters on attribute values.
+        :return: A ResultSet of PageElements.
+        :rtype: bs4.element.ResultSet
+        """
+        _stacklevel = kwargs.pop('_stacklevel', 2)
+        return self._find_all(
+            name, attrs, string, limit,
+            self.previous_siblings, _stacklevel=_stacklevel+1, **kwargs
+        )
     findPreviousSiblings = find_previous_siblings   # BS3
     fetchPreviousSiblings = find_previous_siblings  # BS2
 
     def find_parent(self, name=None, attrs={}, **kwargs):
-        """Returns the closest parent of this Tag that matches the given
-        criteria."""
+        """Find the closest parent of this PageElement that matches the given
+        criteria.
+
+        All find_* methods take a common set of arguments. See the online
+        documentation for detailed explanations.
+
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :kwargs: A dictionary of filters on attribute values.
+
+        :return: A PageElement.
+        :rtype: bs4.element.Tag | bs4.element.NavigableString
+        """
         # NOTE: We can't use _find_one because findParents takes a different
         # set of arguments.
         r = None
-        l = self.find_parents(name, attrs, 1, **kwargs)
+        l = self.find_parents(name, attrs, 1, _stacklevel=3, **kwargs)
         if l:
             r = l[0]
         return r
     findParent = find_parent  # BS3
 
     def find_parents(self, name=None, attrs={}, limit=None, **kwargs):
-        """Returns the parents of this Tag that match the given
-        criteria."""
+        """Find all parents of this PageElement that match the given criteria.
+
+        All find_* methods take a common set of arguments. See the online
+        documentation for detailed explanations.
 
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :param limit: Stop looking after finding this many results.
+        :kwargs: A dictionary of filters on attribute values.
+
+        :return: A PageElement.
+        :rtype: bs4.element.Tag | bs4.element.NavigableString
+        """
+        _stacklevel = kwargs.pop('_stacklevel', 2)
         return self._find_all(name, attrs, None, limit, self.parents,
-                             **kwargs)
+                              _stacklevel=_stacklevel+1, **kwargs)
     findParents = find_parents   # BS3
     fetchParents = find_parents  # BS2
 
     @property
     def next(self):
+        """The PageElement, if any, that was parsed just after this one.
+
+        :return: A PageElement.
+        :rtype: bs4.element.Tag | bs4.element.NavigableString
+        """
         return self.next_element
 
     @property
     def previous(self):
+        """The PageElement, if any, that was parsed just before this one.
+
+        :return: A PageElement.
+        :rtype: bs4.element.Tag | bs4.element.NavigableString
+        """
         return self.previous_element
 
     #These methods do the real heavy lifting.
 
-    def _find_one(self, method, name, attrs, text, **kwargs):
+    def _find_one(self, method, name, attrs, string, **kwargs):
         r = None
-        l = method(name, attrs, text, 1, **kwargs)
+        l = method(name, attrs, string, 1, _stacklevel=4, **kwargs)
         if l:
             r = l[0]
         return r
 
-    def _find_all(self, name, attrs, text, limit, generator, **kwargs):
+    def _find_all(self, name, attrs, string, limit, generator, **kwargs):
         "Iterates over a generator looking for things that match."
+        _stacklevel = kwargs.pop('_stacklevel', 3)
 
-        if text is None and 'string' in kwargs:
-            text = kwargs['string']
-            del kwargs['string']
+        if string is None and 'text' in kwargs:
+            string = kwargs.pop('text')
+            warnings.warn(
+                "The 'text' argument to find()-type methods is deprecated. Use 'string' instead.",
+                DeprecationWarning, stacklevel=_stacklevel
+            )
 
         if isinstance(name, SoupStrainer):
             strainer = name
         else:
-            strainer = SoupStrainer(name, attrs, text, **kwargs)
+            strainer = SoupStrainer(name, attrs, string, **kwargs)
 
-        if text is None and not limit and not attrs and not kwargs:
+        if string is None and not limit and not attrs and not kwargs:
             if name is True or name is None:
                 # Optimization to find all tags.
                 result = (element for element in generator
@@ -522,9 +813,23 @@ class PageElement(object):
                 return ResultSet(strainer, result)
             elif isinstance(name, str):
                 # Optimization to find all tags with a given name.
+                if name.count(':') == 1:
+                    # This is a name with a prefix. If this is a namespace-aware document,
+                    # we need to match the local name against tag.name. If not,
+                    # we need to match the fully-qualified name against tag.name.
+                    prefix, local_name = name.split(':', 1)
+                else:
+                    prefix = None
+                    local_name = name
                 result = (element for element in generator
                           if isinstance(element, Tag)
-                            and element.name == name)
+                          and (
+                              element.name == name
+                          ) or (
+                              element.name == local_name
+                              and (prefix is None or element.prefix == prefix)
+                          )
+                )
                 return ResultSet(strainer, result)
         results = ResultSet(strainer)
         while True:
@@ -544,6 +849,10 @@ class PageElement(object):
     #NavigableStrings and Tags.
     @property
     def next_elements(self):
+        """All PageElements that were parsed after this one.
+
+        :yield: A sequence of PageElements.
+        """
         i = self.next_element
         while i is not None:
             yield i
@@ -551,6 +860,11 @@ class PageElement(object):
 
     @property
     def next_siblings(self):
+        """All PageElements that are siblings of this one but were parsed
+        later.
+
+        :yield: A sequence of PageElements.
+        """
         i = self.next_sibling
         while i is not None:
             yield i
@@ -558,6 +872,10 @@ class PageElement(object):
 
     @property
     def previous_elements(self):
+        """All PageElements that were parsed before this one.
+
+        :yield: A sequence of PageElements.
+        """
         i = self.previous_element
         while i is not None:
             yield i
@@ -565,6 +883,11 @@ class PageElement(object):
 
     @property
     def previous_siblings(self):
+        """All PageElements that are siblings of this one but were parsed
+        earlier.
+
+        :yield: A sequence of PageElements.
+        """
         i = self.previous_sibling
         while i is not None:
             yield i
@@ -572,87 +895,23 @@ class PageElement(object):
 
     @property
     def parents(self):
+        """All PageElements that are parents of this PageElement.
+
+        :yield: A sequence of PageElements.
+        """
         i = self.parent
         while i is not None:
             yield i
             i = i.parent
 
-    # Methods for supporting CSS selectors.
-
-    tag_name_re = re.compile(r'^[a-zA-Z0-9][-.a-zA-Z0-9:_]*$')
-
-    # /^([a-zA-Z0-9][-.a-zA-Z0-9:_]*)\[(\w+)([=~\|\^\$\*]?)=?"?([^\]"]*)"?\]$/
-    #   \---------------------------/  \---/\-------------/    \-------/
-    #     |                              |         |               |
-    #     |                              |         |           The value
-    #     |                              |    ~,|,^,$,* or =
-    #     |                           Attribute
-    #    Tag
-    attribselect_re = re.compile(
-        r'^(?P<tag>[a-zA-Z0-9][-.a-zA-Z0-9:_]*)?\[(?P<attribute>[\w-]+)(?P<operator>[=~\|\^\$\*]?)' +
-        r'=?"?(?P<value>[^\]"]*)"?\]$'
-        )
-
-    def _attr_value_as_string(self, value, default=None):
-        """Force an attribute value into a string representation.
+    @property
+    def decomposed(self):
+        """Check whether a PageElement has been decomposed.
 
-        A multi-valued attribute will be converted into a
-        space-separated stirng.
+        :rtype: bool
         """
-        value = self.get(value, default)
-        if isinstance(value, list) or isinstance(value, tuple):
-            value =" ".join(value)
-        return value
-
-    def _tag_name_matches_and(self, function, tag_name):
-        if not tag_name:
-            return function
-        else:
-            def _match(tag):
-                return tag.name == tag_name and function(tag)
-            return _match
-
-    def _attribute_checker(self, operator, attribute, value=''):
-        """Create a function that performs a CSS selector operation.
-
-        Takes an operator, attribute and optional value. Returns a
-        function that will return True for elements that match that
-        combination.
-        """
-        if operator == '=':
-            # string representation of `attribute` is equal to `value`
-            return lambda el: el._attr_value_as_string(attribute) == value
-        elif operator == '~':
-            # space-separated list representation of `attribute`
-            # contains `value`
-            def _includes_value(element):
-                attribute_value = element.get(attribute, [])
-                if not isinstance(attribute_value, list):
-                    attribute_value = attribute_value.split()
-                return value in attribute_value
-            return _includes_value
-        elif operator == '^':
-            # string representation of `attribute` starts with `value`
-            return lambda el: el._attr_value_as_string(
-                attribute, '').startswith(value)
-        elif operator == '$':
-            # string represenation of `attribute` ends with `value`
-            return lambda el: el._attr_value_as_string(
-                attribute, '').endswith(value)
-        elif operator == '*':
-            # string representation of `attribute` contains `value`
-            return lambda el: value in el._attr_value_as_string(attribute, '')
-        elif operator == '|':
-            # string representation of `attribute` is either exactly
-            # `value` or starts with `value` and then a dash.
-            def _is_or_starts_with_dash(element):
-                attribute_value = element._attr_value_as_string(attribute, '')
-                return (attribute_value == value or attribute_value.startswith(
-                        value + '-'))
-            return _is_or_starts_with_dash
-        else:
-            return lambda el: el.has_attr(attribute)
-
+        return getattr(self, '_decomposed', False) or False
+   
     # Old non-property versions of the generators, for backwards
     # compatibility with BS3.
     def nextGenerator(self):
@@ -672,6 +931,11 @@ class PageElement(object):
 
 
 class NavigableString(str, PageElement):
+    """A Python Unicode string that is part of a parse tree.
+
+    When Beautiful Soup parses the markup <b>penguin</b>, it will
+    create a NavigableString for the string "penguin".
+    """
 
     PREFIX = ''
     SUFFIX = ''
@@ -691,12 +955,22 @@ class NavigableString(str, PageElement):
         u.setup()
         return u
 
-    def __copy__(self):
+    def __deepcopy__(self, memo, recursive=False):
         """A copy of a NavigableString has the same contents and class
         as the original, but it is not connected to the parse tree.
+
+        :param recursive: This parameter is ignored; it's only defined
+           so that NavigableString.__deepcopy__ implements the same
+           signature as Tag.__deepcopy__.
         """
         return type(self)(self)
 
+    def __copy__(self):
+        """A copy of a NavigableString can only be a deep copy, because
+        only one PageElement can occupy a given place in a parse tree.
+        """
+        return self.__deepcopy__({})
+
     def __getnewargs__(self):
         return (str(self),)
 
@@ -712,55 +986,146 @@ class NavigableString(str, PageElement):
                     self.__class__.__name__, attr))
 
     def output_ready(self, formatter="minimal"):
+        """Run the string through the provided formatter.
+
+        :param formatter: A Formatter object, or a string naming one of the standard formatters.
+        """
         output = self.format_string(self, formatter)
         return self.PREFIX + output + self.SUFFIX
 
     @property
     def name(self):
+        """Since a NavigableString is not a Tag, it has no .name.
+
+        This property is implemented so that code like this doesn't crash
+        when run on a mixture of Tag and NavigableString objects:
+            [x.name for x in tag.children]
+        """
         return None
 
     @name.setter
     def name(self, name):
+        """Prevent NavigableString.name from ever being set."""
         raise AttributeError("A NavigableString cannot be given a name.")
 
+    def _all_strings(self, strip=False, types=PageElement.default):
+        """Yield all strings of certain classes, possibly stripping them.
+
+        This makes it easy for NavigableString to implement methods
+        like get_text() as conveniences, creating a consistent
+        text-extraction API across all PageElements.
+
+        :param strip: If True, all strings will be stripped before being
+            yielded.
+
+        :param types: A tuple of NavigableString subclasses. If this
+            NavigableString isn't one of those subclasses, the
+            sequence will be empty. By default, the subclasses
+            considered are NavigableString and CData objects. That
+            means no comments, processing instructions, etc.
+
+        :yield: A sequence that either contains this string, or is empty.
+
+        """
+        if types is self.default:
+            # This is kept in Tag because it's full of subclasses of
+            # this class, which aren't defined until later in the file.
+            types = Tag.DEFAULT_INTERESTING_STRING_TYPES
+
+        # Do nothing if the caller is looking for specific types of
+        # string, and we're of a different type.
+        #
+        # We check specific types instead of using isinstance(self,
+        # types) because all of these classes subclass
+        # NavigableString. Anyone who's using this feature probably
+        # wants generic NavigableStrings but not other stuff.
+        my_type = type(self)
+        if types is not None:
+            if isinstance(types, type):
+                # Looking for a single type.
+                if my_type is not types:
+                    return
+            elif my_type not in types:
+                # Looking for one of a list of types.
+                return
+
+        value = self
+        if strip:
+            value = value.strip()
+        if len(value) > 0:
+            yield value
+    strings = property(_all_strings)
+
 class PreformattedString(NavigableString):
     """A NavigableString not subject to the normal formatting rules.
 
-    The string will be passed into the formatter (to trigger side effects),
-    but the return value will be ignored.
+    This is an abstract class used for special kinds of strings such
+    as comments (the Comment class) and CDATA blocks (the CData
+    class).
     """
 
-    def output_ready(self, formatter="minimal"):
-        """CData strings are passed into the formatter.
-        But the return value is ignored."""
-        self.format_string(self, formatter)
+    PREFIX = ''
+    SUFFIX = ''
+
+    def output_ready(self, formatter=None):
+        """Make this string ready for output by adding any subclass-specific
+            prefix or suffix.
+
+        :param formatter: A Formatter object, or a string naming one
+            of the standard formatters. The string will be passed into the
+            Formatter, but only to trigger any side effects: the return
+            value is ignored.
+
+        :return: The string, with any subclass-specific prefix and
+           suffix added on.
+        """
+        if formatter is not None:
+            ignore = self.format_string(self, formatter)
         return self.PREFIX + self + self.SUFFIX
 
 class CData(PreformattedString):
-
+    """A CDATA block."""
     PREFIX = '<![CDATA['
     SUFFIX = ']]>'
 
 class ProcessingInstruction(PreformattedString):
+    """A SGML processing instruction."""
 
     PREFIX = '<?'
     SUFFIX = '>'
 
-class Comment(PreformattedString):
+class XMLProcessingInstruction(ProcessingInstruction):
+    """An XML processing instruction."""
+    PREFIX = '<?'
+    SUFFIX = '?>'
 
+class Comment(PreformattedString):
+    """An HTML or XML comment."""
     PREFIX = '<!--'
     SUFFIX = '-->'
 
 
 class Declaration(PreformattedString):
+    """An XML declaration."""
     PREFIX = '<?'
     SUFFIX = '?>'
 
 
 class Doctype(PreformattedString):
-
+    """A document type declaration."""
     @classmethod
     def for_name_and_ids(cls, name, pub_id, system_id):
+        """Generate an appropriate document type declaration for a given
+        public ID and system ID.
+
+        :param name: The name of the document's root element, e.g. 'html'.
+        :param pub_id: The Formal Public Identifier for this document type,
+            e.g. '-//W3C//DTD XHTML 1.1//EN'
+        :param system_id: The system identifier for this document type,
+            e.g. 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'
+
+        :return: A Doctype.
+        """
         value = name or ''
         if pub_id is not None:
             value += ' PUBLIC "%s"' % pub_id
@@ -775,14 +1140,105 @@ class Doctype(PreformattedString):
     SUFFIX = '>\n'
 
 
+class Stylesheet(NavigableString):
+    """A NavigableString representing an stylesheet (probably
+    CSS).
+
+    Used to distinguish embedded stylesheets from textual content.
+    """
+    pass
+
+
+class Script(NavigableString):
+    """A NavigableString representing an executable script (probably
+    Javascript).
+
+    Used to distinguish executable code from textual content.
+    """
+    pass
+
+
+class TemplateString(NavigableString):
+    """A NavigableString representing a string found inside an HTML
+    template embedded in a larger document.
+
+    Used to distinguish such strings from the main body of the document.
+    """
+    pass
+
+
+class RubyTextString(NavigableString):
+    """A NavigableString representing the contents of the <rt> HTML
+    element.
+
+    https://dev.w3.org/html5/spec-LC/text-level-semantics.html#the-rt-element
+
+    Can be used to distinguish such strings from the strings they're
+    annotating.
+    """
+    pass
+
+
+class RubyParenthesisString(NavigableString):
+    """A NavigableString representing the contents of the <rp> HTML
+    element.
+
+    https://dev.w3.org/html5/spec-LC/text-level-semantics.html#the-rp-element
+    """
+    pass
+
+
 class Tag(PageElement):
+    """Represents an HTML or XML tag that is part of a parse tree, along
+    with its attributes and contents.
 
-    """Represents a found HTML tag with its attributes and contents."""
+    When Beautiful Soup parses the markup <b>penguin</b>, it will
+    create a Tag object representing the <b> tag.
+    """
 
     def __init__(self, parser=None, builder=None, name=None, namespace=None,
-                 prefix=None, attrs=None, parent=None, previous=None):
-        "Basic constructor."
-
+                 prefix=None, attrs=None, parent=None, previous=None,
+                 is_xml=None, sourceline=None, sourcepos=None,
+                 can_be_empty_element=None, cdata_list_attributes=None,
+                 preserve_whitespace_tags=None,
+                 interesting_string_types=None,
+                 namespaces=None
+    ):
+        """Basic constructor.
+
+        :param parser: A BeautifulSoup object.
+        :param builder: A TreeBuilder.
+        :param name: The name of the tag.
+        :param namespace: The URI of this Tag's XML namespace, if any.
+        :param prefix: The prefix for this Tag's XML namespace, if any.
+        :param attrs: A dictionary of this Tag's attribute values.
+        :param parent: The PageElement to use as this Tag's parent.
+        :param previous: The PageElement that was parsed immediately before
+            this tag.
+        :param is_xml: If True, this is an XML tag. Otherwise, this is an
+            HTML tag.
+        :param sourceline: The line number where this tag was found in its
+            source document.
+        :param sourcepos: The character position within `sourceline` where this
+            tag was found.
+        :param can_be_empty_element: If True, this tag should be
+            represented as <tag/>. If False, this tag should be represented
+            as <tag></tag>.
+        :param cdata_list_attributes: A list of attributes whose values should
+            be treated as CDATA if they ever show up on this tag.
+        :param preserve_whitespace_tags: A list of tag names whose contents
+            should have their whitespace preserved.
+        :param interesting_string_types: This is a NavigableString
+            subclass or a tuple of them. When iterating over this
+            Tag's strings in methods like Tag.strings or Tag.get_text,
+            these are the types of strings that are interesting enough
+            to be considered. The default is to consider
+            NavigableString and CData the only interesting string
+            subtypes.
+        :param namespaces: A dictionary mapping currently active
+            namespace prefixes to URIs. This can be used later to
+            construct CSS selectors.
+        """
         if parser is None:
             self.parser_class = None
         else:
@@ -793,7 +1249,12 @@ class Tag(PageElement):
             raise ValueError("No value provided for new tag's name.")
         self.name = name
         self.namespace = namespace
+        self._namespaces = namespaces or {}
         self.prefix = prefix
+        if ((not builder or builder.store_line_numbers)
+            and (sourceline is not None or sourcepos is not None)):
+            self.sourceline = sourceline
+            self.sourcepos = sourcepos
         if attrs is None:
             attrs = {}
         elif attrs:
@@ -804,32 +1265,109 @@ class Tag(PageElement):
                 attrs = dict(attrs)
         else:
             attrs = dict(attrs)
+
+        # If possible, determine ahead of time whether this tag is an
+        # XML tag.
+        if builder:
+            self.known_xml = builder.is_xml
+        else:
+            self.known_xml = is_xml
         self.attrs = attrs
         self.contents = []
         self.setup(parent, previous)
         self.hidden = False
 
-        # Set up any substitutions, such as the charset in a META tag.
-        if builder is not None:
+        if builder is None:
+            # In the absence of a TreeBuilder, use whatever values were
+            # passed in here. They're probably None, unless this is a copy of some
+            # other tag.
+            self.can_be_empty_element = can_be_empty_element
+            self.cdata_list_attributes = cdata_list_attributes
+            self.preserve_whitespace_tags = preserve_whitespace_tags
+            self.interesting_string_types = interesting_string_types
+        else:
+            # Set up any substitutions for this tag, such as the charset in a META tag.
             builder.set_up_substitutions(self)
+
+            # Ask the TreeBuilder whether this tag might be an empty-element tag.
             self.can_be_empty_element = builder.can_be_empty_element(name)
-        else:
-            self.can_be_empty_element = False
+
+            # Keep track of the list of attributes of this tag that
+            # might need to be treated as a list.
+            #
+            # For performance reasons, we store the whole data structure
+            # rather than asking the question of every tag. Asking would
+            # require building a new data structure every time, and
+            # (unlike can_be_empty_element), we almost never need
+            # to check this.
+            self.cdata_list_attributes = builder.cdata_list_attributes
+
+            # Keep track of the names that might cause this tag to be treated as a
+            # whitespace-preserved tag.
+            self.preserve_whitespace_tags = builder.preserve_whitespace_tags
+
+            if self.name in builder.string_containers:
+                # This sort of tag uses a special string container
+                # subclass for most of its strings. When we ask the
+                self.interesting_string_types = builder.string_containers[self.name]
+            else:
+                self.interesting_string_types = self.DEFAULT_INTERESTING_STRING_TYPES
 
     parserClass = _alias("parser_class")  # BS3
 
-    def __copy__(self):
-        """A copy of a Tag is a new Tag, unconnected to the parse tree.
+    def __deepcopy__(self, memo, recursive=True):
+        """A deepcopy of a Tag is a new Tag, unconnected to the parse tree.
         Its contents are a copy of the old Tag's contents.
         """
-        clone = type(self)(None, self.builder, self.name, self.namespace,
-                           self.nsprefix, self.attrs)
+        clone = self._clone()
+
+        if recursive:
+            # Clone this tag's descendants recursively, but without
+            # making any recursive function calls.
+            tag_stack = [clone]
+            for event, element in self._event_stream(self.descendants):
+                if event is Tag.END_ELEMENT_EVENT:
+                    # Stop appending incoming Tags to the Tag that was
+                    # just closed.
+                    tag_stack.pop()
+                else:
+                    descendant_clone = element.__deepcopy__(
+                        memo, recursive=False
+                    )
+                    # Add to its parent's .contents
+                    tag_stack[-1].append(descendant_clone)
+
+                    if event is Tag.START_ELEMENT_EVENT:
+                        # Add the Tag itself to the stack so that its
+                        # children will be .appended to it.
+                        tag_stack.append(descendant_clone)
+        return clone
+
+    def __copy__(self):
+        """A copy of a Tag must always be a deep copy, because a Tag's
+        children can only have one parent at a time.
+        """
+        return self.__deepcopy__({})
+
+    def _clone(self):
+        """Create a new Tag just like this one, but with no
+        contents and unattached to any parse tree.
+
+        This is the first step in the deepcopy process.
+        """
+        clone = type(self)(
+            None, None, self.name, self.namespace,
+            self.prefix, self.attrs, is_xml=self._is_xml,
+            sourceline=self.sourceline, sourcepos=self.sourcepos,
+            can_be_empty_element=self.can_be_empty_element,
+            cdata_list_attributes=self.cdata_list_attributes,
+            preserve_whitespace_tags=self.preserve_whitespace_tags,
+            interesting_string_types=self.interesting_string_types
+        )
         for attr in ('can_be_empty_element', 'hidden'):
             setattr(clone, attr, getattr(self, attr))
-        for child in self.contents:
-            clone.append(child.__copy__())
         return clone
-
+    
     @property
     def is_empty_element(self):
         """Is this tag an empty-element tag? (aka a self-closing tag)
@@ -850,13 +1388,17 @@ class Tag(PageElement):
 
     @property
     def string(self):
-        """Convenience property to get the single string within this tag.
+        """Convenience property to get the single string within this
+        PageElement.
 
-        :Return: If this tag has a single string child, return value
-         is that string. If this tag has no children, or more than one
-         child, return value is None. If this tag has one child tag,
+        TODO It might make sense to have NavigableString.string return
+        itself.
+
+        :return: If this element has a single string child, return
+         value is that string. If this element has one child tag,
          return value is the 'string' attribute of the child tag,
-         recursively.
+         recursively. If this element is itself a string, has no
+         children, or has more than one child, return value is None.
         """
         if len(self.contents) != 1:
             return None
@@ -867,57 +1409,75 @@ class Tag(PageElement):
 
     @string.setter
     def string(self, string):
+        """Replace this PageElement's contents with `string`."""
         self.clear()
         self.append(string.__class__(string))
 
-    def _all_strings(self, strip=False, types=(NavigableString, CData)):
+    DEFAULT_INTERESTING_STRING_TYPES = (NavigableString, CData)
+    def _all_strings(self, strip=False, types=PageElement.default):
         """Yield all strings of certain classes, possibly stripping them.
 
-        By default, yields only NavigableString and CData objects. So
-        no comments, processing instructions, etc.
+        :param strip: If True, all strings will be stripped before being
+            yielded.
+
+        :param types: A tuple of NavigableString subclasses. Any strings of
+            a subclass not found in this list will be ignored. By
+            default, the subclasses considered are the ones found in
+            self.interesting_string_types. If that's not specified,
+            only NavigableString and CData objects will be
+            considered. That means no comments, processing
+            instructions, etc.
+
+        :yield: A sequence of strings.
+
         """
+        if types is self.default:
+            types = self.interesting_string_types
+
         for descendant in self.descendants:
-            if (
-                (types is None and not isinstance(descendant, NavigableString))
-                or
-                (types is not None and type(descendant) not in types)):
+            if (types is None and not isinstance(descendant, NavigableString)):
+                continue
+            descendant_type = type(descendant)
+            if isinstance(types, type):
+                if descendant_type is not types:
+                    # We're not interested in strings of this type.
+                    continue
+            elif types is not None and descendant_type not in types:
+                # We're not interested in strings of this type.
                 continue
             if strip:
                 descendant = descendant.strip()
                 if len(descendant) == 0:
                     continue
             yield descendant
-
     strings = property(_all_strings)
 
-    @property
-    def stripped_strings(self):
-        for string in self._all_strings(True):
-            yield string
+    def decompose(self):
+        """Recursively destroys this PageElement and its children.
 
-    def get_text(self, separator="", strip=False,
-                 types=(NavigableString, CData)):
-        """
-        Get all child strings, concatenated using the given separator.
-        """
-        return separator.join([s for s in self._all_strings(
-                    strip, types=types)])
-    getText = get_text
-    text = property(get_text)
+        This element will be removed from the tree and wiped out; so
+        will everything beneath it.
 
-    def decompose(self):
-        """Recursively destroys the contents of this tree."""
+        The behavior of a decomposed PageElement is undefined and you
+        should never use one for anything, but if you need to _check_
+        whether an element has been decomposed, you can use the
+        `decomposed` property.
+        """
         self.extract()
         i = self
         while i is not None:
-            next = i.next_element
+            n = i.next_element
             i.__dict__.clear()
             i.contents = []
-            i = next
+            i._decomposed = True
+            i = n
 
     def clear(self, decompose=False):
-        """
-        Extract all children. If decompose is True, decompose instead.
+        """Wipe out all children of this PageElement by calling extract()
+           on them.
+
+        :param decompose: If this is True, decompose() (a more
+            destructive method) will be called instead of extract().
         """
         if decompose:
             for element in self.contents[:]:
@@ -929,10 +1489,51 @@ class Tag(PageElement):
             for element in self.contents[:]:
                 element.extract()
 
-    def index(self, element):
+    def smooth(self):
+        """Smooth out this element's children by consolidating consecutive
+        strings.
+
+        This makes pretty-printed output look more natural following a
+        lot of operations that modified the tree.
         """
-        Find the index of a child by identity, not value. Avoids issues with
-        tag.contents.index(element) getting the index of equal elements.
+        # Mark the first position of every pair of children that need
+        # to be consolidated.  Do this rather than making a copy of
+        # self.contents, since in most cases very few strings will be
+        # affected.
+        marked = []
+        for i, a in enumerate(self.contents):
+            if isinstance(a, Tag):
+                # Recursively smooth children.
+                a.smooth()
+            if i == len(self.contents)-1:
+                # This is the last item in .contents, and it's not a
+                # tag. There's no chance it needs any work.
+                continue
+            b = self.contents[i+1]
+            if (isinstance(a, NavigableString)
+                and isinstance(b, NavigableString)
+                and not isinstance(a, PreformattedString)
+                and not isinstance(b, PreformattedString)
+            ):
+                marked.append(i)
+
+        # Go over the marked positions in reverse order, so that
+        # removing items from .contents won't affect the remaining
+        # positions.
+        for i in reversed(marked):
+            a = self.contents[i]
+            b = self.contents[i+1]
+            b.extract()
+            n = NavigableString(a+b)
+            a.replace_with(n)
+
+    def index(self, element):
+        """Find the index of a child by identity, not value.
+
+        Avoids issues with tag.contents.index(element) getting the
+        index of equal elements.
+
+        :param element: Look for this PageElement in `self.contents`.
         """
         for i, child in enumerate(self.contents):
             if child is element:
@@ -945,23 +1546,38 @@ class Tag(PageElement):
         attribute."""
         return self.attrs.get(key, default)
 
+    def get_attribute_list(self, key, default=None):
+        """The same as get(), but always returns a list.
+
+        :param key: The attribute to look for.
+        :param default: Use this value if the attribute is not present
+            on this PageElement.
+        :return: A list of values, probably containing only a single
+            value.
+        """
+        value = self.get(key, default)
+        if not isinstance(value, list):
+            value = [value]
+        return value
+
     def has_attr(self, key):
+        """Does this PageElement have an attribute with the given name?"""
         return key in self.attrs
 
     def __hash__(self):
         return str(self).__hash__()
 
     def __getitem__(self, key):
-        """tag[key] returns the value of the 'key' attribute for the tag,
+        """tag[key] returns the value of the 'key' attribute for the Tag,
         and throws an exception if it's not there."""
         return self.attrs[key]
 
     def __iter__(self):
-        "Iterating over a tag iterates over its contents."
+        "Iterating over a Tag iterates over its contents."
         return iter(self.contents)
 
     def __len__(self):
-        "The length of a tag is the length of its list of contents."
+        "The length of a Tag is the length of its list of contents."
         return len(self.contents)
 
     def __contains__(self, x):
@@ -981,29 +1597,33 @@ class Tag(PageElement):
         self.attrs.pop(key, None)
 
     def __call__(self, *args, **kwargs):
-        """Calling a tag like a function is the same as calling its
+        """Calling a Tag like a function is the same as calling its
         find_all() method. Eg. tag('a') returns a list of all the A tags
         found within this tag."""
         return self.find_all(*args, **kwargs)
 
     def __getattr__(self, tag):
-        #print "Getattr %s.%s" % (self.__class__, tag)
+        """Calling tag.subtag is the same as calling tag.find(name="subtag")"""
+        #print("Getattr %s.%s" % (self.__class__, tag))
         if len(tag) > 3 and tag.endswith('Tag'):
             # BS3: soup.aTag -> "soup.find("a")
             tag_name = tag[:-3]
             warnings.warn(
-                '.%sTag is deprecated, use .find("%s") instead.' % (
-                    tag_name, tag_name))
+                '.%(name)sTag is deprecated, use .find("%(name)s") instead. If you really were looking for a tag called %(name)sTag, use .find("%(name)sTag")' % dict(
+                    name=tag_name
+                ),
+                DeprecationWarning, stacklevel=2
+            )
             return self.find(tag_name)
         # We special case contents to avoid recursion.
-        elif not tag.startswith("__") and not tag=="contents":
+        elif not tag.startswith("__") and not tag == "contents":
             return self.find(tag)
         raise AttributeError(
             "'%s' object has no attribute '%s'" % (self.__class__, tag))
 
     def __eq__(self, other):
-        """Returns true iff this tag has the same name, the same attributes,
-        and the same contents (recursively) as the given tag."""
+        """Returns true iff this Tag has the same name, the same attributes,
+        and the same contents (recursively) as `other`."""
         if self is other:
             return True
         if (not hasattr(other, 'name') or
@@ -1019,69 +1639,235 @@ class Tag(PageElement):
         return True
 
     def __ne__(self, other):
-        """Returns true iff this tag is not identical to the other tag,
+        """Returns true iff this Tag is not identical to `other`,
         as defined in __eq__."""
         return not self == other
 
     def __repr__(self, encoding="unicode-escape"):
-        """Renders this tag as a string."""
-        if PY3K:
-            # "The return value must be a string object", i.e. Unicode
-            return self.decode()
-        else:
-            # "The return value must be a string object", i.e. a bytestring.
-            # By convention, the return value of __repr__ should also be
-            # an ASCII string.
-            return self.encode(encoding)
+        """Renders this PageElement as a string.
 
-    def __unicode__(self):
+        :param encoding: The encoding to use (Python 2 only).
+            TODO: This is now ignored and a warning should be issued
+            if a value is provided.
+        :return: A (Unicode) string.
+        """
+        # "The return value must be a string object", i.e. Unicode
         return self.decode()
 
-    def __str__(self):
-        if PY3K:
-            return self.decode()
-        else:
-            return self.encode()
+    def __unicode__(self):
+        """Renders this PageElement as a Unicode string."""
+        return self.decode()
 
-    if PY3K:
-        __str__ = __repr__ = __unicode__
+    __str__ = __repr__ = __unicode__
 
     def encode(self, encoding=DEFAULT_OUTPUT_ENCODING,
                indent_level=None, formatter="minimal",
                errors="xmlcharrefreplace"):
+        """Render a bytestring representation of this PageElement and its
+        contents.
+
+        :param encoding: The destination encoding.
+        :param indent_level: Each line of the rendering will be
+           indented this many levels. (The formatter decides what a
+           'level' means in terms of spaces or other characters
+           output.) Used internally in recursive calls while
+           pretty-printing.
+        :param formatter: A Formatter object, or a string naming one of
+            the standard formatters.
+        :param errors: An error handling strategy such as
+            'xmlcharrefreplace'. This value is passed along into
+            encode() and its value should be one of the constants
+            defined by Python.
+        :return: A bytestring.
+
+        """
         # Turn the data structure into Unicode, then encode the
         # Unicode.
         u = self.decode(indent_level, encoding, formatter)
         return u.encode(encoding, errors)
 
-    def _should_pretty_print(self, indent_level):
-        """Should this tag be pretty-printed?"""
-        return (
-            indent_level is not None and
-            (self.name not in HTMLAwareEntitySubstitution.preformatted_tags
-             or self._is_xml))
-
     def decode(self, indent_level=None,
                eventual_encoding=DEFAULT_OUTPUT_ENCODING,
-               formatter="minimal"):
-        """Returns a Unicode representation of this tag and its contents.
+               formatter="minimal",
+               iterator=None):
+        pieces = []
+        # First off, turn a non-Formatter `formatter` into a Formatter
+        # object. This will stop the lookup from happening over and
+        # over again.
+        if not isinstance(formatter, Formatter):
+            formatter = self.formatter_for_name(formatter)
+
+        if indent_level is True:
+            indent_level = 0
+
+        # The currently active tag that put us into string literal
+        # mode. Until this element is closed, children will be treated
+        # as string literals and not pretty-printed. String literal
+        # mode is turned on immediately after this tag begins, and
+        # turned off immediately before it's closed. This means there
+        # will be whitespace before and after the tag itself.
+        string_literal_tag = None
+
+        for event, element in self._event_stream(iterator):
+            if event in (Tag.START_ELEMENT_EVENT, Tag.EMPTY_ELEMENT_EVENT):
+                piece = element._format_tag(
+                    eventual_encoding, formatter, opening=True
+                )
+            elif event is Tag.END_ELEMENT_EVENT:
+                piece = element._format_tag(
+                    eventual_encoding, formatter, opening=False
+                )
+                if indent_level is not None:
+                    indent_level -= 1
+            else:
+                piece = element.output_ready(formatter)
+
+            # Now we need to apply the 'prettiness' -- extra
+            # whitespace before and/or after this tag. This can get
+            # complicated because certain tags, like <pre> and
+            # <script>, can't be prettified, since adding whitespace would
+            # change the meaning of the content.
+
+            # The default behavior is to add whitespace before and
+            # after an element when string literal mode is off, and to
+            # leave things as they are when string literal mode is on.
+            if string_literal_tag:
+                indent_before = indent_after = False
+            else:
+                indent_before = indent_after = True
+
+            # The only time the behavior is more complex than that is
+            # when we encounter an opening or closing tag that might
+            # put us into or out of string literal mode.
+            if (event is Tag.START_ELEMENT_EVENT
+                and not string_literal_tag
+                and not element._should_pretty_print()):
+                    # We are about to enter string literal mode. Add
+                    # whitespace before this tag, but not after. We
+                    # will stay in string literal mode until this tag
+                    # is closed.
+                    indent_before = True
+                    indent_after = False
+                    string_literal_tag = element
+            elif (event is Tag.END_ELEMENT_EVENT
+                  and element is string_literal_tag):
+                # We are about to exit string literal mode by closing
+                # the tag that sent us into that mode. Add whitespace
+                # after this tag, but not before.
+                indent_before = False
+                indent_after = True
+                string_literal_tag = None
+
+            # Now we know whether to add whitespace before and/or
+            # after this element.
+            if indent_level is not None:
+                if (indent_before or indent_after):
+                    if isinstance(element, NavigableString):
+                        piece = piece.strip()
+                    if piece:
+                        piece = self._indent_string(
+                            piece, indent_level, formatter,
+                            indent_before, indent_after
+                        )
+                if event == Tag.START_ELEMENT_EVENT:
+                    indent_level += 1
+            pieces.append(piece)
+        return "".join(pieces)
+
+    # Names for the different events yielded by _event_stream
+    START_ELEMENT_EVENT = object()
+    END_ELEMENT_EVENT = object()
+    EMPTY_ELEMENT_EVENT = object()
+    STRING_ELEMENT_EVENT = object()
+
+    def _event_stream(self, iterator=None):
+        """Yield a sequence of events that can be used to reconstruct the DOM
+        for this element.
+
+        This lets us recreate the nested structure of this element
+        (e.g. when formatting it as a string) without using recursive
+        method calls.
+
+        This is similar in concept to the SAX API, but it's a simpler
+        interface designed for internal use. The events are different
+        from SAX and the arguments associated with the events are Tags
+        and other Beautiful Soup objects.
+
+        :param iterator: An alternate iterator to use when traversing
+         the tree.
+        """
+        tag_stack = []
 
-        :param eventual_encoding: The tag is destined to be
-           encoded into this encoding. This method is _not_
-           responsible for performing that encoding. This information
-           is passed in so that it can be substituted in if the
-           document contains a <META> tag that mentions the document's
-           encoding.
+        iterator = iterator or self.self_and_descendants
+
+        for c in iterator:
+            # If the parent of the element we're about to yield is not
+            # the tag currently on the stack, it means that the tag on
+            # the stack closed before this element appeared.
+            while tag_stack and c.parent != tag_stack[-1]:
+                now_closed_tag = tag_stack.pop()
+                yield Tag.END_ELEMENT_EVENT, now_closed_tag
+
+            if isinstance(c, Tag):
+                if c.is_empty_element:
+                    yield Tag.EMPTY_ELEMENT_EVENT, c
+                else:
+                    yield Tag.START_ELEMENT_EVENT, c
+                    tag_stack.append(c)
+                    continue
+            else:
+                yield Tag.STRING_ELEMENT_EVENT, c
+
+        while tag_stack:
+            now_closed_tag = tag_stack.pop()
+            yield Tag.END_ELEMENT_EVENT, now_closed_tag
+
+    def _indent_string(self, s, indent_level, formatter,
+                       indent_before, indent_after):
+        """Add indentation whitespace before and/or after a string.
+
+        :param s: The string to amend with whitespace.
+        :param indent_level: The indentation level; affects how much
+           whitespace goes before the string.
+        :param indent_before: Whether or not to add whitespace
+           before the string.
+        :param indent_after: Whether or not to add whitespace
+           (a newline) after the string.
         """
+        space_before = ''
+        if indent_before and indent_level:
+            space_before = (formatter.indent * indent_level)
 
-        # First off, turn a string formatter into a function. This
-        # will stop the lookup from happening over and over again.
-        if not isinstance(formatter, collections.abc.Callable):
-            formatter = self._formatter_for_name(formatter)
+        space_after = ''
+        if indent_after:
+            space_after = "\n"
 
-        attrs = []
-        if self.attrs:
-            for key, val in sorted(self.attrs.items()):
+        return space_before + s + space_after
+
+    def _format_tag(self, eventual_encoding, formatter, opening):
+        if self.hidden:
+            # A hidden tag is invisible, although its contents
+            # are visible.
+            return ''
+
+        # A tag starts with the < character (see below).
+
+        # Then the / character, if this is a closing tag.
+        closing_slash = ''
+        if not opening:
+            closing_slash = '/'
+
+        # Then an optional namespace prefix.
+        prefix = ''
+        if self.prefix:
+            prefix = self.prefix + ":"
+
+        # Then a list of attribute values, if this is an opening tag.
+        attribute_string = ''
+        if opening:
+            attributes = formatter.attributes(self)
+            attrs = []
+            for key, val in attributes:
                 if val is None:
                     decoded = key
                 else:
@@ -1090,71 +1876,52 @@ class Tag(PageElement):
                     elif not isinstance(val, str):
                         val = str(val)
                     elif (
-                        isinstance(val, AttributeValueWithCharsetSubstitution)
-                        and eventual_encoding is not None):
+                            isinstance(val, AttributeValueWithCharsetSubstitution)
+                            and eventual_encoding is not None
+                    ):
                         val = val.encode(eventual_encoding)
 
-                    text = self.format_string(val, formatter)
+                    text = formatter.attribute_value(val)
                     decoded = (
                         str(key) + '='
-                        + EntitySubstitution.quoted_attribute_value(text))
+                        + formatter.quoted_attribute_value(text))
                 attrs.append(decoded)
-        close = ''
-        closeTag = ''
-
-        prefix = ''
-        if self.prefix:
-            prefix = self.prefix + ":"
+            if attrs:
+                attribute_string = ' ' + ' '.join(attrs)
 
+        # Then an optional closing slash (for a void element in an
+        # XML document).
+        void_element_closing_slash = ''
         if self.is_empty_element:
-            close = '/'
-        else:
-            closeTag = '</%s%s>' % (prefix, self.name)
-
-        pretty_print = self._should_pretty_print(indent_level)
-        space = ''
-        indent_space = ''
-        if indent_level is not None:
-            indent_space = (' ' * (indent_level - 1))
-        if pretty_print:
-            space = indent_space
-            indent_contents = indent_level + 1
-        else:
-            indent_contents = None
-        contents = self.decode_contents(
-            indent_contents, eventual_encoding, formatter)
+            void_element_closing_slash = formatter.void_element_close_prefix or ''
 
-        if self.hidden:
-            # This is the 'document root' object.
-            s = contents
-        else:
-            s = []
-            attribute_string = ''
-            if attrs:
-                attribute_string = ' ' + ' '.join(attrs)
-            if indent_level is not None:
-                # Even if this particular tag is not pretty-printed,
-                # we should indent up to the start of the tag.
-                s.append(indent_space)
-            s.append('<%s%s%s%s>' % (
-                    prefix, self.name, attribute_string, close))
-            if pretty_print:
-                s.append("\n")
-            s.append(contents)
-            if pretty_print and contents and contents[-1] != "\n":
-                s.append("\n")
-            if pretty_print and closeTag:
-                s.append(space)
-            s.append(closeTag)
-            if indent_level is not None and closeTag and self.next_sibling:
-                # Even if this particular tag is not pretty-printed,
-                # we're now done with the tag, and we should add a
-                # newline if appropriate.
-                s.append("\n")
-            s = ''.join(s)
-        return s
+        # Put it all together.
+        return '<' + closing_slash + prefix + self.name + attribute_string + void_element_closing_slash + '>'
+
+    def _should_pretty_print(self, indent_level=1):
+        """Should this tag be pretty-printed?
+
+        Most of them should, but some (such as <pre> in HTML
+        documents) should not.
+        """
+        return (
+            indent_level is not None
+            and (
+                not self.preserve_whitespace_tags
+                or self.name not in self.preserve_whitespace_tags
+            )
+        )
 
     def prettify(self, encoding=None, formatter="minimal"):
+        """Pretty-print this PageElement as a string.
+
+        :param encoding: The eventual encoding of the string. If this is None,
+            a Unicode string will be returned.
+        :param formatter: A Formatter object, or a string naming one of
+            the standard formatters.
+        :return: A Unicode string (if encoding==None) or a bytestring
+            (otherwise).
+        """
         if encoding is None:
             return self.decode(True, formatter=formatter)
         else:
@@ -1166,62 +1933,50 @@ class Tag(PageElement):
         """Renders the contents of this tag as a Unicode string.
 
         :param indent_level: Each line of the rendering will be
-           indented this many spaces.
+           indented this many levels. (The formatter decides what a
+           'level' means in terms of spaces or other characters
+           output.) Used internally in recursive calls while
+           pretty-printing.
 
         :param eventual_encoding: The tag is destined to be
-           encoded into this encoding. This method is _not_
+           encoded into this encoding. decode_contents() is _not_
            responsible for performing that encoding. This information
            is passed in so that it can be substituted in if the
            document contains a <META> tag that mentions the document's
            encoding.
 
-        :param formatter: The output formatter responsible for converting
-           entities to Unicode characters.
-        """
-        # First off, turn a string formatter into a function. This
-        # will stop the lookup from happening over and over again.
-        if not isinstance(formatter, collections.abc.Callable):
-            formatter = self._formatter_for_name(formatter)
-
-        pretty_print = (indent_level is not None)
-        s = []
-        for c in self:
-            text = None
-            if isinstance(c, NavigableString):
-                text = c.output_ready(formatter)
-            elif isinstance(c, Tag):
-                s.append(c.decode(indent_level, eventual_encoding,
-                                  formatter))
-            if text and indent_level and not self.name == 'pre':
-                text = text.strip()
-            if text:
-                if pretty_print and not self.name == 'pre':
-                    s.append(" " * (indent_level - 1))
-                s.append(text)
-                if pretty_print and not self.name == 'pre':
-                    s.append("\n")
-        return ''.join(s)
+        :param formatter: A Formatter object, or a string naming one of
+            the standard Formatters.
+
+        """
+        return self.decode(indent_level, eventual_encoding, formatter,
+                           iterator=self.descendants)
 
     def encode_contents(
         self, indent_level=None, encoding=DEFAULT_OUTPUT_ENCODING,
         formatter="minimal"):
-        """Renders the contents of this tag as a bytestring.
+        """Renders the contents of this PageElement as a bytestring.
 
         :param indent_level: Each line of the rendering will be
-           indented this many spaces.
+           indented this many levels. (The formatter decides what a
+           'level' means in terms of spaces or other characters
+           output.) Used internally in recursive calls while
+           pretty-printing.
 
         :param eventual_encoding: The bytestring will be in this encoding.
 
-        :param formatter: The output formatter responsible for converting
-           entities to Unicode characters.
-        """
+        :param formatter: A Formatter object, or a string naming one of
+            the standard Formatters.
 
+        :return: A bytestring.
+        """
         contents = self.decode_contents(indent_level, encoding, formatter)
         return contents.encode(encoding)
 
     # Old method for BS3 compatibility
     def renderContents(self, encoding=DEFAULT_OUTPUT_ENCODING,
                        prettyPrint=False, indentLevel=0):
+        """Deprecated method for BS3 compatibility."""
         if not prettyPrint:
             indentLevel = None
         return self.encode_contents(
@@ -1229,44 +1984,88 @@ class Tag(PageElement):
 
     #Soup methods
 
-    def find(self, name=None, attrs={}, recursive=True, text=None,
+    def find(self, name=None, attrs={}, recursive=True, string=None,
              **kwargs):
-        """Return only the first child of this Tag matching the given
-        criteria."""
+        """Look in the children of this PageElement and find the first
+        PageElement that matches the given criteria.
+
+        All find_* methods take a common set of arguments. See the online
+        documentation for detailed explanations.
+
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :param recursive: If this is True, find() will perform a
+            recursive search of this PageElement's children. Otherwise,
+            only the direct children will be considered.
+        :param limit: Stop looking after finding this many results.
+        :kwargs: A dictionary of filters on attribute values.
+        :return: A PageElement.
+        :rtype: bs4.element.Tag | bs4.element.NavigableString
+        """
         r = None
-        l = self.find_all(name, attrs, recursive, text, 1, **kwargs)
+        l = self.find_all(name, attrs, recursive, string, 1, _stacklevel=3,
+                          **kwargs)
         if l:
             r = l[0]
         return r
-    findChild = find
+    findChild = find #BS2
 
-    def find_all(self, name=None, attrs={}, recursive=True, text=None,
+    def find_all(self, name=None, attrs={}, recursive=True, string=None,
                  limit=None, **kwargs):
-        """Extracts a list of Tag objects that match the given
-        criteria.  You can specify the name of the Tag and any
-        attributes you want the Tag to have.
-
-        The value of a key-value pair in the 'attrs' map can be a
-        string, a list of strings, a regular expression object, or a
-        callable that takes a string and returns whether or not the
-        string matches for some custom definition of 'matches'. The
-        same is true of the tag name."""
-
+        """Look in the children of this PageElement and find all
+        PageElements that match the given criteria.
+
+        All find_* methods take a common set of arguments. See the online
+        documentation for detailed explanations.
+
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :param recursive: If this is True, find_all() will perform a
+            recursive search of this PageElement's children. Otherwise,
+            only the direct children will be considered.
+        :param limit: Stop looking after finding this many results.
+        :kwargs: A dictionary of filters on attribute values.
+        :return: A ResultSet of PageElements.
+        :rtype: bs4.element.ResultSet
+        """
         generator = self.descendants
         if not recursive:
             generator = self.children
-        return self._find_all(name, attrs, text, limit, generator, **kwargs)
+        _stacklevel = kwargs.pop('_stacklevel', 2)
+        return self._find_all(name, attrs, string, limit, generator,
+                              _stacklevel=_stacklevel+1, **kwargs)
     findAll = find_all       # BS3
     findChildren = find_all  # BS2
 
     #Generator methods
     @property
     def children(self):
+        """Iterate over all direct children of this PageElement.
+
+        :yield: A sequence of PageElements.
+        """
         # return iter() to make the purpose of the method clear
         return iter(self.contents)  # XXX This seems to be untested.
 
     @property
+    def self_and_descendants(self):
+        """Iterate over this PageElement and its children in a
+        breadth-first sequence.
+
+        :yield: A sequence of PageElements.
+        """
+        if not self.hidden:
+            yield self
+        for i in self.descendants:
+            yield i
+
+    @property
     def descendants(self):
+        """Iterate over all children of this PageElement in a
+        breadth-first sequence.
+
+        :yield: A sequence of PageElements.
+        """
         if not len(self.contents):
             return
         stopNode = self._last_descendant().next_element
@@ -1276,262 +2075,102 @@ class Tag(PageElement):
             current = current.next_element
 
     # CSS selector code
+    def select_one(self, selector, namespaces=None, **kwargs):
+        """Perform a CSS selection operation on the current element.
 
-    _selector_combinators = ['>', '+', '~']
-    _select_debug = False
-    def select_one(self, selector):
-        """Perform a CSS selection operation on the current element."""
-        value = self.select(selector, limit=1)
-        if value:
-            return value[0]
-        return None
+        :param selector: A CSS selector.
 
-    def select(self, selector, _candidate_generator=None, limit=None):
-        """Perform a CSS selection operation on the current element."""
-
-        # Handle grouping selectors if ',' exists, ie: p,a
-        if ',' in selector:
-            context = []
-            for partial_selector in selector.split(','):
-                partial_selector = partial_selector.strip()
-                if partial_selector == '':
-                    raise ValueError('Invalid group selection syntax: %s' % selector)
-                candidates = self.select(partial_selector, limit=limit)
-                for candidate in candidates:
-                    if candidate not in context:
-                        context.append(candidate)
-
-                if limit and len(context) >= limit:
-                    break
-            return context
+        :param namespaces: A dictionary mapping namespace prefixes
+           used in the CSS selector to namespace URIs. By default,
+           Beautiful Soup will use the prefixes it encountered while
+           parsing the document.
 
-        tokens = selector.split()
-        current_context = [self]
+        :param kwargs: Keyword arguments to be passed into Soup Sieve's
+           soupsieve.select() method.
 
-        if tokens[-1] in self._selector_combinators:
-            raise ValueError(
-                'Final combinator "%s" is missing an argument.' % tokens[-1])
+        :return: A Tag.
+        :rtype: bs4.element.Tag
+        """
+        return self.css.select_one(selector, namespaces, **kwargs)
 
-        if self._select_debug:
-            print('Running CSS selector "%s"' % selector)
+    def select(self, selector, namespaces=None, limit=None, **kwargs):
+        """Perform a CSS selection operation on the current element.
 
-        for index, token in enumerate(tokens):
-            new_context = []
-            new_context_ids = set([])
+        This uses the SoupSieve library.
 
-            if tokens[index-1] in self._selector_combinators:
-                # This token was consumed by the previous combinator. Skip it.
-                if self._select_debug:
-                    print('  Token was consumed by the previous combinator.')
-                continue
+        :param selector: A string containing a CSS selector.
 
-            if self._select_debug:
-                print(' Considering token "%s"' % token)
-            recursive_candidate_generator = None
-            tag_name = None
-
-            # Each operation corresponds to a checker function, a rule
-            # for determining whether a candidate matches the
-            # selector. Candidates are generated by the active
-            # iterator.
-            checker = None
-
-            m = self.attribselect_re.match(token)
-            if m is not None:
-                # Attribute selector
-                tag_name, attribute, operator, value = m.groups()
-                checker = self._attribute_checker(operator, attribute, value)
-
-            elif '#' in token:
-                # ID selector
-                tag_name, tag_id = token.split('#', 1)
-                def id_matches(tag):
-                    return tag.get('id', None) == tag_id
-                checker = id_matches
-
-            elif '.' in token:
-                # Class selector
-                tag_name, klass = token.split('.', 1)
-                classes = set(klass.split('.'))
-                def classes_match(candidate):
-                    return classes.issubset(candidate.get('class', []))
-                checker = classes_match
-
-            elif ':' in token:
-                # Pseudo-class
-                tag_name, pseudo = token.split(':', 1)
-                if tag_name == '':
-                    raise ValueError(
-                        "A pseudo-class must be prefixed with a tag name.")
-                pseudo_attributes = re.match(r'([a-zA-Z\d-]+)\(([a-zA-Z\d]+)\)', pseudo)
-                found = []
-                if pseudo_attributes is None:
-                    pseudo_type = pseudo
-                    pseudo_value = None
-                else:
-                    pseudo_type, pseudo_value = pseudo_attributes.groups()
-                if pseudo_type == 'nth-of-type':
-                    try:
-                        pseudo_value = int(pseudo_value)
-                    except:
-                        raise NotImplementedError(
-                            'Only numeric values are currently supported for the nth-of-type pseudo-class.')
-                    if pseudo_value < 1:
-                        raise ValueError(
-                            'nth-of-type pseudo-class value must be at least 1.')
-                    class Counter(object):
-                        def __init__(self, destination):
-                            self.count = 0
-                            self.destination = destination
-
-                        def nth_child_of_type(self, tag):
-                            self.count += 1
-                            if self.count == self.destination:
-                                return True
-                            if self.count > self.destination:
-                                # Stop the generator that's sending us
-                                # these things.
-                                raise StopIteration()
-                            return False
-                    checker = Counter(pseudo_value).nth_child_of_type
-                else:
-                    raise NotImplementedError(
-                        'Only the following pseudo-classes are implemented: nth-of-type.')
-
-            elif token == '*':
-                # Star selector -- matches everything
-                pass
-            elif token == '>':
-                # Run the next token as a CSS selector against the
-                # direct children of each tag in the current context.
-                recursive_candidate_generator = lambda tag: tag.children
-            elif token == '~':
-                # Run the next token as a CSS selector against the
-                # siblings of each tag in the current context.
-                recursive_candidate_generator = lambda tag: tag.next_siblings
-            elif token == '+':
-                # For each tag in the current context, run the next
-                # token as a CSS selector against the tag's next
-                # sibling that's a tag.
-                def next_tag_sibling(tag):
-                    yield tag.find_next_sibling(True)
-                recursive_candidate_generator = next_tag_sibling
-
-            elif self.tag_name_re.match(token):
-                # Just a tag name.
-                tag_name = token
-            else:
-                raise ValueError(
-                    'Unsupported or invalid CSS selector: "%s"' % token)
-            if recursive_candidate_generator:
-                # This happens when the selector looks like  "> foo".
-                #
-                # The generator calls select() recursively on every
-                # member of the current context, passing in a different
-                # candidate generator and a different selector.
-                #
-                # In the case of "> foo", the candidate generator is
-                # one that yields a tag's direct children (">"), and
-                # the selector is "foo".
-                next_token = tokens[index+1]
-                def recursive_select(tag):
-                    if self._select_debug:
-                        print('    Calling select("%s") recursively on %s %s' % (next_token, tag.name, tag.attrs))
-                        print('-' * 40)
-                    for i in tag.select(next_token, recursive_candidate_generator):
-                        if self._select_debug:
-                            print('(Recursive select picked up candidate %s %s)' % (i.name, i.attrs))
-                        yield i
-                    if self._select_debug:
-                        print('-' * 40)
-                _use_candidate_generator = recursive_select
-            elif _candidate_generator is None:
-                # By default, a tag's candidates are all of its
-                # children. If tag_name is defined, only yield tags
-                # with that name.
-                if self._select_debug:
-                    if tag_name:
-                        check = "[any]"
-                    else:
-                        check = tag_name
-                    print('   Default candidate generator, tag name="%s"' % check)
-                if self._select_debug:
-                    # This is redundant with later code, but it stops
-                    # a bunch of bogus tags from cluttering up the
-                    # debug log.
-                    def default_candidate_generator(tag):
-                        for child in tag.descendants:
-                            if not isinstance(child, Tag):
-                                continue
-                            if tag_name and not child.name == tag_name:
-                                continue
-                            yield child
-                    _use_candidate_generator = default_candidate_generator
-                else:
-                    _use_candidate_generator = lambda tag: tag.descendants
-            else:
-                _use_candidate_generator = _candidate_generator
-
-            count = 0
-            for tag in current_context:
-                if self._select_debug:
-                    print("    Running candidate generator on %s %s" % (
-                        tag.name, repr(tag.attrs)))
-                for candidate in _use_candidate_generator(tag):
-                    if not isinstance(candidate, Tag):
-                        continue
-                    if tag_name and candidate.name != tag_name:
-                        continue
-                    if checker is not None:
-                        try:
-                            result = checker(candidate)
-                        except StopIteration:
-                            # The checker has decided we should no longer
-                            # run the generator.
-                            break
-                    if checker is None or result:
-                        if self._select_debug:
-                            print("     SUCCESS %s %s" % (candidate.name, repr(candidate.attrs)))
-                        if id(candidate) not in new_context_ids:
-                            # If a tag matches a selector more than once,
-                            # don't include it in the context more than once.
-                            new_context.append(candidate)
-                            new_context_ids.add(id(candidate))
-                            if limit and len(new_context) >= limit:
-                                break
-                    elif self._select_debug:
-                        print("     FAILURE %s %s" % (candidate.name, repr(candidate.attrs)))
-
-
-            current_context = new_context
-
-        if self._select_debug:
-            print("Final verdict:")
-            for i in current_context:
-                print(" %s %s" % (i.name, i.attrs))
-        return current_context
+        :param namespaces: A dictionary mapping namespace prefixes
+           used in the CSS selector to namespace URIs. By default,
+           Beautiful Soup will use the prefixes it encountered while
+           parsing the document.
+
+        :param limit: After finding this number of results, stop looking.
+
+        :param kwargs: Keyword arguments to be passed into SoupSieve's
+           soupsieve.select() method.
+
+        :return: A ResultSet of Tags.
+        :rtype: bs4.element.ResultSet
+        """
+        return self.css.select(selector, namespaces, limit, **kwargs)
+
+    @property
+    def css(self):
+        """Return an interface to the CSS selector API."""
+        return CSS(self)
 
     # Old names for backwards compatibility
     def childGenerator(self):
+        """Deprecated generator."""
         return self.children
 
     def recursiveChildGenerator(self):
+        """Deprecated generator."""
         return self.descendants
 
     def has_key(self, key):
-        """This was kind of misleading because has_key() (attributes)
-        was different from __in__ (contents). has_key() is gone in
-        Python 3, anyway."""
-        warnings.warn('has_key is deprecated. Use has_attr("%s") instead.' % (
-                key))
+        """Deprecated method. This was kind of misleading because has_key()
+        (attributes) was different from __in__ (contents).
+
+        has_key() is gone in Python 3, anyway.
+        """
+        warnings.warn(
+            'has_key is deprecated. Use has_attr(key) instead.',
+            DeprecationWarning, stacklevel=2
+        )
         return self.has_attr(key)
 
 # Next, a couple classes to represent queries and their results.
 class SoupStrainer(object):
     """Encapsulates a number of ways of matching a markup element (tag or
-    text)."""
+    string).
+
+    This is primarily used to underpin the find_* methods, but you can
+    create one yourself and pass it in as `parse_only` to the
+    `BeautifulSoup` constructor, to parse a subset of a large
+    document.
+    """
+
+    def __init__(self, name=None, attrs={}, string=None, **kwargs):
+        """Constructor.
+
+        The SoupStrainer constructor takes the same arguments passed
+        into the find_* methods. See the online documentation for
+        detailed explanations.
+
+        :param name: A filter on tag name.
+        :param attrs: A dictionary of filters on attribute values.
+        :param string: A filter for a NavigableString with specific text.
+        :kwargs: A dictionary of filters on attribute values.
+        """
+        if string is None and 'text' in kwargs:
+            string = kwargs.pop('text')
+            warnings.warn(
+                "The 'text' argument to the SoupStrainer constructor is deprecated. Use 'string' instead.",
+                DeprecationWarning, stacklevel=2
+            )
 
-    def __init__(self, name=None, attrs={}, text=None, **kwargs):
         self.name = self._normalize_search_value(name)
         if not isinstance(attrs, dict):
             # Treat a non-dict value for attrs as a search for the 'class'
@@ -1556,12 +2195,15 @@ class SoupStrainer(object):
             normalized_attrs[key] = self._normalize_search_value(value)
 
         self.attrs = normalized_attrs
-        self.text = self._normalize_search_value(text)
+        self.string = self._normalize_search_value(string)
+
+        # DEPRECATED but just in case someone is checking this.
+        self.text = self.string
 
     def _normalize_search_value(self, value):
         # Leave it alone if it's a Unicode string, a callable, a
         # regular expression, a boolean, or None.
-        if (isinstance(value, str) or isinstance(value, collections.abc.Callable) or hasattr(value, 'match')
+        if (isinstance(value, str) or isinstance(value, Callable) or hasattr(value, 'match')
             or isinstance(value, bool) or value is None):
             return value
 
@@ -1589,19 +2231,40 @@ class SoupStrainer(object):
         return str(str(value))
 
     def __str__(self):
-        if self.text:
-            return self.text
+        """A human-readable representation of this SoupStrainer."""
+        if self.string:
+            return self.string
         else:
             return "%s|%s" % (self.name, self.attrs)
 
     def search_tag(self, markup_name=None, markup_attrs={}):
+        """Check whether a Tag with the given name and attributes would
+        match this SoupStrainer.
+
+        Used prospectively to decide whether to even bother creating a Tag
+        object.
+
+        :param markup_name: A tag name as found in some markup.
+        :param markup_attrs: A dictionary of attributes as found in some markup.
+
+        :return: True if the prospective tag would match this SoupStrainer;
+            False otherwise.
+        """
         found = None
         markup = None
         if isinstance(markup_name, Tag):
             markup = markup_name
             markup_attrs = markup
+
+        if isinstance(self.name, str):
+            # Optimization for a very common case where the user is
+            # searching for a tag with one specific name, and we're
+            # looking at a tag with a different name.
+            if markup and not markup.prefix and self.name != markup.name:
+                 return False
+
         call_function_with_tag_data = (
-            isinstance(self.name, collections.abc.Callable)
+            isinstance(self.name, Callable)
             and not isinstance(markup_name, Tag))
 
         if ((not self.name)
@@ -1630,13 +2293,22 @@ class SoupStrainer(object):
                     found = markup
                 else:
                     found = markup_name
-        if found and self.text and not self._matches(found.string, self.text):
+        if found and self.string and not self._matches(found.string, self.string):
             found = None
         return found
+
+    # For BS3 compatibility.
     searchTag = search_tag
 
     def search(self, markup):
-        # print 'looking for %s in %s' % (self, markup)
+        """Find all items in `markup` that match this SoupStrainer.
+
+        Used by the core _find_all() method, which is ultimately
+        called by all find_* methods.
+
+        :param markup: A PageElement or a list of them.
+        """
+        # print('looking for %s in %s' % (self, markup))
         found = None
         # If given a list of items, scan it for a text element that
         # matches.
@@ -1649,49 +2321,44 @@ class SoupStrainer(object):
         # If it's a Tag, make sure its name or attributes match.
         # Don't bother with Tags if we're searching for text.
         elif isinstance(markup, Tag):
-            if not self.text or self.name or self.attrs:
+            if not self.string or self.name or self.attrs:
                 found = self.search_tag(markup)
         # If it's text, make sure the text matches.
         elif isinstance(markup, NavigableString) or \
                  isinstance(markup, str):
-            if not self.name and not self.attrs and self._matches(markup, self.text):
+            if not self.name and not self.attrs and self._matches(markup, self.string):
                 found = markup
         else:
             raise Exception(
                 "I don't know how to match against a %s" % markup.__class__)
         return found
 
-    def _matches(self, markup, match_against):
-        # print u"Matching %s against %s" % (markup, match_against)
+    def _matches(self, markup, match_against, already_tried=None):
+        # print(u"Matching %s against %s" % (markup, match_against))
         result = False
         if isinstance(markup, list) or isinstance(markup, tuple):
             # This should only happen when searching a multi-valued attribute
             # like 'class'.
-            if (isinstance(match_against, str)
-                and ' ' in match_against):
-                # A bit of a special case. If they try to match "foo
-                # bar" on a multivalue attribute's value, only accept
-                # the literal value "foo bar"
-                #
-                # XXX This is going to be pretty slow because we keep
-                # splitting match_against. But it shouldn't come up
-                # too often.
-                return (whitespace_re.split(match_against) == markup)
-            else:
-                for item in markup:
-                    if self._matches(item, match_against):
-                        return True
-                return False
+            for item in markup:
+                if self._matches(item, match_against):
+                    return True
+            # We didn't match any particular value of the multivalue
+            # attribute, but maybe we match the attribute value when
+            # considered as a string.
+            if self._matches(' '.join(markup), match_against):
+                return True
+            return False
 
         if match_against is True:
             # True matches any non-None value.
             return markup is not None
 
-        if isinstance(match_against, collections.abc.Callable):
+        if isinstance(match_against, Callable):
             return match_against(markup)
 
         # Custom callables take the tag as an argument, but all
         # other ways of matching match the tag name as a string.
+        original_markup = markup
         if isinstance(markup, Tag):
             markup = markup.name
 
@@ -1702,23 +2369,67 @@ class SoupStrainer(object):
             # None matches None, False, an empty string, an empty list, and so on.
             return not match_against
 
-        if isinstance(match_against, str):
+        if (hasattr(match_against, '__iter__')
+            and not isinstance(match_against, str)):
+            # We're asked to match against an iterable of items.
+            # The markup must be match at least one item in the
+            # iterable. We'll try each one in turn.
+            #
+            # To avoid infinite recursion we need to keep track of
+            # items we've already seen.
+            if not already_tried:
+                already_tried = set()
+            for item in match_against:
+                if item.__hash__:
+                    key = item
+                else:
+                    key = id(item)
+                if key in already_tried:
+                    continue
+                else:
+                    already_tried.add(key)
+                    if self._matches(original_markup, item, already_tried):
+                        return True
+            else:
+                return False
+
+        # Beyond this point we might need to run the test twice: once against
+        # the tag's name and once against its prefixed name.
+        match = False
+
+        if not match and isinstance(match_against, str):
             # Exact string match
-            return markup == match_against
+            match = markup == match_against
 
-        if hasattr(match_against, 'match'):
+        if not match and hasattr(match_against, 'search'):
             # Regexp match
             return match_against.search(markup)
 
-        if hasattr(match_against, '__iter__'):
-            # The markup must be an exact match against something
-            # in the iterable.
-            return markup in match_against
+        if (not match
+            and isinstance(original_markup, Tag)
+            and original_markup.prefix):
+            # Try the whole thing again with the prefixed tag name.
+            return self._matches(
+                original_markup.prefix + ':' + original_markup.name, match_against
+            )
+
+        return match
 
 
 class ResultSet(list):
     """A ResultSet is just a list that keeps track of the SoupStrainer
     that created it."""
     def __init__(self, source, result=()):
+        """Constructor.
+
+        :param source: A SoupStrainer.
+        :param result: A list of PageElements.
+        """
         super(ResultSet, self).__init__(result)
         self.source = source
+
+    def __getattr__(self, key):
+        """Raise a helpful exception to explain a common code fix."""
+        raise AttributeError(
+            "ResultSet object has no attribute '%s'. You're probably treating a list of elements like a single element. Did you call find_all() when you meant to call find()?" % key
+        )
diff --git a/bitbake/lib/bs4/formatter.py b/bitbake/lib/bs4/formatter.py
new file mode 100644
index 0000000000..9fa1b57cb6
--- /dev/null
+++ b/bitbake/lib/bs4/formatter.py
@@ -0,0 +1,185 @@
+from bs4.dammit import EntitySubstitution
+
+class Formatter(EntitySubstitution):
+    """Describes a strategy to use when outputting a parse tree to a string.
+
+    Some parts of this strategy come from the distinction between
+    HTML4, HTML5, and XML. Others are configurable by the user.
+
+    Formatters are passed in as the `formatter` argument to methods
+    like `PageElement.encode`. Most people won't need to think about
+    formatters, and most people who need to think about them can pass
+    in one of these predefined strings as `formatter` rather than
+    making a new Formatter object:
+
+    For HTML documents:
+     * 'html' - HTML entity substitution for generic HTML documents. (default)
+     * 'html5' - HTML entity substitution for HTML5 documents, as
+                 well as some optimizations in the way tags are rendered.
+     * 'minimal' - Only make the substitutions necessary to guarantee
+                   valid HTML.
+     * None - Do not perform any substitution. This will be faster
+              but may result in invalid markup.
+
+    For XML documents:
+     * 'html' - Entity substitution for XHTML documents.
+     * 'minimal' - Only make the substitutions necessary to guarantee
+                   valid XML. (default)
+     * None - Do not perform any substitution. This will be faster
+              but may result in invalid markup.
+    """
+    # Registries of XML and HTML formatters.
+    XML_FORMATTERS = {}
+    HTML_FORMATTERS = {}
+
+    HTML = 'html'
+    XML = 'xml'
+
+    HTML_DEFAULTS = dict(
+        cdata_containing_tags=set(["script", "style"]),
+    )
+
+    def _default(self, language, value, kwarg):
+        if value is not None:
+            return value
+        if language == self.XML:
+            return set()
+        return self.HTML_DEFAULTS[kwarg]
+
+    def __init__(
+            self, language=None, entity_substitution=None,
+            void_element_close_prefix='/', cdata_containing_tags=None,
+            empty_attributes_are_booleans=False, indent=1,
+    ):
+        r"""Constructor.
+
+        :param language: This should be Formatter.XML if you are formatting
+           XML markup and Formatter.HTML if you are formatting HTML markup.
+
+        :param entity_substitution: A function to call to replace special
+           characters with XML/HTML entities. For examples, see 
+           bs4.dammit.EntitySubstitution.substitute_html and substitute_xml.
+        :param void_element_close_prefix: By default, void elements
+           are represented as <tag/> (XML rules) rather than <tag>
+           (HTML rules). To get <tag>, pass in the empty string.
+        :param cdata_containing_tags: The list of tags that are defined
+           as containing CDATA in this dialect. For example, in HTML,
+           <script> and <style> tags are defined as containing CDATA,
+           and their contents should not be formatted.
+        :param blank_attributes_are_booleans: Render attributes whose value
+            is the empty string as HTML-style boolean attributes.
+            (Attributes whose value is None are always rendered this way.)
+
+        :param indent: If indent is a non-negative integer or string,
+            then the contents of elements will be indented
+            appropriately when pretty-printing. An indent level of 0,
+            negative, or "" will only insert newlines. Using a
+            positive integer indent indents that many spaces per
+            level. If indent is a string (such as "\t"), that string
+            is used to indent each level. The default behavior is to
+            indent one space per level.
+        """
+        self.language = language
+        self.entity_substitution = entity_substitution
+        self.void_element_close_prefix = void_element_close_prefix
+        self.cdata_containing_tags = self._default(
+            language, cdata_containing_tags, 'cdata_containing_tags'
+        )
+        self.empty_attributes_are_booleans=empty_attributes_are_booleans
+        if indent is None:
+            indent = 0
+        if isinstance(indent, int):
+            if indent < 0:
+                indent = 0
+            indent = ' ' * indent
+        elif isinstance(indent, str):
+            indent = indent
+        else:
+            indent = ' '
+        self.indent = indent
+
+    def substitute(self, ns):
+        """Process a string that needs to undergo entity substitution.
+        This may be a string encountered in an attribute value or as
+        text.
+
+        :param ns: A string.
+        :return: A string with certain characters replaced by named
+           or numeric entities.
+        """
+        if not self.entity_substitution:
+            return ns
+        from .element import NavigableString
+        if (isinstance(ns, NavigableString)
+            and ns.parent is not None
+            and ns.parent.name in self.cdata_containing_tags):
+            # Do nothing.
+            return ns
+        # Substitute.
+        return self.entity_substitution(ns)
+
+    def attribute_value(self, value):
+        """Process the value of an attribute.
+
+        :param ns: A string.
+        :return: A string with certain characters replaced by named
+           or numeric entities.
+        """
+        return self.substitute(value)
+    
+    def attributes(self, tag):
+        """Reorder a tag's attributes however you want.
+        
+        By default, attributes are sorted alphabetically. This makes
+        behavior consistent between Python 2 and Python 3, and preserves
+        backwards compatibility with older versions of Beautiful Soup.
+
+        If `empty_boolean_attributes` is True, then attributes whose
+        values are set to the empty string will be treated as boolean
+        attributes.
+        """
+        if tag.attrs is None:
+            return []
+        return sorted(
+            (k, (None if self.empty_attributes_are_booleans and v == '' else v))
+            for k, v in list(tag.attrs.items())
+        )
+   
+class HTMLFormatter(Formatter):
+    """A generic Formatter for HTML."""
+    REGISTRY = {}
+    def __init__(self, *args, **kwargs):
+        super(HTMLFormatter, self).__init__(self.HTML, *args, **kwargs)
+
+    
+class XMLFormatter(Formatter):
+    """A generic Formatter for XML."""
+    REGISTRY = {}
+    def __init__(self, *args, **kwargs):
+        super(XMLFormatter, self).__init__(self.XML, *args, **kwargs)
+
+
+# Set up aliases for the default formatters.
+HTMLFormatter.REGISTRY['html'] = HTMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_html
+)
+HTMLFormatter.REGISTRY["html5"] = HTMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_html,
+    void_element_close_prefix=None,
+    empty_attributes_are_booleans=True,
+)
+HTMLFormatter.REGISTRY["minimal"] = HTMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_xml
+)
+HTMLFormatter.REGISTRY[None] = HTMLFormatter(
+    entity_substitution=None
+)
+XMLFormatter.REGISTRY["html"] =  XMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_html
+)
+XMLFormatter.REGISTRY["minimal"] = XMLFormatter(
+    entity_substitution=EntitySubstitution.substitute_xml
+)
+XMLFormatter.REGISTRY[None] = Formatter(
+    Formatter(Formatter.XML, entity_substitution=None)
+)
diff --git a/bitbake/lib/bs4/testing.py b/bitbake/lib/bs4/testing.py
deleted file mode 100644
index 6584ecf303..0000000000
--- a/bitbake/lib/bs4/testing.py
+++ /dev/null
@@ -1,686 +0,0 @@
-"""Helper classes for tests."""
-
-__license__ = "MIT"
-
-import pickle
-import copy
-import unittest
-from unittest import TestCase
-from bs4 import BeautifulSoup
-from bs4.element import (
-    CharsetMetaAttributeValue,
-    Comment,
-    ContentMetaAttributeValue,
-    Doctype,
-    SoupStrainer,
-)
-
-from bs4.builder._htmlparser import HTMLParserTreeBuilder
-default_builder = HTMLParserTreeBuilder
-
-
-class SoupTest(unittest.TestCase):
-
-    @property
-    def default_builder(self):
-        return default_builder()
-
-    def soup(self, markup, **kwargs):
-        """Build a Beautiful Soup object from markup."""
-        builder = kwargs.pop('builder', self.default_builder)
-        return BeautifulSoup(markup, builder=builder, **kwargs)
-
-    def document_for(self, markup):
-        """Turn an HTML fragment into a document.
-
-        The details depend on the builder.
-        """
-        return self.default_builder.test_fragment_to_document(markup)
-
-    def assertSoupEquals(self, to_parse, compare_parsed_to=None):
-        builder = self.default_builder
-        obj = BeautifulSoup(to_parse, builder=builder)
-        if compare_parsed_to is None:
-            compare_parsed_to = to_parse
-
-        self.assertEqual(obj.decode(), self.document_for(compare_parsed_to))
-
-    def assertConnectedness(self, element):
-        """Ensure that next_element and previous_element are properly
-        set for all descendants of the given element.
-        """
-        earlier = None
-        for e in element.descendants:
-            if earlier:
-                self.assertEqual(e, earlier.next_element)
-                self.assertEqual(earlier, e.previous_element)
-            earlier = e
-
-class HTMLTreeBuilderSmokeTest(SoupTest):
-
-    """A basic test of a treebuilder's competence.
-
-    Any HTML treebuilder, present or future, should be able to pass
-    these tests. With invalid markup, there's room for interpretation,
-    and different parsers can handle it differently. But with the
-    markup in these tests, there's not much room for interpretation.
-    """
-
-    def test_pickle_and_unpickle_identity(self):
-        # Pickling a tree, then unpickling it, yields a tree identical
-        # to the original.
-        tree = self.soup("<a><b>foo</a>")
-        dumped = pickle.dumps(tree, 2)
-        loaded = pickle.loads(dumped)
-        self.assertEqual(loaded.__class__, BeautifulSoup)
-        self.assertEqual(loaded.decode(), tree.decode())
-
-    def assertDoctypeHandled(self, doctype_fragment):
-        """Assert that a given doctype string is handled correctly."""
-        doctype_str, soup = self._document_with_doctype(doctype_fragment)
-
-        # Make sure a Doctype object was created.
-        doctype = soup.contents[0]
-        self.assertEqual(doctype.__class__, Doctype)
-        self.assertEqual(doctype, doctype_fragment)
-        self.assertEqual(str(soup)[:len(doctype_str)], doctype_str)
-
-        # Make sure that the doctype was correctly associated with the
-        # parse tree and that the rest of the document parsed.
-        self.assertEqual(soup.p.contents[0], 'foo')
-
-    def _document_with_doctype(self, doctype_fragment):
-        """Generate and parse a document with the given doctype."""
-        doctype = '<!DOCTYPE %s>' % doctype_fragment
-        markup = doctype + '\n<p>foo</p>'
-        soup = self.soup(markup)
-        return doctype, soup
-
-    def test_normal_doctypes(self):
-        """Make sure normal, everyday HTML doctypes are handled correctly."""
-        self.assertDoctypeHandled("html")
-        self.assertDoctypeHandled(
-            'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"')
-
-    def test_empty_doctype(self):
-        soup = self.soup("<!DOCTYPE>")
-        doctype = soup.contents[0]
-        self.assertEqual("", doctype.strip())
-
-    def test_public_doctype_with_url(self):
-        doctype = 'html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"'
-        self.assertDoctypeHandled(doctype)
-
-    def test_system_doctype(self):
-        self.assertDoctypeHandled('foo SYSTEM "http://www.example.com/"')
-
-    def test_namespaced_system_doctype(self):
-        # We can handle a namespaced doctype with a system ID.
-        self.assertDoctypeHandled('xsl:stylesheet SYSTEM "htmlent.dtd"')
-
-    def test_namespaced_public_doctype(self):
-        # Test a namespaced doctype with a public id.
-        self.assertDoctypeHandled('xsl:stylesheet PUBLIC "htmlent.dtd"')
-
-    def test_real_xhtml_document(self):
-        """A real XHTML document should come out more or less the same as it went in."""
-        markup = b"""<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head><title>Hello.</title></head>
-<body>Goodbye.</body>
-</html>"""
-        soup = self.soup(markup)
-        self.assertEqual(
-            soup.encode("utf-8").replace(b"\n", b""),
-            markup.replace(b"\n", b""))
-
-    def test_processing_instruction(self):
-        markup = b"""<?PITarget PIContent?>"""
-        soup = self.soup(markup)
-        self.assertEqual(markup, soup.encode("utf8"))
-
-    def test_deepcopy(self):
-        """Make sure you can copy the tree builder.
-
-        This is important because the builder is part of a
-        BeautifulSoup object, and we want to be able to copy that.
-        """
-        copy.deepcopy(self.default_builder)
-
-    def test_p_tag_is_never_empty_element(self):
-        """A <p> tag is never designated as an empty-element tag.
-
-        Even if the markup shows it as an empty-element tag, it
-        shouldn't be presented that way.
-        """
-        soup = self.soup("<p/>")
-        self.assertFalse(soup.p.is_empty_element)
-        self.assertEqual(str(soup.p), "<p></p>")
-
-    def test_unclosed_tags_get_closed(self):
-        """A tag that's not closed by the end of the document should be closed.
-
-        This applies to all tags except empty-element tags.
-        """
-        self.assertSoupEquals("<p>", "<p></p>")
-        self.assertSoupEquals("<b>", "<b></b>")
-
-        self.assertSoupEquals("<br>", "<br/>")
-
-    def test_br_is_always_empty_element_tag(self):
-        """A <br> tag is designated as an empty-element tag.
-
-        Some parsers treat <br></br> as one <br/> tag, some parsers as
-        two tags, but it should always be an empty-element tag.
-        """
-        soup = self.soup("<br></br>")
-        self.assertTrue(soup.br.is_empty_element)
-        self.assertEqual(str(soup.br), "<br/>")
-
-    def test_nested_formatting_elements(self):
-        self.assertSoupEquals("<em><em></em></em>")
-
-    def test_double_head(self):
-        html = '''<!DOCTYPE html>
-<html>
-<head>
-<title>Ordinary HEAD element test</title>
-</head>
-<script type="text/javascript">
-alert("Help!");
-</script>
-<body>
-Hello, world!
-</body>
-</html>
-'''
-        soup = self.soup(html)
-        self.assertEqual("text/javascript", soup.find('script')['type'])
-
-    def test_comment(self):
-        # Comments are represented as Comment objects.
-        markup = "<p>foo<!--foobar-->baz</p>"
-        self.assertSoupEquals(markup)
-
-        soup = self.soup(markup)
-        comment = soup.find(text="foobar")
-        self.assertEqual(comment.__class__, Comment)
-
-        # The comment is properly integrated into the tree.
-        foo = soup.find(text="foo")
-        self.assertEqual(comment, foo.next_element)
-        baz = soup.find(text="baz")
-        self.assertEqual(comment, baz.previous_element)
-
-    def test_preserved_whitespace_in_pre_and_textarea(self):
-        """Whitespace must be preserved in <pre> and <textarea> tags."""
-        self.assertSoupEquals("<pre>   </pre>")
-        self.assertSoupEquals("<textarea> woo  </textarea>")
-
-    def test_nested_inline_elements(self):
-        """Inline elements can be nested indefinitely."""
-        b_tag = "<b>Inside a B tag</b>"
-        self.assertSoupEquals(b_tag)
-
-        nested_b_tag = "<p>A <i>nested <b>tag</b></i></p>"
-        self.assertSoupEquals(nested_b_tag)
-
-        double_nested_b_tag = "<p>A <a>doubly <i>nested <b>tag</b></i></a></p>"
-        self.assertSoupEquals(nested_b_tag)
-
-    def test_nested_block_level_elements(self):
-        """Block elements can be nested."""
-        soup = self.soup('<blockquote><p><b>Foo</b></p></blockquote>')
-        blockquote = soup.blockquote
-        self.assertEqual(blockquote.p.b.string, 'Foo')
-        self.assertEqual(blockquote.b.string, 'Foo')
-
-    def test_correctly_nested_tables(self):
-        """One table can go inside another one."""
-        markup = ('<table id="1">'
-                  '<tr>'
-                  "<td>Here's another table:"
-                  '<table id="2">'
-                  '<tr><td>foo</td></tr>'
-                  '</table></td>')
-
-        self.assertSoupEquals(
-            markup,
-            '<table id="1"><tr><td>Here\'s another table:'
-            '<table id="2"><tr><td>foo</td></tr></table>'
-            '</td></tr></table>')
-
-        self.assertSoupEquals(
-            "<table><thead><tr><td>Foo</td></tr></thead>"
-            "<tbody><tr><td>Bar</td></tr></tbody>"
-            "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
-
-    def test_deeply_nested_multivalued_attribute(self):
-        # html5lib can set the attributes of the same tag many times
-        # as it rearranges the tree. This has caused problems with
-        # multivalued attributes.
-        markup = '<table><div><div class="css"></div></div></table>'
-        soup = self.soup(markup)
-        self.assertEqual(["css"], soup.div.div['class'])
-
-    def test_multivalued_attribute_on_html(self):
-        # html5lib uses a different API to set the attributes ot the
-        # <html> tag. This has caused problems with multivalued
-        # attributes.
-        markup = '<html class="a b"></html>'
-        soup = self.soup(markup)
-        self.assertEqual(["a", "b"], soup.html['class'])
-
-    def test_angle_brackets_in_attribute_values_are_escaped(self):
-        self.assertSoupEquals('<a b="<a>"></a>', '<a b="&lt;a&gt;"></a>')
-
-    def test_entities_in_attributes_converted_to_unicode(self):
-        expect = '<p id="pi\N{LATIN SMALL LETTER N WITH TILDE}ata"></p>'
-        self.assertSoupEquals('<p id="pi&#241;ata"></p>', expect)
-        self.assertSoupEquals('<p id="pi&#xf1;ata"></p>', expect)
-        self.assertSoupEquals('<p id="pi&#Xf1;ata"></p>', expect)
-        self.assertSoupEquals('<p id="pi&ntilde;ata"></p>', expect)
-
-    def test_entities_in_text_converted_to_unicode(self):
-        expect = '<p>pi\N{LATIN SMALL LETTER N WITH TILDE}ata</p>'
-        self.assertSoupEquals("<p>pi&#241;ata</p>", expect)
-        self.assertSoupEquals("<p>pi&#xf1;ata</p>", expect)
-        self.assertSoupEquals("<p>pi&#Xf1;ata</p>", expect)
-        self.assertSoupEquals("<p>pi&ntilde;ata</p>", expect)
-
-    def test_quot_entity_converted_to_quotation_mark(self):
-        self.assertSoupEquals("<p>I said &quot;good day!&quot;</p>",
-                              '<p>I said "good day!"</p>')
-
-    def test_out_of_range_entity(self):
-        expect = "\N{REPLACEMENT CHARACTER}"
-        self.assertSoupEquals("&#10000000000000;", expect)
-        self.assertSoupEquals("&#x10000000000000;", expect)
-        self.assertSoupEquals("&#1000000000;", expect)
-
-    def test_multipart_strings(self):
-        "Mostly to prevent a recurrence of a bug in the html5lib treebuilder."
-        soup = self.soup("<html><h2>\nfoo</h2><p></p></html>")
-        self.assertEqual("p", soup.h2.string.next_element.name)
-        self.assertEqual("p", soup.p.name)
-        self.assertConnectedness(soup)
-
-    def test_head_tag_between_head_and_body(self):
-        "Prevent recurrence of a bug in the html5lib treebuilder."
-        content = """<html><head></head>
-  <link></link>
-  <body>foo</body>
-</html>
-"""
-        soup = self.soup(content)
-        self.assertNotEqual(None, soup.html.body)
-        self.assertConnectedness(soup)
-
-    def test_multiple_copies_of_a_tag(self):
-        "Prevent recurrence of a bug in the html5lib treebuilder."
-        content = """<!DOCTYPE html>
-<html>
- <body>
-   <article id="a" >
-   <div><a href="1"></div>
-   <footer>
-     <a href="2"></a>
-   </footer>
-  </article>
-  </body>
-</html>
-"""
-        soup = self.soup(content)
-        self.assertConnectedness(soup.article)
-
-    def test_basic_namespaces(self):
-        """Parsers don't need to *understand* namespaces, but at the
-        very least they should not choke on namespaces or lose
-        data."""
-
-        markup = b'<html xmlns="http://www.w3.org/1999/xhtml" xmlns:mathml="http://www.w3.org/1998/Math/MathML" xmlns:svg="http://www.w3.org/2000/svg"><head></head><body><mathml:msqrt>4</mathml:msqrt><b svg:fill="red"></b></body></html>'
-        soup = self.soup(markup)
-        self.assertEqual(markup, soup.encode())
-        html = soup.html
-        self.assertEqual('http://www.w3.org/1999/xhtml', soup.html['xmlns'])
-        self.assertEqual(
-            'http://www.w3.org/1998/Math/MathML', soup.html['xmlns:mathml'])
-        self.assertEqual(
-            'http://www.w3.org/2000/svg', soup.html['xmlns:svg'])
-
-    def test_multivalued_attribute_value_becomes_list(self):
-        markup = b'<a class="foo bar">'
-        soup = self.soup(markup)
-        self.assertEqual(['foo', 'bar'], soup.a['class'])
-
-    #
-    # Generally speaking, tests below this point are more tests of
-    # Beautiful Soup than tests of the tree builders. But parsers are
-    # weird, so we run these tests separately for every tree builder
-    # to detect any differences between them.
-    #
-
-    def test_can_parse_unicode_document(self):
-        # A seemingly innocuous document... but it's in Unicode! And
-        # it contains characters that can't be represented in the
-        # encoding found in the  declaration! The horror!
-        markup = '<html><head><meta encoding="euc-jp"></head><body>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</body>'
-        soup = self.soup(markup)
-        self.assertEqual('Sacr\xe9 bleu!', soup.body.string)
-
-    def test_soupstrainer(self):
-        """Parsers should be able to work with SoupStrainers."""
-        strainer = SoupStrainer("b")
-        soup = self.soup("A <b>bold</b> <meta/> <i>statement</i>",
-                         parse_only=strainer)
-        self.assertEqual(soup.decode(), "<b>bold</b>")
-
-    def test_single_quote_attribute_values_become_double_quotes(self):
-        self.assertSoupEquals("<foo attr='bar'></foo>",
-                              '<foo attr="bar"></foo>')
-
-    def test_attribute_values_with_nested_quotes_are_left_alone(self):
-        text = """<foo attr='bar "brawls" happen'>a</foo>"""
-        self.assertSoupEquals(text)
-
-    def test_attribute_values_with_double_nested_quotes_get_quoted(self):
-        text = """<foo attr='bar "brawls" happen'>a</foo>"""
-        soup = self.soup(text)
-        soup.foo['attr'] = 'Brawls happen at "Bob\'s Bar"'
-        self.assertSoupEquals(
-            soup.foo.decode(),
-            """<foo attr="Brawls happen at &quot;Bob\'s Bar&quot;">a</foo>""")
-
-    def test_ampersand_in_attribute_value_gets_escaped(self):
-        self.assertSoupEquals('<this is="really messed up & stuff"></this>',
-                              '<this is="really messed up &amp; stuff"></this>')
-
-        self.assertSoupEquals(
-            '<a href="http://example.org?a=1&b=2;3">foo</a>',
-            '<a href="http://example.org?a=1&amp;b=2;3">foo</a>')
-
-    def test_escaped_ampersand_in_attribute_value_is_left_alone(self):
-        self.assertSoupEquals('<a href="http://example.org?a=1&amp;b=2;3"></a>')
-
-    def test_entities_in_strings_converted_during_parsing(self):
-        # Both XML and HTML entities are converted to Unicode characters
-        # during parsing.
-        text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
-        expected = "<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>"
-        self.assertSoupEquals(text, expected)
-
-    def test_smart_quotes_converted_on_the_way_in(self):
-        # Microsoft smart quotes are converted to Unicode characters during
-        # parsing.
-        quote = b"<p>\x91Foo\x92</p>"
-        soup = self.soup(quote)
-        self.assertEqual(
-            soup.p.string,
-            "\N{LEFT SINGLE QUOTATION MARK}Foo\N{RIGHT SINGLE QUOTATION MARK}")
-
-    def test_non_breaking_spaces_converted_on_the_way_in(self):
-        soup = self.soup("<a>&nbsp;&nbsp;</a>")
-        self.assertEqual(soup.a.string, "\N{NO-BREAK SPACE}" * 2)
-
-    def test_entities_converted_on_the_way_out(self):
-        text = "<p>&lt;&lt;sacr&eacute;&#32;bleu!&gt;&gt;</p>"
-        expected = "<p>&lt;&lt;sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</p>".encode("utf-8")
-        soup = self.soup(text)
-        self.assertEqual(soup.p.encode("utf-8"), expected)
-
-    def test_real_iso_latin_document(self):
-        # Smoke test of interrelated functionality, using an
-        # easy-to-understand document.
-
-        # Here it is in Unicode. Note that it claims to be in ISO-Latin-1.
-        unicode_html = '<html><head><meta content="text/html; charset=ISO-Latin-1" http-equiv="Content-type"/></head><body><p>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</p></body></html>'
-
-        # That's because we're going to encode it into ISO-Latin-1, and use
-        # that to test.
-        iso_latin_html = unicode_html.encode("iso-8859-1")
-
-        # Parse the ISO-Latin-1 HTML.
-        soup = self.soup(iso_latin_html)
-        # Encode it to UTF-8.
-        result = soup.encode("utf-8")
-
-        # What do we expect the result to look like? Well, it would
-        # look like unicode_html, except that the META tag would say
-        # UTF-8 instead of ISO-Latin-1.
-        expected = unicode_html.replace("ISO-Latin-1", "utf-8")
-
-        # And, of course, it would be in UTF-8, not Unicode.
-        expected = expected.encode("utf-8")
-
-        # Ta-da!
-        self.assertEqual(result, expected)
-
-    def test_real_shift_jis_document(self):
-        # Smoke test to make sure the parser can handle a document in
-        # Shift-JIS encoding, without choking.
-        shift_jis_html = (
-            b'<html><head></head><body><pre>'
-            b'\x82\xb1\x82\xea\x82\xcdShift-JIS\x82\xc5\x83R\x81[\x83f'
-            b'\x83B\x83\x93\x83O\x82\xb3\x82\xea\x82\xbd\x93\xfa\x96{\x8c'
-            b'\xea\x82\xcc\x83t\x83@\x83C\x83\x8b\x82\xc5\x82\xb7\x81B'
-            b'</pre></body></html>')
-        unicode_html = shift_jis_html.decode("shift-jis")
-        soup = self.soup(unicode_html)
-
-        # Make sure the parse tree is correctly encoded to various
-        # encodings.
-        self.assertEqual(soup.encode("utf-8"), unicode_html.encode("utf-8"))
-        self.assertEqual(soup.encode("euc_jp"), unicode_html.encode("euc_jp"))
-
-    def test_real_hebrew_document(self):
-        # A real-world test to make sure we can convert ISO-8859-9 (a
-        # Hebrew encoding) to UTF-8.
-        hebrew_document = b'<html><head><title>Hebrew (ISO 8859-8) in Visual Directionality</title></head><body><h1>Hebrew (ISO 8859-8) in Visual Directionality</h1>\xed\xe5\xec\xf9</body></html>'
-        soup = self.soup(
-            hebrew_document, from_encoding="iso8859-8")
-        self.assertEqual(soup.original_encoding, 'iso8859-8')
-        self.assertEqual(
-            soup.encode('utf-8'),
-            hebrew_document.decode("iso8859-8").encode("utf-8"))
-
-    def test_meta_tag_reflects_current_encoding(self):
-        # Here's the <meta> tag saying that a document is
-        # encoded in Shift-JIS.
-        meta_tag = ('<meta content="text/html; charset=x-sjis" '
-                    'http-equiv="Content-type"/>')
-
-        # Here's a document incorporating that meta tag.
-        shift_jis_html = (
-            '<html><head>\n%s\n'
-            '<meta http-equiv="Content-language" content="ja"/>'
-            '</head><body>Shift-JIS markup goes here.') % meta_tag
-        soup = self.soup(shift_jis_html)
-
-        # Parse the document, and the charset is seemingly unaffected.
-        parsed_meta = soup.find('meta', {'http-equiv': 'Content-type'})
-        content = parsed_meta['content']
-        self.assertEqual('text/html; charset=x-sjis', content)
-
-        # But that value is actually a ContentMetaAttributeValue object.
-        self.assertTrue(isinstance(content, ContentMetaAttributeValue))
-
-        # And it will take on a value that reflects its current
-        # encoding.
-        self.assertEqual('text/html; charset=utf8', content.encode("utf8"))
-
-        # For the rest of the story, see TestSubstitutions in
-        # test_tree.py.
-
-    def test_html5_style_meta_tag_reflects_current_encoding(self):
-        # Here's the <meta> tag saying that a document is
-        # encoded in Shift-JIS.
-        meta_tag = ('<meta id="encoding" charset="x-sjis" />')
-
-        # Here's a document incorporating that meta tag.
-        shift_jis_html = (
-            '<html><head>\n%s\n'
-            '<meta http-equiv="Content-language" content="ja"/>'
-            '</head><body>Shift-JIS markup goes here.') % meta_tag
-        soup = self.soup(shift_jis_html)
-
-        # Parse the document, and the charset is seemingly unaffected.
-        parsed_meta = soup.find('meta', id="encoding")
-        charset = parsed_meta['charset']
-        self.assertEqual('x-sjis', charset)
-
-        # But that value is actually a CharsetMetaAttributeValue object.
-        self.assertTrue(isinstance(charset, CharsetMetaAttributeValue))
-
-        # And it will take on a value that reflects its current
-        # encoding.
-        self.assertEqual('utf8', charset.encode("utf8"))
-
-    def test_tag_with_no_attributes_can_have_attributes_added(self):
-        data = self.soup("<a>text</a>")
-        data.a['foo'] = 'bar'
-        self.assertEqual('<a foo="bar">text</a>', data.a.decode())
-
-class XMLTreeBuilderSmokeTest(SoupTest):
-
-    def test_pickle_and_unpickle_identity(self):
-        # Pickling a tree, then unpickling it, yields a tree identical
-        # to the original.
-        tree = self.soup("<a><b>foo</a>")
-        dumped = pickle.dumps(tree, 2)
-        loaded = pickle.loads(dumped)
-        self.assertEqual(loaded.__class__, BeautifulSoup)
-        self.assertEqual(loaded.decode(), tree.decode())
-
-    def test_docstring_generated(self):
-        soup = self.soup("<root/>")
-        self.assertEqual(
-            soup.encode(), b'<?xml version="1.0" encoding="utf-8"?>\n<root/>')
-
-    def test_xml_declaration(self):
-        markup = b"""<?xml version="1.0" encoding="utf8"?>\n<foo/>"""
-        soup = self.soup(markup)
-        self.assertEqual(markup, soup.encode("utf8"))
-
-    def test_real_xhtml_document(self):
-        """A real XHTML document should come out *exactly* the same as it went in."""
-        markup = b"""<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN">
-<html xmlns="http://www.w3.org/1999/xhtml">
-<head><title>Hello.</title></head>
-<body>Goodbye.</body>
-</html>"""
-        soup = self.soup(markup)
-        self.assertEqual(
-            soup.encode("utf-8"), markup)
-
-    def test_formatter_processes_script_tag_for_xml_documents(self):
-        doc = """
-  <script type="text/javascript">
-  </script>
-"""
-        soup = BeautifulSoup(doc, "lxml-xml")
-        # lxml would have stripped this while parsing, but we can add
-        # it later.
-        soup.script.string = 'console.log("< < hey > > ");'
-        encoded = soup.encode()
-        self.assertTrue(b"&lt; &lt; hey &gt; &gt;" in encoded)
-
-    def test_can_parse_unicode_document(self):
-        markup = '<?xml version="1.0" encoding="euc-jp"><root>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</root>'
-        soup = self.soup(markup)
-        self.assertEqual('Sacr\xe9 bleu!', soup.root.string)
-
-    def test_popping_namespaced_tag(self):
-        markup = '<rss xmlns:dc="foo"><dc:creator>b</dc:creator><dc:date>2012-07-02T20:33:42Z</dc:date><dc:rights>c</dc:rights><image>d</image></rss>'
-        soup = self.soup(markup)
-        self.assertEqual(
-            str(soup.rss), markup)
-
-    def test_docstring_includes_correct_encoding(self):
-        soup = self.soup("<root/>")
-        self.assertEqual(
-            soup.encode("latin1"),
-            b'<?xml version="1.0" encoding="latin1"?>\n<root/>')
-
-    def test_large_xml_document(self):
-        """A large XML document should come out the same as it went in."""
-        markup = (b'<?xml version="1.0" encoding="utf-8"?>\n<root>'
-                  + b'0' * (2**12)
-                  + b'</root>')
-        soup = self.soup(markup)
-        self.assertEqual(soup.encode("utf-8"), markup)
-
-
-    def test_tags_are_empty_element_if_and_only_if_they_are_empty(self):
-        self.assertSoupEquals("<p>", "<p/>")
-        self.assertSoupEquals("<p>foo</p>")
-
-    def test_namespaces_are_preserved(self):
-        markup = '<root xmlns:a="http://example.com/" xmlns:b="http://example.net/"><a:foo>This tag is in the a namespace</a:foo><b:foo>This tag is in the b namespace</b:foo></root>'
-        soup = self.soup(markup)
-        root = soup.root
-        self.assertEqual("http://example.com/", root['xmlns:a'])
-        self.assertEqual("http://example.net/", root['xmlns:b'])
-
-    def test_closing_namespaced_tag(self):
-        markup = '<p xmlns:dc="http://purl.org/dc/elements/1.1/"><dc:date>20010504</dc:date></p>'
-        soup = self.soup(markup)
-        self.assertEqual(str(soup.p), markup)
-
-    def test_namespaced_attributes(self):
-        markup = '<foo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"><bar xsi:schemaLocation="http://www.example.com"/></foo>'
-        soup = self.soup(markup)
-        self.assertEqual(str(soup.foo), markup)
-
-    def test_namespaced_attributes_xml_namespace(self):
-        markup = '<foo xml:lang="fr">bar</foo>'
-        soup = self.soup(markup)
-        self.assertEqual(str(soup.foo), markup)
-
-class HTML5TreeBuilderSmokeTest(HTMLTreeBuilderSmokeTest):
-    """Smoke test for a tree builder that supports HTML5."""
-
-    def test_real_xhtml_document(self):
-        # Since XHTML is not HTML5, HTML5 parsers are not tested to handle
-        # XHTML documents in any particular way.
-        pass
-
-    def test_html_tags_have_namespace(self):
-        markup = "<a>"
-        soup = self.soup(markup)
-        self.assertEqual("http://www.w3.org/1999/xhtml", soup.a.namespace)
-
-    def test_svg_tags_have_namespace(self):
-        markup = '<svg><circle/></svg>'
-        soup = self.soup(markup)
-        namespace = "http://www.w3.org/2000/svg"
-        self.assertEqual(namespace, soup.svg.namespace)
-        self.assertEqual(namespace, soup.circle.namespace)
-
-
-    def test_mathml_tags_have_namespace(self):
-        markup = '<math><msqrt>5</msqrt></math>'
-        soup = self.soup(markup)
-        namespace = 'http://www.w3.org/1998/Math/MathML'
-        self.assertEqual(namespace, soup.math.namespace)
-        self.assertEqual(namespace, soup.msqrt.namespace)
-
-    def test_xml_declaration_becomes_comment(self):
-        markup = '<?xml version="1.0" encoding="utf-8"?><html></html>'
-        soup = self.soup(markup)
-        self.assertTrue(isinstance(soup.contents[0], Comment))
-        self.assertEqual(soup.contents[0], '?xml version="1.0" encoding="utf-8"?')
-        self.assertEqual("html", soup.contents[0].next_element.name)
-
-def skipIf(condition, reason):
-   def nothing(test, *args, **kwargs):
-       return None
-
-   def decorator(test_item):
-       if condition:
-           return nothing
-       else:
-           return test_item
-
-   return decorator
diff --git a/bitbake/lib/bs4/tests/__init__.py b/bitbake/lib/bs4/tests/__init__.py
deleted file mode 100644
index 142c8cc3f1..0000000000
--- a/bitbake/lib/bs4/tests/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-"The beautifulsoup tests."
diff --git a/bitbake/lib/bs4/tests/test_builder_registry.py b/bitbake/lib/bs4/tests/test_builder_registry.py
deleted file mode 100644
index 90cad82933..0000000000
--- a/bitbake/lib/bs4/tests/test_builder_registry.py
+++ /dev/null
@@ -1,147 +0,0 @@
-"""Tests of the builder registry."""
-
-import unittest
-import warnings
-
-from bs4 import BeautifulSoup
-from bs4.builder import (
-    builder_registry as registry,
-    HTMLParserTreeBuilder,
-    TreeBuilderRegistry,
-)
-
-try:
-    from bs4.builder import HTML5TreeBuilder
-    HTML5LIB_PRESENT = True
-except ImportError:
-    HTML5LIB_PRESENT = False
-
-try:
-    from bs4.builder import (
-        LXMLTreeBuilderForXML,
-        LXMLTreeBuilder,
-        )
-    LXML_PRESENT = True
-except ImportError:
-    LXML_PRESENT = False
-
-
-class BuiltInRegistryTest(unittest.TestCase):
-    """Test the built-in registry with the default builders registered."""
-
-    def test_combination(self):
-        if LXML_PRESENT:
-            self.assertEqual(registry.lookup('fast', 'html'),
-                             LXMLTreeBuilder)
-
-        if LXML_PRESENT:
-            self.assertEqual(registry.lookup('permissive', 'xml'),
-                             LXMLTreeBuilderForXML)
-        self.assertEqual(registry.lookup('strict', 'html'),
-                          HTMLParserTreeBuilder)
-        if HTML5LIB_PRESENT:
-            self.assertEqual(registry.lookup('html5lib', 'html'),
-                              HTML5TreeBuilder)
-
-    def test_lookup_by_markup_type(self):
-        if LXML_PRESENT:
-            self.assertEqual(registry.lookup('html'), LXMLTreeBuilder)
-            self.assertEqual(registry.lookup('xml'), LXMLTreeBuilderForXML)
-        else:
-            self.assertEqual(registry.lookup('xml'), None)
-            if HTML5LIB_PRESENT:
-                self.assertEqual(registry.lookup('html'), HTML5TreeBuilder)
-            else:
-                self.assertEqual(registry.lookup('html'), HTMLParserTreeBuilder)
-
-    def test_named_library(self):
-        if LXML_PRESENT:
-            self.assertEqual(registry.lookup('lxml', 'xml'),
-                             LXMLTreeBuilderForXML)
-            self.assertEqual(registry.lookup('lxml', 'html'),
-                             LXMLTreeBuilder)
-        if HTML5LIB_PRESENT:
-            self.assertEqual(registry.lookup('html5lib'),
-                              HTML5TreeBuilder)
-
-        self.assertEqual(registry.lookup('html.parser'),
-                          HTMLParserTreeBuilder)
-
-    def test_beautifulsoup_constructor_does_lookup(self):
-
-        with warnings.catch_warnings(record=True) as w:
-            # This will create a warning about not explicitly
-            # specifying a parser, but we'll ignore it.
-
-            # You can pass in a string.
-            BeautifulSoup("", features="html")
-            # Or a list of strings.
-            BeautifulSoup("", features=["html", "fast"])
-
-        # You'll get an exception if BS can't find an appropriate
-        # builder.
-        self.assertRaises(ValueError, BeautifulSoup,
-                          "", features="no-such-feature")
-
-class RegistryTest(unittest.TestCase):
-    """Test the TreeBuilderRegistry class in general."""
-
-    def setUp(self):
-        self.registry = TreeBuilderRegistry()
-
-    def builder_for_features(self, *feature_list):
-        cls = type('Builder_' + '_'.join(feature_list),
-                   (object,), {'features' : feature_list})
-
-        self.registry.register(cls)
-        return cls
-
-    def test_register_with_no_features(self):
-        builder = self.builder_for_features()
-
-        # Since the builder advertises no features, you can't find it
-        # by looking up features.
-        self.assertEqual(self.registry.lookup('foo'), None)
-
-        # But you can find it by doing a lookup with no features, if
-        # this happens to be the only registered builder.
-        self.assertEqual(self.registry.lookup(), builder)
-
-    def test_register_with_features_makes_lookup_succeed(self):
-        builder = self.builder_for_features('foo', 'bar')
-        self.assertEqual(self.registry.lookup('foo'), builder)
-        self.assertEqual(self.registry.lookup('bar'), builder)
-
-    def test_lookup_fails_when_no_builder_implements_feature(self):
-        builder = self.builder_for_features('foo', 'bar')
-        self.assertEqual(self.registry.lookup('baz'), None)
-
-    def test_lookup_gets_most_recent_registration_when_no_feature_specified(self):
-        builder1 = self.builder_for_features('foo')
-        builder2 = self.builder_for_features('bar')
-        self.assertEqual(self.registry.lookup(), builder2)
-
-    def test_lookup_fails_when_no_tree_builders_registered(self):
-        self.assertEqual(self.registry.lookup(), None)
-
-    def test_lookup_gets_most_recent_builder_supporting_all_features(self):
-        has_one = self.builder_for_features('foo')
-        has_the_other = self.builder_for_features('bar')
-        has_both_early = self.builder_for_features('foo', 'bar', 'baz')
-        has_both_late = self.builder_for_features('foo', 'bar', 'quux')
-        lacks_one = self.builder_for_features('bar')
-        has_the_other = self.builder_for_features('foo')
-
-        # There are two builders featuring 'foo' and 'bar', but
-        # the one that also features 'quux' was registered later.
-        self.assertEqual(self.registry.lookup('foo', 'bar'),
-                          has_both_late)
-
-        # There is only one builder featuring 'foo', 'bar', and 'baz'.
-        self.assertEqual(self.registry.lookup('foo', 'bar', 'baz'),
-                          has_both_early)
-
-    def test_lookup_fails_when_cannot_reconcile_requested_features(self):
-        builder1 = self.builder_for_features('foo', 'bar')
-        builder2 = self.builder_for_features('foo', 'baz')
-        self.assertEqual(self.registry.lookup('bar', 'baz'), None)
diff --git a/bitbake/lib/bs4/tests/test_docs.py b/bitbake/lib/bs4/tests/test_docs.py
deleted file mode 100644
index d1d76a33bf..0000000000
--- a/bitbake/lib/bs4/tests/test_docs.py
+++ /dev/null
@@ -1,32 +0,0 @@
-"Test harness for doctests."
-
-# pylint: disable-msg=E0611,W0142
-
-__metaclass__ = type
-__all__ = [
-    'additional_tests',
-    ]
-
-import doctest
-#from pkg_resources import (
-#    resource_filename, resource_exists, resource_listdir, cleanup_resources)
-
-DOCTEST_FLAGS = (
-    doctest.ELLIPSIS |
-    doctest.NORMALIZE_WHITESPACE |
-    doctest.REPORT_NDIFF)
-
-# def additional_tests():
-#     "Run the doc tests (README.txt and docs/*, if any exist)"
-#     doctest_files = [
-#         os.path.abspath(resource_filename('bs4', 'README.txt'))]
-#     if resource_exists('bs4', 'docs'):
-#         for name in resource_listdir('bs4', 'docs'):
-#             if name.endswith('.txt'):
-#                 doctest_files.append(
-#                     os.path.abspath(
-#                         resource_filename('bs4', 'docs/%s' % name)))
-#     kwargs = dict(module_relative=False, optionflags=DOCTEST_FLAGS)
-#     atexit.register(cleanup_resources)
-#     return unittest.TestSuite((
-#         doctest.DocFileSuite(*doctest_files, **kwargs)))
diff --git a/bitbake/lib/bs4/tests/test_html5lib.py b/bitbake/lib/bs4/tests/test_html5lib.py
deleted file mode 100644
index a7494ca5ba..0000000000
--- a/bitbake/lib/bs4/tests/test_html5lib.py
+++ /dev/null
@@ -1,98 +0,0 @@
-"""Tests to ensure that the html5lib tree builder generates good trees."""
-
-import warnings
-
-try:
-    from bs4.builder import HTML5TreeBuilder
-    HTML5LIB_PRESENT = True
-except ImportError as e:
-    HTML5LIB_PRESENT = False
-from bs4.element import SoupStrainer
-from bs4.testing import (
-    HTML5TreeBuilderSmokeTest,
-    SoupTest,
-    skipIf,
-)
-
-@skipIf(
-    not HTML5LIB_PRESENT,
-    "html5lib seems not to be present, not testing its tree builder.")
-class HTML5LibBuilderSmokeTest(SoupTest, HTML5TreeBuilderSmokeTest):
-    """See ``HTML5TreeBuilderSmokeTest``."""
-
-    @property
-    def default_builder(self):
-        return HTML5TreeBuilder()
-
-    def test_soupstrainer(self):
-        # The html5lib tree builder does not support SoupStrainers.
-        strainer = SoupStrainer("b")
-        markup = "<p>A <b>bold</b> statement.</p>"
-        with warnings.catch_warnings(record=True) as w:
-            soup = self.soup(markup, parse_only=strainer)
-        self.assertEqual(
-            soup.decode(), self.document_for(markup))
-
-        self.assertTrue(
-            "the html5lib tree builder doesn't support parse_only" in
-            str(w[0].message))
-
-    def test_correctly_nested_tables(self):
-        """html5lib inserts <tbody> tags where other parsers don't."""
-        markup = ('<table id="1">'
-                  '<tr>'
-                  "<td>Here's another table:"
-                  '<table id="2">'
-                  '<tr><td>foo</td></tr>'
-                  '</table></td>')
-
-        self.assertSoupEquals(
-            markup,
-            '<table id="1"><tbody><tr><td>Here\'s another table:'
-            '<table id="2"><tbody><tr><td>foo</td></tr></tbody></table>'
-            '</td></tr></tbody></table>')
-
-        self.assertSoupEquals(
-            "<table><thead><tr><td>Foo</td></tr></thead>"
-            "<tbody><tr><td>Bar</td></tr></tbody>"
-            "<tfoot><tr><td>Baz</td></tr></tfoot></table>")
-
-    def test_xml_declaration_followed_by_doctype(self):
-        markup = '''<?xml version="1.0" encoding="utf-8"?>
-<!DOCTYPE html>
-<html>
-  <head>
-  </head>
-  <body>
-   <p>foo</p>
-  </body>
-</html>'''
-        soup = self.soup(markup)
-        # Verify that we can reach the <p> tag; this means the tree is connected.
-        self.assertEqual(b"<p>foo</p>", soup.p.encode())
-
-    def test_reparented_markup(self):
-        markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>'
-        soup = self.soup(markup)
-        self.assertEqual("<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p></body>", soup.body.decode())
-        self.assertEqual(2, len(soup.find_all('p')))
-
-
-    def test_reparented_markup_ends_with_whitespace(self):
-        markup = '<p><em>foo</p>\n<p>bar<a></a></em></p>\n'
-        soup = self.soup(markup)
-        self.assertEqual("<body><p><em>foo</em></p><em>\n</em><p><em>bar<a></a></em></p>\n</body>", soup.body.decode())
-        self.assertEqual(2, len(soup.find_all('p')))
-
-    def test_processing_instruction(self):
-        """Processing instructions become comments."""
-        markup = b"""<?PITarget PIContent?>"""
-        soup = self.soup(markup)
-        assert str(soup).startswith("<!--?PITarget PIContent?-->")
-
-    def test_cloned_multivalue_node(self):
-        markup = b"""<a class="my_class"><p></a>"""
-        soup = self.soup(markup)
-        a1, a2 = soup.find_all('a')
-        self.assertEqual(a1, a2)
-        assert a1 is not a2
diff --git a/bitbake/lib/bs4/tests/test_htmlparser.py b/bitbake/lib/bs4/tests/test_htmlparser.py
deleted file mode 100644
index 30a25e6709..0000000000
--- a/bitbake/lib/bs4/tests/test_htmlparser.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""Tests to ensure that the html.parser tree builder generates good
-trees."""
-
-import pickle
-from bs4.testing import SoupTest, HTMLTreeBuilderSmokeTest
-from bs4.builder import HTMLParserTreeBuilder
-
-class HTMLParserTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest):
-
-    @property
-    def default_builder(self):
-        return HTMLParserTreeBuilder()
-
-    def test_namespaced_system_doctype(self):
-        # html.parser can't handle namespaced doctypes, so skip this one.
-        pass
-
-    def test_namespaced_public_doctype(self):
-        # html.parser can't handle namespaced doctypes, so skip this one.
-        pass
-
-    def test_builder_is_pickled(self):
-        """Unlike most tree builders, HTMLParserTreeBuilder and will
-        be restored after pickling.
-        """
-        tree = self.soup("<a><b>foo</a>")
-        dumped = pickle.dumps(tree, 2)
-        loaded = pickle.loads(dumped)
-        self.assertTrue(isinstance(loaded.builder, type(tree.builder)))
-
-
diff --git a/bitbake/lib/bs4/tests/test_lxml.py b/bitbake/lib/bs4/tests/test_lxml.py
deleted file mode 100644
index 6b6cdd07cb..0000000000
--- a/bitbake/lib/bs4/tests/test_lxml.py
+++ /dev/null
@@ -1,70 +0,0 @@
-"""Tests to ensure that the lxml tree builder generates good trees."""
-
-import warnings
-
-try:
-    import lxml.etree
-    LXML_PRESENT = True
-    LXML_VERSION = lxml.etree.LXML_VERSION
-except ImportError as e:
-    LXML_PRESENT = False
-    LXML_VERSION = (0,)
-
-if LXML_PRESENT:
-    from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
-
-from bs4 import BeautifulStoneSoup
-from bs4.testing import skipIf
-from bs4.testing import (
-    HTMLTreeBuilderSmokeTest,
-    XMLTreeBuilderSmokeTest,
-    SoupTest,
-    skipIf,
-)
-
-@skipIf(
-    not LXML_PRESENT,
-    "lxml seems not to be present, not testing its tree builder.")
-class LXMLTreeBuilderSmokeTest(SoupTest, HTMLTreeBuilderSmokeTest):
-    """See ``HTMLTreeBuilderSmokeTest``."""
-
-    @property
-    def default_builder(self):
-        return LXMLTreeBuilder()
-
-    def test_out_of_range_entity(self):
-        self.assertSoupEquals(
-            "<p>foo&#10000000000000;bar</p>", "<p>foobar</p>")
-        self.assertSoupEquals(
-            "<p>foo&#x10000000000000;bar</p>", "<p>foobar</p>")
-        self.assertSoupEquals(
-            "<p>foo&#1000000000;bar</p>", "<p>foobar</p>")
-
-    # In lxml < 2.3.5, an empty doctype causes a segfault. Skip this
-    # test if an old version of lxml is installed.
-
-    @skipIf(
-        not LXML_PRESENT or LXML_VERSION < (2,3,5,0),
-        "Skipping doctype test for old version of lxml to avoid segfault.")
-    def test_empty_doctype(self):
-        soup = self.soup("<!DOCTYPE>")
-        doctype = soup.contents[0]
-        self.assertEqual("", doctype.strip())
-
-    def test_beautifulstonesoup_is_xml_parser(self):
-        # Make sure that the deprecated BSS class uses an xml builder
-        # if one is installed.
-        with warnings.catch_warnings(record=True) as w:
-            soup = BeautifulStoneSoup("<b />")
-        self.assertEqual("<b/>", str(soup.b))
-        self.assertTrue("BeautifulStoneSoup class is deprecated" in str(w[0].message))
-
-@skipIf(
-    not LXML_PRESENT,
-    "lxml seems not to be present, not testing its XML tree builder.")
-class LXMLXMLTreeBuilderSmokeTest(SoupTest, XMLTreeBuilderSmokeTest):
-    """See ``HTMLTreeBuilderSmokeTest``."""
-
-    @property
-    def default_builder(self):
-        return LXMLTreeBuilderForXML()
diff --git a/bitbake/lib/bs4/tests/test_soup.py b/bitbake/lib/bs4/tests/test_soup.py
deleted file mode 100644
index 6ad3cb3765..0000000000
--- a/bitbake/lib/bs4/tests/test_soup.py
+++ /dev/null
@@ -1,479 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Tests of Beautiful Soup as a whole."""
-
-import logging
-import unittest
-import sys
-import tempfile
-
-from bs4 import BeautifulSoup
-from bs4.element import (
-    CharsetMetaAttributeValue,
-    ContentMetaAttributeValue,
-    SoupStrainer,
-    NamespacedAttribute,
-    )
-import bs4.dammit
-from bs4.dammit import (
-    EntitySubstitution,
-    UnicodeDammit,
-    EncodingDetector,
-)
-from bs4.testing import (
-    SoupTest,
-    skipIf,
-)
-import warnings
-
-try:
-    from bs4.builder import LXMLTreeBuilder, LXMLTreeBuilderForXML
-    LXML_PRESENT = True
-except ImportError as e:
-    LXML_PRESENT = False
-
-PYTHON_2_PRE_2_7 = (sys.version_info < (2,7))
-PYTHON_3_PRE_3_2 = (sys.version_info[0] == 3 and sys.version_info < (3,2))
-
-class TestConstructor(SoupTest):
-
-    def test_short_unicode_input(self):
-        data = "<h1>éé</h1>"
-        soup = self.soup(data)
-        self.assertEqual("éé", soup.h1.string)
-
-    def test_embedded_null(self):
-        data = "<h1>foo\0bar</h1>"
-        soup = self.soup(data)
-        self.assertEqual("foo\0bar", soup.h1.string)
-
-    def test_exclude_encodings(self):
-        utf8_data = "Räksmörgås".encode("utf-8")
-        soup = self.soup(utf8_data, exclude_encodings=["utf-8"])
-        self.assertEqual("windows-1252", soup.original_encoding)
-
-
-class TestWarnings(SoupTest):
-
-    def _no_parser_specified(self, s, is_there=True):
-        v = s.startswith(BeautifulSoup.NO_PARSER_SPECIFIED_WARNING[:80])
-        self.assertTrue(v)
-
-    def test_warning_if_no_parser_specified(self):
-        with warnings.catch_warnings(record=True) as w:
-            soup = self.soup("<a><b></b></a>")
-        msg = str(w[0].message)
-        self._assert_no_parser_specified(msg)
-
-    def test_warning_if_parser_specified_too_vague(self):
-        with warnings.catch_warnings(record=True) as w:
-            soup = self.soup("<a><b></b></a>", "html")
-        msg = str(w[0].message)
-        self._assert_no_parser_specified(msg)
-
-    def test_no_warning_if_explicit_parser_specified(self):
-        with warnings.catch_warnings(record=True) as w:
-            soup = self.soup("<a><b></b></a>", "html.parser")
-        self.assertEqual([], w)
-
-    def test_parseOnlyThese_renamed_to_parse_only(self):
-        with warnings.catch_warnings(record=True) as w:
-            soup = self.soup("<a><b></b></a>", parseOnlyThese=SoupStrainer("b"))
-        msg = str(w[0].message)
-        self.assertTrue("parseOnlyThese" in msg)
-        self.assertTrue("parse_only" in msg)
-        self.assertEqual(b"<b></b>", soup.encode())
-
-    def test_fromEncoding_renamed_to_from_encoding(self):
-        with warnings.catch_warnings(record=True) as w:
-            utf8 = b"\xc3\xa9"
-            soup = self.soup(utf8, fromEncoding="utf8")
-        msg = str(w[0].message)
-        self.assertTrue("fromEncoding" in msg)
-        self.assertTrue("from_encoding" in msg)
-        self.assertEqual("utf8", soup.original_encoding)
-
-    def test_unrecognized_keyword_argument(self):
-        self.assertRaises(
-            TypeError, self.soup, "<a>", no_such_argument=True)
-
-class TestWarnings(SoupTest):
-
-    def test_disk_file_warning(self):
-        filehandle = tempfile.NamedTemporaryFile()
-        filename = filehandle.name
-        try:
-            with warnings.catch_warnings(record=True) as w:
-                soup = self.soup(filename)
-            msg = str(w[0].message)
-            self.assertTrue("looks like a filename" in msg)
-        finally:
-            filehandle.close()
-
-        # The file no longer exists, so Beautiful Soup will no longer issue the warning.
-        with warnings.catch_warnings(record=True) as w:
-            soup = self.soup(filename)
-        self.assertEqual(0, len(w))
-
-    def test_url_warning(self):
-        with warnings.catch_warnings(record=True) as w:
-            soup = self.soup("http://www.crummy.com/")
-        msg = str(w[0].message)
-        self.assertTrue("looks like a URL" in msg)
-
-        with warnings.catch_warnings(record=True) as w:
-            soup = self.soup("http://www.crummy.com/ is great")
-        self.assertEqual(0, len(w))
-
-class TestSelectiveParsing(SoupTest):
-
-    def test_parse_with_soupstrainer(self):
-        markup = "No<b>Yes</b><a>No<b>Yes <c>Yes</c></b>"
-        strainer = SoupStrainer("b")
-        soup = self.soup(markup, parse_only=strainer)
-        self.assertEqual(soup.encode(), b"<b>Yes</b><b>Yes <c>Yes</c></b>")
-
-
-class TestEntitySubstitution(unittest.TestCase):
-    """Standalone tests of the EntitySubstitution class."""
-    def setUp(self):
-        self.sub = EntitySubstitution
-
-    def test_simple_html_substitution(self):
-        # Unicode characters corresponding to named HTML entites
-        # are substituted, and no others.
-        s = "foo\u2200\N{SNOWMAN}\u00f5bar"
-        self.assertEqual(self.sub.substitute_html(s),
-                          "foo&forall;\N{SNOWMAN}&otilde;bar")
-
-    def test_smart_quote_substitution(self):
-        # MS smart quotes are a common source of frustration, so we
-        # give them a special test.
-        quotes = b"\x91\x92foo\x93\x94"
-        dammit = UnicodeDammit(quotes)
-        self.assertEqual(self.sub.substitute_html(dammit.markup),
-                          "&lsquo;&rsquo;foo&ldquo;&rdquo;")
-
-    def test_xml_converstion_includes_no_quotes_if_make_quoted_attribute_is_false(self):
-        s = 'Welcome to "my bar"'
-        self.assertEqual(self.sub.substitute_xml(s, False), s)
-
-    def test_xml_attribute_quoting_normally_uses_double_quotes(self):
-        self.assertEqual(self.sub.substitute_xml("Welcome", True),
-                          '"Welcome"')
-        self.assertEqual(self.sub.substitute_xml("Bob's Bar", True),
-                          '"Bob\'s Bar"')
-
-    def test_xml_attribute_quoting_uses_single_quotes_when_value_contains_double_quotes(self):
-        s = 'Welcome to "my bar"'
-        self.assertEqual(self.sub.substitute_xml(s, True),
-                          "'Welcome to \"my bar\"'")
-
-    def test_xml_attribute_quoting_escapes_single_quotes_when_value_contains_both_single_and_double_quotes(self):
-        s = 'Welcome to "Bob\'s Bar"'
-        self.assertEqual(
-            self.sub.substitute_xml(s, True),
-            '"Welcome to &quot;Bob\'s Bar&quot;"')
-
-    def test_xml_quotes_arent_escaped_when_value_is_not_being_quoted(self):
-        quoted = 'Welcome to "Bob\'s Bar"'
-        self.assertEqual(self.sub.substitute_xml(quoted), quoted)
-
-    def test_xml_quoting_handles_angle_brackets(self):
-        self.assertEqual(
-            self.sub.substitute_xml("foo<bar>"),
-            "foo&lt;bar&gt;")
-
-    def test_xml_quoting_handles_ampersands(self):
-        self.assertEqual(self.sub.substitute_xml("AT&T"), "AT&amp;T")
-
-    def test_xml_quoting_including_ampersands_when_they_are_part_of_an_entity(self):
-        self.assertEqual(
-            self.sub.substitute_xml("&Aacute;T&T"),
-            "&amp;Aacute;T&amp;T")
-
-    def test_xml_quoting_ignoring_ampersands_when_they_are_part_of_an_entity(self):
-        self.assertEqual(
-            self.sub.substitute_xml_containing_entities("&Aacute;T&T"),
-            "&Aacute;T&amp;T")
-
-    def test_quotes_not_html_substituted(self):
-        """There's no need to do this except inside attribute values."""
-        text = 'Bob\'s "bar"'
-        self.assertEqual(self.sub.substitute_html(text), text)
-
-
-class TestEncodingConversion(SoupTest):
-    # Test Beautiful Soup's ability to decode and encode from various
-    # encodings.
-
-    def setUp(self):
-        super(TestEncodingConversion, self).setUp()
-        self.unicode_data = '<html><head><meta charset="utf-8"/></head><body><foo>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</foo></body></html>'
-        self.utf8_data = self.unicode_data.encode("utf-8")
-        # Just so you know what it looks like.
-        self.assertEqual(
-            self.utf8_data,
-            b'<html><head><meta charset="utf-8"/></head><body><foo>Sacr\xc3\xa9 bleu!</foo></body></html>')
-
-    def test_ascii_in_unicode_out(self):
-        # ASCII input is converted to Unicode. The original_encoding
-        # attribute is set to 'utf-8', a superset of ASCII.
-        chardet = bs4.dammit.chardet_dammit
-        logging.disable(logging.WARNING)
-        try:
-            def noop(str):
-                return None
-            # Disable chardet, which will realize that the ASCII is ASCII.
-            bs4.dammit.chardet_dammit = noop
-            ascii = b"<foo>a</foo>"
-            soup_from_ascii = self.soup(ascii)
-            unicode_output = soup_from_ascii.decode()
-            self.assertTrue(isinstance(unicode_output, str))
-            self.assertEqual(unicode_output, self.document_for(ascii.decode()))
-            self.assertEqual(soup_from_ascii.original_encoding.lower(), "utf-8")
-        finally:
-            logging.disable(logging.NOTSET)
-            bs4.dammit.chardet_dammit = chardet
-
-    def test_unicode_in_unicode_out(self):
-        # Unicode input is left alone. The original_encoding attribute
-        # is not set.
-        soup_from_unicode = self.soup(self.unicode_data)
-        self.assertEqual(soup_from_unicode.decode(), self.unicode_data)
-        self.assertEqual(soup_from_unicode.foo.string, 'Sacr\xe9 bleu!')
-        self.assertEqual(soup_from_unicode.original_encoding, None)
-
-    def test_utf8_in_unicode_out(self):
-        # UTF-8 input is converted to Unicode. The original_encoding
-        # attribute is set.
-        soup_from_utf8 = self.soup(self.utf8_data)
-        self.assertEqual(soup_from_utf8.decode(), self.unicode_data)
-        self.assertEqual(soup_from_utf8.foo.string, 'Sacr\xe9 bleu!')
-
-    def test_utf8_out(self):
-        # The internal data structures can be encoded as UTF-8.
-        soup_from_unicode = self.soup(self.unicode_data)
-        self.assertEqual(soup_from_unicode.encode('utf-8'), self.utf8_data)
-
-    @skipIf(
-        PYTHON_2_PRE_2_7 or PYTHON_3_PRE_3_2,
-        "Bad HTMLParser detected; skipping test of non-ASCII characters in attribute name.")
-    def test_attribute_name_containing_unicode_characters(self):
-        markup = '<div><a \N{SNOWMAN}="snowman"></a></div>'
-        self.assertEqual(self.soup(markup).div.encode("utf8"), markup.encode("utf8"))
-
-class TestUnicodeDammit(unittest.TestCase):
-    """Standalone tests of UnicodeDammit."""
-
-    def test_unicode_input(self):
-        markup = "I'm already Unicode! \N{SNOWMAN}"
-        dammit = UnicodeDammit(markup)
-        self.assertEqual(dammit.unicode_markup, markup)
-
-    def test_smart_quotes_to_unicode(self):
-        markup = b"<foo>\x91\x92\x93\x94</foo>"
-        dammit = UnicodeDammit(markup)
-        self.assertEqual(
-            dammit.unicode_markup, "<foo>\u2018\u2019\u201c\u201d</foo>")
-
-    def test_smart_quotes_to_xml_entities(self):
-        markup = b"<foo>\x91\x92\x93\x94</foo>"
-        dammit = UnicodeDammit(markup, smart_quotes_to="xml")
-        self.assertEqual(
-            dammit.unicode_markup, "<foo>&#x2018;&#x2019;&#x201C;&#x201D;</foo>")
-
-    def test_smart_quotes_to_html_entities(self):
-        markup = b"<foo>\x91\x92\x93\x94</foo>"
-        dammit = UnicodeDammit(markup, smart_quotes_to="html")
-        self.assertEqual(
-            dammit.unicode_markup, "<foo>&lsquo;&rsquo;&ldquo;&rdquo;</foo>")
-
-    def test_smart_quotes_to_ascii(self):
-        markup = b"<foo>\x91\x92\x93\x94</foo>"
-        dammit = UnicodeDammit(markup, smart_quotes_to="ascii")
-        self.assertEqual(
-            dammit.unicode_markup, """<foo>''""</foo>""")
-
-    def test_detect_utf8(self):
-        utf8 = b"Sacr\xc3\xa9 bleu! \xe2\x98\x83"
-        dammit = UnicodeDammit(utf8)
-        self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
-        self.assertEqual(dammit.unicode_markup, 'Sacr\xe9 bleu! \N{SNOWMAN}')
-
-
-    def test_convert_hebrew(self):
-        hebrew = b"\xed\xe5\xec\xf9"
-        dammit = UnicodeDammit(hebrew, ["iso-8859-8"])
-        self.assertEqual(dammit.original_encoding.lower(), 'iso-8859-8')
-        self.assertEqual(dammit.unicode_markup, '\u05dd\u05d5\u05dc\u05e9')
-
-    def test_dont_see_smart_quotes_where_there_are_none(self):
-        utf_8 = b"\343\202\261\343\203\274\343\202\277\343\202\244 Watch"
-        dammit = UnicodeDammit(utf_8)
-        self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
-        self.assertEqual(dammit.unicode_markup.encode("utf-8"), utf_8)
-
-    def test_ignore_inappropriate_codecs(self):
-        utf8_data = "Räksmörgås".encode("utf-8")
-        dammit = UnicodeDammit(utf8_data, ["iso-8859-8"])
-        self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
-
-    def test_ignore_invalid_codecs(self):
-        utf8_data = "Räksmörgås".encode("utf-8")
-        for bad_encoding in ['.utf8', '...', 'utF---16.!']:
-            dammit = UnicodeDammit(utf8_data, [bad_encoding])
-            self.assertEqual(dammit.original_encoding.lower(), 'utf-8')
-
-    def test_exclude_encodings(self):
-        # This is UTF-8.
-        utf8_data = "Räksmörgås".encode("utf-8")
-
-        # But if we exclude UTF-8 from consideration, the guess is
-        # Windows-1252.
-        dammit = UnicodeDammit(utf8_data, exclude_encodings=["utf-8"])
-        self.assertEqual(dammit.original_encoding.lower(), 'windows-1252')
-
-        # And if we exclude that, there is no valid guess at all.
-        dammit = UnicodeDammit(
-            utf8_data, exclude_encodings=["utf-8", "windows-1252"])
-        self.assertEqual(dammit.original_encoding, None)
-
-    def test_encoding_detector_replaces_junk_in_encoding_name_with_replacement_character(self):
-        detected = EncodingDetector(
-            b'<?xml version="1.0" encoding="UTF-\xdb" ?>')
-        encodings = list(detected.encodings)
-        assert 'utf-\N{REPLACEMENT CHARACTER}' in encodings
-
-    def test_detect_html5_style_meta_tag(self):
-
-        for data in (
-            b'<html><meta charset="euc-jp" /></html>',
-            b"<html><meta charset='euc-jp' /></html>",
-            b"<html><meta charset=euc-jp /></html>",
-            b"<html><meta charset=euc-jp/></html>"):
-            dammit = UnicodeDammit(data, is_html=True)
-            self.assertEqual(
-                "euc-jp", dammit.original_encoding)
-
-    def test_last_ditch_entity_replacement(self):
-        # This is a UTF-8 document that contains bytestrings
-        # completely incompatible with UTF-8 (ie. encoded with some other
-        # encoding).
-        #
-        # Since there is no consistent encoding for the document,
-        # Unicode, Dammit will eventually encode the document as UTF-8
-        # and encode the incompatible characters as REPLACEMENT
-        # CHARACTER.
-        #
-        # If chardet is installed, it will detect that the document
-        # can be converted into ISO-8859-1 without errors. This happens
-        # to be the wrong encoding, but it is a consistent encoding, so the
-        # code we're testing here won't run.
-        #
-        # So we temporarily disable chardet if it's present.
-        doc = b"""\357\273\277<?xml version="1.0" encoding="UTF-8"?>
-<html><b>\330\250\330\252\330\261</b>
-<i>\310\322\321\220\312\321\355\344</i></html>"""
-        chardet = bs4.dammit.chardet_dammit
-        logging.disable(logging.WARNING)
-        try:
-            def noop(str):
-                return None
-            bs4.dammit.chardet_dammit = noop
-            dammit = UnicodeDammit(doc)
-            self.assertEqual(True, dammit.contains_replacement_characters)
-            self.assertTrue("\ufffd" in dammit.unicode_markup)
-
-            soup = BeautifulSoup(doc, "html.parser")
-            self.assertTrue(soup.contains_replacement_characters)
-        finally:
-            logging.disable(logging.NOTSET)
-            bs4.dammit.chardet_dammit = chardet
-
-    def test_byte_order_mark_removed(self):
-        # A document written in UTF-16LE will have its byte order marker stripped.
-        data = b'\xff\xfe<\x00a\x00>\x00\xe1\x00\xe9\x00<\x00/\x00a\x00>\x00'
-        dammit = UnicodeDammit(data)
-        self.assertEqual("<a>áé</a>", dammit.unicode_markup)
-        self.assertEqual("utf-16le", dammit.original_encoding)
-
-    def test_detwingle(self):
-        # Here's a UTF8 document.
-        utf8 = ("\N{SNOWMAN}" * 3).encode("utf8")
-
-        # Here's a Windows-1252 document.
-        windows_1252 = (
-            "\N{LEFT DOUBLE QUOTATION MARK}Hi, I like Windows!"
-            "\N{RIGHT DOUBLE QUOTATION MARK}").encode("windows_1252")
-
-        # Through some unholy alchemy, they've been stuck together.
-        doc = utf8 + windows_1252 + utf8
-
-        # The document can't be turned into UTF-8:
-        self.assertRaises(UnicodeDecodeError, doc.decode, "utf8")
-
-        # Unicode, Dammit thinks the whole document is Windows-1252,
-        # and decodes it into "☃☃☃“Hi, I like Windows!�☃☃☃"
-
-        # But if we run it through fix_embedded_windows_1252, it's fixed:
-
-        fixed = UnicodeDammit.detwingle(doc)
-        self.assertEqual(
-            "☃☃☃“Hi, I like Windows!�☃☃☃", fixed.decode("utf8"))
-
-    def test_detwingle_ignores_multibyte_characters(self):
-        # Each of these characters has a UTF-8 representation ending
-        # in \x93. \x93 is a smart quote if interpreted as
-        # Windows-1252. But our code knows to skip over multibyte
-        # UTF-8 characters, so they'll survive the process unscathed.
-        for tricky_unicode_char in (
-            "\N{LATIN SMALL LIGATURE OE}", # 2-byte char '\xc5\x93'
-            "\N{LATIN SUBSCRIPT SMALL LETTER X}", # 3-byte char '\xe2\x82\x93'
-            "\xf0\x90\x90\x93", # This is a CJK character, not sure which one.
-            ):
-            input = tricky_unicode_char.encode("utf8")
-            self.assertTrue(input.endswith(b'\x93'))
-            output = UnicodeDammit.detwingle(input)
-            self.assertEqual(output, input)
-
-class TestNamedspacedAttribute(SoupTest):
-
-    def test_name_may_be_none(self):
-        a = NamespacedAttribute("xmlns", None)
-        self.assertEqual(a, "xmlns")
-
-    def test_attribute_is_equivalent_to_colon_separated_string(self):
-        a = NamespacedAttribute("a", "b")
-        self.assertEqual("a:b", a)
-
-    def test_attributes_are_equivalent_if_prefix_and_name_identical(self):
-        a = NamespacedAttribute("a", "b", "c")
-        b = NamespacedAttribute("a", "b", "c")
-        self.assertEqual(a, b)
-
-        # The actual namespace is not considered.
-        c = NamespacedAttribute("a", "b", None)
-        self.assertEqual(a, c)
-
-        # But name and prefix are important.
-        d = NamespacedAttribute("a", "z", "c")
-        self.assertNotEqual(a, d)
-
-        e = NamespacedAttribute("z", "b", "c")
-        self.assertNotEqual(a, e)
-
-
-class TestAttributeValueWithCharsetSubstitution(unittest.TestCase):
-
-    def test_content_meta_attribute_value(self):
-        value = CharsetMetaAttributeValue("euc-jp")
-        self.assertEqual("euc-jp", value)
-        self.assertEqual("euc-jp", value.original_value)
-        self.assertEqual("utf8", value.encode("utf8"))
-
-
-    def test_content_meta_attribute_value(self):
-        value = ContentMetaAttributeValue("text/html; charset=euc-jp")
-        self.assertEqual("text/html; charset=euc-jp", value)
-        self.assertEqual("text/html; charset=euc-jp", value.original_value)
-        self.assertEqual("text/html; charset=utf8", value.encode("utf8"))
diff --git a/bitbake/lib/bs4/tests/test_tree.py b/bitbake/lib/bs4/tests/test_tree.py
deleted file mode 100644
index 8e5c66426e..0000000000
--- a/bitbake/lib/bs4/tests/test_tree.py
+++ /dev/null
@@ -1,2004 +0,0 @@
-# -*- coding: utf-8 -*-
-"""Tests for Beautiful Soup's tree traversal methods.
-
-The tree traversal methods are the main advantage of using Beautiful
-Soup over just using a parser.
-
-Different parsers will build different Beautiful Soup trees given the
-same markup, but all Beautiful Soup trees can be traversed with the
-methods tested here.
-"""
-
-import copy
-import pickle
-import re
-import warnings
-from bs4 import BeautifulSoup
-from bs4.builder import builder_registry
-from bs4.element import (
-    PY3K,
-    CData,
-    Comment,
-    Declaration,
-    Doctype,
-    NavigableString,
-    SoupStrainer,
-    Tag,
-)
-from bs4.testing import SoupTest
-
-XML_BUILDER_PRESENT = (builder_registry.lookup("xml") is not None)
-LXML_PRESENT = (builder_registry.lookup("lxml") is not None)
-
-class TreeTest(SoupTest):
-
-    def assertSelects(self, tags, should_match):
-        """Make sure that the given tags have the correct text.
-
-        This is used in tests that define a bunch of tags, each
-        containing a single string, and then select certain strings by
-        some mechanism.
-        """
-        self.assertEqual([tag.string for tag in tags], should_match)
-
-    def assertSelectsIDs(self, tags, should_match):
-        """Make sure that the given tags have the correct IDs.
-
-        This is used in tests that define a bunch of tags, each
-        containing a single string, and then select certain strings by
-        some mechanism.
-        """
-        self.assertEqual([tag['id'] for tag in tags], should_match)
-
-
-class TestFind(TreeTest):
-    """Basic tests of the find() method.
-
-    find() just calls find_all() with limit=1, so it's not tested all
-    that thouroughly here.
-    """
-
-    def test_find_tag(self):
-        soup = self.soup("<a>1</a><b>2</b><a>3</a><b>4</b>")
-        self.assertEqual(soup.find("b").string, "2")
-
-    def test_unicode_text_find(self):
-        soup = self.soup('<h1>Räksmörgås</h1>')
-        self.assertEqual(soup.find(string='Räksmörgås'), 'Räksmörgås')
-
-    def test_unicode_attribute_find(self):
-        soup = self.soup('<h1 id="Räksmörgås">here it is</h1>')
-        str(soup)
-        self.assertEqual("here it is", soup.find(id='Räksmörgås').text)
-
-
-    def test_find_everything(self):
-        """Test an optimization that finds all tags."""
-        soup = self.soup("<a>foo</a><b>bar</b>")
-        self.assertEqual(2, len(soup.find_all()))
-
-    def test_find_everything_with_name(self):
-        """Test an optimization that finds all tags with a given name."""
-        soup = self.soup("<a>foo</a><b>bar</b><a>baz</a>")
-        self.assertEqual(2, len(soup.find_all('a')))
-
-class TestFindAll(TreeTest):
-    """Basic tests of the find_all() method."""
-
-    def test_find_all_text_nodes(self):
-        """You can search the tree for text nodes."""
-        soup = self.soup("<html>Foo<b>bar</b>\xbb</html>")
-        # Exact match.
-        self.assertEqual(soup.find_all(string="bar"), ["bar"])
-        self.assertEqual(soup.find_all(text="bar"), ["bar"])
-        # Match any of a number of strings.
-        self.assertEqual(
-            soup.find_all(text=["Foo", "bar"]), ["Foo", "bar"])
-        # Match a regular expression.
-        self.assertEqual(soup.find_all(text=re.compile('.*')),
-                         ["Foo", "bar", '\xbb'])
-        # Match anything.
-        self.assertEqual(soup.find_all(text=True),
-                         ["Foo", "bar", '\xbb'])
-
-    def test_find_all_limit(self):
-        """You can limit the number of items returned by find_all."""
-        soup = self.soup("<a>1</a><a>2</a><a>3</a><a>4</a><a>5</a>")
-        self.assertSelects(soup.find_all('a', limit=3), ["1", "2", "3"])
-        self.assertSelects(soup.find_all('a', limit=1), ["1"])
-        self.assertSelects(
-            soup.find_all('a', limit=10), ["1", "2", "3", "4", "5"])
-
-        # A limit of 0 means no limit.
-        self.assertSelects(
-            soup.find_all('a', limit=0), ["1", "2", "3", "4", "5"])
-
-    def test_calling_a_tag_is_calling_findall(self):
-        soup = self.soup("<a>1</a><b>2<a id='foo'>3</a></b>")
-        self.assertSelects(soup('a', limit=1), ["1"])
-        self.assertSelects(soup.b(id="foo"), ["3"])
-
-    def test_find_all_with_self_referential_data_structure_does_not_cause_infinite_recursion(self):
-        soup = self.soup("<a></a>")
-        # Create a self-referential list.
-        l = []
-        l.append(l)
-
-        # Without special code in _normalize_search_value, this would cause infinite
-        # recursion.
-        self.assertEqual([], soup.find_all(l))
-
-    def test_find_all_resultset(self):
-        """All find_all calls return a ResultSet"""
-        soup = self.soup("<a></a>")
-        result = soup.find_all("a")
-        self.assertTrue(hasattr(result, "source"))
-
-        result = soup.find_all(True)
-        self.assertTrue(hasattr(result, "source"))
-
-        result = soup.find_all(text="foo")
-        self.assertTrue(hasattr(result, "source"))
-
-
-class TestFindAllBasicNamespaces(TreeTest):
-
-    def test_find_by_namespaced_name(self):
-        soup = self.soup('<mathml:msqrt>4</mathml:msqrt><a svg:fill="red">')
-        self.assertEqual("4", soup.find("mathml:msqrt").string)
-        self.assertEqual("a", soup.find(attrs= { "svg:fill" : "red" }).name)
-
-
-class TestFindAllByName(TreeTest):
-    """Test ways of finding tags by tag name."""
-
-    def setUp(self):
-        super(TreeTest, self).setUp()
-        self.tree =  self.soup("""<a>First tag.</a>
-                                  <b>Second tag.</b>
-                                  <c>Third <a>Nested tag.</a> tag.</c>""")
-
-    def test_find_all_by_tag_name(self):
-        # Find all the <a> tags.
-        self.assertSelects(
-            self.tree.find_all('a'), ['First tag.', 'Nested tag.'])
-
-    def test_find_all_by_name_and_text(self):
-        self.assertSelects(
-            self.tree.find_all('a', text='First tag.'), ['First tag.'])
-
-        self.assertSelects(
-            self.tree.find_all('a', text=True), ['First tag.', 'Nested tag.'])
-
-        self.assertSelects(
-            self.tree.find_all('a', text=re.compile("tag")),
-            ['First tag.', 'Nested tag.'])
-
-
-    def test_find_all_on_non_root_element(self):
-        # You can call find_all on any node, not just the root.
-        self.assertSelects(self.tree.c.find_all('a'), ['Nested tag.'])
-
-    def test_calling_element_invokes_find_all(self):
-        self.assertSelects(self.tree('a'), ['First tag.', 'Nested tag.'])
-
-    def test_find_all_by_tag_strainer(self):
-        self.assertSelects(
-            self.tree.find_all(SoupStrainer('a')),
-            ['First tag.', 'Nested tag.'])
-
-    def test_find_all_by_tag_names(self):
-        self.assertSelects(
-            self.tree.find_all(['a', 'b']),
-            ['First tag.', 'Second tag.', 'Nested tag.'])
-
-    def test_find_all_by_tag_dict(self):
-        self.assertSelects(
-            self.tree.find_all({'a' : True, 'b' : True}),
-            ['First tag.', 'Second tag.', 'Nested tag.'])
-
-    def test_find_all_by_tag_re(self):
-        self.assertSelects(
-            self.tree.find_all(re.compile('^[ab]$')),
-            ['First tag.', 'Second tag.', 'Nested tag.'])
-
-    def test_find_all_with_tags_matching_method(self):
-        # You can define an oracle method that determines whether
-        # a tag matches the search.
-        def id_matches_name(tag):
-            return tag.name == tag.get('id')
-
-        tree = self.soup("""<a id="a">Match 1.</a>
-                            <a id="1">Does not match.</a>
-                            <b id="b">Match 2.</a>""")
-
-        self.assertSelects(
-            tree.find_all(id_matches_name), ["Match 1.", "Match 2."])
-
-
-class TestFindAllByAttribute(TreeTest):
-
-    def test_find_all_by_attribute_name(self):
-        # You can pass in keyword arguments to find_all to search by
-        # attribute.
-        tree = self.soup("""
-                         <a id="first">Matching a.</a>
-                         <a id="second">
-                          Non-matching <b id="first">Matching b.</b>a.
-                         </a>""")
-        self.assertSelects(tree.find_all(id='first'),
-                           ["Matching a.", "Matching b."])
-
-    def test_find_all_by_utf8_attribute_value(self):
-        peace = "�ולש".encode("utf8")
-        data = '<a title="�ולש"></a>'.encode("utf8")
-        soup = self.soup(data)
-        self.assertEqual([soup.a], soup.find_all(title=peace))
-        self.assertEqual([soup.a], soup.find_all(title=peace.decode("utf8")))
-        self.assertEqual([soup.a], soup.find_all(title=[peace, "something else"]))
-
-    def test_find_all_by_attribute_dict(self):
-        # You can pass in a dictionary as the argument 'attrs'. This
-        # lets you search for attributes like 'name' (a fixed argument
-        # to find_all) and 'class' (a reserved word in Python.)
-        tree = self.soup("""
-                         <a name="name1" class="class1">Name match.</a>
-                         <a name="name2" class="class2">Class match.</a>
-                         <a name="name3" class="class3">Non-match.</a>
-                         <name1>A tag called 'name1'.</name1>
-                         """)
-
-        # This doesn't do what you want.
-        self.assertSelects(tree.find_all(name='name1'),
-                           ["A tag called 'name1'."])
-        # This does what you want.
-        self.assertSelects(tree.find_all(attrs={'name' : 'name1'}),
-                           ["Name match."])
-
-        self.assertSelects(tree.find_all(attrs={'class' : 'class2'}),
-                           ["Class match."])
-
-    def test_find_all_by_class(self):
-        tree = self.soup("""
-                         <a class="1">Class 1.</a>
-                         <a class="2">Class 2.</a>
-                         <b class="1">Class 1.</b>
-                         <c class="3 4">Class 3 and 4.</c>
-                         """)
-
-        # Passing in the class_ keyword argument will search against
-        # the 'class' attribute.
-        self.assertSelects(tree.find_all('a', class_='1'), ['Class 1.'])
-        self.assertSelects(tree.find_all('c', class_='3'), ['Class 3 and 4.'])
-        self.assertSelects(tree.find_all('c', class_='4'), ['Class 3 and 4.'])
-
-        # Passing in a string to 'attrs' will also search the CSS class.
-        self.assertSelects(tree.find_all('a', '1'), ['Class 1.'])
-        self.assertSelects(tree.find_all(attrs='1'), ['Class 1.', 'Class 1.'])
-        self.assertSelects(tree.find_all('c', '3'), ['Class 3 and 4.'])
-        self.assertSelects(tree.find_all('c', '4'), ['Class 3 and 4.'])
-
-    def test_find_by_class_when_multiple_classes_present(self):
-        tree = self.soup("<gar class='foo bar'>Found it</gar>")
-
-        f = tree.find_all("gar", class_=re.compile("o"))
-        self.assertSelects(f, ["Found it"])
-
-        f = tree.find_all("gar", class_=re.compile("a"))
-        self.assertSelects(f, ["Found it"])
-
-        # Since the class is not the string "foo bar", but the two
-        # strings "foo" and "bar", this will not find anything.
-        f = tree.find_all("gar", class_=re.compile("o b"))
-        self.assertSelects(f, [])
-
-    def test_find_all_with_non_dictionary_for_attrs_finds_by_class(self):
-        soup = self.soup("<a class='bar'>Found it</a>")
-
-        self.assertSelects(soup.find_all("a", re.compile("ba")), ["Found it"])
-
-        def big_attribute_value(value):
-            return len(value) > 3
-
-        self.assertSelects(soup.find_all("a", big_attribute_value), [])
-
-        def small_attribute_value(value):
-            return len(value) <= 3
-
-        self.assertSelects(
-            soup.find_all("a", small_attribute_value), ["Found it"])
-
-    def test_find_all_with_string_for_attrs_finds_multiple_classes(self):
-        soup = self.soup('<a class="foo bar"></a><a class="foo"></a>')
-        a, a2 = soup.find_all("a")
-        self.assertEqual([a, a2], soup.find_all("a", "foo"))
-        self.assertEqual([a], soup.find_all("a", "bar"))
-
-        # If you specify the class as a string that contains a
-        # space, only that specific value will be found.
-        self.assertEqual([a], soup.find_all("a", class_="foo bar"))
-        self.assertEqual([a], soup.find_all("a", "foo bar"))
-        self.assertEqual([], soup.find_all("a", "bar foo"))
-
-    def test_find_all_by_attribute_soupstrainer(self):
-        tree = self.soup("""
-                         <a id="first">Match.</a>
-                         <a id="second">Non-match.</a>""")
-
-        strainer = SoupStrainer(attrs={'id' : 'first'})
-        self.assertSelects(tree.find_all(strainer), ['Match.'])
-
-    def test_find_all_with_missing_atribute(self):
-        # You can pass in None as the value of an attribute to find_all.
-        # This will match tags that do not have that attribute set.
-        tree = self.soup("""<a id="1">ID present.</a>
-                            <a>No ID present.</a>
-                            <a id="">ID is empty.</a>""")
-        self.assertSelects(tree.find_all('a', id=None), ["No ID present."])
-
-    def test_find_all_with_defined_attribute(self):
-        # You can pass in None as the value of an attribute to find_all.
-        # This will match tags that have that attribute set to any value.
-        tree = self.soup("""<a id="1">ID present.</a>
-                            <a>No ID present.</a>
-                            <a id="">ID is empty.</a>""")
-        self.assertSelects(
-            tree.find_all(id=True), ["ID present.", "ID is empty."])
-
-    def test_find_all_with_numeric_attribute(self):
-        # If you search for a number, it's treated as a string.
-        tree = self.soup("""<a id=1>Unquoted attribute.</a>
-                            <a id="1">Quoted attribute.</a>""")
-
-        expected = ["Unquoted attribute.", "Quoted attribute."]
-        self.assertSelects(tree.find_all(id=1), expected)
-        self.assertSelects(tree.find_all(id="1"), expected)
-
-    def test_find_all_with_list_attribute_values(self):
-        # You can pass a list of attribute values instead of just one,
-        # and you'll get tags that match any of the values.
-        tree = self.soup("""<a id="1">1</a>
-                            <a id="2">2</a>
-                            <a id="3">3</a>
-                            <a>No ID.</a>""")
-        self.assertSelects(tree.find_all(id=["1", "3", "4"]),
-                           ["1", "3"])
-
-    def test_find_all_with_regular_expression_attribute_value(self):
-        # You can pass a regular expression as an attribute value, and
-        # you'll get tags whose values for that attribute match the
-        # regular expression.
-        tree = self.soup("""<a id="a">One a.</a>
-                            <a id="aa">Two as.</a>
-                            <a id="ab">Mixed as and bs.</a>
-                            <a id="b">One b.</a>
-                            <a>No ID.</a>""")
-
-        self.assertSelects(tree.find_all(id=re.compile("^a+$")),
-                           ["One a.", "Two as."])
-
-    def test_find_by_name_and_containing_string(self):
-        soup = self.soup("<b>foo</b><b>bar</b><a>foo</a>")
-        a = soup.a
-
-        self.assertEqual([a], soup.find_all("a", text="foo"))
-        self.assertEqual([], soup.find_all("a", text="bar"))
-        self.assertEqual([], soup.find_all("a", text="bar"))
-
-    def test_find_by_name_and_containing_string_when_string_is_buried(self):
-        soup = self.soup("<a>foo</a><a><b><c>foo</c></b></a>")
-        self.assertEqual(soup.find_all("a"), soup.find_all("a", text="foo"))
-
-    def test_find_by_attribute_and_containing_string(self):
-        soup = self.soup('<b id="1">foo</b><a id="2">foo</a>')
-        a = soup.a
-
-        self.assertEqual([a], soup.find_all(id=2, text="foo"))
-        self.assertEqual([], soup.find_all(id=1, text="bar"))
-
-
-
-
-class TestIndex(TreeTest):
-    """Test Tag.index"""
-    def test_index(self):
-        tree = self.soup("""<div>
-                            <a>Identical</a>
-                            <b>Not identical</b>
-                            <a>Identical</a>
-
-                            <c><d>Identical with child</d></c>
-                            <b>Also not identical</b>
-                            <c><d>Identical with child</d></c>
-                            </div>""")
-        div = tree.div
-        for i, element in enumerate(div.contents):
-            self.assertEqual(i, div.index(element))
-        self.assertRaises(ValueError, tree.index, 1)
-
-
-class TestParentOperations(TreeTest):
-    """Test navigation and searching through an element's parents."""
-
-    def setUp(self):
-        super(TestParentOperations, self).setUp()
-        self.tree = self.soup('''<ul id="empty"></ul>
-                                 <ul id="top">
-                                  <ul id="middle">
-                                   <ul id="bottom">
-                                    <b>Start here</b>
-                                   </ul>
-                                  </ul>''')
-        self.start = self.tree.b
-
-
-    def test_parent(self):
-        self.assertEqual(self.start.parent['id'], 'bottom')
-        self.assertEqual(self.start.parent.parent['id'], 'middle')
-        self.assertEqual(self.start.parent.parent.parent['id'], 'top')
-
-    def test_parent_of_top_tag_is_soup_object(self):
-        top_tag = self.tree.contents[0]
-        self.assertEqual(top_tag.parent, self.tree)
-
-    def test_soup_object_has_no_parent(self):
-        self.assertEqual(None, self.tree.parent)
-
-    def test_find_parents(self):
-        self.assertSelectsIDs(
-            self.start.find_parents('ul'), ['bottom', 'middle', 'top'])
-        self.assertSelectsIDs(
-            self.start.find_parents('ul', id="middle"), ['middle'])
-
-    def test_find_parent(self):
-        self.assertEqual(self.start.find_parent('ul')['id'], 'bottom')
-        self.assertEqual(self.start.find_parent('ul', id='top')['id'], 'top')
-
-    def test_parent_of_text_element(self):
-        text = self.tree.find(text="Start here")
-        self.assertEqual(text.parent.name, 'b')
-
-    def test_text_element_find_parent(self):
-        text = self.tree.find(text="Start here")
-        self.assertEqual(text.find_parent('ul')['id'], 'bottom')
-
-    def test_parent_generator(self):
-        parents = [parent['id'] for parent in self.start.parents
-                   if parent is not None and 'id' in parent.attrs]
-        self.assertEqual(parents, ['bottom', 'middle', 'top'])
-
-
-class ProximityTest(TreeTest):
-
-    def setUp(self):
-        super(TreeTest, self).setUp()
-        self.tree = self.soup(
-            '<html id="start"><head></head><body><b id="1">One</b><b id="2">Two</b><b id="3">Three</b></body></html>')
-
-
-class TestNextOperations(ProximityTest):
-
-    def setUp(self):
-        super(TestNextOperations, self).setUp()
-        self.start = self.tree.b
-
-    def test_next(self):
-        self.assertEqual(self.start.next_element, "One")
-        self.assertEqual(self.start.next_element.next_element['id'], "2")
-
-    def test_next_of_last_item_is_none(self):
-        last = self.tree.find(text="Three")
-        self.assertEqual(last.next_element, None)
-
-    def test_next_of_root_is_none(self):
-        # The document root is outside the next/previous chain.
-        self.assertEqual(self.tree.next_element, None)
-
-    def test_find_all_next(self):
-        self.assertSelects(self.start.find_all_next('b'), ["Two", "Three"])
-        self.start.find_all_next(id=3)
-        self.assertSelects(self.start.find_all_next(id=3), ["Three"])
-
-    def test_find_next(self):
-        self.assertEqual(self.start.find_next('b')['id'], '2')
-        self.assertEqual(self.start.find_next(text="Three"), "Three")
-
-    def test_find_next_for_text_element(self):
-        text = self.tree.find(text="One")
-        self.assertEqual(text.find_next("b").string, "Two")
-        self.assertSelects(text.find_all_next("b"), ["Two", "Three"])
-
-    def test_next_generator(self):
-        start = self.tree.find(text="Two")
-        successors = [node for node in start.next_elements]
-        # There are two successors: the final <b> tag and its text contents.
-        tag, contents = successors
-        self.assertEqual(tag['id'], '3')
-        self.assertEqual(contents, "Three")
-
-class TestPreviousOperations(ProximityTest):
-
-    def setUp(self):
-        super(TestPreviousOperations, self).setUp()
-        self.end = self.tree.find(text="Three")
-
-    def test_previous(self):
-        self.assertEqual(self.end.previous_element['id'], "3")
-        self.assertEqual(self.end.previous_element.previous_element, "Two")
-
-    def test_previous_of_first_item_is_none(self):
-        first = self.tree.find('html')
-        self.assertEqual(first.previous_element, None)
-
-    def test_previous_of_root_is_none(self):
-        # The document root is outside the next/previous chain.
-        # XXX This is broken!
-        #self.assertEqual(self.tree.previous_element, None)
-        pass
-
-    def test_find_all_previous(self):
-        # The <b> tag containing the "Three" node is the predecessor
-        # of the "Three" node itself, which is why "Three" shows up
-        # here.
-        self.assertSelects(
-            self.end.find_all_previous('b'), ["Three", "Two", "One"])
-        self.assertSelects(self.end.find_all_previous(id=1), ["One"])
-
-    def test_find_previous(self):
-        self.assertEqual(self.end.find_previous('b')['id'], '3')
-        self.assertEqual(self.end.find_previous(text="One"), "One")
-
-    def test_find_previous_for_text_element(self):
-        text = self.tree.find(text="Three")
-        self.assertEqual(text.find_previous("b").string, "Three")
-        self.assertSelects(
-            text.find_all_previous("b"), ["Three", "Two", "One"])
-
-    def test_previous_generator(self):
-        start = self.tree.find(text="One")
-        predecessors = [node for node in start.previous_elements]
-
-        # There are four predecessors: the <b> tag containing "One"
-        # the <body> tag, the <head> tag, and the <html> tag.
-        b, body, head, html = predecessors
-        self.assertEqual(b['id'], '1')
-        self.assertEqual(body.name, "body")
-        self.assertEqual(head.name, "head")
-        self.assertEqual(html.name, "html")
-
-
-class SiblingTest(TreeTest):
-
-    def setUp(self):
-        super(SiblingTest, self).setUp()
-        markup = '''<html>
-                    <span id="1">
-                     <span id="1.1"></span>
-                    </span>
-                    <span id="2">
-                     <span id="2.1"></span>
-                    </span>
-                    <span id="3">
-                     <span id="3.1"></span>
-                    </span>
-                    <span id="4"></span>
-                    </html>'''
-        # All that whitespace looks good but makes the tests more
-        # difficult. Get rid of it.
-        markup = re.compile("\n\s*").sub("", markup)
-        self.tree = self.soup(markup)
-
-
-class TestNextSibling(SiblingTest):
-
-    def setUp(self):
-        super(TestNextSibling, self).setUp()
-        self.start = self.tree.find(id="1")
-
-    def test_next_sibling_of_root_is_none(self):
-        self.assertEqual(self.tree.next_sibling, None)
-
-    def test_next_sibling(self):
-        self.assertEqual(self.start.next_sibling['id'], '2')
-        self.assertEqual(self.start.next_sibling.next_sibling['id'], '3')
-
-        # Note the difference between next_sibling and next_element.
-        self.assertEqual(self.start.next_element['id'], '1.1')
-
-    def test_next_sibling_may_not_exist(self):
-        self.assertEqual(self.tree.html.next_sibling, None)
-
-        nested_span = self.tree.find(id="1.1")
-        self.assertEqual(nested_span.next_sibling, None)
-
-        last_span = self.tree.find(id="4")
-        self.assertEqual(last_span.next_sibling, None)
-
-    def test_find_next_sibling(self):
-        self.assertEqual(self.start.find_next_sibling('span')['id'], '2')
-
-    def test_next_siblings(self):
-        self.assertSelectsIDs(self.start.find_next_siblings("span"),
-                              ['2', '3', '4'])
-
-        self.assertSelectsIDs(self.start.find_next_siblings(id='3'), ['3'])
-
-    def test_next_sibling_for_text_element(self):
-        soup = self.soup("Foo<b>bar</b>baz")
-        start = soup.find(text="Foo")
-        self.assertEqual(start.next_sibling.name, 'b')
-        self.assertEqual(start.next_sibling.next_sibling, 'baz')
-
-        self.assertSelects(start.find_next_siblings('b'), ['bar'])
-        self.assertEqual(start.find_next_sibling(text="baz"), "baz")
-        self.assertEqual(start.find_next_sibling(text="nonesuch"), None)
-
-
-class TestPreviousSibling(SiblingTest):
-
-    def setUp(self):
-        super(TestPreviousSibling, self).setUp()
-        self.end = self.tree.find(id="4")
-
-    def test_previous_sibling_of_root_is_none(self):
-        self.assertEqual(self.tree.previous_sibling, None)
-
-    def test_previous_sibling(self):
-        self.assertEqual(self.end.previous_sibling['id'], '3')
-        self.assertEqual(self.end.previous_sibling.previous_sibling['id'], '2')
-
-        # Note the difference between previous_sibling and previous_element.
-        self.assertEqual(self.end.previous_element['id'], '3.1')
-
-    def test_previous_sibling_may_not_exist(self):
-        self.assertEqual(self.tree.html.previous_sibling, None)
-
-        nested_span = self.tree.find(id="1.1")
-        self.assertEqual(nested_span.previous_sibling, None)
-
-        first_span = self.tree.find(id="1")
-        self.assertEqual(first_span.previous_sibling, None)
-
-    def test_find_previous_sibling(self):
-        self.assertEqual(self.end.find_previous_sibling('span')['id'], '3')
-
-    def test_previous_siblings(self):
-        self.assertSelectsIDs(self.end.find_previous_siblings("span"),
-                              ['3', '2', '1'])
-
-        self.assertSelectsIDs(self.end.find_previous_siblings(id='1'), ['1'])
-
-    def test_previous_sibling_for_text_element(self):
-        soup = self.soup("Foo<b>bar</b>baz")
-        start = soup.find(text="baz")
-        self.assertEqual(start.previous_sibling.name, 'b')
-        self.assertEqual(start.previous_sibling.previous_sibling, 'Foo')
-
-        self.assertSelects(start.find_previous_siblings('b'), ['bar'])
-        self.assertEqual(start.find_previous_sibling(text="Foo"), "Foo")
-        self.assertEqual(start.find_previous_sibling(text="nonesuch"), None)
-
-
-class TestTagCreation(SoupTest):
-    """Test the ability to create new tags."""
-    def test_new_tag(self):
-        soup = self.soup("")
-        new_tag = soup.new_tag("foo", bar="baz")
-        self.assertTrue(isinstance(new_tag, Tag))
-        self.assertEqual("foo", new_tag.name)
-        self.assertEqual(dict(bar="baz"), new_tag.attrs)
-        self.assertEqual(None, new_tag.parent)
-
-    def test_tag_inherits_self_closing_rules_from_builder(self):
-        if XML_BUILDER_PRESENT:
-            xml_soup = BeautifulSoup("", "lxml-xml")
-            xml_br = xml_soup.new_tag("br")
-            xml_p = xml_soup.new_tag("p")
-
-            # Both the <br> and <p> tag are empty-element, just because
-            # they have no contents.
-            self.assertEqual(b"<br/>", xml_br.encode())
-            self.assertEqual(b"<p/>", xml_p.encode())
-
-        html_soup = BeautifulSoup("", "html.parser")
-        html_br = html_soup.new_tag("br")
-        html_p = html_soup.new_tag("p")
-
-        # The HTML builder users HTML's rules about which tags are
-        # empty-element tags, and the new tags reflect these rules.
-        self.assertEqual(b"<br/>", html_br.encode())
-        self.assertEqual(b"<p></p>", html_p.encode())
-
-    def test_new_string_creates_navigablestring(self):
-        soup = self.soup("")
-        s = soup.new_string("foo")
-        self.assertEqual("foo", s)
-        self.assertTrue(isinstance(s, NavigableString))
-
-    def test_new_string_can_create_navigablestring_subclass(self):
-        soup = self.soup("")
-        s = soup.new_string("foo", Comment)
-        self.assertEqual("foo", s)
-        self.assertTrue(isinstance(s, Comment))
-
-class TestTreeModification(SoupTest):
-
-    def test_attribute_modification(self):
-        soup = self.soup('<a id="1"></a>')
-        soup.a['id'] = 2
-        self.assertEqual(soup.decode(), self.document_for('<a id="2"></a>'))
-        del(soup.a['id'])
-        self.assertEqual(soup.decode(), self.document_for('<a></a>'))
-        soup.a['id2'] = 'foo'
-        self.assertEqual(soup.decode(), self.document_for('<a id2="foo"></a>'))
-
-    def test_new_tag_creation(self):
-        builder = builder_registry.lookup('html')()
-        soup = self.soup("<body></body>", builder=builder)
-        a = Tag(soup, builder, 'a')
-        ol = Tag(soup, builder, 'ol')
-        a['href'] = 'http://foo.com/'
-        soup.body.insert(0, a)
-        soup.body.insert(1, ol)
-        self.assertEqual(
-            soup.body.encode(),
-            b'<body><a href="http://foo.com/"></a><ol></ol></body>')
-
-    def test_append_to_contents_moves_tag(self):
-        doc = """<p id="1">Don't leave me <b>here</b>.</p>
-                <p id="2">Don\'t leave!</p>"""
-        soup = self.soup(doc)
-        second_para = soup.find(id='2')
-        bold = soup.b
-
-        # Move the <b> tag to the end of the second paragraph.
-        soup.find(id='2').append(soup.b)
-
-        # The <b> tag is now a child of the second paragraph.
-        self.assertEqual(bold.parent, second_para)
-
-        self.assertEqual(
-            soup.decode(), self.document_for(
-                '<p id="1">Don\'t leave me .</p>\n'
-                '<p id="2">Don\'t leave!<b>here</b></p>'))
-
-    def test_replace_with_returns_thing_that_was_replaced(self):
-        text = "<a></a><b><c></c></b>"
-        soup = self.soup(text)
-        a = soup.a
-        new_a = a.replace_with(soup.c)
-        self.assertEqual(a, new_a)
-
-    def test_unwrap_returns_thing_that_was_replaced(self):
-        text = "<a><b></b><c></c></a>"
-        soup = self.soup(text)
-        a = soup.a
-        new_a = a.unwrap()
-        self.assertEqual(a, new_a)
-
-    def test_replace_with_and_unwrap_give_useful_exception_when_tag_has_no_parent(self):
-        soup = self.soup("<a><b>Foo</b></a><c>Bar</c>")
-        a = soup.a
-        a.extract()
-        self.assertEqual(None, a.parent)
-        self.assertRaises(ValueError, a.unwrap)
-        self.assertRaises(ValueError, a.replace_with, soup.c)
-
-    def test_replace_tag_with_itself(self):
-        text = "<a><b></b><c>Foo<d></d></c></a><a><e></e></a>"
-        soup = self.soup(text)
-        c = soup.c
-        soup.c.replace_with(c)
-        self.assertEqual(soup.decode(), self.document_for(text))
-
-    def test_replace_tag_with_its_parent_raises_exception(self):
-        text = "<a><b></b></a>"
-        soup = self.soup(text)
-        self.assertRaises(ValueError, soup.b.replace_with, soup.a)
-
-    def test_insert_tag_into_itself_raises_exception(self):
-        text = "<a><b></b></a>"
-        soup = self.soup(text)
-        self.assertRaises(ValueError, soup.a.insert, 0, soup.a)
-
-    def test_replace_with_maintains_next_element_throughout(self):
-        soup = self.soup('<p><a>one</a><b>three</b></p>')
-        a = soup.a
-        b = a.contents[0]
-        # Make it so the <a> tag has two text children.
-        a.insert(1, "two")
-
-        # Now replace each one with the empty string.
-        left, right = a.contents
-        left.replaceWith('')
-        right.replaceWith('')
-
-        # The <b> tag is still connected to the tree.
-        self.assertEqual("three", soup.b.string)
-
-    def test_replace_final_node(self):
-        soup = self.soup("<b>Argh!</b>")
-        soup.find(text="Argh!").replace_with("Hooray!")
-        new_text = soup.find(text="Hooray!")
-        b = soup.b
-        self.assertEqual(new_text.previous_element, b)
-        self.assertEqual(new_text.parent, b)
-        self.assertEqual(new_text.previous_element.next_element, new_text)
-        self.assertEqual(new_text.next_element, None)
-
-    def test_consecutive_text_nodes(self):
-        # A builder should never create two consecutive text nodes,
-        # but if you insert one next to another, Beautiful Soup will
-        # handle it correctly.
-        soup = self.soup("<a><b>Argh!</b><c></c></a>")
-        soup.b.insert(1, "Hooray!")
-
-        self.assertEqual(
-            soup.decode(), self.document_for(
-                "<a><b>Argh!Hooray!</b><c></c></a>"))
-
-        new_text = soup.find(text="Hooray!")
-        self.assertEqual(new_text.previous_element, "Argh!")
-        self.assertEqual(new_text.previous_element.next_element, new_text)
-
-        self.assertEqual(new_text.previous_sibling, "Argh!")
-        self.assertEqual(new_text.previous_sibling.next_sibling, new_text)
-
-        self.assertEqual(new_text.next_sibling, None)
-        self.assertEqual(new_text.next_element, soup.c)
-
-    def test_insert_string(self):
-        soup = self.soup("<a></a>")
-        soup.a.insert(0, "bar")
-        soup.a.insert(0, "foo")
-        # The string were added to the tag.
-        self.assertEqual(["foo", "bar"], soup.a.contents)
-        # And they were converted to NavigableStrings.
-        self.assertEqual(soup.a.contents[0].next_element, "bar")
-
-    def test_insert_tag(self):
-        builder = self.default_builder
-        soup = self.soup(
-            "<a><b>Find</b><c>lady!</c><d></d></a>", builder=builder)
-        magic_tag = Tag(soup, builder, 'magictag')
-        magic_tag.insert(0, "the")
-        soup.a.insert(1, magic_tag)
-
-        self.assertEqual(
-            soup.decode(), self.document_for(
-                "<a><b>Find</b><magictag>the</magictag><c>lady!</c><d></d></a>"))
-
-        # Make sure all the relationships are hooked up correctly.
-        b_tag = soup.b
-        self.assertEqual(b_tag.next_sibling, magic_tag)
-        self.assertEqual(magic_tag.previous_sibling, b_tag)
-
-        find = b_tag.find(text="Find")
-        self.assertEqual(find.next_element, magic_tag)
-        self.assertEqual(magic_tag.previous_element, find)
-
-        c_tag = soup.c
-        self.assertEqual(magic_tag.next_sibling, c_tag)
-        self.assertEqual(c_tag.previous_sibling, magic_tag)
-
-        the = magic_tag.find(text="the")
-        self.assertEqual(the.parent, magic_tag)
-        self.assertEqual(the.next_element, c_tag)
-        self.assertEqual(c_tag.previous_element, the)
-
-    def test_append_child_thats_already_at_the_end(self):
-        data = "<a><b></b></a>"
-        soup = self.soup(data)
-        soup.a.append(soup.b)
-        self.assertEqual(data, soup.decode())
-
-    def test_move_tag_to_beginning_of_parent(self):
-        data = "<a><b></b><c></c><d></d></a>"
-        soup = self.soup(data)
-        soup.a.insert(0, soup.d)
-        self.assertEqual("<a><d></d><b></b><c></c></a>", soup.decode())
-
-    def test_insert_works_on_empty_element_tag(self):
-        # This is a little strange, since most HTML parsers don't allow
-        # markup like this to come through. But in general, we don't
-        # know what the parser would or wouldn't have allowed, so
-        # I'm letting this succeed for now.
-        soup = self.soup("<br/>")
-        soup.br.insert(1, "Contents")
-        self.assertEqual(str(soup.br), "<br>Contents</br>")
-
-    def test_insert_before(self):
-        soup = self.soup("<a>foo</a><b>bar</b>")
-        soup.b.insert_before("BAZ")
-        soup.a.insert_before("QUUX")
-        self.assertEqual(
-            soup.decode(), self.document_for("QUUX<a>foo</a>BAZ<b>bar</b>"))
-
-        soup.a.insert_before(soup.b)
-        self.assertEqual(
-            soup.decode(), self.document_for("QUUX<b>bar</b><a>foo</a>BAZ"))
-
-    def test_insert_after(self):
-        soup = self.soup("<a>foo</a><b>bar</b>")
-        soup.b.insert_after("BAZ")
-        soup.a.insert_after("QUUX")
-        self.assertEqual(
-            soup.decode(), self.document_for("<a>foo</a>QUUX<b>bar</b>BAZ"))
-        soup.b.insert_after(soup.a)
-        self.assertEqual(
-            soup.decode(), self.document_for("QUUX<b>bar</b><a>foo</a>BAZ"))
-
-    def test_insert_after_raises_exception_if_after_has_no_meaning(self):
-        soup = self.soup("")
-        tag = soup.new_tag("a")
-        string = soup.new_string("")
-        self.assertRaises(ValueError, string.insert_after, tag)
-        self.assertRaises(NotImplementedError, soup.insert_after, tag)
-        self.assertRaises(ValueError, tag.insert_after, tag)
-
-    def test_insert_before_raises_notimplementederror_if_before_has_no_meaning(self):
-        soup = self.soup("")
-        tag = soup.new_tag("a")
-        string = soup.new_string("")
-        self.assertRaises(ValueError, string.insert_before, tag)
-        self.assertRaises(NotImplementedError, soup.insert_before, tag)
-        self.assertRaises(ValueError, tag.insert_before, tag)
-
-    def test_replace_with(self):
-        soup = self.soup(
-                "<p>There's <b>no</b> business like <b>show</b> business</p>")
-        no, show = soup.find_all('b')
-        show.replace_with(no)
-        self.assertEqual(
-            soup.decode(),
-            self.document_for(
-                "<p>There's  business like <b>no</b> business</p>"))
-
-        self.assertEqual(show.parent, None)
-        self.assertEqual(no.parent, soup.p)
-        self.assertEqual(no.next_element, "no")
-        self.assertEqual(no.next_sibling, " business")
-
-    def test_replace_first_child(self):
-        data = "<a><b></b><c></c></a>"
-        soup = self.soup(data)
-        soup.b.replace_with(soup.c)
-        self.assertEqual("<a><c></c></a>", soup.decode())
-
-    def test_replace_last_child(self):
-        data = "<a><b></b><c></c></a>"
-        soup = self.soup(data)
-        soup.c.replace_with(soup.b)
-        self.assertEqual("<a><b></b></a>", soup.decode())
-
-    def test_nested_tag_replace_with(self):
-        soup = self.soup(
-            """<a>We<b>reserve<c>the</c><d>right</d></b></a><e>to<f>refuse</f><g>service</g></e>""")
-
-        # Replace the entire <b> tag and its contents ("reserve the
-        # right") with the <f> tag ("refuse").
-        remove_tag = soup.b
-        move_tag = soup.f
-        remove_tag.replace_with(move_tag)
-
-        self.assertEqual(
-            soup.decode(), self.document_for(
-                "<a>We<f>refuse</f></a><e>to<g>service</g></e>"))
-
-        # The <b> tag is now an orphan.
-        self.assertEqual(remove_tag.parent, None)
-        self.assertEqual(remove_tag.find(text="right").next_element, None)
-        self.assertEqual(remove_tag.previous_element, None)
-        self.assertEqual(remove_tag.next_sibling, None)
-        self.assertEqual(remove_tag.previous_sibling, None)
-
-        # The <f> tag is now connected to the <a> tag.
-        self.assertEqual(move_tag.parent, soup.a)
-        self.assertEqual(move_tag.previous_element, "We")
-        self.assertEqual(move_tag.next_element.next_element, soup.e)
-        self.assertEqual(move_tag.next_sibling, None)
-
-        # The gap where the <f> tag used to be has been mended, and
-        # the word "to" is now connected to the <g> tag.
-        to_text = soup.find(text="to")
-        g_tag = soup.g
-        self.assertEqual(to_text.next_element, g_tag)
-        self.assertEqual(to_text.next_sibling, g_tag)
-        self.assertEqual(g_tag.previous_element, to_text)
-        self.assertEqual(g_tag.previous_sibling, to_text)
-
-    def test_unwrap(self):
-        tree = self.soup("""
-            <p>Unneeded <em>formatting</em> is unneeded</p>
-            """)
-        tree.em.unwrap()
-        self.assertEqual(tree.em, None)
-        self.assertEqual(tree.p.text, "Unneeded formatting is unneeded")
-
-    def test_wrap(self):
-        soup = self.soup("I wish I was bold.")
-        value = soup.string.wrap(soup.new_tag("b"))
-        self.assertEqual(value.decode(), "<b>I wish I was bold.</b>")
-        self.assertEqual(
-            soup.decode(), self.document_for("<b>I wish I was bold.</b>"))
-
-    def test_wrap_extracts_tag_from_elsewhere(self):
-        soup = self.soup("<b></b>I wish I was bold.")
-        soup.b.next_sibling.wrap(soup.b)
-        self.assertEqual(
-            soup.decode(), self.document_for("<b>I wish I was bold.</b>"))
-
-    def test_wrap_puts_new_contents_at_the_end(self):
-        soup = self.soup("<b>I like being bold.</b>I wish I was bold.")
-        soup.b.next_sibling.wrap(soup.b)
-        self.assertEqual(2, len(soup.b.contents))
-        self.assertEqual(
-            soup.decode(), self.document_for(
-                "<b>I like being bold.I wish I was bold.</b>"))
-
-    def test_extract(self):
-        soup = self.soup(
-            '<html><body>Some content. <div id="nav">Nav crap</div> More content.</body></html>')
-
-        self.assertEqual(len(soup.body.contents), 3)
-        extracted = soup.find(id="nav").extract()
-
-        self.assertEqual(
-            soup.decode(), "<html><body>Some content.  More content.</body></html>")
-        self.assertEqual(extracted.decode(), '<div id="nav">Nav crap</div>')
-
-        # The extracted tag is now an orphan.
-        self.assertEqual(len(soup.body.contents), 2)
-        self.assertEqual(extracted.parent, None)
-        self.assertEqual(extracted.previous_element, None)
-        self.assertEqual(extracted.next_element.next_element, None)
-
-        # The gap where the extracted tag used to be has been mended.
-        content_1 = soup.find(text="Some content. ")
-        content_2 = soup.find(text=" More content.")
-        self.assertEqual(content_1.next_element, content_2)
-        self.assertEqual(content_1.next_sibling, content_2)
-        self.assertEqual(content_2.previous_element, content_1)
-        self.assertEqual(content_2.previous_sibling, content_1)
-
-    def test_extract_distinguishes_between_identical_strings(self):
-        soup = self.soup("<a>foo</a><b>bar</b>")
-        foo_1 = soup.a.string
-        bar_1 = soup.b.string
-        foo_2 = soup.new_string("foo")
-        bar_2 = soup.new_string("bar")
-        soup.a.append(foo_2)
-        soup.b.append(bar_2)
-
-        # Now there are two identical strings in the <a> tag, and two
-        # in the <b> tag. Let's remove the first "foo" and the second
-        # "bar".
-        foo_1.extract()
-        bar_2.extract()
-        self.assertEqual(foo_2, soup.a.string)
-        self.assertEqual(bar_2, soup.b.string)
-
-    def test_extract_multiples_of_same_tag(self):
-        soup = self.soup("""
-<html>
-<head>
-<script>foo</script>
-</head>
-<body>
- <script>bar</script>
- <a></a>
-</body>
-<script>baz</script>
-</html>""")
-        [soup.script.extract() for i in soup.find_all("script")]
-        self.assertEqual("<body>\n\n<a></a>\n</body>", str(soup.body))
-
-
-    def test_extract_works_when_element_is_surrounded_by_identical_strings(self):
-        soup = self.soup(
- '<html>\n'
- '<body>hi</body>\n'
- '</html>')
-        soup.find('body').extract()
-        self.assertEqual(None, soup.find('body'))
-
-
-    def test_clear(self):
-        """Tag.clear()"""
-        soup = self.soup("<p><a>String <em>Italicized</em></a> and another</p>")
-        # clear using extract()
-        a = soup.a
-        soup.p.clear()
-        self.assertEqual(len(soup.p.contents), 0)
-        self.assertTrue(hasattr(a, "contents"))
-
-        # clear using decompose()
-        em = a.em
-        a.clear(decompose=True)
-        self.assertEqual(0, len(em.contents))
-
-    def test_string_set(self):
-        """Tag.string = 'string'"""
-        soup = self.soup("<a></a> <b><c></c></b>")
-        soup.a.string = "foo"
-        self.assertEqual(soup.a.contents, ["foo"])
-        soup.b.string = "bar"
-        self.assertEqual(soup.b.contents, ["bar"])
-
-    def test_string_set_does_not_affect_original_string(self):
-        soup = self.soup("<a><b>foo</b><c>bar</c>")
-        soup.b.string = soup.c.string
-        self.assertEqual(soup.a.encode(), b"<a><b>bar</b><c>bar</c></a>")
-
-    def test_set_string_preserves_class_of_string(self):
-        soup = self.soup("<a></a>")
-        cdata = CData("foo")
-        soup.a.string = cdata
-        self.assertTrue(isinstance(soup.a.string, CData))
-
-class TestElementObjects(SoupTest):
-    """Test various features of element objects."""
-
-    def test_len(self):
-        """The length of an element is its number of children."""
-        soup = self.soup("<top>1<b>2</b>3</top>")
-
-        # The BeautifulSoup object itself contains one element: the
-        # <top> tag.
-        self.assertEqual(len(soup.contents), 1)
-        self.assertEqual(len(soup), 1)
-
-        # The <top> tag contains three elements: the text node "1", the
-        # <b> tag, and the text node "3".
-        self.assertEqual(len(soup.top), 3)
-        self.assertEqual(len(soup.top.contents), 3)
-
-    def test_member_access_invokes_find(self):
-        """Accessing a Python member .foo invokes find('foo')"""
-        soup = self.soup('<b><i></i></b>')
-        self.assertEqual(soup.b, soup.find('b'))
-        self.assertEqual(soup.b.i, soup.find('b').find('i'))
-        self.assertEqual(soup.a, None)
-
-    def test_deprecated_member_access(self):
-        soup = self.soup('<b><i></i></b>')
-        with warnings.catch_warnings(record=True) as w:
-            tag = soup.bTag
-        self.assertEqual(soup.b, tag)
-        self.assertEqual(
-            '.bTag is deprecated, use .find("b") instead.',
-            str(w[0].message))
-
-    def test_has_attr(self):
-        """has_attr() checks for the presence of an attribute.
-
-        Please note note: has_attr() is different from
-        __in__. has_attr() checks the tag's attributes and __in__
-        checks the tag's chidlren.
-        """
-        soup = self.soup("<foo attr='bar'>")
-        self.assertTrue(soup.foo.has_attr('attr'))
-        self.assertFalse(soup.foo.has_attr('attr2'))
-
-
-    def test_attributes_come_out_in_alphabetical_order(self):
-        markup = '<b a="1" z="5" m="3" f="2" y="4"></b>'
-        self.assertSoupEquals(markup, '<b a="1" f="2" m="3" y="4" z="5"></b>')
-
-    def test_string(self):
-        # A tag that contains only a text node makes that node
-        # available as .string.
-        soup = self.soup("<b>foo</b>")
-        self.assertEqual(soup.b.string, 'foo')
-
-    def test_empty_tag_has_no_string(self):
-        # A tag with no children has no .stirng.
-        soup = self.soup("<b></b>")
-        self.assertEqual(soup.b.string, None)
-
-    def test_tag_with_multiple_children_has_no_string(self):
-        # A tag with no children has no .string.
-        soup = self.soup("<a>foo<b></b><b></b></b>")
-        self.assertEqual(soup.b.string, None)
-
-        soup = self.soup("<a>foo<b></b>bar</b>")
-        self.assertEqual(soup.b.string, None)
-
-        # Even if all the children are strings, due to trickery,
-        # it won't work--but this would be a good optimization.
-        soup = self.soup("<a>foo</b>")
-        soup.a.insert(1, "bar")
-        self.assertEqual(soup.a.string, None)
-
-    def test_tag_with_recursive_string_has_string(self):
-        # A tag with a single child which has a .string inherits that
-        # .string.
-        soup = self.soup("<a><b>foo</b></a>")
-        self.assertEqual(soup.a.string, "foo")
-        self.assertEqual(soup.string, "foo")
-
-    def test_lack_of_string(self):
-        """Only a tag containing a single text node has a .string."""
-        soup = self.soup("<b>f<i>e</i>o</b>")
-        self.assertFalse(soup.b.string)
-
-        soup = self.soup("<b></b>")
-        self.assertFalse(soup.b.string)
-
-    def test_all_text(self):
-        """Tag.text and Tag.get_text(sep=u"") -> all child text, concatenated"""
-        soup = self.soup("<a>a<b>r</b>   <r> t </r></a>")
-        self.assertEqual(soup.a.text, "ar  t ")
-        self.assertEqual(soup.a.get_text(strip=True), "art")
-        self.assertEqual(soup.a.get_text(","), "a,r, , t ")
-        self.assertEqual(soup.a.get_text(",", strip=True), "a,r,t")
-
-    def test_get_text_ignores_comments(self):
-        soup = self.soup("foo<!--IGNORE-->bar")
-        self.assertEqual(soup.get_text(), "foobar")
-
-        self.assertEqual(
-            soup.get_text(types=(NavigableString, Comment)), "fooIGNOREbar")
-        self.assertEqual(
-            soup.get_text(types=None), "fooIGNOREbar")
-
-    def test_all_strings_ignores_comments(self):
-        soup = self.soup("foo<!--IGNORE-->bar")
-        self.assertEqual(['foo', 'bar'], list(soup.strings))
-
-class TestCDAtaListAttributes(SoupTest):
-
-    """Testing cdata-list attributes like 'class'.
-    """
-    def test_single_value_becomes_list(self):
-        soup = self.soup("<a class='foo'>")
-        self.assertEqual(["foo"],soup.a['class'])
-
-    def test_multiple_values_becomes_list(self):
-        soup = self.soup("<a class='foo bar'>")
-        self.assertEqual(["foo", "bar"], soup.a['class'])
-
-    def test_multiple_values_separated_by_weird_whitespace(self):
-        soup = self.soup("<a class='foo\tbar\nbaz'>")
-        self.assertEqual(["foo", "bar", "baz"],soup.a['class'])
-
-    def test_attributes_joined_into_string_on_output(self):
-        soup = self.soup("<a class='foo\tbar'>")
-        self.assertEqual(b'<a class="foo bar"></a>', soup.a.encode())
-
-    def test_accept_charset(self):
-        soup = self.soup('<form accept-charset="ISO-8859-1 UTF-8">')
-        self.assertEqual(['ISO-8859-1', 'UTF-8'], soup.form['accept-charset'])
-
-    def test_cdata_attribute_applying_only_to_one_tag(self):
-        data = '<a accept-charset="ISO-8859-1 UTF-8"></a>'
-        soup = self.soup(data)
-        # We saw in another test that accept-charset is a cdata-list
-        # attribute for the <form> tag. But it's not a cdata-list
-        # attribute for any other tag.
-        self.assertEqual('ISO-8859-1 UTF-8', soup.a['accept-charset'])
-
-    def test_string_has_immutable_name_property(self):
-        string = self.soup("s").string
-        self.assertEqual(None, string.name)
-        def t():
-            string.name = 'foo'
-        self.assertRaises(AttributeError, t)
-
-class TestPersistence(SoupTest):
-    "Testing features like pickle and deepcopy."
-
-    def setUp(self):
-        super(TestPersistence, self).setUp()
-        self.page = """<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"
-"http://www.w3.org/TR/REC-html40/transitional.dtd">
-<html>
-<head>
-<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
-<title>Beautiful Soup: We called him Tortoise because he taught us.</title>
-<link rev="made" href="mailto:leonardr@segfault.org">
-<meta name="Description" content="Beautiful Soup: an HTML parser optimized for screen-scraping.">
-<meta name="generator" content="Markov Approximation 1.4 (module: leonardr)">
-<meta name="author" content="Leonard Richardson">
-</head>
-<body>
-<a href="foo">foo</a>
-<a href="foo"><b>bar</b></a>
-</body>
-</html>"""
-        self.tree = self.soup(self.page)
-
-    def test_pickle_and_unpickle_identity(self):
-        # Pickling a tree, then unpickling it, yields a tree identical
-        # to the original.
-        dumped = pickle.dumps(self.tree, 2)
-        loaded = pickle.loads(dumped)
-        self.assertEqual(loaded.__class__, BeautifulSoup)
-        self.assertEqual(loaded.decode(), self.tree.decode())
-
-    def test_deepcopy_identity(self):
-        # Making a deepcopy of a tree yields an identical tree.
-        copied = copy.deepcopy(self.tree)
-        self.assertEqual(copied.decode(), self.tree.decode())
-
-    def test_unicode_pickle(self):
-        # A tree containing Unicode characters can be pickled.
-        html = "<b>\N{SNOWMAN}</b>"
-        soup = self.soup(html)
-        dumped = pickle.dumps(soup, pickle.HIGHEST_PROTOCOL)
-        loaded = pickle.loads(dumped)
-        self.assertEqual(loaded.decode(), soup.decode())
-
-    def test_copy_navigablestring_is_not_attached_to_tree(self):
-        html = "<b>Foo<a></a></b><b>Bar</b>"
-        soup = self.soup(html)
-        s1 = soup.find(string="Foo")
-        s2 = copy.copy(s1)
-        self.assertEqual(s1, s2)
-        self.assertEqual(None, s2.parent)
-        self.assertEqual(None, s2.next_element)
-        self.assertNotEqual(None, s1.next_sibling)
-        self.assertEqual(None, s2.next_sibling)
-        self.assertEqual(None, s2.previous_element)
-
-    def test_copy_navigablestring_subclass_has_same_type(self):
-        html = "<b><!--Foo--></b>"
-        soup = self.soup(html)
-        s1 = soup.string
-        s2 = copy.copy(s1)
-        self.assertEqual(s1, s2)
-        self.assertTrue(isinstance(s2, Comment))
-
-    def test_copy_entire_soup(self):
-        html = "<div><b>Foo<a></a></b><b>Bar</b></div>end"
-        soup = self.soup(html)
-        soup_copy = copy.copy(soup)
-        self.assertEqual(soup, soup_copy)
-
-    def test_copy_tag_copies_contents(self):
-        html = "<div><b>Foo<a></a></b><b>Bar</b></div>end"
-        soup = self.soup(html)
-        div = soup.div
-        div_copy = copy.copy(div)
-
-        # The two tags look the same, and evaluate to equal.
-        self.assertEqual(str(div), str(div_copy))
-        self.assertEqual(div, div_copy)
-
-        # But they're not the same object.
-        self.assertFalse(div is div_copy)
-
-        # And they don't have the same relation to the parse tree. The
-        # copy is not associated with a parse tree at all.
-        self.assertEqual(None, div_copy.parent)
-        self.assertEqual(None, div_copy.previous_element)
-        self.assertEqual(None, div_copy.find(string='Bar').next_element)
-        self.assertNotEqual(None, div.find(string='Bar').next_element)
-
-class TestSubstitutions(SoupTest):
-
-    def test_default_formatter_is_minimal(self):
-        markup = "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
-        soup = self.soup(markup)
-        decoded = soup.decode(formatter="minimal")
-        # The < is converted back into &lt; but the e-with-acute is left alone.
-        self.assertEqual(
-            decoded,
-            self.document_for(
-                "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"))
-
-    def test_formatter_html(self):
-        markup = "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
-        soup = self.soup(markup)
-        decoded = soup.decode(formatter="html")
-        self.assertEqual(
-            decoded,
-            self.document_for("<b>&lt;&lt;Sacr&eacute; bleu!&gt;&gt;</b>"))
-
-    def test_formatter_minimal(self):
-        markup = "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
-        soup = self.soup(markup)
-        decoded = soup.decode(formatter="minimal")
-        # The < is converted back into &lt; but the e-with-acute is left alone.
-        self.assertEqual(
-            decoded,
-            self.document_for(
-                "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"))
-
-    def test_formatter_null(self):
-        markup = "<b>&lt;&lt;Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!&gt;&gt;</b>"
-        soup = self.soup(markup)
-        decoded = soup.decode(formatter=None)
-        # Neither the angle brackets nor the e-with-acute are converted.
-        # This is not valid HTML, but it's what the user wanted.
-        self.assertEqual(decoded,
-                          self.document_for("<b><<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>></b>"))
-
-    def test_formatter_custom(self):
-        markup = "<b>&lt;foo&gt;</b><b>bar</b>"
-        soup = self.soup(markup)
-        decoded = soup.decode(formatter = lambda x: x.upper())
-        # Instead of normal entity conversion code, the custom
-        # callable is called on every string.
-        self.assertEqual(
-            decoded,
-            self.document_for("<b><FOO></b><b>BAR</b>"))
-
-    def test_formatter_is_run_on_attribute_values(self):
-        markup = '<a href="http://a.com?a=b&c=é">e</a>'
-        soup = self.soup(markup)
-        a = soup.a
-
-        expect_minimal = '<a href="http://a.com?a=b&amp;c=é">e</a>'
-
-        self.assertEqual(expect_minimal, a.decode())
-        self.assertEqual(expect_minimal, a.decode(formatter="minimal"))
-
-        expect_html = '<a href="http://a.com?a=b&amp;c=&eacute;">e</a>'
-        self.assertEqual(expect_html, a.decode(formatter="html"))
-
-        self.assertEqual(markup, a.decode(formatter=None))
-        expect_upper = '<a href="HTTP://A.COM?A=B&C=É">E</a>'
-        self.assertEqual(expect_upper, a.decode(formatter=lambda x: x.upper()))
-
-    def test_formatter_skips_script_tag_for_html_documents(self):
-        doc = """
-  <script type="text/javascript">
-   console.log("< < hey > > ");
-  </script>
-"""
-        encoded = BeautifulSoup(doc, 'html.parser').encode()
-        self.assertTrue(b"< < hey > >" in encoded)
-
-    def test_formatter_skips_style_tag_for_html_documents(self):
-        doc = """
-  <style type="text/css">
-   console.log("< < hey > > ");
-  </style>
-"""
-        encoded = BeautifulSoup(doc, 'html.parser').encode()
-        self.assertTrue(b"< < hey > >" in encoded)
-
-    def test_prettify_leaves_preformatted_text_alone(self):
-        soup = self.soup("<div>  foo  <pre>  \tbar\n  \n  </pre>  baz  ")
-        # Everything outside the <pre> tag is reformatted, but everything
-        # inside is left alone.
-        self.assertEqual(
-            '<div>\n foo\n <pre>  \tbar\n  \n  </pre>\n baz\n</div>',
-            soup.div.prettify())
-
-    def test_prettify_accepts_formatter(self):
-        soup = BeautifulSoup("<html><body>foo</body></html>", 'html.parser')
-        pretty = soup.prettify(formatter = lambda x: x.upper())
-        self.assertTrue("FOO" in pretty)
-
-    def test_prettify_outputs_unicode_by_default(self):
-        soup = self.soup("<a></a>")
-        self.assertEqual(str, type(soup.prettify()))
-
-    def test_prettify_can_encode_data(self):
-        soup = self.soup("<a></a>")
-        self.assertEqual(bytes, type(soup.prettify("utf-8")))
-
-    def test_html_entity_substitution_off_by_default(self):
-        markup = "<b>Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!</b>"
-        soup = self.soup(markup)
-        encoded = soup.b.encode("utf-8")
-        self.assertEqual(encoded, markup.encode('utf-8'))
-
-    def test_encoding_substitution(self):
-        # Here's the <meta> tag saying that a document is
-        # encoded in Shift-JIS.
-        meta_tag = ('<meta content="text/html; charset=x-sjis" '
-                    'http-equiv="Content-type"/>')
-        soup = self.soup(meta_tag)
-
-        # Parse the document, and the charset apprears unchanged.
-        self.assertEqual(soup.meta['content'], 'text/html; charset=x-sjis')
-
-        # Encode the document into some encoding, and the encoding is
-        # substituted into the meta tag.
-        utf_8 = soup.encode("utf-8")
-        self.assertTrue(b"charset=utf-8" in utf_8)
-
-        euc_jp = soup.encode("euc_jp")
-        self.assertTrue(b"charset=euc_jp" in euc_jp)
-
-        shift_jis = soup.encode("shift-jis")
-        self.assertTrue(b"charset=shift-jis" in shift_jis)
-
-        utf_16_u = soup.encode("utf-16").decode("utf-16")
-        self.assertTrue("charset=utf-16" in utf_16_u)
-
-    def test_encoding_substitution_doesnt_happen_if_tag_is_strained(self):
-        markup = ('<head><meta content="text/html; charset=x-sjis" '
-                    'http-equiv="Content-type"/></head><pre>foo</pre>')
-
-        # Beautiful Soup used to try to rewrite the meta tag even if the
-        # meta tag got filtered out by the strainer. This test makes
-        # sure that doesn't happen.
-        strainer = SoupStrainer('pre')
-        soup = self.soup(markup, parse_only=strainer)
-        self.assertEqual(soup.contents[0].name, 'pre')
-
-class TestEncoding(SoupTest):
-    """Test the ability to encode objects into strings."""
-
-    def test_unicode_string_can_be_encoded(self):
-        html = "<b>\N{SNOWMAN}</b>"
-        soup = self.soup(html)
-        self.assertEqual(soup.b.string.encode("utf-8"),
-                          "\N{SNOWMAN}".encode("utf-8"))
-
-    def test_tag_containing_unicode_string_can_be_encoded(self):
-        html = "<b>\N{SNOWMAN}</b>"
-        soup = self.soup(html)
-        self.assertEqual(
-            soup.b.encode("utf-8"), html.encode("utf-8"))
-
-    def test_encoding_substitutes_unrecognized_characters_by_default(self):
-        html = "<b>\N{SNOWMAN}</b>"
-        soup = self.soup(html)
-        self.assertEqual(soup.b.encode("ascii"), b"<b>&#9731;</b>")
-
-    def test_encoding_can_be_made_strict(self):
-        html = "<b>\N{SNOWMAN}</b>"
-        soup = self.soup(html)
-        self.assertRaises(
-            UnicodeEncodeError, soup.encode, "ascii", errors="strict")
-
-    def test_decode_contents(self):
-        html = "<b>\N{SNOWMAN}</b>"
-        soup = self.soup(html)
-        self.assertEqual("\N{SNOWMAN}", soup.b.decode_contents())
-
-    def test_encode_contents(self):
-        html = "<b>\N{SNOWMAN}</b>"
-        soup = self.soup(html)
-        self.assertEqual(
-            "\N{SNOWMAN}".encode("utf8"), soup.b.encode_contents(
-                encoding="utf8"))
-
-    def test_deprecated_renderContents(self):
-        html = "<b>\N{SNOWMAN}</b>"
-        soup = self.soup(html)
-        self.assertEqual(
-            "\N{SNOWMAN}".encode("utf8"), soup.b.renderContents())
-
-    def test_repr(self):
-        html = "<b>\N{SNOWMAN}</b>"
-        soup = self.soup(html)
-        if PY3K:
-            self.assertEqual(html, repr(soup))
-        else:
-            self.assertEqual(b'<b>\\u2603</b>', repr(soup))
-
-class TestNavigableStringSubclasses(SoupTest):
-
-    def test_cdata(self):
-        # None of the current builders turn CDATA sections into CData
-        # objects, but you can create them manually.
-        soup = self.soup("")
-        cdata = CData("foo")
-        soup.insert(1, cdata)
-        self.assertEqual(str(soup), "<![CDATA[foo]]>")
-        self.assertEqual(soup.find(text="foo"), "foo")
-        self.assertEqual(soup.contents[0], "foo")
-
-    def test_cdata_is_never_formatted(self):
-        """Text inside a CData object is passed into the formatter.
-
-        But the return value is ignored.
-        """
-
-        self.count = 0
-        def increment(*args):
-            self.count += 1
-            return "BITTER FAILURE"
-
-        soup = self.soup("")
-        cdata = CData("<><><>")
-        soup.insert(1, cdata)
-        self.assertEqual(
-            b"<![CDATA[<><><>]]>", soup.encode(formatter=increment))
-        self.assertEqual(1, self.count)
-
-    def test_doctype_ends_in_newline(self):
-        # Unlike other NavigableString subclasses, a DOCTYPE always ends
-        # in a newline.
-        doctype = Doctype("foo")
-        soup = self.soup("")
-        soup.insert(1, doctype)
-        self.assertEqual(soup.encode(), b"<!DOCTYPE foo>\n")
-
-    def test_declaration(self):
-        d = Declaration("foo")
-        self.assertEqual("<?foo?>", d.output_ready())
-
-class TestSoupSelector(TreeTest):
-
-    HTML = """
-<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN"
-"http://www.w3.org/TR/html4/strict.dtd">
-<html>
-<head>
-<title>The title</title>
-<link rel="stylesheet" href="blah.css" type="text/css" id="l1">
-</head>
-<body>
-<custom-dashed-tag class="dashed" id="dash1">Hello there.</custom-dashed-tag>
-<div id="main" class="fancy">
-<div id="inner">
-<h1 id="header1">An H1</h1>
-<p>Some text</p>
-<p class="onep" id="p1">Some more text</p>
-<h2 id="header2">An H2</h2>
-<p class="class1 class2 class3" id="pmulti">Another</p>
-<a href="http://bob.example.org/" rel="friend met" id="bob">Bob</a>
-<h2 id="header3">Another H2</h2>
-<a id="me" href="http://simonwillison.net/" rel="me">me</a>
-<span class="s1">
-<a href="#" id="s1a1">span1a1</a>
-<a href="#" id="s1a2">span1a2 <span id="s1a2s1">test</span></a>
-<span class="span2">
-<a href="#" id="s2a1">span2a1</a>
-</span>
-<span class="span3"></span>
-<custom-dashed-tag class="dashed" id="dash2"/>
-<div data-tag="dashedvalue" id="data1"/>
-</span>
-</div>
-<x id="xid">
-<z id="zida"/>
-<z id="zidab"/>
-<z id="zidac"/>
-</x>
-<y id="yid">
-<z id="zidb"/>
-</y>
-<p lang="en" id="lang-en">English</p>
-<p lang="en-gb" id="lang-en-gb">English UK</p>
-<p lang="en-us" id="lang-en-us">English US</p>
-<p lang="fr" id="lang-fr">French</p>
-</div>
-
-<div id="footer">
-</div>
-"""
-
-    def setUp(self):
-        self.soup = BeautifulSoup(self.HTML, 'html.parser')
-
-    def assertSelects(self, selector, expected_ids):
-        el_ids = [el['id'] for el in self.soup.select(selector)]
-        el_ids.sort()
-        expected_ids.sort()
-        self.assertEqual(expected_ids, el_ids,
-            "Selector %s, expected [%s], got [%s]" % (
-                selector, ', '.join(expected_ids), ', '.join(el_ids)
-            )
-        )
-
-    assertSelect = assertSelects
-
-    def assertSelectMultiple(self, *tests):
-        for selector, expected_ids in tests:
-            self.assertSelect(selector, expected_ids)
-
-    def test_one_tag_one(self):
-        els = self.soup.select('title')
-        self.assertEqual(len(els), 1)
-        self.assertEqual(els[0].name, 'title')
-        self.assertEqual(els[0].contents, ['The title'])
-
-    def test_one_tag_many(self):
-        els = self.soup.select('div')
-        self.assertEqual(len(els), 4)
-        for div in els:
-            self.assertEqual(div.name, 'div')
-
-        el = self.soup.select_one('div')
-        self.assertEqual('main', el['id'])
-
-    def test_select_one_returns_none_if_no_match(self):
-        match = self.soup.select_one('nonexistenttag')
-        self.assertEqual(None, match)
-
-
-    def test_tag_in_tag_one(self):
-        els = self.soup.select('div div')
-        self.assertSelects('div div', ['inner', 'data1'])
-
-    def test_tag_in_tag_many(self):
-        for selector in ('html div', 'html body div', 'body div'):
-            self.assertSelects(selector, ['data1', 'main', 'inner', 'footer'])
-
-    def test_tag_no_match(self):
-        self.assertEqual(len(self.soup.select('del')), 0)
-
-    def test_invalid_tag(self):
-        self.assertRaises(ValueError, self.soup.select, 'tag%t')
-
-    def test_select_dashed_tag_ids(self):
-        self.assertSelects('custom-dashed-tag', ['dash1', 'dash2'])
-
-    def test_select_dashed_by_id(self):
-        dashed = self.soup.select('custom-dashed-tag[id=\"dash2\"]')
-        self.assertEqual(dashed[0].name, 'custom-dashed-tag')
-        self.assertEqual(dashed[0]['id'], 'dash2')
-
-    def test_dashed_tag_text(self):
-        self.assertEqual(self.soup.select('body > custom-dashed-tag')[0].text, 'Hello there.')
-
-    def test_select_dashed_matches_find_all(self):
-        self.assertEqual(self.soup.select('custom-dashed-tag'), self.soup.find_all('custom-dashed-tag'))
-
-    def test_header_tags(self):
-        self.assertSelectMultiple(
-            ('h1', ['header1']),
-            ('h2', ['header2', 'header3']),
-        )
-
-    def test_class_one(self):
-        for selector in ('.onep', 'p.onep', 'html p.onep'):
-            els = self.soup.select(selector)
-            self.assertEqual(len(els), 1)
-            self.assertEqual(els[0].name, 'p')
-            self.assertEqual(els[0]['class'], ['onep'])
-
-    def test_class_mismatched_tag(self):
-        els = self.soup.select('div.onep')
-        self.assertEqual(len(els), 0)
-
-    def test_one_id(self):
-        for selector in ('div#inner', '#inner', 'div div#inner'):
-            self.assertSelects(selector, ['inner'])
-
-    def test_bad_id(self):
-        els = self.soup.select('#doesnotexist')
-        self.assertEqual(len(els), 0)
-
-    def test_items_in_id(self):
-        els = self.soup.select('div#inner p')
-        self.assertEqual(len(els), 3)
-        for el in els:
-            self.assertEqual(el.name, 'p')
-        self.assertEqual(els[1]['class'], ['onep'])
-        self.assertFalse(els[0].has_attr('class'))
-
-    def test_a_bunch_of_emptys(self):
-        for selector in ('div#main del', 'div#main div.oops', 'div div#main'):
-            self.assertEqual(len(self.soup.select(selector)), 0)
-
-    def test_multi_class_support(self):
-        for selector in ('.class1', 'p.class1', '.class2', 'p.class2',
-            '.class3', 'p.class3', 'html p.class2', 'div#inner .class2'):
-            self.assertSelects(selector, ['pmulti'])
-
-    def test_multi_class_selection(self):
-        for selector in ('.class1.class3', '.class3.class2',
-                         '.class1.class2.class3'):
-            self.assertSelects(selector, ['pmulti'])
-
-    def test_child_selector(self):
-        self.assertSelects('.s1 > a', ['s1a1', 's1a2'])
-        self.assertSelects('.s1 > a span', ['s1a2s1'])
-
-    def test_child_selector_id(self):
-        self.assertSelects('.s1 > a#s1a2 span', ['s1a2s1'])
-
-    def test_attribute_equals(self):
-        self.assertSelectMultiple(
-            ('p[class="onep"]', ['p1']),
-            ('p[id="p1"]', ['p1']),
-            ('[class="onep"]', ['p1']),
-            ('[id="p1"]', ['p1']),
-            ('link[rel="stylesheet"]', ['l1']),
-            ('link[type="text/css"]', ['l1']),
-            ('link[href="blah.css"]', ['l1']),
-            ('link[href="no-blah.css"]', []),
-            ('[rel="stylesheet"]', ['l1']),
-            ('[type="text/css"]', ['l1']),
-            ('[href="blah.css"]', ['l1']),
-            ('[href="no-blah.css"]', []),
-            ('p[href="no-blah.css"]', []),
-            ('[href="no-blah.css"]', []),
-        )
-
-    def test_attribute_tilde(self):
-        self.assertSelectMultiple(
-            ('p[class~="class1"]', ['pmulti']),
-            ('p[class~="class2"]', ['pmulti']),
-            ('p[class~="class3"]', ['pmulti']),
-            ('[class~="class1"]', ['pmulti']),
-            ('[class~="class2"]', ['pmulti']),
-            ('[class~="class3"]', ['pmulti']),
-            ('a[rel~="friend"]', ['bob']),
-            ('a[rel~="met"]', ['bob']),
-            ('[rel~="friend"]', ['bob']),
-            ('[rel~="met"]', ['bob']),
-        )
-
-    def test_attribute_startswith(self):
-        self.assertSelectMultiple(
-            ('[rel^="style"]', ['l1']),
-            ('link[rel^="style"]', ['l1']),
-            ('notlink[rel^="notstyle"]', []),
-            ('[rel^="notstyle"]', []),
-            ('link[rel^="notstyle"]', []),
-            ('link[href^="bla"]', ['l1']),
-            ('a[href^="http://"]', ['bob', 'me']),
-            ('[href^="http://"]', ['bob', 'me']),
-            ('[id^="p"]', ['pmulti', 'p1']),
-            ('[id^="m"]', ['me', 'main']),
-            ('div[id^="m"]', ['main']),
-            ('a[id^="m"]', ['me']),
-            ('div[data-tag^="dashed"]', ['data1'])
-        )
-
-    def test_attribute_endswith(self):
-        self.assertSelectMultiple(
-            ('[href$=".css"]', ['l1']),
-            ('link[href$=".css"]', ['l1']),
-            ('link[id$="1"]', ['l1']),
-            ('[id$="1"]', ['data1', 'l1', 'p1', 'header1', 's1a1', 's2a1', 's1a2s1', 'dash1']),
-            ('div[id$="1"]', ['data1']),
-            ('[id$="noending"]', []),
-        )
-
-    def test_attribute_contains(self):
-        self.assertSelectMultiple(
-            # From test_attribute_startswith
-            ('[rel*="style"]', ['l1']),
-            ('link[rel*="style"]', ['l1']),
-            ('notlink[rel*="notstyle"]', []),
-            ('[rel*="notstyle"]', []),
-            ('link[rel*="notstyle"]', []),
-            ('link[href*="bla"]', ['l1']),
-            ('[href*="http://"]', ['bob', 'me']),
-            ('[id*="p"]', ['pmulti', 'p1']),
-            ('div[id*="m"]', ['main']),
-            ('a[id*="m"]', ['me']),
-            # From test_attribute_endswith
-            ('[href*=".css"]', ['l1']),
-            ('link[href*=".css"]', ['l1']),
-            ('link[id*="1"]', ['l1']),
-            ('[id*="1"]', ['data1', 'l1', 'p1', 'header1', 's1a1', 's1a2', 's2a1', 's1a2s1', 'dash1']),
-            ('div[id*="1"]', ['data1']),
-            ('[id*="noending"]', []),
-            # New for this test
-            ('[href*="."]', ['bob', 'me', 'l1']),
-            ('a[href*="."]', ['bob', 'me']),
-            ('link[href*="."]', ['l1']),
-            ('div[id*="n"]', ['main', 'inner']),
-            ('div[id*="nn"]', ['inner']),
-            ('div[data-tag*="edval"]', ['data1'])
-        )
-
-    def test_attribute_exact_or_hypen(self):
-        self.assertSelectMultiple(
-            ('p[lang|="en"]', ['lang-en', 'lang-en-gb', 'lang-en-us']),
-            ('[lang|="en"]', ['lang-en', 'lang-en-gb', 'lang-en-us']),
-            ('p[lang|="fr"]', ['lang-fr']),
-            ('p[lang|="gb"]', []),
-        )
-
-    def test_attribute_exists(self):
-        self.assertSelectMultiple(
-            ('[rel]', ['l1', 'bob', 'me']),
-            ('link[rel]', ['l1']),
-            ('a[rel]', ['bob', 'me']),
-            ('[lang]', ['lang-en', 'lang-en-gb', 'lang-en-us', 'lang-fr']),
-            ('p[class]', ['p1', 'pmulti']),
-            ('[blah]', []),
-            ('p[blah]', []),
-            ('div[data-tag]', ['data1'])
-        )
-
-    def test_unsupported_pseudoclass(self):
-        self.assertRaises(
-            NotImplementedError, self.soup.select, "a:no-such-pseudoclass")
-
-        self.assertRaises(
-            NotImplementedError, self.soup.select, "a:nth-of-type(a)")
-
-
-    def test_nth_of_type(self):
-        # Try to select first paragraph
-        els = self.soup.select('div#inner p:nth-of-type(1)')
-        self.assertEqual(len(els), 1)
-        self.assertEqual(els[0].string, 'Some text')
-
-        # Try to select third paragraph
-        els = self.soup.select('div#inner p:nth-of-type(3)')
-        self.assertEqual(len(els), 1)
-        self.assertEqual(els[0].string, 'Another')
-
-        # Try to select (non-existent!) fourth paragraph
-        els = self.soup.select('div#inner p:nth-of-type(4)')
-        self.assertEqual(len(els), 0)
-
-        # Pass in an invalid value.
-        self.assertRaises(
-            ValueError, self.soup.select, 'div p:nth-of-type(0)')
-
-    def test_nth_of_type_direct_descendant(self):
-        els = self.soup.select('div#inner > p:nth-of-type(1)')
-        self.assertEqual(len(els), 1)
-        self.assertEqual(els[0].string, 'Some text')
-
-    def test_id_child_selector_nth_of_type(self):
-        self.assertSelects('#inner > p:nth-of-type(2)', ['p1'])
-
-    def test_select_on_element(self):
-        # Other tests operate on the tree; this operates on an element
-        # within the tree.
-        inner = self.soup.find("div", id="main")
-        selected = inner.select("div")
-        # The <div id="inner"> tag was selected. The <div id="footer">
-        # tag was not.
-        self.assertSelectsIDs(selected, ['inner', 'data1'])
-
-    def test_overspecified_child_id(self):
-        self.assertSelects(".fancy #inner", ['inner'])
-        self.assertSelects(".normal #inner", [])
-
-    def test_adjacent_sibling_selector(self):
-        self.assertSelects('#p1 + h2', ['header2'])
-        self.assertSelects('#p1 + h2 + p', ['pmulti'])
-        self.assertSelects('#p1 + #header2 + .class1', ['pmulti'])
-        self.assertEqual([], self.soup.select('#p1 + p'))
-
-    def test_general_sibling_selector(self):
-        self.assertSelects('#p1 ~ h2', ['header2', 'header3'])
-        self.assertSelects('#p1 ~ #header2', ['header2'])
-        self.assertSelects('#p1 ~ h2 + a', ['me'])
-        self.assertSelects('#p1 ~ h2 + [rel="me"]', ['me'])
-        self.assertEqual([], self.soup.select('#inner ~ h2'))
-
-    def test_dangling_combinator(self):
-        self.assertRaises(ValueError, self.soup.select, 'h1 >')
-
-    def test_sibling_combinator_wont_select_same_tag_twice(self):
-        self.assertSelects('p[lang] ~ p', ['lang-en-gb', 'lang-en-us', 'lang-fr'])
-
-    # Test the selector grouping operator (the comma)
-    def test_multiple_select(self):
-        self.assertSelects('x, y', ['xid', 'yid'])
-
-    def test_multiple_select_with_no_space(self):
-        self.assertSelects('x,y', ['xid', 'yid'])
-
-    def test_multiple_select_with_more_space(self):
-        self.assertSelects('x,    y', ['xid', 'yid'])
-
-    def test_multiple_select_duplicated(self):
-        self.assertSelects('x, x', ['xid'])
-
-    def test_multiple_select_sibling(self):
-        self.assertSelects('x, y ~ p[lang=fr]', ['xid', 'lang-fr'])
-
-    def test_multiple_select_tag_and_direct_descendant(self):
-        self.assertSelects('x, y > z', ['xid', 'zidb'])
-
-    def test_multiple_select_direct_descendant_and_tags(self):
-        self.assertSelects('div > x, y, z', ['xid', 'yid', 'zida', 'zidb', 'zidab', 'zidac'])
-
-    def test_multiple_select_indirect_descendant(self):
-        self.assertSelects('div x,y,  z', ['xid', 'yid', 'zida', 'zidb', 'zidab', 'zidac'])
-
-    def test_invalid_multiple_select(self):
-        self.assertRaises(ValueError, self.soup.select, ',x, y')
-        self.assertRaises(ValueError, self.soup.select, 'x,,y')
-
-    def test_multiple_select_attrs(self):
-        self.assertSelects('p[lang=en], p[lang=en-gb]', ['lang-en', 'lang-en-gb'])
-
-    def test_multiple_select_ids(self):
-        self.assertSelects('x, y > z[id=zida], z[id=zidab], z[id=zidb]', ['xid', 'zidb', 'zidab'])
-
-    def test_multiple_select_nested(self):
-        self.assertSelects('body > div > x, y > z', ['xid', 'zidb'])
-
-
-
diff --git a/bitbake/lib/codegen.py b/bitbake/lib/codegen.py
index 62a6748c47..018b283177 100644
--- a/bitbake/lib/codegen.py
+++ b/bitbake/lib/codegen.py
@@ -392,14 +392,8 @@ class SourceGenerator(NodeVisitor):
     def visit_Name(self, node):
         self.write(node.id)
 
-    def visit_Str(self, node):
-        self.write(repr(node.s))
-
-    def visit_Bytes(self, node):
-        self.write(repr(node.s))
-
-    def visit_Num(self, node):
-        self.write(repr(node.n))
+    def visit_Constant(self, node):
+        self.write(repr(node.value))
 
     def visit_Tuple(self, node):
         self.write('(')
diff --git a/bitbake/lib/hashserv/__init__.py b/bitbake/lib/hashserv/__init__.py
index 5f2e101e52..ac891e0174 100644
--- a/bitbake/lib/hashserv/__init__.py
+++ b/bitbake/lib/hashserv/__init__.py
@@ -5,129 +5,104 @@
 
 import asyncio
 from contextlib import closing
-import re
-import sqlite3
 import itertools
 import json
+from collections import namedtuple
+from urllib.parse import urlparse
+from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS
 
-UNIX_PREFIX = "unix://"
-
-ADDR_TYPE_UNIX = 0
-ADDR_TYPE_TCP = 1
-
-# The Python async server defaults to a 64K receive buffer, so we hardcode our
-# maximum chunk size. It would be better if the client and server reported to
-# each other what the maximum chunk sizes were, but that will slow down the
-# connection setup with a round trip delay so I'd rather not do that unless it
-# is necessary
-DEFAULT_MAX_CHUNK = 32 * 1024
-
-TABLE_DEFINITION = (
-    ("method", "TEXT NOT NULL"),
-    ("outhash", "TEXT NOT NULL"),
-    ("taskhash", "TEXT NOT NULL"),
-    ("unihash", "TEXT NOT NULL"),
-    ("created", "DATETIME"),
-
-    # Optional fields
-    ("owner", "TEXT"),
-    ("PN", "TEXT"),
-    ("PV", "TEXT"),
-    ("PR", "TEXT"),
-    ("task", "TEXT"),
-    ("outhash_siginfo", "TEXT"),
-)
-
-TABLE_COLUMNS = tuple(name for name, _ in TABLE_DEFINITION)
-
-def setup_database(database, sync=True):
-    db = sqlite3.connect(database)
-    db.row_factory = sqlite3.Row
-
-    with closing(db.cursor()) as cursor:
-        cursor.execute('''
-            CREATE TABLE IF NOT EXISTS tasks_v2 (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                %s
-                UNIQUE(method, outhash, taskhash)
-                )
-            ''' % " ".join("%s %s," % (name, typ) for name, typ in TABLE_DEFINITION))
-        cursor.execute('PRAGMA journal_mode = WAL')
-        cursor.execute('PRAGMA synchronous = %s' % ('NORMAL' if sync else 'OFF'))
-
-        # Drop old indexes
-        cursor.execute('DROP INDEX IF EXISTS taskhash_lookup')
-        cursor.execute('DROP INDEX IF EXISTS outhash_lookup')
-
-        # Create new indexes
-        cursor.execute('CREATE INDEX IF NOT EXISTS taskhash_lookup_v2 ON tasks_v2 (method, taskhash, created)')
-        cursor.execute('CREATE INDEX IF NOT EXISTS outhash_lookup_v2 ON tasks_v2 (method, outhash)')
-
-    return db
-
-
-def parse_address(addr):
-    if addr.startswith(UNIX_PREFIX):
-        return (ADDR_TYPE_UNIX, (addr[len(UNIX_PREFIX):],))
-    else:
-        m = re.match(r'\[(?P<host>[^\]]*)\]:(?P<port>\d+)$', addr)
-        if m is not None:
-            host = m.group('host')
-            port = m.group('port')
-        else:
-            host, port = addr.split(':')
+User = namedtuple("User", ("username", "permissions"))
 
-        return (ADDR_TYPE_TCP, (host, int(port)))
 
+def create_server(
+    addr,
+    dbname,
+    *,
+    sync=True,
+    upstream=None,
+    read_only=False,
+    db_username=None,
+    db_password=None,
+    anon_perms=None,
+    admin_username=None,
+    admin_password=None,
+    reuseport=False,
+):
+    def sqlite_engine():
+        from .sqlite import DatabaseEngine
 
-def chunkify(msg, max_chunk):
-    if len(msg) < max_chunk - 1:
-        yield ''.join((msg, "\n"))
-    else:
-        yield ''.join((json.dumps({
-                'chunk-stream': None
-            }), "\n"))
+        return DatabaseEngine(dbname, sync)
 
-        args = [iter(msg)] * (max_chunk - 1)
-        for m in map(''.join, itertools.zip_longest(*args, fillvalue='')):
-            yield ''.join(itertools.chain(m, "\n"))
-        yield "\n"
+    def sqlalchemy_engine():
+        from .sqlalchemy import DatabaseEngine
 
+        return DatabaseEngine(dbname, db_username, db_password)
 
-def create_server(addr, dbname, *, sync=True, upstream=None, read_only=False):
     from . import server
-    db = setup_database(dbname, sync=sync)
-    s = server.Server(db, upstream=upstream, read_only=read_only)
+
+    if "://" in dbname:
+        db_engine = sqlalchemy_engine()
+    else:
+        db_engine = sqlite_engine()
+
+    if anon_perms is None:
+        anon_perms = server.DEFAULT_ANON_PERMS
+
+    s = server.Server(
+        db_engine,
+        upstream=upstream,
+        read_only=read_only,
+        anon_perms=anon_perms,
+        admin_username=admin_username,
+        admin_password=admin_password,
+    )
 
     (typ, a) = parse_address(addr)
     if typ == ADDR_TYPE_UNIX:
         s.start_unix_server(*a)
+    elif typ == ADDR_TYPE_WS:
+        url = urlparse(a[0])
+        s.start_websocket_server(url.hostname, url.port, reuseport=reuseport)
     else:
-        s.start_tcp_server(*a)
+        s.start_tcp_server(*a, reuseport=reuseport)
 
     return s
 
 
-def create_client(addr):
+def create_client(addr, username=None, password=None):
     from . import client
-    c = client.Client()
 
-    (typ, a) = parse_address(addr)
-    if typ == ADDR_TYPE_UNIX:
-        c.connect_unix(*a)
-    else:
-        c.connect_tcp(*a)
+    c = client.Client(username, password)
+
+    try:
+        (typ, a) = parse_address(addr)
+        if typ == ADDR_TYPE_UNIX:
+            c.connect_unix(*a)
+        elif typ == ADDR_TYPE_WS:
+            c.connect_websocket(*a)
+        else:
+            c.connect_tcp(*a)
+        return c
+    except Exception as e:
+        c.close()
+        raise e
 
-    return c
 
-async def create_async_client(addr):
+async def create_async_client(addr, username=None, password=None):
     from . import client
-    c = client.AsyncClient()
 
-    (typ, a) = parse_address(addr)
-    if typ == ADDR_TYPE_UNIX:
-        await c.connect_unix(*a)
-    else:
-        await c.connect_tcp(*a)
+    c = client.AsyncClient(username, password)
+
+    try:
+        (typ, a) = parse_address(addr)
+        if typ == ADDR_TYPE_UNIX:
+            await c.connect_unix(*a)
+        elif typ == ADDR_TYPE_WS:
+            await c.connect_websocket(*a)
+        else:
+            await c.connect_tcp(*a)
 
-    return c
+        return c
+    except Exception as e:
+        await c.close()
+        raise e
diff --git a/bitbake/lib/hashserv/client.py b/bitbake/lib/hashserv/client.py
index e05c1eb568..8cb18050a6 100644
--- a/bitbake/lib/hashserv/client.py
+++ b/bitbake/lib/hashserv/client.py
@@ -3,231 +3,378 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-import asyncio
-import json
 import logging
 import socket
-import os
-from . import chunkify, DEFAULT_MAX_CHUNK, create_async_client
+import asyncio
+import bb.asyncrpc
+import json
+from . import create_async_client
 
 
 logger = logging.getLogger("hashserv.client")
 
 
-class HashConnectionError(Exception):
-    pass
+class Batch(object):
+    def __init__(self):
+        self.done = False
+        self.cond = asyncio.Condition()
+        self.pending = []
+        self.results = []
+        self.sent_count = 0
 
+    async def recv(self, socket):
+        while True:
+            async with self.cond:
+                await self.cond.wait_for(lambda: self.pending or self.done)
 
-class AsyncClient(object):
-    MODE_NORMAL = 0
-    MODE_GET_STREAM = 1
+                if not self.pending:
+                    if self.done:
+                        return
+                    continue
 
-    def __init__(self):
-        self.reader = None
-        self.writer = None
-        self.mode = self.MODE_NORMAL
-        self.max_chunk = DEFAULT_MAX_CHUNK
+            r = await socket.recv()
+            self.results.append(r)
 
-    async def connect_tcp(self, address, port):
-        async def connect_sock():
-            return await asyncio.open_connection(address, port)
+            async with self.cond:
+                self.pending.pop(0)
 
-        self._connect_sock = connect_sock
+    async def send(self, socket, msgs):
+        try:
+            # In the event of a restart due to a reconnect, all in-flight
+            # messages need to be resent first to keep to result count in sync
+            for m in self.pending:
+                await socket.send(m)
 
-    async def connect_unix(self, path):
-        async def connect_sock():
-            return await asyncio.open_unix_connection(path)
+            for m in msgs:
+                # Add the message to the pending list before attempting to send
+                # it so that if the send fails it will be retried
+                async with self.cond:
+                    self.pending.append(m)
+                    self.cond.notify()
+                    self.sent_count += 1
 
-        self._connect_sock = connect_sock
+                await socket.send(m)
 
-    async def connect(self):
-        if self.reader is None or self.writer is None:
-            (self.reader, self.writer) = await self._connect_sock()
+        finally:
+            async with self.cond:
+                self.done = True
+                self.cond.notify()
+
+    async def process(self, socket, msgs):
+        await asyncio.gather(
+            self.recv(socket),
+            self.send(socket, msgs),
+        )
 
-            self.writer.write("OEHASHEQUIV 1.1\n\n".encode("utf-8"))
-            await self.writer.drain()
+        if len(self.results) != self.sent_count:
+            raise ValueError(
+                f"Expected result count {len(self.results)}. Expected {self.sent_count}"
+            )
 
-            cur_mode = self.mode
-            self.mode = self.MODE_NORMAL
-            await self._set_mode(cur_mode)
+        return self.results
 
-    async def close(self):
-        self.reader = None
 
-        if self.writer is not None:
-            self.writer.close()
-            self.writer = None
+class AsyncClient(bb.asyncrpc.AsyncClient):
+    MODE_NORMAL = 0
+    MODE_GET_STREAM = 1
+    MODE_EXIST_STREAM = 2
+    MODE_MARK_STREAM = 3
 
-    async def _send_wrapper(self, proc):
-        count = 0
-        while True:
-            try:
-                await self.connect()
-                return await proc()
-            except (
-                OSError,
-                HashConnectionError,
-                json.JSONDecodeError,
-                UnicodeDecodeError,
-            ) as e:
-                logger.warning("Error talking to server: %s" % e)
-                if count >= 3:
-                    if not isinstance(e, HashConnectionError):
-                        raise HashConnectionError(str(e))
-                    raise e
-                await self.close()
-                count += 1
-
-    async def send_message(self, msg):
-        async def get_line():
-            line = await self.reader.readline()
-            if not line:
-                raise HashConnectionError("Connection closed")
-
-            line = line.decode("utf-8")
-
-            if not line.endswith("\n"):
-                raise HashConnectionError("Bad message %r" % message)
-
-            return line
+    def __init__(self, username=None, password=None):
+        super().__init__("OEHASHEQUIV", "1.1", logger)
+        self.mode = self.MODE_NORMAL
+        self.username = username
+        self.password = password
+        self.saved_become_user = None
 
-        async def proc():
-            for c in chunkify(json.dumps(msg), self.max_chunk):
-                self.writer.write(c.encode("utf-8"))
-            await self.writer.drain()
+    async def setup_connection(self):
+        await super().setup_connection()
+        self.mode = self.MODE_NORMAL
+        if self.username:
+            # Save off become user temporarily because auth() resets it
+            become = self.saved_become_user
+            await self.auth(self.username, self.password)
 
-            l = await get_line()
+            if become:
+                await self.become_user(become)
 
-            m = json.loads(l)
-            if m and "chunk-stream" in m:
-                lines = []
-                while True:
-                    l = (await get_line()).rstrip("\n")
-                    if not l:
-                        break
-                    lines.append(l)
+    async def send_stream_batch(self, mode, msgs):
+        """
+        Does a "batch" process of stream messages. This sends the query
+        messages as fast as possible, and simultaneously attempts to read the
+        messages back. This helps to mitigate the effects of latency to the
+        hash equivalence server be allowing multiple queries to be "in-flight"
+        at once
 
-                m = json.loads("".join(lines))
+        The implementation does more complicated tracking using a count of sent
+        messages so that `msgs` can be a generator function (i.e. its length is
+        unknown)
 
-            return m
+        """
 
-        return await self._send_wrapper(proc)
+        b = Batch()
 
-    async def send_stream(self, msg):
         async def proc():
-            self.writer.write(("%s\n" % msg).encode("utf-8"))
-            await self.writer.drain()
-            l = await self.reader.readline()
-            if not l:
-                raise HashConnectionError("Connection closed")
-            return l.decode("utf-8").rstrip()
+            nonlocal b
+
+            await self._set_mode(mode)
+            return await b.process(self.socket, msgs)
 
         return await self._send_wrapper(proc)
 
+    async def invoke(self, *args, skip_mode=False, **kwargs):
+        # It's OK if connection errors cause a failure here, because the mode
+        # is also reset to normal on a new connection
+        if not skip_mode:
+            await self._set_mode(self.MODE_NORMAL)
+        return await super().invoke(*args, **kwargs)
+
     async def _set_mode(self, new_mode):
-        if new_mode == self.MODE_NORMAL and self.mode == self.MODE_GET_STREAM:
-            r = await self.send_stream("END")
+        async def stream_to_normal():
+            # Check if already in normal mode (e.g. due to a connection reset)
+            if self.mode == self.MODE_NORMAL:
+                return "ok"
+            await self.socket.send("END")
+            return await self.socket.recv()
+
+        async def normal_to_stream(command):
+            r = await self.invoke({command: None}, skip_mode=True)
             if r != "ok":
-                raise HashConnectionError("Bad response from server %r" % r)
-        elif new_mode == self.MODE_GET_STREAM and self.mode == self.MODE_NORMAL:
-            r = await self.send_message({"get-stream": None})
+                self.check_invoke_error(r)
+                raise ConnectionError(
+                    f"Unable to transition to stream mode: Bad response from server {r!r}"
+                )
+            self.logger.debug("Mode is now %s", command)
+
+        if new_mode == self.mode:
+            return
+
+        self.logger.debug("Transitioning mode %s -> %s", self.mode, new_mode)
+
+        # Always transition to normal mode before switching to any other mode
+        if self.mode != self.MODE_NORMAL:
+            r = await self._send_wrapper(stream_to_normal)
             if r != "ok":
-                raise HashConnectionError("Bad response from server %r" % r)
-        elif new_mode != self.mode:
-            raise Exception(
-                "Undefined mode transition %r -> %r" % (self.mode, new_mode)
-            )
+                self.check_invoke_error(r)
+                raise ConnectionError(
+                    f"Unable to transition to normal mode: Bad response from server {r!r}"
+                )
+            self.logger.debug("Mode is now normal")
+
+        if new_mode == self.MODE_GET_STREAM:
+            await normal_to_stream("get-stream")
+        elif new_mode == self.MODE_EXIST_STREAM:
+            await normal_to_stream("exists-stream")
+        elif new_mode == self.MODE_MARK_STREAM:
+            await normal_to_stream("gc-mark-stream")
+        elif new_mode != self.MODE_NORMAL:
+            raise Exception("Undefined mode transition {self.mode!r} -> {new_mode!r}")
 
         self.mode = new_mode
 
     async def get_unihash(self, method, taskhash):
-        await self._set_mode(self.MODE_GET_STREAM)
-        r = await self.send_stream("%s %s" % (method, taskhash))
-        if not r:
-            return None
-        return r
+        r = await self.get_unihash_batch([(method, taskhash)])
+        return r[0]
+
+    async def get_unihash_batch(self, args):
+        result = await self.send_stream_batch(
+            self.MODE_GET_STREAM,
+            (f"{method} {taskhash}" for method, taskhash in args),
+        )
+        return [r if r else None for r in result]
 
     async def report_unihash(self, taskhash, method, outhash, unihash, extra={}):
-        await self._set_mode(self.MODE_NORMAL)
         m = extra.copy()
         m["taskhash"] = taskhash
         m["method"] = method
         m["outhash"] = outhash
         m["unihash"] = unihash
-        return await self.send_message({"report": m})
+        return await self.invoke({"report": m})
 
     async def report_unihash_equiv(self, taskhash, method, unihash, extra={}):
-        await self._set_mode(self.MODE_NORMAL)
         m = extra.copy()
         m["taskhash"] = taskhash
         m["method"] = method
         m["unihash"] = unihash
-        return await self.send_message({"report-equiv": m})
+        return await self.invoke({"report-equiv": m})
 
     async def get_taskhash(self, method, taskhash, all_properties=False):
-        await self._set_mode(self.MODE_NORMAL)
-        return await self.send_message(
+        return await self.invoke(
             {"get": {"taskhash": taskhash, "method": method, "all": all_properties}}
         )
 
-    async def get_outhash(self, method, outhash, taskhash):
-        await self._set_mode(self.MODE_NORMAL)
-        return await self.send_message(
-            {"get-outhash": {"outhash": outhash, "taskhash": taskhash, "method": method}}
+    async def unihash_exists(self, unihash):
+        r = await self.unihash_exists_batch([unihash])
+        return r[0]
+
+    async def unihash_exists_batch(self, unihashes):
+        result = await self.send_stream_batch(self.MODE_EXIST_STREAM, unihashes)
+        return [r == "true" for r in result]
+
+    async def get_outhash(self, method, outhash, taskhash, with_unihash=True):
+        return await self.invoke(
+            {
+                "get-outhash": {
+                    "outhash": outhash,
+                    "taskhash": taskhash,
+                    "method": method,
+                    "with_unihash": with_unihash,
+                }
+            }
         )
 
     async def get_stats(self):
-        await self._set_mode(self.MODE_NORMAL)
-        return await self.send_message({"get-stats": None})
+        return await self.invoke({"get-stats": None})
 
     async def reset_stats(self):
-        await self._set_mode(self.MODE_NORMAL)
-        return await self.send_message({"reset-stats": None})
+        return await self.invoke({"reset-stats": None})
 
     async def backfill_wait(self):
-        await self._set_mode(self.MODE_NORMAL)
-        return (await self.send_message({"backfill-wait": None}))["tasks"]
+        return (await self.invoke({"backfill-wait": None}))["tasks"]
+
+    async def remove(self, where):
+        return await self.invoke({"remove": {"where": where}})
+
+    async def clean_unused(self, max_age):
+        return await self.invoke({"clean-unused": {"max_age_seconds": max_age}})
+
+    async def auth(self, username, token):
+        result = await self.invoke({"auth": {"username": username, "token": token}})
+        self.username = username
+        self.password = token
+        self.saved_become_user = None
+        return result
+
+    async def refresh_token(self, username=None):
+        m = {}
+        if username:
+            m["username"] = username
+        result = await self.invoke({"refresh-token": m})
+        if (
+            self.username
+            and not self.saved_become_user
+            and result["username"] == self.username
+        ):
+            self.password = result["token"]
+        return result
 
+    async def set_user_perms(self, username, permissions):
+        return await self.invoke(
+            {"set-user-perms": {"username": username, "permissions": permissions}}
+        )
 
-class Client(object):
-    def __init__(self):
-        self.client = AsyncClient()
-        self.loop = asyncio.new_event_loop()
+    async def get_user(self, username=None):
+        m = {}
+        if username:
+            m["username"] = username
+        return await self.invoke({"get-user": m})
+
+    async def get_all_users(self):
+        return (await self.invoke({"get-all-users": {}}))["users"]
+
+    async def new_user(self, username, permissions):
+        return await self.invoke(
+            {"new-user": {"username": username, "permissions": permissions}}
+        )
+
+    async def delete_user(self, username):
+        return await self.invoke({"delete-user": {"username": username}})
+
+    async def become_user(self, username):
+        result = await self.invoke({"become-user": {"username": username}})
+        if username == self.username:
+            self.saved_become_user = None
+        else:
+            self.saved_become_user = username
+        return result
+
+    async def get_db_usage(self):
+        return (await self.invoke({"get-db-usage": {}}))["usage"]
+
+    async def get_db_query_columns(self):
+        return (await self.invoke({"get-db-query-columns": {}}))["columns"]
+
+    async def gc_status(self):
+        return await self.invoke({"gc-status": {}})
+
+    async def gc_mark(self, mark, where):
+        """
+        Starts a new garbage collection operation identified by "mark". If
+        garbage collection is already in progress with "mark", the collection
+        is continued.
+
+        All unihash entries that match the "where" clause are marked to be
+        kept. In addition, any new entries added to the database after this
+        command will be automatically marked with "mark"
+        """
+        return await self.invoke({"gc-mark": {"mark": mark, "where": where}})
+
+    async def gc_mark_stream(self, mark, rows):
+        """
+        Similar to `gc-mark`, but accepts a list of "where" key-value pair
+        conditions. It utilizes stream mode to mark hashes, which helps reduce
+        the impact of latency when communicating with the hash equivalence
+        server.
+        """
+        def row_to_dict(row):
+            pairs = row.split()
+            return dict(zip(pairs[::2], pairs[1::2]))
+
+        responses = await self.send_stream_batch(
+            self.MODE_MARK_STREAM,
+            (json.dumps({"mark": mark, "where": row_to_dict(row)}) for row in rows),
+        )
+
+        return {"count": sum(int(json.loads(r)["count"]) for r in responses)}
 
-        for call in (
+    async def gc_sweep(self, mark):
+        """
+        Finishes garbage collection for "mark". All unihash entries that have
+        not been marked will be deleted.
+
+        It is recommended to clean unused outhash entries after running this to
+        cleanup any dangling outhashes
+        """
+        return await self.invoke({"gc-sweep": {"mark": mark}})
+
+
+class Client(bb.asyncrpc.Client):
+    def __init__(self, username=None, password=None):
+        self.username = username
+        self.password = password
+
+        super().__init__()
+        self._add_methods(
             "connect_tcp",
-            "close",
+            "connect_websocket",
             "get_unihash",
+            "get_unihash_batch",
             "report_unihash",
             "report_unihash_equiv",
             "get_taskhash",
+            "unihash_exists",
+            "unihash_exists_batch",
+            "get_outhash",
             "get_stats",
             "reset_stats",
             "backfill_wait",
-        ):
-            downcall = getattr(self.client, call)
-            setattr(self, call, self._get_downcall_wrapper(downcall))
-
-    def _get_downcall_wrapper(self, downcall):
-        def wrapper(*args, **kwargs):
-            return self.loop.run_until_complete(downcall(*args, **kwargs))
-
-        return wrapper
-
-    def connect_unix(self, path):
-        # AF_UNIX has path length issues so chdir here to workaround
-        cwd = os.getcwd()
-        try:
-            os.chdir(os.path.dirname(path))
-            self.loop.run_until_complete(self.client.connect_unix(os.path.basename(path)))
-            self.loop.run_until_complete(self.client.connect())
-        finally:
-            os.chdir(cwd)
-
-    @property
-    def max_chunk(self):
-        return self.client.max_chunk
+            "remove",
+            "clean_unused",
+            "auth",
+            "refresh_token",
+            "set_user_perms",
+            "get_user",
+            "get_all_users",
+            "new_user",
+            "delete_user",
+            "become_user",
+            "get_db_usage",
+            "get_db_query_columns",
+            "gc_status",
+            "gc_mark",
+            "gc_mark_stream",
+            "gc_sweep",
+        )
 
-    @max_chunk.setter
-    def max_chunk(self, value):
-        self.client.max_chunk = value
+    def _get_async_client(self):
+        return AsyncClient(self.username, self.password)
diff --git a/bitbake/lib/hashserv/server.py b/bitbake/lib/hashserv/server.py
index a0dc0c170f..58f95c7bcd 100644
--- a/bitbake/lib/hashserv/server.py
+++ b/bitbake/lib/hashserv/server.py
@@ -3,20 +3,52 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-from contextlib import closing, contextmanager
-from datetime import datetime
+from datetime import datetime, timedelta
 import asyncio
-import json
 import logging
 import math
-import os
-import signal
-import socket
-import sys
 import time
-from . import chunkify, DEFAULT_MAX_CHUNK, create_async_client, TABLE_COLUMNS
+import os
+import base64
+import json
+import hashlib
+from . import create_async_client
+import bb.asyncrpc
+
+logger = logging.getLogger("hashserv.server")
+
+
+# This permission only exists to match nothing
+NONE_PERM = "@none"
+
+READ_PERM = "@read"
+REPORT_PERM = "@report"
+DB_ADMIN_PERM = "@db-admin"
+USER_ADMIN_PERM = "@user-admin"
+ALL_PERM = "@all"
+
+ALL_PERMISSIONS = {
+    READ_PERM,
+    REPORT_PERM,
+    DB_ADMIN_PERM,
+    USER_ADMIN_PERM,
+    ALL_PERM,
+}
+
+DEFAULT_ANON_PERMS = (
+    READ_PERM,
+    REPORT_PERM,
+    DB_ADMIN_PERM,
+)
+
+TOKEN_ALGORITHM = "sha256"
+
+# 48 bytes of random data will result in 64 characters when base64
+# encoded. This number also ensures that the base64 encoding won't have any
+# trailing '=' characters.
+TOKEN_SIZE = 48
 
-logger = logging.getLogger('hashserv.server')
+SALT_SIZE = 8
 
 
 class Measurement(object):
@@ -106,522 +138,773 @@ class Stats(object):
         return math.sqrt(self.s / (self.num - 1))
 
     def todict(self):
-        return {k: getattr(self, k) for k in ('num', 'total_time', 'max_time', 'average', 'stdev')}
-
-
-class ClientError(Exception):
-    pass
-
-class ServerError(Exception):
-    pass
-
-def insert_task(cursor, data, ignore=False):
-    keys = sorted(data.keys())
-    query = '''INSERT%s INTO tasks_v2 (%s) VALUES (%s)''' % (
-        " OR IGNORE" if ignore else "",
-        ', '.join(keys),
-        ', '.join(':' + k for k in keys))
-    cursor.execute(query, data)
-
-async def copy_from_upstream(client, db, method, taskhash):
-    d = await client.get_taskhash(method, taskhash, True)
-    if d is not None:
-        # Filter out unknown columns
-        d = {k: v for k, v in d.items() if k in TABLE_COLUMNS}
-        keys = sorted(d.keys())
-
-        with closing(db.cursor()) as cursor:
-            insert_task(cursor, d)
-            db.commit()
-
-    return d
-
-async def copy_outhash_from_upstream(client, db, method, outhash, taskhash):
-    d = await client.get_outhash(method, outhash, taskhash)
-    if d is not None:
-        # Filter out unknown columns
-        d = {k: v for k, v in d.items() if k in TABLE_COLUMNS}
-        keys = sorted(d.keys())
-
-        with closing(db.cursor()) as cursor:
-            insert_task(cursor, d)
-            db.commit()
-
-    return d
-
-class ServerClient(object):
-    FAST_QUERY = 'SELECT taskhash, method, unihash FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1'
-    ALL_QUERY =  'SELECT *                         FROM tasks_v2 WHERE method=:method AND taskhash=:taskhash ORDER BY created ASC LIMIT 1'
-    OUTHASH_QUERY = '''
-        -- Find tasks with a matching outhash (that is, tasks that
-        -- are equivalent)
-        SELECT * FROM tasks_v2 WHERE method=:method AND outhash=:outhash
-
-        -- If there is an exact match on the taskhash, return it.
-        -- Otherwise return the oldest matching outhash of any
-        -- taskhash
-        ORDER BY CASE WHEN taskhash=:taskhash THEN 1 ELSE 2 END,
-            created ASC
-
-        -- Only return one row
-        LIMIT 1
-        '''
-
-    def __init__(self, reader, writer, db, request_stats, backfill_queue, upstream, read_only):
-        self.reader = reader
-        self.writer = writer
-        self.db = db
-        self.request_stats = request_stats
-        self.max_chunk = DEFAULT_MAX_CHUNK
-        self.backfill_queue = backfill_queue
-        self.upstream = upstream
-
-        self.handlers = {
-            'get': self.handle_get,
-            'get-outhash': self.handle_get_outhash,
-            'get-stream': self.handle_get_stream,
-            'get-stats': self.handle_get_stats,
-            'chunk-stream': self.handle_chunk,
+        return {
+            k: getattr(self, k)
+            for k in ("num", "total_time", "max_time", "average", "stdev")
         }
 
-        if not read_only:
-            self.handlers.update({
-                'report': self.handle_report,
-                'report-equiv': self.handle_equivreport,
-                'reset-stats': self.handle_reset_stats,
-                'backfill-wait': self.handle_backfill_wait,
-            })
 
-    async def process_requests(self):
-        if self.upstream is not None:
-            self.upstream_client = await create_async_client(self.upstream)
-        else:
-            self.upstream_client = None
+token_refresh_semaphore = asyncio.Lock()
 
-        try:
 
+async def new_token():
+    # Prevent malicious users from using this API to deduce the entropy
+    # pool on the server and thus be able to guess a token. *All* token
+    # refresh requests lock the same global semaphore and then sleep for a
+    # short time. The effectively rate limits the total number of requests
+    # than can be made across all clients to 10/second, which should be enough
+    # since you have to be an authenticated users to make the request in the
+    # first place
+    async with token_refresh_semaphore:
+        await asyncio.sleep(0.1)
+        raw = os.getrandom(TOKEN_SIZE, os.GRND_NONBLOCK)
 
-            self.addr = self.writer.get_extra_info('peername')
-            logger.debug('Client %r connected' % (self.addr,))
+    return base64.b64encode(raw, b"._").decode("utf-8")
 
-            # Read protocol and version
-            protocol = await self.reader.readline()
-            if protocol is None:
-                return
 
-            (proto_name, proto_version) = protocol.decode('utf-8').rstrip().split()
-            if proto_name != 'OEHASHEQUIV':
-                return
+def new_salt():
+    return os.getrandom(SALT_SIZE, os.GRND_NONBLOCK).hex()
 
-            proto_version = tuple(int(v) for v in proto_version.split('.'))
-            if proto_version < (1, 0) or proto_version > (1, 1):
-                return
 
-            # Read headers. Currently, no headers are implemented, so look for
-            # an empty line to signal the end of the headers
-            while True:
-                line = await self.reader.readline()
-                if line is None:
-                    return
+def hash_token(algo, salt, token):
+    h = hashlib.new(algo)
+    h.update(salt.encode("utf-8"))
+    h.update(token.encode("utf-8"))
+    return ":".join([algo, salt, h.hexdigest()])
 
-                line = line.decode('utf-8').rstrip()
-                if not line:
-                    break
 
-            # Handle messages
-            while True:
-                d = await self.read_message()
-                if d is None:
-                    break
-                await self.dispatch_message(d)
-                await self.writer.drain()
-        except ClientError as e:
-            logger.error(str(e))
-        finally:
-            if self.upstream_client is not None:
-                await self.upstream_client.close()
+def permissions(*permissions, allow_anon=True, allow_self_service=False):
+    """
+    Function decorator that can be used to decorate an RPC function call and
+    check that the current users permissions match the require permissions.
 
-            self.writer.close()
+    If allow_anon is True, the user will also be allowed to make the RPC call
+    if the anonymous user permissions match the permissions.
 
-    async def dispatch_message(self, msg):
-        for k in self.handlers.keys():
-            if k in msg:
-                logger.debug('Handling %s' % k)
-                if 'stream' in k:
-                    await self.handlers[k](msg[k])
+    If allow_self_service is True, and the "username" property in the request
+    is the currently logged in user, or not specified, the user will also be
+    allowed to make the request. This allows users to access normal privileged
+    API, as long as they are only modifying their own user properties (e.g.
+    users can be allowed to reset their own token without @user-admin
+    permissions, but not the token for any other user.
+    """
+
+    def wrapper(func):
+        async def wrap(self, request):
+            if allow_self_service and self.user is not None:
+                username = request.get("username", self.user.username)
+                if username == self.user.username:
+                    request["username"] = self.user.username
+                    return await func(self, request)
+
+            if not self.user_has_permissions(*permissions, allow_anon=allow_anon):
+                if not self.user:
+                    username = "Anonymous user"
+                    user_perms = self.server.anon_perms
                 else:
-                    with self.request_stats.start_sample() as self.request_sample, \
-                            self.request_sample.measure():
-                        await self.handlers[k](msg[k])
-                return
+                    username = self.user.username
+                    user_perms = self.user.permissions
+
+                self.logger.info(
+                    "User %s with permissions %r denied from calling %s. Missing permissions(s) %r",
+                    username,
+                    ", ".join(user_perms),
+                    func.__name__,
+                    ", ".join(permissions),
+                )
+                raise bb.asyncrpc.InvokeError(
+                    f"{username} is not allowed to access permissions(s) {', '.join(permissions)}"
+                )
+
+            return await func(self, request)
+
+        return wrap
+
+    return wrapper
+
+
+class ServerClient(bb.asyncrpc.AsyncServerConnection):
+    def __init__(self, socket, server):
+        super().__init__(socket, "OEHASHEQUIV", server.logger)
+        self.server = server
+        self.max_chunk = bb.asyncrpc.DEFAULT_MAX_CHUNK
+        self.user = None
+
+        self.handlers.update(
+            {
+                "get": self.handle_get,
+                "get-outhash": self.handle_get_outhash,
+                "get-stream": self.handle_get_stream,
+                "exists-stream": self.handle_exists_stream,
+                "get-stats": self.handle_get_stats,
+                "get-db-usage": self.handle_get_db_usage,
+                "get-db-query-columns": self.handle_get_db_query_columns,
+                # Not always read-only, but internally checks if the server is
+                # read-only
+                "report": self.handle_report,
+                "auth": self.handle_auth,
+                "get-user": self.handle_get_user,
+                "get-all-users": self.handle_get_all_users,
+                "become-user": self.handle_become_user,
+            }
+        )
 
-        raise ClientError("Unrecognized command %r" % msg)
+        if not self.server.read_only:
+            self.handlers.update(
+                {
+                    "report-equiv": self.handle_equivreport,
+                    "reset-stats": self.handle_reset_stats,
+                    "backfill-wait": self.handle_backfill_wait,
+                    "remove": self.handle_remove,
+                    "gc-mark": self.handle_gc_mark,
+                    "gc-mark-stream": self.handle_gc_mark_stream,
+                    "gc-sweep": self.handle_gc_sweep,
+                    "gc-status": self.handle_gc_status,
+                    "clean-unused": self.handle_clean_unused,
+                    "refresh-token": self.handle_refresh_token,
+                    "set-user-perms": self.handle_set_perms,
+                    "new-user": self.handle_new_user,
+                    "delete-user": self.handle_delete_user,
+                }
+            )
 
-    def write_message(self, msg):
-        for c in chunkify(json.dumps(msg), self.max_chunk):
-            self.writer.write(c.encode('utf-8'))
+    def raise_no_user_error(self, username):
+        raise bb.asyncrpc.InvokeError(f"No user named '{username}' exists")
 
-    async def read_message(self):
-        l = await self.reader.readline()
-        if not l:
-            return None
+    def user_has_permissions(self, *permissions, allow_anon=True):
+        permissions = set(permissions)
+        if allow_anon:
+            if ALL_PERM in self.server.anon_perms:
+                return True
 
-        try:
-            message = l.decode('utf-8')
+            if not permissions - self.server.anon_perms:
+                return True
 
-            if not message.endswith('\n'):
-                return None
+        if self.user is None:
+            return False
 
-            return json.loads(message)
-        except (json.JSONDecodeError, UnicodeDecodeError) as e:
-            logger.error('Bad message from client: %r' % message)
-            raise e
+        if ALL_PERM in self.user.permissions:
+            return True
 
-    async def handle_chunk(self, request):
-        lines = []
-        try:
-            while True:
-                l = await self.reader.readline()
-                l = l.rstrip(b"\n").decode("utf-8")
-                if not l:
-                    break
-                lines.append(l)
+        if not permissions - self.user.permissions:
+            return True
 
-            msg = json.loads(''.join(lines))
-        except (json.JSONDecodeError, UnicodeDecodeError) as e:
-            logger.error('Bad message from client: %r' % message)
-            raise e
+        return False
 
-        if 'chunk-stream' in msg:
-            raise ClientError("Nested chunks are not allowed")
+    def validate_proto_version(self):
+        return self.proto_version > (1, 0) and self.proto_version <= (1, 1)
 
-        await self.dispatch_message(msg)
+    async def process_requests(self):
+        async with self.server.db_engine.connect(self.logger) as db:
+            self.db = db
+            if self.server.upstream is not None:
+                self.upstream_client = await create_async_client(self.server.upstream)
+            else:
+                self.upstream_client = None
 
-    async def handle_get(self, request):
-        method = request['method']
-        taskhash = request['taskhash']
+            try:
+                await super().process_requests()
+            finally:
+                if self.upstream_client is not None:
+                    await self.upstream_client.close()
 
-        if request.get('all', False):
-            row = self.query_equivalent(method, taskhash, self.ALL_QUERY)
-        else:
-            row = self.query_equivalent(method, taskhash, self.FAST_QUERY)
+    async def dispatch_message(self, msg):
+        for k in self.handlers.keys():
+            if k in msg:
+                self.logger.debug("Handling %s" % k)
+                if "stream" in k:
+                    return await self.handlers[k](msg[k])
+                else:
+                    with self.server.request_stats.start_sample() as self.request_sample, self.request_sample.measure():
+                        return await self.handlers[k](msg[k])
 
-        if row is not None:
-            logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash']))
-            d = {k: row[k] for k in row.keys()}
-        elif self.upstream_client is not None:
-            d = await copy_from_upstream(self.upstream_client, self.db, method, taskhash)
+        raise bb.asyncrpc.ClientError("Unrecognized command %r" % msg)
+
+    @permissions(READ_PERM)
+    async def handle_get(self, request):
+        method = request["method"]
+        taskhash = request["taskhash"]
+        fetch_all = request.get("all", False)
+
+        return await self.get_unihash(method, taskhash, fetch_all)
+
+    async def get_unihash(self, method, taskhash, fetch_all=False):
+        d = None
+
+        if fetch_all:
+            row = await self.db.get_unihash_by_taskhash_full(method, taskhash)
+            if row is not None:
+                d = {k: row[k] for k in row.keys()}
+            elif self.upstream_client is not None:
+                d = await self.upstream_client.get_taskhash(method, taskhash, True)
+                await self.update_unified(d)
         else:
-            d = None
+            row = await self.db.get_equivalent(method, taskhash)
+
+            if row is not None:
+                d = {k: row[k] for k in row.keys()}
+            elif self.upstream_client is not None:
+                d = await self.upstream_client.get_taskhash(method, taskhash)
+                await self.db.insert_unihash(d["method"], d["taskhash"], d["unihash"])
 
-        self.write_message(d)
+        return d
 
+    @permissions(READ_PERM)
     async def handle_get_outhash(self, request):
-        with closing(self.db.cursor()) as cursor:
-            cursor.execute(self.OUTHASH_QUERY,
-                           {k: request[k] for k in ('method', 'outhash', 'taskhash')})
+        method = request["method"]
+        outhash = request["outhash"]
+        taskhash = request["taskhash"]
+        with_unihash = request.get("with_unihash", True)
 
-            row = cursor.fetchone()
+        return await self.get_outhash(method, outhash, taskhash, with_unihash)
+
+    async def get_outhash(self, method, outhash, taskhash, with_unihash=True):
+        d = None
+        if with_unihash:
+            row = await self.db.get_unihash_by_outhash(method, outhash)
+        else:
+            row = await self.db.get_outhash(method, outhash)
 
         if row is not None:
-            logger.debug('Found equivalent outhash %s -> %s', (row['outhash'], row['unihash']))
             d = {k: row[k] for k in row.keys()}
-        else:
-            d = None
+        elif self.upstream_client is not None:
+            d = await self.upstream_client.get_outhash(method, outhash, taskhash)
+            await self.update_unified(d)
 
-        self.write_message(d)
+        return d
 
-    async def handle_get_stream(self, request):
-        self.write_message('ok')
+    async def update_unified(self, data):
+        if data is None:
+            return
+
+        await self.db.insert_unihash(data["method"], data["taskhash"], data["unihash"])
+        await self.db.insert_outhash(data)
+
+    async def _stream_handler(self, handler):
+        await self.socket.send_message("ok")
 
         while True:
             upstream = None
 
-            l = await self.reader.readline()
+            l = await self.socket.recv()
             if not l:
-                return
+                break
 
             try:
                 # This inner loop is very sensitive and must be as fast as
                 # possible (which is why the request sample is handled manually
                 # instead of using 'with', and also why logging statements are
                 # commented out.
-                self.request_sample = self.request_stats.start_sample()
+                self.request_sample = self.server.request_stats.start_sample()
                 request_measure = self.request_sample.measure()
                 request_measure.start()
 
-                l = l.decode('utf-8').rstrip()
-                if l == 'END':
-                    self.writer.write('ok\n'.encode('utf-8'))
-                    return
-
-                (method, taskhash) = l.split()
-                #logger.debug('Looking up %s %s' % (method, taskhash))
-                row = self.query_equivalent(method, taskhash, self.FAST_QUERY)
-                if row is not None:
-                    msg = ('%s\n' % row['unihash']).encode('utf-8')
-                    #logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash']))
-                elif self.upstream_client is not None:
-                    upstream = await self.upstream_client.get_unihash(method, taskhash)
-                    if upstream:
-                        msg = ("%s\n" % upstream).encode("utf-8")
-                    else:
-                        msg = "\n".encode("utf-8")
-                else:
-                    msg = '\n'.encode('utf-8')
+                if l == "END":
+                    break
 
-                self.writer.write(msg)
+                msg = await handler(l)
+                await self.socket.send(msg)
             finally:
                 request_measure.end()
                 self.request_sample.end()
 
-            await self.writer.drain()
+        await self.socket.send("ok")
+        return self.NO_RESPONSE
 
-            # Post to the backfill queue after writing the result to minimize
-            # the turn around time on a request
-            if upstream is not None:
-                await self.backfill_queue.put((method, taskhash))
+    @permissions(READ_PERM)
+    async def handle_get_stream(self, request):
+        async def handler(l):
+            (method, taskhash) = l.split()
+            # self.logger.debug('Looking up %s %s' % (method, taskhash))
+            row = await self.db.get_equivalent(method, taskhash)
 
-    async def handle_report(self, data):
-        with closing(self.db.cursor()) as cursor:
-            cursor.execute(self.OUTHASH_QUERY,
-                           {k: data[k] for k in ('method', 'outhash', 'taskhash')})
-
-            row = cursor.fetchone()
-
-            if row is None and self.upstream_client:
-                # Try upstream
-                row = await copy_outhash_from_upstream(self.upstream_client,
-                                                       self.db,
-                                                       data['method'],
-                                                       data['outhash'],
-                                                       data['taskhash'])
-
-            # If no matching outhash was found, or one *was* found but it
-            # wasn't an exact match on the taskhash, a new entry for this
-            # taskhash should be added
-            if row is None or row['taskhash'] != data['taskhash']:
-                # If a row matching the outhash was found, the unihash for
-                # the new taskhash should be the same as that one.
-                # Otherwise the caller provided unihash is used.
-                unihash = data['unihash']
-                if row is not None:
-                    unihash = row['unihash']
-
-                insert_data = {
-                    'method': data['method'],
-                    'outhash': data['outhash'],
-                    'taskhash': data['taskhash'],
-                    'unihash': unihash,
-                    'created': datetime.now()
-                }
+            if row is not None:
+                # self.logger.debug('Found equivalent task %s -> %s', (row['taskhash'], row['unihash']))
+                return row["unihash"]
 
-                for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'):
-                    if k in data:
-                        insert_data[k] = data[k]
+            if self.upstream_client is not None:
+                upstream = await self.upstream_client.get_unihash(method, taskhash)
+                if upstream:
+                    await self.server.backfill_queue.put((method, taskhash))
+                    return upstream
 
-                insert_task(cursor, insert_data)
-                self.db.commit()
+            return ""
 
-                logger.info('Adding taskhash %s with unihash %s',
-                            data['taskhash'], unihash)
+        return await self._stream_handler(handler)
 
-                d = {
-                    'taskhash': data['taskhash'],
-                    'method': data['method'],
-                    'unihash': unihash
-                }
-            else:
-                d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
+    @permissions(READ_PERM)
+    async def handle_exists_stream(self, request):
+        async def handler(l):
+            if await self.db.unihash_exists(l):
+                return "true"
 
-        self.write_message(d)
+            if self.upstream_client is not None:
+                if await self.upstream_client.unihash_exists(l):
+                    return "true"
 
-    async def handle_equivreport(self, data):
-        with closing(self.db.cursor()) as cursor:
-            insert_data = {
-                'method': data['method'],
-                'outhash': "",
-                'taskhash': data['taskhash'],
-                'unihash': data['unihash'],
-                'created': datetime.now()
-            }
+            return "false"
 
-            for k in ('owner', 'PN', 'PV', 'PR', 'task', 'outhash_siginfo'):
-                if k in data:
-                    insert_data[k] = data[k]
+        return await self._stream_handler(handler)
 
-            insert_task(cursor, insert_data, ignore=True)
-            self.db.commit()
+    async def report_readonly(self, data):
+        method = data["method"]
+        outhash = data["outhash"]
+        taskhash = data["taskhash"]
 
-            # Fetch the unihash that will be reported for the taskhash. If the
-            # unihash matches, it means this row was inserted (or the mapping
-            # was already valid)
-            row = self.query_equivalent(data['method'], data['taskhash'], self.FAST_QUERY)
+        info = await self.get_outhash(method, outhash, taskhash)
+        if info:
+            unihash = info["unihash"]
+        else:
+            unihash = data["unihash"]
 
-            if row['unihash'] == data['unihash']:
-                logger.info('Adding taskhash equivalence for %s with unihash %s',
-                                data['taskhash'], row['unihash'])
+        return {
+            "taskhash": taskhash,
+            "method": method,
+            "unihash": unihash,
+        }
 
-            d = {k: row[k] for k in ('taskhash', 'method', 'unihash')}
+    # Since this can be called either read only or to report, the check to
+    # report is made inside the function
+    @permissions(READ_PERM)
+    async def handle_report(self, data):
+        if self.server.read_only or not self.user_has_permissions(REPORT_PERM):
+            return await self.report_readonly(data)
+
+        outhash_data = {
+            "method": data["method"],
+            "outhash": data["outhash"],
+            "taskhash": data["taskhash"],
+            "created": datetime.now(),
+        }
 
-        self.write_message(d)
+        for k in ("owner", "PN", "PV", "PR", "task", "outhash_siginfo"):
+            if k in data:
+                outhash_data[k] = data[k]
 
+        if self.user:
+            outhash_data["owner"] = self.user.username
 
-    async def handle_get_stats(self, request):
-        d = {
-            'requests': self.request_stats.todict(),
+        # Insert the new entry, unless it already exists
+        if await self.db.insert_outhash(outhash_data):
+            # If this row is new, check if it is equivalent to another
+            # output hash
+            row = await self.db.get_equivalent_for_outhash(
+                data["method"], data["outhash"], data["taskhash"]
+            )
+
+            if row is not None:
+                # A matching output hash was found. Set our taskhash to the
+                # same unihash since they are equivalent
+                unihash = row["unihash"]
+            else:
+                # No matching output hash was found. This is probably the
+                # first outhash to be added.
+                unihash = data["unihash"]
+
+                # Query upstream to see if it has a unihash we can use
+                if self.upstream_client is not None:
+                    upstream_data = await self.upstream_client.get_outhash(
+                        data["method"], data["outhash"], data["taskhash"]
+                    )
+                    if upstream_data is not None:
+                        unihash = upstream_data["unihash"]
+
+            await self.db.insert_unihash(data["method"], data["taskhash"], unihash)
+
+        unihash_data = await self.get_unihash(data["method"], data["taskhash"])
+        if unihash_data is not None:
+            unihash = unihash_data["unihash"]
+        else:
+            unihash = data["unihash"]
+
+        return {
+            "taskhash": data["taskhash"],
+            "method": data["method"],
+            "unihash": unihash,
         }
 
-        self.write_message(d)
+    @permissions(READ_PERM, REPORT_PERM)
+    async def handle_equivreport(self, data):
+        await self.db.insert_unihash(data["method"], data["taskhash"], data["unihash"])
+
+        # Fetch the unihash that will be reported for the taskhash. If the
+        # unihash matches, it means this row was inserted (or the mapping
+        # was already valid)
+        row = await self.db.get_equivalent(data["method"], data["taskhash"])
+
+        if row["unihash"] == data["unihash"]:
+            self.logger.info(
+                "Adding taskhash equivalence for %s with unihash %s",
+                data["taskhash"],
+                row["unihash"],
+            )
+
+        return {k: row[k] for k in ("taskhash", "method", "unihash")}
 
+    @permissions(READ_PERM)
+    async def handle_get_stats(self, request):
+        return {
+            "requests": self.server.request_stats.todict(),
+        }
+
+    @permissions(DB_ADMIN_PERM)
     async def handle_reset_stats(self, request):
         d = {
-            'requests': self.request_stats.todict(),
+            "requests": self.server.request_stats.todict(),
         }
 
-        self.request_stats.reset()
-        self.write_message(d)
+        self.server.request_stats.reset()
+        return d
 
+    @permissions(READ_PERM)
     async def handle_backfill_wait(self, request):
         d = {
-            'tasks': self.backfill_queue.qsize(),
+            "tasks": self.server.backfill_queue.qsize(),
         }
-        await self.backfill_queue.join()
-        self.write_message(d)
+        await self.server.backfill_queue.join()
+        return d
 
-    def query_equivalent(self, method, taskhash, query):
-        # This is part of the inner loop and must be as fast as possible
-        try:
-            cursor = self.db.cursor()
-            cursor.execute(query, {'method': method, 'taskhash': taskhash})
-            return cursor.fetchone()
-        except:
-            cursor.close()
+    @permissions(DB_ADMIN_PERM)
+    async def handle_remove(self, request):
+        condition = request["where"]
+        if not isinstance(condition, dict):
+            raise TypeError("Bad condition type %s" % type(condition))
 
+        return {"count": await self.db.remove(condition)}
 
-class Server(object):
-    def __init__(self, db, loop=None, upstream=None, read_only=False):
-        if upstream and read_only:
-            raise ServerError("Read-only hashserv cannot pull from an upstream server")
+    @permissions(DB_ADMIN_PERM)
+    async def handle_gc_mark(self, request):
+        condition = request["where"]
+        mark = request["mark"]
 
-        self.request_stats = Stats()
-        self.db = db
+        if not isinstance(condition, dict):
+            raise TypeError("Bad condition type %s" % type(condition))
 
-        if loop is None:
-            self.loop = asyncio.new_event_loop()
-            self.close_loop = True
-        else:
-            self.loop = loop
-            self.close_loop = False
+        if not isinstance(mark, str):
+            raise TypeError("Bad mark type %s" % type(mark))
 
-        self.upstream = upstream
-        self.read_only = read_only
+        return {"count": await self.db.gc_mark(mark, condition)}
 
-        self._cleanup_socket = None
+    @permissions(DB_ADMIN_PERM)
+    async def handle_gc_mark_stream(self, request):
+        async def handler(line):
+            try:
+                decoded_line = json.loads(line)
+            except json.JSONDecodeError as exc:
+                raise bb.asyncrpc.InvokeError(
+                    "Could not decode JSONL input '%s'" % line
+                ) from exc
 
-    def start_tcp_server(self, host, port):
-        self.server = self.loop.run_until_complete(
-            asyncio.start_server(self.handle_client, host, port, loop=self.loop)
-        )
+            try:
+                mark = decoded_line["mark"]
+                condition = decoded_line["where"]
+                if not isinstance(mark, str):
+                    raise TypeError("Bad mark type %s" % type(mark))
 
-        for s in self.server.sockets:
-            logger.info('Listening on %r' % (s.getsockname(),))
-            # Newer python does this automatically. Do it manually here for
-            # maximum compatibility
-            s.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
-            s.setsockopt(socket.SOL_TCP, socket.TCP_QUICKACK, 1)
+                if not isinstance(condition, dict):
+                    raise TypeError("Bad condition type %s" % type(condition))
+            except KeyError as exc:
+                raise bb.asyncrpc.InvokeError(
+                    "Input line is missing key '%s' " % exc
+                ) from exc
 
-        name = self.server.sockets[0].getsockname()
-        if self.server.sockets[0].family == socket.AF_INET6:
-            self.address = "[%s]:%d" % (name[0], name[1])
-        else:
-            self.address = "%s:%d" % (name[0], name[1])
+            return json.dumps({"count": await self.db.gc_mark(mark, condition)})
 
-    def start_unix_server(self, path):
-        def cleanup():
-            os.unlink(path)
+        return await self._stream_handler(handler)
 
-        cwd = os.getcwd()
-        try:
-            # Work around path length limits in AF_UNIX
-            os.chdir(os.path.dirname(path))
-            self.server = self.loop.run_until_complete(
-                asyncio.start_unix_server(self.handle_client, os.path.basename(path), loop=self.loop)
+    @permissions(DB_ADMIN_PERM)
+    async def handle_gc_sweep(self, request):
+        mark = request["mark"]
+
+        if not isinstance(mark, str):
+            raise TypeError("Bad mark type %s" % type(mark))
+
+        current_mark = await self.db.get_current_gc_mark()
+
+        if not current_mark or mark != current_mark:
+            raise bb.asyncrpc.InvokeError(
+                f"'{mark}' is not the current mark. Refusing to sweep"
             )
-        finally:
-            os.chdir(cwd)
 
-        logger.info('Listening on %r' % path)
+        count = await self.db.gc_sweep()
+
+        return {"count": count}
 
-        self._cleanup_socket = cleanup
-        self.address = "unix://%s" % os.path.abspath(path)
+    @permissions(DB_ADMIN_PERM)
+    async def handle_gc_status(self, request):
+        (keep_rows, remove_rows, current_mark) = await self.db.gc_status()
+        return {
+            "keep": keep_rows,
+            "remove": remove_rows,
+            "mark": current_mark,
+        }
+
+    @permissions(DB_ADMIN_PERM)
+    async def handle_clean_unused(self, request):
+        max_age = request["max_age_seconds"]
+        oldest = datetime.now() - timedelta(seconds=-max_age)
+        return {"count": await self.db.clean_unused(oldest)}
+
+    @permissions(DB_ADMIN_PERM)
+    async def handle_get_db_usage(self, request):
+        return {"usage": await self.db.get_usage()}
+
+    @permissions(DB_ADMIN_PERM)
+    async def handle_get_db_query_columns(self, request):
+        return {"columns": await self.db.get_query_columns()}
+
+    # The authentication API is always allowed
+    async def handle_auth(self, request):
+        username = str(request["username"])
+        token = str(request["token"])
+
+        async def fail_auth():
+            nonlocal username
+            # Rate limit bad login attempts
+            await asyncio.sleep(1)
+            raise bb.asyncrpc.InvokeError(f"Unable to authenticate as {username}")
+
+        user, db_token = await self.db.lookup_user_token(username)
+
+        if not user or not db_token:
+            await fail_auth()
 
-    async def handle_client(self, reader, writer):
-        # writer.transport.set_write_buffer_limits(0)
         try:
-            client = ServerClient(reader, writer, self.db, self.request_stats, self.backfill_queue, self.upstream, self.read_only)
-            await client.process_requests()
-        except Exception as e:
-            import traceback
-            logger.error('Error from client: %s' % str(e), exc_info=True)
-            traceback.print_exc()
-            writer.close()
-        logger.info('Client disconnected')
-
-    @contextmanager
-    def _backfill_worker(self):
-        async def backfill_worker_task():
-            client = await create_async_client(self.upstream)
-            try:
-                while True:
-                    item = await self.backfill_queue.get()
-                    if item is None:
-                        self.backfill_queue.task_done()
-                        break
-                    method, taskhash = item
-                    await copy_from_upstream(client, self.db, method, taskhash)
-                    self.backfill_queue.task_done()
-            finally:
-                await client.close()
+            algo, salt, _ = db_token.split(":")
+        except ValueError:
+            await fail_auth()
 
-        async def join_worker(worker):
-            await self.backfill_queue.put(None)
-            await worker
+        if hash_token(algo, salt, token) != db_token:
+            await fail_auth()
 
-        if self.upstream is not None:
-            worker = asyncio.ensure_future(backfill_worker_task())
-            try:
-                yield
-            finally:
-                self.loop.run_until_complete(join_worker(worker))
-        else:
-            yield
+        self.user = user
+
+        self.logger.info("Authenticated as %s", username)
+
+        return {
+            "result": True,
+            "username": self.user.username,
+            "permissions": sorted(list(self.user.permissions)),
+        }
 
-    def serve_forever(self):
-        def signal_handler():
-            self.loop.stop()
+    @permissions(USER_ADMIN_PERM, allow_self_service=True, allow_anon=False)
+    async def handle_refresh_token(self, request):
+        username = str(request["username"])
 
-        asyncio.set_event_loop(self.loop)
+        token = await new_token()
+
+        updated = await self.db.set_user_token(
+            username,
+            hash_token(TOKEN_ALGORITHM, new_salt(), token),
+        )
+        if not updated:
+            self.raise_no_user_error(username)
+
+        return {"username": username, "token": token}
+
+    def get_perm_arg(self, arg):
+        if not isinstance(arg, list):
+            raise bb.asyncrpc.InvokeError("Unexpected type for permissions")
+
+        arg = set(arg)
         try:
-            self.backfill_queue = asyncio.Queue()
+            arg.remove(NONE_PERM)
+        except KeyError:
+            pass
+
+        unknown_perms = arg - ALL_PERMISSIONS
+        if unknown_perms:
+            raise bb.asyncrpc.InvokeError(
+                "Unknown permissions %s" % ", ".join(sorted(list(unknown_perms)))
+            )
+
+        return sorted(list(arg))
+
+    def return_perms(self, permissions):
+        if ALL_PERM in permissions:
+            return sorted(list(ALL_PERMISSIONS))
+        return sorted(list(permissions))
 
-            self.loop.add_signal_handler(signal.SIGTERM, signal_handler)
+    @permissions(USER_ADMIN_PERM, allow_anon=False)
+    async def handle_set_perms(self, request):
+        username = str(request["username"])
+        permissions = self.get_perm_arg(request["permissions"])
 
-            with self._backfill_worker():
-                try:
-                    self.loop.run_forever()
-                except KeyboardInterrupt:
-                    pass
+        if not await self.db.set_user_perms(username, permissions):
+            self.raise_no_user_error(username)
 
-                self.server.close()
+        return {
+            "username": username,
+            "permissions": self.return_perms(permissions),
+        }
+
+    @permissions(USER_ADMIN_PERM, allow_self_service=True, allow_anon=False)
+    async def handle_get_user(self, request):
+        username = str(request["username"])
 
-            self.loop.run_until_complete(self.server.wait_closed())
-            logger.info('Server shutting down')
-        finally:
-            if self.close_loop:
-                if sys.version_info >= (3, 6):
-                    self.loop.run_until_complete(self.loop.shutdown_asyncgens())
-                self.loop.close()
+        user = await self.db.lookup_user(username)
+        if user is None:
+            return None
+
+        return {
+            "username": user.username,
+            "permissions": self.return_perms(user.permissions),
+        }
+
+    @permissions(USER_ADMIN_PERM, allow_anon=False)
+    async def handle_get_all_users(self, request):
+        users = await self.db.get_all_users()
+        return {
+            "users": [
+                {
+                    "username": u.username,
+                    "permissions": self.return_perms(u.permissions),
+                }
+                for u in users
+            ]
+        }
 
-            if self._cleanup_socket is not None:
-                self._cleanup_socket()
+    @permissions(USER_ADMIN_PERM, allow_anon=False)
+    async def handle_new_user(self, request):
+        username = str(request["username"])
+        permissions = self.get_perm_arg(request["permissions"])
+
+        token = await new_token()
+
+        inserted = await self.db.new_user(
+            username,
+            permissions,
+            hash_token(TOKEN_ALGORITHM, new_salt(), token),
+        )
+        if not inserted:
+            raise bb.asyncrpc.InvokeError(f"Cannot create new user '{username}'")
+
+        return {
+            "username": username,
+            "permissions": self.return_perms(permissions),
+            "token": token,
+        }
+
+    @permissions(USER_ADMIN_PERM, allow_self_service=True, allow_anon=False)
+    async def handle_delete_user(self, request):
+        username = str(request["username"])
+
+        if not await self.db.delete_user(username):
+            self.raise_no_user_error(username)
+
+        return {"username": username}
+
+    @permissions(USER_ADMIN_PERM, allow_anon=False)
+    async def handle_become_user(self, request):
+        username = str(request["username"])
+
+        user = await self.db.lookup_user(username)
+        if user is None:
+            raise bb.asyncrpc.InvokeError(f"User {username} doesn't exist")
+
+        self.user = user
+
+        self.logger.info("Became user %s", username)
+
+        return {
+            "username": self.user.username,
+            "permissions": self.return_perms(self.user.permissions),
+        }
+
+
+class Server(bb.asyncrpc.AsyncServer):
+    def __init__(
+        self,
+        db_engine,
+        upstream=None,
+        read_only=False,
+        anon_perms=DEFAULT_ANON_PERMS,
+        admin_username=None,
+        admin_password=None,
+    ):
+        if upstream and read_only:
+            raise bb.asyncrpc.ServerError(
+                "Read-only hashserv cannot pull from an upstream server"
+            )
+
+        disallowed_perms = set(anon_perms) - set(
+            [NONE_PERM, READ_PERM, REPORT_PERM, DB_ADMIN_PERM]
+        )
+
+        if disallowed_perms:
+            raise bb.asyncrpc.ServerError(
+                f"Permission(s) {' '.join(disallowed_perms)} are not allowed for anonymous users"
+            )
+
+        super().__init__(logger)
+
+        self.request_stats = Stats()
+        self.db_engine = db_engine
+        self.upstream = upstream
+        self.read_only = read_only
+        self.backfill_queue = None
+        self.anon_perms = set(anon_perms)
+        self.admin_username = admin_username
+        self.admin_password = admin_password
+
+        self.logger.info(
+            "Anonymous user permissions are: %s", ", ".join(self.anon_perms)
+        )
+
+    def accept_client(self, socket):
+        return ServerClient(socket, self)
+
+    async def create_admin_user(self):
+        admin_permissions = (ALL_PERM,)
+        async with self.db_engine.connect(self.logger) as db:
+            added = await db.new_user(
+                self.admin_username,
+                admin_permissions,
+                hash_token(TOKEN_ALGORITHM, new_salt(), self.admin_password),
+            )
+            if added:
+                self.logger.info("Created admin user '%s'", self.admin_username)
+            else:
+                await db.set_user_perms(
+                    self.admin_username,
+                    admin_permissions,
+                )
+                await db.set_user_token(
+                    self.admin_username,
+                    hash_token(TOKEN_ALGORITHM, new_salt(), self.admin_password),
+                )
+                self.logger.info("Admin user '%s' updated", self.admin_username)
+
+    async def backfill_worker_task(self):
+        async with await create_async_client(
+            self.upstream
+        ) as client, self.db_engine.connect(self.logger) as db:
+            while True:
+                item = await self.backfill_queue.get()
+                if item is None:
+                    self.backfill_queue.task_done()
+                    break
+
+                method, taskhash = item
+                d = await client.get_taskhash(method, taskhash)
+                if d is not None:
+                    await db.insert_unihash(d["method"], d["taskhash"], d["unihash"])
+                self.backfill_queue.task_done()
+
+    def start(self):
+        tasks = super().start()
+        if self.upstream:
+            self.backfill_queue = asyncio.Queue()
+            tasks += [self.backfill_worker_task()]
+
+        self.loop.run_until_complete(self.db_engine.create())
+
+        if self.admin_username:
+            self.loop.run_until_complete(self.create_admin_user())
+
+        return tasks
+
+    async def stop(self):
+        if self.backfill_queue is not None:
+            await self.backfill_queue.put(None)
+        await super().stop()
diff --git a/bitbake/lib/hashserv/sqlalchemy.py b/bitbake/lib/hashserv/sqlalchemy.py
new file mode 100644
index 0000000000..f7b0226a7a
--- /dev/null
+++ b/bitbake/lib/hashserv/sqlalchemy.py
@@ -0,0 +1,598 @@
+#! /usr/bin/env python3
+#
+# Copyright (C) 2023 Garmin Ltd.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import logging
+from datetime import datetime
+from . import User
+
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.pool import NullPool
+from sqlalchemy import (
+    MetaData,
+    Column,
+    Table,
+    Text,
+    Integer,
+    UniqueConstraint,
+    DateTime,
+    Index,
+    select,
+    insert,
+    exists,
+    literal,
+    and_,
+    delete,
+    update,
+    func,
+    inspect,
+)
+import sqlalchemy.engine
+from sqlalchemy.orm import declarative_base
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.dialects.postgresql import insert as postgres_insert
+
+Base = declarative_base()
+
+
+class UnihashesV3(Base):
+    __tablename__ = "unihashes_v3"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    method = Column(Text, nullable=False)
+    taskhash = Column(Text, nullable=False)
+    unihash = Column(Text, nullable=False)
+    gc_mark = Column(Text, nullable=False)
+
+    __table_args__ = (
+        UniqueConstraint("method", "taskhash"),
+        Index("taskhash_lookup_v4", "method", "taskhash"),
+        Index("unihash_lookup_v1", "unihash"),
+    )
+
+
+class OuthashesV2(Base):
+    __tablename__ = "outhashes_v2"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    method = Column(Text, nullable=False)
+    taskhash = Column(Text, nullable=False)
+    outhash = Column(Text, nullable=False)
+    created = Column(DateTime)
+    owner = Column(Text)
+    PN = Column(Text)
+    PV = Column(Text)
+    PR = Column(Text)
+    task = Column(Text)
+    outhash_siginfo = Column(Text)
+
+    __table_args__ = (
+        UniqueConstraint("method", "taskhash", "outhash"),
+        Index("outhash_lookup_v3", "method", "outhash"),
+    )
+
+
+class Users(Base):
+    __tablename__ = "users"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    username = Column(Text, nullable=False)
+    token = Column(Text, nullable=False)
+    permissions = Column(Text)
+
+    __table_args__ = (UniqueConstraint("username"),)
+
+
+class Config(Base):
+    __tablename__ = "config"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    name = Column(Text, nullable=False)
+    value = Column(Text)
+    __table_args__ = (
+        UniqueConstraint("name"),
+        Index("config_lookup", "name"),
+    )
+
+
+#
+# Old table versions
+#
+DeprecatedBase = declarative_base()
+
+
+class UnihashesV2(DeprecatedBase):
+    __tablename__ = "unihashes_v2"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    method = Column(Text, nullable=False)
+    taskhash = Column(Text, nullable=False)
+    unihash = Column(Text, nullable=False)
+
+    __table_args__ = (
+        UniqueConstraint("method", "taskhash"),
+        Index("taskhash_lookup_v3", "method", "taskhash"),
+    )
+
+
+class DatabaseEngine(object):
+    def __init__(self, url, username=None, password=None):
+        self.logger = logging.getLogger("hashserv.sqlalchemy")
+        self.url = sqlalchemy.engine.make_url(url)
+
+        if username is not None:
+            self.url = self.url.set(username=username)
+
+        if password is not None:
+            self.url = self.url.set(password=password)
+
+    async def create(self):
+        def check_table_exists(conn, name):
+            return inspect(conn).has_table(name)
+
+        self.logger.info("Using database %s", self.url)
+        if self.url.drivername == 'postgresql+psycopg':
+            # Psygopg 3 (psygopg) driver can handle async connection pooling
+            self.engine = create_async_engine(self.url, max_overflow=-1)
+        else:
+            self.engine = create_async_engine(self.url, poolclass=NullPool)
+
+        async with self.engine.begin() as conn:
+            # Create tables
+            self.logger.info("Creating tables...")
+            await conn.run_sync(Base.metadata.create_all)
+
+            if await conn.run_sync(check_table_exists, UnihashesV2.__tablename__):
+                self.logger.info("Upgrading Unihashes V2 -> V3...")
+                statement = insert(UnihashesV3).from_select(
+                    ["id", "method", "unihash", "taskhash", "gc_mark"],
+                    select(
+                        UnihashesV2.id,
+                        UnihashesV2.method,
+                        UnihashesV2.unihash,
+                        UnihashesV2.taskhash,
+                        literal("").label("gc_mark"),
+                    ),
+                )
+                self.logger.debug("%s", statement)
+                await conn.execute(statement)
+
+                await conn.run_sync(Base.metadata.drop_all, [UnihashesV2.__table__])
+                self.logger.info("Upgrade complete")
+
+    def connect(self, logger):
+        return Database(self.engine, logger)
+
+
+def map_row(row):
+    if row is None:
+        return None
+    return dict(**row._mapping)
+
+
+def map_user(row):
+    if row is None:
+        return None
+    return User(
+        username=row.username,
+        permissions=set(row.permissions.split()),
+    )
+
+
+def _make_condition_statement(table, condition):
+    where = {}
+    for c in table.__table__.columns:
+        if c.key in condition and condition[c.key] is not None:
+            where[c] = condition[c.key]
+
+    return [(k == v) for k, v in where.items()]
+
+
+class Database(object):
+    def __init__(self, engine, logger):
+        self.engine = engine
+        self.db = None
+        self.logger = logger
+
+    async def __aenter__(self):
+        self.db = await self.engine.connect()
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.close()
+
+    async def close(self):
+        await self.db.close()
+        self.db = None
+
+    async def _execute(self, statement):
+        self.logger.debug("%s", statement)
+        return await self.db.execute(statement)
+
+    async def _set_config(self, name, value):
+        while True:
+            result = await self._execute(
+                update(Config).where(Config.name == name).values(value=value)
+            )
+
+            if result.rowcount == 0:
+                self.logger.debug("Config '%s' not found. Adding it", name)
+                try:
+                    await self._execute(insert(Config).values(name=name, value=value))
+                except IntegrityError:
+                    # Race. Try again
+                    continue
+
+            break
+
+    def _get_config_subquery(self, name, default=None):
+        if default is not None:
+            return func.coalesce(
+                select(Config.value).where(Config.name == name).scalar_subquery(),
+                default,
+            )
+        return select(Config.value).where(Config.name == name).scalar_subquery()
+
+    async def _get_config(self, name):
+        result = await self._execute(select(Config.value).where(Config.name == name))
+        row = result.first()
+        if row is None:
+            return None
+        return row.value
+
+    async def get_unihash_by_taskhash_full(self, method, taskhash):
+        async with self.db.begin():
+            result = await self._execute(
+                select(
+                    OuthashesV2,
+                    UnihashesV3.unihash.label("unihash"),
+                )
+                .join(
+                    UnihashesV3,
+                    and_(
+                        UnihashesV3.method == OuthashesV2.method,
+                        UnihashesV3.taskhash == OuthashesV2.taskhash,
+                    ),
+                )
+                .where(
+                    OuthashesV2.method == method,
+                    OuthashesV2.taskhash == taskhash,
+                )
+                .order_by(
+                    OuthashesV2.created.asc(),
+                )
+                .limit(1)
+            )
+            return map_row(result.first())
+
+    async def get_unihash_by_outhash(self, method, outhash):
+        async with self.db.begin():
+            result = await self._execute(
+                select(OuthashesV2, UnihashesV3.unihash.label("unihash"))
+                .join(
+                    UnihashesV3,
+                    and_(
+                        UnihashesV3.method == OuthashesV2.method,
+                        UnihashesV3.taskhash == OuthashesV2.taskhash,
+                    ),
+                )
+                .where(
+                    OuthashesV2.method == method,
+                    OuthashesV2.outhash == outhash,
+                )
+                .order_by(
+                    OuthashesV2.created.asc(),
+                )
+                .limit(1)
+            )
+            return map_row(result.first())
+
+    async def unihash_exists(self, unihash):
+        async with self.db.begin():
+            result = await self._execute(
+                select(UnihashesV3).where(UnihashesV3.unihash == unihash).limit(1)
+            )
+
+            return result.first() is not None
+
+    async def get_outhash(self, method, outhash):
+        async with self.db.begin():
+            result = await self._execute(
+                select(OuthashesV2)
+                .where(
+                    OuthashesV2.method == method,
+                    OuthashesV2.outhash == outhash,
+                )
+                .order_by(
+                    OuthashesV2.created.asc(),
+                )
+                .limit(1)
+            )
+            return map_row(result.first())
+
+    async def get_equivalent_for_outhash(self, method, outhash, taskhash):
+        async with self.db.begin():
+            result = await self._execute(
+                select(
+                    OuthashesV2.taskhash.label("taskhash"),
+                    UnihashesV3.unihash.label("unihash"),
+                )
+                .join(
+                    UnihashesV3,
+                    and_(
+                        UnihashesV3.method == OuthashesV2.method,
+                        UnihashesV3.taskhash == OuthashesV2.taskhash,
+                    ),
+                )
+                .where(
+                    OuthashesV2.method == method,
+                    OuthashesV2.outhash == outhash,
+                    OuthashesV2.taskhash != taskhash,
+                )
+                .order_by(
+                    OuthashesV2.created.asc(),
+                )
+                .limit(1)
+            )
+            return map_row(result.first())
+
+    async def get_equivalent(self, method, taskhash):
+        async with self.db.begin():
+            result = await self._execute(
+                select(
+                    UnihashesV3.unihash,
+                    UnihashesV3.method,
+                    UnihashesV3.taskhash,
+                ).where(
+                    UnihashesV3.method == method,
+                    UnihashesV3.taskhash == taskhash,
+                )
+            )
+            return map_row(result.first())
+
+    async def remove(self, condition):
+        async def do_remove(table):
+            where = _make_condition_statement(table, condition)
+            if where:
+                async with self.db.begin():
+                    result = await self._execute(delete(table).where(*where))
+                return result.rowcount
+
+            return 0
+
+        count = 0
+        count += await do_remove(UnihashesV3)
+        count += await do_remove(OuthashesV2)
+
+        return count
+
+    async def get_current_gc_mark(self):
+        async with self.db.begin():
+            return await self._get_config("gc-mark")
+
+    async def gc_status(self):
+        async with self.db.begin():
+            gc_mark_subquery = self._get_config_subquery("gc-mark", "")
+
+            result = await self._execute(
+                select(func.count())
+                .select_from(UnihashesV3)
+                .where(UnihashesV3.gc_mark == gc_mark_subquery)
+            )
+            keep_rows = result.scalar()
+
+            result = await self._execute(
+                select(func.count())
+                .select_from(UnihashesV3)
+                .where(UnihashesV3.gc_mark != gc_mark_subquery)
+            )
+            remove_rows = result.scalar()
+
+            return (keep_rows, remove_rows, await self._get_config("gc-mark"))
+
+    async def gc_mark(self, mark, condition):
+        async with self.db.begin():
+            await self._set_config("gc-mark", mark)
+
+            where = _make_condition_statement(UnihashesV3, condition)
+            if not where:
+                return 0
+
+            result = await self._execute(
+                update(UnihashesV3)
+                .values(gc_mark=self._get_config_subquery("gc-mark", ""))
+                .where(*where)
+            )
+            return result.rowcount
+
+    async def gc_sweep(self):
+        async with self.db.begin():
+            result = await self._execute(
+                delete(UnihashesV3).where(
+                    # A sneaky conditional that provides some errant use
+                    # protection: If the config mark is NULL, this will not
+                    # match any rows because No default is specified in the
+                    # select statement
+                    UnihashesV3.gc_mark
+                    != self._get_config_subquery("gc-mark")
+                )
+            )
+            await self._set_config("gc-mark", None)
+
+            return result.rowcount
+
+    async def clean_unused(self, oldest):
+        async with self.db.begin():
+            result = await self._execute(
+                delete(OuthashesV2).where(
+                    OuthashesV2.created < oldest,
+                    ~(
+                        select(UnihashesV3.id)
+                        .where(
+                            UnihashesV3.method == OuthashesV2.method,
+                            UnihashesV3.taskhash == OuthashesV2.taskhash,
+                        )
+                        .limit(1)
+                        .exists()
+                    ),
+                )
+            )
+            return result.rowcount
+
+    async def insert_unihash(self, method, taskhash, unihash):
+        # Postgres specific ignore on insert duplicate
+        if self.engine.name == "postgresql":
+            statement = (
+                postgres_insert(UnihashesV3)
+                .values(
+                    method=method,
+                    taskhash=taskhash,
+                    unihash=unihash,
+                    gc_mark=self._get_config_subquery("gc-mark", ""),
+                )
+                .on_conflict_do_nothing(index_elements=("method", "taskhash"))
+            )
+        else:
+            statement = insert(UnihashesV3).values(
+                method=method,
+                taskhash=taskhash,
+                unihash=unihash,
+                gc_mark=self._get_config_subquery("gc-mark", ""),
+            )
+
+        try:
+            async with self.db.begin():
+                result = await self._execute(statement)
+                return result.rowcount != 0
+        except IntegrityError:
+            self.logger.debug(
+                "%s, %s, %s already in unihash database", method, taskhash, unihash
+            )
+            return False
+
+    async def insert_outhash(self, data):
+        outhash_columns = set(c.key for c in OuthashesV2.__table__.columns)
+
+        data = {k: v for k, v in data.items() if k in outhash_columns}
+
+        if "created" in data and not isinstance(data["created"], datetime):
+            data["created"] = datetime.fromisoformat(data["created"])
+
+        # Postgres specific ignore on insert duplicate
+        if self.engine.name == "postgresql":
+            statement = (
+                postgres_insert(OuthashesV2)
+                .values(**data)
+                .on_conflict_do_nothing(
+                    index_elements=("method", "taskhash", "outhash")
+                )
+            )
+        else:
+            statement = insert(OuthashesV2).values(**data)
+
+        try:
+            async with self.db.begin():
+                result = await self._execute(statement)
+                return result.rowcount != 0
+        except IntegrityError:
+            self.logger.debug(
+                "%s, %s already in outhash database", data["method"], data["outhash"]
+            )
+            return False
+
+    async def _get_user(self, username):
+        async with self.db.begin():
+            result = await self._execute(
+                select(
+                    Users.username,
+                    Users.permissions,
+                    Users.token,
+                ).where(
+                    Users.username == username,
+                )
+            )
+            return result.first()
+
+    async def lookup_user_token(self, username):
+        row = await self._get_user(username)
+        if not row:
+            return None, None
+        return map_user(row), row.token
+
+    async def lookup_user(self, username):
+        return map_user(await self._get_user(username))
+
+    async def set_user_token(self, username, token):
+        async with self.db.begin():
+            result = await self._execute(
+                update(Users)
+                .where(
+                    Users.username == username,
+                )
+                .values(
+                    token=token,
+                )
+            )
+            return result.rowcount != 0
+
+    async def set_user_perms(self, username, permissions):
+        async with self.db.begin():
+            result = await self._execute(
+                update(Users)
+                .where(Users.username == username)
+                .values(permissions=" ".join(permissions))
+            )
+            return result.rowcount != 0
+
+    async def get_all_users(self):
+        async with self.db.begin():
+            result = await self._execute(
+                select(
+                    Users.username,
+                    Users.permissions,
+                )
+            )
+            return [map_user(row) for row in result]
+
+    async def new_user(self, username, permissions, token):
+        try:
+            async with self.db.begin():
+                await self._execute(
+                    insert(Users).values(
+                        username=username,
+                        permissions=" ".join(permissions),
+                        token=token,
+                    )
+                )
+            return True
+        except IntegrityError as e:
+            self.logger.debug("Cannot create new user %s: %s", username, e)
+            return False
+
+    async def delete_user(self, username):
+        async with self.db.begin():
+            result = await self._execute(
+                delete(Users).where(Users.username == username)
+            )
+            return result.rowcount != 0
+
+    async def get_usage(self):
+        usage = {}
+        async with self.db.begin() as session:
+            for name, table in Base.metadata.tables.items():
+                result = await self._execute(
+                    statement=select(func.count()).select_from(table)
+                )
+                usage[name] = {
+                    "rows": result.scalar(),
+                }
+
+        return usage
+
+    async def get_query_columns(self):
+        columns = set()
+        for table in (UnihashesV3, OuthashesV2):
+            for c in table.__table__.columns:
+                if not isinstance(c.type, Text):
+                    continue
+                columns.add(c.key)
+
+        return list(columns)
diff --git a/bitbake/lib/hashserv/sqlite.py b/bitbake/lib/hashserv/sqlite.py
new file mode 100644
index 0000000000..976504d7f4
--- /dev/null
+++ b/bitbake/lib/hashserv/sqlite.py
@@ -0,0 +1,579 @@
+#! /usr/bin/env python3
+#
+# Copyright (C) 2023 Garmin Ltd.
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+from datetime import datetime, timezone
+import sqlite3
+import logging
+from contextlib import closing
+from . import User
+
+logger = logging.getLogger("hashserv.sqlite")
+
+UNIHASH_TABLE_DEFINITION = (
+    ("method", "TEXT NOT NULL", "UNIQUE"),
+    ("taskhash", "TEXT NOT NULL", "UNIQUE"),
+    ("unihash", "TEXT NOT NULL", ""),
+    ("gc_mark", "TEXT NOT NULL", ""),
+)
+
+UNIHASH_TABLE_COLUMNS = tuple(name for name, _, _ in UNIHASH_TABLE_DEFINITION)
+
+OUTHASH_TABLE_DEFINITION = (
+    ("method", "TEXT NOT NULL", "UNIQUE"),
+    ("taskhash", "TEXT NOT NULL", "UNIQUE"),
+    ("outhash", "TEXT NOT NULL", "UNIQUE"),
+    ("created", "DATETIME", ""),
+    # Optional fields
+    ("owner", "TEXT", ""),
+    ("PN", "TEXT", ""),
+    ("PV", "TEXT", ""),
+    ("PR", "TEXT", ""),
+    ("task", "TEXT", ""),
+    ("outhash_siginfo", "TEXT", ""),
+)
+
+OUTHASH_TABLE_COLUMNS = tuple(name for name, _, _ in OUTHASH_TABLE_DEFINITION)
+
+USERS_TABLE_DEFINITION = (
+    ("username", "TEXT NOT NULL", "UNIQUE"),
+    ("token", "TEXT NOT NULL", ""),
+    ("permissions", "TEXT NOT NULL", ""),
+)
+
+USERS_TABLE_COLUMNS = tuple(name for name, _, _ in USERS_TABLE_DEFINITION)
+
+
+CONFIG_TABLE_DEFINITION = (
+    ("name", "TEXT NOT NULL", "UNIQUE"),
+    ("value", "TEXT", ""),
+)
+
+CONFIG_TABLE_COLUMNS = tuple(name for name, _, _ in CONFIG_TABLE_DEFINITION)
+
+
+def adapt_datetime_iso(val):
+    """Adapt datetime.datetime to UTC ISO 8601 date."""
+    return val.astimezone(timezone.utc).isoformat()
+
+
+sqlite3.register_adapter(datetime, adapt_datetime_iso)
+
+
+def convert_datetime(val):
+    """Convert ISO 8601 datetime to datetime.datetime object."""
+    return datetime.fromisoformat(val.decode())
+
+
+sqlite3.register_converter("DATETIME", convert_datetime)
+
+
+def _make_table(cursor, name, definition):
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS {name} (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            {fields}
+            UNIQUE({unique})
+            )
+        """.format(
+            name=name,
+            fields=" ".join("%s %s," % (name, typ) for name, typ, _ in definition),
+            unique=", ".join(
+                name for name, _, flags in definition if "UNIQUE" in flags
+            ),
+        )
+    )
+
+
+def map_user(row):
+    if row is None:
+        return None
+    return User(
+        username=row["username"],
+        permissions=set(row["permissions"].split()),
+    )
+
+
+def _make_condition_statement(columns, condition):
+    where = {}
+    for c in columns:
+        if c in condition and condition[c] is not None:
+            where[c] = condition[c]
+
+    return where, " AND ".join("%s=:%s" % (k, k) for k in where.keys())
+
+
+def _get_sqlite_version(cursor):
+    cursor.execute("SELECT sqlite_version()")
+
+    version = []
+    for v in cursor.fetchone()[0].split("."):
+        try:
+            version.append(int(v))
+        except ValueError:
+            version.append(v)
+
+    return tuple(version)
+
+
+def _schema_table_name(version):
+    if version >= (3, 33):
+        return "sqlite_schema"
+
+    return "sqlite_master"
+
+
+class DatabaseEngine(object):
+    def __init__(self, dbname, sync):
+        self.dbname = dbname
+        self.logger = logger
+        self.sync = sync
+
+    async def create(self):
+        db = sqlite3.connect(self.dbname)
+        db.row_factory = sqlite3.Row
+
+        with closing(db.cursor()) as cursor:
+            _make_table(cursor, "unihashes_v3", UNIHASH_TABLE_DEFINITION)
+            _make_table(cursor, "outhashes_v2", OUTHASH_TABLE_DEFINITION)
+            _make_table(cursor, "users", USERS_TABLE_DEFINITION)
+            _make_table(cursor, "config", CONFIG_TABLE_DEFINITION)
+
+            cursor.execute("PRAGMA journal_mode = WAL")
+            cursor.execute(
+                "PRAGMA synchronous = %s" % ("NORMAL" if self.sync else "OFF")
+            )
+
+            # Drop old indexes
+            cursor.execute("DROP INDEX IF EXISTS taskhash_lookup")
+            cursor.execute("DROP INDEX IF EXISTS outhash_lookup")
+            cursor.execute("DROP INDEX IF EXISTS taskhash_lookup_v2")
+            cursor.execute("DROP INDEX IF EXISTS outhash_lookup_v2")
+            cursor.execute("DROP INDEX IF EXISTS taskhash_lookup_v3")
+
+            # TODO: Upgrade from tasks_v2?
+            cursor.execute("DROP TABLE IF EXISTS tasks_v2")
+
+            # Create new indexes
+            cursor.execute(
+                "CREATE INDEX IF NOT EXISTS taskhash_lookup_v4 ON unihashes_v3 (method, taskhash)"
+            )
+            cursor.execute(
+                "CREATE INDEX IF NOT EXISTS unihash_lookup_v1 ON unihashes_v3 (unihash)"
+            )
+            cursor.execute(
+                "CREATE INDEX IF NOT EXISTS outhash_lookup_v3 ON outhashes_v2 (method, outhash)"
+            )
+            cursor.execute("CREATE INDEX IF NOT EXISTS config_lookup ON config (name)")
+
+            sqlite_version = _get_sqlite_version(cursor)
+
+            cursor.execute(
+                f"""
+                SELECT name FROM {_schema_table_name(sqlite_version)} WHERE type = 'table' AND name = 'unihashes_v2'
+                """
+            )
+            if cursor.fetchone():
+                self.logger.info("Upgrading Unihashes V2 -> V3...")
+                cursor.execute(
+                    """
+                    INSERT INTO unihashes_v3 (id, method, unihash, taskhash, gc_mark)
+                    SELECT id, method, unihash, taskhash, '' FROM unihashes_v2
+                    """
+                )
+                cursor.execute("DROP TABLE unihashes_v2")
+                db.commit()
+                self.logger.info("Upgrade complete")
+
+    def connect(self, logger):
+        return Database(logger, self.dbname, self.sync)
+
+
+class Database(object):
+    def __init__(self, logger, dbname, sync):
+        self.dbname = dbname
+        self.logger = logger
+
+        self.db = sqlite3.connect(self.dbname)
+        self.db.row_factory = sqlite3.Row
+
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute("PRAGMA journal_mode = WAL")
+            cursor.execute(
+                "PRAGMA synchronous = %s" % ("NORMAL" if sync else "OFF")
+            )
+
+            self.sqlite_version = _get_sqlite_version(cursor)
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.close()
+
+    async def _set_config(self, cursor, name, value):
+        cursor.execute(
+            """
+            INSERT OR REPLACE INTO config (id, name, value) VALUES
+            ((SELECT id FROM config WHERE name=:name), :name, :value)
+            """,
+            {
+                "name": name,
+                "value": value,
+            },
+        )
+
+    async def _get_config(self, cursor, name):
+        cursor.execute(
+            "SELECT value FROM config WHERE name=:name",
+            {
+                "name": name,
+            },
+        )
+        row = cursor.fetchone()
+        if row is None:
+            return None
+        return row["value"]
+
+    async def close(self):
+        self.db.close()
+
+    async def get_unihash_by_taskhash_full(self, method, taskhash):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT *, unihashes_v3.unihash AS unihash FROM outhashes_v2
+                INNER JOIN unihashes_v3 ON unihashes_v3.method=outhashes_v2.method AND unihashes_v3.taskhash=outhashes_v2.taskhash
+                WHERE outhashes_v2.method=:method AND outhashes_v2.taskhash=:taskhash
+                ORDER BY outhashes_v2.created ASC
+                LIMIT 1
+                """,
+                {
+                    "method": method,
+                    "taskhash": taskhash,
+                },
+            )
+            return cursor.fetchone()
+
+    async def get_unihash_by_outhash(self, method, outhash):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT *, unihashes_v3.unihash AS unihash FROM outhashes_v2
+                INNER JOIN unihashes_v3 ON unihashes_v3.method=outhashes_v2.method AND unihashes_v3.taskhash=outhashes_v2.taskhash
+                WHERE outhashes_v2.method=:method AND outhashes_v2.outhash=:outhash
+                ORDER BY outhashes_v2.created ASC
+                LIMIT 1
+                """,
+                {
+                    "method": method,
+                    "outhash": outhash,
+                },
+            )
+            return cursor.fetchone()
+
+    async def unihash_exists(self, unihash):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT * FROM unihashes_v3 WHERE unihash=:unihash
+                LIMIT 1
+                """,
+                {
+                    "unihash": unihash,
+                },
+            )
+            return cursor.fetchone() is not None
+
+    async def get_outhash(self, method, outhash):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT * FROM outhashes_v2
+                WHERE outhashes_v2.method=:method AND outhashes_v2.outhash=:outhash
+                ORDER BY outhashes_v2.created ASC
+                LIMIT 1
+                """,
+                {
+                    "method": method,
+                    "outhash": outhash,
+                },
+            )
+            return cursor.fetchone()
+
+    async def get_equivalent_for_outhash(self, method, outhash, taskhash):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT outhashes_v2.taskhash AS taskhash, unihashes_v3.unihash AS unihash FROM outhashes_v2
+                INNER JOIN unihashes_v3 ON unihashes_v3.method=outhashes_v2.method AND unihashes_v3.taskhash=outhashes_v2.taskhash
+                -- Select any matching output hash except the one we just inserted
+                WHERE outhashes_v2.method=:method AND outhashes_v2.outhash=:outhash AND outhashes_v2.taskhash!=:taskhash
+                -- Pick the oldest hash
+                ORDER BY outhashes_v2.created ASC
+                LIMIT 1
+                """,
+                {
+                    "method": method,
+                    "outhash": outhash,
+                    "taskhash": taskhash,
+                },
+            )
+            return cursor.fetchone()
+
+    async def get_equivalent(self, method, taskhash):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                "SELECT taskhash, method, unihash FROM unihashes_v3 WHERE method=:method AND taskhash=:taskhash",
+                {
+                    "method": method,
+                    "taskhash": taskhash,
+                },
+            )
+            return cursor.fetchone()
+
+    async def remove(self, condition):
+        def do_remove(columns, table_name, cursor):
+            where, clause = _make_condition_statement(columns, condition)
+            if where:
+                query = f"DELETE FROM {table_name} WHERE {clause}"
+                cursor.execute(query, where)
+                return cursor.rowcount
+
+            return 0
+
+        count = 0
+        with closing(self.db.cursor()) as cursor:
+            count += do_remove(OUTHASH_TABLE_COLUMNS, "outhashes_v2", cursor)
+            count += do_remove(UNIHASH_TABLE_COLUMNS, "unihashes_v3", cursor)
+            self.db.commit()
+
+        return count
+
+    async def get_current_gc_mark(self):
+        with closing(self.db.cursor()) as cursor:
+            return await self._get_config(cursor, "gc-mark")
+
+    async def gc_status(self):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT COUNT() FROM unihashes_v3 WHERE
+                    gc_mark=COALESCE((SELECT value FROM config WHERE name='gc-mark'), '')
+                """
+            )
+            keep_rows = cursor.fetchone()[0]
+
+            cursor.execute(
+                """
+                SELECT COUNT() FROM unihashes_v3 WHERE
+                    gc_mark!=COALESCE((SELECT value FROM config WHERE name='gc-mark'), '')
+                """
+            )
+            remove_rows = cursor.fetchone()[0]
+
+            current_mark = await self._get_config(cursor, "gc-mark")
+
+            return (keep_rows, remove_rows, current_mark)
+
+    async def gc_mark(self, mark, condition):
+        with closing(self.db.cursor()) as cursor:
+            await self._set_config(cursor, "gc-mark", mark)
+
+            where, clause = _make_condition_statement(UNIHASH_TABLE_COLUMNS, condition)
+
+            new_rows = 0
+            if where:
+                cursor.execute(
+                    f"""
+                    UPDATE unihashes_v3 SET
+                        gc_mark=COALESCE((SELECT value FROM config WHERE name='gc-mark'), '')
+                    WHERE {clause}
+                    """,
+                    where,
+                )
+                new_rows = cursor.rowcount
+
+            self.db.commit()
+            return new_rows
+
+    async def gc_sweep(self):
+        with closing(self.db.cursor()) as cursor:
+            # NOTE: COALESCE is not used in this query so that if the current
+            # mark is NULL, nothing will happen
+            cursor.execute(
+                """
+                DELETE FROM unihashes_v3 WHERE
+                    gc_mark!=(SELECT value FROM config WHERE name='gc-mark')
+                """
+            )
+            count = cursor.rowcount
+            await self._set_config(cursor, "gc-mark", None)
+
+            self.db.commit()
+            return count
+
+    async def clean_unused(self, oldest):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                DELETE FROM outhashes_v2 WHERE created<:oldest AND NOT EXISTS (
+                    SELECT unihashes_v3.id FROM unihashes_v3 WHERE unihashes_v3.method=outhashes_v2.method AND unihashes_v3.taskhash=outhashes_v2.taskhash LIMIT 1
+                )
+                """,
+                {
+                    "oldest": oldest,
+                },
+            )
+            self.db.commit()
+            return cursor.rowcount
+
+    async def insert_unihash(self, method, taskhash, unihash):
+        with closing(self.db.cursor()) as cursor:
+            prevrowid = cursor.lastrowid
+            cursor.execute(
+                """
+                INSERT OR IGNORE INTO unihashes_v3 (method, taskhash, unihash, gc_mark) VALUES
+                    (
+                    :method,
+                    :taskhash,
+                    :unihash,
+                    COALESCE((SELECT value FROM config WHERE name='gc-mark'), '')
+                    )
+                """,
+                {
+                    "method": method,
+                    "taskhash": taskhash,
+                    "unihash": unihash,
+                },
+            )
+            self.db.commit()
+            return cursor.lastrowid != prevrowid
+
+    async def insert_outhash(self, data):
+        data = {k: v for k, v in data.items() if k in OUTHASH_TABLE_COLUMNS}
+        keys = sorted(data.keys())
+        query = "INSERT OR IGNORE INTO outhashes_v2 ({fields}) VALUES({values})".format(
+            fields=", ".join(keys),
+            values=", ".join(":" + k for k in keys),
+        )
+        with closing(self.db.cursor()) as cursor:
+            prevrowid = cursor.lastrowid
+            cursor.execute(query, data)
+            self.db.commit()
+            return cursor.lastrowid != prevrowid
+
+    def _get_user(self, username):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                SELECT username, permissions, token FROM users WHERE username=:username
+                """,
+                {
+                    "username": username,
+                },
+            )
+            return cursor.fetchone()
+
+    async def lookup_user_token(self, username):
+        row = self._get_user(username)
+        if row is None:
+            return None, None
+        return map_user(row), row["token"]
+
+    async def lookup_user(self, username):
+        return map_user(self._get_user(username))
+
+    async def set_user_token(self, username, token):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                UPDATE users SET token=:token WHERE username=:username
+                """,
+                {
+                    "username": username,
+                    "token": token,
+                },
+            )
+            self.db.commit()
+            return cursor.rowcount != 0
+
+    async def set_user_perms(self, username, permissions):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                UPDATE users SET permissions=:permissions WHERE username=:username
+                """,
+                {
+                    "username": username,
+                    "permissions": " ".join(permissions),
+                },
+            )
+            self.db.commit()
+            return cursor.rowcount != 0
+
+    async def get_all_users(self):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute("SELECT username, permissions FROM users")
+            return [map_user(r) for r in cursor.fetchall()]
+
+    async def new_user(self, username, permissions, token):
+        with closing(self.db.cursor()) as cursor:
+            try:
+                cursor.execute(
+                    """
+                    INSERT INTO users (username, token, permissions) VALUES (:username, :token, :permissions)
+                    """,
+                    {
+                        "username": username,
+                        "token": token,
+                        "permissions": " ".join(permissions),
+                    },
+                )
+                self.db.commit()
+                return True
+            except sqlite3.IntegrityError:
+                return False
+
+    async def delete_user(self, username):
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                """
+                DELETE FROM users WHERE username=:username
+                """,
+                {
+                    "username": username,
+                },
+            )
+            self.db.commit()
+            return cursor.rowcount != 0
+
+    async def get_usage(self):
+        usage = {}
+        with closing(self.db.cursor()) as cursor:
+            cursor.execute(
+                f"""
+                SELECT name FROM {_schema_table_name(self.sqlite_version)} WHERE type = 'table' AND name NOT LIKE 'sqlite_%'
+                """
+            )
+            for row in cursor.fetchall():
+                cursor.execute(
+                    """
+                    SELECT COUNT() FROM %s
+                    """
+                    % row["name"],
+                )
+                usage[row["name"]] = {
+                    "rows": cursor.fetchone()[0],
+                }
+        return usage
+
+    async def get_query_columns(self):
+        columns = set()
+        for name, typ, _ in UNIHASH_TABLE_DEFINITION + OUTHASH_TABLE_DEFINITION:
+            if typ.startswith("TEXT"):
+                columns.add(name)
+        return list(columns)
diff --git a/bitbake/lib/hashserv/tests.py b/bitbake/lib/hashserv/tests.py
index 1a696481e3..da3f8e0884 100644
--- a/bitbake/lib/hashserv/tests.py
+++ b/bitbake/lib/hashserv/tests.py
@@ -6,7 +6,8 @@
 #
 
 from . import create_server, create_client
-from .client import HashConnectionError
+from .server import DEFAULT_ANON_PERMS, ALL_PERMISSIONS
+from bb.asyncrpc import InvokeError
 import hashlib
 import logging
 import multiprocessing
@@ -16,72 +17,161 @@ import tempfile
 import threading
 import unittest
 import socket
-
-def _run_server(server, idx):
-    # logging.basicConfig(level=logging.DEBUG, filename='bbhashserv.log', filemode='w',
-    #                     format='%(levelname)s %(filename)s:%(lineno)d %(message)s')
-    sys.stdout = open('bbhashserv-%d.log' % idx, 'w')
+import time
+import signal
+import subprocess
+import json
+import re
+from pathlib import Path
+
+
+THIS_DIR = Path(__file__).parent
+BIN_DIR = THIS_DIR.parent.parent / "bin"
+
+def server_prefunc(server, idx):
+    logging.basicConfig(level=logging.DEBUG, filename='bbhashserv-%d.log' % idx, filemode='w',
+                        format='%(levelname)s %(filename)s:%(lineno)d %(message)s')
+    server.logger.debug("Running server %d" % idx)
+    sys.stdout = open('bbhashserv-stdout-%d.log' % idx, 'w')
     sys.stderr = sys.stdout
-    server.serve_forever()
-
 
 class HashEquivalenceTestSetup(object):
     METHOD = 'TestMethod'
 
     server_index = 0
+    client_index = 0
 
-    def start_server(self, dbpath=None, upstream=None, read_only=False):
+    def start_server(self, dbpath=None, upstream=None, read_only=False, prefunc=server_prefunc, anon_perms=DEFAULT_ANON_PERMS, admin_username=None, admin_password=None):
         self.server_index += 1
         if dbpath is None:
-            dbpath = os.path.join(self.temp_dir.name, "db%d.sqlite" % self.server_index)
+            dbpath = self.make_dbpath()
+
+        def cleanup_server(server):
+            if server.process.exitcode is not None:
+                return
 
-        def cleanup_thread(thread):
-            thread.terminate()
-            thread.join()
+            server.process.terminate()
+            server.process.join()
 
         server = create_server(self.get_server_addr(self.server_index),
                                dbpath,
                                upstream=upstream,
-                               read_only=read_only)
+                               read_only=read_only,
+                               anon_perms=anon_perms,
+                               admin_username=admin_username,
+                               admin_password=admin_password)
         server.dbpath = dbpath
 
-        server.thread = multiprocessing.Process(target=_run_server, args=(server, self.server_index))
-        server.thread.start()
-        self.addCleanup(cleanup_thread, server.thread)
+        server.serve_as_process(prefunc=prefunc, args=(self.server_index,))
+        self.addCleanup(cleanup_server, server)
+
+        return server
+
+    def make_dbpath(self):
+        return os.path.join(self.temp_dir.name, "db%d.sqlite" % self.server_index)
 
+    def start_client(self, server_address, username=None, password=None):
         def cleanup_client(client):
             client.close()
 
-        client = create_client(server.address)
+        client = create_client(server_address, username=username, password=password)
         self.addCleanup(cleanup_client, client)
 
-        return (client, server)
+        return client
 
-    def setUp(self):
-        if sys.version_info < (3, 5, 0):
-            self.skipTest('Python 3.5 or later required')
+    def start_test_server(self):
+        self.server = self.start_server()
+        return self.server.address
+
+    def start_auth_server(self):
+        auth_server = self.start_server(self.server.dbpath, anon_perms=[], admin_username="admin", admin_password="password")
+        self.auth_server_address = auth_server.address
+        self.admin_client = self.start_client(auth_server.address, username="admin", password="password")
+        return self.admin_client
+
+    def auth_client(self, user):
+        return self.start_client(self.auth_server_address, user["username"], user["token"])
 
+    def setUp(self):
         self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-hashserv')
         self.addCleanup(self.temp_dir.cleanup)
 
-        (self.client, self.server) = self.start_server()
+        self.server_address = self.start_test_server()
+
+        self.client = self.start_client(self.server_address)
 
     def assertClientGetHash(self, client, taskhash, unihash):
         result = client.get_unihash(self.METHOD, taskhash)
         self.assertEqual(result, unihash)
 
+    def assertUserPerms(self, user, permissions):
+        with self.auth_client(user) as client:
+            info = client.get_user()
+            self.assertEqual(info, {
+                "username": user["username"],
+                "permissions": permissions,
+            })
 
-class HashEquivalenceCommonTests(object):
-    def test_create_hash(self):
+    def assertUserCanAuth(self, user):
+        with self.start_client(self.auth_server_address) as client:
+            client.auth(user["username"], user["token"])
+
+    def assertUserCannotAuth(self, user):
+        with self.start_client(self.auth_server_address) as client, self.assertRaises(InvokeError):
+            client.auth(user["username"], user["token"])
+
+    def create_test_hash(self, client):
         # Simple test that hashes can be created
         taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
         outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
         unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
 
-        self.assertClientGetHash(self.client, taskhash, None)
+        self.assertClientGetHash(client, taskhash, None)
 
-        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        result = client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+        return taskhash, outhash, unihash
+
+    def run_hashclient(self, args, **kwargs):
+        try:
+            p = subprocess.run(
+                [BIN_DIR / "bitbake-hashclient"] + args,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                encoding="utf-8",
+                **kwargs
+            )
+        except subprocess.CalledProcessError as e:
+            print(e.output)
+            raise e
+
+        print(p.stdout)
+        return p
+
+
+class HashEquivalenceCommonTests(object):
+    def auth_perms(self, *permissions):
+        self.client_index += 1
+        user = self.create_user(f"user-{self.client_index}", permissions)
+        return self.auth_client(user)
+
+    def create_user(self, username, permissions, *, client=None):
+        def remove_user(username):
+            try:
+                self.admin_client.delete_user(username)
+            except bb.asyncrpc.InvokeError:
+                pass
+
+        if client is None:
+            client = self.admin_client
+
+        user = client.new_user(username, permissions)
+        self.addCleanup(remove_user, username)
+
+        return user
+
+    def test_create_hash(self):
+        return self.create_test_hash(self.client)
 
     def test_create_equivalent(self):
         # Tests that a second reported task with the same outhash will be
@@ -123,6 +213,57 @@ class HashEquivalenceCommonTests(object):
 
         self.assertClientGetHash(self.client, taskhash, unihash)
 
+    def test_remove_taskhash(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+        result = self.client.remove({"taskhash": taskhash})
+        self.assertGreater(result["count"], 0)
+        self.assertClientGetHash(self.client, taskhash, None)
+
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash)
+        self.assertIsNone(result_outhash)
+
+    def test_remove_unihash(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+        result = self.client.remove({"unihash": unihash})
+        self.assertGreater(result["count"], 0)
+        self.assertClientGetHash(self.client, taskhash, None)
+
+    def test_remove_outhash(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+        result = self.client.remove({"outhash": outhash})
+        self.assertGreater(result["count"], 0)
+
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash)
+        self.assertIsNone(result_outhash)
+
+    def test_remove_method(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+        result = self.client.remove({"method": self.METHOD})
+        self.assertGreater(result["count"], 0)
+        self.assertClientGetHash(self.client, taskhash, None)
+
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash)
+        self.assertIsNone(result_outhash)
+
+    def test_clean_unused(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+
+        # Clean the database, which should not remove anything because all hashes an in-use
+        result = self.client.clean_unused(0)
+        self.assertEqual(result["count"], 0)
+        self.assertClientGetHash(self.client, taskhash, unihash)
+
+        # Remove the unihash. The row in the outhash table should still be present
+        self.client.remove({"unihash": unihash})
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash, False)
+        self.assertIsNotNone(result_outhash)
+
+        # Now clean with no minimum age which will remove the outhash
+        result = self.client.clean_unused(0)
+        self.assertEqual(result["count"], 1)
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash, False)
+        self.assertIsNone(result_outhash)
+
     def test_huge_message(self):
         # Simple test that hashes can be created
         taskhash = 'c665584ee6817aa99edfc77a44dd853828279370'
@@ -138,16 +279,21 @@ class HashEquivalenceCommonTests(object):
         })
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
 
-        result = self.client.get_taskhash(self.METHOD, taskhash, True)
-        self.assertEqual(result['taskhash'], taskhash)
-        self.assertEqual(result['unihash'], unihash)
-        self.assertEqual(result['method'], self.METHOD)
-        self.assertEqual(result['outhash'], outhash)
-        self.assertEqual(result['outhash_siginfo'], siginfo)
+        result_unihash = self.client.get_taskhash(self.METHOD, taskhash, True)
+        self.assertEqual(result_unihash['taskhash'], taskhash)
+        self.assertEqual(result_unihash['unihash'], unihash)
+        self.assertEqual(result_unihash['method'], self.METHOD)
+
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash)
+        self.assertEqual(result_outhash['taskhash'], taskhash)
+        self.assertEqual(result_outhash['method'], self.METHOD)
+        self.assertEqual(result_outhash['unihash'], unihash)
+        self.assertEqual(result_outhash['outhash'], outhash)
+        self.assertEqual(result_outhash['outhash_siginfo'], siginfo)
 
     def test_stress(self):
         def query_server(failures):
-            client = Client(self.server.address)
+            client = Client(self.server_address)
             try:
                 for i in range(1000):
                     taskhash = hashlib.sha256()
@@ -186,8 +332,10 @@ class HashEquivalenceCommonTests(object):
         # the side client. It also verifies that the results are pulled into
         # the downstream database by checking that the downstream and side servers
         # match after the downstream is done waiting for all backfill tasks
-        (down_client, down_server) = self.start_server(upstream=self.server.address)
-        (side_client, side_server) = self.start_server(dbpath=down_server.dbpath)
+        down_server = self.start_server(upstream=self.server_address)
+        down_client = self.start_client(down_server.address)
+        side_server = self.start_server(dbpath=down_server.dbpath)
+        side_client = self.start_client(side_server.address)
 
         def check_hash(taskhash, unihash, old_sidehash):
             nonlocal down_client
@@ -258,15 +406,57 @@ class HashEquivalenceCommonTests(object):
         result = down_client.report_unihash(taskhash6, self.METHOD, outhash5, unihash6)
         self.assertEqual(result['unihash'], unihash5, 'Server failed to copy unihash from upstream')
 
+        # Tests read through from server with
+        taskhash7 = '9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74'
+        outhash7 = '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69'
+        unihash7 = '05d2a63c81e32f0a36542ca677e8ad852365c538'
+        self.client.report_unihash(taskhash7, self.METHOD, outhash7, unihash7)
+
+        result = down_client.get_taskhash(self.METHOD, taskhash7, True)
+        self.assertEqual(result['unihash'], unihash7, 'Server failed to copy unihash from upstream')
+        self.assertEqual(result['outhash'], outhash7, 'Server failed to copy unihash from upstream')
+        self.assertEqual(result['taskhash'], taskhash7, 'Server failed to copy unihash from upstream')
+        self.assertEqual(result['method'], self.METHOD)
+
+        taskhash8 = '86978a4c8c71b9b487330b0152aade10c1ee58aa'
+        outhash8 = 'ca8c128e9d9e4a28ef24d0508aa20b5cf880604eacd8f65c0e366f7e0cc5fbcf'
+        unihash8 = 'd8bcf25369d40590ad7d08c84d538982f2023e01'
+        self.client.report_unihash(taskhash8, self.METHOD, outhash8, unihash8)
+
+        result = down_client.get_outhash(self.METHOD, outhash8, taskhash8)
+        self.assertEqual(result['unihash'], unihash8, 'Server failed to copy unihash from upstream')
+        self.assertEqual(result['outhash'], outhash8, 'Server failed to copy unihash from upstream')
+        self.assertEqual(result['taskhash'], taskhash8, 'Server failed to copy unihash from upstream')
+        self.assertEqual(result['method'], self.METHOD)
+
+        taskhash9 = 'ae6339531895ddf5b67e663e6a374ad8ec71d81c'
+        outhash9 = 'afc78172c81880ae10a1fec994b5b4ee33d196a001a1b66212a15ebe573e00b5'
+        unihash9 = '6662e699d6e3d894b24408ff9a4031ef9b038ee8'
+        self.client.report_unihash(taskhash9, self.METHOD, outhash9, unihash9)
+
+        result = down_client.get_taskhash(self.METHOD, taskhash9, False)
+        self.assertEqual(result['unihash'], unihash9, 'Server failed to copy unihash from upstream')
+        self.assertEqual(result['taskhash'], taskhash9, 'Server failed to copy unihash from upstream')
+        self.assertEqual(result['method'], self.METHOD)
+
+    def test_unihash_exsits(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+        self.assertTrue(self.client.unihash_exists(unihash))
+        self.assertFalse(self.client.unihash_exists('6662e699d6e3d894b24408ff9a4031ef9b038ee8'))
+
     def test_ro_server(self):
-        (ro_client, ro_server) = self.start_server(dbpath=self.server.dbpath, read_only=True)
+        rw_server = self.start_server()
+        rw_client = self.start_client(rw_server.address)
+
+        ro_server = self.start_server(dbpath=rw_server.dbpath, read_only=True)
+        ro_client = self.start_client(ro_server.address)
 
         # Report a hash via the read-write server
         taskhash = '35788efcb8dfb0a02659d81cf2bfd695fb30faf9'
         outhash = '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f'
         unihash = 'f46d3fbb439bd9b921095da657a4de906510d2cd'
 
-        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        result = rw_client.report_unihash(taskhash, self.METHOD, outhash, unihash)
         self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
 
         # Check the hash via the read-only server
@@ -277,11 +467,976 @@ class HashEquivalenceCommonTests(object):
         outhash2 = '3c979c3db45c569f51ab7626a4651074be3a9d11a84b1db076f5b14f7d39db44'
         unihash2 = '90e9bc1d1f094c51824adca7f8ea79a048d68824'
 
-        with self.assertRaises(HashConnectionError):
-            ro_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
+        result = ro_client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
+        self.assertEqual(result['unihash'], unihash2)
 
         # Ensure that the database was not modified
+        self.assertClientGetHash(rw_client, taskhash2, None)
+
+
+    def test_slow_server_start(self):
+        # Ensures that the server will exit correctly even if it gets a SIGTERM
+        # before entering the main loop
+
+        event = multiprocessing.Event()
+
+        def prefunc(server, idx):
+            nonlocal event
+            server_prefunc(server, idx)
+            event.wait()
+
+        def do_nothing(signum, frame):
+            pass
+
+        old_signal = signal.signal(signal.SIGTERM, do_nothing)
+        self.addCleanup(signal.signal, signal.SIGTERM, old_signal)
+
+        server = self.start_server(prefunc=prefunc)
+        server.process.terminate()
+        time.sleep(30)
+        event.set()
+        server.process.join(300)
+        self.assertIsNotNone(server.process.exitcode, "Server did not exit in a timely manner!")
+
+    def test_diverging_report_race(self):
+        # Tests that a reported task will correctly pick up an updated unihash
+
+        # This is a baseline report added to the database to ensure that there
+        # is something to match against as equivalent
+        outhash1 = 'afd11c366050bcd75ad763e898e4430e2a60659b26f83fbb22201a60672019fa'
+        taskhash1 = '3bde230c743fc45ab61a065d7a1815fbfa01c4740e4c895af2eb8dc0f684a4ab'
+        unihash1 = '3bde230c743fc45ab61a065d7a1815fbfa01c4740e4c895af2eb8dc0f684a4ab'
+        result = self.client.report_unihash(taskhash1, self.METHOD, outhash1, unihash1)
+
+        # Add a report that is equivalent to Task 1. It should ignore the
+        # provided unihash and report the unihash from task 1
+        taskhash2 = '6259ae8263bd94d454c086f501c37e64c4e83cae806902ca95b4ab513546b273'
+        unihash2 = taskhash2
+        result = self.client.report_unihash(taskhash2, self.METHOD, outhash1, unihash2)
+        self.assertEqual(result['unihash'], unihash1)
+
+        # Add another report for Task 2, but with a different outhash (e.g. the
+        # task is non-deterministic). It should still be marked with the Task 1
+        # unihash because it has the Task 2 taskhash, which is equivalent to
+        # Task 1
+        outhash3 = 'd2187ee3a8966db10b34fe0e863482288d9a6185cb8ef58a6c1c6ace87a2f24c'
+        result = self.client.report_unihash(taskhash2, self.METHOD, outhash3, unihash2)
+        self.assertEqual(result['unihash'], unihash1)
+
+
+    def test_diverging_report_reverse_race(self):
+        # Same idea as the previous test, but Tasks 2 and 3 are reported in
+        # reverse order the opposite order
+
+        outhash1 = 'afd11c366050bcd75ad763e898e4430e2a60659b26f83fbb22201a60672019fa'
+        taskhash1 = '3bde230c743fc45ab61a065d7a1815fbfa01c4740e4c895af2eb8dc0f684a4ab'
+        unihash1 = '3bde230c743fc45ab61a065d7a1815fbfa01c4740e4c895af2eb8dc0f684a4ab'
+        result = self.client.report_unihash(taskhash1, self.METHOD, outhash1, unihash1)
+
+        taskhash2 = '6259ae8263bd94d454c086f501c37e64c4e83cae806902ca95b4ab513546b273'
+        unihash2 = taskhash2
+
+        # Report Task 3 first. Since there is nothing else in the database it
+        # will use the client provided unihash
+        outhash3 = 'd2187ee3a8966db10b34fe0e863482288d9a6185cb8ef58a6c1c6ace87a2f24c'
+        result = self.client.report_unihash(taskhash2, self.METHOD, outhash3, unihash2)
+        self.assertEqual(result['unihash'], unihash2)
+
+        # Report Task 2. This is equivalent to Task 1 but there is already a mapping for
+        # taskhash2 so it will report unihash2
+        result = self.client.report_unihash(taskhash2, self.METHOD, outhash1, unihash2)
+        self.assertEqual(result['unihash'], unihash2)
+
+        # The originally reported unihash for Task 3 should be unchanged even if it
+        # shares a taskhash with Task 2
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
+    def test_get_unihash_batch(self):
+        TEST_INPUT = (
+            # taskhash                                   outhash                                                            unihash
+            ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e','218e57509998197d570e2c98512d0105985dffc9'),
+            # Duplicated taskhash with multiple output hashes and unihashes.
+            ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'),
+            # Equivalent hash
+            ("044c2ec8aaf480685a00ff6ff49e6162e6ad34e1", '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', "def64766090d28f627e816454ed46894bb3aab36"),
+            ("e3da00593d6a7fb435c7e2114976c59c5fd6d561", "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a", "3b5d3d83f07f259e9086fcb422c855286e18a57d"),
+            ('35788efcb8dfb0a02659d81cf2bfd695fb30faf9', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2cd'),
+            ('35788efcb8dfb0a02659d81cf2bfd695fb30fafa', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2ce'),
+            ('9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74', '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69', '05d2a63c81e32f0a36542ca677e8ad852365c538'),
+        )
+        EXTRA_QUERIES = (
+            "6b6be7a84ab179b4240c4302518dc3f6",
+        )
+
+        for taskhash, outhash, unihash in TEST_INPUT:
+            self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+
+
+        result = self.client.get_unihash_batch(
+            [(self.METHOD, data[0]) for data in TEST_INPUT] +
+            [(self.METHOD, e) for e in EXTRA_QUERIES]
+        )
+
+        self.assertListEqual(result, [
+            "218e57509998197d570e2c98512d0105985dffc9",
+            "218e57509998197d570e2c98512d0105985dffc9",
+            "218e57509998197d570e2c98512d0105985dffc9",
+            "3b5d3d83f07f259e9086fcb422c855286e18a57d",
+            "f46d3fbb439bd9b921095da657a4de906510d2cd",
+            "f46d3fbb439bd9b921095da657a4de906510d2cd",
+            "05d2a63c81e32f0a36542ca677e8ad852365c538",
+            None,
+        ])
+
+    def test_unihash_exists_batch(self):
+        TEST_INPUT = (
+            # taskhash                                   outhash                                                            unihash
+            ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', 'afe240a439959ce86f5e322f8c208e1fedefea9e813f2140c81af866cc9edf7e','218e57509998197d570e2c98512d0105985dffc9'),
+            # Duplicated taskhash with multiple output hashes and unihashes.
+            ('8aa96fcffb5831b3c2c0cb75f0431e3f8b20554a', '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', 'ae9a7d252735f0dafcdb10e2e02561ca3a47314c'),
+            # Equivalent hash
+            ("044c2ec8aaf480685a00ff6ff49e6162e6ad34e1", '0904a7fe3dc712d9fd8a74a616ddca2a825a8ee97adf0bd3fc86082c7639914d', "def64766090d28f627e816454ed46894bb3aab36"),
+            ("e3da00593d6a7fb435c7e2114976c59c5fd6d561", "1cf8713e645f491eb9c959d20b5cae1c47133a292626dda9b10709857cbe688a", "3b5d3d83f07f259e9086fcb422c855286e18a57d"),
+            ('35788efcb8dfb0a02659d81cf2bfd695fb30faf9', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2cd'),
+            ('35788efcb8dfb0a02659d81cf2bfd695fb30fafa', '2765d4a5884be49b28601445c2760c5f21e7e5c0ee2b7e3fce98fd7e5970796f', 'f46d3fbb439bd9b921095da657a4de906510d2ce'),
+            ('9d81d76242cc7cfaf7bf74b94b9cd2e29324ed74', '8470d56547eea6236d7c81a644ce74670ca0bbda998e13c629ef6bb3f0d60b69', '05d2a63c81e32f0a36542ca677e8ad852365c538'),
+        )
+        EXTRA_QUERIES = (
+            "6b6be7a84ab179b4240c4302518dc3f6",
+        )
+
+        result_unihashes = set()
+
+
+        for taskhash, outhash, unihash in TEST_INPUT:
+            result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+            result_unihashes.add(result["unihash"])
+
+        query = []
+        expected = []
+
+        for _, _, unihash in TEST_INPUT:
+            query.append(unihash)
+            expected.append(unihash in result_unihashes)
+
+
+        for unihash in EXTRA_QUERIES:
+            query.append(unihash)
+            expected.append(False)
+
+        result = self.client.unihash_exists_batch(query)
+        self.assertListEqual(result, expected)
+
+    def test_auth_read_perms(self):
+        admin_client = self.start_auth_server()
+
+        # Create hashes with non-authenticated server
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+
+        # Validate hash can be retrieved using authenticated client
+        with self.auth_perms("@read") as client:
+            self.assertClientGetHash(client, taskhash, unihash)
+
+        with self.auth_perms() as client, self.assertRaises(InvokeError):
+            self.assertClientGetHash(client, taskhash, unihash)
+
+    def test_auth_report_perms(self):
+        admin_client = self.start_auth_server()
+
+        # Without read permission, the user is completely denied
+        with self.auth_perms() as client, self.assertRaises(InvokeError):
+            self.create_test_hash(client)
+
+        # Read permission allows the call to succeed, but it doesn't record
+        # anythin in the database
+        with self.auth_perms("@read") as client:
+            taskhash, outhash, unihash = self.create_test_hash(client)
+            self.assertClientGetHash(client, taskhash, None)
+
+        # Report permission alone is insufficient
+        with self.auth_perms("@report") as client, self.assertRaises(InvokeError):
+            self.create_test_hash(client)
+
+        # Read and report permission actually modify the database
+        with self.auth_perms("@read", "@report") as client:
+            taskhash, outhash, unihash = self.create_test_hash(client)
+            self.assertClientGetHash(client, taskhash, unihash)
+
+    def test_auth_no_token_refresh_from_anon_user(self):
+        self.start_auth_server()
+
+        with self.start_client(self.auth_server_address) as client, self.assertRaises(InvokeError):
+            client.refresh_token()
+
+    def test_auth_self_token_refresh(self):
+        admin_client = self.start_auth_server()
+
+        # Create a new user with no permissions
+        user = self.create_user("test-user", [])
+
+        with self.auth_client(user) as client:
+            new_user = client.refresh_token()
+
+        self.assertEqual(user["username"], new_user["username"])
+        self.assertNotEqual(user["token"], new_user["token"])
+        self.assertUserCanAuth(new_user)
+        self.assertUserCannotAuth(user)
+
+        # Explicitly specifying with your own username is fine also
+        with self.auth_client(new_user) as client:
+            new_user2 = client.refresh_token(user["username"])
+
+        self.assertEqual(user["username"], new_user2["username"])
+        self.assertNotEqual(user["token"], new_user2["token"])
+        self.assertUserCanAuth(new_user2)
+        self.assertUserCannotAuth(new_user)
+        self.assertUserCannotAuth(user)
+
+    def test_auth_token_refresh(self):
+        admin_client = self.start_auth_server()
+
+        user = self.create_user("test-user", [])
+
+        with self.auth_perms() as client, self.assertRaises(InvokeError):
+            client.refresh_token(user["username"])
+
+        with self.auth_perms("@user-admin") as client:
+            new_user = client.refresh_token(user["username"])
+
+        self.assertEqual(user["username"], new_user["username"])
+        self.assertNotEqual(user["token"], new_user["token"])
+        self.assertUserCanAuth(new_user)
+        self.assertUserCannotAuth(user)
+
+    def test_auth_self_get_user(self):
+        admin_client = self.start_auth_server()
+
+        user = self.create_user("test-user", [])
+        user_info = user.copy()
+        del user_info["token"]
+
+        with self.auth_client(user) as client:
+            info = client.get_user()
+            self.assertEqual(info, user_info)
+
+            # Explicitly asking for your own username is fine also
+            info = client.get_user(user["username"])
+            self.assertEqual(info, user_info)
+
+    def test_auth_get_user(self):
+        admin_client = self.start_auth_server()
+
+        user = self.create_user("test-user", [])
+        user_info = user.copy()
+        del user_info["token"]
+
+        with self.auth_perms() as client, self.assertRaises(InvokeError):
+            client.get_user(user["username"])
+
+        with self.auth_perms("@user-admin") as client:
+            info = client.get_user(user["username"])
+            self.assertEqual(info, user_info)
+
+            info = client.get_user("nonexist-user")
+            self.assertIsNone(info)
+
+    def test_auth_reconnect(self):
+        admin_client = self.start_auth_server()
+
+        user = self.create_user("test-user", [])
+        user_info = user.copy()
+        del user_info["token"]
+
+        with self.auth_client(user) as client:
+            info = client.get_user()
+            self.assertEqual(info, user_info)
+
+            client.disconnect()
+
+            info = client.get_user()
+            self.assertEqual(info, user_info)
+
+    def test_auth_delete_user(self):
+        admin_client = self.start_auth_server()
+
+        user = self.create_user("test-user", [])
+
+        # self service
+        with self.auth_client(user) as client:
+            client.delete_user(user["username"])
+
+        self.assertIsNone(admin_client.get_user(user["username"]))
+        user = self.create_user("test-user", [])
+
+        with self.auth_perms() as client, self.assertRaises(InvokeError):
+            client.delete_user(user["username"])
+
+        with self.auth_perms("@user-admin") as client:
+            client.delete_user(user["username"])
+
+        # User doesn't exist, so even though the permission is correct, it's an
+        # error
+        with self.auth_perms("@user-admin") as client, self.assertRaises(InvokeError):
+            client.delete_user(user["username"])
+
+    def test_auth_set_user_perms(self):
+        admin_client = self.start_auth_server()
+
+        user = self.create_user("test-user", [])
+
+        self.assertUserPerms(user, [])
+
+        # No self service to change permissions
+        with self.auth_client(user) as client, self.assertRaises(InvokeError):
+            client.set_user_perms(user["username"], ["@all"])
+        self.assertUserPerms(user, [])
+
+        with self.auth_perms() as client, self.assertRaises(InvokeError):
+            client.set_user_perms(user["username"], ["@all"])
+        self.assertUserPerms(user, [])
+
+        with self.auth_perms("@user-admin") as client:
+            client.set_user_perms(user["username"], ["@all"])
+        self.assertUserPerms(user, sorted(list(ALL_PERMISSIONS)))
+
+        # Bad permissions
+        with self.auth_perms("@user-admin") as client, self.assertRaises(InvokeError):
+            client.set_user_perms(user["username"], ["@this-is-not-a-permission"])
+        self.assertUserPerms(user, sorted(list(ALL_PERMISSIONS)))
+
+    def test_auth_get_all_users(self):
+        admin_client = self.start_auth_server()
+
+        user = self.create_user("test-user", [])
+
+        with self.auth_client(user) as client, self.assertRaises(InvokeError):
+            client.get_all_users()
+
+        # Give the test user the correct permission
+        admin_client.set_user_perms(user["username"], ["@user-admin"])
+
+        with self.auth_client(user) as client:
+            all_users = client.get_all_users()
+
+        # Convert to a dictionary for easier comparison
+        all_users = {u["username"]: u for u in all_users}
+
+        self.assertEqual(all_users,
+            {
+                "admin": {
+                    "username": "admin",
+                    "permissions": sorted(list(ALL_PERMISSIONS)),
+                },
+                "test-user": {
+                    "username": "test-user",
+                    "permissions": ["@user-admin"],
+                }
+            }
+        )
+
+    def test_auth_new_user(self):
+        self.start_auth_server()
+
+        permissions = ["@read", "@report", "@db-admin", "@user-admin"]
+        permissions.sort()
+
+        with self.auth_perms() as client, self.assertRaises(InvokeError):
+            self.create_user("test-user", permissions, client=client)
+
+        with self.auth_perms("@user-admin") as client:
+            user = self.create_user("test-user", permissions, client=client)
+            self.assertIn("token", user)
+            self.assertEqual(user["username"], "test-user")
+            self.assertEqual(user["permissions"], permissions)
+
+    def test_auth_become_user(self):
+        admin_client = self.start_auth_server()
+
+        user = self.create_user("test-user", ["@read", "@report"])
+        user_info = user.copy()
+        del user_info["token"]
+
+        with self.auth_perms() as client, self.assertRaises(InvokeError):
+            client.become_user(user["username"])
+
+        with self.auth_perms("@user-admin") as client:
+            become = client.become_user(user["username"])
+            self.assertEqual(become, user_info)
+
+            info = client.get_user()
+            self.assertEqual(info, user_info)
+
+            # Verify become user is preserved across disconnect
+            client.disconnect()
+
+            info = client.get_user()
+            self.assertEqual(info, user_info)
+
+            # test-user doesn't have become_user permissions, so this should
+            # not work
+            with self.assertRaises(InvokeError):
+                client.become_user(user["username"])
+
+        # No self-service of become
+        with self.auth_client(user) as client, self.assertRaises(InvokeError):
+            client.become_user(user["username"])
+
+        # Give test user permissions to become
+        admin_client.set_user_perms(user["username"], ["@user-admin"])
+
+        # It's possible to become yourself (effectively a noop)
+        with self.auth_perms("@user-admin") as client:
+            become = client.become_user(client.username)
+
+    def test_auth_gc(self):
+        admin_client = self.start_auth_server()
+
+        with self.auth_perms() as client, self.assertRaises(InvokeError):
+            client.gc_mark("ABC", {"unihash": "123"})
+
+        with self.auth_perms() as client, self.assertRaises(InvokeError):
+            client.gc_status()
+
+        with self.auth_perms() as client, self.assertRaises(InvokeError):
+            client.gc_sweep("ABC")
+
+        with self.auth_perms("@db-admin") as client:
+            client.gc_mark("ABC", {"unihash": "123"})
+
+        with self.auth_perms("@db-admin") as client:
+            client.gc_status()
+
+        with self.auth_perms("@db-admin") as client:
+            client.gc_sweep("ABC")
+
+    def test_get_db_usage(self):
+        usage = self.client.get_db_usage()
+
+        self.assertTrue(isinstance(usage, dict))
+        for name in usage.keys():
+            self.assertTrue(isinstance(usage[name], dict))
+            self.assertIn("rows", usage[name])
+            self.assertTrue(isinstance(usage[name]["rows"], int))
+
+    def test_get_db_query_columns(self):
+        columns = self.client.get_db_query_columns()
+
+        self.assertTrue(isinstance(columns, list))
+        self.assertTrue(len(columns) > 0)
+
+        for col in columns:
+            self.client.remove({col: ""})
+
+    def test_auth_is_owner(self):
+        admin_client = self.start_auth_server()
+
+        user = self.create_user("test-user", ["@read", "@report"])
+        with self.auth_client(user) as client:
+            taskhash, outhash, unihash = self.create_test_hash(client)
+            data = client.get_taskhash(self.METHOD, taskhash, True)
+            self.assertEqual(data["owner"], user["username"])
+
+    def test_gc(self):
+        taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
+        outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
+        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+
+        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+
+        taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
+        outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
+        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+
+        result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
+        # Mark the first unihash to be kept
+        ret = self.client.gc_mark("ABC", {"unihash": unihash, "method": self.METHOD})
+        self.assertEqual(ret, {"count": 1})
+
+        ret = self.client.gc_status()
+        self.assertEqual(ret, {"mark": "ABC", "keep": 1, "remove": 1})
+
+        # Second hash is still there; mark doesn't delete hashes
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
+        ret = self.client.gc_sweep("ABC")
+        self.assertEqual(ret, {"count": 1})
+
+        # Hash is gone. Taskhash is returned for second hash
+        self.assertClientGetHash(self.client, taskhash2, None)
+        # First hash is still present
+        self.assertClientGetHash(self.client, taskhash, unihash)
+
+    def test_gc_stream(self):
+        taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
+        outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
+        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+
+        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+
+        taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
+        outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
+        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+
+        result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
+        taskhash3 = 'a1117c1f5a7c9ab2f5a39cc6fe5e6152169d09c0'
+        outhash3 = '7289c414905303700a1117c1f5a7c9ab2f5a39cc6fe5e6152169d09c04f9a53c'
+        unihash3 = '905303700a1117c1f5a7c9ab2f5a39cc6fe5e615'
+
+        result = self.client.report_unihash(taskhash3, self.METHOD, outhash3, unihash3)
+        self.assertClientGetHash(self.client, taskhash3, unihash3)
+
+        # Mark the first unihash to be kept
+        ret = self.client.gc_mark_stream("ABC", (f"unihash {h}" for h in [unihash, unihash2]))
+        self.assertEqual(ret, {"count": 2})
+
+        ret = self.client.gc_status()
+        self.assertEqual(ret, {"mark": "ABC", "keep": 2, "remove": 1})
+
+        # Third hash is still there; mark doesn't delete hashes
+        self.assertClientGetHash(self.client, taskhash3, unihash3)
+
+        ret = self.client.gc_sweep("ABC")
+        self.assertEqual(ret, {"count": 1})
+
+        # Hash is gone. Taskhash is returned for second hash
+        self.assertClientGetHash(self.client, taskhash3, None)
+        # First hash is still present
+        self.assertClientGetHash(self.client, taskhash, unihash)
+        # Second hash is still present
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
+    def test_gc_switch_mark(self):
+        taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
+        outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
+        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+
+        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+
+        taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
+        outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
+        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+
+        result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
+        # Mark the first unihash to be kept
+        ret = self.client.gc_mark("ABC", {"unihash": unihash, "method": self.METHOD})
+        self.assertEqual(ret, {"count": 1})
+
+        ret = self.client.gc_status()
+        self.assertEqual(ret, {"mark": "ABC", "keep": 1, "remove": 1})
+
+        # Second hash is still there; mark doesn't delete hashes
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
+        # Switch to a different mark and mark the second hash. This will start
+        # a new collection cycle
+        ret = self.client.gc_mark("DEF", {"unihash": unihash2, "method": self.METHOD})
+        self.assertEqual(ret, {"count": 1})
+
+        ret = self.client.gc_status()
+        self.assertEqual(ret, {"mark": "DEF", "keep": 1, "remove": 1})
+
+        # Both hashes are still present
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+        self.assertClientGetHash(self.client, taskhash, unihash)
+
+        # Sweep with the new mark
+        ret = self.client.gc_sweep("DEF")
+        self.assertEqual(ret, {"count": 1})
+
+        # First hash is gone, second is kept
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+        self.assertClientGetHash(self.client, taskhash, None)
+
+    def test_gc_switch_sweep_mark(self):
+        taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
+        outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
+        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+
+        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+
+        taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
+        outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
+        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+
+        result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
+        # Mark the first unihash to be kept
+        ret = self.client.gc_mark("ABC", {"unihash": unihash, "method": self.METHOD})
+        self.assertEqual(ret, {"count": 1})
+
+        ret = self.client.gc_status()
+        self.assertEqual(ret, {"mark": "ABC", "keep": 1, "remove": 1})
+
+        # Sweeping with a different mark raises an error
+        with self.assertRaises(InvokeError):
+            self.client.gc_sweep("DEF")
+
+        # Both hashes are present
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+        self.assertClientGetHash(self.client, taskhash, unihash)
+
+    def test_gc_new_hashes(self):
+        taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
+        outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
+        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+
+        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+
+        # Start a new garbage collection
+        ret = self.client.gc_mark("ABC", {"unihash": unihash, "method": self.METHOD})
+        self.assertEqual(ret, {"count": 1})
+
+        ret = self.client.gc_status()
+        self.assertEqual(ret, {"mark": "ABC", "keep": 1, "remove": 0})
+
+        # Add second hash. It should inherit the mark from the current garbage
+        # collection operation
+
+        taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
+        outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
+        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+
+        result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
+        # Sweep should remove nothing
+        ret = self.client.gc_sweep("ABC")
+        self.assertEqual(ret, {"count": 0})
+
+        # Both hashes are present
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+        self.assertClientGetHash(self.client, taskhash, unihash)
+
+
+class TestHashEquivalenceClient(HashEquivalenceTestSetup, unittest.TestCase):
+    def get_server_addr(self, server_idx):
+        return "unix://" + os.path.join(self.temp_dir.name, 'sock%d' % server_idx)
+
+    def test_get(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+
+        p = self.run_hashclient(["--address", self.server_address, "get", self.METHOD, taskhash])
+        data = json.loads(p.stdout)
+        self.assertEqual(data["unihash"], unihash)
+        self.assertEqual(data["outhash"], outhash)
+        self.assertEqual(data["taskhash"], taskhash)
+        self.assertEqual(data["method"], self.METHOD)
+
+    def test_get_outhash(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+
+        p = self.run_hashclient(["--address", self.server_address, "get-outhash", self.METHOD, outhash, taskhash])
+        data = json.loads(p.stdout)
+        self.assertEqual(data["unihash"], unihash)
+        self.assertEqual(data["outhash"], outhash)
+        self.assertEqual(data["taskhash"], taskhash)
+        self.assertEqual(data["method"], self.METHOD)
+
+    def test_stats(self):
+        p = self.run_hashclient(["--address", self.server_address, "stats"], check=True)
+        json.loads(p.stdout)
+
+    def test_stress(self):
+        self.run_hashclient(["--address", self.server_address, "stress"], check=True)
+
+    def test_unihash_exsits(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+
+        p = self.run_hashclient([
+            "--address", self.server_address,
+            "unihash-exists", unihash,
+        ], check=True)
+        self.assertEqual(p.stdout.strip(), "true")
+
+        p = self.run_hashclient([
+            "--address", self.server_address,
+            "unihash-exists", '6662e699d6e3d894b24408ff9a4031ef9b038ee8',
+        ], check=True)
+        self.assertEqual(p.stdout.strip(), "false")
+
+    def test_unihash_exsits_quiet(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+
+        p = self.run_hashclient([
+            "--address", self.server_address,
+            "unihash-exists", unihash,
+            "--quiet",
+        ])
+        self.assertEqual(p.returncode, 0)
+        self.assertEqual(p.stdout.strip(), "")
+
+        p = self.run_hashclient([
+            "--address", self.server_address,
+            "unihash-exists", '6662e699d6e3d894b24408ff9a4031ef9b038ee8',
+            "--quiet",
+        ])
+        self.assertEqual(p.returncode, 1)
+        self.assertEqual(p.stdout.strip(), "")
+
+    def test_remove_taskhash(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+        self.run_hashclient([
+            "--address", self.server_address,
+            "remove",
+            "--where", "taskhash", taskhash,
+        ], check=True)
+        self.assertClientGetHash(self.client, taskhash, None)
+
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash)
+        self.assertIsNone(result_outhash)
+
+    def test_remove_unihash(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+        self.run_hashclient([
+            "--address", self.server_address,
+            "remove",
+            "--where", "unihash", unihash,
+        ], check=True)
+        self.assertClientGetHash(self.client, taskhash, None)
+
+    def test_remove_outhash(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+        self.run_hashclient([
+            "--address", self.server_address,
+            "remove",
+            "--where", "outhash", outhash,
+        ], check=True)
+
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash)
+        self.assertIsNone(result_outhash)
+
+    def test_remove_method(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+        self.run_hashclient([
+            "--address", self.server_address,
+            "remove",
+            "--where", "method", self.METHOD,
+        ], check=True)
+        self.assertClientGetHash(self.client, taskhash, None)
+
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash)
+        self.assertIsNone(result_outhash)
+
+    def test_clean_unused(self):
+        taskhash, outhash, unihash = self.create_test_hash(self.client)
+
+        # Clean the database, which should not remove anything because all hashes an in-use
+        self.run_hashclient([
+            "--address", self.server_address,
+            "clean-unused", "0",
+        ], check=True)
+        self.assertClientGetHash(self.client, taskhash, unihash)
+
+        # Remove the unihash. The row in the outhash table should still be present
+        self.run_hashclient([
+            "--address", self.server_address,
+            "remove",
+            "--where", "unihash", unihash,
+        ], check=True)
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash, False)
+        self.assertIsNotNone(result_outhash)
+
+        # Now clean with no minimum age which will remove the outhash
+        self.run_hashclient([
+            "--address", self.server_address,
+            "clean-unused", "0",
+        ], check=True)
+        result_outhash = self.client.get_outhash(self.METHOD, outhash, taskhash, False)
+        self.assertIsNone(result_outhash)
+
+    def test_refresh_token(self):
+        admin_client = self.start_auth_server()
+
+        user = admin_client.new_user("test-user", ["@read", "@report"])
+
+        p = self.run_hashclient([
+            "--address", self.auth_server_address,
+            "--login", user["username"],
+            "--password", user["token"],
+            "refresh-token"
+        ], check=True)
+
+        new_token = None
+        for l in p.stdout.splitlines():
+            l = l.rstrip()
+            m = re.match(r'Token: +(.*)$', l)
+            if m is not None:
+                new_token = m.group(1)
+
+        self.assertTrue(new_token)
+
+        print("New token is %r" % new_token)
+
+        self.run_hashclient([
+            "--address", self.auth_server_address,
+            "--login", user["username"],
+            "--password", new_token,
+            "get-user"
+        ], check=True)
+
+    def test_set_user_perms(self):
+        admin_client = self.start_auth_server()
+
+        user = admin_client.new_user("test-user", ["@read"])
+
+        self.run_hashclient([
+            "--address", self.auth_server_address,
+            "--login", admin_client.username,
+            "--password", admin_client.password,
+            "set-user-perms",
+            "-u", user["username"],
+            "@read", "@report",
+        ], check=True)
+
+        new_user = admin_client.get_user(user["username"])
+
+        self.assertEqual(set(new_user["permissions"]), {"@read", "@report"})
+
+    def test_get_user(self):
+        admin_client = self.start_auth_server()
+
+        user = admin_client.new_user("test-user", ["@read"])
+
+        p = self.run_hashclient([
+            "--address", self.auth_server_address,
+            "--login", admin_client.username,
+            "--password", admin_client.password,
+            "get-user",
+            "-u", user["username"],
+        ], check=True)
+
+        self.assertIn("Username:", p.stdout)
+        self.assertIn("Permissions:", p.stdout)
+
+        p = self.run_hashclient([
+            "--address", self.auth_server_address,
+            "--login", user["username"],
+            "--password", user["token"],
+            "get-user",
+        ], check=True)
+
+        self.assertIn("Username:", p.stdout)
+        self.assertIn("Permissions:", p.stdout)
+
+    def test_get_all_users(self):
+        admin_client = self.start_auth_server()
+
+        admin_client.new_user("test-user1", ["@read"])
+        admin_client.new_user("test-user2", ["@read"])
+
+        p = self.run_hashclient([
+            "--address", self.auth_server_address,
+            "--login", admin_client.username,
+            "--password", admin_client.password,
+            "get-all-users",
+        ], check=True)
+
+        self.assertIn("admin", p.stdout)
+        self.assertIn("test-user1", p.stdout)
+        self.assertIn("test-user2", p.stdout)
+
+    def test_new_user(self):
+        admin_client = self.start_auth_server()
+
+        p = self.run_hashclient([
+            "--address", self.auth_server_address,
+            "--login", admin_client.username,
+            "--password", admin_client.password,
+            "new-user",
+            "-u", "test-user",
+            "@read", "@report",
+        ], check=True)
+
+        new_token = None
+        for l in p.stdout.splitlines():
+            l = l.rstrip()
+            m = re.match(r'Token: +(.*)$', l)
+            if m is not None:
+                new_token = m.group(1)
+
+        self.assertTrue(new_token)
+
+        user = {
+            "username": "test-user",
+            "token": new_token,
+        }
+
+        self.assertUserPerms(user, ["@read", "@report"])
+
+    def test_delete_user(self):
+        admin_client = self.start_auth_server()
+
+        user = admin_client.new_user("test-user", ["@read"])
+
+        p = self.run_hashclient([
+            "--address", self.auth_server_address,
+            "--login", admin_client.username,
+            "--password", admin_client.password,
+            "delete-user",
+            "-u", user["username"],
+        ], check=True)
+
+        self.assertIsNone(admin_client.get_user(user["username"]))
+
+    def test_get_db_usage(self):
+        p = self.run_hashclient([
+            "--address", self.server_address,
+            "get-db-usage",
+        ], check=True)
+
+    def test_get_db_query_columns(self):
+        p = self.run_hashclient([
+            "--address", self.server_address,
+            "get-db-query-columns",
+        ], check=True)
+
+    def test_gc(self):
+        taskhash = '53b8dce672cb6d0c73170be43f540460bfc347b4'
+        outhash = '5a9cb1649625f0bf41fc7791b635cd9c2d7118c7f021ba87dcd03f72b67ce7a8'
+        unihash = 'f37918cc02eb5a520b1aff86faacbc0a38124646'
+
+        result = self.client.report_unihash(taskhash, self.METHOD, outhash, unihash)
+        self.assertEqual(result['unihash'], unihash, 'Server returned bad unihash')
+
+        taskhash2 = '3bf6f1e89d26205aec90da04854fbdbf73afe6b4'
+        outhash2 = '77623a549b5b1a31e3732dfa8fe61d7ce5d44b3370f253c5360e136b852967b4'
+        unihash2 = 'af36b199320e611fbb16f1f277d3ee1d619ca58b'
+
+        result = self.client.report_unihash(taskhash2, self.METHOD, outhash2, unihash2)
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
+        # Mark the first unihash to be kept
+        self.run_hashclient([
+            "--address", self.server_address,
+            "gc-mark", "ABC",
+            "--where", "unihash", unihash,
+            "--where", "method", self.METHOD
+        ], check=True)
+
+        # Second hash is still there; mark doesn't delete hashes
+        self.assertClientGetHash(self.client, taskhash2, unihash2)
+
+        self.run_hashclient([
+            "--address", self.server_address,
+            "gc-sweep", "ABC",
+        ], check=True)
+
+        # Hash is gone. Taskhash is returned for second hash
         self.assertClientGetHash(self.client, taskhash2, None)
+        # First hash is still present
+        self.assertClientGetHash(self.client, taskhash, unihash)
 
 
 class TestHashEquivalenceUnixServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase):
@@ -314,3 +1469,77 @@ class TestHashEquivalenceTCPServer(HashEquivalenceTestSetup, HashEquivalenceComm
         # If IPv6 is enabled, it should be safe to use localhost directly, in general
         # case it is more reliable to resolve the IP address explicitly.
         return socket.gethostbyname("localhost") + ":0"
+
+
+class TestHashEquivalenceWebsocketServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase):
+    def setUp(self):
+        try:
+            import websockets
+        except ImportError as e:
+            self.skipTest(str(e))
+
+        super().setUp()
+
+    def get_server_addr(self, server_idx):
+        # Some hosts cause asyncio module to misbehave, when IPv6 is not enabled.
+        # If IPv6 is enabled, it should be safe to use localhost directly, in general
+        # case it is more reliable to resolve the IP address explicitly.
+        host = socket.gethostbyname("localhost")
+        return "ws://%s:0" % host
+
+
+class TestHashEquivalenceWebsocketsSQLAlchemyServer(TestHashEquivalenceWebsocketServer):
+    def setUp(self):
+        try:
+            import sqlalchemy
+            import aiosqlite
+        except ImportError as e:
+            self.skipTest(str(e))
+
+        super().setUp()
+
+    def make_dbpath(self):
+        return "sqlite+aiosqlite:///%s" % os.path.join(self.temp_dir.name, "db%d.sqlite" % self.server_index)
+
+
+class TestHashEquivalenceExternalServer(HashEquivalenceTestSetup, HashEquivalenceCommonTests, unittest.TestCase):
+    def get_env(self, name):
+        v = os.environ.get(name)
+        if not v:
+            self.skipTest(f'{name} not defined to test an external server')
+        return v
+
+    def start_test_server(self):
+        return self.get_env('BB_TEST_HASHSERV')
+
+    def start_server(self, *args, **kwargs):
+        self.skipTest('Cannot start local server when testing external servers')
+
+    def start_auth_server(self):
+
+        self.auth_server_address = self.server_address
+        self.admin_client = self.start_client(
+            self.server_address,
+            username=self.get_env('BB_TEST_HASHSERV_USERNAME'),
+            password=self.get_env('BB_TEST_HASHSERV_PASSWORD'),
+        )
+        return self.admin_client
+
+    def setUp(self):
+        super().setUp()
+        if "BB_TEST_HASHSERV_USERNAME" in os.environ:
+            self.client = self.start_client(
+                self.server_address,
+                username=os.environ["BB_TEST_HASHSERV_USERNAME"],
+                password=os.environ["BB_TEST_HASHSERV_PASSWORD"],
+            )
+        self.client.remove({"method": self.METHOD})
+
+    def tearDown(self):
+        self.client.remove({"method": self.METHOD})
+        super().tearDown()
+
+
+    def test_auth_get_all_users(self):
+        self.skipTest("Cannot test all users with external server")
+
diff --git a/bitbake/lib/layerindexlib/__init__.py b/bitbake/lib/layerindexlib/__init__.py
index 9ca127b9df..c3265ddaa1 100644
--- a/bitbake/lib/layerindexlib/__init__.py
+++ b/bitbake/lib/layerindexlib/__init__.py
@@ -6,7 +6,6 @@
 import datetime
 
 import logging
-import imp
 import os
 
 from collections import OrderedDict
@@ -179,9 +178,9 @@ class LayerIndex():
         '''Load the layerindex.
 
            indexURI - An index to load.  (Use multiple calls to load multiple indexes)
-           
+
            reload - If reload is True, then any previously loaded indexes will be forgotten.
-           
+
            load - List of elements to load.  Default loads all items.
                   Note: plugs may ignore this.
 
@@ -199,7 +198,7 @@ The format of the indexURI:
 
   For example:
 
-  http://layers.openembedded.org/layerindex/api/;branch=master;desc=OpenEmbedded%20Layer%20Index
+  https://layers.openembedded.org/layerindex/api/;branch=master;desc=OpenEmbedded%20Layer%20Index
   cooker://
 '''
         if reload:
@@ -384,7 +383,14 @@ layerBranches set.  If not, they are effectively blank.'''
 
                 # Get a list of dependencies and then recursively process them
                 for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]:
-                    deplayerbranch = layerdependency.dependency_layerBranch
+                    try:
+                        deplayerbranch = layerdependency.dependency_layerBranch
+                    except AttributeError as e:
+                            logger.error('LayerBranch does not exist for dependent layer {}:{}\n' \
+                                '       Cannot continue successfully.\n' \
+                                '       You might be able to resolve this by checking out the layer locally.\n' \
+                                '       Consider reaching out the to the layer maintainers or the layerindex admins' \
+                                .format(layerdependency.dependency.name, layerbranch.branch.name))
 
                     if ignores and deplayerbranch.layer.name in ignores:
                         continue
@@ -577,7 +583,7 @@ This function is used to implement debugging and provide the user info.
 #   index['config'] - configuration data for this index
 #   index['branches'] - dictionary of Branch objects, by id number
 #   index['layerItems'] - dictionary of layerItem objects, by id number
-#   ...etc...  (See: http://layers.openembedded.org/layerindex/api/)
+#   ...etc...  (See: https://layers.openembedded.org/layerindex/api/)
 #
 # The class needs to manage the 'index' entries and allow easily adding
 # of new items, as well as simply loading of the items.
@@ -847,7 +853,7 @@ class LayerIndexObj():
                     continue
 
                 for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]:
-                    deplayerbranch = layerdependency.dependency_layerBranch
+                    deplayerbranch = layerdependency.dependency_layerBranch or None
 
                     if ignores and deplayerbranch.layer.name in ignores:
                         continue
@@ -1279,7 +1285,7 @@ class Recipe(LayerIndexItemObj_LayerBranch):
                     filename, filepath, pn, pv, layerbranch,
                     summary="", description="", section="", license="",
                     homepage="", bugtracker="", provides="", bbclassextend="",
-                    inherits="", blacklisted="", updated=None):
+                    inherits="", disallowed="", updated=None):
         self.id = id
         self.filename = filename
         self.filepath = filepath
@@ -1295,7 +1301,7 @@ class Recipe(LayerIndexItemObj_LayerBranch):
         self.bbclassextend = bbclassextend
         self.inherits = inherits
         self.updated = updated or datetime.datetime.today().isoformat()
-        self.blacklisted = blacklisted
+        self.disallowed = disallowed
         if isinstance(layerbranch, LayerBranch):
             self.layerbranch = layerbranch
         else:
diff --git a/bitbake/lib/layerindexlib/cooker.py b/bitbake/lib/layerindexlib/cooker.py
index 2de6e5faa0..ced3e06360 100644
--- a/bitbake/lib/layerindexlib/cooker.py
+++ b/bitbake/lib/layerindexlib/cooker.py
@@ -279,7 +279,7 @@ class CookerPlugin(layerindexlib.plugin.IndexPlugin):
                                    summary=pn, description=pn, section='?',
                                    license='?', homepage='?', bugtracker='?',
                                    provides='?', bbclassextend='?', inherits='?',
-                                   blacklisted='?', layerbranch=depBranchId)
+                                   disallowed='?', layerbranch=depBranchId)
 
                     index = addElement("recipes", [recipe], index)
 
diff --git a/bitbake/lib/layerindexlib/restapi.py b/bitbake/lib/layerindexlib/restapi.py
index 26a1c9674e..81d99b02ea 100644
--- a/bitbake/lib/layerindexlib/restapi.py
+++ b/bitbake/lib/layerindexlib/restapi.py
@@ -31,7 +31,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
             The return value is a LayerIndexObj.
 
             url is the url to the rest api of the layer index, such as:
-            http://layers.openembedded.org/layerindex/api/
+            https://layers.openembedded.org/layerindex/api/
 
             Or a local file...
         """
@@ -138,7 +138,7 @@ class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
             The return value is a LayerIndexObj.
 
             ud is the parsed url to the rest api of the layer index, such as:
-            http://layers.openembedded.org/layerindex/api/
+            https://layers.openembedded.org/layerindex/api/
         """
 
         def _get_json_response(apiurl=None, username=None, password=None, retry=True):
diff --git a/bitbake/lib/layerindexlib/tests/restapi.py b/bitbake/lib/layerindexlib/tests/restapi.py
index 33b5c1c4c8..71f0ae8a9d 100644
--- a/bitbake/lib/layerindexlib/tests/restapi.py
+++ b/bitbake/lib/layerindexlib/tests/restapi.py
@@ -22,7 +22,7 @@ class LayerIndexWebRestApiTest(LayersTest):
         self.assertFalse(os.environ.get("BB_SKIP_NETTESTS") == "yes", msg="BB_SKIP_NETTESTS set, but we tried to test anyway")
         LayersTest.setUp(self)
         self.layerindex = layerindexlib.LayerIndex(self.d)
-        self.layerindex.load_layerindex('http://layers.openembedded.org/layerindex/api/;branch=sumo', load=['layerDependencies'])
+        self.layerindex.load_layerindex('https://layers.openembedded.org/layerindex/api/;branch=sumo', load=['layerDependencies'])
 
     @skipIfNoNetwork()
     def test_layerindex_is_empty(self):
diff --git a/bitbake/lib/ply/yacc.py b/bitbake/lib/ply/yacc.py
index 46e7dc96f6..529f85b081 100644
--- a/bitbake/lib/ply/yacc.py
+++ b/bitbake/lib/ply/yacc.py
@@ -1122,7 +1122,6 @@ class LRParser:
 # manipulate the rules that make up a grammar. 
 # -----------------------------------------------------------------------------
 
-import re
 
 # regex matching identifiers
 _is_identifier = re.compile(r'^[a-zA-Z0-9_-]+$')
@@ -2797,11 +2796,15 @@ class ParserReflect(object):
     # Compute a signature over the grammar
     def signature(self):
         try:
-            from hashlib import md5
+            import hashlib
         except ImportError:
-            from md5 import md5
+            raise RuntimeError("Unable to import hashlib")
+        try:
+            sig = hashlib.new('MD5', usedforsecurity=False)
+        except TypeError:
+            # Some configurations don't appear to support two arguments
+            sig = hashlib.new('MD5')
         try:
-            sig = md5()
             if self.start:
                 sig.update(self.start.encode('latin-1'))
             if self.prec:
diff --git a/bitbake/lib/progressbar/progressbar.py b/bitbake/lib/progressbar/progressbar.py
index e2b6ba1083..d4da10ab75 100644
--- a/bitbake/lib/progressbar/progressbar.py
+++ b/bitbake/lib/progressbar/progressbar.py
@@ -253,7 +253,7 @@ class ProgressBar(object):
             if (self.maxval is not UnknownLength
                 and not 0 <= value <= self.maxval):
 
-                raise ValueError('Value out of range')
+                self.maxval = value
 
             self.currval = value
 
diff --git a/bitbake/lib/prserv/__init__.py b/bitbake/lib/prserv/__init__.py
index 9961040b58..ffc5a40a28 100644
--- a/bitbake/lib/prserv/__init__.py
+++ b/bitbake/lib/prserv/__init__.py
@@ -1,18 +1,95 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-__version__ = "1.0.0"
 
-import os, time
-import sys,logging
+__version__ = "2.0.0"
+
+import logging
+logger = logging.getLogger("BitBake.PRserv")
+
+from bb.asyncrpc.client import parse_address, ADDR_TYPE_UNIX, ADDR_TYPE_WS
+
+def create_server(addr, dbpath, upstream=None, read_only=False):
+    from . import serv
+
+    s = serv.PRServer(dbpath, upstream=upstream, read_only=read_only)
+    host, port = addr.split(":")
+    s.start_tcp_server(host, int(port))
+
+    return s
+
+def increase_revision(ver):
+    """Take a revision string such as "1" or "1.2.3" or even a number and increase its last number
+    This fails if the last number is not an integer"""
+
+    fields=str(ver).split('.')
+    last = fields[-1]
+
+    try:
+         val = int(last)
+    except Exception as e:
+         logger.critical("Unable to increase revision value %s: %s" % (ver, e))
+         raise e
+
+    return ".".join(fields[0:-1] + [ str(val + 1) ])
+
+def _revision_greater_or_equal(rev1, rev2):
+    """Compares x.y.z revision numbers, using integer comparison
+    Returns True if rev1 is greater or equal to rev2"""
+
+    fields1 = rev1.split(".")
+    fields2 = rev2.split(".")
+    l1 = len(fields1)
+    l2 = len(fields2)
+
+    for i in range(l1):
+       val1 = int(fields1[i])
+       if i < l2:
+           val2 = int(fields2[i])
+           if val2 < val1:
+              return True
+           elif val2 > val1:
+              return False
+       else:
+          return True
+    return True
+
+def revision_smaller(rev1, rev2):
+    """Compares x.y.z revision numbers, using integer comparison
+    Returns True if rev1 is strictly smaller than rev2"""
+    return not(_revision_greater_or_equal(rev1, rev2))
+
+def revision_greater(rev1, rev2):
+    """Compares x.y.z revision numbers, using integer comparison
+    Returns True if rev1 is strictly greater than rev2"""
+    return _revision_greater_or_equal(rev1, rev2) and (rev1 != rev2)
+
+def create_client(addr):
+    from . import client
+
+    c = client.PRClient()
+
+    try:
+        (typ, a) = parse_address(addr)
+        c.connect_tcp(*a)
+        return c
+    except Exception as e:
+        c.close()
+        raise e
+
+async def create_async_client(addr):
+    from . import client
+
+    c = client.PRAsyncClient()
 
-def init_logger(logfile, loglevel):
-    numeric_level = getattr(logging, loglevel.upper(), None)
-    if not isinstance(numeric_level, int):
-        raise ValueError('Invalid log level: %s' % loglevel)
-    FORMAT = '%(asctime)-15s %(message)s'
-    logging.basicConfig(level=numeric_level, filename=logfile, format=FORMAT)
+    try:
+        (typ, a) = parse_address(addr)
+        await c.connect_tcp(*a)
+        return c
 
-class NotFoundError(Exception):
-    pass
+    except Exception as e:
+        await c.close()
+        raise e
diff --git a/bitbake/lib/prserv/client.py b/bitbake/lib/prserv/client.py
new file mode 100644
index 0000000000..9f5794c433
--- /dev/null
+++ b/bitbake/lib/prserv/client.py
@@ -0,0 +1,72 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import logging
+import bb.asyncrpc
+from . import create_async_client
+
+logger = logging.getLogger("BitBake.PRserv")
+
+class PRAsyncClient(bb.asyncrpc.AsyncClient):
+    def __init__(self):
+        super().__init__("PRSERVICE", "1.0", logger)
+
+    async def getPR(self, version, pkgarch, checksum, history=False):
+        response = await self.invoke(
+            {"get-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "history": history}}
+        )
+        if response:
+            return response["value"]
+
+    async def test_pr(self, version, pkgarch, checksum, history=False):
+        response = await self.invoke(
+            {"test-pr": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "history": history}}
+        )
+        if response:
+            return response["value"]
+
+    async def test_package(self, version, pkgarch):
+        response = await self.invoke(
+            {"test-package": {"version": version, "pkgarch": pkgarch}}
+        )
+        if response:
+            return response["value"]
+
+    async def max_package_pr(self, version, pkgarch):
+        response = await self.invoke(
+            {"max-package-pr": {"version": version, "pkgarch": pkgarch}}
+        )
+        if response:
+            return response["value"]
+
+    async def importone(self, version, pkgarch, checksum, value):
+        response = await self.invoke(
+            {"import-one": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "value": value}}
+        )
+        if response:
+            return response["value"]
+
+    async def export(self, version, pkgarch, checksum, colinfo, history=False):
+        response = await self.invoke(
+            {"export": {"version": version, "pkgarch": pkgarch, "checksum": checksum, "colinfo": colinfo, "history": history}}
+        )
+        if response:
+            return (response["metainfo"], response["datainfo"])
+
+    async def is_readonly(self):
+        response = await self.invoke(
+            {"is-readonly": {}}
+        )
+        if response:
+            return response["readonly"]
+
+class PRClient(bb.asyncrpc.Client):
+    def __init__(self):
+        super().__init__()
+        self._add_methods("getPR", "test_pr", "test_package", "max_package_pr", "importone", "export", "is_readonly")
+
+    def _get_async_client(self):
+        return PRAsyncClient()
diff --git a/bitbake/lib/prserv/db.py b/bitbake/lib/prserv/db.py
index cb2a2461e0..2da493ddf5 100644
--- a/bitbake/lib/prserv/db.py
+++ b/bitbake/lib/prserv/db.py
@@ -1,4 +1,6 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
@@ -6,19 +8,13 @@ import logging
 import os.path
 import errno
 import prserv
-import time
+import sqlite3
 
-try:
-    import sqlite3
-except ImportError:
-    from pysqlite2 import dbapi2 as sqlite3
+from contextlib import closing
+from . import increase_revision, revision_greater, revision_smaller
 
 logger = logging.getLogger("BitBake.PRserv")
 
-sqlversion = sqlite3.sqlite_version_info
-if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
-    raise Exception("sqlite3 version 3.3.0 or later is required.")
-
 #
 # "No History" mode - for a given query tuple (version, pkgarch, checksum),
 # the returned value will be the largest among all the values of the same
@@ -27,212 +23,232 @@ if sqlversion[0] < 3 or (sqlversion[0] == 3 and sqlversion[1] < 3):
 # "History" mode - Return a new higher value for previously unseen query
 # tuple (version, pkgarch, checksum), otherwise return historical value.
 # Value can decrement if returning to a previous build.
-#
 
 class PRTable(object):
-    def __init__(self, conn, table, nohist):
+    def __init__(self, conn, table, read_only):
         self.conn = conn
-        self.nohist = nohist
-        self.dirty = False
-        if nohist:
-            self.table = "%s_nohist" % table 
-        else:
-            self.table = "%s_hist" % table 
-
-        self._execute("CREATE TABLE IF NOT EXISTS %s \
-                    (version TEXT NOT NULL, \
-                    pkgarch TEXT NOT NULL,  \
-                    checksum TEXT NOT NULL, \
-                    value INTEGER, \
-                    PRIMARY KEY (version, pkgarch, checksum));" % self.table)
-
-    def _execute(self, *query):
-        """Execute a query, waiting to acquire a lock if necessary"""
-        start = time.time()
-        end = start + 20
-        while True:
-            try:
-                return self.conn.execute(*query)
-            except sqlite3.OperationalError as exc:
-                if 'is locked' in str(exc) and end > time.time():
-                    continue
-                raise exc
-
-    def sync(self):
-        self.conn.commit()
-        self._execute("BEGIN EXCLUSIVE TRANSACTION")
-
-    def sync_if_dirty(self):
-        if self.dirty:
-            self.sync()
-            self.dirty = False
-
-    def _getValueHist(self, version, pkgarch, checksum):
-        data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
-                           (version, pkgarch, checksum))
-        row=data.fetchone()
-        if row is not None:
-            return row[0]
-        else:
-            #no value found, try to insert
-            try:
-                self._execute("INSERT INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
-                           % (self.table,self.table),
-                           (version,pkgarch, checksum,version, pkgarch))
-            except sqlite3.IntegrityError as exc:
-                logger.error(str(exc))
-
-            self.dirty = True
-
-            data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
-                               (version, pkgarch, checksum))
+        self.read_only = read_only
+        self.table = table
+
+        # Creating the table even if the server is read-only.
+        # This avoids a race condition if a shared database
+        # is accessed by a read-only server first.
+
+        with closing(self.conn.cursor()) as cursor:
+            cursor.execute("CREATE TABLE IF NOT EXISTS %s \
+                        (version TEXT NOT NULL, \
+                        pkgarch TEXT NOT NULL,  \
+                        checksum TEXT NOT NULL, \
+                        value TEXT, \
+                        PRIMARY KEY (version, pkgarch, checksum, value));" % self.table)
+            self.conn.commit()
+
+    def _extremum_value(self, rows, is_max):
+        value = None
+
+        for row in rows:
+            current_value = row[0]
+            if value is None:
+                value = current_value
+            else:
+                if is_max:
+                    is_new_extremum = revision_greater(current_value, value)
+                else:
+                    is_new_extremum = revision_smaller(current_value, value)
+                if  is_new_extremum:
+                    value = current_value
+        return value
+
+    def _max_value(self, rows):
+        return self._extremum_value(rows, True)
+
+    def _min_value(self, rows):
+        return self._extremum_value(rows, False)
+
+    def test_package(self, version, pkgarch):
+        """Returns whether the specified package version is found in the database for the specified architecture"""
+
+        # Just returns the value if found or None otherwise
+        with closing(self.conn.cursor()) as cursor:
+            data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=?;" % self.table,
+                               (version, pkgarch))
             row=data.fetchone()
             if row is not None:
-                return row[0]
+                return True
             else:
-                raise prserv.NotFoundError
-
-    def _getValueNohist(self, version, pkgarch, checksum):
-        data=self._execute("SELECT value FROM %s \
-                            WHERE version=? AND pkgarch=? AND checksum=? AND \
-                            value >= (select max(value) from %s where version=? AND pkgarch=?);" 
-                            % (self.table, self.table),
-                            (version, pkgarch, checksum, version, pkgarch))
-        row=data.fetchone()
-        if row is not None:
-            return row[0]
-        else:
-            #no value found, try to insert
-            try:
-                self._execute("INSERT OR REPLACE INTO %s VALUES (?, ?, ?, (select ifnull(max(value)+1,0) from %s where version=? AND pkgarch=?));"
-                               % (self.table,self.table),
-                               (version, pkgarch, checksum, version, pkgarch))
-            except sqlite3.IntegrityError as exc:
-                logger.error(str(exc))
-                self.conn.rollback()
-
-            self.dirty = True
-
-            data=self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
-                               (version, pkgarch, checksum))
+                return False
+
+    def test_checksum_value(self, version, pkgarch, checksum, value):
+        """Returns whether the specified value is found in the database for the specified package, architecture and checksum"""
+
+        with closing(self.conn.cursor()) as cursor:
+            data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and checksum=? and value=?;" % self.table,
+                               (version, pkgarch, checksum, value))
             row=data.fetchone()
             if row is not None:
-                return row[0]
+                return True
             else:
-                raise prserv.NotFoundError
+                return False
 
-    def getValue(self, version, pkgarch, checksum):
-        if self.nohist:
-            return self._getValueNohist(version, pkgarch, checksum)
-        else:
-            return self._getValueHist(version, pkgarch, checksum)
-
-    def _importHist(self, version, pkgarch, checksum, value):
-        val = None 
-        data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
-                           (version, pkgarch, checksum))
-        row = data.fetchone()
-        if row is not None:
-            val=row[0]
+    def test_value(self, version, pkgarch, value):
+        """Returns whether the specified value is found in the database for the specified package and architecture"""
+
+        # Just returns the value if found or None otherwise
+        with closing(self.conn.cursor()) as cursor:
+            data=cursor.execute("SELECT value FROM %s WHERE version=? AND pkgarch=? and value=?;" % self.table,
+                               (version, pkgarch, value))
+            row=data.fetchone()
+            if row is not None:
+                return True
+            else:
+                return False
+
+
+    def find_package_max_value(self, version, pkgarch):
+        """Returns the greatest value for (version, pkgarch), or None if not found. Doesn't create a new value"""
+
+        with closing(self.conn.cursor()) as cursor:
+            data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=?;" % (self.table),
+                                 (version, pkgarch))
+            rows = data.fetchall()
+            value = self._max_value(rows)
+            return value
+
+    def find_value(self, version, pkgarch, checksum, history=False):
+        """Returns the value for the specified checksum if found or None otherwise."""
+
+        if history:
+            return self.find_min_value(version, pkgarch, checksum)
         else:
-            #no value found, try to insert
-            try:
-                self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);"  % (self.table),
+            return self.find_max_value(version, pkgarch, checksum)
+
+
+    def _find_extremum_value(self, version, pkgarch, checksum, is_max):
+        """Returns the maximum (if is_max is True) or minimum (if is_max is False) value
+           for (version, pkgarch, checksum), or None if not found. Doesn't create a new value"""
+
+        with closing(self.conn.cursor()) as cursor:
+            data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=? AND checksum=?;" % (self.table),
+                                 (version, pkgarch, checksum))
+            rows = data.fetchall()
+            return self._extremum_value(rows, is_max)
+
+    def find_max_value(self, version, pkgarch, checksum):
+        return self._find_extremum_value(version, pkgarch, checksum, True)
+
+    def find_min_value(self, version, pkgarch, checksum):
+        return self._find_extremum_value(version, pkgarch, checksum, False)
+
+    def find_new_subvalue(self, version, pkgarch, base):
+        """Take and increase the greatest "<base>.y" value for (version, pkgarch), or return "<base>.0" if not found.
+        This doesn't store a new value."""
+
+        with closing(self.conn.cursor()) as cursor:
+            data = cursor.execute("SELECT value FROM %s where version=? AND pkgarch=? AND value LIKE '%s.%%';" % (self.table, base),
+                                 (version, pkgarch))
+            rows = data.fetchall()
+            value = self._max_value(rows)
+
+            if value is not None:
+                return increase_revision(value)
+            else:
+                return base + ".0"
+
+    def store_value(self, version, pkgarch, checksum, value):
+        """Store value in the database"""
+
+        if not self.read_only and not self.test_checksum_value(version, pkgarch, checksum, value):
+            with closing(self.conn.cursor()) as cursor:
+                cursor.execute("INSERT INTO %s VALUES (?, ?, ?, ?);"  % (self.table),
                            (version, pkgarch, checksum, value))
-            except sqlite3.IntegrityError as exc:
-                logger.error(str(exc))
+                self.conn.commit()
 
-            self.dirty = True
+    def _get_value(self, version, pkgarch, checksum, history):
 
-            data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=?;" % self.table,
-                           (version, pkgarch, checksum))
-            row = data.fetchone()
-            if row is not None:
-                val = row[0]
-        return val
+        max_value = self.find_package_max_value(version, pkgarch)
 
-    def _importNohist(self, version, pkgarch, checksum, value):
-        try:
-            #try to insert
-            self._execute("INSERT INTO %s VALUES (?, ?, ?, ?);"  % (self.table),
-                           (version, pkgarch, checksum,value))
-        except sqlite3.IntegrityError as exc:
-            #already have the record, try to update
-            try:
-                self._execute("UPDATE %s SET value=? WHERE version=? AND pkgarch=? AND checksum=? AND value<?"  
-                              % (self.table),
-                               (value,version,pkgarch,checksum,value))
-            except sqlite3.IntegrityError as exc:
-                logger.error(str(exc))
-
-        self.dirty = True
-
-        data = self._execute("SELECT value FROM %s WHERE version=? AND pkgarch=? AND checksum=? AND value>=?;" % self.table,
-                            (version,pkgarch,checksum,value))
-        row=data.fetchone()
-        if row is not None:
-            return row[0]
+        if max_value is None:
+            # version, pkgarch completely unknown. Return initial value.
+            return "0"
+
+        value = self.find_value(version, pkgarch, checksum, history)
+
+        if value is None:
+            # version, pkgarch found but not checksum. Create a new value from the maximum one
+            return increase_revision(max_value)
+
+        if history:
+            return value
+
+        # "no history" mode - If the value is not the maximum value for the package, need to increase it.
+        if max_value > value:
+            return increase_revision(max_value)
         else:
-            return None
+            return value
+
+    def get_value(self, version, pkgarch, checksum, history):
+        value = self._get_value(version, pkgarch, checksum, history)
+        if not self.read_only:
+            self.store_value(version, pkgarch, checksum, value)
+        return value
 
     def importone(self, version, pkgarch, checksum, value):
-        if self.nohist:
-            return self._importNohist(version, pkgarch, checksum, value)
-        else:
-            return self._importHist(version, pkgarch, checksum, value)
+        self.store_value(version, pkgarch, checksum, value)
+        return value
 
-    def export(self, version, pkgarch, checksum, colinfo):
+    def export(self, version, pkgarch, checksum, colinfo, history=False):
         metainfo = {}
-        #column info 
-        if colinfo:
-            metainfo['tbl_name'] = self.table
-            metainfo['core_ver'] = prserv.__version__
-            metainfo['col_info'] = []
-            data = self._execute("PRAGMA table_info(%s);" % self.table)
+        with closing(self.conn.cursor()) as cursor:
+            #column info
+            if colinfo:
+                metainfo["tbl_name"] = self.table
+                metainfo["core_ver"] = prserv.__version__
+                metainfo["col_info"] = []
+                data = cursor.execute("PRAGMA table_info(%s);" % self.table)
+                for row in data:
+                    col = {}
+                    col["name"] = row["name"]
+                    col["type"] = row["type"]
+                    col["notnull"] = row["notnull"]
+                    col["dflt_value"] = row["dflt_value"]
+                    col["pk"] = row["pk"]
+                    metainfo["col_info"].append(col)
+
+            #data info
+            datainfo = []
+
+            if history:
+                sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
+            else:
+                sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
+                        (SELECT version, pkgarch, max(value) as maxvalue FROM %s GROUP BY version, pkgarch) as T2 \
+                        WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
+            sqlarg = []
+            where = ""
+            if version:
+                where += "AND T1.version=? "
+                sqlarg.append(str(version))
+            if pkgarch:
+                where += "AND T1.pkgarch=? "
+                sqlarg.append(str(pkgarch))
+            if checksum:
+                where += "AND T1.checksum=? "
+                sqlarg.append(str(checksum))
+
+            sqlstmt += where + ";"
+
+            if len(sqlarg):
+                data = cursor.execute(sqlstmt, tuple(sqlarg))
+            else:
+                data = cursor.execute(sqlstmt)
             for row in data:
-                col = {}
-                col['name'] = row['name']
-                col['type'] = row['type']
-                col['notnull'] = row['notnull']
-                col['dflt_value'] = row['dflt_value']
-                col['pk'] = row['pk']
-                metainfo['col_info'].append(col)
-
-        #data info
-        datainfo = []
-
-        if self.nohist:
-            sqlstmt = "SELECT T1.version, T1.pkgarch, T1.checksum, T1.value FROM %s as T1, \
-                    (SELECT version,pkgarch,max(value) as maxvalue FROM %s GROUP BY version,pkgarch) as T2 \
-                    WHERE T1.version=T2.version AND T1.pkgarch=T2.pkgarch AND T1.value=T2.maxvalue " % (self.table, self.table)
-        else:
-            sqlstmt = "SELECT * FROM %s as T1 WHERE 1=1 " % self.table
-        sqlarg = []
-        where = ""
-        if version:
-            where += "AND T1.version=? "
-            sqlarg.append(str(version))
-        if pkgarch:
-            where += "AND T1.pkgarch=? "
-            sqlarg.append(str(pkgarch))
-        if checksum:
-            where += "AND T1.checksum=? "
-            sqlarg.append(str(checksum))
-
-        sqlstmt += where + ";"
-
-        if len(sqlarg):
-            data = self._execute(sqlstmt, tuple(sqlarg))
-        else:
-            data = self._execute(sqlstmt)
-        for row in data:
-            if row['version']:
-                col = {}
-                col['version'] = row['version']
-                col['pkgarch'] = row['pkgarch']
-                col['checksum'] = row['checksum']
-                col['value'] = row['value']
-                datainfo.append(col)
+                if row["version"]:
+                    col = {}
+                    col["version"] = row["version"]
+                    col["pkgarch"] = row["pkgarch"]
+                    col["checksum"] = row["checksum"]
+                    col["value"] = row["value"]
+                    datainfo.append(col)
         return (metainfo, datainfo)
 
     def dump_db(self, fd):
@@ -240,41 +256,46 @@ class PRTable(object):
         for line in self.conn.iterdump():
             writeCount = writeCount + len(line) + 1
             fd.write(line)
-            fd.write('\n')
+            fd.write("\n")
         return writeCount
 
 class PRData(object):
     """Object representing the PR database"""
-    def __init__(self, filename, nohist=True):
+    def __init__(self, filename, read_only=False):
         self.filename=os.path.abspath(filename)
-        self.nohist=nohist
+        self.read_only = read_only
         #build directory hierarchy
         try:
             os.makedirs(os.path.dirname(self.filename))
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise e
-        self.connection=sqlite3.connect(self.filename, isolation_level="EXCLUSIVE", check_same_thread = False)
+        uri = "file:%s%s" % (self.filename, "?mode=ro" if self.read_only else "")
+        logger.debug("Opening PRServ database '%s'" % (uri))
+        self.connection=sqlite3.connect(uri, uri=True)
         self.connection.row_factory=sqlite3.Row
-        self.connection.execute("pragma synchronous = off;")
-        self.connection.execute("PRAGMA journal_mode = MEMORY;")
+        self.connection.execute("PRAGMA synchronous = OFF;")
+        self.connection.execute("PRAGMA journal_mode = WAL;")
+        self.connection.commit()
         self._tables={}
 
     def disconnect(self):
+        self.connection.commit()
         self.connection.close()
 
-    def __getitem__(self,tblname):
+    def __getitem__(self, tblname):
         if not isinstance(tblname, str):
             raise TypeError("tblname argument must be a string, not '%s'" %
                             type(tblname))
         if tblname in self._tables:
             return self._tables[tblname]
         else:
-            tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.nohist)
+            tableobj = self._tables[tblname] = PRTable(self.connection, tblname, self.read_only)
             return tableobj
 
     def __delitem__(self, tblname):
         if tblname in self._tables:
             del self._tables[tblname]
         logger.info("drop table %s" % (tblname))
-        self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname) 
+        self.connection.execute("DROP TABLE IF EXISTS %s;" % tblname)
+        self.connection.commit()
diff --git a/bitbake/lib/prserv/serv.py b/bitbake/lib/prserv/serv.py
index 25dcf8a0ee..e175886308 100644
--- a/bitbake/lib/prserv/serv.py
+++ b/bitbake/lib/prserv/serv.py
@@ -1,354 +1,326 @@
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
 import os,sys,logging
 import signal, time
-from xmlrpc.server import SimpleXMLRPCServer, SimpleXMLRPCRequestHandler
-import threading
-import queue
 import socket
 import io
 import sqlite3
-import bb.server.xmlrpcclient
 import prserv
 import prserv.db
 import errno
-import select
+from . import create_async_client, revision_smaller, increase_revision
+import bb.asyncrpc
 
 logger = logging.getLogger("BitBake.PRserv")
 
-if sys.hexversion < 0x020600F0:
-    print("Sorry, python 2.6 or later is required.")
-    sys.exit(1)
+PIDPREFIX = "/tmp/PRServer_%s_%s.pid"
+singleton = None
 
-class Handler(SimpleXMLRPCRequestHandler):
-    def _dispatch(self,method,params):
+class PRServerClient(bb.asyncrpc.AsyncServerConnection):
+    def __init__(self, socket, server):
+        super().__init__(socket, "PRSERVICE", server.logger)
+        self.server = server
+
+        self.handlers.update({
+            "get-pr": self.handle_get_pr,
+            "test-pr": self.handle_test_pr,
+            "test-package": self.handle_test_package,
+            "max-package-pr": self.handle_max_package_pr,
+            "import-one": self.handle_import_one,
+            "export": self.handle_export,
+            "is-readonly": self.handle_is_readonly,
+        })
+
+    def validate_proto_version(self):
+        return (self.proto_version == (1, 0))
+
+    async def dispatch_message(self, msg):
         try:
-            value=self.server.funcs[method](*params)
+            return await super().dispatch_message(msg)
         except:
-            import traceback
-            traceback.print_exc()
             raise
-        return value
 
-PIDPREFIX = "/tmp/PRServer_%s_%s.pid"
-singleton = None
+    async def handle_test_pr(self, request):
+        '''Finds the PR value corresponding to the request. If not found, returns None and doesn't insert a new value'''
+        version = request["version"]
+        pkgarch = request["pkgarch"]
+        checksum = request["checksum"]
+        history = request["history"]
 
+        value = self.server.table.find_value(version, pkgarch, checksum, history)
+        return {"value": value}
 
-class PRServer(SimpleXMLRPCServer):
-    def __init__(self, dbfile, logfile, interface, daemon=True):
-        ''' constructor '''
-        try:
-            SimpleXMLRPCServer.__init__(self, interface,
-                                        logRequests=False, allow_none=True)
-        except socket.error:
-            ip=socket.gethostbyname(interface[0])
-            port=interface[1]
-            msg="PR Server unable to bind to %s:%s\n" % (ip, port)
-            sys.stderr.write(msg)
-            raise PRServiceConfigError
+    async def handle_test_package(self, request):
+        '''Tells whether there are entries for (version, pkgarch) in the db. Returns True or False'''
+        version = request["version"]
+        pkgarch = request["pkgarch"]
 
-        self.dbfile=dbfile
-        self.daemon=daemon
-        self.logfile=logfile
-        self.working_thread=None
-        self.host, self.port = self.socket.getsockname()
-        self.pidfile=PIDPREFIX % (self.host, self.port)
-
-        self.register_function(self.getPR, "getPR")
-        self.register_function(self.quit, "quit")
-        self.register_function(self.ping, "ping")
-        self.register_function(self.export, "export")
-        self.register_function(self.dump_db, "dump_db")
-        self.register_function(self.importone, "importone")
-        self.register_introspection_functions()
-
-        self.quitpipein, self.quitpipeout = os.pipe()
-
-        self.requestqueue = queue.Queue()
-        self.handlerthread = threading.Thread(target = self.process_request_thread)
-        self.handlerthread.daemon = False
-
-    def process_request_thread(self):
-        """Same as in BaseServer but as a thread.
-
-        In addition, exception handling is done here.
-
-        """
-        iter_count = 1
-        # 60 iterations between syncs or sync if dirty every ~30 seconds
-        iterations_between_sync = 60
-
-        bb.utils.set_process_name("PRServ Handler")
-
-        while not self.quitflag:
-            try:
-                (request, client_address) = self.requestqueue.get(True, 30)
-            except queue.Empty:
-                self.table.sync_if_dirty()
-                continue
-            if request is None:
-                continue
-            try:
-                self.finish_request(request, client_address)
-                self.shutdown_request(request)
-                iter_count = (iter_count + 1) % iterations_between_sync
-                if iter_count == 0:
-                    self.table.sync_if_dirty()
-            except:
-                self.handle_error(request, client_address)
-                self.shutdown_request(request)
-                self.table.sync()
-            self.table.sync_if_dirty()
-
-    def sigint_handler(self, signum, stack):
-        if self.table:
-            self.table.sync()
-
-    def sigterm_handler(self, signum, stack):
-        if self.table:
-            self.table.sync()
-        self.quit()
-        self.requestqueue.put((None, None))
-
-    def process_request(self, request, client_address):
-        self.requestqueue.put((request, client_address))
-
-    def export(self, version=None, pkgarch=None, checksum=None, colinfo=True):
-        try:
-            return self.table.export(version, pkgarch, checksum, colinfo)
-        except sqlite3.Error as exc:
-            logger.error(str(exc))
-            return None
-
-    def dump_db(self):
-        """
-        Returns a script (string) that reconstructs the state of the
-        entire database at the time this function is called. The script
-        language is defined by the backing database engine, which is a
-        function of server configuration.
-        Returns None if the database engine does not support dumping to
-        script or if some other error is encountered in processing.
-        """
-        buff = io.StringIO()
-        try:
-            self.table.sync()
-            self.table.dump_db(buff)
-            return buff.getvalue()
-        except Exception as exc:
-            logger.error(str(exc))
-            return None
-        finally:
-            buff.close()
+        value = self.server.table.test_package(version, pkgarch)
+        return {"value": value}
 
-    def importone(self, version, pkgarch, checksum, value):
-        return self.table.importone(version, pkgarch, checksum, value)
+    async def handle_max_package_pr(self, request):
+        '''Finds the greatest PR value for (version, pkgarch) in the db. Returns None if no entry was found'''
+        version = request["version"]
+        pkgarch = request["pkgarch"]
 
-    def ping(self):
-        return not self.quitflag
+        value = self.server.table.find_package_max_value(version, pkgarch)
+        return {"value": value}
 
-    def getinfo(self):
-        return (self.host, self.port)
+    async def handle_get_pr(self, request):
+        version = request["version"]
+        pkgarch = request["pkgarch"]
+        checksum = request["checksum"]
+        history = request["history"]
 
-    def getPR(self, version, pkgarch, checksum):
-        try:
-            return self.table.getValue(version, pkgarch, checksum)
-        except prserv.NotFoundError:
-            logger.error("can not find value for (%s, %s)",version, checksum)
-            return None
-        except sqlite3.Error as exc:
-            logger.error(str(exc))
-            return None
-
-    def quit(self):
-        self.quitflag=True
-        os.write(self.quitpipeout, b"q")
-        os.close(self.quitpipeout)
-        return
-
-    def work_forever(self,):
-        self.quitflag = False
-        # This timeout applies to the poll in TCPServer, we need the select 
-        # below to wake on our quit pipe closing. We only ever call into handle_request
-        # if there is data there.
-        self.timeout = 0.01
-
-        bb.utils.set_process_name("PRServ")
-
-        # DB connection must be created after all forks
-        self.db = prserv.db.PRData(self.dbfile)
-        self.table = self.db["PRMAIN"]
+        if self.upstream_client is None:
+            value = self.server.table.get_value(version, pkgarch, checksum, history)
+            return {"value": value}
 
-        logger.info("Started PRServer with DBfile: %s, IP: %s, PORT: %s, PID: %s" %
-                     (self.dbfile, self.host, self.port, str(os.getpid())))
-
-        self.handlerthread.start()
-        while not self.quitflag:
-            ready = select.select([self.fileno(), self.quitpipein], [], [], 30)
-            if self.quitflag:
-                break
-            if self.fileno() in ready[0]:
-                self.handle_request()
-        self.handlerthread.join()
-        self.db.disconnect()
-        logger.info("PRServer: stopping...")
-        self.server_close()
-        os.close(self.quitpipein)
-        return
+        # We have an upstream server.
+        # Check whether the local server already knows the requested configuration.
+        # If the configuration is a new one, the generated value we will add will
+        # depend on what's on the upstream server. That's why we're calling find_value()
+        # instead of get_value() directly.
 
-    def start(self):
-        if self.daemon:
-            pid = self.daemonize()
-        else:
-            pid = self.fork()
-            self.pid = pid
+        value = self.server.table.find_value(version, pkgarch, checksum, history)
+        upstream_max = await self.upstream_client.max_package_pr(version, pkgarch)
 
-        # Ensure both the parent sees this and the child from the work_forever log entry above
-        logger.info("Started PRServer with DBfile: %s, IP: %s, PORT: %s, PID: %s" %
-                     (self.dbfile, self.host, self.port, str(pid)))
+        if value is not None:
 
-    def delpid(self):
-        os.remove(self.pidfile)
+            # The configuration is already known locally.
 
-    def daemonize(self):
-        """
-        See Advanced Programming in the UNIX, Sec 13.3
-        """
-        try:
-            pid = os.fork()
-            if pid > 0:
-                os.waitpid(pid, 0)
-                #parent return instead of exit to give control 
-                return pid
-        except OSError as e:
-            raise Exception("%s [%d]" % (e.strerror, e.errno))
-
-        os.setsid()
-        """
-        fork again to make sure the daemon is not session leader, 
-        which prevents it from acquiring controlling terminal
-        """
-        try:
-            pid = os.fork()
-            if pid > 0: #parent
-                os._exit(0)
-        except OSError as e:
-            raise Exception("%s [%d]" % (e.strerror, e.errno))
+            if history:
+                value = self.server.table.get_value(version, pkgarch, checksum, history)
+            else:
+                existing_value = value
+                # In "no history", we need to make sure the value doesn't decrease
+                # and is at least greater than the maximum upstream value
+                # and the maximum local value
 
-        self.cleanup_handles()
-        os._exit(0)
+                local_max = self.server.table.find_package_max_value(version, pkgarch)
+                if revision_smaller(value, local_max):
+                    value = increase_revision(local_max)
+
+                if revision_smaller(value, upstream_max):
+                    # Ask upstream whether it knows the checksum
+                    upstream_value = await self.upstream_client.test_pr(version, pkgarch, checksum)
+                    if upstream_value is None:
+                        # Upstream doesn't have our checksum, let create a new one
+                        value = upstream_max + ".0"
+                    else:
+                        # Fine to take the same value as upstream
+                        value = upstream_max
+
+                if not value == existing_value and not self.server.read_only:
+                    self.server.table.store_value(version, pkgarch, checksum, value)
+
+            return {"value": value}
+
+        # The configuration is a new one for the local server
+        # Let's ask the upstream server whether it knows it
+
+        known_upstream = await self.upstream_client.test_package(version, pkgarch)
+
+        if not known_upstream:
+
+            # The package is not known upstream, must be a local-only package
+            # Let's compute the PR number using the local-only method
+
+            value = self.server.table.get_value(version, pkgarch, checksum, history)
+            return {"value": value}
+
+        # The package is known upstream, let's ask the upstream server
+        # whether it knows our new output hash
+
+        value = await self.upstream_client.test_pr(version, pkgarch, checksum)
+
+        if value is not None:
+
+            # Upstream knows this output hash, let's store it and use it too.
+
+            if not self.server.read_only:
+                self.server.table.store_value(version, pkgarch, checksum, value)
+            # If the local server is read only, won't be able to store the new
+            # value in the database and will have to keep asking the upstream server
+            return {"value": value}
+
+        # The output hash doesn't exist upstream, get the most recent number from upstream (x)
+        # Then, we want to have a new PR value for the local server: x.y
+
+        upstream_max = await self.upstream_client.max_package_pr(version, pkgarch)
+        # Here we know that the package is known upstream, so upstream_max can't be None
+        subvalue = self.server.table.find_new_subvalue(version, pkgarch, upstream_max)
+
+        if not self.server.read_only:
+            self.server.table.store_value(version, pkgarch, checksum, subvalue)
+
+        return {"value": subvalue}
+
+    async def process_requests(self):
+        if self.server.upstream is not None:
+            self.upstream_client = await create_async_client(self.server.upstream)
+        else:
+            self.upstream_client = None
 
-    def fork(self):
-        try:
-            pid = os.fork()
-            if pid > 0:
-                self.socket.close() # avoid ResourceWarning in parent
-                return pid
-        except OSError as e:
-            raise Exception("%s [%d]" % (e.strerror, e.errno))
-
-        bb.utils.signal_on_parent_exit("SIGTERM")
-        self.cleanup_handles()
-        os._exit(0)
-
-    def cleanup_handles(self):
-        signal.signal(signal.SIGINT, self.sigint_handler)
-        signal.signal(signal.SIGTERM, self.sigterm_handler)
-        os.chdir("/")
-
-        sys.stdout.flush()
-        sys.stderr.flush()
-
-        # We could be called from a python thread with io.StringIO as
-        # stdout/stderr or it could be 'real' unix fd forking where we need
-        # to physically close the fds to prevent the program launching us from
-        # potentially hanging on a pipe. Handle both cases.
-        si = open('/dev/null', 'r')
-        try:
-            os.dup2(si.fileno(),sys.stdin.fileno())
-        except (AttributeError, io.UnsupportedOperation):
-            sys.stdin = si
-        so = open(self.logfile, 'a+')
         try:
-            os.dup2(so.fileno(),sys.stdout.fileno())
-        except (AttributeError, io.UnsupportedOperation):
-            sys.stdout = so
+            await super().process_requests()
+        finally:
+            if self.upstream_client is not None:
+                await self.upstream_client.close()
+
+    async def handle_import_one(self, request):
+        response = None
+        if not self.server.read_only:
+            version = request["version"]
+            pkgarch = request["pkgarch"]
+            checksum = request["checksum"]
+            value = request["value"]
+
+            value = self.server.table.importone(version, pkgarch, checksum, value)
+            if value is not None:
+                response = {"value": value}
+
+        return response
+
+    async def handle_export(self, request):
+        version = request["version"]
+        pkgarch = request["pkgarch"]
+        checksum = request["checksum"]
+        colinfo = request["colinfo"]
+        history = request["history"]
+
         try:
-            os.dup2(so.fileno(),sys.stderr.fileno())
-        except (AttributeError, io.UnsupportedOperation):
-            sys.stderr = so
-
-        # Clear out all log handlers prior to the fork() to avoid calling
-        # event handlers not part of the PRserver
-        for logger_iter in logging.Logger.manager.loggerDict.keys():
-            logging.getLogger(logger_iter).handlers = []
-
-        # Ensure logging makes it to the logfile
-        streamhandler = logging.StreamHandler()
-        streamhandler.setLevel(logging.DEBUG)
-        formatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
-        streamhandler.setFormatter(formatter)
-        logger.addHandler(streamhandler)
-
-        # write pidfile
-        pid = str(os.getpid())
-        with open(self.pidfile, 'w') as pf:
-            pf.write("%s\n" % pid)
-
-        self.work_forever()
-        self.delpid()
+            (metainfo, datainfo) = self.server.table.export(version, pkgarch, checksum, colinfo, history)
+        except sqlite3.Error as exc:
+            self.logger.error(str(exc))
+            metainfo = datainfo = None
 
-class PRServSingleton(object):
-    def __init__(self, dbfile, logfile, interface):
+        return {"metainfo": metainfo, "datainfo": datainfo}
+
+    async def handle_is_readonly(self, request):
+        return {"readonly": self.server.read_only}
+
+class PRServer(bb.asyncrpc.AsyncServer):
+    def __init__(self, dbfile, read_only=False, upstream=None):
+        super().__init__(logger)
         self.dbfile = dbfile
-        self.logfile = logfile
-        self.interface = interface
-        self.host = None
-        self.port = None
+        self.table = None
+        self.read_only = read_only
+        self.upstream = upstream
+
+    def accept_client(self, socket):
+        return PRServerClient(socket, self)
 
     def start(self):
-        self.prserv = PRServer(self.dbfile, self.logfile, self.interface, daemon=False)
-        self.prserv.start()
-        self.host, self.port = self.prserv.getinfo()
+        tasks = super().start()
+        self.db = prserv.db.PRData(self.dbfile, read_only=self.read_only)
+        self.table = self.db["PRMAIN"]
+
+        self.logger.info("Started PRServer with DBfile: %s, Address: %s, PID: %s" %
+                     (self.dbfile, self.address, str(os.getpid())))
 
-    def getinfo(self):
-        return (self.host, self.port)
+        if self.upstream is not None:
+            self.logger.info("And upstream PRServer: %s " % (self.upstream))
 
-class PRServerConnection(object):
-    def __init__(self, host, port):
-        if is_local_special(host, port):
-            host, port = singleton.getinfo()
+        return tasks
+
+    async def stop(self):
+        self.db.disconnect()
+        await super().stop()
+
+class PRServSingleton(object):
+    def __init__(self, dbfile, logfile, host, port, upstream):
+        self.dbfile = dbfile
+        self.logfile = logfile
         self.host = host
         self.port = port
-        self.connection, self.transport = bb.server.xmlrpcclient._create_server(self.host, self.port)
-
-    def terminate(self):
-        try:
-            logger.info("Terminating PRServer...")
-            self.connection.quit()
-        except Exception as exc:
-            sys.stderr.write("%s\n" % str(exc))
+        self.upstream = upstream
 
-    def getPR(self, version, pkgarch, checksum):
-        return self.connection.getPR(version, pkgarch, checksum)
+    def start(self):
+        self.prserv = PRServer(self.dbfile, upstream=self.upstream)
+        self.prserv.start_tcp_server(socket.gethostbyname(self.host), self.port)
+        self.process = self.prserv.serve_as_process(log_level=logging.WARNING)
 
-    def ping(self):
-        return self.connection.ping()
+        if not self.prserv.address:
+            raise PRServiceConfigError
+        if not self.port:
+            self.port = int(self.prserv.address.rsplit(":", 1)[1])
 
-    def export(self,version=None, pkgarch=None, checksum=None, colinfo=True):
-        return self.connection.export(version, pkgarch, checksum, colinfo)
+def run_as_daemon(func, pidfile, logfile):
+    """
+    See Advanced Programming in the UNIX, Sec 13.3
+    """
+    try:
+        pid = os.fork()
+        if pid > 0:
+            os.waitpid(pid, 0)
+            #parent return instead of exit to give control
+            return pid
+    except OSError as e:
+        raise Exception("%s [%d]" % (e.strerror, e.errno))
 
-    def dump_db(self):
-        return self.connection.dump_db()
+    os.setsid()
+    """
+    fork again to make sure the daemon is not session leader,
+    which prevents it from acquiring controlling terminal
+    """
+    try:
+        pid = os.fork()
+        if pid > 0: #parent
+            os._exit(0)
+    except OSError as e:
+        raise Exception("%s [%d]" % (e.strerror, e.errno))
 
-    def importone(self, version, pkgarch, checksum, value):
-        return self.connection.importone(version, pkgarch, checksum, value)
+    os.chdir("/")
 
-    def getinfo(self):
-        return self.host, self.port
+    sys.stdout.flush()
+    sys.stderr.flush()
 
-def start_daemon(dbfile, host, port, logfile):
+    # We could be called from a python thread with io.StringIO as
+    # stdout/stderr or it could be 'real' unix fd forking where we need
+    # to physically close the fds to prevent the program launching us from
+    # potentially hanging on a pipe. Handle both cases.
+    si = open("/dev/null", "r")
+    try:
+        os.dup2(si.fileno(), sys.stdin.fileno())
+    except (AttributeError, io.UnsupportedOperation):
+        sys.stdin = si
+    so = open(logfile, "a+")
+    try:
+        os.dup2(so.fileno(), sys.stdout.fileno())
+    except (AttributeError, io.UnsupportedOperation):
+        sys.stdout = so
+    try:
+        os.dup2(so.fileno(), sys.stderr.fileno())
+    except (AttributeError, io.UnsupportedOperation):
+        sys.stderr = so
+
+    # Clear out all log handlers prior to the fork() to avoid calling
+    # event handlers not part of the PRserver
+    for logger_iter in logging.Logger.manager.loggerDict.keys():
+        logging.getLogger(logger_iter).handlers = []
+
+    # Ensure logging makes it to the logfile
+    streamhandler = logging.StreamHandler()
+    streamhandler.setLevel(logging.DEBUG)
+    formatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
+    streamhandler.setFormatter(formatter)
+    logger.addHandler(streamhandler)
+
+    # write pidfile
+    pid = str(os.getpid())
+    with open(pidfile, "w") as pf:
+        pf.write("%s\n" % pid)
+
+    func()
+    os.remove(pidfile)
+    os._exit(0)
+
+def start_daemon(dbfile, host, port, logfile, read_only=False, upstream=None):
     ip = socket.gethostbyname(host)
     pidfile = PIDPREFIX % (ip, port)
     try:
@@ -362,15 +334,13 @@ def start_daemon(dbfile, host, port, logfile):
                             % pidfile)
         return 1
 
-    server = PRServer(os.path.abspath(dbfile), os.path.abspath(logfile), (ip,port))
-    server.start()
+    dbfile = os.path.abspath(dbfile)
+    def daemon_main():
+        server = PRServer(dbfile, read_only=read_only, upstream=upstream)
+        server.start_tcp_server(ip, port)
+        server.serve_forever()
 
-    # Sometimes, the port (i.e. localhost:0) indicated by the user does not match with
-    # the one the server actually is listening, so at least warn the user about it
-    _,rport = server.getinfo()
-    if port != rport:
-        sys.stdout.write("Server is listening at port %s instead of %s\n"
-                         % (rport,port))
+    run_as_daemon(daemon_main, pidfile, os.path.abspath(logfile))
     return 0
 
 def stop_daemon(host, port):
@@ -388,37 +358,28 @@ def stop_daemon(host, port):
         # so at least advise the user which ports the corresponding server is listening
         ports = []
         portstr = ""
-        for pf in glob.glob(PIDPREFIX % (ip,'*')):
+        for pf in glob.glob(PIDPREFIX % (ip, "*")):
             bn = os.path.basename(pf)
             root, _ = os.path.splitext(bn)
-            ports.append(root.split('_')[-1])
+            ports.append(root.split("_")[-1])
         if len(ports):
-            portstr = "Wrong port? Other ports listening at %s: %s" % (host, ' '.join(ports))
+            portstr = "Wrong port? Other ports listening at %s: %s" % (host, " ".join(ports))
 
         sys.stderr.write("pidfile %s does not exist. Daemon not running? %s\n"
-                         % (pidfile,portstr))
+                         % (pidfile, portstr))
         return 1
 
     try:
-        PRServerConnection(ip, port).terminate()
-    except:
-        logger.critical("Stop PRService %s:%d failed" % (host,port))
-
-    try:
-        if pid:
-            wait_timeout = 0
-            print("Waiting for pr-server to exit.")
-            while is_running(pid) and wait_timeout < 50:
-                time.sleep(0.1)
-                wait_timeout += 1
+        if is_running(pid):
+            print("Sending SIGTERM to pr-server.")
+            os.kill(pid, signal.SIGTERM)
+            time.sleep(0.1)
 
-            if is_running(pid):
-                print("Sending SIGTERM to pr-server.")
-                os.kill(pid,signal.SIGTERM)
-                time.sleep(0.1)
-
-            if os.path.exists(pidfile):
-                os.remove(pidfile)
+        try:
+            os.remove(pidfile)
+        except FileNotFoundError:
+            # The PID file might have been removed by the exiting process
+            pass
 
     except OSError as e:
         err = str(e)
@@ -436,7 +397,7 @@ def is_running(pid):
     return True
 
 def is_local_special(host, port):
-    if host.strip().upper() == 'localhost'.upper() and (not port):
+    if (host == "localhost" or host == "127.0.0.1") and not port:
         return True
     else:
         return False
@@ -447,7 +408,7 @@ class PRServiceConfigError(Exception):
 def auto_start(d):
     global singleton
 
-    host_params = list(filter(None, (d.getVar('PRSERV_HOST') or '').split(':')))
+    host_params = list(filter(None, (d.getVar("PRSERV_HOST") or "").split(":")))
     if not host_params:
         # Shutdown any existing PR Server
         auto_shutdown()
@@ -456,11 +417,16 @@ def auto_start(d):
     if len(host_params) != 2:
         # Shutdown any existing PR Server
         auto_shutdown()
-        logger.critical('\n'.join(['PRSERV_HOST: incorrect format',
+        logger.critical("\n".join(["PRSERV_HOST: incorrect format",
                 'Usage: PRSERV_HOST = "<hostname>:<port>"']))
         raise PRServiceConfigError
 
-    if is_local_special(host_params[0], int(host_params[1])):
+    host = host_params[0].strip().lower()
+    port = int(host_params[1])
+
+    upstream = d.getVar("PRSERV_UPSTREAM") or None
+
+    if is_local_special(host, port):
         import bb.utils
         cachedir = (d.getVar("PERSISTENT_DIR") or d.getVar("CACHE"))
         if not cachedir:
@@ -474,39 +440,43 @@ def auto_start(d):
                auto_shutdown()
         if not singleton:
             bb.utils.mkdirhier(cachedir)
-            singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), ("localhost",0))
+            singleton = PRServSingleton(os.path.abspath(dbfile), os.path.abspath(logfile), host, port, upstream)
             singleton.start()
     if singleton:
-        host, port = singleton.getinfo()
-    else:
-        host = host_params[0]
-        port = int(host_params[1])
+        host = singleton.host
+        port = singleton.port
 
     try:
-        connection = PRServerConnection(host,port)
-        connection.ping()
-        realhost, realport = connection.getinfo()
-        return str(realhost) + ":" + str(realport)
-        
+        ping(host, port)
+        return str(host) + ":" + str(port)
+
     except Exception:
         logger.critical("PRservice %s:%d not available" % (host, port))
         raise PRServiceConfigError
 
 def auto_shutdown():
     global singleton
-    if singleton:
-        host, port = singleton.getinfo()
-        try:
-            PRServerConnection(host, port).terminate()
-        except:
-            logger.critical("Stop PRService %s:%d failed" % (host,port))
-
-        try:
-            os.waitpid(singleton.prserv.pid, 0)
-        except ChildProcessError:
-            pass
+    if singleton and singleton.process:
+        singleton.process.terminate()
+        singleton.process.join()
         singleton = None
 
 def ping(host, port):
-    conn=PRServerConnection(host, port)
-    return conn.ping()
+    from . import client
+
+    with client.PRClient() as conn:
+        conn.connect_tcp(host, port)
+        return conn.ping()
+
+def connect(host, port):
+    from . import client
+
+    global singleton
+
+    if host.strip().lower() == "localhost" and not port:
+        host = "localhost"
+        port = singleton.port
+
+    conn = client.PRClient()
+    conn.connect_tcp(host, port)
+    return conn
diff --git a/bitbake/lib/prserv/tests.py b/bitbake/lib/prserv/tests.py
new file mode 100644
index 0000000000..df0c003003
--- /dev/null
+++ b/bitbake/lib/prserv/tests.py
@@ -0,0 +1,388 @@
+#! /usr/bin/env python3
+#
+# Copyright (C) 2024 BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+from . import create_server, create_client, increase_revision, revision_greater, revision_smaller, _revision_greater_or_equal
+import prserv.db as db
+from bb.asyncrpc import InvokeError
+import logging
+import os
+import sys
+import tempfile
+import unittest
+import socket
+import subprocess
+from pathlib import Path
+
+THIS_DIR = Path(__file__).parent
+BIN_DIR = THIS_DIR.parent.parent / "bin"
+
+version = "dummy-1.0-r0"
+pkgarch = "core2-64"
+other_arch = "aarch64"
+
+checksumX = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4f0"
+checksum0 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a0"
+checksum1 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a1"
+checksum2 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a2"
+checksum3 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a3"
+checksum4 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a4"
+checksum5 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a5"
+checksum6 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a6"
+checksum7 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a7"
+checksum8 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a8"
+checksum9 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4a9"
+checksum10 = "51bf8189dbe9ea81fa6dd89608bf19380c437a9cf12f6c6239887801ba4ab4aa"
+
+def server_prefunc(server, name):
+    logging.basicConfig(level=logging.DEBUG, filename='prserv-%s.log' % name, filemode='w',
+                        format='%(levelname)s %(filename)s:%(lineno)d %(message)s')
+    server.logger.debug("Running server %s" % name)
+    sys.stdout = open('prserv-stdout-%s.log' % name, 'w')
+    sys.stderr = sys.stdout
+
+class PRTestSetup(object):
+
+    def start_server(self, name, dbfile, upstream=None, read_only=False, prefunc=server_prefunc):
+
+        def cleanup_server(server):
+            if server.process.exitcode is not None:
+                return
+            server.process.terminate()
+            server.process.join()
+
+        server = create_server(socket.gethostbyname("localhost") + ":0",
+                               dbfile,
+                               upstream=upstream,
+                               read_only=read_only)
+
+        server.serve_as_process(prefunc=prefunc, args=(name,))
+        self.addCleanup(cleanup_server, server)
+
+        return server
+
+    def start_client(self, server_address):
+        def cleanup_client(client):
+            client.close()
+
+        client = create_client(server_address)
+        self.addCleanup(cleanup_client, client)
+
+        return client
+
+class FunctionTests(unittest.TestCase):
+
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv')
+        self.addCleanup(self.temp_dir.cleanup)
+
+    def test_increase_revision(self):
+        self.assertEqual(increase_revision("1"), "2")
+        self.assertEqual(increase_revision("1.0"), "1.1")
+        self.assertEqual(increase_revision("1.1.1"), "1.1.2")
+        self.assertEqual(increase_revision("1.1.1.3"), "1.1.1.4")
+        self.assertEqual(increase_revision("9"), "10")
+        self.assertEqual(increase_revision("1.9"), "1.10")
+        self.assertRaises(ValueError, increase_revision, "1.a")
+        self.assertRaises(ValueError, increase_revision, "1.")
+        self.assertRaises(ValueError, increase_revision, "")
+
+    def test_revision_greater_or_equal(self):
+        self.assertTrue(_revision_greater_or_equal("2", "2"))
+        self.assertTrue(_revision_greater_or_equal("2", "1"))
+        self.assertTrue(_revision_greater_or_equal("10", "2"))
+        self.assertTrue(_revision_greater_or_equal("1.10", "1.2"))
+        self.assertFalse(_revision_greater_or_equal("1.2", "1.10"))
+        self.assertTrue(_revision_greater_or_equal("1.10", "1"))
+        self.assertTrue(_revision_greater_or_equal("1.10.1", "1.10"))
+        self.assertFalse(_revision_greater_or_equal("1.10.1", "1.10.2"))
+        self.assertTrue(_revision_greater_or_equal("1.10.1", "1.10.1"))
+        self.assertTrue(_revision_greater_or_equal("1.10.1", "1"))
+        self.assertTrue(revision_greater("1.20", "1.3"))
+        self.assertTrue(revision_smaller("1.3", "1.20"))
+
+    # DB tests
+
+    def test_db(self):
+        dbfile = os.path.join(self.temp_dir.name, "testtable.sqlite3")
+
+        self.db = db.PRData(dbfile)
+        self.table = self.db["PRMAIN"]
+
+        self.table.store_value(version, pkgarch, checksum0, "0")
+        self.table.store_value(version, pkgarch, checksum1, "1")
+        # "No history" mode supports multiple PRs for the same checksum
+        self.table.store_value(version, pkgarch, checksum0, "2")
+        self.table.store_value(version, pkgarch, checksum2, "1.0")
+
+        self.assertTrue(self.table.test_package(version, pkgarch))
+        self.assertFalse(self.table.test_package(version, other_arch))
+
+        self.assertTrue(self.table.test_value(version, pkgarch, "0"))
+        self.assertTrue(self.table.test_value(version, pkgarch, "1"))
+        self.assertTrue(self.table.test_value(version, pkgarch, "2"))
+
+        self.assertEqual(self.table.find_package_max_value(version, pkgarch), "2")
+
+        self.assertEqual(self.table.find_min_value(version, pkgarch, checksum0), "0")
+        self.assertEqual(self.table.find_max_value(version, pkgarch, checksum0), "2")
+
+        # Test history modes
+        self.assertEqual(self.table.find_value(version, pkgarch, checksum0, True), "0")
+        self.assertEqual(self.table.find_value(version, pkgarch, checksum0, False), "2")
+
+        self.assertEqual(self.table.find_new_subvalue(version, pkgarch, "3"), "3.0")
+        self.assertEqual(self.table.find_new_subvalue(version, pkgarch, "1"), "1.1")
+
+        # Revision comparison tests
+        self.table.store_value(version, pkgarch, checksum1, "1.3")
+        self.table.store_value(version, pkgarch, checksum1, "1.20")
+        self.assertEqual(self.table.find_min_value(version, pkgarch, checksum1), "1")
+        self.assertEqual(self.table.find_max_value(version, pkgarch, checksum1), "1.20")
+
+class PRBasicTests(PRTestSetup, unittest.TestCase):
+
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv')
+        self.addCleanup(self.temp_dir.cleanup)
+
+        dbfile = os.path.join(self.temp_dir.name, "prtest-basic.sqlite3")
+
+        self.server1 = self.start_server("basic", dbfile)
+        self.client1 = self.start_client(self.server1.address)
+
+    def test_basic(self):
+
+        # Checks on non existing configuration
+
+        result = self.client1.test_pr(version, pkgarch, checksum0)
+        self.assertIsNone(result, "test_pr should return 'None' for a non existing PR")
+
+        result = self.client1.test_package(version, pkgarch)
+        self.assertFalse(result, "test_package should return 'False' for a non existing PR")
+
+        result = self.client1.max_package_pr(version, pkgarch)
+        self.assertIsNone(result, "max_package_pr should return 'None' for a non existing PR")
+
+        # Add a first configuration
+
+        result = self.client1.getPR(version, pkgarch, checksum0)
+        self.assertEqual(result, "0", "getPR: initial PR of a package should be '0'")
+
+        result = self.client1.test_pr(version, pkgarch, checksum0)
+        self.assertEqual(result, "0", "test_pr should return '0' here, matching the result of getPR")
+
+        result = self.client1.test_package(version, pkgarch)
+        self.assertTrue(result, "test_package should return 'True' for an existing PR")
+
+        result = self.client1.max_package_pr(version, pkgarch)
+        self.assertEqual(result, "0", "max_package_pr should return '0' in the current test series")
+
+        # Check that the same request gets the same value
+
+        result = self.client1.getPR(version, pkgarch, checksum0)
+        self.assertEqual(result, "0", "getPR: asking for the same PR a second time in a row should return the same value.")
+
+        # Add new configurations
+
+        result = self.client1.getPR(version, pkgarch, checksum1)
+        self.assertEqual(result, "1", "getPR: second PR of a package should be '1'")
+
+        result = self.client1.test_pr(version, pkgarch, checksum1)
+        self.assertEqual(result, "1", "test_pr should return '1' here, matching the result of getPR")
+
+        result = self.client1.max_package_pr(version, pkgarch)
+        self.assertEqual(result, "1", "max_package_pr should return '1' in the current test series")
+
+        result = self.client1.getPR(version, pkgarch, checksum2)
+        self.assertEqual(result, "2", "getPR: second PR of a package should be '2'")
+
+        result = self.client1.test_pr(version, pkgarch, checksum2)
+        self.assertEqual(result, "2", "test_pr should return '2' here, matching the result of getPR")
+
+        result = self.client1.max_package_pr(version, pkgarch)
+        self.assertEqual(result, "2", "max_package_pr should return '2' in the current test series")
+
+        result = self.client1.getPR(version, pkgarch, checksum3)
+        self.assertEqual(result, "3", "getPR: second PR of a package should be '3'")
+
+        result = self.client1.test_pr(version, pkgarch, checksum3)
+        self.assertEqual(result, "3", "test_pr should return '3' here, matching the result of getPR")
+
+        result = self.client1.max_package_pr(version, pkgarch)
+        self.assertEqual(result, "3", "max_package_pr should return '3' in the current test series")
+
+        # Ask again for the first configuration
+
+        result = self.client1.getPR(version, pkgarch, checksum0)
+        self.assertEqual(result, "4", "getPR: should return '4' in this configuration")
+
+        # Ask again with explicit "no history" mode
+
+        result = self.client1.getPR(version, pkgarch, checksum0, False)
+        self.assertEqual(result, "4", "getPR: should return '4' in this configuration")
+
+        # Ask again with explicit "history" mode. This should return the first recorded PR for checksum0
+
+        result = self.client1.getPR(version, pkgarch, checksum0, True)
+        self.assertEqual(result, "0", "getPR: should return '0' in this configuration")
+
+        # Check again that another pkgarg resets the counters
+
+        result = self.client1.test_pr(version, other_arch, checksum0)
+        self.assertIsNone(result, "test_pr should return 'None' for a non existing PR")
+
+        result = self.client1.test_package(version, other_arch)
+        self.assertFalse(result, "test_package should return 'False' for a non existing PR")
+
+        result = self.client1.max_package_pr(version, other_arch)
+        self.assertIsNone(result, "max_package_pr should return 'None' for a non existing PR")
+
+        # Now add the configuration
+
+        result = self.client1.getPR(version, other_arch, checksum0)
+        self.assertEqual(result, "0", "getPR: initial PR of a package should be '0'")
+
+        result = self.client1.test_pr(version, other_arch, checksum0)
+        self.assertEqual(result, "0", "test_pr should return '0' here, matching the result of getPR")
+
+        result = self.client1.test_package(version, other_arch)
+        self.assertTrue(result, "test_package should return 'True' for an existing PR")
+
+        result = self.client1.max_package_pr(version, other_arch)
+        self.assertEqual(result, "0", "max_package_pr should return '0' in the current test series")
+
+        result = self.client1.is_readonly()
+        self.assertFalse(result, "Server should not be described as 'read-only'")
+
+class PRUpstreamTests(PRTestSetup, unittest.TestCase):
+
+    def setUp(self):
+
+        self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv')
+        self.addCleanup(self.temp_dir.cleanup)
+
+        dbfile2 = os.path.join(self.temp_dir.name, "prtest-upstream2.sqlite3")
+        self.server2 = self.start_server("upstream2", dbfile2)
+        self.client2 = self.start_client(self.server2.address)
+
+        dbfile1 = os.path.join(self.temp_dir.name, "prtest-upstream1.sqlite3")
+        self.server1 = self.start_server("upstream1", dbfile1, upstream=self.server2.address)
+        self.client1 = self.start_client(self.server1.address)
+
+        dbfile0 = os.path.join(self.temp_dir.name, "prtest-local.sqlite3")
+        self.server0 = self.start_server("local", dbfile0, upstream=self.server1.address)
+        self.client0 = self.start_client(self.server0.address)
+        self.shared_db = dbfile0
+
+    def test_upstream_and_readonly(self):
+
+        # For identical checksums, all servers should return the same PR
+
+        result = self.client2.getPR(version, pkgarch, checksum0)
+        self.assertEqual(result, "0", "getPR: initial PR of a package should be '0'")
+
+        result = self.client1.getPR(version, pkgarch, checksum0)
+        self.assertEqual(result, "0", "getPR: initial PR of a package should be '0' (same as upstream)")
+
+        result = self.client0.getPR(version, pkgarch, checksum0)
+        self.assertEqual(result, "0", "getPR: initial PR of a package should be '0' (same as upstream)")
+
+        # Now introduce new checksums on server1 for, same version
+
+        result = self.client1.getPR(version, pkgarch, checksum1)
+        self.assertEqual(result, "0.0", "getPR: first PR of a package which has a different checksum upstream should be '0.0'")
+
+        result = self.client1.getPR(version, pkgarch, checksum2)
+        self.assertEqual(result, "0.1", "getPR: second PR of a package that has a different checksum upstream should be '0.1'")
+
+        # Now introduce checksums on server0 for, same version
+
+        result = self.client1.getPR(version, pkgarch, checksum1)
+        self.assertEqual(result, "0.2", "getPR: can't decrease for known PR")
+
+        result = self.client1.getPR(version, pkgarch, checksum2)
+        self.assertEqual(result, "0.3")
+
+        result = self.client1.max_package_pr(version, pkgarch)
+        self.assertEqual(result, "0.3")
+
+        result = self.client0.getPR(version, pkgarch, checksum3)
+        self.assertEqual(result, "0.3.0", "getPR: first PR of a package that doesn't exist upstream should be '0.3.0'")
+
+        result = self.client0.getPR(version, pkgarch, checksum4)
+        self.assertEqual(result, "0.3.1", "getPR: second PR of a package that doesn't exist upstream should be '0.3.1'")
+
+        result = self.client0.getPR(version, pkgarch, checksum3)
+        self.assertEqual(result, "0.3.2")
+
+        # More upstream updates
+        # Here, we assume no communication between server2 and server0. server2 only impacts server0
+        # after impacting server1
+
+        self.assertEqual(self.client2.getPR(version, pkgarch, checksum5), "1")
+        self.assertEqual(self.client1.getPR(version, pkgarch, checksum6), "1.0")
+        self.assertEqual(self.client1.getPR(version, pkgarch, checksum7), "1.1")
+        self.assertEqual(self.client0.getPR(version, pkgarch, checksum8), "1.1.0")
+        self.assertEqual(self.client0.getPR(version, pkgarch, checksum9), "1.1.1")
+
+        # "history" mode tests
+
+        self.assertEqual(self.client2.getPR(version, pkgarch, checksum0, True), "0")
+        self.assertEqual(self.client1.getPR(version, pkgarch, checksum2, True), "0.1")
+        self.assertEqual(self.client0.getPR(version, pkgarch, checksum3, True), "0.3.0")
+
+        # More "no history" mode tests
+
+        self.assertEqual(self.client2.getPR(version, pkgarch, checksum0), "2")
+        self.assertEqual(self.client1.getPR(version, pkgarch, checksum0), "2") # Same as upstream
+        self.assertEqual(self.client0.getPR(version, pkgarch, checksum0), "2") # Same as upstream
+        self.assertEqual(self.client1.getPR(version, pkgarch, checksum7), "3") # This could be surprising, but since the previous revision was "2", increasing it yields "3".
+                                                                               # We don't know how many upstream servers we have
+        # Start read-only server with server1 as upstream
+        self.server_ro = self.start_server("local-ro", self.shared_db, upstream=self.server1.address, read_only=True)
+        self.client_ro = self.start_client(self.server_ro.address)
+
+        self.assertTrue(self.client_ro.is_readonly(), "Database should be described as 'read-only'")
+
+        # Checks on non existing configurations
+        self.assertIsNone(self.client_ro.test_pr(version, pkgarch, checksumX))
+        self.assertFalse(self.client_ro.test_package("unknown", pkgarch))
+
+        # Look up existing configurations
+        self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum0), "3") # "no history" mode
+        self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum0, True), "0") # "history" mode
+        self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum3), "3")
+        self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum3, True), "0.3.0")
+        self.assertEqual(self.client_ro.max_package_pr(version, pkgarch), "2") # normal as "3" was never saved
+
+        # Try to insert a new value. Here this one is know upstream.
+        self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum7), "3")
+        # Try to insert a completely new value. As the max upstream value is already "3", it should be "3.0"
+        self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum10), "3.0")
+        # Same with another value which only exists in the upstream upstream server
+        # This time, as the upstream server doesn't know it, it will ask its upstream server. So that's a known one.
+        self.assertEqual(self.client_ro.getPR(version, pkgarch, checksum9), "3")
+
+class ScriptTests(unittest.TestCase):
+
+    def setUp(self):
+
+        self.temp_dir = tempfile.TemporaryDirectory(prefix='bb-prserv')
+        self.addCleanup(self.temp_dir.cleanup)
+        self.dbfile = os.path.join(self.temp_dir.name, "prtest.sqlite3")
+
+    def test_1_start_bitbake_prserv(self):
+        try:
+            subprocess.check_call([BIN_DIR / "bitbake-prserv", "--start", "-f", self.dbfile])
+        except subprocess.CalledProcessError as e:
+            self.fail("Failed to start bitbake-prserv: %s" % e.returncode)
+
+    def test_2_stop_bitbake_prserv(self):
+        try:
+            subprocess.check_call([BIN_DIR / "bitbake-prserv", "--stop"])
+        except subprocess.CalledProcessError as e:
+            self.fail("Failed to stop bitbake-prserv: %s" % e.returncode)
diff --git a/bitbake/lib/pyinotify.py b/bitbake/lib/pyinotify.py
index 6ae40a2d76..3c5dab0312 100644
--- a/bitbake/lib/pyinotify.py
+++ b/bitbake/lib/pyinotify.py
@@ -52,7 +52,6 @@ from collections import deque
 from datetime import datetime, timedelta
 import time
 import re
-import asyncore
 import glob
 import locale
 import subprocess
@@ -596,14 +595,24 @@ class _ProcessEvent:
         @type event: Event object
         @return: By convention when used from the ProcessEvent class:
                  - Returning False or None (default value) means keep on
-                 executing next chained functors (see chain.py example).
+                   executing next chained functors (see chain.py example).
                  - Returning True instead means do not execute next
                    processing functions.
         @rtype: bool
         @raise ProcessEventError: Event object undispatchable,
                                   unknown event.
         """
-        stripped_mask = event.mask - (event.mask & IN_ISDIR)
+        stripped_mask = event.mask & ~IN_ISDIR
+        # Bitbake hack - we see event masks of 0x6, i.e., IN_MODIFY & IN_ATTRIB.
+        # The kernel inotify code can set more than one of the bits in the mask,
+        # fsnotify_change() in linux/fsnotify.h is quite clear that IN_ATTRIB,
+        # IN_MODIFY and IN_ACCESS can arrive together.
+        # This breaks the code below which assume only one mask bit is ever
+        # set in an event. We don't care about attrib or access in bitbake so
+        # drop those.
+        if stripped_mask & IN_MODIFY:
+            stripped_mask &= ~(IN_ATTRIB | IN_ACCESS)
+
         maskname = EventsCodes.ALL_VALUES.get(stripped_mask)
         if maskname is None:
             raise ProcessEventError("Unknown mask 0x%08x" % stripped_mask)
@@ -1475,35 +1484,6 @@ class ThreadedNotifier(threading.Thread, Notifier):
         self.loop()
 
 
-class AsyncNotifier(asyncore.file_dispatcher, Notifier):
-    """
-    This notifier inherits from asyncore.file_dispatcher in order to be able to
-    use pyinotify along with the asyncore framework.
-
-    """
-    def __init__(self, watch_manager, default_proc_fun=None, read_freq=0,
-                 threshold=0, timeout=None, channel_map=None):
-        """
-        Initializes the async notifier. The only additional parameter is
-        'channel_map' which is the optional asyncore private map. See
-        Notifier class for the meaning of the others parameters.
-
-        """
-        Notifier.__init__(self, watch_manager, default_proc_fun, read_freq,
-                          threshold, timeout)
-        asyncore.file_dispatcher.__init__(self, self._fd, channel_map)
-
-    def handle_read(self):
-        """
-        When asyncore tells us we can read from the fd, we proceed processing
-        events. This method can be overridden for handling a notification
-        differently.
-
-        """
-        self.read_events()
-        self.process_events()
-
-
 class TornadoAsyncNotifier(Notifier):
     """
     Tornado ioloop adapter.
diff --git a/bitbake/lib/toaster/bldcollector/urls.py b/bitbake/lib/toaster/bldcollector/urls.py
index efd67a81a5..3c34070351 100644
--- a/bitbake/lib/toaster/bldcollector/urls.py
+++ b/bitbake/lib/toaster/bldcollector/urls.py
@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-from django.conf.urls import url
+from django.urls import re_path as url
 
 import bldcollector.views
 
diff --git a/bitbake/lib/toaster/bldcollector/views.py b/bitbake/lib/toaster/bldcollector/views.py
index 04cd8b3dd4..bdf38ae6e8 100644
--- a/bitbake/lib/toaster/bldcollector/views.py
+++ b/bitbake/lib/toaster/bldcollector/views.py
@@ -14,8 +14,11 @@ import subprocess
 import toastermain
 from django.views.decorators.csrf import csrf_exempt
 
+from toastermain.logs import log_view_mixin
+
 
 @csrf_exempt
+@log_view_mixin
 def eventfile(request):
     """ Receives a file by POST, and runs toaster-eventreply on this file """
     if request.method != "POST":
diff --git a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
index 75674ccbf1..577e765f11 100644
--- a/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
+++ b/bitbake/lib/toaster/bldcontrol/localhostbecontroller.py
@@ -200,7 +200,7 @@ class LocalhostBEController(BuildEnvironmentController):
                 localdirpath = os.path.join(localdirname, dirpath)
                 logger.debug("localhostbecontroller: localdirpath expects '%s'" % localdirpath)
                 if not os.path.exists(localdirpath):
-                    raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting." % (localdirpath, giturl, commit))
+                    raise BuildSetupException("Cannot find layer git path '%s' in checked out repository '%s:%s'. Exiting." % (localdirpath, giturl, commit))
 
                 if name != "bitbake":
                     layerlist.append("%03d:%s" % (index,localdirpath.rstrip("/")))
@@ -467,7 +467,7 @@ class LocalhostBEController(BuildEnvironmentController):
             logger.debug("localhostbecontroller: waiting for bblock content to appear")
             time.sleep(1)
         else:
-            raise BuildSetupException("Cannot find bitbake server lock file '%s'. Aborting." % bblock)
+            raise BuildSetupException("Cannot find bitbake server lock file '%s'. Exiting." % bblock)
 
         with open(bblock) as fplock:
             for line in fplock:
diff --git a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
index 19f659ec41..834e32b36f 100644
--- a/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
+++ b/bitbake/lib/toaster/bldcontrol/management/commands/runbuilds.py
@@ -180,6 +180,77 @@ class Command(BaseCommand):
         except Exception as e:
             logger.warning("runbuilds: schedule exception %s" % str(e))
 
+    # Test to see if a build pre-maturely died due to a bitbake crash
+    def check_dead_builds(self):
+        do_cleanup = False
+        try:
+            for br in BuildRequest.objects.filter(state=BuildRequest.REQ_INPROGRESS):
+                # Get the build directory
+                if br.project.builddir:
+                    builddir =  br.project.builddir
+                else:
+                    builddir = '%s-toaster-%d' % (br.environment.builddir,br.project.id)
+                # Check log to see if there is a recent traceback
+                toaster_ui_log = os.path.join(builddir, 'toaster_ui.log')
+                test_file = os.path.join(builddir, '._toaster_check.txt')
+                os.system("tail -n 50 %s > %s" % (os.path.join(builddir, 'toaster_ui.log'),test_file))
+                traceback_text = ''
+                is_traceback = False
+                with open(test_file,'r') as test_file_fd:
+                    test_file_tail = test_file_fd.readlines()
+                    for line in test_file_tail:
+                        if line.startswith('Traceback (most recent call last):'):
+                            traceback_text = line
+                            is_traceback = True
+                        elif line.startswith('NOTE: ToasterUI waiting for events'):
+                            # Ignore any traceback before new build start
+                            traceback_text = ''
+                            is_traceback = False
+                        elif line.startswith('Note: Toaster traceback auto-stop'):
+                            # Ignore any traceback before this previous traceback catch
+                            traceback_text = ''
+                            is_traceback = False
+                        elif is_traceback:
+                            traceback_text += line
+                # Test the results
+                is_stop = False
+                if is_traceback:
+                    # Found a traceback
+                    errtype = 'Bitbake crash'
+                    errmsg = 'Bitbake crash\n' + traceback_text
+                    state = BuildRequest.REQ_FAILED
+                    # Clean up bitbake files
+                    bitbake_lock = os.path.join(builddir, 'bitbake.lock')
+                    if os.path.isfile(bitbake_lock):
+                        os.remove(bitbake_lock)
+                    bitbake_sock = os.path.join(builddir, 'bitbake.sock')
+                    if os.path.isfile(bitbake_sock):
+                        os.remove(bitbake_sock)
+                    if os.path.isfile(test_file):
+                        os.remove(test_file)
+                    # Add note to ignore this traceback on next check
+                    os.system('echo "Note: Toaster traceback auto-stop" >> %s' % toaster_ui_log)
+                    is_stop = True
+                # Add more tests here
+                #elif ...
+                # Stop the build request?
+                if is_stop:
+                    brerror = BRError(
+                        req = br,
+                        errtype = errtype,
+                        errmsg = errmsg,
+                        traceback = traceback_text,
+                        )
+                    brerror.save()
+                    br.state = state
+                    br.save()
+                    do_cleanup = True
+            # Do cleanup
+            if do_cleanup:
+                self.cleanup()
+        except Exception as e:
+            logger.error("runbuilds: Error in check_dead_builds %s" % e)
+
     def handle(self, **options):
         pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
                                     ".runbuilds.pid")
@@ -187,10 +258,18 @@ class Command(BaseCommand):
         with open(pidfile_path, 'w') as pidfile:
             pidfile.write("%s" % os.getpid())
 
+        # Clean up any stale/failed builds from previous Toaster run
         self.runbuild()
 
         signal.signal(signal.SIGUSR1, lambda sig, frame: None)
 
         while True:
-            signal.pause()
-            self.runbuild()
+            sigset = signal.sigtimedwait([signal.SIGUSR1], 5)
+            if sigset:
+                for sig in sigset:
+                    # Consume each captured pending event
+                    self.runbuild()
+            else:
+                # Check for build exceptions
+                self.check_dead_builds()
+
diff --git a/bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py b/bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py
new file mode 100644
index 0000000000..45b477d02c
--- /dev/null
+++ b/bitbake/lib/toaster/bldcontrol/migrations/0008_models_bigautofield.py
@@ -0,0 +1,48 @@
+# Generated by Django 3.2.12 on 2022-03-06 03:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bldcontrol', '0007_brlayers_optional_gitinfo'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='brbitbake',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='brerror',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='brlayer',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='brtarget',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='brvariable',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='buildenvironment',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='buildrequest',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]
diff --git a/bitbake/lib/toaster/bldcontrol/models.py b/bitbake/lib/toaster/bldcontrol/models.py
index c2f302da24..42750e7180 100644
--- a/bitbake/lib/toaster/bldcontrol/models.py
+++ b/bitbake/lib/toaster/bldcontrol/models.py
@@ -4,7 +4,7 @@
 
 from __future__ import unicode_literals
 from django.db import models
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 from orm.models import Project, Build, Layer_Version
 
 import logging
@@ -124,7 +124,7 @@ class BuildRequest(models.Model):
         return self.brvariable_set.get(name="MACHINE").value
 
     def __str__(self):
-        return force_text('%s %s' % (self.project, self.get_state_display()))
+        return force_str('%s %s' % (self.project, self.get_state_display()))
 
 # These tables specify the settings for running an actual build.
 # They MUST be kept in sync with the tables in orm.models.Project*
diff --git a/bitbake/lib/toaster/logs/.gitignore b/bitbake/lib/toaster/logs/.gitignore
new file mode 100644
index 0000000000..e5ebf25a49
--- /dev/null
+++ b/bitbake/lib/toaster/logs/.gitignore
@@ -0,0 +1 @@
+*.log*
diff --git a/bitbake/lib/toaster/manage.py b/bitbake/lib/toaster/manage.py
index ae32619d12..f8de49c264 100755
--- a/bitbake/lib/toaster/manage.py
+++ b/bitbake/lib/toaster/manage.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 #
+# Copyright BitBake Contributors
+#
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
diff --git a/bitbake/lib/toaster/orm/fixtures/README b/bitbake/lib/toaster/orm/fixtures/README
index 1b1c660aac..7cd745e26b 100644
--- a/bitbake/lib/toaster/orm/fixtures/README
+++ b/bitbake/lib/toaster/orm/fixtures/README
@@ -27,4 +27,4 @@ Data can be provided in XML, JSON and if installed YAML formats.
 
 Use the django management command manage.py loaddata <your fixture file>
 For further information see the Django command documentation at:
-https://docs.djangoproject.com/en/1.8/ref/django-admin/#django-admin-loaddata
+https://docs.djangoproject.com/en/3.2/ref/django-admin/#django-admin-loaddata
diff --git a/bitbake/lib/toaster/orm/fixtures/check_fixtures.py b/bitbake/lib/toaster/orm/fixtures/check_fixtures.py
new file mode 100755
index 0000000000..ae3722e0f6
--- /dev/null
+++ b/bitbake/lib/toaster/orm/fixtures/check_fixtures.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2025 Linux Foundation
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import json
+import urllib.request
+
+import gen_fixtures as fixtures
+
+RELEASE_URL = "https://dashboard.yoctoproject.org/releases.json"
+
+with urllib.request.urlopen(RELEASE_URL) as response:
+    if response.getcode() == 200:
+        data = response.read().decode("utf-8")
+        releases = json.loads(data)
+    else:
+        print("Couldn't access %s: %s" % (RELEASE_URL, reponse.getcode()))
+        exit(1)
+
+
+# grab the recent release branches and add master, so we can ignore old branches
+active_releases = [
+    e["release_codename"].lower() for e in releases if e["series"] == "current"
+]
+active_releases.append("master")
+active_releases.append("head")
+
+fixtures_releases = [x[0].lower() for x in fixtures.current_releases]
+
+if set(active_releases) != set(fixtures_releases):
+    print("WARNING: Active releases don't match toaster configured releases, the difference is: %s" % set(active_releases).difference(set(fixtures_releases)))
+    print("Active releases: %s" % sorted(active_releases))
+    print("Toaster configured releases: %s" % sorted(fixtures_releases))
+else:
+    print("Success, configuration matches")
+
diff --git a/bitbake/lib/toaster/orm/fixtures/gen_fixtures.py b/bitbake/lib/toaster/orm/fixtures/gen_fixtures.py
new file mode 100755
index 0000000000..6201f679b9
--- /dev/null
+++ b/bitbake/lib/toaster/orm/fixtures/gen_fixtures.py
@@ -0,0 +1,451 @@
+#!/usr/bin/env python3
+# ex:ts=4:sw=4:sts=4:et
+# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
+#
+# Generate Toaster Fixtures for 'poky.xml' and 'oe-core.xml'
+#
+# Copyright (C) 2022      Wind River Systems
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Edit the 'current_releases' table for each new release cycle
+#
+# Usage: ./get_fixtures --all
+#
+
+import os
+import sys
+import argparse
+
+verbose = False
+
+####################################
+# Releases
+#
+# https://wiki.yoctoproject.org/wiki/Releases
+#
+# NOTE: for the current releases table, it helps to keep continuing releases
+# in the same table positions since this minimizes the patch diff for review.
+# The order of the table does not matter since Toaster presents them sorted.
+#
+# Traditionally, the two most current releases are included in addition to the
+# 'master' branch and the local installation's 'HEAD'.
+# It is also policy to include all active LTS releases.
+#
+
+# [Codename, Yocto Project Version, Release Date, Current Version, Support Level, Poky Version, BitBake branch]
+current_releases = [
+    # Release slot #1
+    ['Scarthgap','5.0','April 2024','5.0.0 (April 2024)','Long Term Support (until April 2028)','','2.8'],
+    # Release slot #2 'local'
+    ['HEAD','HEAD','','Local Yocto Project','HEAD','','HEAD'],
+    # Release slot #3 'master'
+    ['Master','master','','Yocto Project master','master','','master'],
+    # Release slot #4
+    ['Whinlatter','5.3','October 2025','5.3.0 (October 2024)','Support for 7 months (until May 2026)','','2.14'],
+    ['Walnascar','5.2','April 2025','5.2.0 (April 2025)','Support for 7 months (until October 2025)','','2.12'],
+    #['Styhead','5.1','November 2024','5.1.0 (November 2024)','Support for 7 months (until May 2025)','','2.10'],
+    #['Nanbield','4.3','November 2023','4.3.0 (November 2023)','Support for 7 months (until May 2024)','','2.6'],
+    #['Mickledore','4.2','April 2023','4.2.0 (April 2023)','Support for 7 months (until October 2023)','','2.4'],
+    #['Langdale','4.1','October 2022','4.1.2 (January 2023)','Support for 7 months (until May 2023)','','2.2'],
+    ['Kirkstone','4.0','April 2022','4.0.8 (March 2023)','Stable - Long Term Support (until Apr. 2024)','','2.0'],
+    #['Honister','3.4','October 2021','3.4.2 (February 2022)','Support for 7 months (until May 2022)','26.0','1.52'],
+    #['Hardknott','3.3','April 2021','3.3.5 (March 2022)','Stable - Support for 13 months (until Apr. 2022)','25.0','1.50'],
+    #['Gatesgarth','3.2','Oct 2020','3.2.4 (May 2021)','EOL','24.0','1.48'],
+    #['Dunfell','3.1','April 2020','3.1.23 (February 2023)','Stable - Long Term Support (until Apr. 2024)','23.0','1.46'],
+]
+
+default_poky_layers = [
+    'openembedded-core',
+    'meta-poky',
+    'meta-yocto-bsp',
+]
+
+default_oe_core_layers = [
+    'openembedded-core',
+]
+
+####################################
+# Templates
+
+prolog_template = '''\
+<?xml version="1.0" encoding="utf-8"?>
+<django-objects version="1.0">
+  <!-- Set the project default value for DISTRO -->
+  <object model="orm.toastersetting" pk="1">
+    <field type="CharField" name="name">DEFCONF_DISTRO</field>
+    <field type="CharField" name="value">{{distro}}</field>
+  </object>
+'''
+
+#<!-- Bitbake versions which correspond to the metadata release -->')
+bitbakeversion_poky_template = '''\
+  <object model="orm.bitbakeversion" pk="{{bitbake_id}}">
+    <field type="CharField" name="name">{{name}}</field>
+    <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="branch">{{branch}}</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+'''
+bitbakeversion_oecore_template = '''\
+  <object model="orm.bitbakeversion" pk="{{bitbake_id}}">
+    <field type="CharField" name="name">{{name}}</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">{{bitbakeversion}}</field>
+  </object>
+'''
+
+# <!-- Releases available -->
+releases_available_template = '''\
+  <object model="orm.release" pk="{{ra_count}}">
+    <field type="CharField" name="name">{{name}}</field>
+    <field type="CharField" name="description">{{description}}</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">{{ra_count}}</field>
+    <field type="CharField" name="branch_name">{{release}}</field>
+    <field type="TextField" name="helptext">Toaster will run your builds {{help_source}}.</field>
+  </object>
+'''
+
+# <!-- Default project layers for each release -->
+default_layers_template = '''\
+  <object model="orm.releasedefaultlayer" pk="{{rdl_count}}">
+    <field rel="ManyToOneRel" to="orm.release" name="release">{{release_id}}</field>
+    <field type="CharField" name="layer_name">{{layer}}</field>
+  </object>
+'''
+
+default_layers_preface = '''\
+  <!-- Default layers provided by poky
+       openembedded-core
+       meta-poky
+       meta-yocto-bsp
+  -->
+'''
+
+layer_poky_template = '''\
+  <object model="orm.layer" pk="{{layer_id}}">
+    <field type="CharField" name="name">{{layer}}</field>
+    <field type="CharField" name="layer_index_url"></field>
+    <field type="CharField" name="vcs_url">{{vcs_url}}</field>
+    <field type="CharField" name="vcs_web_url">{{vcs_web_url}}</field>
+    <field type="CharField" name="vcs_web_tree_base_url">{{vcs_web_tree_base_url}}</field>
+    <field type="CharField" name="vcs_web_file_base_url">{{vcs_web_file_base_url}}</field>
+  </object>
+'''
+
+layer_oe_core_template = '''\
+  <object model="orm.layer" pk="{{layer_id}}">
+    <field type="CharField" name="name">{{layer}}</field>
+    <field type="CharField" name="vcs_url">{{vcs_url}}</field>
+    <field type="CharField" name="vcs_web_url">{{vcs_web_url}}</field>
+    <field type="CharField" name="vcs_web_tree_base_url">{{vcs_web_tree_base_url}}</field>
+    <field type="CharField" name="vcs_web_file_base_url">{{vcs_web_file_base_url}}</field>
+  </object>
+'''
+
+layer_version_template = '''\
+  <object model="orm.layer_version" pk="{{lv_count}}">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">{{layer_id}}</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">{{release_id}}</field>
+    <field type="CharField" name="branch">{{branch}}</field>
+    <field type="CharField" name="dirpath">{{dirpath}}</field>
+  </object>
+'''
+
+layer_version_HEAD_template = '''\
+  <object model="orm.layer_version" pk="{{lv_count}}">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">{{layer_id}}</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">{{release_id}}</field>
+    <field type="CharField" name="branch">{{branch}}</field>
+    <field type="CharField" name="commit">{{commit}}</field>
+    <field type="CharField" name="dirpath">{{dirpath}}</field>
+  </object>
+'''
+
+layer_version_oe_core_template = '''\
+  <object model="orm.layer_version" pk="1">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
+    <field type="CharField" name="local_path">OE-CORE-LAYER-DIR</field>
+    <field type="CharField" name="branch">HEAD</field>
+    <field type="CharField" name="dirpath">meta</field>
+    <field type="IntegerField" name="layer_source">0</field>
+  </object>
+'''
+
+epilog_template = '''\
+</django-objects>
+'''
+
+#################################
+# Helper Routines
+#
+
+def print_str(str,fd):
+    # Avoid extra newline at end
+    if str and (str[-1] == '\n'):
+        str = str[0:-1]
+    print(str,file=fd)
+
+def print_template(template,params,fd):
+    for line in template.split('\n'):
+        p = line.find('{{')
+        while p > 0:
+            q = line.find('}}')
+            key = line[p+2:q]
+            if key in params:
+                line = line[0:p] + params[key] + line[q+2:]
+            else:
+                line = line[0:p] + '?' + key + '?' + line[q+2:]
+            p = line.find('{{')
+        if line:
+            print(line,file=fd)
+
+#################################
+# Generate poky.xml
+#
+
+def generate_poky():
+    fd = open('poky.xml','w')
+
+    params = {}
+    params['distro'] = 'poky'
+    print_template(prolog_template,params,fd)
+    print_str('',fd)
+
+    print_str('  <!-- Bitbake versions which correspond to the metadata release -->',fd)
+    for i,release in enumerate(current_releases):
+        params = {}
+        params['release'] = release[0]
+        params['Release'] = release[0]
+        params['release_version'] = release[1]
+        if not (params['release'] in ('HEAD')):    # 'master',
+            params['release'] = params['release'][0].lower() + params['release'][1:]
+        params['name'] = params['release']
+        params['bitbake_id'] = str(i+1)
+        params['branch'] = params['release']
+        print_template(bitbakeversion_poky_template,params,fd)
+    print_str('',fd)
+
+    print_str('',fd)
+    print_str('  <!-- Releases available -->',fd)
+    for i,release in enumerate(current_releases):
+        params = {}
+        params['release'] = release[0]
+        params['Release'] = release[0]
+        params['release_version'] = release[1]
+        if not (params['release'] in ('HEAD')): #'master',
+            params['release'] = params['release'][0].lower() + params['release'][1:]
+        params['h_release'] = '?h={{release}}'
+        params['name'] = params['release']
+        params['ra_count'] = str(i+1)
+        params['branch'] = params['release']
+
+        if 'HEAD' == params['release']:
+            params['help_source'] = 'with the version of the Yocto Project you have cloned or downloaded to your computer'
+            params['description'] = 'Local Yocto Project'
+            params['name'] = 'local'
+        else:
+            params['help_source'] = 'using the tip of the &lt;a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/{{h_release}}"&gt;Yocto Project {{Release}} branch&lt;/a&gt;'
+            params['description'] = 'Yocto Project {{release_version}} "{{Release}}"'
+        if 'master' == params['release']:
+            params['h_release'] = ''
+            params['description'] = 'Yocto Project master'
+
+        print_template(releases_available_template,params,fd)
+    print_str('',fd)
+
+    print_str('  <!-- Default project layers for each release -->',fd)
+    rdl_count = 1
+    for i,release in enumerate(current_releases):
+        for j,layer in enumerate(default_poky_layers):
+            params = {}
+            params['layer'] = layer
+            params['release'] = release[0]
+            params['Release'] = release[0]
+            params['release_version'] = release[1]
+            if not (params['release'] in ('master','HEAD')):
+                params['release'] = params['release'][0].lower() + params['release'][1:]
+            params['release_id'] = str(i+1)
+            params['rdl_count'] = str(rdl_count)
+            params['branch'] = params['release']
+            print_template(default_layers_template,params,fd)
+            rdl_count += 1
+    print_str('',fd)
+
+    print_str(default_layers_preface,fd)
+    lv_count = 1
+    for i,layer in enumerate(default_poky_layers):
+        params = {}
+        params['layer'] = layer
+        params['layer_id'] = str(i+1)
+        params['vcs_url'] = 'git://git.yoctoproject.org/poky'
+        params['vcs_web_url'] = 'https://git.yoctoproject.org/cgit/cgit.cgi/poky'
+        params['vcs_web_tree_base_url'] = 'https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%'
+        params['vcs_web_file_base_url'] = 'https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%'
+
+        if i:
+            print_str('',fd)
+        print_template(layer_poky_template,params,fd)
+        for j,release in enumerate(current_releases):
+            params['release'] = release[0]
+            params['Release'] = release[0]
+            params['release_version'] = release[1]
+            if not (params['release'] in ('master','HEAD')):
+                params['release'] = params['release'][0].lower() + params['release'][1:]
+            params['release_id'] = str(j+1)
+            params['lv_count'] = str(lv_count)
+            params['branch'] = params['release']
+            params['commit'] = params['release']
+
+            params['dirpath'] = params['layer']
+            if params['layer'] in ('openembedded-core'):   #'openembedded-core',
+                params['dirpath'] = 'meta'
+
+            if 'HEAD' == params['release']:
+                print_template(layer_version_HEAD_template,params,fd)
+            else:
+                print_template(layer_version_template,params,fd)
+            lv_count += 1
+
+    print_str(epilog_template,fd)
+    fd.close()
+
+#################################
+# Generate oe-core.xml
+#
+
+def generate_oe_core():
+    fd = open('oe-core.xml','w')
+
+    params = {}
+    params['distro'] = 'nodistro'
+    print_template(prolog_template,params,fd)
+    print_str('',fd)
+
+    print_str('  <!-- Bitbake versions which correspond to the metadata release -->',fd)
+    for i,release in enumerate(current_releases):
+        params = {}
+        params['release'] = release[0]
+        params['Release'] = release[0]
+        params['bitbakeversion'] = release[6]
+        params['release_version'] = release[1]
+        if not (params['release'] in ('HEAD')):    # 'master',
+            params['release'] = params['release'][0].lower() + params['release'][1:]
+        params['name'] = params['release']
+        params['bitbake_id'] = str(i+1)
+        params['branch'] = params['release']
+        print_template(bitbakeversion_oecore_template,params,fd)
+    print_str('',fd)
+
+    print_str('  <!-- Releases available -->',fd)
+    for i,release in enumerate(current_releases):
+        params = {}
+        params['release'] = release[0]
+        params['Release'] = release[0]
+        params['release_version'] = release[1]
+        if not (params['release'] in ('HEAD')): #'master',
+            params['release'] = params['release'][0].lower() + params['release'][1:]
+        params['h_release'] = '?h={{release}}'
+        params['name'] = params['release']
+        params['ra_count'] = str(i+1)
+        params['branch'] = params['release']
+
+        if 'HEAD' == params['release']:
+            params['help_source'] = 'with the version of OpenEmbedded that you have cloned or downloaded to your computer'
+            params['description'] = 'Local Openembedded'
+            params['name'] = 'local'
+        else:
+            params['help_source'] = 'using the tip of the &lt;a href=\\"https://cgit.openembedded.org/openembedded-core/log/{{h_release}}\\"&gt;OpenEmbedded {{Release}}&lt;/a&gt; branch'
+            params['description'] = 'Openembedded {{Release}}'
+        if 'master' == params['release']:
+            params['h_release'] = ''
+            params['description'] = 'OpenEmbedded core master'
+            params['Release'] = params['release']
+
+        print_template(releases_available_template,params,fd)
+    print_str('',fd)
+
+    print_str('  <!-- Default layers for each release -->',fd)
+    rdl_count = 1
+    for i,release in enumerate(current_releases):
+        for j,layer in enumerate(default_oe_core_layers):
+            params = {}
+            params['layer'] = layer
+            params['release'] = release[0]
+            params['Release'] = release[0]
+            params['release_version'] = release[1]
+            if not (params['release'] in ('master','HEAD')):
+                params['release'] = params['release'][0].lower() + params['release'][1:]
+            params['release_id'] = str(i+1)
+            params['rdl_count'] = str(rdl_count)
+            params['branch'] = params['release']
+            print_template(default_layers_template,params,fd)
+            rdl_count += 1
+    print_str('',fd)
+
+    print_str('',fd)
+    print_str('  <!-- Layer for the Local release -->',fd)
+    lv_count = 1
+    for i,layer in enumerate(default_oe_core_layers):
+        params = {}
+        params['layer'] = layer
+        params['layer_id'] = str(i+1)
+        params['vcs_url'] = 'git://git.openembedded.org/openembedded-core'
+        params['vcs_web_url'] = 'https://cgit.openembedded.org/openembedded-core'
+        params['vcs_web_tree_base_url'] = 'https://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch%'
+        params['vcs_web_file_base_url'] = 'https://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch%'
+        if i:
+            print_str('',fd)
+        print_template(layer_oe_core_template,params,fd)
+
+        print_template(layer_version_oe_core_template,params,fd)
+    print_str('',fd)
+
+    print_str(epilog_template,fd)
+    fd.close()
+
+#################################
+# Help
+#
+
+def list_releases():
+    print("Release    ReleaseVer  BitbakeVer Support Level")
+    print("========== =========== ========== ==============================================")
+    for release in current_releases:
+        print("%10s %10s %11s %s" % (release[0],release[1],release[6],release[4]))
+
+#################################
+# main
+#
+
+def main(argv):
+    global verbose
+
+    parser = argparse.ArgumentParser(description='gen_fixtures.py: table generate the fixture files')
+    parser.add_argument('--poky', '-p', action='store_const', const='poky', dest='command', help='Generate the poky.xml file')
+    parser.add_argument('--oe-core', '-o', action='store_const', const='oe_core', dest='command', help='Generate the oe-core.xml file')
+    parser.add_argument('--all', '-a', action='store_const', const='all', dest='command', help='Generate all fixture files')
+    parser.add_argument('--list', '-l', action='store_const', const='list', dest='command', help='List the release table')
+    parser.add_argument('--verbose', '-v', action='store_true', dest='verbose', help='Enable verbose debugging output')
+    args = parser.parse_args()
+
+    verbose = args.verbose
+    if 'poky' == args.command:
+        generate_poky()
+    elif 'oe_core' == args.command:
+        generate_oe_core()
+    elif 'all' == args.command:
+        generate_poky()
+        generate_oe_core()
+    elif 'all' == args.command:
+        list_releases()
+    elif 'list' == args.command:
+        list_releases()
+
+    else:
+        print("No command for 'gen_fixtures.py' selected")
+
+if __name__ == '__main__':
+    main(sys.argv[1:])
diff --git a/bitbake/lib/toaster/orm/fixtures/oe-core.xml b/bitbake/lib/toaster/orm/fixtures/oe-core.xml
index 026d94869a..264231d139 100644
--- a/bitbake/lib/toaster/orm/fixtures/oe-core.xml
+++ b/bitbake/lib/toaster/orm/fixtures/oe-core.xml
@@ -8,9 +8,9 @@
 
   <!-- Bitbake versions which correspond to the metadata release -->
   <object model="orm.bitbakeversion" pk="1">
-    <field type="CharField" name="name">dunfell</field>
+    <field type="CharField" name="name">scarthgap</field>
     <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
-    <field type="CharField" name="branch">1.46</field>
+    <field type="CharField" name="branch">2.8</field>
   </object>
   <object model="orm.bitbakeversion" pk="2">
     <field type="CharField" name="name">HEAD</field>
@@ -23,18 +23,33 @@
     <field type="CharField" name="branch">master</field>
   </object>
   <object model="orm.bitbakeversion" pk="4">
-    <field type="CharField" name="name">gatesgarth</field>
+    <field type="CharField" name="name">whinlatter</field>
     <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
-    <field type="CharField" name="branch">1.48</field>
+    <field type="CharField" name="branch">2.14</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="5">
+    <field type="CharField" name="name">walnascar</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">2.12</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="6">
+    <field type="CharField" name="name">styhead</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">2.10</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="7">
+    <field type="CharField" name="name">kirkstone</field>
+    <field type="CharField" name="giturl">git://git.openembedded.org/bitbake</field>
+    <field type="CharField" name="branch">2.0</field>
   </object>
 
   <!-- Releases available -->
   <object model="orm.release" pk="1">
-    <field type="CharField" name="name">dunfell</field>
-    <field type="CharField" name="description">Openembedded Dunfell</field>
+    <field type="CharField" name="name">scarthgap</field>
+    <field type="CharField" name="description">Openembedded Scarthgap</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
-    <field type="CharField" name="branch_name">dunfell</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=dunfell\"&gt;OpenEmbedded Dunfell&lt;/a&gt; branch.</field>
+    <field type="CharField" name="branch_name">scarthgap</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=scarthgap\"&gt;OpenEmbedded Scarthgap&lt;/a&gt; branch.</field>
   </object>
   <object model="orm.release" pk="2">
     <field type="CharField" name="name">local</field>
@@ -48,14 +63,35 @@
     <field type="CharField" name="description">OpenEmbedded core master</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">3</field>
     <field type="CharField" name="branch_name">master</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/\"&gt;OpenEmbedded master&lt;/a&gt; branch.</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"https://cgit.openembedded.org/openembedded-core/log/\"&gt;OpenEmbedded master&lt;/a&gt; branch.</field>
   </object>
   <object model="orm.release" pk="4">
-    <field type="CharField" name="name">gatesgarth</field>
-    <field type="CharField" name="description">Openembedded Gatesgarth</field>
+    <field type="CharField" name="name">whinlatter</field>
+    <field type="CharField" name="description">Openembedded Whinlatter</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">4</field>
-    <field type="CharField" name="branch_name">gatesgarth</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"http://cgit.openembedded.org/openembedded-core/log/?h=gatesgarth\"&gt;OpenEmbedded Gatesgarth&lt;/a&gt; branch.</field>
+    <field type="CharField" name="branch_name">whinlatter</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=whinlatter\"&gt;OpenEmbedded Whinlatter&lt;/a&gt; branch.</field>
+  </object>
+  <object model="orm.release" pk="5">
+    <field type="CharField" name="name">walnascar</field>
+    <field type="CharField" name="description">Openembedded Walnascar</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">5</field>
+    <field type="CharField" name="branch_name">walnascar</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=walnascar\"&gt;OpenEmbedded Walnascar&lt;/a&gt; branch.</field>
+  </object>
+  <object model="orm.release" pk="6">
+    <field type="CharField" name="name">styhead</field>
+    <field type="CharField" name="description">Openembedded Styhead</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">6</field>
+    <field type="CharField" name="branch_name">styhead</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=styhead\"&gt;OpenEmbedded Styhead&lt;/a&gt; branch.</field>
+  </object>
+  <object model="orm.release" pk="7">
+    <field type="CharField" name="name">kirkstone</field>
+    <field type="CharField" name="description">Openembedded Kirkstone</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">7</field>
+    <field type="CharField" name="branch_name">kirkstone</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href=\"https://cgit.openembedded.org/openembedded-core/log/?h=kirkstone\"&gt;OpenEmbedded Kirkstone&lt;/a&gt; branch.</field>
   </object>
 
   <!-- Default layers for each release -->
@@ -75,15 +111,27 @@
     <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
     <field type="CharField" name="layer_name">openembedded-core</field>
   </object>
+  <object model="orm.releasedefaultlayer" pk="5">
+    <field rel="ManyToOneRel" to="orm.release" name="release">5</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="6">
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="7">
+    <field rel="ManyToOneRel" to="orm.release" name="release">7</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
 
 
   <!-- Layer for the Local release -->
   <object model="orm.layer" pk="1">
     <field type="CharField" name="name">openembedded-core</field>
     <field type="CharField" name="vcs_url">git://git.openembedded.org/openembedded-core</field>
-    <field type="CharField" name="vcs_web_url">http://cgit.openembedded.org/openembedded-core</field>
-    <field type="CharField" name="vcs_web_tree_base_url">http://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch%</field>
-    <field type="CharField" name="vcs_web_file_base_url">http://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch%</field>
+    <field type="CharField" name="vcs_web_url">https://cgit.openembedded.org/openembedded-core</field>
+    <field type="CharField" name="vcs_web_tree_base_url">https://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch%</field>
+    <field type="CharField" name="vcs_web_file_base_url">https://cgit.openembedded.org/openembedded-core/tree/%path%?h=%branch%</field>
   </object>
   <object model="orm.layer_version" pk="1">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
diff --git a/bitbake/lib/toaster/orm/fixtures/poky.xml b/bitbake/lib/toaster/orm/fixtures/poky.xml
index a468a54c49..6cf4f0687a 100644
--- a/bitbake/lib/toaster/orm/fixtures/poky.xml
+++ b/bitbake/lib/toaster/orm/fixtures/poky.xml
@@ -8,9 +8,9 @@
 
   <!-- Bitbake versions which correspond to the metadata release -->
   <object model="orm.bitbakeversion" pk="1">
-    <field type="CharField" name="name">dunfell</field>
+    <field type="CharField" name="name">scarthgap</field>
     <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
-    <field type="CharField" name="branch">dunfell</field>
+    <field type="CharField" name="branch">scarthgap</field>
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
   <object model="orm.bitbakeversion" pk="2">
@@ -26,20 +26,38 @@
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
   <object model="orm.bitbakeversion" pk="4">
-    <field type="CharField" name="name">gatesgarth</field>
+    <field type="CharField" name="name">whinlatter</field>
     <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
-    <field type="CharField" name="branch">gatesgarth</field>
+    <field type="CharField" name="branch">whinlatter</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="5">
+    <field type="CharField" name="name">walnascar</field>
+    <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="branch">walnascar</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="6">
+    <field type="CharField" name="name">styhead</field>
+    <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="branch">styhead</field>
+    <field type="CharField" name="dirpath">bitbake</field>
+  </object>
+  <object model="orm.bitbakeversion" pk="7">
+    <field type="CharField" name="name">kirkstone</field>
+    <field type="CharField" name="giturl">git://git.yoctoproject.org/poky</field>
+    <field type="CharField" name="branch">kirkstone</field>
     <field type="CharField" name="dirpath">bitbake</field>
   </object>
 
 
   <!-- Releases available -->
   <object model="orm.release" pk="1">
-    <field type="CharField" name="name">dunfell</field>
-    <field type="CharField" name="description">Yocto Project 3.1 "Dunfell"</field>
+    <field type="CharField" name="name">scarthgap</field>
+    <field type="CharField" name="description">Yocto Project 5.0 "Scarthgap"</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">1</field>
-    <field type="CharField" name="branch_name">dunfell</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=dunfell"&gt;Yocto Project Dunfell branch&lt;/a&gt;.</field>
+    <field type="CharField" name="branch_name">scarthgap</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=scarthgap"&gt;Yocto Project Scarthgap branch&lt;/a&gt;.</field>
   </object>
   <object model="orm.release" pk="2">
     <field type="CharField" name="name">local</field>
@@ -53,14 +71,35 @@
     <field type="CharField" name="description">Yocto Project master</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">3</field>
     <field type="CharField" name="branch_name">master</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/"&gt;Yocto Project Master branch&lt;/a&gt;.</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/"&gt;Yocto Project Master branch&lt;/a&gt;.</field>
   </object>
   <object model="orm.release" pk="4">
-    <field type="CharField" name="name">gatesgarth</field>
-    <field type="CharField" name="description">Yocto Project 3.2 "Gatesgarth"</field>
+    <field type="CharField" name="name">whinlatter</field>
+    <field type="CharField" name="description">Yocto Project 5.3 "Whinlatter"</field>
     <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">4</field>
-    <field type="CharField" name="branch_name">gatesgarth</field>
-    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="http://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=gatesgarth"&gt;Yocto Project Gatesgarth branch&lt;/a&gt;.</field>
+    <field type="CharField" name="branch_name">whinlatter</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=whinlatter"&gt;Yocto Project Whinlatter branch&lt;/a&gt;.</field>
+  </object>
+  <object model="orm.release" pk="5">
+    <field type="CharField" name="name">walnascar</field>
+    <field type="CharField" name="description">Yocto Project 5.2 "Walnascar"</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">5</field>
+    <field type="CharField" name="branch_name">walnascar</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=walnascar"&gt;Yocto Project Walnascar branch&lt;/a&gt;.</field>
+  </object>
+  <object model="orm.release" pk="6">
+    <field type="CharField" name="name">styhead</field>
+    <field type="CharField" name="description">Yocto Project 5.1 "Styhead"</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">6</field>
+    <field type="CharField" name="branch_name">styhead</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=styhead"&gt;Yocto Project Styhead branch&lt;/a&gt;.</field>
+  </object>
+  <object model="orm.release" pk="7">
+    <field type="CharField" name="name">kirkstone</field>
+    <field type="CharField" name="description">Yocto Project 4.0 "Kirkstone"</field>
+    <field rel="ManyToOneRel" to="orm.bitbakeversion" name="bitbake_version">7</field>
+    <field type="CharField" name="branch_name">kirkstone</field>
+    <field type="TextField" name="helptext">Toaster will run your builds using the tip of the &lt;a href="https://git.yoctoproject.org/cgit/cgit.cgi/poky/log/?h=kirkstone"&gt;Yocto Project Kirkstone branch&lt;/a&gt;.</field>
   </object>
 
   <!-- Default project layers for each release -->
@@ -112,6 +151,42 @@
     <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
     <field type="CharField" name="layer_name">meta-yocto-bsp</field>
   </object>
+  <object model="orm.releasedefaultlayer" pk="13">
+    <field rel="ManyToOneRel" to="orm.release" name="release">5</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="14">
+    <field rel="ManyToOneRel" to="orm.release" name="release">5</field>
+    <field type="CharField" name="layer_name">meta-poky</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="15">
+    <field rel="ManyToOneRel" to="orm.release" name="release">5</field>
+    <field type="CharField" name="layer_name">meta-yocto-bsp</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="16">
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="17">
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
+    <field type="CharField" name="layer_name">meta-poky</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="18">
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
+    <field type="CharField" name="layer_name">meta-yocto-bsp</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="19">
+    <field rel="ManyToOneRel" to="orm.release" name="release">7</field>
+    <field type="CharField" name="layer_name">openembedded-core</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="20">
+    <field rel="ManyToOneRel" to="orm.release" name="release">7</field>
+    <field type="CharField" name="layer_name">meta-poky</field>
+  </object>
+  <object model="orm.releasedefaultlayer" pk="21">
+    <field rel="ManyToOneRel" to="orm.release" name="release">7</field>
+    <field type="CharField" name="layer_name">meta-yocto-bsp</field>
+  </object>
 
   <!-- Default layers provided by poky
        openembedded-core
@@ -122,15 +197,15 @@
     <field type="CharField" name="name">openembedded-core</field>
     <field type="CharField" name="layer_index_url"></field>
     <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
-    <field type="CharField" name="vcs_web_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky</field>
-    <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
-    <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
+    <field type="CharField" name="vcs_web_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky</field>
+    <field type="CharField" name="vcs_web_tree_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
+    <field type="CharField" name="vcs_web_file_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
   </object>
   <object model="orm.layer_version" pk="1">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
-    <field type="CharField" name="branch">dunfell</field>
+    <field type="CharField" name="branch">scarthgap</field>
     <field type="CharField" name="dirpath">meta</field>
   </object>
   <object model="orm.layer_version" pk="2">
@@ -152,7 +227,28 @@
     <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
-    <field type="CharField" name="branch">gatesgarth</field>
+    <field type="CharField" name="branch">whinlatter</field>
+    <field type="CharField" name="dirpath">meta</field>
+  </object>
+  <object model="orm.layer_version" pk="5">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">5</field>
+    <field type="CharField" name="branch">walnascar</field>
+    <field type="CharField" name="dirpath">meta</field>
+  </object>
+  <object model="orm.layer_version" pk="6">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
+    <field type="CharField" name="branch">styhead</field>
+    <field type="CharField" name="dirpath">meta</field>
+  </object>
+  <object model="orm.layer_version" pk="7">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">1</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">7</field>
+    <field type="CharField" name="branch">kirkstone</field>
     <field type="CharField" name="dirpath">meta</field>
   </object>
 
@@ -160,18 +256,18 @@
     <field type="CharField" name="name">meta-poky</field>
     <field type="CharField" name="layer_index_url"></field>
     <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
-    <field type="CharField" name="vcs_web_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky</field>
-    <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
-    <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
+    <field type="CharField" name="vcs_web_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky</field>
+    <field type="CharField" name="vcs_web_tree_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
+    <field type="CharField" name="vcs_web_file_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
   </object>
-  <object model="orm.layer_version" pk="5">
+  <object model="orm.layer_version" pk="8">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
-    <field type="CharField" name="branch">dunfell</field>
+    <field type="CharField" name="branch">scarthgap</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
-  <object model="orm.layer_version" pk="6">
+  <object model="orm.layer_version" pk="9">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
@@ -179,18 +275,39 @@
     <field type="CharField" name="commit">HEAD</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
-  <object model="orm.layer_version" pk="7">
+  <object model="orm.layer_version" pk="10">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
     <field type="CharField" name="branch">master</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
-  <object model="orm.layer_version" pk="8">
+  <object model="orm.layer_version" pk="11">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
-    <field type="CharField" name="branch">gatesgarth</field>
+    <field type="CharField" name="branch">whinlatter</field>
+    <field type="CharField" name="dirpath">meta-poky</field>
+  </object>
+  <object model="orm.layer_version" pk="12">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">5</field>
+    <field type="CharField" name="branch">walnascar</field>
+    <field type="CharField" name="dirpath">meta-poky</field>
+  </object>
+  <object model="orm.layer_version" pk="13">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
+    <field type="CharField" name="branch">styhead</field>
+    <field type="CharField" name="dirpath">meta-poky</field>
+  </object>
+  <object model="orm.layer_version" pk="14">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">2</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">7</field>
+    <field type="CharField" name="branch">kirkstone</field>
     <field type="CharField" name="dirpath">meta-poky</field>
   </object>
 
@@ -198,18 +315,18 @@
     <field type="CharField" name="name">meta-yocto-bsp</field>
     <field type="CharField" name="layer_index_url"></field>
     <field type="CharField" name="vcs_url">git://git.yoctoproject.org/poky</field>
-    <field type="CharField" name="vcs_web_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky</field>
-    <field type="CharField" name="vcs_web_tree_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
-    <field type="CharField" name="vcs_web_file_base_url">http://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
+    <field type="CharField" name="vcs_web_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky</field>
+    <field type="CharField" name="vcs_web_tree_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
+    <field type="CharField" name="vcs_web_file_base_url">https://git.yoctoproject.org/cgit/cgit.cgi/poky/tree/%path%?h=%branch%</field>
   </object>
-  <object model="orm.layer_version" pk="9">
+  <object model="orm.layer_version" pk="15">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">1</field>
-    <field type="CharField" name="branch">dunfell</field>
+    <field type="CharField" name="branch">scarthgap</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
-  <object model="orm.layer_version" pk="10">
+  <object model="orm.layer_version" pk="16">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">2</field>
@@ -217,18 +334,39 @@
     <field type="CharField" name="commit">HEAD</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
-  <object model="orm.layer_version" pk="11">
+  <object model="orm.layer_version" pk="17">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">3</field>
     <field type="CharField" name="branch">master</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
-  <object model="orm.layer_version" pk="12">
+  <object model="orm.layer_version" pk="18">
     <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
     <field type="IntegerField" name="layer_source">0</field>
     <field rel="ManyToOneRel" to="orm.release" name="release">4</field>
-    <field type="CharField" name="branch">gatesgarth</field>
+    <field type="CharField" name="branch">whinlatter</field>
+    <field type="CharField" name="dirpath">meta-yocto-bsp</field>
+  </object>
+  <object model="orm.layer_version" pk="19">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">5</field>
+    <field type="CharField" name="branch">walnascar</field>
+    <field type="CharField" name="dirpath">meta-yocto-bsp</field>
+  </object>
+  <object model="orm.layer_version" pk="20">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">6</field>
+    <field type="CharField" name="branch">styhead</field>
+    <field type="CharField" name="dirpath">meta-yocto-bsp</field>
+  </object>
+  <object model="orm.layer_version" pk="21">
+    <field rel="ManyToOneRel" to="orm.layer" name="layer">3</field>
+    <field type="IntegerField" name="layer_source">0</field>
+    <field rel="ManyToOneRel" to="orm.release" name="release">7</field>
+    <field type="CharField" name="branch">kirkstone</field>
     <field type="CharField" name="dirpath">meta-yocto-bsp</field>
   </object>
 </django-objects>
diff --git a/bitbake/lib/toaster/orm/fixtures/settings.xml b/bitbake/lib/toaster/orm/fixtures/settings.xml
index 78c0fdca7f..02c26a6974 100644
--- a/bitbake/lib/toaster/orm/fixtures/settings.xml
+++ b/bitbake/lib/toaster/orm/fixtures/settings.xml
@@ -12,14 +12,14 @@
   </object>
   <object model="orm.toastersetting" pk="4">
     <field type="CharField" name="name">DEFCONF_MACHINE</field>
-    <field type="CharField" name="value">qemux86</field>
+    <field type="CharField" name="value">qemux86-64</field>
   </object>
   <object model="orm.toastersetting" pk="5">
     <field type="CharField" name="name">DEFCONF_SSTATE_DIR</field>
     <field type="CharField" name="value">${TOPDIR}/../sstate-cache</field>
   </object>
   <object model="orm.toastersetting" pk="6">
-    <field type="CharField" name="name">DEFCONF_IMAGE_INSTALL_append</field>
+    <field type="CharField" name="name">DEFCONF_IMAGE_INSTALL:append</field>
     <field type="CharField" name="value"></field>
   </object>
   <object model="orm.toastersetting" pk="7">
diff --git a/bitbake/lib/toaster/orm/management/commands/lsupdates.py b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
index 2fbd7be3d2..6d64830ebd 100644
--- a/bitbake/lib/toaster/orm/management/commands/lsupdates.py
+++ b/bitbake/lib/toaster/orm/management/commands/lsupdates.py
@@ -21,7 +21,7 @@ import threading
 import time
 logger = logging.getLogger("toaster")
 
-DEFAULT_LAYERINDEX_SERVER = "http://layers.openembedded.org/layerindex/api/"
+DEFAULT_LAYERINDEX_SERVER = "https://layers.openembedded.org/layerindex/api/"
 
 # Add path to bitbake modules for layerindexlib
 # lib/toaster/orm/management/commands/lsupdates.py (abspath)
@@ -40,7 +40,7 @@ class Spinner(threading.Thread):
     """ A simple progress spinner to indicate download/parsing is happening"""
     def __init__(self, *args, **kwargs):
         super(Spinner, self).__init__(*args, **kwargs)
-        self.setDaemon(True)
+        self.daemon = True
         self.signal = True
 
     def run(self):
@@ -87,13 +87,13 @@ class Command(BaseCommand):
 
         # update branches; only those that we already have names listed in the
         # Releases table
-        whitelist_branch_names = [rel.branch_name
-                                  for rel in Release.objects.all()]
-        if len(whitelist_branch_names) == 0:
+        allowed_branch_names = [rel.branch_name
+                                for rel in Release.objects.all()]
+        if len(allowed_branch_names) == 0:
             raise Exception("Failed to make list of branches to fetch")
 
         logger.info("Fetching metadata for %s",
-                    " ".join(whitelist_branch_names))
+                    " ".join(allowed_branch_names))
 
         # We require a non-empty bb.data, but we can fake it with a dictionary
         layerindex = layerindexlib.LayerIndex({"DUMMY" : "VALUE"})
@@ -101,8 +101,8 @@ class Command(BaseCommand):
         http_progress = Spinner()
         http_progress.start()
 
-        if whitelist_branch_names:
-            url_branches = ";branch=%s" % ','.join(whitelist_branch_names)
+        if allowed_branch_names:
+            url_branches = ";branch=%s" % ','.join(allowed_branch_names)
         else:
             url_branches = ""
         layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches))
diff --git a/bitbake/lib/toaster/orm/migrations/0020_models_bigautofield.py b/bitbake/lib/toaster/orm/migrations/0020_models_bigautofield.py
new file mode 100644
index 0000000000..f19b5dddbc
--- /dev/null
+++ b/bitbake/lib/toaster/orm/migrations/0020_models_bigautofield.py
@@ -0,0 +1,173 @@
+# Generated by Django 3.2.12 on 2022-03-06 03:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0019_django_2_2'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='bitbakeversion',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='build',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='distro',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='helptext',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='layer',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='layer_version',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='layerversiondependency',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='logmessage',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='machine',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='package',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='package_dependency',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='package_file',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='project',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='projectlayer',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='projecttarget',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='projectvariable',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='provides',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='recipe',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='recipe_dependency',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='release',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='releasedefaultlayer',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='target',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='target_file',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='target_image_file',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='target_installed_package',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='targetkernelfile',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='targetsdkfile',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='task_dependency',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='toastersetting',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='variable',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='variablehistory',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]
diff --git a/bitbake/lib/toaster/orm/migrations/0021_eventlogsimports.py b/bitbake/lib/toaster/orm/migrations/0021_eventlogsimports.py
new file mode 100644
index 0000000000..328eb5753c
--- /dev/null
+++ b/bitbake/lib/toaster/orm/migrations/0021_eventlogsimports.py
@@ -0,0 +1,22 @@
+# Generated by Django 4.2.5 on 2023-11-23 18:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('orm', '0020_models_bigautofield'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='EventLogsImports',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=255)),
+                ('imported', models.BooleanField(default=False)),
+                ('build_id', models.IntegerField(blank=True, null=True)),
+            ],
+        ),
+    ]
diff --git a/bitbake/lib/toaster/orm/models.py b/bitbake/lib/toaster/orm/models.py
index 7f7e922ade..e2f488ed89 100644
--- a/bitbake/lib/toaster/orm/models.py
+++ b/bitbake/lib/toaster/orm/models.py
@@ -58,7 +58,6 @@ if 'sqlite' in settings.DATABASES['default']['ENGINE']:
             return _base_insert(self, *args, **kwargs)
     QuerySet._insert = _insert
 
-    from django.utils import six
     def _create_object_from_params(self, lookup, params):
         """
         Tries to create an object using passed params.
@@ -80,7 +79,6 @@ if 'sqlite' in settings.DATABASES['default']['ENGINE']:
     # end of HACK
 
 class GitURLValidator(validators.URLValidator):
-    import re
     regex = re.compile(
         r'^(?:ssh|git|http|ftp)s?://'  # http:// or https://
         r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # domain...
@@ -108,7 +106,7 @@ class ToasterSetting(models.Model):
 
 
 class ProjectManager(models.Manager):
-    def create_project(self, name, release, existing_project=None):
+    def create_project(self, name, release, existing_project=None, imported=False):
         if existing_project and (release is not None):
             prj = existing_project
             prj.bitbake_version = release.bitbake_version
@@ -135,19 +133,19 @@ class ProjectManager(models.Manager):
 
         if release is None:
             return prj
-
-        for rdl in release.releasedefaultlayer_set.all():
-            lv = Layer_Version.objects.filter(
-                layer__name=rdl.layer_name,
-                release=release).first()
-
-            if lv:
-                ProjectLayer.objects.create(project=prj,
-                                            layercommit=lv,
-                                            optional=False)
-            else:
-                logger.warning("Default project layer %s not found" %
-                               rdl.layer_name)
+        if not imported:
+            for rdl in release.releasedefaultlayer_set.all():
+                lv = Layer_Version.objects.filter(
+                    layer__name=rdl.layer_name,
+                    release=release).first()
+
+                if lv:
+                    ProjectLayer.objects.create(project=prj,
+                                                layercommit=lv,
+                                                optional=False)
+                else:
+                    logger.warning("Default project layer %s not found" %
+                                rdl.layer_name)
 
         return prj
 
@@ -1390,9 +1388,6 @@ class Machine(models.Model):
         return "Machine " + self.name + "(" + self.description + ")"
 
 
-
-
-
 class BitbakeVersion(models.Model):
 
     name = models.CharField(max_length=32, unique = True)
@@ -1504,7 +1499,7 @@ class Layer_Version(models.Model):
     # code lifted, with adaptations, from the layerindex-web application
     # https://git.yoctoproject.org/cgit/cgit.cgi/layerindex-web/
     def _handle_url_path(self, base_url, path):
-        import re, posixpath
+        import posixpath
         if base_url:
             if self.dirpath:
                 if path:
@@ -1717,9 +1712,9 @@ class CustomImageRecipe(Recipe):
 
     def generate_recipe_file_contents(self):
         """Generate the contents for the recipe file."""
-        # If we have no excluded packages we only need to _append
+        # If we have no excluded packages we only need to :append
         if self.excludes_set.count() == 0:
-            packages_conf = "IMAGE_INSTALL_append = \" "
+            packages_conf = "IMAGE_INSTALL:append = \" "
 
             for pkg in self.appends_set.all():
                 packages_conf += pkg.name+' '
@@ -1734,7 +1729,7 @@ class CustomImageRecipe(Recipe):
         packages_conf += "\""
 
         base_recipe_path = self.get_base_recipe_file()
-        if base_recipe_path:
+        if base_recipe_path and os.path.isfile(base_recipe_path):
             base_recipe = open(base_recipe_path, 'r').read()
         else:
             # Pass back None to trigger error message to user
@@ -1854,6 +1849,8 @@ def signal_runbuilds():
             os.kill(int(pidf.read()), SIGUSR1)
     except FileNotFoundError:
         logger.info("Stopping existing runbuilds: no current process found")
+    except ProcessLookupError:
+        logger.warning("Stopping existing runbuilds: process lookup not found")
 
 class Distro(models.Model):
     search_allowed_fields = ["name", "description", "layer_version__layer__name"]
@@ -1870,6 +1867,15 @@ class Distro(models.Model):
     def __unicode__(self):
         return "Distro " + self.name + "(" + self.description + ")"
 
+class EventLogsImports(models.Model):
+    name = models.CharField(max_length=255)
+    imported = models.BooleanField(default=False)
+    build_id = models.IntegerField(blank=True, null=True)
+
+    def __str__(self):
+        return self.name
+
+
 django.db.models.signals.post_save.connect(invalidate_cache)
 django.db.models.signals.post_delete.connect(invalidate_cache)
 django.db.models.signals.m2m_changed.connect(invalidate_cache)
diff --git a/bitbake/lib/toaster/pytest.ini b/bitbake/lib/toaster/pytest.ini
new file mode 100644
index 0000000000..071c65fcd5
--- /dev/null
+++ b/bitbake/lib/toaster/pytest.ini
@@ -0,0 +1,16 @@
+# -- FILE: pytest.ini (or tox.ini)
+[pytest]
+# --create-db - force re creation of the test database
+# https://pytest-django.readthedocs.io/en/latest/database.html#create-db-force-re-creation-of-the-test-database
+
+# --html=report.html --self-contained-html
+# https://docs.pytest.org/en/latest/usage.html#creating-html-reports
+# https://pytest-html.readthedocs.io/en/latest/user_guide.html#creating-a-self-contained-report
+addopts = --create-db --html="Toaster Tests Report.html" --self-contained-html 
+
+# Define environment variables using pytest-env
+# A pytest plugin that enables you to set environment variables in the pytest.ini file.
+# https://pypi.org/project/pytest-env/
+env =
+    TOASTER_BUILDSERVER=1
+    DJANGO_SETTINGS_MODULE=toastermain.settings_test
diff --git a/bitbake/lib/toaster/tests/browser/selenium_helpers_base.py b/bitbake/lib/toaster/tests/browser/selenium_helpers_base.py
index 644d45fe58..6953541ab5 100644
--- a/bitbake/lib/toaster/tests/browser/selenium_helpers_base.py
+++ b/bitbake/lib/toaster/tests/browser/selenium_helpers_base.py
@@ -19,11 +19,15 @@ import os
 import time
 import unittest
 
+import pytest
 from selenium import webdriver
+from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.common.by import By
 from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
 from selenium.common.exceptions import NoSuchElementException, \
-        StaleElementReferenceException, TimeoutException
+        StaleElementReferenceException, TimeoutException, \
+        SessionNotCreatedException, WebDriverException
 
 def create_selenium_driver(cls,browser='chrome'):
     # set default browser string based on env (if available)
@@ -32,9 +36,32 @@ def create_selenium_driver(cls,browser='chrome'):
         browser = env_browser
 
     if browser == 'chrome':
-        return webdriver.Chrome(
-            service_args=["--verbose", "--log-path=selenium.log"]
-        )
+        options = webdriver.ChromeOptions()
+        options.add_argument('--headless')
+        options.add_argument('--disable-infobars')
+        options.add_argument('--disable-dev-shm-usage')
+        options.add_argument('--no-sandbox')
+        options.add_argument('--remote-debugging-port=9222')
+        try:
+            return webdriver.Chrome(options=options)
+        except SessionNotCreatedException as e:
+            exit_message = "Halting tests prematurely to avoid cascading errors."
+            # check if chrome / chromedriver exists
+            chrome_path = os.popen("find ~/.cache/selenium/chrome/ -name 'chrome' -type f -print -quit").read().strip()
+            if not chrome_path:
+                pytest.exit(f"Failed to install/find chrome.\n{exit_message}")
+            chromedriver_path = os.popen("find ~/.cache/selenium/chromedriver/ -name 'chromedriver' -type f -print -quit").read().strip()
+            if not chromedriver_path:
+                pytest.exit(f"Failed to install/find chromedriver.\n{exit_message}")
+            # check if depends on each are fulfilled
+            depends_chrome = os.popen(f"ldd {chrome_path} | grep 'not found'").read().strip()
+            if depends_chrome:
+                pytest.exit(f"Missing chrome dependencies.\n{depends_chrome}\n{exit_message}")
+            depends_chromedriver = os.popen(f"ldd {chromedriver_path} | grep 'not found'").read().strip()
+            if depends_chromedriver:
+                pytest.exit(f"Missing chromedriver dependencies.\n{depends_chromedriver}\n{exit_message}")
+            # print original error otherwise
+            pytest.exit(f"Failed to start chromedriver.\n{e}\n{exit_message}")
     elif browser == 'firefox':
         return webdriver.Firefox()
     elif browser == 'marionette':
@@ -63,10 +90,12 @@ class Wait(WebDriverWait):
     Subclass of WebDriverWait with predetermined timeout and poll
     frequency. Also deals with a wider variety of exceptions.
     """
-    _TIMEOUT = 10
+    _TIMEOUT = 20
     _POLL_FREQUENCY = 0.5
 
-    def __init__(self, driver):
+    def __init__(self, driver, timeout=_TIMEOUT, poll=_POLL_FREQUENCY):
+        self._TIMEOUT = timeout
+        self._POLL_FREQUENCY = poll
         super(Wait, self).__init__(driver, self._TIMEOUT, self._POLL_FREQUENCY)
 
     def until(self, method, message=''):
@@ -85,6 +114,9 @@ class Wait(WebDriverWait):
                 pass
             except StaleElementReferenceException:
                 pass
+            except WebDriverException:
+                # selenium.common.exceptions.WebDriverException: Message: unknown error: unhandled inspector error: {"code":-32000,"message":"Node with given id does not belong to the document"}
+                pass
 
             time.sleep(self._poll)
             if time.time() > end_time:
@@ -138,6 +170,8 @@ class SeleniumTestCaseBase(unittest.TestCase):
         """ Clean up webdriver driver """
 
         cls.driver.quit()
+        # Allow driver resources to be properly freed before proceeding with further tests
+        time.sleep(5)
         super(SeleniumTestCaseBase, cls).tearDownClass()
 
     def get(self, url):
@@ -151,13 +185,20 @@ class SeleniumTestCaseBase(unittest.TestCase):
         abs_url = '%s%s' % (self.live_server_url, url)
         self.driver.get(abs_url)
 
+        try:  # Ensure page is loaded before proceeding
+            self.wait_until_visible("#global-nav")
+        except NoSuchElementException:
+            self.driver.implicitly_wait(3)
+        except TimeoutException:
+            self.driver.implicitly_wait(3)
+
     def find(self, selector):
         """ Find single element by CSS selector """
-        return self.driver.find_element_by_css_selector(selector)
+        return self.driver.find_element(By.CSS_SELECTOR, selector)
 
     def find_all(self, selector):
         """ Find all elements matching CSS selector """
-        return self.driver.find_elements_by_css_selector(selector)
+        return self.driver.find_elements(By.CSS_SELECTOR, selector)
 
     def element_exists(self, selector):
         """
@@ -170,20 +211,43 @@ class SeleniumTestCaseBase(unittest.TestCase):
         """ Return the element which currently has focus on the page """
         return self.driver.switch_to.active_element
 
-    def wait_until_present(self, selector):
+    def wait_until_present(self, selector, timeout=Wait._TIMEOUT):
         """ Wait until element matching CSS selector is on the page """
         is_present = lambda driver: self.find(selector)
         msg = 'An element matching "%s" should be on the page' % selector
-        element = Wait(self.driver).until(is_present, msg)
+        element = Wait(self.driver, timeout=timeout).until(is_present, msg)
         return element
 
-    def wait_until_visible(self, selector):
+    def wait_until_visible(self, selector, timeout=Wait._TIMEOUT):
         """ Wait until element matching CSS selector is visible on the page """
         is_visible = lambda driver: self.find(selector).is_displayed()
         msg = 'An element matching "%s" should be visible' % selector
-        Wait(self.driver).until(is_visible, msg)
+        Wait(self.driver, timeout=timeout).until(is_visible, msg)
+        return self.find(selector)
+
+    def wait_until_not_visible(self, selector, timeout=Wait._TIMEOUT):
+        """ Wait until element matching CSS selector is not visible on the page """
+        is_visible = lambda driver: self.find(selector).is_displayed()
+        msg = 'An element matching "%s" should be visible' % selector
+        Wait(self.driver, timeout=timeout).until_not(is_visible, msg)
         return self.find(selector)
 
+    def wait_until_clickable(self, selector, timeout=Wait._TIMEOUT):
+        """ Wait until element matching CSS selector is visible on the page """
+        WebDriverWait(self.driver, timeout=timeout).until(lambda driver: self.driver.execute_script("return jQuery.active == 0"))
+        is_clickable = lambda driver: (self.find(selector).is_displayed() and self.find(selector).is_enabled())
+        msg = 'An element matching "%s" should be clickable' % selector
+        Wait(self.driver, timeout=timeout).until(is_clickable, msg)
+        return self.find(selector)
+
+    def wait_until_element_clickable(self, finder, timeout=Wait._TIMEOUT):
+        """ Wait until element is clickable """
+        WebDriverWait(self.driver, timeout=timeout).until(lambda driver: self.driver.execute_script("return jQuery.active == 0"))
+        is_clickable = lambda driver: (finder(driver).is_displayed() and finder(driver).is_enabled())
+        msg = 'A matching element never became be clickable'
+        Wait(self.driver, timeout=timeout).until(is_clickable, msg)
+        return finder(self.driver)
+
     def wait_until_focused(self, selector):
         """ Wait until element matching CSS selector has focus """
         is_focused = \
diff --git a/bitbake/lib/toaster/tests/browser/test_all_builds_page.py b/bitbake/lib/toaster/tests/browser/test_all_builds_page.py
index 8423d3dab2..9ab81fb11b 100644
--- a/bitbake/lib/toaster/tests/browser/test_all_builds_page.py
+++ b/bitbake/lib/toaster/tests/browser/test_all_builds_page.py
@@ -7,13 +7,18 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import os
 import re
 
 from django.urls import reverse
+from selenium.webdriver.support.select import Select
 from django.utils import timezone
+from bldcontrol.models import BuildRequest
 from tests.browser.selenium_helpers import SeleniumTestCase
 
-from orm.models import BitbakeVersion, Release, Project, Build, Target
+from orm.models import BitbakeVersion, Layer, Layer_Version, Recipe, Release, Project, Build, Target, Task
+
+from selenium.webdriver.common.by import By
 
 
 class TestAllBuildsPage(SeleniumTestCase):
@@ -23,7 +28,8 @@ class TestAllBuildsPage(SeleniumTestCase):
     CLI_BUILDS_PROJECT_NAME = 'command line builds'
 
     def setUp(self):
-        bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='bbv1', giturl=f'{builldir}/',
                                             branch='master', dirpath='')
         release = Release.objects.create(name='release1',
                                          bitbake_version=bbv)
@@ -69,7 +75,7 @@ class TestAllBuildsPage(SeleniumTestCase):
             '[data-role="data-recent-build-buildtime-field"]' % build.id
 
         # because this loads via Ajax, wait for it to be visible
-        self.wait_until_present(selector)
+        self.wait_until_visible(selector)
 
         build_time_spans = self.find_all(selector)
 
@@ -79,7 +85,7 @@ class TestAllBuildsPage(SeleniumTestCase):
 
     def _get_row_for_build(self, build):
         """ Get the table row for the build from the all builds table """
-        self.wait_until_present('#allbuildstable')
+        self.wait_until_visible('#allbuildstable')
 
         rows = self.find_all('#allbuildstable tr')
 
@@ -91,7 +97,7 @@ class TestAllBuildsPage(SeleniumTestCase):
         found_row = None
         for row in rows:
 
-            outcome_links = row.find_elements_by_css_selector(selector)
+            outcome_links = row.find_elements(By.CSS_SELECTOR, selector)
             if len(outcome_links) == 1:
                 found_row = row
                 break
@@ -100,6 +106,66 @@ class TestAllBuildsPage(SeleniumTestCase):
 
         return found_row
 
+    def _get_create_builds(self, **kwargs):
+        """ Create a build and return the build object """
+        build1 = Build.objects.create(**self.project1_build_success)
+        build2 = Build.objects.create(**self.project1_build_failure)
+
+        # add some targets to these builds so they have recipe links
+        # (and so we can find the row in the ToasterTable corresponding to
+        # a particular build)
+        Target.objects.create(build=build1, target='foo')
+        Target.objects.create(build=build2, target='bar')
+
+        if kwargs:
+            # Create kwargs.get('success') builds with success status with target
+            # and kwargs.get('failure') builds with failure status with target
+            for i in range(kwargs.get('success', 0)):
+                now = timezone.now()
+                self.project1_build_success['started_on'] = now
+                self.project1_build_success[
+                    'completed_on'] = now - timezone.timedelta(days=i)
+                build = Build.objects.create(**self.project1_build_success)
+                Target.objects.create(build=build,
+                                      target=f'{i}_success_recipe',
+                                      task=f'{i}_success_task')
+
+                self._set_buildRequest_and_task_on_build(build)
+            for i in range(kwargs.get('failure', 0)):
+                now = timezone.now()
+                self.project1_build_failure['started_on'] = now
+                self.project1_build_failure[
+                    'completed_on'] = now - timezone.timedelta(days=i)
+                build = Build.objects.create(**self.project1_build_failure)
+                Target.objects.create(build=build,
+                                      target=f'{i}_fail_recipe',
+                                      task=f'{i}_fail_task')
+                self._set_buildRequest_and_task_on_build(build)
+        return build1, build2
+
+    def _create_recipe(self):
+        """ Add a recipe to the database and return it """
+        layer = Layer.objects.create()
+        layer_version = Layer_Version.objects.create(layer=layer)
+        return Recipe.objects.create(name='recipe_foo', layer_version=layer_version)
+
+    def _set_buildRequest_and_task_on_build(self, build):
+        """ Set buildRequest and task on build """
+        build.recipes_parsed = 1
+        build.save()
+        buildRequest = BuildRequest.objects.create(
+            build=build,
+            project=self.project1,
+            state=BuildRequest.REQ_COMPLETED)
+        build.build_request = buildRequest
+        recipe = self._create_recipe()
+        task = Task.objects.create(build=build,
+                                   recipe=recipe,
+                                   task_name='task',
+                                   outcome=Task.OUTCOME_SUCCESS)
+        task.save()
+        build.save()
+
     def test_show_tasks_with_suffix(self):
         """ Task should be shown as suffix on build name """
         build = Build.objects.create(**self.project1_build_success)
@@ -109,7 +175,7 @@ class TestAllBuildsPage(SeleniumTestCase):
 
         url = reverse('all-builds')
         self.get(url)
-        self.wait_until_present('td[class="target"]')
+        self.wait_until_visible('td[class="target"]')
 
         cell = self.find('td[class="target"]')
         content = cell.get_attribute('innerHTML')
@@ -126,23 +192,26 @@ class TestAllBuildsPage(SeleniumTestCase):
         but should be shown for other builds
         """
         build1 = Build.objects.create(**self.project1_build_success)
-        default_build = Build.objects.create(**self.default_project_build_success)
+        default_build = Build.objects.create(
+            **self.default_project_build_success)
 
         url = reverse('all-builds')
         self.get(url)
 
-        # shouldn't see a rebuild button for command-line builds
-        selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % default_build.id
-        run_again_button = self.find_all(selector)
-        self.assertEqual(len(run_again_button), 0,
-                         'should not see a rebuild button for cli builds')
-
         # should see a rebuild button for non-command-line builds
+        self.wait_until_visible('#allbuildstable tbody tr')
+        self.wait_until_visible('.rebuild-btn')
         selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % build1.id
         run_again_button = self.find_all(selector)
         self.assertEqual(len(run_again_button), 1,
                          'should see a rebuild button for non-cli builds')
 
+        # shouldn't see a rebuild button for command-line builds
+        selector = 'div[data-latest-build-result="%s"] .rebuild-btn' % default_build.id
+        run_again_button = self.find_all(selector)
+        self.assertEqual(len(run_again_button), 0,
+                         'should not see a rebuild button for cli builds')
+
     def test_tooltips_on_project_name(self):
         """
         Test tooltips shown next to project name in the main table
@@ -156,6 +225,7 @@ class TestAllBuildsPage(SeleniumTestCase):
 
         url = reverse('all-builds')
         self.get(url)
+        self.wait_until_visible('#allbuildstable')
 
         # get the project name cells from the table
         cells = self.find_all('#allbuildstable td[class="project"]')
@@ -164,7 +234,7 @@ class TestAllBuildsPage(SeleniumTestCase):
 
         for cell in cells:
             content = cell.get_attribute('innerHTML')
-            help_icons = cell.find_elements_by_css_selector(selector)
+            help_icons = cell.find_elements(By.CSS_SELECTOR, selector)
 
             if re.search(self.PROJECT_NAME, content):
                 # no help icon next to non-cli project name
@@ -184,38 +254,224 @@ class TestAllBuildsPage(SeleniumTestCase):
         recent builds area; failed builds should not have links on the time column,
         or in the recent builds area
         """
-        build1 = Build.objects.create(**self.project1_build_success)
-        build2 = Build.objects.create(**self.project1_build_failure)
-
-        # add some targets to these builds so they have recipe links
-        # (and so we can find the row in the ToasterTable corresponding to
-        # a particular build)
-        Target.objects.create(build=build1, target='foo')
-        Target.objects.create(build=build2, target='bar')
+        build1, build2 = self._get_create_builds()
 
         url = reverse('all-builds')
         self.get(url)
+        self.wait_until_visible('#allbuildstable')
 
         # test recent builds area for successful build
         element = self._get_build_time_element(build1)
-        links = element.find_elements_by_css_selector('a')
+        links = element.find_elements(By.CSS_SELECTOR, 'a')
         msg = 'should be a link on the build time for a successful recent build'
-        self.assertEquals(len(links), 1, msg)
+        self.assertEqual(len(links), 1, msg)
 
         # test recent builds area for failed build
         element = self._get_build_time_element(build2)
-        links = element.find_elements_by_css_selector('a')
+        links = element.find_elements(By.CSS_SELECTOR, 'a')
         msg = 'should not be a link on the build time for a failed recent build'
-        self.assertEquals(len(links), 0, msg)
+        self.assertEqual(len(links), 0, msg)
 
         # test the time column for successful build
         build1_row = self._get_row_for_build(build1)
-        links = build1_row.find_elements_by_css_selector('td.time a')
+        links = build1_row.find_elements(By.CSS_SELECTOR, 'td.time a')
         msg = 'should be a link on the build time for a successful build'
-        self.assertEquals(len(links), 1, msg)
+        self.assertEqual(len(links), 1, msg)
 
         # test the time column for failed build
         build2_row = self._get_row_for_build(build2)
-        links = build2_row.find_elements_by_css_selector('td.time a')
+        links = build2_row.find_elements(By.CSS_SELECTOR, 'td.time a')
         msg = 'should not be a link on the build time for a failed build'
-        self.assertEquals(len(links), 0, msg)
+        self.assertEqual(len(links), 0, msg)
+
+    def test_builds_table_search_box(self):
+        """ Test the search box in the builds table on the all builds page """
+        self._get_create_builds()
+
+        url = reverse('all-builds')
+        self.get(url)
+
+        # Check search box is present and works
+        self.wait_until_visible('#allbuildstable tbody tr')
+        search_box = self.find('#search-input-allbuildstable')
+        self.assertTrue(search_box.is_displayed())
+
+        # Check that we can search for a build by recipe name
+        search_box.send_keys('foo')
+        search_btn = self.find('#search-submit-allbuildstable')
+        search_btn.click()
+        self.wait_until_visible('#allbuildstable tbody tr')
+        rows = self.find_all('#allbuildstable tbody tr')
+        self.assertTrue(len(rows) >= 1)
+
+    def test_filtering_on_failure_tasks_column(self):
+        """ Test the filtering on failure tasks column in the builds table on the all builds page """
+        def _check_if_filter_failed_tasks_column_is_visible():
+            # check if failed tasks filter column is visible, if not click on it
+            # Check edit column
+            edit_column = self.find('#edit-columns-button')
+            self.assertTrue(edit_column.is_displayed())
+            edit_column.click()
+            # Check dropdown is visible
+            self.wait_until_visible('ul.dropdown-menu.editcol')
+            filter_fails_task_checkbox = self.find('#checkbox-failed_tasks')
+            if not filter_fails_task_checkbox.is_selected():
+                filter_fails_task_checkbox.click()
+            edit_column.click()
+
+        self._get_create_builds(success=10, failure=10)
+
+        url = reverse('all-builds')
+        self.get(url)
+
+        # Check filtering on failure tasks column
+        self.wait_until_visible('#allbuildstable tbody tr')
+        _check_if_filter_failed_tasks_column_is_visible()
+        failed_tasks_filter = self.find('#failed_tasks_filter')
+        failed_tasks_filter.click()
+        # Check popup is visible
+        self.wait_until_visible('#filter-modal-allbuildstable')
+        self.assertTrue(
+            self.find('#filter-modal-allbuildstable').is_displayed())
+        # Check that we can filter by failure tasks
+        build_without_failure_tasks = self.find(
+            '#failed_tasks_filter\\:without_failed_tasks')
+        build_without_failure_tasks.click()
+        # click on apply button
+        self.find('#filter-modal-allbuildstable .btn-primary').click()
+        self.wait_until_visible('#allbuildstable tbody tr')
+        # Check if filter is applied, by checking if failed_tasks_filter has btn-primary class
+        self.assertTrue(self.find('#failed_tasks_filter').get_attribute(
+            'class').find('btn-primary') != -1)
+
+    def test_filtering_on_completedOn_column(self):
+        """ Test the filtering on completed_on column in the builds table on the all builds page """
+        self._get_create_builds(success=10, failure=10)
+
+        url = reverse('all-builds')
+        self.get(url)
+
+        # Check filtering on failure tasks column
+        self.wait_until_visible('#allbuildstable tbody tr')
+        completed_on_filter = self.find('#completed_on_filter')
+        completed_on_filter.click()
+        # Check popup is visible
+        self.wait_until_visible('#filter-modal-allbuildstable')
+        self.assertTrue(
+            self.find('#filter-modal-allbuildstable').is_displayed())
+        # Check that we can filter by failure tasks
+        build_without_failure_tasks = self.find(
+            '#completed_on_filter\\:date_range')
+        build_without_failure_tasks.click()
+        # click on apply button
+        self.find('#filter-modal-allbuildstable .btn-primary').click()
+        self.wait_until_visible('#allbuildstable tbody tr')
+        # Check if filter is applied, by checking if completed_on_filter has btn-primary class
+        self.assertTrue(self.find('#completed_on_filter').get_attribute(
+            'class').find('btn-primary') != -1)
+
+        # Filter by date range
+        self.find('#completed_on_filter').click()
+        self.wait_until_visible('#filter-modal-allbuildstable')
+        date_ranges = self.driver.find_elements(
+            By.XPATH, '//input[@class="form-control hasDatepicker"]')
+        today = timezone.now()
+        yestersday = today - timezone.timedelta(days=1)
+        date_ranges[0].send_keys(yestersday.strftime('%Y-%m-%d'))
+        date_ranges[1].send_keys(today.strftime('%Y-%m-%d'))
+        self.find('#filter-modal-allbuildstable .btn-primary').click()
+        self.wait_until_visible('#allbuildstable tbody tr')
+        self.assertTrue(self.find('#completed_on_filter').get_attribute(
+            'class').find('btn-primary') != -1)
+        # Check if filter is applied, number of builds displayed should be 6
+        self.assertTrue(len(self.find_all('#allbuildstable tbody tr')) >= 4)
+
+    def test_builds_table_editColumn(self):
+        """ Test the edit column feature in the builds table on the all builds page """
+        self._get_create_builds(success=10, failure=10)
+
+        def test_edit_column(check_box_id):
+            # Check that we can hide/show table column
+            check_box = self.find(f'#{check_box_id}')
+            th_class = str(check_box_id).replace('checkbox-', '')
+            if check_box.is_selected():
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#allbuildstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+                check_box.click()
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#allbuildstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+            else:
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#allbuildstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+                check_box.click()
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#allbuildstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+        url = reverse('all-builds')
+        self.get(url)
+        self.wait_until_visible('#allbuildstable tbody tr')
+
+        # Check edit column
+        edit_column = self.find('#edit-columns-button')
+        self.assertTrue(edit_column.is_displayed())
+        edit_column.click()
+        # Check dropdown is visible
+        self.wait_until_visible('ul.dropdown-menu.editcol')
+
+        # Check that we can hide the edit column
+        test_edit_column('checkbox-errors_no')
+        test_edit_column('checkbox-failed_tasks')
+        test_edit_column('checkbox-image_files')
+        test_edit_column('checkbox-project')
+        test_edit_column('checkbox-started_on')
+        test_edit_column('checkbox-time')
+        test_edit_column('checkbox-warnings_no')
+
+    def test_builds_table_show_rows(self):
+        """ Test the show rows feature in the builds table on the all builds page """
+        self._get_create_builds(success=100, failure=100)
+
+        def test_show_rows(row_to_show, show_row_link):
+            # Check that we can show rows == row_to_show
+            show_row_link.select_by_value(str(row_to_show))
+            self.wait_until_visible('#allbuildstable tbody tr')
+            # check at least some rows are visible
+            self.assertTrue(
+                len(self.find_all('#allbuildstable tbody tr')) > 0
+            )
+
+        url = reverse('all-builds')
+        self.get(url)
+        self.wait_until_visible('#allbuildstable tbody tr')
+
+        show_rows = self.driver.find_elements(
+            By.XPATH,
+            '//select[@class="form-control pagesize-allbuildstable"]'
+        )
+        # Check show rows
+        for show_row_link in show_rows:
+            show_row_link = Select(show_row_link)
+            test_show_rows(10, show_row_link)
+            test_show_rows(25, show_row_link)
+            test_show_rows(50, show_row_link)
+            test_show_rows(100, show_row_link)
+            test_show_rows(150, show_row_link)
diff --git a/bitbake/lib/toaster/tests/browser/test_all_projects_page.py b/bitbake/lib/toaster/tests/browser/test_all_projects_page.py
index 15b03400f9..05e12892be 100644
--- a/bitbake/lib/toaster/tests/browser/test_all_projects_page.py
+++ b/bitbake/lib/toaster/tests/browser/test_all_projects_page.py
@@ -7,15 +7,20 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import os
 import re
 
 from django.urls import reverse
 from django.utils import timezone
+from selenium.webdriver.support.select import Select
 from tests.browser.selenium_helpers import SeleniumTestCase
 
 from orm.models import BitbakeVersion, Release, Project, Build
 from orm.models import ProjectVariable
 
+from selenium.webdriver.common.by import By
+
+
 class TestAllProjectsPage(SeleniumTestCase):
     """ Browser tests for projects page /projects/ """
 
@@ -25,7 +30,8 @@ class TestAllProjectsPage(SeleniumTestCase):
 
     def setUp(self):
         """ Add default project manually """
-        project = Project.objects.create_project(self.CLI_BUILDS_PROJECT_NAME, None)
+        project = Project.objects.create_project(
+            self.CLI_BUILDS_PROJECT_NAME, None)
         self.default_project = project
         self.default_project.is_default = True
         self.default_project.save()
@@ -35,6 +41,17 @@ class TestAllProjectsPage(SeleniumTestCase):
 
         self.release = None
 
+    def _create_projects(self, nb_project=10):
+        projects = []
+        for i in range(1, nb_project + 1):
+            projects.append(
+                Project(
+                    name='test project {}'.format(i),
+                    release=self.release,
+                )
+            )
+        Project.objects.bulk_create(projects)
+
     def _add_build_to_default_project(self):
         """ Add a build to the default project (not used in all tests) """
         now = timezone.now()
@@ -45,12 +62,14 @@ class TestAllProjectsPage(SeleniumTestCase):
 
     def _add_non_default_project(self):
         """ Add another project """
-        bbv = BitbakeVersion.objects.create(name='test bbv', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='test bbv', giturl=f'{builldir}/',
                                             branch='master', dirpath='')
         self.release = Release.objects.create(name='test release',
                                               branch_name='master',
                                               bitbake_version=bbv)
-        self.project = Project.objects.create_project(self.PROJECT_NAME, self.release)
+        self.project = Project.objects.create_project(
+            self.PROJECT_NAME, self.release)
         self.project.is_default = False
         self.project.save()
 
@@ -62,7 +81,7 @@ class TestAllProjectsPage(SeleniumTestCase):
 
     def _get_row_for_project(self, project_name):
         """ Get the HTML row for a project, or None if not found """
-        self.wait_until_present('#projectstable tbody tr')
+        self.wait_until_visible('#projectstable tbody tr')
         rows = self.find_all('#projectstable tbody tr')
 
         # find the row with a project name matching the one supplied
@@ -93,7 +112,8 @@ class TestAllProjectsPage(SeleniumTestCase):
         url = reverse('all-projects')
         self.get(url)
 
-        default_project_row = self._get_row_for_project(self.default_project.name)
+        default_project_row = self._get_row_for_project(
+            self.default_project.name)
 
         self.assertNotEqual(default_project_row, None,
                             'default project "cli builds" should be in page')
@@ -113,11 +133,12 @@ class TestAllProjectsPage(SeleniumTestCase):
         self.wait_until_visible("#projectstable tr")
 
         # find the row for the default project
-        default_project_row = self._get_row_for_project(self.default_project.name)
+        default_project_row = self._get_row_for_project(
+            self.default_project.name)
 
         # check the release text for the default project
         selector = 'span[data-project-field="release"] span.text-muted'
-        element = default_project_row.find_element_by_css_selector(selector)
+        element = default_project_row.find_element(By.CSS_SELECTOR, selector)
         text = element.text.strip()
         self.assertEqual(text, 'Not applicable',
                          'release should be "not applicable" for default project')
@@ -127,7 +148,7 @@ class TestAllProjectsPage(SeleniumTestCase):
 
         # check the link in the release cell for the other project
         selector = 'span[data-project-field="release"]'
-        element = other_project_row.find_element_by_css_selector(selector)
+        element = other_project_row.find_element(By.CSS_SELECTOR, selector)
         text = element.text.strip()
         self.assertEqual(text, self.release.name,
                          'release name should be shown for non-default project')
@@ -148,11 +169,12 @@ class TestAllProjectsPage(SeleniumTestCase):
         self.wait_until_visible("#projectstable tr")
 
         # find the row for the default project
-        default_project_row = self._get_row_for_project(self.default_project.name)
+        default_project_row = self._get_row_for_project(
+            self.default_project.name)
 
         # check the machine cell for the default project
         selector = 'span[data-project-field="machine"] span.text-muted'
-        element = default_project_row.find_element_by_css_selector(selector)
+        element = default_project_row.find_element(By.CSS_SELECTOR, selector)
         text = element.text.strip()
         self.assertEqual(text, 'Not applicable',
                          'machine should be not applicable for default project')
@@ -162,7 +184,7 @@ class TestAllProjectsPage(SeleniumTestCase):
 
         # check the link in the machine cell for the other project
         selector = 'span[data-project-field="machine"]'
-        element = other_project_row.find_element_by_css_selector(selector)
+        element = other_project_row.find_element(By.CSS_SELECTOR, selector)
         text = element.text.strip()
         self.assertEqual(text, self.MACHINE_NAME,
                          'machine name should be shown for non-default project')
@@ -183,13 +205,15 @@ class TestAllProjectsPage(SeleniumTestCase):
         self.get(reverse('all-projects'))
 
         # find the row for the default project
-        default_project_row = self._get_row_for_project(self.default_project.name)
+        default_project_row = self._get_row_for_project(
+            self.default_project.name)
 
         # check the link on the name field
         selector = 'span[data-project-field="name"] a'
-        element = default_project_row.find_element_by_css_selector(selector)
+        element = default_project_row.find_element(By.CSS_SELECTOR, selector)
         link_url = element.get_attribute('href').strip()
-        expected_url = reverse('projectbuilds', args=(self.default_project.id,))
+        expected_url = reverse(
+            'projectbuilds', args=(self.default_project.id,))
         msg = 'link on default project name should point to builds but was %s' % link_url
         self.assertTrue(link_url.endswith(expected_url), msg)
 
@@ -198,8 +222,116 @@ class TestAllProjectsPage(SeleniumTestCase):
 
         # check the link for the other project
         selector = 'span[data-project-field="name"] a'
-        element = other_project_row.find_element_by_css_selector(selector)
+        element = other_project_row.find_element(By.CSS_SELECTOR, selector)
         link_url = element.get_attribute('href').strip()
         expected_url = reverse('project', args=(self.project.id,))
         msg = 'link on project name should point to configuration but was %s' % link_url
         self.assertTrue(link_url.endswith(expected_url), msg)
+
+    def test_allProject_table_search_box(self):
+        """ Test the search box in the all project table on the all projects page """
+        self._create_projects()
+
+        url = reverse('all-projects')
+        self.get(url)
+
+        # Chseck search box is present and works
+        self.wait_until_visible('#projectstable tbody tr')
+        search_box = self.find('#search-input-projectstable')
+        self.assertTrue(search_box.is_displayed())
+
+        # Check that we can search for a project by project name
+        search_box.send_keys('test project 10')
+        search_btn = self.find('#search-submit-projectstable')
+        search_btn.click()
+        self.wait_until_visible('#projectstable tbody tr')
+        rows = self.find_all('#projectstable tbody tr')
+        self.assertTrue(len(rows) == 1)
+
+    def test_allProject_table_editColumn(self):
+        """ Test the edit column feature in the projects table on the all projects page """
+        self._create_projects()
+
+        def test_edit_column(check_box_id):
+            # Check that we can hide/show table column
+            check_box = self.find(f'#{check_box_id}')
+            th_class = str(check_box_id).replace('checkbox-', '')
+            if check_box.is_selected():
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#projectstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+                check_box.click()
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#projectstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+            else:
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#projectstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+                check_box.click()
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#projectstable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+        url = reverse('all-projects')
+        self.get(url)
+        self.wait_until_visible('#projectstable tbody tr')
+
+        # Check edit column
+        edit_column = self.find('#edit-columns-button')
+        self.assertTrue(edit_column.is_displayed())
+        edit_column.click()
+        # Check dropdown is visible
+        self.wait_until_visible('ul.dropdown-menu.editcol')
+
+        # Check that we can hide the edit column
+        test_edit_column('checkbox-errors')
+        test_edit_column('checkbox-image_files')
+        test_edit_column('checkbox-last_build_outcome')
+        test_edit_column('checkbox-recipe_name')
+        test_edit_column('checkbox-warnings')
+
+    def test_allProject_table_show_rows(self):
+        """ Test the show rows feature in the projects table on the all projects page """
+        self._create_projects(nb_project=200)
+
+        def test_show_rows(row_to_show, show_row_link):
+            # Check that we can show rows == row_to_show
+            show_row_link.select_by_value(str(row_to_show))
+            self.wait_until_visible('#projectstable tbody tr')
+            # check at least some rows are visible
+            self.assertTrue(
+                len(self.find_all('#projectstable tbody tr')) > 0
+            )
+
+        url = reverse('all-projects')
+        self.get(url)
+        self.wait_until_visible('#projectstable tbody tr')
+
+        show_rows = self.driver.find_elements(
+            By.XPATH,
+            '//select[@class="form-control pagesize-projectstable"]'
+        )
+        # Check show rows
+        for show_row_link in show_rows:
+            show_row_link = Select(show_row_link)
+            test_show_rows(10, show_row_link)
+            test_show_rows(25, show_row_link)
+            test_show_rows(50, show_row_link)
+            test_show_rows(100, show_row_link)
+            test_show_rows(150, show_row_link)
diff --git a/bitbake/lib/toaster/tests/browser/test_builddashboard_page.py b/bitbake/lib/toaster/tests/browser/test_builddashboard_page.py
index efcd89b346..82367108e2 100644
--- a/bitbake/lib/toaster/tests/browser/test_builddashboard_page.py
+++ b/bitbake/lib/toaster/tests/browser/test_builddashboard_page.py
@@ -7,6 +7,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import os
 from django.urls import reverse
 from django.utils import timezone
 
@@ -15,11 +16,14 @@ from tests.browser.selenium_helpers import SeleniumTestCase
 from orm.models import Project, Release, BitbakeVersion, Build, LogMessage
 from orm.models import Layer, Layer_Version, Recipe, CustomImageRecipe, Variable
 
+from selenium.webdriver.common.by import By
+
 class TestBuildDashboardPage(SeleniumTestCase):
     """ Tests for the build dashboard /build/X """
 
     def setUp(self):
-        bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='bbv1', giturl=f'{builldir}/',
                                             branch='master', dirpath="")
         release = Release.objects.create(name='release1',
                                          bitbake_version=bbv)
@@ -158,6 +162,7 @@ class TestBuildDashboardPage(SeleniumTestCase):
         """
         url = reverse('builddashboard', args=(build.id,))
         self.get(url)
+        self.wait_until_visible('#global-nav')
 
     def _get_build_dashboard_errors(self, build):
         """
@@ -183,7 +188,7 @@ class TestBuildDashboardPage(SeleniumTestCase):
 
         found = False
         for element in message_elements:
-            log_message_text = element.find_element_by_tag_name('pre').text.strip()
+            log_message_text = element.find_element(By.TAG_NAME, 'pre').text.strip()
             text_matches = (log_message_text == expected_text)
 
             log_message_pk = element.get_attribute('data-log-message-id')
@@ -213,7 +218,7 @@ class TestBuildDashboardPage(SeleniumTestCase):
         the WebElement modal match the list of text values in expected
         """
         # labels containing the radio buttons we're testing for
-        labels = modal.find_elements_by_css_selector(".radio")
+        labels = modal.find_elements(By.CSS_SELECTOR,".radio")
 
         labels_text = [lab.text for lab in labels]
         self.assertEqual(len(labels_text), len(expected))
@@ -248,7 +253,7 @@ class TestBuildDashboardPage(SeleniumTestCase):
         selector = '[data-role="edit-custom-image-trigger"]'
         self.click(selector)
 
-        modal = self.driver.find_element_by_id('edit-custom-image-modal')
+        modal = self.driver.find_element(By.ID, 'edit-custom-image-modal')
         self.wait_until_visible("#edit-custom-image-modal")
 
         # recipes we expect to see in the edit custom image modal
@@ -270,7 +275,7 @@ class TestBuildDashboardPage(SeleniumTestCase):
         selector = '[data-role="new-custom-image-trigger"]'
         self.click(selector)
 
-        modal = self.driver.find_element_by_id('new-custom-image-modal')
+        modal = self.driver.find_element(By.ID,'new-custom-image-modal')
         self.wait_until_visible("#new-custom-image-modal")
 
         # recipes we expect to see in the new custom image modal
diff --git a/bitbake/lib/toaster/tests/browser/test_builddashboard_page_artifacts.py b/bitbake/lib/toaster/tests/browser/test_builddashboard_page_artifacts.py
index c6226d60eb..675825bd40 100644
--- a/bitbake/lib/toaster/tests/browser/test_builddashboard_page_artifacts.py
+++ b/bitbake/lib/toaster/tests/browser/test_builddashboard_page_artifacts.py
@@ -7,6 +7,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import os
 from django.urls import reverse
 from django.utils import timezone
 
@@ -20,7 +21,8 @@ class TestBuildDashboardPageArtifacts(SeleniumTestCase):
     """ Tests for artifacts on the build dashboard /build/X """
 
     def setUp(self):
-        bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='bbv1', giturl=f'{builldir}/',
                                             branch='master', dirpath="")
         release = Release.objects.create(name='release1',
                                          bitbake_version=bbv)
@@ -197,12 +199,12 @@ class TestBuildDashboardPageArtifacts(SeleniumTestCase):
         # check package count and size, link on target name
         selector = '[data-value="target-package-count"]'
         element = self.find(selector)
-        self.assertEquals(element.text, '1',
+        self.assertEqual(element.text, '1',
             'package count should be shown for image builds')
 
         selector = '[data-value="target-package-size"]'
         element = self.find(selector)
-        self.assertEquals(element.text, '1.0 KB',
+        self.assertEqual(element.text, '1.0 KB',
             'package size should be shown for image builds')
 
         selector = '[data-link="target-packages"]'
diff --git a/bitbake/lib/toaster/tests/browser/test_delete_project.py b/bitbake/lib/toaster/tests/browser/test_delete_project.py
new file mode 100644
index 0000000000..1941777ccc
--- /dev/null
+++ b/bitbake/lib/toaster/tests/browser/test_delete_project.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux Inc
+#
+# SPDX-License-Identifier: GPL-2.0-only
+
+import pytest
+from django.urls import reverse
+from selenium.webdriver.support.ui import Select
+from tests.browser.selenium_helpers import SeleniumTestCase
+from orm.models import BitbakeVersion, Project, Release
+from selenium.webdriver.common.by import By
+
+class TestDeleteProject(SeleniumTestCase):
+
+    def setUp(self):
+        bitbake, _ = BitbakeVersion.objects.get_or_create(
+            name="master",
+            giturl="git://master",
+            branch="master",
+            dirpath="master")
+
+        self.release, _ = Release.objects.get_or_create(
+            name="master",
+            description="Yocto Project master",
+            branch_name="master",
+            helptext="latest",
+            bitbake_version=bitbake)
+
+        Release.objects.get_or_create(
+            name="foo",
+            description="Yocto Project foo",
+            branch_name="foo",
+            helptext="latest",
+            bitbake_version=bitbake)
+
+    @pytest.mark.django_db
+    def test_delete_project(self):
+        """ Test delete a project
+            - Check delete modal is visible
+            - Check delete modal has right text
+            - Confirm delete
+            - Check project is deleted
+        """
+        project_name = "project_to_delete"
+        url = reverse('newproject')
+        self.get(url)
+        self.enter_text('#new-project-name', project_name)
+        select = Select(self.find('#projectversion'))
+        select.select_by_value(str(self.release.pk))
+        self.click("#create-project-button")
+        # We should get redirected to the new project's page with the
+        # notification at the top
+        element = self.wait_until_visible('#project-created-notification')
+        self.assertTrue(project_name in element.text,
+                        "New project name not in new project notification")
+        self.assertTrue(Project.objects.filter(name=project_name).count(),
+                        "New project not found in database")
+
+        # Delete project
+        delete_project_link = self.driver.find_element(
+            By.XPATH, '//a[@href="#delete-project-modal"]')
+        delete_project_link.click()
+        
+        # Check delete modal is visible
+        self.wait_until_visible('#delete-project-modal')
+
+        # Check delete modal has right text
+        modal_header_text = self.find('#delete-project-modal .modal-header').text
+        self.assertTrue(
+            "Are you sure you want to delete this project?" in modal_header_text,
+            "Delete project modal header text is wrong")
+
+        modal_body_text = self.find('#delete-project-modal .modal-body').text
+        self.assertTrue(
+            "Cancel its builds currently in progress" in modal_body_text,
+            "Modal body doesn't contain: Cancel its builds currently in progress")
+        self.assertTrue(
+            "Remove its configuration information" in modal_body_text,
+            "Modal body doesn't contain: Remove its configuration information")
+        self.assertTrue(
+            "Remove its imported layers" in modal_body_text,
+            "Modal body doesn't contain: Remove its imported layers")
+        self.assertTrue(
+            "Remove its custom images" in modal_body_text,
+            "Modal body doesn't contain: Remove its custom images")
+        self.assertTrue(
+            "Remove all its build information" in modal_body_text,
+            "Modal body doesn't contain: Remove all its build information")
+
+        # Confirm delete
+        delete_btn = self.find('#delete-project-confirmed')
+        delete_btn.click()
+
+        # Check project is deleted
+        self.wait_until_visible('#change-notification')
+        delete_notification = self.find('#change-notification-msg')
+        self.assertTrue("You have deleted 1 project:" in delete_notification.text)
+        self.assertTrue(project_name in delete_notification.text)
+        self.assertFalse(Project.objects.filter(name=project_name).exists(),
+                        "Project not deleted from database")
diff --git a/bitbake/lib/toaster/tests/browser/test_landing_page.py b/bitbake/lib/toaster/tests/browser/test_landing_page.py
index 8bb64b9f3e..210359d561 100644
--- a/bitbake/lib/toaster/tests/browser/test_landing_page.py
+++ b/bitbake/lib/toaster/tests/browser/test_landing_page.py
@@ -10,8 +10,10 @@
 from django.urls import reverse
 from django.utils import timezone
 from tests.browser.selenium_helpers import SeleniumTestCase
+from selenium.webdriver.common.by import By
+
+from orm.models import Layer, Layer_Version, Project, Build
 
-from orm.models import Project, Build
 
 class TestLandingPage(SeleniumTestCase):
     """ Tests for redirects on the landing page """
@@ -29,12 +31,147 @@ class TestLandingPage(SeleniumTestCase):
         self.project.is_default = True
         self.project.save()
 
+    def test_icon_info_visible_and_clickable(self):
+        """ Test that the information icon is visible and clickable """
+        self.get(reverse('landing'))
+        self.wait_until_visible('#toaster-version-info-sign')
+        info_sign = self.find('#toaster-version-info-sign')
+
+        # check that the info sign is visible
+        self.assertTrue(info_sign.is_displayed())
+
+        # check that the info sign is clickable
+        # and info modal is appearing when clicking on the info sign
+        info_sign.click()  # click on the info sign make attribute 'aria-describedby' visible
+        info_model_id = info_sign.get_attribute('aria-describedby')
+        self.wait_until_visible(f'#{info_model_id}')
+        info_modal = self.find(f'#{info_model_id}')
+        self.assertTrue(info_modal.is_displayed())
+        self.assertTrue("Toaster version information" in info_modal.text)
+
+    def test_documentation_link_displayed(self):
+        """ Test that the documentation link is displayed """
+        self.get(reverse('landing'))
+        self.wait_until_visible('#navbar-docs')
+        documentation_link = self.find('#navbar-docs > a')
+
+        # check that the documentation link is visible
+        self.assertTrue(documentation_link.is_displayed())
+
+        # check browser open new tab toaster manual when clicking on the documentation link
+        self.assertEqual(documentation_link.get_attribute('target'), '_blank')
+        self.assertEqual(
+            documentation_link.get_attribute('href'),
+            'http://docs.yoctoproject.org/toaster-manual/index.html#toaster-user-manual')
+        self.assertTrue("Documentation" in documentation_link.text)
+
+    def test_openembedded_jumbotron_link_visible_and_clickable(self):
+        """ Test OpenEmbedded link jumbotron is visible and clickable: """
+        self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
+        jumbotron = self.find('.jumbotron')
+
+        # check OpenEmbedded
+        openembedded = jumbotron.find_element(By.LINK_TEXT, 'OpenEmbedded')
+        self.assertTrue(openembedded.is_displayed())
+        openembedded.click()
+        self.assertTrue("openembedded.org" in self.driver.current_url)
+
+    def test_bitbake_jumbotron_link_visible_and_clickable(self):
+        """ Test BitBake link jumbotron is visible and clickable: """
+        self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
+        jumbotron = self.find('.jumbotron')
+
+        # check BitBake
+        bitbake = jumbotron.find_element(By.LINK_TEXT, 'BitBake')
+        self.assertTrue(bitbake.is_displayed())
+        bitbake.click()
+        self.assertTrue(
+            "docs.yoctoproject.org/bitbake.html" in self.driver.current_url)
+
+    def test_yoctoproject_jumbotron_link_visible_and_clickable(self):
+        """ Test Yocto Project link jumbotron is visible and clickable: """
+        self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
+        jumbotron = self.find('.jumbotron')
+
+        # check Yocto Project
+        yoctoproject = jumbotron.find_element(By.LINK_TEXT, 'Yocto Project')
+        self.assertTrue(yoctoproject.is_displayed())
+        yoctoproject.click()
+        self.assertTrue("yoctoproject.org" in self.driver.current_url)
+
+    def test_link_setup_using_toaster_visible_and_clickable(self):
+        """ Test big magenta button setting up and using toaster link in jumbotron
+            if visible and clickable
+        """
+        self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
+        jumbotron = self.find('.jumbotron')
+
+        # check Big magenta button
+        big_magenta_button = jumbotron.find_element(By.LINK_TEXT,
+                                                    'Toaster is ready to capture your command line builds'
+                                                    )
+        self.assertTrue(big_magenta_button.is_displayed())
+        big_magenta_button.click()
+        self.assertTrue(
+            "docs.yoctoproject.org/toaster-manual/setup-and-use.html#setting-up-and-using-toaster" in self.driver.current_url)
+
+    def test_link_create_new_project_in_jumbotron_visible_and_clickable(self):
+        """ Test big blue button create new project jumbotron if visible and clickable """
+        # Create a layer and a layer version to make visible the big blue button
+        layer = Layer.objects.create(name='bar')
+        Layer_Version.objects.create(layer=layer)
+
+        self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
+        jumbotron = self.find('.jumbotron')
+
+        # check Big Blue button
+        big_blue_button = jumbotron.find_element(By.LINK_TEXT,
+                                                 'Create your first Toaster project to run manage builds'
+                                                 )
+        self.assertTrue(big_blue_button.is_displayed())
+        big_blue_button.click()
+        self.assertTrue("toastergui/newproject/" in self.driver.current_url)
+
+    def test_toaster_manual_link_visible_and_clickable(self):
+        """ Test Read the Toaster manual link jumbotron is visible and clickable: """
+        self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
+        jumbotron = self.find('.jumbotron')
+
+        # check Read the Toaster manual
+        toaster_manual = jumbotron.find_element(
+            By.LINK_TEXT, 'Read the Toaster manual')
+        self.assertTrue(toaster_manual.is_displayed())
+        toaster_manual.click()
+        self.assertTrue(
+            "https://docs.yoctoproject.org/toaster-manual/index.html#toaster-user-manual" in self.driver.current_url)
+
+    def test_contrib_to_toaster_link_visible_and_clickable(self):
+        """ Test Contribute to Toaster link jumbotron is visible and clickable: """
+        self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
+        jumbotron = self.find('.jumbotron')
+
+        # check Contribute to Toaster
+        contribute_to_toaster = jumbotron.find_element(
+            By.LINK_TEXT, 'Contribute to Toaster')
+        self.assertTrue(contribute_to_toaster.is_displayed())
+        contribute_to_toaster.click()
+        self.assertTrue(
+            "wiki.yoctoproject.org/wiki/contribute_to_toaster" in str(self.driver.current_url).lower())
+
     def test_only_default_project(self):
         """
         No projects except default
         => should see the landing page
         """
         self.get(reverse('landing'))
+        self.wait_until_visible('.jumbotron')
         self.assertTrue(self.LANDING_PAGE_TITLE in self.get_page_source())
 
     def test_default_project_has_build(self):
@@ -67,6 +204,7 @@ class TestLandingPage(SeleniumTestCase):
         user_project.save()
 
         self.get(reverse('landing'))
+        self.wait_until_visible('#projectstable')
 
         elements = self.find_all('#projectstable')
         self.assertEqual(len(elements), 1, 'should redirect to projects')
@@ -87,10 +225,9 @@ class TestLandingPage(SeleniumTestCase):
 
         self.get(reverse('landing'))
 
+        self.wait_until_visible("#latest-builds")
         elements = self.find_all('#allbuildstable')
         self.assertEqual(len(elements), 1, 'should redirect to builds')
         content = self.get_page_source()
         self.assertTrue(self.PROJECT_NAME in content,
                         'should show builds for project %s' % self.PROJECT_NAME)
-        self.assertFalse(self.CLI_BUILDS_PROJECT_NAME in content,
-                         'should not show builds for cli project')
diff --git a/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py b/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py
index 71bdd2aafd..6abfdef699 100644
--- a/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py
+++ b/bitbake/lib/toaster/tests/browser/test_layerdetails_page.py
@@ -8,6 +8,7 @@
 #
 
 from django.urls import reverse
+from selenium.common.exceptions import ElementClickInterceptedException, TimeoutException
 from tests.browser.selenium_helpers import SeleniumTestCase
 
 from orm.models import Layer, Layer_Version, Project, LayerSource, Release
@@ -63,11 +64,12 @@ class TestLayerDetailsPage(SeleniumTestCase):
                            args=(self.project.pk,
                                  self.imported_layer_version.pk))
 
-    def test_edit_layerdetails(self):
+    def test_edit_layerdetails_page(self):
         """ Edit all the editable fields for the layer refresh the page and
         check that the new values exist"""
 
         self.get(self.url)
+        self.wait_until_visible("#add-remove-layer-btn")
 
         self.click("#add-remove-layer-btn")
         self.click("#edit-layer-source")
@@ -97,13 +99,21 @@ class TestLayerDetailsPage(SeleniumTestCase):
                             "Expecting any of \"%s\"but got \"%s\"" %
                             (self.initial_values, value))
 
+            # Make sure the input visible beofre sending keys
+            self.wait_until_clickable("#layer-git input[type=text]")
             inputs.send_keys("-edited")
 
         # Save the new values
         for save_btn in self.find_all(".change-btn"):
             save_btn.click()
 
-        self.click("#save-changes-for-switch")
+        self.wait_until_visible("#save-changes-for-switch")
+        # Ensure scrolled into view
+        self.driver.execute_script('window.scrollTo({behavior: "instant", top: 0, left: 0})')
+        btn_save_chg_for_switch = self.wait_until_clickable(
+            "#save-changes-for-switch")
+        btn_save_chg_for_switch.click()
+
         self.wait_until_visible("#edit-layer-source")
 
         # Refresh the page to see if the new values are returned
@@ -132,7 +142,11 @@ class TestLayerDetailsPage(SeleniumTestCase):
         new_dir = "/home/test/my-meta-dir"
         dir_input.send_keys(new_dir)
 
-        self.click("#save-changes-for-switch")
+        self.wait_until_visible("#save-changes-for-switch")
+        btn_save_chg_for_switch = self.wait_until_clickable(
+            "#save-changes-for-switch")
+        btn_save_chg_for_switch.click()
+
         self.wait_until_visible("#edit-layer-source")
 
         # Refresh the page to see if the new values are returned
@@ -142,6 +156,7 @@ class TestLayerDetailsPage(SeleniumTestCase):
                         "Expected %s in the dir value for layer directory" %
                         new_dir)
 
+
     def test_delete_layer(self):
         """ Delete the layer """
 
@@ -178,6 +193,7 @@ class TestLayerDetailsPage(SeleniumTestCase):
         self.get(self.url)
 
         # Add the layer
+        self.wait_until_clickable("#add-remove-layer-btn")
         self.click("#add-remove-layer-btn")
 
         notification = self.wait_until_visible("#change-notification-msg")
@@ -185,12 +201,17 @@ class TestLayerDetailsPage(SeleniumTestCase):
         expected_text = "You have added 1 layer to your project: %s" % \
             self.imported_layer_version.layer.name
 
-        self.assertTrue(expected_text in notification.text,
+        self.assertIn(expected_text, notification.text,
                         "Expected notification text %s not found was "
                         " \"%s\" instead" %
                         (expected_text, notification.text))
 
+        hide_button = self.find('#hide-alert')
+        hide_button.click()
+        self.wait_until_not_visible('#change-notification')
+
         # Remove the layer
+        self.wait_until_clickable("#add-remove-layer-btn")
         self.click("#add-remove-layer-btn")
 
         notification = self.wait_until_visible("#change-notification-msg")
@@ -198,7 +219,7 @@ class TestLayerDetailsPage(SeleniumTestCase):
         expected_text = "You have removed 1 layer from your project: %s" % \
             self.imported_layer_version.layer.name
 
-        self.assertTrue(expected_text in notification.text,
+        self.assertIn(expected_text, notification.text,
                         "Expected notification text %s not found was "
                         " \"%s\" instead" %
                         (expected_text, notification.text))
diff --git a/bitbake/lib/toaster/tests/browser/test_most_recent_builds_states.py b/bitbake/lib/toaster/tests/browser/test_most_recent_builds_states.py
index 7844aaa395..d7a4c34532 100644
--- a/bitbake/lib/toaster/tests/browser/test_most_recent_builds_states.py
+++ b/bitbake/lib/toaster/tests/browser/test_most_recent_builds_states.py
@@ -6,7 +6,6 @@
 #
 # Copyright (C) 2013-2016 Intel Corporation
 #
-
 from django.urls import reverse
 from django.utils import timezone
 from tests.browser.selenium_helpers import SeleniumTestCase
@@ -14,6 +13,8 @@ from tests.browser.selenium_helpers_base import Wait
 from orm.models import Project, Build, Task, Recipe, Layer, Layer_Version
 from bldcontrol.models import BuildRequest
 
+from selenium.webdriver.common.by import By
+
 class TestMostRecentBuildsStates(SeleniumTestCase):
     """ Test states update correctly in most recent builds area """
 
@@ -45,13 +46,14 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         # build queued; check shown as queued
         selector = base_selector + '[data-build-state="Queued"]'
         element = self.wait_until_visible(selector)
-        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+        self.assertRegex(element.get_attribute('innerHTML'),
             'Build queued', 'build should show queued status')
 
         # waiting for recipes to be parsed
         build.outcome = Build.IN_PROGRESS
         build.recipes_to_parse = recipes_to_parse
         build.recipes_parsed = 0
+        build.save()
 
         build_request.state = BuildRequest.REQ_INPROGRESS
         build_request.save()
@@ -62,7 +64,7 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         element = self.wait_until_visible(selector)
 
         bar_selector = '#recipes-parsed-percentage-bar-%s' % build.id
-        bar_element = element.find_element_by_css_selector(bar_selector)
+        bar_element = element.find_element(By.CSS_SELECTOR, bar_selector)
         self.assertEqual(bar_element.value_of_css_property('width'), '0px',
             'recipe parse progress should be at 0')
 
@@ -73,7 +75,7 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         self.get(url)
 
         element = self.wait_until_visible(selector)
-        bar_element = element.find_element_by_css_selector(bar_selector)
+        bar_element = element.find_element(By.CSS_SELECTOR, bar_selector)
         recipe_bar_updated = lambda driver: \
             bar_element.get_attribute('style') == 'width: 50%;'
         msg = 'recipe parse progress bar should update to 50%'
@@ -94,11 +96,11 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
 
         selector = base_selector + '[data-build-state="Starting"]'
         element = self.wait_until_visible(selector)
-        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+        self.assertRegex(element.get_attribute('innerHTML'),
             'Tasks starting', 'build should show "tasks starting" status')
 
         # first task finished; check tasks progress bar
-        task1.order = 1
+        task1.outcome = Task.OUTCOME_SUCCESS
         task1.save()
 
         self.get(url)
@@ -107,7 +109,7 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         element = self.wait_until_visible(selector)
 
         bar_selector = '#build-pc-done-bar-%s' % build.id
-        bar_element = element.find_element_by_css_selector(bar_selector)
+        bar_element = element.find_element(By.CSS_SELECTOR, bar_selector)
 
         task_bar_updated = lambda driver: \
             bar_element.get_attribute('style') == 'width: 50%;'
@@ -115,13 +117,13 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         element = Wait(self.driver).until(task_bar_updated, msg)
 
         # last task finished; check tasks progress bar updates
-        task2.order = 2
+        task2.outcome = Task.OUTCOME_SUCCESS
         task2.save()
 
         self.get(url)
 
         element = self.wait_until_visible(selector)
-        bar_element = element.find_element_by_css_selector(bar_selector)
+        bar_element = element.find_element(By.CSS_SELECTOR, bar_selector)
         task_bar_updated = lambda driver: \
             bar_element.get_attribute('style') == 'width: 100%;'
         msg = 'tasks progress bar should update to 100%'
@@ -183,7 +185,7 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         selector = '[data-latest-build-result="%s"] ' \
             '[data-build-state="Cancelling"]' % build.id
         element = self.wait_until_visible(selector)
-        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+        self.assertRegex(element.get_attribute('innerHTML'),
             'Cancelling the build', 'build should show "cancelling" status')
 
         # check cancelled state
@@ -195,5 +197,5 @@ class TestMostRecentBuildsStates(SeleniumTestCase):
         selector = '[data-latest-build-result="%s"] ' \
             '[data-build-state="Cancelled"]' % build.id
         element = self.wait_until_visible(selector)
-        self.assertRegexpMatches(element.get_attribute('innerHTML'),
+        self.assertRegex(element.get_attribute('innerHTML'),
             'Build cancelled', 'build should show "cancelled" status')
diff --git a/bitbake/lib/toaster/tests/browser/test_new_custom_image_page.py b/bitbake/lib/toaster/tests/browser/test_new_custom_image_page.py
index 9906ae42a9..bf0304dbec 100644
--- a/bitbake/lib/toaster/tests/browser/test_new_custom_image_page.py
+++ b/bitbake/lib/toaster/tests/browser/test_new_custom_image_page.py
@@ -6,6 +6,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
+from bldcontrol.models import BuildEnvironment
 
 from django.urls import reverse
 from tests.browser.selenium_helpers import SeleniumTestCase
@@ -18,6 +19,9 @@ class TestNewCustomImagePage(SeleniumTestCase):
     CUSTOM_IMAGE_NAME = 'roopa-doopa'
 
     def setUp(self):
+        BuildEnvironment.objects.get_or_create(
+            betype=BuildEnvironment.TYPE_LOCAL,
+        )
         release = Release.objects.create(
             name='baz',
             bitbake_version=BitbakeVersion.objects.create(name='v1')
@@ -41,11 +45,16 @@ class TestNewCustomImagePage(SeleniumTestCase):
         )
 
         # add a fake image recipe to the layer that can be customised
+        builldir = os.environ.get('BUILDDIR', './')
         self.recipe = Recipe.objects.create(
             name='core-image-minimal',
             layer_version=layer_version,
+            file_path=f'{builldir}/core-image-minimal.bb',
             is_image=True
         )
+        # create a tmp file for the recipe
+        with open(self.recipe.file_path, 'w') as f:
+            f.write('foo')
 
         # another project with a custom image already in it
         project2 = Project.objects.create(name='whoop', release=release)
@@ -81,6 +90,7 @@ class TestNewCustomImagePage(SeleniumTestCase):
         """
         url = reverse('newcustomimage', args=(self.project.id,))
         self.get(url)
+        self.wait_until_visible('#global-nav')
 
         self.click('button[data-recipe="%s"]' % self.recipe.id)
 
@@ -128,7 +138,7 @@ class TestNewCustomImagePage(SeleniumTestCase):
         """
         self._create_custom_image(self.recipe.name)
         element = self.wait_until_visible('#invalid-name-help')
-        self.assertRegexpMatches(element.text.strip(),
+        self.assertRegex(element.text.strip(),
                                  'image with this name already exists')
 
     def test_new_duplicates_project_image(self):
@@ -146,4 +156,4 @@ class TestNewCustomImagePage(SeleniumTestCase):
         self._create_custom_image(custom_image_name)
         element = self.wait_until_visible('#invalid-name-help')
         expected = 'An image with this name already exists in this project'
-        self.assertRegexpMatches(element.text.strip(), expected)
+        self.assertRegex(element.text.strip(), expected)
diff --git a/bitbake/lib/toaster/tests/browser/test_new_project_page.py b/bitbake/lib/toaster/tests/browser/test_new_project_page.py
index e20a1f686e..e50f236c32 100644
--- a/bitbake/lib/toaster/tests/browser/test_new_project_page.py
+++ b/bitbake/lib/toaster/tests/browser/test_new_project_page.py
@@ -6,11 +6,11 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
-
 from django.urls import reverse
 from tests.browser.selenium_helpers import SeleniumTestCase
 from selenium.webdriver.support.ui import Select
 from selenium.common.exceptions import InvalidElementStateException
+from selenium.webdriver.common.by import By
 
 from orm.models import Project, Release, BitbakeVersion
 
@@ -47,7 +47,7 @@ class TestNewProjectPage(SeleniumTestCase):
 
         url = reverse('newproject')
         self.get(url)
-
+        self.wait_until_visible('#new-project-name')
         self.enter_text('#new-project-name', project_name)
 
         select = Select(self.find('#projectversion'))
@@ -57,7 +57,8 @@ class TestNewProjectPage(SeleniumTestCase):
 
         # We should get redirected to the new project's page with the
         # notification at the top
-        element = self.wait_until_visible('#project-created-notification')
+        element = self.wait_until_visible(
+            '#project-created-notification')
 
         self.assertTrue(project_name in element.text,
                         "New project name not in new project notification")
@@ -78,15 +79,20 @@ class TestNewProjectPage(SeleniumTestCase):
 
         url = reverse('newproject')
         self.get(url)
+        self.wait_until_visible('#new-project-name')
 
         self.enter_text('#new-project-name', project_name)
 
         select = Select(self.find('#projectversion'))
         select.select_by_value(str(self.release.pk))
 
-        element = self.wait_until_visible('#hint-error-project-name')
+        radio = self.driver.find_element(By.ID, 'type-new')
+        radio.click()
+
+        self.wait_until_visible('#hint-error-project-name')
+        element = self.find('#hint-error-project-name')
 
-        self.assertTrue(("Project names must be unique" in element.text),
+        self.assertIn("Project names must be unique", element.text,
                         "Did not find unique project name error message")
 
         # Try and click it anyway, if it submits we'll have a new project in
diff --git a/bitbake/lib/toaster/tests/browser/test_project_builds_page.py b/bitbake/lib/toaster/tests/browser/test_project_builds_page.py
index 51717e72d4..0dba33b9c8 100644
--- a/bitbake/lib/toaster/tests/browser/test_project_builds_page.py
+++ b/bitbake/lib/toaster/tests/browser/test_project_builds_page.py
@@ -7,6 +7,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import os
 import re
 
 from django.urls import reverse
@@ -22,7 +23,8 @@ class TestProjectBuildsPage(SeleniumTestCase):
     CLI_BUILDS_PROJECT_NAME = 'command line builds'
 
     def setUp(self):
-        bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='bbv1', giturl=f'{builldir}/',
                                             branch='master', dirpath='')
         release = Release.objects.create(name='release1',
                                          bitbake_version=bbv)
diff --git a/bitbake/lib/toaster/tests/browser/test_project_config_page.py b/bitbake/lib/toaster/tests/browser/test_project_config_page.py
index 944bcb2631..b9de541efa 100644
--- a/bitbake/lib/toaster/tests/browser/test_project_config_page.py
+++ b/bitbake/lib/toaster/tests/browser/test_project_config_page.py
@@ -7,10 +7,12 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import os
 from django.urls import reverse
 from tests.browser.selenium_helpers import SeleniumTestCase
 
 from orm.models import BitbakeVersion, Release, Project, ProjectVariable
+from selenium.webdriver.common.by import By
 
 class TestProjectConfigsPage(SeleniumTestCase):
     """ Test data at /project/X/builds is displayed correctly """
@@ -21,7 +23,8 @@ class TestProjectConfigsPage(SeleniumTestCase):
         'any of these characters'
 
     def setUp(self):
-        bbv = BitbakeVersion.objects.create(name='bbv1', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='bbv1', giturl=f'{builldir}/',
                                             branch='master', dirpath='')
         release = Release.objects.create(name='release1',
                                          bitbake_version=bbv)
@@ -66,7 +69,7 @@ class TestProjectConfigsPage(SeleniumTestCase):
 
         self.enter_text('#new-imagefs_types', imagefs_type)
 
-        checkboxes = self.driver.find_elements_by_xpath("//input[@class='fs-checkbox-fstypes']")
+        checkboxes = self.driver.find_elements(By.XPATH, "//input[@class='fs-checkbox-fstypes']")
 
         for checkbox in checkboxes:
             if checkbox.get_attribute("value") == "btrfs":
@@ -95,7 +98,7 @@ class TestProjectConfigsPage(SeleniumTestCase):
         for checkbox in checkboxes:
             if checkbox.get_attribute("value") == "cpio":
                checkbox.click()
-               element = self.driver.find_element_by_id('new-imagefs_types')
+               element = self.driver.find_element(By.ID, 'new-imagefs_types')
 
                self.wait_until_visible('#new-imagefs_types')
 
@@ -129,7 +132,7 @@ class TestProjectConfigsPage(SeleniumTestCase):
         self.assertTrue((self.INVALID_PATH_START_TEXT in element.text), msg)
 
         # downloads dir path has a space
-        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.driver.find_element(By.ID, 'new-dl_dir').clear()
         self.enter_text('#new-dl_dir', '/foo/bar a')
 
         element = self.wait_until_visible('#hintError-dl_dir')
@@ -137,7 +140,7 @@ class TestProjectConfigsPage(SeleniumTestCase):
         self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
 
         # downloads dir path starts with ${...} but has a space
-        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.driver.find_element(By.ID,'new-dl_dir').clear()
         self.enter_text('#new-dl_dir', '${TOPDIR}/down foo')
 
         element = self.wait_until_visible('#hintError-dl_dir')
@@ -145,18 +148,18 @@ class TestProjectConfigsPage(SeleniumTestCase):
         self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
 
         # downloads dir path starts with /
-        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.driver.find_element(By.ID,'new-dl_dir').clear()
         self.enter_text('#new-dl_dir', '/bar/foo')
 
-        hidden_element = self.driver.find_element_by_id('hintError-dl_dir')
+        hidden_element = self.driver.find_element(By.ID,'hintError-dl_dir')
         self.assertEqual(hidden_element.is_displayed(), False,
             'downloads directory path valid but treated as invalid')
 
         # downloads dir path starts with ${...}
-        self.driver.find_element_by_id('new-dl_dir').clear()
+        self.driver.find_element(By.ID,'new-dl_dir').clear()
         self.enter_text('#new-dl_dir', '${TOPDIR}/down')
 
-        hidden_element = self.driver.find_element_by_id('hintError-dl_dir')
+        hidden_element = self.driver.find_element(By.ID,'hintError-dl_dir')
         self.assertEqual(hidden_element.is_displayed(), False,
             'downloads directory path valid but treated as invalid')
 
@@ -184,7 +187,7 @@ class TestProjectConfigsPage(SeleniumTestCase):
         self.assertTrue((self.INVALID_PATH_START_TEXT in element.text), msg)
 
         # path has a space
-        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.driver.find_element(By.ID, 'new-sstate_dir').clear()
         self.enter_text('#new-sstate_dir', '/foo/bar a')
 
         element = self.wait_until_visible('#hintError-sstate_dir')
@@ -192,7 +195,7 @@ class TestProjectConfigsPage(SeleniumTestCase):
         self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
 
         # path starts with ${...} but has a space
-        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.driver.find_element(By.ID,'new-sstate_dir').clear()
         self.enter_text('#new-sstate_dir', '${TOPDIR}/down foo')
 
         element = self.wait_until_visible('#hintError-sstate_dir')
@@ -200,18 +203,18 @@ class TestProjectConfigsPage(SeleniumTestCase):
         self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
 
         # path starts with /
-        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.driver.find_element(By.ID,'new-sstate_dir').clear()
         self.enter_text('#new-sstate_dir', '/bar/foo')
 
-        hidden_element = self.driver.find_element_by_id('hintError-sstate_dir')
+        hidden_element = self.driver.find_element(By.ID, 'hintError-sstate_dir')
         self.assertEqual(hidden_element.is_displayed(), False,
             'sstate directory path valid but treated as invalid')
 
         # paths starts with ${...}
-        self.driver.find_element_by_id('new-sstate_dir').clear()
+        self.driver.find_element(By.ID, 'new-sstate_dir').clear()
         self.enter_text('#new-sstate_dir', '${TOPDIR}/down')
 
-        hidden_element = self.driver.find_element_by_id('hintError-sstate_dir')
+        hidden_element = self.driver.find_element(By.ID, 'hintError-sstate_dir')
         self.assertEqual(hidden_element.is_displayed(), False,
             'sstate directory path valid but treated as invalid')
 
diff --git a/bitbake/lib/toaster/tests/browser/test_sample.py b/bitbake/lib/toaster/tests/browser/test_sample.py
index b0067c21cd..f04f1d9a16 100644
--- a/bitbake/lib/toaster/tests/browser/test_sample.py
+++ b/bitbake/lib/toaster/tests/browser/test_sample.py
@@ -27,3 +27,13 @@ class TestSample(SeleniumTestCase):
         self.get(url)
         brand_link = self.find('.toaster-navbar-brand a.brand')
         self.assertEqual(brand_link.text.strip(), 'Toaster')
+
+    def test_no_builds_message(self):
+        """ Test that a message is shown when there are no builds """
+        url = reverse('all-builds')
+        self.get(url)
+        self.wait_until_visible('#empty-state-allbuildstable')  # wait for the empty state div to appear
+        div_msg = self.find('#empty-state-allbuildstable .alert-info')
+
+        msg = 'Sorry - no data found'
+        self.assertEqual(div_msg.text, msg)
diff --git a/bitbake/lib/toaster/tests/browser/test_toastertable_ui.py b/bitbake/lib/toaster/tests/browser/test_toastertable_ui.py
index e82d5ec654..691aca1ef0 100644
--- a/bitbake/lib/toaster/tests/browser/test_toastertable_ui.py
+++ b/bitbake/lib/toaster/tests/browser/test_toastertable_ui.py
@@ -8,11 +8,13 @@
 #
 
 from datetime import datetime
+import os
 
 from django.urls import reverse
 from django.utils import timezone
 from tests.browser.selenium_helpers import SeleniumTestCase
 from orm.models import BitbakeVersion, Release, Project, Build
+from selenium.webdriver.common.by import By
 
 class TestToasterTableUI(SeleniumTestCase):
     """
@@ -33,7 +35,7 @@ class TestToasterTableUI(SeleniumTestCase):
         table: WebElement for a ToasterTable
         """
         selector = 'thead a.sorted'
-        heading = table.find_element_by_css_selector(selector)
+        heading = table.find_element(By.CSS_SELECTOR, selector)
         return heading.get_attribute('innerHTML').strip()
 
     def _get_datetime_from_cell(self, row, selector):
@@ -45,7 +47,7 @@ class TestToasterTableUI(SeleniumTestCase):
         selector: CSS selector to use to find the cell containing the date time
         string
         """
-        cell = row.find_element_by_css_selector(selector)
+        cell = row.find_element(By.CSS_SELECTOR, selector)
         cell_text = cell.get_attribute('innerHTML').strip()
         return datetime.strptime(cell_text, '%d/%m/%y %H:%M')
 
@@ -58,7 +60,8 @@ class TestToasterTableUI(SeleniumTestCase):
         later = now + timezone.timedelta(hours=1)
         even_later = later + timezone.timedelta(hours=1)
 
-        bbv = BitbakeVersion.objects.create(name='test bbv', giturl='/tmp/',
+        builldir = os.environ.get('BUILDDIR', './')
+        bbv = BitbakeVersion.objects.create(name='test bbv', giturl=f'{builldir}/',
                                             branch='master', dirpath='')
         release = Release.objects.create(name='test release',
                                          branch_name='master',
@@ -105,7 +108,7 @@ class TestToasterTableUI(SeleniumTestCase):
         self.click('#checkbox-started_on')
 
         # sort by started_on column
-        links = table.find_elements_by_css_selector('th.started_on a')
+        links = table.find_elements(By.CSS_SELECTOR, 'th.started_on a')
         for link in links:
             if link.get_attribute('innerHTML').strip() == 'Started on':
                 link.click()
diff --git a/bitbake/lib/toaster/tests/builds/buildtest.py b/bitbake/lib/toaster/tests/builds/buildtest.py
index 872bbd3775..e54d561334 100644
--- a/bitbake/lib/toaster/tests/builds/buildtest.py
+++ b/bitbake/lib/toaster/tests/builds/buildtest.py
@@ -88,7 +88,7 @@ def load_build_environment():
 class BuildTest(unittest.TestCase):
 
     PROJECT_NAME = "Testbuild"
-    BUILDDIR = "/tmp/build/"
+    BUILDDIR = os.environ.get("BUILDDIR")
 
     def build(self, target):
         # So that the buildinfo helper uses the test database'
@@ -116,10 +116,19 @@ class BuildTest(unittest.TestCase):
         project = Project.objects.create_project(name=BuildTest.PROJECT_NAME,
                                                  release=release)
 
+        passthrough_variable_names = ["SSTATE_DIR", "DL_DIR", "SSTATE_MIRRORS", "BB_HASHSERVE", "BB_HASHSERVE_UPSTREAM"]
+        for variable_name in passthrough_variable_names:
+            current_variable = os.environ.get(variable_name)
+            if current_variable:
+                ProjectVariable.objects.get_or_create(
+                    name=variable_name,
+                    value=current_variable,
+                    project=project)
+
         if os.environ.get("TOASTER_TEST_USE_SSTATE_MIRROR"):
             ProjectVariable.objects.get_or_create(
                 name="SSTATE_MIRRORS",
-                value="file://.* http://autobuilder.yoctoproject.org/pub/sstate/PATH;downloadfilename=PATH",
+                value="file://.* http://sstate.yoctoproject.org/all/PATH;downloadfilename=PATH",
                 project=project)
 
         ProjectTarget.objects.create(project=project,
diff --git a/bitbake/lib/toaster/tests/builds/test_core_image_min.py b/bitbake/lib/toaster/tests/builds/test_core_image_min.py
index 44b6cbec7b..c5bfdbfbb5 100644
--- a/bitbake/lib/toaster/tests/builds/test_core_image_min.py
+++ b/bitbake/lib/toaster/tests/builds/test_core_image_min.py
@@ -10,6 +10,7 @@
 # Ionut Chisanovici, Paul Eggleton and Cristian Iorga
 
 import os
+import pytest
 
 from django.db.models import Q
 
@@ -20,12 +21,13 @@ from orm.models import CustomImagePackage
 
 from tests.builds.buildtest import BuildTest
 
-
+@pytest.mark.order(4)
+@pytest.mark.django_db(True)
 class BuildCoreImageMinimal(BuildTest):
     """Build core-image-minimal and test the results"""
 
     def setUp(self):
-        self.completed_build = self.build("core-image-minimal")
+        self.completed_build = self.target_already_built("core-image-minimal")
 
     # Check if build name is unique - tc_id=795
     def test_Build_Unique_Name(self):
@@ -44,17 +46,6 @@ class BuildCoreImageMinimal(BuildTest):
                          total_builds,
                          msg='Build cooker log path is not unique')
 
-    # Check if task order is unique for one build - tc=824
-    def test_Task_Unique_Order(self):
-        total_task_order = Task.objects.filter(
-            build=self.built).values('order').count()
-        distinct_task_order = Task.objects.filter(
-            build=self.completed_build).values('order').distinct().count()
-
-        self.assertEqual(total_task_order,
-                         distinct_task_order,
-                         msg='Errors task order is not unique')
-
     # Check task order sequence for one build - tc=825
     def test_Task_Order_Sequence(self):
         cnt_err = []
@@ -98,7 +89,6 @@ class BuildCoreImageMinimal(BuildTest):
                                                         'task_name',
                                                         'sstate_result')
         cnt_err = []
-
         for task in tasks:
             if (task['sstate_result'] != Task.SSTATE_NA and
                     task['sstate_result'] != Task.SSTATE_MISS):
@@ -221,6 +211,7 @@ class BuildCoreImageMinimal(BuildTest):
     # orm_build.outcome=0 then if the file exists and its size matches
     # the file_size value. Need to add the tc in the test run
     def test_Target_File_Name_Populated(self):
+        cnt_err = []
         builds = Build.objects.filter(outcome=0).values('id')
         for build in builds:
             targets = Target.objects.filter(
@@ -230,7 +221,6 @@ class BuildCoreImageMinimal(BuildTest):
                     target_id=target['id']).values('id',
                                                    'file_name',
                                                    'file_size')
-                cnt_err = []
                 for file_info in target_files:
                     target_id = file_info['id']
                     target_file_name = file_info['file_name']
diff --git a/bitbake/lib/toaster/tests/commands/test_loaddata.py b/bitbake/lib/toaster/tests/commands/test_loaddata.py
index 9e8d5553cf..7d04f030ee 100644
--- a/bitbake/lib/toaster/tests/commands/test_loaddata.py
+++ b/bitbake/lib/toaster/tests/commands/test_loaddata.py
@@ -6,13 +6,13 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
-
+import pytest
 from django.test import TestCase
 from django.core import management
 
 from orm.models import Layer_Version, Layer, Release, ToasterSetting
 
-
+@pytest.mark.order(2)
 class TestLoadDataFixtures(TestCase):
     """ Test loading our 3 provided fixtures """
     def test_run_loaddata_poky_command(self):
diff --git a/bitbake/lib/toaster/tests/commands/test_lsupdates.py b/bitbake/lib/toaster/tests/commands/test_lsupdates.py
index 3c4fbe0550..30c6eeb4ac 100644
--- a/bitbake/lib/toaster/tests/commands/test_lsupdates.py
+++ b/bitbake/lib/toaster/tests/commands/test_lsupdates.py
@@ -7,12 +7,13 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import pytest
 from django.test import TestCase
 from django.core import management
 
 from orm.models import Layer_Version, Machine, Recipe
 
-
+@pytest.mark.order(3)
 class TestLayerIndexUpdater(TestCase):
     def test_run_lsupdates_command(self):
         # Load some release information for us to fetch from the layer index
diff --git a/bitbake/lib/toaster/tests/commands/test_runbuilds.py b/bitbake/lib/toaster/tests/commands/test_runbuilds.py
index e223b95fcb..849c227edc 100644
--- a/bitbake/lib/toaster/tests/commands/test_runbuilds.py
+++ b/bitbake/lib/toaster/tests/commands/test_runbuilds.py
@@ -19,12 +19,14 @@ import time
 import subprocess
 import signal
 
+import logging
+
 
 class KillRunbuilds(threading.Thread):
     """ Kill the runbuilds process after an amount of time """
     def __init__(self, *args, **kwargs):
         super(KillRunbuilds, self).__init__(*args, **kwargs)
-        self.setDaemon(True)
+        self.daemon = True
 
     def run(self):
         time.sleep(5)
@@ -34,9 +36,12 @@ class KillRunbuilds(threading.Thread):
         pidfile_path = os.path.join(os.environ.get("BUILDDIR", "."),
                                     ".runbuilds.pid")
 
-        with open(pidfile_path) as pidfile:
-            pid = pidfile.read()
-            os.kill(int(pid), signal.SIGTERM)
+        try:
+            with open(pidfile_path) as pidfile:
+                pid = pidfile.read()
+                os.kill(int(pid), signal.SIGTERM)
+        except ProcessLookupError:
+            logging.warning("Runbuilds not running or already killed")
 
 
 class TestCommands(TestCase):
diff --git a/bitbake/lib/toaster/tests/db/test_db.py b/bitbake/lib/toaster/tests/db/test_db.py
index 0410422276..072ab94363 100644
--- a/bitbake/lib/toaster/tests/db/test_db.py
+++ b/bitbake/lib/toaster/tests/db/test_db.py
@@ -23,6 +23,7 @@
 # SOFTWARE.
 
 import sys
+import pytest
 
 try:
     from StringIO import StringIO
@@ -47,7 +48,7 @@ def capture(command, *args, **kwargs):
 def makemigrations():
     management.call_command('makemigrations')
 
-
+@pytest.mark.order(1)
 class MigrationTest(TestCase):
 
     def testPendingMigration(self):
diff --git a/bitbake/lib/toaster/tests/functional/functional_helpers.py b/bitbake/lib/toaster/tests/functional/functional_helpers.py
index 5c4ea71794..e28f2024f5 100644
--- a/bitbake/lib/toaster/tests/functional/functional_helpers.py
+++ b/bitbake/lib/toaster/tests/functional/functional_helpers.py
@@ -11,35 +11,58 @@ import os
 import logging
 import subprocess
 import signal
-import time
 import re
+import requests
 
+from django.urls import reverse
 from tests.browser.selenium_helpers_base import SeleniumTestCaseBase
-from tests.builds.buildtest import load_build_environment
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.select import Select
+from selenium.common.exceptions import NoSuchElementException
 
 logger = logging.getLogger("toaster")
+toaster_processes = []
 
 class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
-    wait_toaster_time = 5
+    wait_toaster_time = 10
 
     @classmethod
     def setUpClass(cls):
         # So that the buildinfo helper uses the test database'
         if os.environ.get('DJANGO_SETTINGS_MODULE', '') != \
             'toastermain.settings_test':
-            raise RuntimeError("Please initialise django with the tests settings:  " \
+            raise RuntimeError("Please initialise django with the tests settings:  "
                 "DJANGO_SETTINGS_MODULE='toastermain.settings_test'")
 
-        load_build_environment()
+        # Wait for any known toaster processes to exit
+        global toaster_processes
+        for toaster_process in toaster_processes:
+            try:
+                os.waitpid(toaster_process, os.WNOHANG)
+            except ChildProcessError:
+                pass
 
         # start toaster
         cmd = "bash -c 'source toaster start'"
-        p = subprocess.Popen(
+        start_process = subprocess.Popen(
             cmd,
             cwd=os.environ.get("BUILDDIR"),
             shell=True)
-        if p.wait() != 0:
-            raise RuntimeError("Can't initialize toaster")
+        toaster_processes = [start_process.pid]
+        if start_process.wait() != 0:
+            port_use = os.popen("lsof -i -P -n | grep '8000 (LISTEN)'").read().strip()
+            message = ''
+            if port_use:
+                process_id = port_use.split()[1]
+                process = os.popen(f"ps -o cmd= -p {process_id}").read().strip()
+                message = f"Port 8000 occupied by {process}"
+            raise RuntimeError(f"Can't initialize toaster. {message}")
+
+        builddir = os.environ.get("BUILDDIR")
+        with open(os.path.join(builddir, '.toastermain.pid'), 'r') as f:
+            toaster_processes.append(int(f.read()))
+        with open(os.path.join(builddir, '.runbuilds.pid'), 'r') as f:
+            toaster_processes.append(int(f.read()))
 
         super(SeleniumFunctionalTestCase, cls).setUpClass()
         cls.live_server_url = 'http://localhost:8000/'
@@ -48,22 +71,30 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
     def tearDownClass(cls):
         super(SeleniumFunctionalTestCase, cls).tearDownClass()
 
-        # XXX: source toaster stop gets blocked, to review why?
-        # from now send SIGTERM by hand
-        time.sleep(cls.wait_toaster_time)
-        builddir = os.environ.get("BUILDDIR")
+        global toaster_processes
 
-        with open(os.path.join(builddir, '.toastermain.pid'), 'r') as f:
-            toastermain_pid = int(f.read())
-            os.kill(toastermain_pid, signal.SIGTERM)
-        with open(os.path.join(builddir, '.runbuilds.pid'), 'r') as f:
-            runbuilds_pid = int(f.read())
-            os.kill(runbuilds_pid, signal.SIGTERM)
+        cmd = "bash -c 'source toaster stop'"
+        stop_process = subprocess.Popen(
+            cmd,
+            cwd=os.environ.get("BUILDDIR"),
+            shell=True)
+        # Toaster stop has been known to hang in these tests so force kill if it stalls
+        try:
+            if stop_process.wait(cls.wait_toaster_time) != 0:
+                raise Exception('Toaster stop process failed')
+        except Exception as e:
+            if e is subprocess.TimeoutExpired:
+                print('Toaster stop process took too long. Force killing toaster...')
+            else:
+                print('Toaster stop process failed. Force killing toaster...')
+            stop_process.kill()
+            for toaster_process in toaster_processes:
+                os.kill(toaster_process, signal.SIGTERM)
 
 
     def get_URL(self):
          rc=self.get_page_source()
-         project_url=re.search("(projectPageUrl\s:\s\")(.*)(\",)",rc)
+         project_url=re.search(r"(projectPageUrl\s:\s\")(.*)(\",)",rc)
          return project_url.group(2)
 
 
@@ -74,8 +105,8 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
         """
         try:
             table_element = self.get_table_element(table_id)
-            element = table_element.find_element_by_link_text(link_text)
-        except self.NoSuchElementException:
+            element = table_element.find_element(By.LINK_TEXT, link_text)
+        except NoSuchElementException:
             print('no element found')
             raise
         return element
@@ -85,8 +116,8 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
 #return whole-table element
             element_xpath = "//*[@id='" + table_id + "']"
             try:
-                element = self.driver.find_element_by_xpath(element_xpath)
-            except self.NoSuchElementException:
+                element = self.driver.find_element(By.XPATH, element_xpath)
+            except NoSuchElementException:
                 raise
             return element
         row = coordinate[0]
@@ -95,8 +126,8 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
 #return whole-row element
             element_xpath = "//*[@id='" + table_id + "']/tbody/tr[" + str(row) + "]"
             try:
-                element = self.driver.find_element_by_xpath(element_xpath)
-            except self.NoSuchElementException:
+                element = self.driver.find_element(By.XPATH, element_xpath)
+            except NoSuchElementException:
                 return False
             return element
 #now we are looking for an element with specified X and Y
@@ -104,7 +135,90 @@ class SeleniumFunctionalTestCase(SeleniumTestCaseBase):
 
         element_xpath = "//*[@id='" + table_id + "']/tbody/tr[" + str(row) + "]/td[" + str(column) + "]"
         try:
-            element = self.driver.find_element_by_xpath(element_xpath)
-        except self.NoSuchElementException:
+            element = self.driver.find_element(By.XPATH, element_xpath)
+        except NoSuchElementException:
             return False
         return element
+
+    def create_new_project(
+        self,
+        project_name,
+        release,
+        release_title,
+        merge_toaster_settings,
+    ):
+        """ Create/Test new project using:
+          - Project Name: Any string
+          - Release: Any string
+          - Merge Toaster settings: True or False
+        """
+
+        # Obtain a CSRF token from a suitable URL
+        projs = requests.get(self.live_server_url + reverse('newproject'))
+        csrftoken = projs.cookies.get('csrftoken')
+
+        # Use the projects typeahead to find out if the project already exists
+        req = requests.get(self.live_server_url + reverse('xhr_projectstypeahead'), {'search': project_name, 'format' : 'json'})
+        data = req.json()
+        # Delete any existing projects
+        for result in data['results']:
+            del_url = reverse('xhr_project', args=(result['id'],))
+            del_response = requests.delete(self.live_server_url + del_url, cookies={'csrftoken': csrftoken}, headers={'X-CSRFToken': csrftoken})
+            self.assertEqual(del_response.status_code, 200)
+
+        self.get(reverse('newproject'))
+        self.wait_until_visible('#new-project-name')
+        self.driver.find_element(By.ID,
+                                 "new-project-name").send_keys(project_name)
+
+        select = Select(self.find('#projectversion'))
+        select.select_by_value(release)
+
+        # check merge toaster settings
+        checkbox = self.find('.checkbox-mergeattr')
+        if merge_toaster_settings:
+            if not checkbox.is_selected():
+                checkbox.click()
+        else:
+            if checkbox.is_selected():
+                checkbox.click()
+
+        self.wait_until_clickable('#create-project-button')
+
+        self.driver.find_element(By.ID, "create-project-button").click()
+
+        element = self.wait_until_visible('#project-created-notification')
+        self.assertTrue(
+            self.element_exists('#project-created-notification'),
+            f"Project:{project_name} creation notification not shown"
+        )
+        self.assertTrue(
+            project_name in element.text,
+            f"New project name:{project_name} not in new project notification"
+        )
+
+        # Use the projects typeahead again to check the project now exists
+        req = requests.get(self.live_server_url + reverse('xhr_projectstypeahead'), {'search': project_name, 'format' : 'json'})
+        data = req.json()
+        self.assertGreater(len(data['results']), 0, f"New project:{project_name} not found in database")
+
+        project_id = data['results'][0]['id']
+
+        self.wait_until_visible('#project-release-title')
+
+        # check release
+        if release_title is not None:
+            self.assertTrue(re.search(
+                release_title,
+                self.driver.find_element(By.XPATH,
+                                     "//span[@id='project-release-title']"
+                                     ).text),
+                            'The project release is not defined')
+
+        return project_id
+        
+    def load_projects_page_helper(self):        
+        self.wait_until_present('#projectstable')
+        # Need to wait for some data in the table too
+        self.wait_until_present('td[class="updated"]')
+
diff --git a/bitbake/lib/toaster/tests/functional/test_create_new_project.py b/bitbake/lib/toaster/tests/functional/test_create_new_project.py
new file mode 100644
index 0000000000..66213c736e
--- /dev/null
+++ b/bitbake/lib/toaster/tests/functional/test_create_new_project.py
@@ -0,0 +1,124 @@
+#! /usr/bin/env python3
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import re
+import pytest
+from django.urls import reverse
+from selenium.webdriver.support.select import Select
+from tests.functional.functional_helpers import SeleniumFunctionalTestCase
+from selenium.webdriver.common.by import By
+
+class TestCreateNewProject(SeleniumFunctionalTestCase):
+
+    def test_create_new_project_master(self):
+        """ Test create new project using:
+          - Project Name: Any string
+          - Release: Yocto Project master (option value: 3)
+          - Merge Toaster settings: False
+        """
+        release = '3'
+        release_title = 'Yocto Project master'
+        project_name = 'projectmaster'
+        self.create_new_project(
+            project_name,
+            release,
+            release_title,
+            False,
+        )
+
+    def test_create_new_project_scarthgap(self):
+        """ Test create new project using:
+          - Project Name: Any string
+          - Release: Yocto Project 5.0 "Scarthgap" (option value: 1)
+          - Merge Toaster settings: True
+        """
+        release = '1'
+        release_title = 'Yocto Project 5.0 "Scarthgap"'
+        project_name = 'projectscarthgap'
+        self.create_new_project(
+            project_name,
+            release,
+            release_title,
+            True,
+        )
+
+    def test_create_new_project_kirkstone(self):
+        """ Test create new project using:
+          - Project Name: Any string
+          - Release: Yocto Project 4.0 "Kirkstone" (option value: 6)
+          - Merge Toaster settings: True
+        """
+        release = '7'
+        release_title = 'Yocto Project 4.0 "Kirkstone"'
+        project_name = 'projectkirkstone'
+        self.create_new_project(
+            project_name,
+            release,
+            release_title,
+            True,
+        )
+
+    def test_create_new_project_local(self):
+        """ Test create new project using:
+          - Project Name: Any string
+          - Release: Local Yocto Project (option value: 2)
+          - Merge Toaster settings: True
+        """
+        release = '2'
+        release_title = 'Local Yocto Project'
+        project_name = 'projectlocal'
+        self.create_new_project(
+            project_name,
+            release,
+            release_title,
+            True,
+        )
+
+    def test_create_new_project_without_name(self):
+        """ Test create new project without project name """
+        self.get(reverse('newproject'))
+
+        select = Select(self.find('#projectversion'))
+        select.select_by_value(str(3))
+
+        # Check input name has required attribute
+        input_name = self.driver.find_element(By.ID, "new-project-name")
+        self.assertIsNotNone(input_name.get_attribute('required'),
+                        'Input name has not required attribute')
+
+        # Check create button is disabled
+        create_btn = self.driver.find_element(By.ID, "create-project-button")
+        self.assertIsNotNone(create_btn.get_attribute('disabled'),
+                        'Create button is not disabled')
+
+    def test_import_new_project(self):
+        """ Test import new project using:
+          - Project Name: Any string
+          - Project type: select (Import command line project)
+          - Import existing project directory: Wrong Path
+        """
+        project_name = 'projectimport'
+        self.get(reverse('newproject'))
+        self.driver.find_element(By.ID,
+                                 "new-project-name").send_keys(project_name)
+        # select import project
+        self.find('#type-import').click()
+
+        # set wrong path
+        wrong_path = '/wrongpath'
+        self.driver.find_element(By.ID,
+                                 "import-project-dir").send_keys(wrong_path)
+        self.driver.find_element(By.ID, "create-project-button").click()
+
+        self.wait_until_visible('.alert-danger')
+
+        # check error message
+        self.assertTrue(self.element_exists('.alert-danger'),
+                        'Alert message not shown')
+        self.assertTrue(wrong_path in self.find('.alert-danger').text,
+                        "Wrong path not in alert message")
diff --git a/bitbake/lib/toaster/tests/functional/test_functional_basic.py b/bitbake/lib/toaster/tests/functional/test_functional_basic.py
index 5683e3873e..d5c9708617 100644
--- a/bitbake/lib/toaster/tests/functional/test_functional_basic.py
+++ b/bitbake/lib/toaster/tests/functional/test_functional_basic.py
@@ -8,223 +8,250 @@
 #
 
 import re
+from django.urls import reverse
+import pytest
 from tests.functional.functional_helpers import SeleniumFunctionalTestCase
 from orm.models import Project
+from selenium.webdriver.common.by import By
+
+from tests.functional.utils import get_projectId_from_url
+
 
 class FuntionalTestBasic(SeleniumFunctionalTestCase):
+    """Basic functional tests for Toaster"""
+    project_id = None
+    project_url = None
 
-#   testcase (1514)
-    def test_create_slenium_project(self):
-        project_name = 'selenium-project'
-        self.get('')
-        self.driver.find_element_by_link_text("To start building, create your first Toaster project").click()
-        self.driver.find_element_by_id("new-project-name").send_keys(project_name)
-        self.driver.find_element_by_id('projectversion').click()
-        self.driver.find_element_by_id("create-project-button").click()
-        element = self.wait_until_visible('#project-created-notification')
-        self.assertTrue(self.element_exists('#project-created-notification'),'Project creation notification not shown')
-        self.assertTrue(project_name in element.text,
-                        "New project name not in new project notification")
-        self.assertTrue(Project.objects.filter(name=project_name).count(),
-                        "New project not found in database")
+    def setUp(self):
+        super(FuntionalTestBasic, self).setUp()
+        if not FuntionalTestBasic.project_id:
+            FuntionalTestBasic.project_id = self.create_new_project('selenium-project', '3', None, False)
 
  #  testcase (1515)
     def test_verify_left_bar_menu(self):
-        self.get('')
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        self.wait_until_present('#config-nav')
         self.assertTrue(self.element_exists('#config-nav'),'Configuration Tab does not exist')
         project_URL=self.get_URL()
-        self.driver.find_element_by_xpath('//a[@href="'+project_URL+'"]').click()
+        self.driver.find_element(By.XPATH, '//a[@href="'+project_URL+'"]').click()
 
         try:
-            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'customimages/"'+"]").click()
-            self.assertTrue(re.search("Custom images",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'Custom images information is not loading properly')
+            self.wait_until_present('#config-nav')
+            self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'customimages/"'+"]").click()
+            self.wait_until_present('#filter-modal-customimagestable')
         except:
             self.fail(msg='No Custom images tab available')
+        self.assertTrue(re.search("Custom images",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'Custom images information is not loading properly')
 
         try:
-            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'images/"'+"]").click()
-            self.assertTrue(re.search("Compatible image recipes",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Compatible image recipes information is not loading properly')
+            self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'images/"'+"]").click()
+            self.wait_until_present('#filter-modal-imagerecipestable')
         except:
             self.fail(msg='No Compatible image tab available')
+        self.assertTrue(re.search("Compatible image recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible image recipes information is not loading properly')
 
         try:
-            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'softwarerecipes/"'+"]").click()
-            self.assertTrue(re.search("Compatible software recipes",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Compatible software recipe information is not loading properly')
+            self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'softwarerecipes/"'+"]").click()
+            self.wait_until_present('#filter-modal-softwarerecipestable')
         except:
             self.fail(msg='No Compatible software recipe tab available')
+        self.assertTrue(re.search("Compatible software recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible software recipe information is not loading properly')
 
         try:
-            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'machines/"'+"]").click()
-            self.assertTrue(re.search("Compatible machines",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Compatible machine information is not loading properly')
+            self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'machines/"'+"]").click()
+            self.wait_until_present('#filter-modal-machinestable')
         except:
             self.fail(msg='No Compatible machines tab available')
+        self.assertTrue(re.search("Compatible machines",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible machine information is not loading properly')
 
         try:
-            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'layers/"'+"]").click()
-            self.assertTrue(re.search("Compatible layers",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Compatible layer information is not loading properly')
+            self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'layers/"'+"]").click()
+            self.wait_until_present('#filter-modal-layerstable')
         except:
             self.fail(msg='No Compatible layers tab available')
+        self.assertTrue(re.search("Compatible layers",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Compatible layer information is not loading properly')
 
         try:
-            self.driver.find_element_by_xpath("//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'configuration"'+"]").click()
-            self.assertTrue(re.search("Bitbake variables",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Bitbake variables information is not loading properly')
+            self.driver.find_element(By.XPATH, "//*[@id='config-nav']/ul/li/a[@href="+'"'+project_URL+'configuration"'+"]").click()
+            self.wait_until_present('#configvar-list')
         except:
             self.fail(msg='No Bitbake variables tab available')
+        self.assertTrue(re.search("Bitbake variables",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Bitbake variables information is not loading properly')
 
 #   testcase (1516)
     def test_review_configuration_information(self):
-        self.get('')
-        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
         project_URL=self.get_URL()
 
+        # Machine section of page
+        self.wait_until_visible('#machine-section')
+        self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
+        self.assertTrue(re.search("qemux86-64",self.driver.find_element(By.XPATH, "//span[@id='project-machine-name']").text),'The machine type is not assigned')
         try:
-           self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
-           self.assertTrue(re.search("qemux86",self.driver.find_element_by_xpath("//span[@id='project-machine-name']").text),'The machine type is not assigned')
-           self.driver.find_element_by_xpath("//span[@id='change-machine-toggle']").click()
+           self.driver.find_element(By.XPATH, "//span[@id='change-machine-toggle']").click()
            self.wait_until_visible('#select-machine-form')
            self.wait_until_visible('#cancel-machine-change')
-           self.driver.find_element_by_xpath("//form[@id='select-machine-form']/a[@id='cancel-machine-change']").click()
+           self.driver.find_element(By.XPATH, "//form[@id='select-machine-form']/a[@id='cancel-machine-change']").click()
         except:
            self.fail(msg='The machine information is wrong in the configuration page')
 
+        # Most built recipes section
+        self.wait_until_visible('#no-most-built')
         try:
-           self.driver.find_element_by_id('no-most-built')
+           self.driver.find_element(By.ID, 'no-most-built')
         except:
            self.fail(msg='No Most built information in project detail page')
 
-        try:
-           self.assertTrue(re.search("Yocto Project master",self.driver.find_element_by_xpath("//span[@id='project-release-title']").text),'The project release is not defined')
-        except:
-           self.fail(msg='No project release title information in project detail page')
+        # Project Release title
+        self.assertTrue(re.search("Yocto Project master",self.driver.find_element(By.XPATH, "//span[@id='project-release-title']").text), 'The project release is not defined in the project detail page')
 
+        # List of layers in project
+        self.wait_until_visible('#layer-container')
+        self.driver.find_element(By.XPATH, "//div[@id='layer-container']")
+        self.assertTrue(re.search("3",self.driver.find_element(By.ID, "project-layers-count").text),'There should be 3 layers listed in the layer count')
         try:
-           self.driver.find_element_by_xpath("//div[@id='layer-container']")
-           self.assertTrue(re.search("3",self.driver.find_element_by_id("project-layers-count").text),'There should be 3 layers listed in the layer count')
-           layer_list = self.driver.find_element_by_id("layers-in-project-list")
-           layers = layer_list.find_elements_by_tag_name("li")
-           for layer in layers:
-               if re.match ("openembedded-core",layer.text):
-                    print ("openembedded-core layer is a default layer in the project configuration")
-               elif re.match ("meta-poky",layer.text):
-                  print ("meta-poky layer is a default layer in the project configuration")
-               elif re.match ("meta-yocto-bsp",layer.text):
-                  print ("meta-yocto-bsp is a default layer in the project configuratoin")
-               else:
-                  self.fail(msg='default layers are missing from the project configuration')
+           layer_list = self.driver.find_element(By.ID, "layers-in-project-list")
+           layers = layer_list.find_elements(By.TAG_NAME, "li")
         except:
            self.fail(msg='No Layer information in project detail page')
 
+        for layer in layers:
+            if re.match ("openembedded-core", layer.text):
+                print ("openembedded-core layer is a default layer in the project configuration")
+            elif re.match ("meta-poky", layer.text):
+                 print ("meta-poky layer is a default layer in the project configuration")
+            elif re.match ("meta-yocto-bsp", layer.text):
+                 print ("meta-yocto-bsp is a default layer in the project configuratoin")
+            else:
+                 self.fail(msg='default layers are missing from the project configuration')
+
 #   testcase (1517)
     def test_verify_machine_information(self):
-        self.get('')
-        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
 
+        self.wait_until_visible('#machine-section')
+        self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
+        self.wait_until_visible('#project-machine-name')
+        self.assertTrue(re.search("qemux86-64",self.driver.find_element(By.ID, "project-machine-name").text),'The machine type is not assigned')
         try:
-            self.assertTrue(self.element_exists('#machine-section'),'Machine section for the project configuration page does not exist')
-            self.assertTrue(re.search("qemux86",self.driver.find_element_by_id("project-machine-name").text),'The machine type is not assigned')
-            self.driver.find_element_by_id("change-machine-toggle").click()
+            self.driver.find_element(By.ID, "change-machine-toggle").click()
             self.wait_until_visible('#select-machine-form')
             self.wait_until_visible('#cancel-machine-change')
-            self.driver.find_element_by_id("cancel-machine-change").click()
+            self.driver.find_element(By.ID, "cancel-machine-change").click()
         except:
             self.fail(msg='The machine information is wrong in the configuration page')
 
 #   testcase (1518)
     def test_verify_most_built_recipes_information(self):
-        self.get('')
-        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        self.wait_until_present('#config-nav')
         project_URL=self.get_URL()
 
+        self.wait_until_visible('#no-most-built')
+        self.assertTrue(re.search("You haven't built any recipes yet",self.driver.find_element(By.ID, "no-most-built").text),'Default message of no builds is not present')
         try:
-            self.assertTrue(re.search("You haven't built any recipes yet",self.driver.find_element_by_id("no-most-built").text),'Default message of no builds is not present')
-            self.driver.find_element_by_xpath("//div[@id='no-most-built']/p/a[@href="+'"'+project_URL+'images/"'+"]").click()
-            self.assertTrue(re.search("Compatible image recipes",self.driver.find_element_by_xpath("//div[@class='col-md-10']").text),'The Choose a recipe to build link  is not working  properly')
+            self.driver.find_element(By.XPATH, "//div[@id='no-most-built']/p/a[@href="+'"'+project_URL+'images/"'+"]").click()
         except:
             self.fail(msg='No Most built information in project detail page')
+        self.wait_until_visible('#config-nav')
+        self.assertTrue(re.search("Compatible image recipes",self.driver.find_element(By.XPATH, "//div[@class='col-md-10']").text),'The Choose a recipe to build link  is not working  properly')
 
 #   testcase (1519)
     def test_verify_project_release_information(self):
-        self.get('')
-        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
-
-        try:
-            self.assertTrue(re.search("Yocto Project master",self.driver.find_element_by_id("project-release-title").text),'The project release is not defined')
-        except:
-            self.fail(msg='No project release title information in project detail page')
+        self.wait_until_visible('#project-release-title')
+        self.assertTrue(re.search("Yocto Project master",self.driver.find_element(By.ID, "project-release-title").text), 'No project release title information in project detail page')
 
 #   testcase (1520)
     def test_verify_layer_information(self):
-        self.get('')
-        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        self.wait_until_present('#config-nav')
         project_URL=self.get_URL()
+        self.wait_until_visible('#layer-container')
+        self.driver.find_element(By.XPATH, "//div[@id='layer-container']")
+        self.wait_until_visible('#project-layers-count')
+        self.assertTrue(re.search("3",self.driver.find_element(By.ID, "project-layers-count").text),'There should be 3 layers listed in the layer count')
 
         try:
-           self.driver.find_element_by_xpath("//div[@id='layer-container']")
-           self.assertTrue(re.search("3",self.driver.find_element_by_id("project-layers-count").text),'There should be 3 layers listed in the layer count')
-           layer_list = self.driver.find_element_by_id("layers-in-project-list")
-           layers = layer_list.find_elements_by_tag_name("li")
+           layer_list = self.driver.find_element(By.ID, "layers-in-project-list")
+           layers = layer_list.find_elements(By.TAG_NAME, "li")
+        except:
+            self.fail(msg='No Layer information in project detail page')
 
-           for layer in layers:
-               if re.match ("openembedded-core",layer.text):
-                    print ("openembedded-core layer is a default layer in the project configuration")
-               elif re.match ("meta-poky",layer.text):
-                  print ("meta-poky layer is a default layer in the project configuration")
-               elif re.match ("meta-yocto-bsp",layer.text):
-                  print ("meta-yocto-bsp is a default layer in the project configuratoin")
-               else:
-                  self.fail(msg='default layers are missing from the project configuration')
+        for layer in layers:
+            if re.match ("openembedded-core",layer.text):
+                print ("openembedded-core layer is a default layer in the project configuration")
+            elif re.match ("meta-poky",layer.text):
+                print ("meta-poky layer is a default layer in the project configuration")
+            elif re.match ("meta-yocto-bsp",layer.text):
+                print ("meta-yocto-bsp is a default layer in the project configuratoin")
+            else:
+                self.fail(msg='default layers are missing from the project configuration')
 
-           self.driver.find_element_by_xpath("//input[@id='layer-add-input']")
-           self.driver.find_element_by_xpath("//button[@id='add-layer-btn']")
-           self.driver.find_element_by_xpath("//div[@id='layer-container']/form[@class='form-inline']/p/a[@id='view-compatible-layers']")
-           self.driver.find_element_by_xpath("//div[@id='layer-container']/form[@class='form-inline']/p/a[@href="+'"'+project_URL+'importlayer"'+"]")
+        try:
+           self.driver.find_element(By.XPATH, "//input[@id='layer-add-input']")
+           self.driver.find_element(By.XPATH, "//button[@id='add-layer-btn']")
+           self.driver.find_element(By.XPATH, "//div[@id='layer-container']/form[@class='form-inline']/p/a[@id='view-compatible-layers']")
+           self.driver.find_element(By.XPATH, "//div[@id='layer-container']/form[@class='form-inline']/p/a[@href="+'"'+project_URL+'importlayer"'+"]")
         except:
-            self.fail(msg='No Layer information in project detail page')
+            self.fail(msg='Layer configuration controls missing')
 
 #   testcase (1521)
     def test_verify_project_detail_links(self):
-        self.get('')
-        self.driver.find_element_by_xpath("//div[@id='global-nav']/ul/li/a[@href="+'"'+'/toastergui/projects/'+'"'+"]").click()
-        self.wait_until_visible('#projectstable')
+        self.get(reverse('all-projects'))
+        self.load_projects_page_helper()
         self.find_element_by_link_text_in_table('projectstable', 'selenium-project').click()
+        self.wait_until_present('#config-nav')
         project_URL=self.get_URL()
+        self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").click()
+        self.wait_until_visible('#topbar-configuration-tab')
+        self.assertTrue(re.search("Configuration",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").text), 'Configuration tab in project topbar is misspelled')
 
-        self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").click()
-        self.assertTrue(re.search("Configuration",self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li[@id='topbar-configuration-tab']/a[@href="+'"'+project_URL+'"'+"]").text), 'Configuration tab in project topbar is misspelled')
+        try:
+            self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").click()
+        except:
+            self.fail(msg='Builds tab information is not present')
 
+        self.wait_until_visible('#project-topbar')
+        self.assertTrue(re.search("Builds",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").text), 'Builds tab in project topbar is misspelled')
         try:
-            self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").click()
-            self.assertTrue(re.search("Builds",self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'builds/"'+"]").text), 'Builds tab in project topbar is misspelled')
-            self.driver.find_element_by_xpath("//div[@id='empty-state-projectbuildstable']")
+            self.driver.find_element(By.XPATH, "//div[@id='empty-state-projectbuildstable']")
         except:
             self.fail(msg='Builds tab information is not present')
 
         try:
-            self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").click()
-            self.assertTrue(re.search("Import layer",self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").text), 'Import layer tab in project topbar is misspelled')
-            self.driver.find_element_by_xpath("//fieldset[@id='repo-select']")
-            self.driver.find_element_by_xpath("//fieldset[@id='git-repo']")
+            self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").click()
         except:
             self.fail(msg='Import layer tab not loading properly')
 
+        self.wait_until_visible('#project-topbar')
+        self.assertTrue(re.search("Import layer",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'importlayer"'+"]").text), 'Import layer tab in project topbar is misspelled')
         try:
-            self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").click()
-            self.assertTrue(re.search("New custom image",self.driver.find_element_by_xpath("//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").text), 'New custom image tab in project topbar is misspelled')
-            self.assertTrue(re.search("Select the image recipe you want to customise",self.driver.find_element_by_xpath("//div[@class='col-md-12']/h2").text),'The new custom image tab is not loading correctly')
+            self.driver.find_element(By.XPATH, "//fieldset[@id='repo-select']")
+            self.driver.find_element(By.XPATH, "//fieldset[@id='git-repo']")
+        except:
+            self.fail(msg='Import layer tab not loading properly')
+
+        try:
+            self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").click()
         except:
             self.fail(msg='New custom image tab not loading properly')
 
+        self.wait_until_visible('#project-topbar')
+        self.assertTrue(re.search("New custom image",self.driver.find_element(By.XPATH, "//div[@id='project-topbar']/ul[@class='nav nav-tabs']/li/a[@href="+'"'+project_URL+'newcustomimage/"'+"]").text), 'New custom image tab in project topbar is misspelled')
+        self.assertTrue(re.search("Select the image recipe you want to customise",self.driver.find_element(By.XPATH, "//div[@class='col-md-12']/h2").text),'The new custom image tab is not loading correctly')
+
 
 
diff --git a/bitbake/lib/toaster/tests/functional/test_project_config.py b/bitbake/lib/toaster/tests/functional/test_project_config.py
new file mode 100644
index 0000000000..fcb1bc3284
--- /dev/null
+++ b/bitbake/lib/toaster/tests/functional/test_project_config.py
@@ -0,0 +1,294 @@
+#! /usr/bin/env python3 #
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import string
+import pytest
+from django.urls import reverse
+from selenium.webdriver import Keys
+from selenium.webdriver.support.select import Select
+from selenium.common.exceptions import TimeoutException
+from tests.functional.functional_helpers import SeleniumFunctionalTestCase
+from selenium.webdriver.common.by import By
+
+from .utils import get_projectId_from_url
+
+class TestProjectConfig(SeleniumFunctionalTestCase):
+    project_id = None
+    PROJECT_NAME = 'TestProjectConfig'
+    INVALID_PATH_START_TEXT = 'The directory path should either start with a /'
+    INVALID_PATH_CHAR_TEXT = 'The directory path cannot include spaces or ' \
+        'any of these characters'
+
+    def _get_config_nav_item(self, index):
+        config_nav = self.find('#config-nav')
+        return config_nav.find_elements(By.TAG_NAME, 'li')[index]
+
+    def _navigate_bbv_page(self):
+        """ Navigate to project BitBake variables page """
+        # check if the menu is displayed
+        if TestProjectConfig.project_id is None:
+            TestProjectConfig.project_id = self.create_new_project(self.PROJECT_NAME, '3', None, True)
+
+        url = reverse('projectconf', args=(TestProjectConfig.project_id,))
+        self.get(url)
+        self.wait_until_visible('#config-nav')
+        bbv_page_link = self._get_config_nav_item(9)
+        bbv_page_link.click()
+        self.wait_until_visible('#config-nav')
+
+    def test_no_underscore_iamgefs_type(self):
+        """
+        Should not accept IMAGEFS_TYPE with an underscore
+        """
+        self._navigate_bbv_page()
+        imagefs_type = "foo_bar"
+
+        self.wait_until_visible('#change-image_fstypes-icon')
+
+        self.click('#change-image_fstypes-icon')
+
+        self.enter_text('#new-imagefs_types', imagefs_type)
+
+        element = self.wait_until_visible('#hintError-image-fs_type')
+
+        self.assertTrue(("A valid image type cannot include underscores" in element.text),
+                        "Did not find underscore error message")
+
+    def test_checkbox_verification(self):
+        """
+        Should automatically check the checkbox if user enters value
+        text box, if value is there in the checkbox.
+        """
+        self._navigate_bbv_page()
+
+        imagefs_type = "btrfs"
+
+        self.wait_until_visible('#change-image_fstypes-icon')
+
+        self.click('#change-image_fstypes-icon')
+
+        self.enter_text('#new-imagefs_types', imagefs_type)
+
+        checkboxes = self.driver.find_elements(By.XPATH, "//input[@class='fs-checkbox-fstypes']")
+
+        for checkbox in checkboxes:
+            if checkbox.get_attribute("value") == "btrfs":
+               self.assertEqual(checkbox.is_selected(), True)
+
+    def test_textbox_with_checkbox_verification(self):
+        """
+        Should automatically add or remove value in textbox, if user checks
+        or unchecks checkboxes.
+        """
+        self._navigate_bbv_page()
+
+        self.wait_until_visible('#change-image_fstypes-icon')
+        self.click('#change-image_fstypes-icon')
+
+        checkboxes_selector = '.fs-checkbox-fstypes'
+
+        self.wait_until_visible(checkboxes_selector)
+        checkboxes = self.find_all(checkboxes_selector)
+
+        for checkbox in checkboxes:
+            if checkbox.get_attribute("value") == "cpio":
+               checkbox.click()
+               self.wait_until_visible('#new-imagefs_types')
+               element = self.driver.find_element(By.ID, 'new-imagefs_types')
+
+               self.assertTrue(("cpio" in element.get_attribute('value'),
+                               "Imagefs not added into the textbox"))
+               checkbox.click()
+               self.assertTrue(("cpio" not in element.text),
+                               "Image still present in the textbox")
+
+    def test_set_download_dir(self):
+        """
+        Validate the allowed and disallowed types in the directory field for
+        DL_DIR
+        """
+        self._navigate_bbv_page()
+
+        # activate the input to edit download dir
+        try:
+            change_dl_dir_btn = self.wait_until_visible('#change-dl_dir-icon')
+        except TimeoutException:
+            # If download dir is not displayed, test is skipped
+            change_dl_dir_btn = None
+
+        if change_dl_dir_btn:
+            change_dl_dir_btn.click()
+
+            # downloads dir path doesn't start with / or ${...}
+            input_field = self.wait_until_visible('#new-dl_dir')
+            input_field.clear()
+            self.enter_text('#new-dl_dir', 'home/foo')
+            element = self.wait_until_visible('#hintError-initialChar-dl_dir')
+
+            msg = 'downloads directory path starts with invalid character but ' \
+                'treated as valid'
+            self.assertTrue((self.INVALID_PATH_START_TEXT in element.text), msg)
+
+            # downloads dir path has a space
+            self.driver.find_element(By.ID, 'new-dl_dir').clear()
+            self.enter_text('#new-dl_dir', '/foo/bar a')
+
+            element = self.wait_until_visible('#hintError-dl_dir')
+            msg = 'downloads directory path characters invalid but treated as valid'
+            self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+            # downloads dir path starts with ${...} but has a space
+            self.driver.find_element(By.ID,'new-dl_dir').clear()
+            self.enter_text('#new-dl_dir', '${TOPDIR}/down foo')
+
+            element = self.wait_until_visible('#hintError-dl_dir')
+            msg = 'downloads directory path characters invalid but treated as valid'
+            self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+            # downloads dir path starts with /
+            self.driver.find_element(By.ID,'new-dl_dir').clear()
+            self.enter_text('#new-dl_dir', '/bar/foo')
+
+            hidden_element = self.driver.find_element(By.ID,'hintError-dl_dir')
+            self.assertEqual(hidden_element.is_displayed(), False,
+                'downloads directory path valid but treated as invalid')
+
+            # downloads dir path starts with ${...}
+            self.driver.find_element(By.ID,'new-dl_dir').clear()
+            self.enter_text('#new-dl_dir', '${TOPDIR}/down')
+
+            hidden_element = self.driver.find_element(By.ID,'hintError-dl_dir')
+            self.assertEqual(hidden_element.is_displayed(), False,
+                'downloads directory path valid but treated as invalid')
+
+    def test_set_sstate_dir(self):
+        """
+        Validate the allowed and disallowed types in the directory field for
+        SSTATE_DIR
+        """
+        self._navigate_bbv_page()
+
+        try:
+            btn_chg_sstate_dir = self.wait_until_visible('#change-sstate_dir-icon')
+            self.click('#change-sstate_dir-icon')
+        except TimeoutException:
+            # If sstate_dir is not displayed, test is skipped
+            btn_chg_sstate_dir = None
+
+        if btn_chg_sstate_dir:  # Skip continuation if sstate_dir is not displayed
+            # path doesn't start with / or ${...}
+            input_field = self.wait_until_visible('#new-sstate_dir')
+            input_field.clear()
+            self.enter_text('#new-sstate_dir', 'home/foo')
+            element = self.wait_until_visible('#hintError-initialChar-sstate_dir')
+
+            msg = 'sstate directory path starts with invalid character but ' \
+                'treated as valid'
+            self.assertTrue((self.INVALID_PATH_START_TEXT in element.text), msg)
+
+            # path has a space
+            self.driver.find_element(By.ID, 'new-sstate_dir').clear()
+            self.enter_text('#new-sstate_dir', '/foo/bar a')
+
+            element = self.wait_until_visible('#hintError-sstate_dir')
+            msg = 'sstate directory path characters invalid but treated as valid'
+            self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+            # path starts with ${...} but has a space
+            self.driver.find_element(By.ID,'new-sstate_dir').clear()
+            self.enter_text('#new-sstate_dir', '${TOPDIR}/down foo')
+
+            element = self.wait_until_visible('#hintError-sstate_dir')
+            msg = 'sstate directory path characters invalid but treated as valid'
+            self.assertTrue((self.INVALID_PATH_CHAR_TEXT in element.text), msg)
+
+            # path starts with /
+            self.driver.find_element(By.ID,'new-sstate_dir').clear()
+            self.enter_text('#new-sstate_dir', '/bar/foo')
+
+            hidden_element = self.driver.find_element(By.ID, 'hintError-sstate_dir')
+            self.assertEqual(hidden_element.is_displayed(), False,
+                'sstate directory path valid but treated as invalid')
+
+            # paths starts with ${...}
+            self.driver.find_element(By.ID, 'new-sstate_dir').clear()
+            self.enter_text('#new-sstate_dir', '${TOPDIR}/down')
+
+            hidden_element = self.driver.find_element(By.ID, 'hintError-sstate_dir')
+            self.assertEqual(hidden_element.is_displayed(), False,
+                'sstate directory path valid but treated as invalid')
+
+    def _change_bbv_value(self, **kwargs):
+        var_name, field, btn_id, input_id, value, save_btn, *_ = kwargs.values()
+        """ Change bitbake variable value """
+        self._navigate_bbv_page()
+        self.wait_until_visible(f'#{btn_id}')
+        if kwargs.get('new_variable'):
+            self.find(f"#{btn_id}").clear()
+            self.enter_text(f"#{btn_id}", f"{var_name}")
+        else:
+            self.click(f'#{btn_id}')
+
+        self.wait_until_visible(f'#{input_id}')
+
+        if kwargs.get('is_select'):
+            select = Select(self.find(f'#{input_id}'))
+            select.select_by_visible_text(value)
+        else:
+            self.find(f"#{input_id}").clear()
+            self.enter_text(f'#{input_id}', f'{value}')
+        self.click(f'#{save_btn}')
+        value_displayed = str(self.wait_until_visible(f'#{field}').text).lower()
+        msg = f'{var_name} variable not changed'
+        self.assertTrue(str(value).lower() in value_displayed, msg)
+
+    def test_change_distro_var(self):
+        """ Test changing distro variable """
+        self._change_bbv_value(
+            var_name='DISTRO',
+            field='distro',
+            btn_id='change-distro-icon',
+            input_id='new-distro',
+            value='poky-changed',
+            save_btn="apply-change-distro",
+        )
+
+    def test_set_image_install_append_var(self):
+        """ Test setting IMAGE_INSTALL:append variable """
+        self._change_bbv_value(
+            var_name='IMAGE_INSTALL:append',
+            field='image_install',
+            btn_id='change-image_install-icon',
+            input_id='new-image_install',
+            value='bash, apt, busybox',
+            save_btn="apply-change-image_install",
+        )
+
+    def test_set_package_classes_var(self):
+        """ Test setting PACKAGE_CLASSES variable """
+        self._change_bbv_value(
+            var_name='PACKAGE_CLASSES',
+            field='package_classes',
+            btn_id='change-package_classes-icon',
+            input_id='package_classes-select',
+            value='package_deb',
+            save_btn="apply-change-package_classes",
+            is_select=True,
+        )
+
+    def test_create_new_bbv(self):
+        """ Test creating new bitbake variable """
+        self._change_bbv_value(
+            var_name='New_Custom_Variable',
+            field='configvar-list',
+            btn_id='variable',
+            input_id='value',
+            value='new variable value',
+            save_btn="add-configvar-button",
+            new_variable=True
+        )
diff --git a/bitbake/lib/toaster/tests/functional/test_project_page.py b/bitbake/lib/toaster/tests/functional/test_project_page.py
new file mode 100644
index 0000000000..429d86feba
--- /dev/null
+++ b/bitbake/lib/toaster/tests/functional/test_project_page.py
@@ -0,0 +1,775 @@
+#! /usr/bin/env python3 #
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import os
+import string
+import time
+from unittest import skip
+import pytest
+from django.urls import reverse
+from django.utils import timezone
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.support.select import Select
+from selenium.common.exceptions import TimeoutException
+from tests.functional.functional_helpers import SeleniumFunctionalTestCase
+from orm.models import Build, Project, Target
+from selenium.webdriver.common.by import By
+
+from .utils import get_projectId_from_url, wait_until_build, wait_until_build_cancelled
+
+class TestProjectPageBase(SeleniumFunctionalTestCase):
+    project_id = None
+    PROJECT_NAME = 'TestProjectPage'
+
+    def _navigate_to_project_page(self):
+        # Navigate to project page
+        if TestProjectPageBase.project_id is None:
+            TestProjectPageBase.project_id = self.create_new_project(self.PROJECT_NAME, '3', None, True)
+
+        url = reverse('project', args=(TestProjectPageBase.project_id,))
+        self.get(url)
+        self.wait_until_visible('#config-nav')
+
+    def _get_create_builds(self, **kwargs):
+        """ Create a build and return the build object """
+        # parameters for builds to associate with the projects
+        now = timezone.now()
+        self.project1_build_success = {
+            'project': Project.objects.get(id=TestProjectPageBase.project_id),
+            'started_on': now,
+            'completed_on': now,
+            'outcome': Build.SUCCEEDED
+        }
+
+        self.project1_build_failure = {
+            'project': Project.objects.get(id=TestProjectPageBase.project_id),
+            'started_on': now,
+            'completed_on': now,
+            'outcome': Build.FAILED
+        }
+        build1 = Build.objects.create(**self.project1_build_success)
+        build2 = Build.objects.create(**self.project1_build_failure)
+
+        # add some targets to these builds so they have recipe links
+        # (and so we can find the row in the ToasterTable corresponding to
+        # a particular build)
+        Target.objects.create(build=build1, target='foo')
+        Target.objects.create(build=build2, target='bar')
+
+        if kwargs:
+            # Create kwargs.get('success') builds with success status with target
+            # and kwargs.get('failure') builds with failure status with target
+            for i in range(kwargs.get('success', 0)):
+                now = timezone.now()
+                self.project1_build_success['started_on'] = now
+                self.project1_build_success[
+                    'completed_on'] = now - timezone.timedelta(days=i)
+                build = Build.objects.create(**self.project1_build_success)
+                Target.objects.create(build=build,
+                                      target=f'{i}_success_recipe',
+                                      task=f'{i}_success_task')
+
+            for i in range(kwargs.get('failure', 0)):
+                now = timezone.now()
+                self.project1_build_failure['started_on'] = now
+                self.project1_build_failure[
+                    'completed_on'] = now - timezone.timedelta(days=i)
+                build = Build.objects.create(**self.project1_build_failure)
+                Target.objects.create(build=build,
+                                      target=f'{i}_fail_recipe',
+                                      task=f'{i}_fail_task')
+        return build1, build2
+
+    def _mixin_test_table_edit_column(
+            self,
+            table_id,
+            edit_btn_id,
+            list_check_box_id: list
+    ):
+        # Check edit column
+        finder = lambda driver: self.find(f'#{edit_btn_id}')
+        edit_column = self.wait_until_element_clickable(finder)
+        self.assertTrue(edit_column.is_displayed())
+        edit_column.click()
+        # Check dropdown is visible
+        self.wait_until_visible('ul.dropdown-menu.editcol')
+        for check_box_id in list_check_box_id:
+            # Check that we can hide/show table column
+            check_box = self.find(f'#{check_box_id}')
+            th_class = str(check_box_id).replace('checkbox-', '')
+            if check_box.is_selected():
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#{table_id} thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+                check_box.click()
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#{table_id} thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+            else:
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#{table_id} thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+                check_box.click()
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#{table_id} thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+
+    def _get_config_nav_item(self, index):
+        config_nav = self.find('#config-nav')
+        return config_nav.find_elements(By.TAG_NAME, 'li')[index]
+
+    def _navigate_to_config_nav(self, nav_id, nav_index):
+        # navigate to the project page
+        self._navigate_to_project_page()
+        # click on "Software recipe" tab
+        soft_recipe = self._get_config_nav_item(nav_index)
+        soft_recipe.click()
+        self.wait_until_visible(f'#{nav_id}')
+
+    def _mixin_test_table_show_rows(self, table_selector, **kwargs):
+        """ Test the show rows feature in the builds table on the all builds page """
+        def test_show_rows(row_to_show, show_row_link):
+            # Check that we can show rows == row_to_show
+            show_row_link.select_by_value(str(row_to_show))
+            self.wait_until_visible(f'#{table_selector} tbody tr')
+            # check at least some rows are visible
+            self.assertTrue(
+                len(self.find_all(f'#{table_selector} tbody tr')) > 0
+            )
+        self.wait_until_present(f'#{table_selector} tbody tr')
+        show_rows = self.driver.find_elements(
+            By.XPATH,
+            f'//select[@class="form-control pagesize-{table_selector}"]'
+        )
+        rows_to_show = [10, 25, 50, 100, 150]
+        to_skip = kwargs.get('to_skip', [])
+        # Check show rows
+        for show_row_link in show_rows:
+            show_row_link = Select(show_row_link)
+            for row_to_show in rows_to_show:
+                if row_to_show not in to_skip:
+                    test_show_rows(row_to_show, show_row_link)
+
+    def _mixin_test_table_search_input(self, **kwargs):
+        input_selector, input_text, searchBtn_selector, table_selector, *_ = kwargs.values()
+        # Test search input
+        self.wait_until_visible(f'#{input_selector}')
+        recipe_input = self.find(f'#{input_selector}')
+        recipe_input.send_keys(input_text)
+        self.find(f'#{searchBtn_selector}').click()
+        self.wait_until_visible(f'#{table_selector} tbody tr')
+        rows = self.find_all(f'#{table_selector} tbody tr')
+        self.assertTrue(len(rows) > 0)
+
+class TestProjectPage(TestProjectPageBase):
+
+    def test_page_header_on_project_page(self):
+        """ Check page header in project page:
+          - AT LEFT -> Logo of Yocto project, displayed, clickable
+          - "Toaster"+" Information icon", displayed, clickable
+          - "Server Icon" + "All builds", displayed, clickable
+          - "Directory Icon" + "All projects", displayed, clickable
+          - "Book Icon" + "Documentation", displayed, clickable
+          - AT RIGHT -> button "New project", displayed, clickable
+        """
+        # navigate to the project page
+        self._navigate_to_project_page()
+
+        # check page header
+        # AT LEFT -> Logo of Yocto project
+        logo = self.driver.find_element(
+            By.XPATH,
+            "//div[@class='toaster-navbar-brand']",
+        )
+        logo_img = logo.find_element(By.TAG_NAME, 'img')
+        self.assertTrue(logo_img.is_displayed(),
+                        'Logo of Yocto project not found')
+        self.assertIn(
+            '/static/img/logo.png', str(logo_img.get_attribute('src')),
+            'Logo of Yocto project not found'
+        )
+        # "Toaster"+" Information icon", clickable
+        toaster = self.driver.find_element(
+            By.XPATH,
+            "//div[@class='toaster-navbar-brand']//a[@class='brand']",
+        )
+        self.assertTrue(toaster.is_displayed(), 'Toaster not found')
+        self.assertEqual(toaster.text, 'Toaster')
+        info_sign = self.find('.glyphicon-info-sign')
+        self.assertTrue(info_sign.is_displayed())
+
+        # "Server Icon" + "All builds"
+        all_builds = self.find('#navbar-all-builds')
+        all_builds_link = all_builds.find_element(By.TAG_NAME, 'a')
+        self.assertIn("All builds", all_builds_link.text)
+        self.assertIn(
+            '/toastergui/builds/', str(all_builds_link.get_attribute('href'))
+        )
+        server_icon = all_builds.find_element(By.TAG_NAME, 'i')
+        self.assertEqual(
+            server_icon.get_attribute('class'), 'glyphicon glyphicon-tasks'
+        )
+        self.assertTrue(server_icon.is_displayed())
+
+        # "Directory Icon" + "All projects"
+        all_projects = self.find('#navbar-all-projects')
+        all_projects_link = all_projects.find_element(By.TAG_NAME, 'a')
+        self.assertIn("All projects", all_projects_link.text)
+        self.assertIn(
+            '/toastergui/projects/', str(all_projects_link.get_attribute(
+                'href'))
+        )
+        dir_icon = all_projects.find_element(By.TAG_NAME, 'i')
+        self.assertEqual(
+            dir_icon.get_attribute('class'), 'icon-folder-open'
+        )
+        self.assertTrue(dir_icon.is_displayed())
+
+        # "Book Icon" + "Documentation"
+        toaster_docs_link = self.find('#navbar-docs')
+        toaster_docs_link_link = toaster_docs_link.find_element(By.TAG_NAME,
+                                                                'a')
+        self.assertIn("Documentation", toaster_docs_link_link.text)
+        self.assertEqual(
+            toaster_docs_link_link.get_attribute('href'), 'http://docs.yoctoproject.org/toaster-manual/index.html#toaster-user-manual'
+        )
+        book_icon = toaster_docs_link.find_element(By.TAG_NAME, 'i')
+        self.assertEqual(
+            book_icon.get_attribute('class'), 'glyphicon glyphicon-book'
+        )
+        self.assertTrue(book_icon.is_displayed())
+
+        # AT RIGHT -> button "New project"
+        new_project_button = self.find('#new-project-button')
+        self.assertTrue(new_project_button.is_displayed())
+        self.assertEqual(new_project_button.text, 'New project')
+        new_project_button.click()
+        self.assertIn(
+            '/toastergui/newproject/', str(self.driver.current_url)
+        )
+
+    def test_edit_project_name(self):
+        """ Test edit project name:
+          - Click on "Edit" icon button
+          - Change project name
+          - Click on "Save" button
+          - Check project name is changed
+        """
+        # navigate to the project page
+        self._navigate_to_project_page()
+
+        # click on "Edit" icon button
+        self.wait_until_visible('#project-name-container')
+        finder = lambda driver: self.find('#project-change-form-toggle')
+        edit_button = self.wait_until_element_clickable(finder)
+        edit_button.click()
+        project_name_input = self.find('#project-name-change-input')
+        self.assertTrue(project_name_input.is_displayed())
+        project_name_input.clear()
+        project_name_input.send_keys('New Name')
+        self.find('#project-name-change-btn').click()
+
+        # check project name is changed
+        self.wait_until_visible('#project-name-container')
+        self.assertIn(
+            'New Name', str(self.find('#project-name-container').text)
+        )
+
+    def test_project_page_tabs(self):
+        """ Test project tabs:
+          - "configuration" tab
+          - "Builds" tab
+          - "Import layers" tab
+          - "New custom image" tab
+          Check search box used to build recipes
+        """
+        # navigate to the project page
+        self._navigate_to_project_page()
+
+        # check "configuration" tab
+        self.wait_until_visible('#topbar-configuration-tab')
+        config_tab = self.find('#topbar-configuration-tab')
+        self.assertEqual(config_tab.get_attribute('class'), 'active')
+        self.assertIn('Configuration', str(config_tab.text))
+        self.assertIn(
+            f"/toastergui/project/{TestProjectPageBase.project_id}", str(self.driver.current_url)
+        )
+
+        def get_tabs():
+            # tabs links list
+            return self.driver.find_elements(
+                By.XPATH,
+                '//div[@id="project-topbar"]//li'
+            )
+
+        def check_tab_link(tab_index, tab_name, url):
+            tab = get_tabs()[tab_index]
+            tab_link = tab.find_element(By.TAG_NAME, 'a')
+            self.assertIn(url, tab_link.get_attribute('href'))
+            self.assertIn(tab_name, tab_link.text)
+            self.assertEqual(tab.get_attribute('class'), 'active')
+
+        # check "Builds" tab
+        builds_tab = get_tabs()[1]
+        builds_tab.find_element(By.TAG_NAME, 'a').click()
+        check_tab_link(
+            1,
+            'Builds',
+            f"/toastergui/project/{TestProjectPageBase.project_id}/builds"
+        )
+
+        # check "Import layers" tab
+        import_layers_tab = get_tabs()[2]
+        import_layers_tab.find_element(By.TAG_NAME, 'a').click()
+        check_tab_link(
+            2,
+            'Import layer',
+            f"/toastergui/project/{TestProjectPageBase.project_id}/importlayer"
+        )
+
+        # check "New custom image" tab
+        new_custom_image_tab = get_tabs()[3]
+        new_custom_image_tab.find_element(By.TAG_NAME, 'a').click()
+        check_tab_link(
+            3,
+            'New custom image',
+            f"/toastergui/project/{TestProjectPageBase.project_id}/newcustomimage"
+        )
+
+        # check search box can be use to build recipes
+        search_box = self.find('#build-input')
+        search_box.send_keys('core-image-minimal')
+        self.find('#build-button').click()
+        self.wait_until_visible('#latest-builds')
+        buildtext = "Loading"
+        while "Loading" in buildtext:
+            time.sleep(1)
+            lastest_builds = self.driver.find_elements(
+                By.XPATH,
+                '//div[@id="latest-builds"]',
+            )
+            last_build = lastest_builds[0]
+            buildtext = last_build.text
+        self.assertIn(
+            'core-image-minimal', str(last_build.text)
+        )
+
+    def test_softwareRecipe_page(self):
+        """ Test software recipe page
+            - Check title "Compatible software recipes" is displayed
+            - Check search input
+            - Check "build recipe" button works
+            - Check software recipe table feature(show/hide column, pagination)
+        """
+        self._navigate_to_config_nav('softwarerecipestable', 4)
+        # check title "Compatible software recipes" is displayed
+        self.assertIn("Compatible software recipes", self.get_page_source())
+        # Test search input
+        self._mixin_test_table_search_input(
+            input_selector='search-input-softwarerecipestable',
+            input_text='busybox',
+            searchBtn_selector='search-submit-softwarerecipestable',
+            table_selector='softwarerecipestable'
+        )
+        # check "build recipe" button works
+        finder = lambda driver: self.find_all('#softwarerecipestable tbody tr')[0].find_element(By.XPATH, '//td[@class="add-del-layers"]/a')
+        build_btn = self.wait_until_element_clickable(finder)
+        build_btn.click()
+        build_state = wait_until_build(self, 'queued cloning starting parsing failed')
+        lastest_builds = self.driver.find_elements(
+            By.XPATH,
+            '//div[@id="latest-builds"]/div'
+        )
+        self.assertTrue(len(lastest_builds) > 0)
+        # Find the latest builds, the last build and then the cancel button
+ 
+        finder = lambda driver: driver.find_elements(By.XPATH, '//div[@id="latest-builds"]/div')[0].find_element(By.XPATH, '//span[@class="cancel-build-btn pull-right alert-link"]')
+        cancel_button = self.wait_until_element_clickable(finder)
+        cancel_button.click()
+        if 'starting' not in build_state:  # change build state when cancelled in starting state
+            wait_until_build_cancelled(self)
+
+        # check software recipe table feature(show/hide column, pagination)
+        self._navigate_to_config_nav('softwarerecipestable', 4)
+        column_list = [
+            'get_description_or_summary',
+            'layer_version__get_vcs_reference',
+            'layer_version__layer__name',
+            'license',
+            'recipe-file',
+            'section',
+            'version',
+        ]
+        self._mixin_test_table_edit_column(
+            'softwarerecipestable',
+            'edit-columns-button',
+            [f'checkbox-{column}' for column in column_list]
+        )
+        self._navigate_to_config_nav('softwarerecipestable', 4)
+        # check show rows(pagination)
+        self._mixin_test_table_show_rows(
+            table_selector='softwarerecipestable',
+            to_skip=[150],
+        )
+
+    def test_machines_page(self):
+        """ Test Machine page
+            - Check if title "Compatible machines" is displayed
+            - Check search input
+            - Check "Select machine" button works
+            - Check "Add layer" button works
+            - Check Machine table feature(show/hide column, pagination)
+        """
+        self._navigate_to_config_nav('machinestable', 5)
+        # check title "Compatible software recipes" is displayed
+        self.assertIn("Compatible machines", self.get_page_source())
+        # Test search input
+        self._mixin_test_table_search_input(
+            input_selector='search-input-machinestable',
+            input_text='qemux86-64',
+            searchBtn_selector='search-submit-machinestable',
+            table_selector='machinestable'
+        )
+        # check "Select machine" button works
+        finder = lambda driver: self.find_all('#machinestable tbody tr')[0].find_element(By.XPATH, '//td[@class="add-del-layers"]')
+        select_btn = self.wait_until_element_clickable(finder)
+        select_btn.click()
+        self.wait_until_visible('#project-machine-name')
+        project_machine_name = self.find('#project-machine-name')
+        self.assertIn(
+            'qemux86-64', project_machine_name.text
+        )
+        # check "Add layer" button works
+        self._navigate_to_config_nav('machinestable', 5)
+        # Search for a machine whit layer not in project
+        self._mixin_test_table_search_input(
+            input_selector='search-input-machinestable',
+            input_text='qemux86-64-tpm2',
+            searchBtn_selector='search-submit-machinestable',
+            table_selector='machinestable'
+        )
+
+        self.wait_until_visible('#machinestable tbody tr')
+        # Locate a machine to add button
+        finder = lambda driver: self.find_all('#machinestable tbody tr')[0].find_element(By.XPATH, '//td[@class="add-del-layers"]')
+        add_btn = self.wait_until_element_clickable(finder)
+        add_btn.click()
+        self.wait_until_visible('#change-notification')
+        change_notification = self.find('#change-notification')
+        self.assertIn(
+            f'You have added 1 layer to your project', str(change_notification.text)
+        )
+
+        finder = lambda driver: self.find('#hide-alert')
+        hide_button = self.wait_until_element_clickable(finder)
+        hide_button.click()
+        self.wait_until_not_visible('#change-notification')
+
+        # check Machine table feature(show/hide column, pagination)
+        self._navigate_to_config_nav('machinestable', 5)
+        column_list = [
+            'description',
+            'layer_version__get_vcs_reference',
+            'layer_version__layer__name',
+            'machinefile',
+        ]
+        self._mixin_test_table_edit_column(
+            'machinestable',
+            'edit-columns-button',
+            [f'checkbox-{column}' for column in column_list]
+        )
+        self._navigate_to_config_nav('machinestable', 5)
+        # check show rows(pagination)
+        self._mixin_test_table_show_rows(
+            table_selector='machinestable',
+            to_skip=[150],
+        )
+
+    def test_layers_page(self):
+        """ Test layers page
+            - Check if title "Compatible layerss" is displayed
+            - Check search input
+            - Check "Add layer" button works
+            - Check "Remove layer" button works
+            - Check layers table feature(show/hide column, pagination)
+        """
+        self._navigate_to_config_nav('layerstable', 6)
+        # check title "Compatible layers" is displayed
+        self.assertIn("Compatible layers", self.get_page_source())
+        # Test search input
+        input_text='meta-tanowrt'
+        self._mixin_test_table_search_input(
+            input_selector='search-input-layerstable',
+            input_text=input_text,
+            searchBtn_selector='search-submit-layerstable',
+            table_selector='layerstable'
+        )
+        # check "Add layer" button works
+        self.wait_until_visible('#layerstable tbody tr')
+        finder = lambda driver: self.find_all('#layerstable tbody tr')[0].find_element(By.XPATH, '//td[@class="add-del-layers"]/a[@data-directive="add"]')
+        add_btn = self.wait_until_element_clickable(finder)
+        add_btn.click()
+        # check modal is displayed
+        self.wait_until_visible('#dependencies-modal')
+        list_dependencies = self.find_all('#dependencies-list li')
+        # click on add-layers button
+        finder = lambda driver: self.driver.find_element(By.XPATH, '//form[@id="dependencies-modal-form"]//button[@class="btn btn-primary"]')
+        add_layers_btn = self.wait_until_element_clickable(finder)
+        add_layers_btn.click()
+        self.wait_until_visible('#change-notification')
+        change_notification = self.find('#change-notification')
+        self.assertIn(
+            f'You have added {len(list_dependencies)+1} layers to your project: {input_text} and its dependencies', str(change_notification.text)
+        )
+
+        finder = lambda driver: self.find('#hide-alert')
+        hide_button = self.wait_until_element_clickable(finder)
+        hide_button.click()
+        self.wait_until_not_visible('#change-notification')
+
+        # check "Remove layer" button works
+        self.wait_until_visible('#layerstable tbody tr')
+        finder = lambda driver: self.find_all('#layerstable tbody tr')[0].find_element(By.XPATH, '//td[@class="add-del-layers"]/a[@data-directive="remove"]')
+        remove_btn = self.wait_until_element_clickable(finder)
+        remove_btn.click()
+        self.wait_until_visible('#change-notification')
+        change_notification = self.find('#change-notification')
+        self.assertIn(
+            f'You have removed 1 layer from your project: {input_text}', str(change_notification.text)
+        )
+
+        finder = lambda driver: self.find('#hide-alert')
+        hide_button = self.wait_until_element_clickable(finder)
+        hide_button.click()
+        self.wait_until_not_visible('#change-notification')
+
+        # check layers table feature(show/hide column, pagination)
+        self._navigate_to_config_nav('layerstable', 6)
+        column_list = [
+            'dependencies',
+            'revision',
+            'layer__vcs_url',
+            'git_subdir',
+            'layer__summary',
+        ]
+        self._mixin_test_table_edit_column(
+            'layerstable',
+            'edit-columns-button',
+            [f'checkbox-{column}' for column in column_list]
+        )
+        self._navigate_to_config_nav('layerstable', 6)
+        # check show rows(pagination)
+        self._mixin_test_table_show_rows(
+            table_selector='layerstable',
+            to_skip=[150],
+        )
+
+    def test_distro_page(self):
+        """ Test distros page
+            - Check if title "Compatible distros" is displayed
+            - Check search input
+            - Check "Add layer" button works
+            - Check distro table feature(show/hide column, pagination)
+        """
+        self._navigate_to_config_nav('distrostable', 7)
+        # check title "Compatible distros" is displayed
+        self.assertIn("Compatible Distros", self.get_page_source())
+        # Test search input
+        input_text='poky-altcfg'
+        self._mixin_test_table_search_input(
+            input_selector='search-input-distrostable',
+            input_text=input_text,
+            searchBtn_selector='search-submit-distrostable',
+            table_selector='distrostable'
+        )
+        # check "Add distro" button works
+        self.wait_until_visible(".add-del-layers")
+        finder = lambda driver: self.find_all('#distrostable tbody tr')[0].find_element(By.XPATH, '//td[@class="add-del-layers"]')
+        add_btn = self.wait_until_element_clickable(finder)
+        add_btn.click()
+        self.wait_until_visible('#change-notification')
+        change_notification = self.find('#change-notification')
+        self.assertIn(
+            f'You have changed the distro to: {input_text}', str(change_notification.text)
+        )
+        # check distro table feature(show/hide column, pagination)
+        self._navigate_to_config_nav('distrostable', 7)
+        column_list = [
+            'description',
+            'templatefile',
+            'layer_version__get_vcs_reference',
+            'layer_version__layer__name',
+        ]
+        self._mixin_test_table_edit_column(
+            'distrostable',
+            'edit-columns-button',
+            [f'checkbox-{column}' for column in column_list]
+        )
+        self._navigate_to_config_nav('distrostable', 7)
+        # check show rows(pagination)
+        self._mixin_test_table_show_rows(
+            table_selector='distrostable',
+            to_skip=[150],
+        )
+
+    def test_single_layer_page(self):
+        """ Test layer details page using meta-poky as an example (assumes is added to start with)
+            - Check if title is displayed
+            - Check add/remove layer button works
+            - Check tabs(layers, recipes, machines) are displayed
+            - Check left section is displayed
+                - Check layer name
+                - Check layer summary
+                - Check layer description
+        """
+        self._navigate_to_config_nav('layerstable', 6)
+        layer_link = self.driver.find_element(By.XPATH, '//tr/td[@class="layer__name"]/a[contains(text(),"meta-poky")]')
+        layer_link.click()
+        self.wait_until_visible('.page-header')
+        # check title is displayed
+        self.assertTrue(self.find('.page-header h1').is_displayed())
+
+        # check remove layer button works
+        finder = lambda driver: self.find('#add-remove-layer-btn')
+        remove_layer_btn = self.wait_until_element_clickable(finder)
+        remove_layer_btn.click()
+        self.wait_until_visible('#change-notification')
+        change_notification = self.find('#change-notification')
+        self.assertIn(
+            f'You have removed 1 layer from your project', str(change_notification.text)
+        )
+        finder = lambda driver: self.find('#hide-alert')
+        hide_button = self.wait_until_element_clickable(finder)
+        hide_button.click()
+        # check add layer button works
+        self.wait_until_not_visible('#change-notification')
+        finder = lambda driver: self.find('#add-remove-layer-btn')
+        add_layer_btn = self.wait_until_element_clickable(finder)
+        add_layer_btn.click()
+        self.wait_until_visible('#change-notification')
+        change_notification = self.find('#change-notification')
+        self.assertIn(
+            f'You have added 1 layer to your project', str(change_notification.text)
+        )
+        finder = lambda driver: self.find('#hide-alert')
+        hide_button = self.wait_until_element_clickable(finder)
+        hide_button.click()
+        self.wait_until_not_visible('#change-notification')
+        # check tabs(layers, recipes, machines) are displayed
+        tabs = self.find_all('.nav-tabs li')
+        self.assertEqual(len(tabs), 3)
+        # Check first tab
+        tabs[0].click()
+        self.assertIn(
+            'active', str(self.find('#information').get_attribute('class'))
+        )
+        # Check second tab (recipes)
+        self.wait_until_visible('.nav-tabs')
+        # Ensure page is scrolled to the top
+        self.driver.execute_script('window.scrollTo({behavior: "instant", top: 0, left: 0})')
+        tabs[1].click()
+        self.assertIn(
+            'active', str(self.find('#recipes').get_attribute('class'))
+        )
+        # Check third tab (machines)
+        self.wait_until_visible('.nav-tabs')
+        # Ensure page is scrolled to the top
+        self.driver.execute_script('window.scrollTo({behavior: "instant", top: 0, left: 0})')
+        tabs[2].click()
+        self.assertIn(
+            'active', str(self.find('#machines').get_attribute('class'))
+        )
+        # Check left section is displayed
+        section = self.find('.well')
+        # Check layer name
+        self.assertTrue(
+            section.find_element(By.XPATH, '//h2[1]').is_displayed()
+        )
+        # Check layer summary
+        self.assertIn("Summary", section.text)
+        # Check layer description
+        self.assertIn("Description", section.text)
+
+@pytest.mark.django_db
+@pytest.mark.order("last")
+class TestProjectPageRecipes(TestProjectPageBase):
+
+    def test_single_recipe_page(self):
+        """ Test recipe page
+            - Check if title is displayed
+            - Check add recipe layer displayed
+            - Check left section is displayed
+                - Check recipe: name, summary, description, Version, Section,
+                License, Approx. packages included, Approx. size, Recipe file
+        """
+        # Use a recipe which is likely to exist in the layer index but not enabled
+        # in poky out the box - xen-image-minimal from meta-virtualization
+        self._navigate_to_project_page()
+        prj = Project.objects.get(pk=TestProjectPageBase.project_id)
+        recipe_id = prj.get_all_compatible_recipes().get(name="xen-image-minimal").pk
+        url = reverse("recipedetails", args=(TestProjectPageBase.project_id, recipe_id))
+        self.get(url)
+        self.wait_until_visible('.page-header')
+        # check title is displayed
+        self.assertTrue(self.find('.page-header h1').is_displayed())
+        # check add recipe layer displayed
+        add_recipe_layer_btn = self.find('#add-layer-btn')
+        self.assertTrue(add_recipe_layer_btn.is_displayed())
+        # check left section is displayed
+        section = self.find('.well')
+        # Check recipe name
+        self.assertTrue(
+            section.find_element(By.XPATH, '//h2[1]').is_displayed()
+        )
+        # Check recipe sections details info are displayed
+        self.assertIn("Summary", section.text)
+        self.assertIn("Description", section.text)
+        self.assertIn("Version", section.text)
+        self.assertIn("Section", section.text)
+        self.assertIn("License", section.text)
+        self.assertIn("Approx. packages included", section.text)
+        self.assertIn("Approx. package size", section.text)
+        self.assertIn("Recipe file", section.text)
+
+    def test_image_recipe_editColumn(self):
+        """ Test the edit column feature in image recipe table on project page """
+        self._get_create_builds(success=10, failure=10)
+
+        url = reverse('projectimagerecipes', args=(TestProjectPageBase.project_id,))
+        self.get(url)
+        self.wait_until_present('#imagerecipestable tbody tr')
+
+        column_list = [
+            'get_description_or_summary', 'layer_version__get_vcs_reference',
+            'layer_version__layer__name', 'license', 'recipe-file', 'section',
+            'version'
+        ]
+
+        # Check that we can hide the edit column
+        self._mixin_test_table_edit_column(
+            'imagerecipestable',
+            'edit-columns-button',
+            [f'checkbox-{column}' for column in column_list]
+        )
+
diff --git a/bitbake/lib/toaster/tests/functional/test_project_page_tab_config.py b/bitbake/lib/toaster/tests/functional/test_project_page_tab_config.py
new file mode 100644
index 0000000000..80c53e1544
--- /dev/null
+++ b/bitbake/lib/toaster/tests/functional/test_project_page_tab_config.py
@@ -0,0 +1,507 @@
+#! /usr/bin/env python3 #
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import string
+import time
+import pytest
+from django.urls import reverse
+from selenium.webdriver import Keys
+from selenium.webdriver.support.select import Select
+from selenium.common.exceptions import ElementClickInterceptedException, NoSuchElementException, TimeoutException
+from tests.functional.functional_helpers import SeleniumFunctionalTestCase
+from selenium.webdriver.common.by import By
+
+from .utils import get_projectId_from_url, wait_until_build, wait_until_build_cancelled
+
+class TestProjectConfigTabBase(SeleniumFunctionalTestCase):
+    PROJECT_NAME = 'TestProjectConfigTab'
+    project_id = None
+
+    def _navigate_to_project_page(self):
+        # Navigate to project page
+        if TestProjectConfigTabBase.project_id is None:
+            TestProjectConfigTabBase.project_id = self.create_new_project(self.PROJECT_NAME, '3', None, True)
+        url = reverse('project', args=(TestProjectConfigTabBase.project_id,))
+        self.get(url)
+        self.wait_until_visible('#config-nav')
+
+    def _create_builds(self):
+        # check search box can be use to build recipes
+        search_box = self.find('#build-input')
+        search_box.send_keys('foo')
+        self.find('#build-button').click()
+        self.wait_until_present('#latest-builds')
+        # loop until reach the parsing state
+        wait_until_build(self, 'queued cloning starting parsing failed')
+        lastest_builds = self.driver.find_elements(
+            By.XPATH,
+            '//div[@id="latest-builds"]/div',
+        )
+        last_build = lastest_builds[0]
+        self.assertIn(
+            'foo', str(last_build.text)
+        )
+        last_build = lastest_builds[0]
+        try:
+            cancel_button = last_build.find_element(
+                By.XPATH,
+                '//span[@class="cancel-build-btn pull-right alert-link"]',
+            )
+            cancel_button.click()
+        except NoSuchElementException:
+            # Skip if the build is already cancelled
+            pass
+        wait_until_build_cancelled(self)
+
+    def _get_tabs(self):
+        # tabs links list
+        return self.driver.find_elements(
+            By.XPATH,
+            '//div[@id="project-topbar"]//li'
+        )
+
+    def _get_config_nav_item(self, index):
+        config_nav = self.find('#config-nav')
+        return config_nav.find_elements(By.TAG_NAME, 'li')[index]
+
+class TestProjectConfigTab(TestProjectConfigTabBase):
+
+    def test_project_config_nav(self):
+        """ Test project config tab navigation:
+        - Check if the menu is displayed and contains the right elements:
+            - Configuration
+            - COMPATIBLE METADATA
+            - Custom images
+            - Image recipes
+            - Software recipes
+            - Machines
+            - Layers
+            - Distro
+            - EXTRA CONFIGURATION
+            - Bitbake variables
+            - Actions
+            - Delete project
+        """
+        self._navigate_to_project_page()
+
+        def _get_config_nav_item(index):
+            config_nav = self.find('#config-nav')
+            return config_nav.find_elements(By.TAG_NAME, 'li')[index]
+
+        def check_config_nav_item(index, item_name, url):
+            item = _get_config_nav_item(index)
+            self.assertIn(item_name, item.text)
+            self.assertEqual(item.get_attribute('class'), 'active')
+            self.assertIn(url, self.driver.current_url)
+
+        # check if the menu contains the right elements
+        # COMPATIBLE METADATA
+        compatible_metadata = _get_config_nav_item(1)
+        self.assertIn(
+            "compatible metadata", compatible_metadata.text.lower()
+        )
+        # EXTRA CONFIGURATION
+        extra_configuration = _get_config_nav_item(8)
+        self.assertIn(
+            "extra configuration", extra_configuration.text.lower()
+        )
+        # Actions
+        actions = _get_config_nav_item(10)
+        self.assertIn("actions", str(actions.text).lower())
+
+        conf_nav_list = [
+            # config
+            [0, 'Configuration',
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}"],
+            # custom images
+            [2, 'Custom images',
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}/customimages"],
+            # image recipes
+            [3, 'Image recipes',
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}/images"],
+            # software recipes
+            [4, 'Software recipes',
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}/softwarerecipes"],
+            # machines
+            [5, 'Machines',
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}/machines"],
+            # layers
+            [6, 'Layers',
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}/layers"],
+            # distro
+            [7, 'Distros',
+                f"/toastergui/project/{TestProjectConfigTabBase.project_id}/distros"],
+            #  [9, 'BitBake variables', f"/toastergui/project/{TestProjectConfigTabBase.project_id}/configuration"],  # bitbake variables
+        ]
+        for index, item_name, url in conf_nav_list:
+            item = _get_config_nav_item(index)
+            if item.get_attribute('class') != 'active':
+                item.click()
+            check_config_nav_item(index, item_name, url)
+
+    def test_image_recipe_editColumn(self):
+        """ Test the edit column feature in image recipe table on project page """
+        def test_edit_column(check_box_id):
+            # Check that we can hide/show table column
+            check_box = self.find(f'#{check_box_id}')
+            th_class = str(check_box_id).replace('checkbox-', '')
+            if check_box.is_selected():
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#imagerecipestable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+                check_box.click()
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#imagerecipestable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+            else:
+                # check if column is hidden in table
+                self.assertFalse(
+                    self.find(
+                        f'#imagerecipestable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is unchecked in EditColumn dropdown, but it's visible in table"
+                )
+                check_box.click()
+                # check if column is visible in table
+                self.assertTrue(
+                    self.find(
+                        f'#imagerecipestable thead th.{th_class}'
+                    ).is_displayed(),
+                    f"The {th_class} column is checked in EditColumn dropdown, but it's not visible in table"
+                )
+
+        self._navigate_to_project_page()
+        # navigate to project image recipe page
+        recipe_image_page_link = self._get_config_nav_item(3)
+        recipe_image_page_link.click()
+        self.wait_until_present('#imagerecipestable tbody tr')
+
+        # Check edit column
+        edit_column = self.find('#edit-columns-button')
+        self.assertTrue(edit_column.is_displayed())
+        edit_column.click()
+        # Check dropdown is visible
+        self.wait_until_visible('ul.dropdown-menu.editcol')
+
+        # Check that we can hide the edit column
+        test_edit_column('checkbox-get_description_or_summary')
+        test_edit_column('checkbox-layer_version__get_vcs_reference')
+        test_edit_column('checkbox-layer_version__layer__name')
+        test_edit_column('checkbox-license')
+        test_edit_column('checkbox-recipe-file')
+        test_edit_column('checkbox-section')
+        test_edit_column('checkbox-version')
+
+    def test_image_recipe_show_rows(self):
+        """ Test the show rows feature in image recipe table on project page """
+        def test_show_rows(row_to_show, show_row_link):
+            # Check that we can show rows == row_to_show
+            show_row_link.select_by_value(str(row_to_show))
+            self.wait_until_visible('#imagerecipestable tbody tr')
+            # check at least some rows are visible
+            self.assertTrue(
+                len(self.find_all('#imagerecipestable tbody tr'))  > 0
+            )
+
+        self._navigate_to_project_page()
+        # navigate to project image recipe page
+        recipe_image_page_link = self._get_config_nav_item(3)
+        recipe_image_page_link.click()
+        self.wait_until_present('#imagerecipestable tbody tr')
+
+        show_rows = self.driver.find_elements(
+            By.XPATH,
+            '//select[@class="form-control pagesize-imagerecipestable"]'
+        )
+        # Check show rows
+        for show_row_link in show_rows:
+            show_row_link = Select(show_row_link)
+            test_show_rows(10, show_row_link)
+            test_show_rows(25, show_row_link)
+            test_show_rows(50, show_row_link)
+            test_show_rows(100, show_row_link)
+            test_show_rows(150, show_row_link)
+
+    def test_project_config_tab_right_section(self):
+        """ Test project config tab right section contains five blocks:
+            - Machine:
+                - check 'Machine' is displayed
+                - check can change Machine
+            - Distro:
+                - check 'Distro' is displayed
+                - check can change Distro
+            - Most built recipes:
+                - check 'Most built recipes' is displayed
+                - check can select a recipe and build it
+            - Project release:
+                - check 'Project release' is displayed
+                - check project has right release displayed
+            - Layers:
+                - check can add a layer if exists
+                - check at least three layers are displayed
+                    - openembedded-core
+                    - meta-poky
+                    - meta-yocto-bsp
+        """
+        project_id = self.create_new_project(self.PROJECT_NAME + "-ST", '3', None, True)
+        url = reverse('project', args=(project_id,))
+        self.get(url)
+        self.wait_until_visible('#config-nav')
+
+        # check if the menu is displayed
+        self.wait_until_visible('#project-page')
+        block_l = self.driver.find_element(
+            By.XPATH, '//*[@id="project-page"]/div[2]')
+        project_release = self.driver.find_element(
+            By.XPATH, '//*[@id="project-page"]/div[1]/div[4]')
+        layers = block_l.find_element(By.ID, 'layer-container')
+
+        def check_machine_distro(self, item_name, new_item_name, block_id):
+            block = self.find(f'#{block_id}')
+            title = block.find_element(By.TAG_NAME, 'h3')
+            self.assertIn(item_name.capitalize(), title.text)
+            edit_btn = self.find(f'#change-{item_name}-toggle')
+            edit_btn.click()
+            self.wait_until_visible(f'#{item_name}-change-input')
+            name_input = self.find(f'#{item_name}-change-input')
+            name_input.clear()
+            name_input.send_keys(new_item_name)
+            change_btn = self.find(f'#{item_name}-change-btn')
+            change_btn.click()
+            self.wait_until_visible(f'#project-{item_name}-name')
+            project_name = self.find(f'#project-{item_name}-name')
+            self.assertIn(new_item_name, project_name.text)
+            # check change notificaiton is displayed
+            change_notification = self.find('#change-notification')
+            self.assertIn(
+                f'You have changed the {item_name} to: {new_item_name}', change_notification.text
+            )
+            hide_button = self.find('#hide-alert')
+            hide_button.click()
+            self.wait_until_not_visible('#change-notification')
+
+        # Machine
+        check_machine_distro(self, 'machine', 'qemux86-64', 'machine-section')
+        # Distro
+        check_machine_distro(self, 'distro', 'poky-altcfg', 'distro-section')
+
+        # Project release
+        title = project_release.find_element(By.TAG_NAME, 'h3')
+        self.assertIn("Project release", title.text)
+        self.assertIn(
+            "Yocto Project master", self.find('#project-release-title').text
+        )
+        # Layers
+        title = layers.find_element(By.TAG_NAME, 'h3')
+        self.assertIn("Layers", title.text)
+        self.wait_until_clickable('#layer-add-input')
+        # check at least three layers are displayed
+        # openembedded-core
+        # meta-poky
+        # meta-yocto-bsp
+        layer_list_items = []
+        starttime = time.time()
+        while len(layer_list_items) < 3:
+            layers_list = self.driver.find_element(By.ID, 'layers-in-project-list')
+            layer_list_items = layers_list.find_elements(By.TAG_NAME, 'li')
+            if time.time() > (starttime + 30):
+                self.fail("Layer list didn't contain at least 3 items within 30s (contained %d)" % len(layer_list_items))
+
+        # remove all layers except the first three layers
+        for i in range(3, len(layer_list_items)):
+            layer_list_items[i].find_element(By.TAG_NAME, 'span').click()
+
+        # check can add a layer if exists
+        add_layer_input = layers.find_element(By.ID, 'layer-add-input')
+        add_layer_input.send_keys('meta-oe')
+        self.wait_until_visible('#layer-container > form > div > span > div')
+        self.wait_until_visible('.dropdown-menu')
+        finder = lambda driver: driver.find_element(By.XPATH, '//*[@id="layer-container"]/form/div/span/div/div/div')
+        dropdown_item = self.wait_until_element_clickable(finder)
+        dropdown_item.click()
+        self.wait_until_clickable('#add-layer-btn')
+        add_layer_btn = layers.find_element(By.ID, 'add-layer-btn')
+        add_layer_btn.click()
+        self.wait_until_visible('#layers-in-project-list')
+
+        # check layer is added
+        layer_list_items = []
+        starttime = time.time()
+        while len(layer_list_items) < 4:
+            layers_list = self.driver.find_element(By.ID, 'layers-in-project-list')
+            layer_list_items = layers_list.find_elements(By.TAG_NAME, 'li')
+            if time.time() > (starttime + 30):
+                self.fail("Layer list didn't contain at least 4 items within 30s (contained %d)" % len(layer_list_items))
+
+    def test_project_page_tab_importlayer(self):
+        """ Test project page tab import layer """
+        self._navigate_to_project_page()
+        # navigate to "Import layers" tab
+        import_layers_tab = self._get_tabs()[2]
+        import_layers_tab.find_element(By.TAG_NAME, 'a').click()
+        self.wait_until_visible('#layer-git-repo-url')
+
+        # Check git repo radio button
+        git_repo_radio = self.find('#git-repo-radio')
+        git_repo_radio.click()
+
+        # Set git repo url
+        input_repo_url = self.find('#layer-git-repo-url')
+        input_repo_url.send_keys('git://git.yoctoproject.org/meta-fake')
+        # Blur the input to trigger the validation
+        input_repo_url.send_keys(Keys.TAB)
+
+        # Check name is set
+        input_layer_name = self.find('#import-layer-name')
+        self.assertTrue(input_layer_name.get_attribute('value') == 'meta-fake')
+
+        # Set branch
+        input_branch = self.find('#layer-git-ref')
+        input_branch.send_keys('master')
+
+        # Import layer
+        self.find('#import-and-add-btn').click()
+
+        # Check layer is added
+        self.wait_until_visible('#layer-container')
+        block_l = self.driver.find_element(
+            By.XPATH, '//*[@id="project-page"]/div[2]')
+        layers = block_l.find_element(By.ID, 'layer-container')
+        layers_list = layers.find_element(By.ID, 'layers-in-project-list')
+        layers_list_items = layers_list.find_elements(By.TAG_NAME, 'li')
+        self.assertIn(
+            'meta-fake', str(layers_list_items[-1].text)
+        )
+
+    def test_project_page_custom_image_no_image(self):
+        """ Test project page tab "New custom image" when no custom image """
+        project_id = self.create_new_project(self.PROJECT_NAME + "-CustomImage", '3', None, True)
+        url = reverse('project', args=(project_id,))
+        self.get(url)
+        self.wait_until_visible('#config-nav')
+
+        # navigate to "Custom image" tab
+        custom_image_section = self._get_config_nav_item(2)
+        custom_image_section.click()
+        self.wait_until_visible('#empty-state-customimagestable')
+
+        # Check message when no custom image
+        self.assertIn(
+            "You have not created any custom images yet.", str(
+                self.find('#empty-state-customimagestable').text
+            )
+        )
+        div_empty_msg = self.find('#empty-state-customimagestable')
+        link_create_custom_image = div_empty_msg.find_element(
+            By.TAG_NAME, 'a')
+        self.assertTrue(TestProjectConfigTabBase.project_id is not None)
+        self.assertIn(
+            f"/toastergui/project/{project_id}/newcustomimage", str(
+                link_create_custom_image.get_attribute('href')
+            )
+        )
+        self.assertIn(
+            "Create your first custom image", str(
+                link_create_custom_image.text
+            )
+        )
+
+    def test_project_page_image_recipe(self):
+        """ Test project page section images
+            - Check image recipes are displayed
+            - Check search input
+            - Check image recipe build button works
+            - Check image recipe table features(show/hide column, pagination)
+        """
+        self._navigate_to_project_page()
+        # navigate to "Images section"
+        images_section = self._get_config_nav_item(3)
+        images_section.click()
+        self.wait_until_visible('#imagerecipestable')
+        rows = self.find_all('#imagerecipestable tbody tr')
+        self.assertTrue(len(rows) > 0)
+
+        # Test search input
+        self.wait_until_visible('#search-input-imagerecipestable')
+        recipe_input = self.find('#search-input-imagerecipestable')
+        recipe_input.send_keys('core-image-minimal')
+        self.find('#search-submit-imagerecipestable').click()
+        self.wait_until_visible('#imagerecipestable tbody tr')
+        rows = self.find_all('#imagerecipestable tbody tr')
+        self.assertTrue(len(rows) > 0)
+
+@pytest.mark.django_db
+@pytest.mark.order("last") 
+class TestProjectConfigTabDB(TestProjectConfigTabBase):
+
+    def test_most_build_recipes(self):
+        """ Test most build recipes block contains"""
+        def rebuild_from_most_build_recipes(recipe_list_items):
+            checkbox = recipe_list_items[0].find_element(By.TAG_NAME, 'input')
+            checkbox.click()
+            build_btn = self.find('#freq-build-btn')
+            build_btn.click()
+            self.wait_until_visible('#latest-builds')
+            wait_until_build(self, 'queued cloning starting parsing failed')
+            lastest_builds = self.driver.find_elements(
+                By.XPATH,
+                '//div[@id="latest-builds"]/div'
+            )
+            self.assertTrue(len(lastest_builds) >= 2)
+            last_build = lastest_builds[0]
+            try:
+                cancel_button = last_build.find_element(
+                    By.XPATH,
+                    '//span[@class="cancel-build-btn pull-right alert-link"]',
+                )
+                cancel_button.click()
+            except NoSuchElementException:
+                # Skip if the build is already cancelled
+                pass
+            wait_until_build_cancelled(self)
+
+        # Create a new project for remaining asserts
+        project_id = self.create_new_project(self.PROJECT_NAME + "-MostBuilt", '2', None, True)
+        url = reverse('project', args=(project_id,))
+        self.get(url)
+        self.wait_until_visible('#config-nav')
+
+        current_url = self.driver.current_url
+        url = current_url.split('?')[0]
+
+        # Create a new builds
+        self._create_builds()
+
+        # back to project page
+        self.driver.get(url)
+
+        self.wait_until_visible('#project-page')
+
+        # Most built recipes
+        most_built_recipes = self.driver.find_element(
+            By.XPATH, '//*[@id="project-page"]/div[1]/div[3]')
+        title = most_built_recipes.find_element(By.TAG_NAME, 'h3')
+        self.assertIn("Most built recipes", title.text)
+        # check can select a recipe and build it
+        self.wait_until_visible('#freq-build-list')
+        recipe_list = self.find('#freq-build-list')
+        recipe_list_items = recipe_list.find_elements(By.TAG_NAME, 'li')
+        self.assertTrue(
+            len(recipe_list_items) > 0,
+            msg="No recipes found in the most built recipes list",
+        )
+        rebuild_from_most_build_recipes(recipe_list_items)
+
diff --git a/bitbake/lib/toaster/tests/functional/utils.py b/bitbake/lib/toaster/tests/functional/utils.py
new file mode 100644
index 0000000000..72345aef9f
--- /dev/null
+++ b/bitbake/lib/toaster/tests/functional/utils.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+
+
+from time import sleep
+from selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException, TimeoutException, WebDriverException
+from selenium.webdriver.common.by import By
+
+from orm.models import Build
+
+
+def wait_until_build(test_instance, state):
+    timeout = 60
+    start_time = 0
+    build_state = ''
+    while True:
+        try:
+            if start_time > timeout:
+                raise TimeoutException(
+                    f'Build did not reach {state} state within {timeout} seconds'
+                )
+            last_build_state = test_instance.driver.find_element(
+                By.XPATH,
+                '//*[@id="latest-builds"]/div[1]//div[@class="build-state"]',
+            )
+            build_state = last_build_state.get_attribute(
+                'data-build-state')
+            state_text = state.lower().split()
+            if any(x in str(build_state).lower() for x in state_text):
+                return str(build_state).lower()
+            if 'failed' in str(build_state).lower():
+                break
+        except NoSuchElementException:
+            pass
+        except TimeoutException:
+            break
+        start_time += 1
+        sleep(1) # take a breath and try again
+
+def wait_until_build_cancelled(test_instance):
+    """ Cancel build take a while sometime, the method is to wait driver action
+        until build being cancelled
+    """
+    timeout = 30
+    start_time = 0
+    while True:
+        try:
+            if start_time > timeout:
+                raise TimeoutException(
+                    f'Build did not reach cancelled state within {timeout} seconds'
+                )
+            last_build_state = test_instance.driver.find_element(
+                By.XPATH,
+                '//*[@id="latest-builds"]/div[1]//div[@class="build-state"]',
+            )
+            build_state = last_build_state.get_attribute(
+                'data-build-state')
+            if 'failed' in str(build_state).lower():
+                break
+            if 'cancelling' in str(build_state).lower():
+                pass
+            if 'cancelled' in str(build_state).lower():
+                break
+        except TimeoutException:
+            break
+        except NoSuchElementException:
+            pass
+        except StaleElementReferenceException:
+            pass
+        except WebDriverException:
+            pass
+        start_time += 1
+        sleep(1) # take a breath and try again
+
+def get_projectId_from_url(url):
+    # url = 'http://domainename.com/toastergui/project/1656/whatever
+    # or url = 'http://domainename.com/toastergui/project/1/
+    # or url = 'http://domainename.com/toastergui/project/186
+    assert '/toastergui/project/' in url, "URL is not valid"
+    url_to_list = url.split('/toastergui/project/')
+    return  int(url_to_list[1].split('/')[0])  # project_id
diff --git a/bitbake/lib/toaster/tests/toaster-tests-requirements.txt b/bitbake/lib/toaster/tests/toaster-tests-requirements.txt
index 4f9fcc46d2..6243c00a36 100644
--- a/bitbake/lib/toaster/tests/toaster-tests-requirements.txt
+++ b/bitbake/lib/toaster/tests/toaster-tests-requirements.txt
@@ -1 +1,9 @@
-selenium==2.49.2
+selenium>=4.13.0
+pytest==7.4.2
+pytest-django==4.5.2
+pytest-env==1.1.0
+pytest-html==4.0.2
+pytest-metadata==3.0.0
+pytest-order==1.1.0
+requests
+
diff --git a/bitbake/lib/toaster/tests/views/test_views.py b/bitbake/lib/toaster/tests/views/test_views.py
index 735d596bcc..e1adfcf86a 100644
--- a/bitbake/lib/toaster/tests/views/test_views.py
+++ b/bitbake/lib/toaster/tests/views/test_views.py
@@ -9,6 +9,8 @@
 
 """Test cases for Toaster GUI and ReST."""
 
+import os
+import pytest
 from django.test import TestCase
 from django.test.client import RequestFactory
 from django.urls import reverse
@@ -19,6 +21,7 @@ from orm.models import Layer_Version, Recipe
 from orm.models import CustomImageRecipe
 from orm.models import CustomImagePackage
 
+from bldcontrol.models import BuildEnvironment
 import inspect
 import toastergui
 
@@ -32,19 +35,32 @@ PROJECT_NAME2 = "test project 2"
 CLI_BUILDS_PROJECT_NAME = 'Command line builds'
 
 
+
 class ViewTests(TestCase):
     """Tests to verify view APIs."""
 
     fixtures = ['toastergui-unittest-data']
+    builldir = os.environ.get('BUILDDIR')
 
     def setUp(self):
 
         self.project = Project.objects.first()
+
         self.recipe1 = Recipe.objects.get(pk=2)
+        # create a file and to recipe1 file_path
+        file_path = f"{self.builldir}/{self.recipe1.name.strip().replace(' ', '-')}.bb"
+        with open(file_path, 'w') as f:
+            f.write('foo')
+        self.recipe1.file_path = file_path
+        self.recipe1.save()
+
         self.customr = CustomImageRecipe.objects.first()
         self.cust_package = CustomImagePackage.objects.first()
         self.package = Package.objects.first()
         self.lver = Layer_Version.objects.first()
+        if BuildEnvironment.objects.count() == 0:
+            BuildEnvironment.objects.create(betype=BuildEnvironment.TYPE_LOCAL)
+
 
     def test_get_base_call_returns_html(self):
         """Basic test for all-projects view"""
@@ -226,7 +242,7 @@ class ViewTests(TestCase):
         recipe = CustomImageRecipe.objects.create(
                      name=name, project=self.project,
                      base_recipe=self.recipe1,
-                     file_path="/tmp/testing",
+                     file_path=f"{self.builldir}/testing",
                      layer_version=self.customr.layer_version)
         url = reverse('xhr_customrecipe_id', args=(recipe.id,))
         response = self.client.delete(url)
@@ -297,7 +313,7 @@ class ViewTests(TestCase):
         """Download the recipe file generated for the custom image"""
 
         # Create a dummy recipe file for the custom image generation to read
-        open("/tmp/a_recipe.bb", 'a').close()
+        open(f"{self.builldir}/a_recipe.bb", 'a').close()
         response = self.client.get(reverse('customrecipedownload',
                                            args=(self.project.id,
                                                  self.customr.id)))
diff --git a/bitbake/lib/toaster/toastergui/api.py b/bitbake/lib/toaster/toastergui/api.py
index b4cdc335ef..e367bd910e 100644
--- a/bitbake/lib/toaster/toastergui/api.py
+++ b/bitbake/lib/toaster/toastergui/api.py
@@ -11,7 +11,7 @@ import os
 import re
 import logging
 import json
-import subprocess
+import glob
 from collections import Counter
 
 from orm.models import Project, ProjectTarget, Build, Layer_Version
@@ -227,20 +227,18 @@ class XhrSetDefaultImageUrl(View):
 #    same logical name
 #  * Each project that uses a layer will have its own
 #    LayerVersion and Project Layer for it
-#  * During the Paroject delete process, when the last
+#  * During the Project delete process, when the last
 #    LayerVersion for a 'local_source_dir' layer is deleted
 #    then the Layer record is deleted to remove orphans
 #
 
 def scan_layer_content(layer,layer_version):
     # if this is a local layer directory, we can immediately scan its content
-    if layer.local_source_dir:
+    if os.path.isdir(layer.local_source_dir):
         try:
             # recipes-*/*/*.bb
-            cmd = '%s %s' % ('ls', os.path.join(layer.local_source_dir,'recipes-*/*/*.bb'))
-            recipes_list = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read()
-            recipes_list = recipes_list.decode("utf-8").strip()
-            if recipes_list and 'No such' not in recipes_list:
+            recipes_list = glob.glob(os.path.join(layer.local_source_dir, 'recipes-*/*/*.bb'))
+            for recipe in recipes_list:
                 for recipe in recipes_list.split('\n'):
                     recipe_path = recipe[recipe.rfind('recipes-'):]
                     recipe_name = recipe[recipe.rfind('/')+1:].replace('.bb','')
@@ -260,6 +258,9 @@ def scan_layer_content(layer,layer_version):
 
         except Exception as e:
             logger.warning("ERROR:scan_layer_content: %s" % e)
+    else:
+        logger.warning("ERROR: wrong path given")
+        raise KeyError("local_source_dir")
 
 class XhrLayer(View):
     """ Delete, Get, Add and Update Layer information
@@ -456,15 +457,18 @@ class XhrLayer(View):
                              'layerdetailurl':
                              layer_dep.get_detailspage_url(project.pk)})
 
-            # Scan the layer's content and update components
-            scan_layer_content(layer,layer_version)
+            # Only scan_layer_content if layer is local
+            if layer_data.get('local_source_dir', None):
+                # Scan the layer's content and update components
+                scan_layer_content(layer,layer_version)
 
         except Layer_Version.DoesNotExist:
             return error_response("layer-dep-not-found")
         except Project.DoesNotExist:
             return error_response("project-not-found")
-        except KeyError:
-            return error_response("incorrect-parameters")
+        except KeyError as e:
+            _log("KeyError: %s" % e)
+            return error_response(f"incorrect-parameters")
 
         return JsonResponse({'error': "ok",
                              'imported_layer': {
diff --git a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
index 4517ed1765..f626572fd1 100644
--- a/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
+++ b/bitbake/lib/toaster/toastergui/fixtures/toastergui-unittest-data.xml
@@ -6,10 +6,22 @@
       <field type="CharField" name="dirpath">b</field>
       <field type="CharField" name="branch">a</field>
   </object>
+  <object pk="1" model="orm.distro">
+     <field type="DateTimeField" name="up_date"><None></None></field>
+     <field to="orm.layer_version" name="layer_version" rel="ManyToOneRel">1</field>
+     <field type="CharField" name="name">poky_distro1</field>
+     <field type="CharField" name="description">poky_distro1 description</field>
+  </object>
+  <object pk="2" model="orm.distro">
+     <field type="DateTimeField" name="up_date"><None></None></field>
+     <field to="orm.layer_version" name="layer_version" rel="ManyToOneRel">2</field>
+     <field type="CharField" name="name">poky_distro2</field>
+     <field type="CharField" name="description">poky_distro2 description</field>
+  </object>
   <object pk="1" model="orm.release">
-     <field type="CharField" name="name">master</field>
+     <field type="CharField" name="name">foo_master</field>
      <field type="CharField" name="description">master project</field>
-     <field to="orm.bitbake_version" name="bitbake_version">1</field>
+     <field to="orm.bitbakeversion" name="bitbake_version">1</field>
   </object>
   <object pk="1" model="orm.project">
     <field type="CharField" name="name">a test project</field>
@@ -34,12 +46,12 @@
   <object pk="1" model="orm.ProjectVariable">
     <field to="orm.project" name="project" rel="ManyToOneRel">1</field>
     <field type="CharField" name="name">MACHINE</field>
-    <field type="TextField" name="value">qemux86</field>
+    <field type="TextField" name="value">qemux86-64</field>
   </object>
   <object pk="2" model="orm.ProjectVariable">
     <field to="orm.project" name="project" rel="ManyToOneRel">2</field>
     <field type="CharField" name="name">MACHINE</field>
-    <field type="TextField" name="value">qemux86</field>
+    <field type="TextField" name="value">qemux86-64</field>
   </object>
   <object pk="1" model="orm.build">
     <field to="orm.project" name="project" rel="ManyToOneRel">1</field>
@@ -67,7 +79,7 @@
   </object>
   <object pk="3" model="orm.build">
     <field to="orm.project" name="project" rel="ManyToOneRel">1</field>
-    <field type="CharField" name="machine">qemux86</field>
+    <field type="CharField" name="machine">qemux86-64</field>
     <field type="CharField" name="distro"></field>
     <field type="CharField" name="distro_version"></field>
     <field type="DateTimeField" name="started_on">2016-02-12T18:46:20.114530+00:00</field>
@@ -79,7 +91,7 @@
   </object>
   <object pk="4" model="orm.build">
     <field to="orm.project" name="project" rel="ManyToOneRel">2</field>
-    <field type="CharField" name="machine">qemux86</field>
+    <field type="CharField" name="machine">qemux86-64</field>
     <field type="CharField" name="distro"></field>
     <field type="CharField" name="distro_version"></field>
     <field type="DateTimeField" name="started_on">2016-02-11T18:46:20.114530+00:00</field>
diff --git a/bitbake/lib/toaster/toastergui/forms.py b/bitbake/lib/toaster/toastergui/forms.py
new file mode 100644
index 0000000000..0f279e06c5
--- /dev/null
+++ b/bitbake/lib/toaster/toastergui/forms.py
@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# BitBake Toaster UI tests implementation
+#
+# Copyright (C) 2023 Savoir-faire Linux
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+from django import forms
+from django.core.validators import FileExtensionValidator
+
+class LoadFileForm(forms.Form):
+    eventlog_file = forms.FileField(widget=forms.FileInput(attrs={'accept': '.json'}))
diff --git a/bitbake/lib/toaster/toastergui/static/css/default.css b/bitbake/lib/toaster/toastergui/static/css/default.css
index 5cd7e211a0..284355e70b 100644
--- a/bitbake/lib/toaster/toastergui/static/css/default.css
+++ b/bitbake/lib/toaster/toastergui/static/css/default.css
@@ -367,3 +367,31 @@ h2.panel-title { font-size: 30px; }
   }
 }
 /* End copied in from newer version of Font-Awesome 4.3.0 */
+
+
+#overlay {
+  display: flex;
+  position: fixed;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  background-color: rgba(0, 0, 0, 0.7);
+  align-items: center;
+  justify-content: center;
+  z-index: 999;
+}
+
+.spinner {
+  border: 6px solid rgba(255, 255, 255, 0.3);
+  border-radius: 50%;
+  border-top: 6px solid #3498db;
+  width: 50px;
+  height: 50px;
+  animation: spin 1s linear infinite;
+}
+
+@keyframes spin {
+  0% { transform: rotate(0deg); }
+  100% { transform: rotate(360deg); }
+}
diff --git a/bitbake/lib/toaster/toastergui/static/css/jquery.dataTables-1.13.8.min.css b/bitbake/lib/toaster/toastergui/static/css/jquery.dataTables-1.13.8.min.css
new file mode 100644
index 0000000000..c0a442ce07
--- /dev/null
+++ b/bitbake/lib/toaster/toastergui/static/css/jquery.dataTables-1.13.8.min.css
@@ -0,0 +1 @@
+:root{--dt-row-selected: 13, 110, 253;--dt-row-selected-text: 255, 255, 255;--dt-row-selected-link: 9, 10, 11;--dt-row-stripe: 0, 0, 0;--dt-row-hover: 0, 0, 0;--dt-column-ordering: 0, 0, 0;--dt-html-background: white}:root.dark{--dt-html-background: rgb(33, 37, 41)}table.dataTable td.dt-control{text-align:center;cursor:pointer}table.dataTable td.dt-control:before{display:inline-block;color:rgba(0, 0, 0, 0.5);content:"â–¶"}table.dataTable tr.dt-hasChild td.dt-control:before{content:"â–¼"}html.dark table.dataTable td.dt-control:before{color:rgba(255, 255, 255, 0.5)}html.dark table.dataTable tr.dt-hasChild td.dt-control:before{color:rgba(255, 255, 255, 0.5)}table.dataTable thead>tr>th.sorting,table.dataTable thead>tr>th.sorting_asc,table.dataTable thead>tr>th.sorting_desc,table.dataTable thead>tr>th.sorting_asc_disabled,table.dataTable thead>tr>th.sorting_desc_disabled,table.dataTable thead>tr>td.sorting,table.dataTable thead>tr>td.sorting_asc,table.dataTable thead>tr>td.sorting_desc,table.dataTable thead>tr>td.sorting_asc_disabled,table.dataTable thead>tr>td.sorting_desc_disabled{cursor:pointer;position:relative;padding-right:26px}table.dataTable thead>tr>th.sorting:before,table.dataTable thead>tr>th.sorting:after,table.dataTable thead>tr>th.sorting_asc:before,table.dataTable thead>tr>th.sorting_asc:after,table.dataTable thead>tr>th.sorting_desc:before,table.dataTable thead>tr>th.sorting_desc:after,table.dataTable thead>tr>th.sorting_asc_disabled:before,table.dataTable thead>tr>th.sorting_asc_disabled:after,table.dataTable thead>tr>th.sorting_desc_disabled:before,table.dataTable thead>tr>th.sorting_desc_disabled:after,table.dataTable thead>tr>td.sorting:before,table.dataTable thead>tr>td.sorting:after,table.dataTable thead>tr>td.sorting_asc:before,table.dataTable thead>tr>td.sorting_asc:after,table.dataTable thead>tr>td.sorting_desc:before,table.dataTable thead>tr>td.sorting_desc:after,table.dataTable thead>tr>td.sorting_asc_disabled:before,table.dataTable thead>tr>td.sorting_asc_disabled:after,table.dataTable thead>tr>td.sorting_desc_disabled:before,table.dataTable thead>tr>td.sorting_desc_disabled:after{position:absolute;display:block;opacity:.125;right:10px;line-height:9px;font-size:.8em}table.dataTable thead>tr>th.sorting:before,table.dataTable thead>tr>th.sorting_asc:before,table.dataTable thead>tr>th.sorting_desc:before,table.dataTable thead>tr>th.sorting_asc_disabled:before,table.dataTable thead>tr>th.sorting_desc_disabled:before,table.dataTable thead>tr>td.sorting:before,table.dataTable thead>tr>td.sorting_asc:before,table.dataTable thead>tr>td.sorting_desc:before,table.dataTable thead>tr>td.sorting_asc_disabled:before,table.dataTable thead>tr>td.sorting_desc_disabled:before{bottom:50%;content:"â–²";content:"â–²"/""}table.dataTable thead>tr>th.sorting:after,table.dataTable thead>tr>th.sorting_asc:after,table.dataTable thead>tr>th.sorting_desc:after,table.dataTable thead>tr>th.sorting_asc_disabled:after,table.dataTable thead>tr>th.sorting_desc_disabled:after,table.dataTable thead>tr>td.sorting:after,table.dataTable thead>tr>td.sorting_asc:after,table.dataTable thead>tr>td.sorting_desc:after,table.dataTable thead>tr>td.sorting_asc_disabled:after,table.dataTable thead>tr>td.sorting_desc_disabled:after{top:50%;content:"â–¼";content:"â–¼"/""}table.dataTable thead>tr>th.sorting_asc:before,table.dataTable thead>tr>th.sorting_desc:after,table.dataTable thead>tr>td.sorting_asc:before,table.dataTable thead>tr>td.sorting_desc:after{opacity:.6}table.dataTable thead>tr>th.sorting_desc_disabled:after,table.dataTable thead>tr>th.sorting_asc_disabled:before,table.dataTable thead>tr>td.sorting_desc_disabled:after,table.dataTable thead>tr>td.sorting_asc_disabled:before{display:none}table.dataTable thead>tr>th:active,table.dataTable thead>tr>td:active{outline:none}div.dataTables_scrollBody>table.dataTable>thead>tr>th:before,div.dataTables_scrollBody>table.dataTable>thead>tr>th:after,div.dataTables_scrollBody>table.dataTable>thead>tr>td:before,div.dataTables_scrollBody>table.dataTable>thead>tr>td:after{display:none}div.dataTables_processing{position:absolute;top:50%;left:50%;width:200px;margin-left:-100px;margin-top:-26px;text-align:center;padding:2px;z-index:10}div.dataTables_processing>div:last-child{position:relative;width:80px;height:15px;margin:1em auto}div.dataTables_processing>div:last-child>div{position:absolute;top:0;width:13px;height:13px;border-radius:50%;background:rgb(13, 110, 253);background:rgb(var(--dt-row-selected));animation-timing-function:cubic-bezier(0, 1, 1, 0)}div.dataTables_processing>div:last-child>div:nth-child(1){left:8px;animation:datatables-loader-1 .6s infinite}div.dataTables_processing>div:last-child>div:nth-child(2){left:8px;animation:datatables-loader-2 .6s infinite}div.dataTables_processing>div:last-child>div:nth-child(3){left:32px;animation:datatables-loader-2 .6s infinite}div.dataTables_processing>div:last-child>div:nth-child(4){left:56px;animation:datatables-loader-3 .6s infinite}@keyframes datatables-loader-1{0%{transform:scale(0)}100%{transform:scale(1)}}@keyframes datatables-loader-3{0%{transform:scale(1)}100%{transform:scale(0)}}@keyframes datatables-loader-2{0%{transform:translate(0, 0)}100%{transform:translate(24px, 0)}}table.dataTable.nowrap th,table.dataTable.nowrap td{white-space:nowrap}table.dataTable th.dt-left,table.dataTable td.dt-left{text-align:left}table.dataTable th.dt-center,table.dataTable td.dt-center,table.dataTable td.dataTables_empty{text-align:center}table.dataTable th.dt-right,table.dataTable td.dt-right{text-align:right}table.dataTable th.dt-justify,table.dataTable td.dt-justify{text-align:justify}table.dataTable th.dt-nowrap,table.dataTable td.dt-nowrap{white-space:nowrap}table.dataTable thead th,table.dataTable thead td,table.dataTable tfoot th,table.dataTable tfoot td{text-align:left}table.dataTable thead th.dt-head-left,table.dataTable thead td.dt-head-left,table.dataTable tfoot th.dt-head-left,table.dataTable tfoot td.dt-head-left{text-align:left}table.dataTable thead th.dt-head-center,table.dataTable thead td.dt-head-center,table.dataTable tfoot th.dt-head-center,table.dataTable tfoot td.dt-head-center{text-align:center}table.dataTable thead th.dt-head-right,table.dataTable thead td.dt-head-right,table.dataTable tfoot th.dt-head-right,table.dataTable tfoot td.dt-head-right{text-align:right}table.dataTable thead th.dt-head-justify,table.dataTable thead td.dt-head-justify,table.dataTable tfoot th.dt-head-justify,table.dataTable tfoot td.dt-head-justify{text-align:justify}table.dataTable thead th.dt-head-nowrap,table.dataTable thead td.dt-head-nowrap,table.dataTable tfoot th.dt-head-nowrap,table.dataTable tfoot td.dt-head-nowrap{white-space:nowrap}table.dataTable tbody th.dt-body-left,table.dataTable tbody td.dt-body-left{text-align:left}table.dataTable tbody th.dt-body-center,table.dataTable tbody td.dt-body-center{text-align:center}table.dataTable tbody th.dt-body-right,table.dataTable tbody td.dt-body-right{text-align:right}table.dataTable tbody th.dt-body-justify,table.dataTable tbody td.dt-body-justify{text-align:justify}table.dataTable tbody th.dt-body-nowrap,table.dataTable tbody td.dt-body-nowrap{white-space:nowrap}table.dataTable{width:100%;margin:0 auto;clear:both;border-collapse:separate;border-spacing:0}table.dataTable thead th,table.dataTable tfoot th{font-weight:bold}table.dataTable>thead>tr>th,table.dataTable>thead>tr>td{padding:10px;border-bottom:1px solid rgba(0, 0, 0, 0.3)}table.dataTable>thead>tr>th:active,table.dataTable>thead>tr>td:active{outline:none}table.dataTable>tfoot>tr>th,table.dataTable>tfoot>tr>td{padding:10px 10px 6px 10px;border-top:1px solid rgba(0, 0, 0, 0.3)}table.dataTable tbody tr{background-color:transparent}table.dataTable tbody tr.selected>*{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.9);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.9);color:rgb(255, 255, 255);color:rgb(var(--dt-row-selected-text))}table.dataTable tbody tr.selected a{color:rgb(9, 10, 11);color:rgb(var(--dt-row-selected-link))}table.dataTable tbody th,table.dataTable tbody td{padding:8px 10px}table.dataTable.row-border>tbody>tr>th,table.dataTable.row-border>tbody>tr>td,table.dataTable.display>tbody>tr>th,table.dataTable.display>tbody>tr>td{border-top:1px solid rgba(0, 0, 0, 0.15)}table.dataTable.row-border>tbody>tr:first-child>th,table.dataTable.row-border>tbody>tr:first-child>td,table.dataTable.display>tbody>tr:first-child>th,table.dataTable.display>tbody>tr:first-child>td{border-top:none}table.dataTable.row-border>tbody>tr.selected+tr.selected>td,table.dataTable.display>tbody>tr.selected+tr.selected>td{border-top-color:#0262ef}table.dataTable.cell-border>tbody>tr>th,table.dataTable.cell-border>tbody>tr>td{border-top:1px solid rgba(0, 0, 0, 0.15);border-right:1px solid rgba(0, 0, 0, 0.15)}table.dataTable.cell-border>tbody>tr>th:first-child,table.dataTable.cell-border>tbody>tr>td:first-child{border-left:1px solid rgba(0, 0, 0, 0.15)}table.dataTable.cell-border>tbody>tr:first-child>th,table.dataTable.cell-border>tbody>tr:first-child>td{border-top:none}table.dataTable.stripe>tbody>tr.odd>*,table.dataTable.display>tbody>tr.odd>*{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.023);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-stripe), 0.023)}table.dataTable.stripe>tbody>tr.odd.selected>*,table.dataTable.display>tbody>tr.odd.selected>*{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.923);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.923)}table.dataTable.hover>tbody>tr:hover>*,table.dataTable.display>tbody>tr:hover>*{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.035);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-hover), 0.035)}table.dataTable.hover>tbody>tr.selected:hover>*,table.dataTable.display>tbody>tr.selected:hover>*{box-shadow:inset 0 0 0 9999px #0d6efd !important;box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 1) !important}table.dataTable.order-column>tbody tr>.sorting_1,table.dataTable.order-column>tbody tr>.sorting_2,table.dataTable.order-column>tbody tr>.sorting_3,table.dataTable.display>tbody tr>.sorting_1,table.dataTable.display>tbody tr>.sorting_2,table.dataTable.display>tbody tr>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.019);box-shadow:inset 0 0 0 9999px rgba(var(--dt-column-ordering), 0.019)}table.dataTable.order-column>tbody tr.selected>.sorting_1,table.dataTable.order-column>tbody tr.selected>.sorting_2,table.dataTable.order-column>tbody tr.selected>.sorting_3,table.dataTable.display>tbody tr.selected>.sorting_1,table.dataTable.display>tbody tr.selected>.sorting_2,table.dataTable.display>tbody tr.selected>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.919);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.919)}table.dataTable.display>tbody>tr.odd>.sorting_1,table.dataTable.order-column.stripe>tbody>tr.odd>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.054);box-shadow:inset 0 0 0 9999px rgba(var(--dt-column-ordering), 0.054)}table.dataTable.display>tbody>tr.odd>.sorting_2,table.dataTable.order-column.stripe>tbody>tr.odd>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.047);box-shadow:inset 0 0 0 9999px rgba(var(--dt-column-ordering), 0.047)}table.dataTable.display>tbody>tr.odd>.sorting_3,table.dataTable.order-column.stripe>tbody>tr.odd>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.039);box-shadow:inset 0 0 0 9999px rgba(var(--dt-column-ordering), 0.039)}table.dataTable.display>tbody>tr.odd.selected>.sorting_1,table.dataTable.order-column.stripe>tbody>tr.odd.selected>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.954);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.954)}table.dataTable.display>tbody>tr.odd.selected>.sorting_2,table.dataTable.order-column.stripe>tbody>tr.odd.selected>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.947);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.947)}table.dataTable.display>tbody>tr.odd.selected>.sorting_3,table.dataTable.order-column.stripe>tbody>tr.odd.selected>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.939);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.939)}table.dataTable.display>tbody>tr.even>.sorting_1,table.dataTable.order-column.stripe>tbody>tr.even>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.019);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.019)}table.dataTable.display>tbody>tr.even>.sorting_2,table.dataTable.order-column.stripe>tbody>tr.even>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.011);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.011)}table.dataTable.display>tbody>tr.even>.sorting_3,table.dataTable.order-column.stripe>tbody>tr.even>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.003);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.003)}table.dataTable.display>tbody>tr.even.selected>.sorting_1,table.dataTable.order-column.stripe>tbody>tr.even.selected>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.919);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.919)}table.dataTable.display>tbody>tr.even.selected>.sorting_2,table.dataTable.order-column.stripe>tbody>tr.even.selected>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.911);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.911)}table.dataTable.display>tbody>tr.even.selected>.sorting_3,table.dataTable.order-column.stripe>tbody>tr.even.selected>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.903);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.903)}table.dataTable.display tbody tr:hover>.sorting_1,table.dataTable.order-column.hover tbody tr:hover>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.082);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-hover), 0.082)}table.dataTable.display tbody tr:hover>.sorting_2,table.dataTable.order-column.hover tbody tr:hover>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.074);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-hover), 0.074)}table.dataTable.display tbody tr:hover>.sorting_3,table.dataTable.order-column.hover tbody tr:hover>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(0, 0, 0, 0.062);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-hover), 0.062)}table.dataTable.display tbody tr:hover.selected>.sorting_1,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_1{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.982);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.982)}table.dataTable.display tbody tr:hover.selected>.sorting_2,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_2{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.974);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.974)}table.dataTable.display tbody tr:hover.selected>.sorting_3,table.dataTable.order-column.hover tbody tr:hover.selected>.sorting_3{box-shadow:inset 0 0 0 9999px rgba(13, 110, 253, 0.962);box-shadow:inset 0 0 0 9999px rgba(var(--dt-row-selected), 0.962)}table.dataTable.no-footer{border-bottom:1px solid rgba(0, 0, 0, 0.3)}table.dataTable.compact thead th,table.dataTable.compact thead td,table.dataTable.compact tfoot th,table.dataTable.compact tfoot td,table.dataTable.compact tbody th,table.dataTable.compact tbody td{padding:4px}table.dataTable th,table.dataTable td{box-sizing:content-box}.dataTables_wrapper{position:relative;clear:both}.dataTables_wrapper .dataTables_length{float:left}.dataTables_wrapper .dataTables_length select{border:1px solid #aaa;border-radius:3px;padding:5px;background-color:transparent;color:inherit;padding:4px}.dataTables_wrapper .dataTables_filter{float:right;text-align:right}.dataTables_wrapper .dataTables_filter input{border:1px solid #aaa;border-radius:3px;padding:5px;background-color:transparent;color:inherit;margin-left:3px}.dataTables_wrapper .dataTables_info{clear:both;float:left;padding-top:.755em}.dataTables_wrapper .dataTables_paginate{float:right;text-align:right;padding-top:.25em}.dataTables_wrapper .dataTables_paginate .paginate_button{box-sizing:border-box;display:inline-block;min-width:1.5em;padding:.5em 1em;margin-left:2px;text-align:center;text-decoration:none !important;cursor:pointer;color:inherit !important;border:1px solid transparent;border-radius:2px;background:transparent}.dataTables_wrapper .dataTables_paginate .paginate_button.current,.dataTables_wrapper .dataTables_paginate .paginate_button.current:hover{color:inherit !important;border:1px solid rgba(0, 0, 0, 0.3);background-color:rgba(0, 0, 0, 0.05);background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, rgba(230, 230, 230, 0.05)), color-stop(100%, rgba(0, 0, 0, 0.05)));background:-webkit-linear-gradient(top, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%);background:-moz-linear-gradient(top, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%);background:-ms-linear-gradient(top, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%);background:-o-linear-gradient(top, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%);background:linear-gradient(to bottom, rgba(230, 230, 230, 0.05) 0%, rgba(0, 0, 0, 0.05) 100%)}.dataTables_wrapper .dataTables_paginate .paginate_button.disabled,.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover,.dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active{cursor:default;color:#666 !important;border:1px solid transparent;background:transparent;box-shadow:none}.dataTables_wrapper .dataTables_paginate .paginate_button:hover{color:white !important;border:1px solid #111;background-color:#111;background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #585858), color-stop(100%, #111));background:-webkit-linear-gradient(top, #585858 0%, #111 100%);background:-moz-linear-gradient(top, #585858 0%, #111 100%);background:-ms-linear-gradient(top, #585858 0%, #111 100%);background:-o-linear-gradient(top, #585858 0%, #111 100%);background:linear-gradient(to bottom, #585858 0%, #111 100%)}.dataTables_wrapper .dataTables_paginate .paginate_button:active{outline:none;background-color:#0c0c0c;background:-webkit-gradient(linear, left top, left bottom, color-stop(0%, #2b2b2b), color-stop(100%, #0c0c0c));background:-webkit-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-moz-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-ms-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:-o-linear-gradient(top, #2b2b2b 0%, #0c0c0c 100%);background:linear-gradient(to bottom, #2b2b2b 0%, #0c0c0c 100%);box-shadow:inset 0 0 3px #111}.dataTables_wrapper .dataTables_paginate .ellipsis{padding:0 1em}.dataTables_wrapper .dataTables_length,.dataTables_wrapper .dataTables_filter,.dataTables_wrapper .dataTables_info,.dataTables_wrapper .dataTables_processing,.dataTables_wrapper .dataTables_paginate{color:inherit}.dataTables_wrapper .dataTables_scroll{clear:both}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody{-webkit-overflow-scrolling:touch}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>th,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>td,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>th,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>td{vertical-align:middle}.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>th>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>thead>tr>td>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>th>div.dataTables_sizing,.dataTables_wrapper .dataTables_scroll div.dataTables_scrollBody>table>tbody>tr>td>div.dataTables_sizing{height:0;overflow:hidden;margin:0 !important;padding:0 !important}.dataTables_wrapper.no-footer .dataTables_scrollBody{border-bottom:1px solid rgba(0, 0, 0, 0.3)}.dataTables_wrapper.no-footer div.dataTables_scrollHead table.dataTable,.dataTables_wrapper.no-footer div.dataTables_scrollBody>table{border-bottom:none}.dataTables_wrapper:after{visibility:hidden;display:block;content:"";clear:both;height:0}@media screen and (max-width: 767px){.dataTables_wrapper .dataTables_info,.dataTables_wrapper .dataTables_paginate{float:none;text-align:center}.dataTables_wrapper .dataTables_paginate{margin-top:.5em}}@media screen and (max-width: 640px){.dataTables_wrapper .dataTables_length,.dataTables_wrapper .dataTables_filter{float:none;text-align:center}.dataTables_wrapper .dataTables_filter{margin-top:.5em}}html.dark{--dt-row-hover: 255, 255, 255;--dt-row-stripe: 255, 255, 255;--dt-column-ordering: 255, 255, 255}html.dark table.dataTable>thead>tr>th,html.dark table.dataTable>thead>tr>td{border-bottom:1px solid rgb(89, 91, 94)}html.dark table.dataTable>thead>tr>th:active,html.dark table.dataTable>thead>tr>td:active{outline:none}html.dark table.dataTable>tfoot>tr>th,html.dark table.dataTable>tfoot>tr>td{border-top:1px solid rgb(89, 91, 94)}html.dark table.dataTable.row-border>tbody>tr>th,html.dark table.dataTable.row-border>tbody>tr>td,html.dark table.dataTable.display>tbody>tr>th,html.dark table.dataTable.display>tbody>tr>td{border-top:1px solid rgb(64, 67, 70)}html.dark table.dataTable.row-border>tbody>tr.selected+tr.selected>td,html.dark table.dataTable.display>tbody>tr.selected+tr.selected>td{border-top-color:#0257d5}html.dark table.dataTable.cell-border>tbody>tr>th,html.dark table.dataTable.cell-border>tbody>tr>td{border-top:1px solid rgb(64, 67, 70);border-right:1px solid rgb(64, 67, 70)}html.dark table.dataTable.cell-border>tbody>tr>th:first-child,html.dark table.dataTable.cell-border>tbody>tr>td:first-child{border-left:1px solid rgb(64, 67, 70)}html.dark .dataTables_wrapper .dataTables_filter input,html.dark .dataTables_wrapper .dataTables_length select{border:1px solid rgba(255, 255, 255, 0.2);background-color:var(--dt-html-background)}html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.current,html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.current:hover{border:1px solid rgb(89, 91, 94);background:rgba(255, 255, 255, 0.15)}html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.disabled,html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:hover,html.dark .dataTables_wrapper .dataTables_paginate .paginate_button.disabled:active{color:#666 !important}html.dark .dataTables_wrapper .dataTables_paginate .paginate_button:hover{border:1px solid rgb(53, 53, 53);background:rgb(53, 53, 53)}html.dark .dataTables_wrapper .dataTables_paginate .paginate_button:active{background:#3a3a3a}
diff --git a/bitbake/lib/toaster/toastergui/static/js/bootstrap.js b/bitbake/lib/toaster/toastergui/static/js/bootstrap-3.4.1.js
index d47d640feb..170bd608f7 100644
--- a/bitbake/lib/toaster/toastergui/static/js/bootstrap.js
+++ b/bitbake/lib/toaster/toastergui/static/js/bootstrap-3.4.1.js
@@ -1,6 +1,6 @@
 /*!
- * Bootstrap v3.3.6 (http://getbootstrap.com)
- * Copyright 2011-2016 Twitter, Inc.
+ * Bootstrap v3.4.1 (https://getbootstrap.com/)
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under the MIT license
  */
 
@@ -11,16 +11,16 @@ if (typeof jQuery === 'undefined') {
 +function ($) {
   'use strict';
   var version = $.fn.jquery.split(' ')[0].split('.')
-  if ((version[0] < 2 && version[1] < 9) || (version[0] == 1 && version[1] == 9 && version[2] < 1) || (version[0] > 2)) {
-    throw new Error('Bootstrap\'s JavaScript requires jQuery version 1.9.1 or higher, but lower than version 3')
+  if ((version[0] < 2 && version[1] < 9) || (version[0] == 1 && version[1] == 9 && version[2] < 1) || (version[0] > 3)) {
+    throw new Error('Bootstrap\'s JavaScript requires jQuery version 1.9.1 or higher, but lower than version 4')
   }
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: transition.js v3.3.6
- * http://getbootstrap.com/javascript/#transitions
+ * Bootstrap: transition.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#transitions
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -28,7 +28,7 @@ if (typeof jQuery === 'undefined') {
 +function ($) {
   'use strict';
 
-  // CSS TRANSITION SUPPORT (Shoutout: http://www.modernizr.com/)
+  // CSS TRANSITION SUPPORT (Shoutout: https://modernizr.com/)
   // ============================================================
 
   function transitionEnd() {
@@ -50,7 +50,7 @@ if (typeof jQuery === 'undefined') {
     return false // explicit for ie8 (  ._.)
   }
 
-  // http://blog.alexmaccaw.com/css-transitions
+  // https://blog.alexmaccaw.com/css-transitions
   $.fn.emulateTransitionEnd = function (duration) {
     var called = false
     var $el = this
@@ -77,10 +77,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: alert.js v3.3.6
- * http://getbootstrap.com/javascript/#alerts
+ * Bootstrap: alert.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#alerts
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -96,7 +96,7 @@ if (typeof jQuery === 'undefined') {
     $(el).on('click', dismiss, this.close)
   }
 
-  Alert.VERSION = '3.3.6'
+  Alert.VERSION = '3.4.1'
 
   Alert.TRANSITION_DURATION = 150
 
@@ -109,7 +109,8 @@ if (typeof jQuery === 'undefined') {
       selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') // strip for ie7
     }
 
-    var $parent = $(selector)
+    selector    = selector === '#' ? [] : selector
+    var $parent = $(document).find(selector)
 
     if (e) e.preventDefault()
 
@@ -172,10 +173,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: button.js v3.3.6
- * http://getbootstrap.com/javascript/#buttons
+ * Bootstrap: button.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#buttons
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -192,7 +193,7 @@ if (typeof jQuery === 'undefined') {
     this.isLoading = false
   }
 
-  Button.VERSION  = '3.3.6'
+  Button.VERSION  = '3.4.1'
 
   Button.DEFAULTS = {
     loadingText: 'loading...'
@@ -214,10 +215,10 @@ if (typeof jQuery === 'undefined') {
 
       if (state == 'loadingText') {
         this.isLoading = true
-        $el.addClass(d).attr(d, d)
+        $el.addClass(d).attr(d, d).prop(d, true)
       } else if (this.isLoading) {
         this.isLoading = false
-        $el.removeClass(d).removeAttr(d)
+        $el.removeClass(d).removeAttr(d).prop(d, false)
       }
     }, this), 0)
   }
@@ -281,10 +282,15 @@ if (typeof jQuery === 'undefined') {
 
   $(document)
     .on('click.bs.button.data-api', '[data-toggle^="button"]', function (e) {
-      var $btn = $(e.target)
-      if (!$btn.hasClass('btn')) $btn = $btn.closest('.btn')
+      var $btn = $(e.target).closest('.btn')
       Plugin.call($btn, 'toggle')
-      if (!($(e.target).is('input[type="radio"]') || $(e.target).is('input[type="checkbox"]'))) e.preventDefault()
+      if (!($(e.target).is('input[type="radio"], input[type="checkbox"]'))) {
+        // Prevent double click on radios, and the double selections (so cancellation) on checkboxes
+        e.preventDefault()
+        // The target component still receive the focus
+        if ($btn.is('input,button')) $btn.trigger('focus')
+        else $btn.find('input:visible,button:visible').first().trigger('focus')
+      }
     })
     .on('focus.bs.button.data-api blur.bs.button.data-api', '[data-toggle^="button"]', function (e) {
       $(e.target).closest('.btn').toggleClass('focus', /^focus(in)?$/.test(e.type))
@@ -293,10 +299,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: carousel.js v3.3.6
- * http://getbootstrap.com/javascript/#carousel
+ * Bootstrap: carousel.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#carousel
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -324,7 +330,7 @@ if (typeof jQuery === 'undefined') {
       .on('mouseleave.bs.carousel', $.proxy(this.cycle, this))
   }
 
-  Carousel.VERSION  = '3.3.6'
+  Carousel.VERSION  = '3.4.1'
 
   Carousel.TRANSITION_DURATION = 600
 
@@ -438,7 +444,9 @@ if (typeof jQuery === 'undefined') {
     var slidEvent = $.Event('slid.bs.carousel', { relatedTarget: relatedTarget, direction: direction }) // yes, "slid"
     if ($.support.transition && this.$element.hasClass('slide')) {
       $next.addClass(type)
-      $next[0].offsetWidth // force reflow
+      if (typeof $next === 'object' && $next.length) {
+        $next[0].offsetWidth // force reflow
+      }
       $active.addClass(direction)
       $next.addClass(direction)
       $active
@@ -500,10 +508,17 @@ if (typeof jQuery === 'undefined') {
   // =================
 
   var clickHandler = function (e) {
-    var href
     var $this   = $(this)
-    var $target = $($this.attr('data-target') || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) // strip for ie7
+    var href    = $this.attr('href')
+    if (href) {
+      href = href.replace(/.*(?=#[^\s]+$)/, '') // strip for ie7
+    }
+
+    var target  = $this.attr('data-target') || href
+    var $target = $(document).find(target)
+
     if (!$target.hasClass('carousel')) return
+
     var options = $.extend({}, $target.data(), $this.data())
     var slideIndex = $this.attr('data-slide-to')
     if (slideIndex) options.interval = false
@@ -531,13 +546,14 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: collapse.js v3.3.6
- * http://getbootstrap.com/javascript/#collapse
+ * Bootstrap: collapse.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#collapse
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
+/* jshint latedef: false */
 
 +function ($) {
   'use strict';
@@ -561,7 +577,7 @@ if (typeof jQuery === 'undefined') {
     if (this.options.toggle) this.toggle()
   }
 
-  Collapse.VERSION  = '3.3.6'
+  Collapse.VERSION  = '3.4.1'
 
   Collapse.TRANSITION_DURATION = 350
 
@@ -668,7 +684,7 @@ if (typeof jQuery === 'undefined') {
   }
 
   Collapse.prototype.getParent = function () {
-    return $(this.options.parent)
+    return $(document).find(this.options.parent)
       .find('[data-toggle="collapse"][data-parent="' + this.options.parent + '"]')
       .each($.proxy(function (i, element) {
         var $element = $(element)
@@ -691,7 +707,7 @@ if (typeof jQuery === 'undefined') {
     var target = $trigger.attr('data-target')
       || (href = $trigger.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '') // strip for ie7
 
-    return $(target)
+    return $(document).find(target)
   }
 
 
@@ -743,10 +759,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: dropdown.js v3.3.6
- * http://getbootstrap.com/javascript/#dropdowns
+ * Bootstrap: dropdown.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#dropdowns
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -763,7 +779,7 @@ if (typeof jQuery === 'undefined') {
     $(element).on('click.bs.dropdown', this.toggle)
   }
 
-  Dropdown.VERSION = '3.3.6'
+  Dropdown.VERSION = '3.4.1'
 
   function getParent($this) {
     var selector = $this.attr('data-target')
@@ -773,7 +789,7 @@ if (typeof jQuery === 'undefined') {
       selector = selector && /#[A-Za-z]/.test(selector) && selector.replace(/.*(?=#[^\s]*$)/, '') // strip for ie7
     }
 
-    var $parent = selector && $(selector)
+    var $parent = selector !== '#' ? $(document).find(selector) : null
 
     return $parent && $parent.length ? $parent : $this.parent()
   }
@@ -909,10 +925,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: modal.js v3.3.6
- * http://getbootstrap.com/javascript/#modals
+ * Bootstrap: modal.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#modals
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -924,15 +940,16 @@ if (typeof jQuery === 'undefined') {
   // ======================
 
   var Modal = function (element, options) {
-    this.options             = options
-    this.$body               = $(document.body)
-    this.$element            = $(element)
-    this.$dialog             = this.$element.find('.modal-dialog')
-    this.$backdrop           = null
-    this.isShown             = null
-    this.originalBodyPad     = null
-    this.scrollbarWidth      = 0
+    this.options = options
+    this.$body = $(document.body)
+    this.$element = $(element)
+    this.$dialog = this.$element.find('.modal-dialog')
+    this.$backdrop = null
+    this.isShown = null
+    this.originalBodyPad = null
+    this.scrollbarWidth = 0
     this.ignoreBackdropClick = false
+    this.fixedContent = '.navbar-fixed-top, .navbar-fixed-bottom'
 
     if (this.options.remote) {
       this.$element
@@ -943,7 +960,7 @@ if (typeof jQuery === 'undefined') {
     }
   }
 
-  Modal.VERSION  = '3.3.6'
+  Modal.VERSION = '3.4.1'
 
   Modal.TRANSITION_DURATION = 300
   Modal.BACKDROP_TRANSITION_DURATION = 150
@@ -960,7 +977,7 @@ if (typeof jQuery === 'undefined') {
 
   Modal.prototype.show = function (_relatedTarget) {
     var that = this
-    var e    = $.Event('show.bs.modal', { relatedTarget: _relatedTarget })
+    var e = $.Event('show.bs.modal', { relatedTarget: _relatedTarget })
 
     this.$element.trigger(e)
 
@@ -1050,7 +1067,9 @@ if (typeof jQuery === 'undefined') {
     $(document)
       .off('focusin.bs.modal') // guard against infinite focus loop
       .on('focusin.bs.modal', $.proxy(function (e) {
-        if (this.$element[0] !== e.target && !this.$element.has(e.target).length) {
+        if (document !== e.target &&
+          this.$element[0] !== e.target &&
+          !this.$element.has(e.target).length) {
           this.$element.trigger('focus')
         }
       }, this))
@@ -1152,7 +1171,7 @@ if (typeof jQuery === 'undefined') {
     var modalIsOverflowing = this.$element[0].scrollHeight > document.documentElement.clientHeight
 
     this.$element.css({
-      paddingLeft:  !this.bodyIsOverflowing && modalIsOverflowing ? this.scrollbarWidth : '',
+      paddingLeft: !this.bodyIsOverflowing && modalIsOverflowing ? this.scrollbarWidth : '',
       paddingRight: this.bodyIsOverflowing && !modalIsOverflowing ? this.scrollbarWidth : ''
     })
   }
@@ -1177,11 +1196,26 @@ if (typeof jQuery === 'undefined') {
   Modal.prototype.setScrollbar = function () {
     var bodyPad = parseInt((this.$body.css('padding-right') || 0), 10)
     this.originalBodyPad = document.body.style.paddingRight || ''
-    if (this.bodyIsOverflowing) this.$body.css('padding-right', bodyPad + this.scrollbarWidth)
+    var scrollbarWidth = this.scrollbarWidth
+    if (this.bodyIsOverflowing) {
+      this.$body.css('padding-right', bodyPad + scrollbarWidth)
+      $(this.fixedContent).each(function (index, element) {
+        var actualPadding = element.style.paddingRight
+        var calculatedPadding = $(element).css('padding-right')
+        $(element)
+          .data('padding-right', actualPadding)
+          .css('padding-right', parseFloat(calculatedPadding) + scrollbarWidth + 'px')
+      })
+    }
   }
 
   Modal.prototype.resetScrollbar = function () {
     this.$body.css('padding-right', this.originalBodyPad)
+    $(this.fixedContent).each(function (index, element) {
+      var padding = $(element).data('padding-right')
+      $(element).removeData('padding-right')
+      element.style.paddingRight = padding ? padding : ''
+    })
   }
 
   Modal.prototype.measureScrollbar = function () { // thx walsh
@@ -1199,8 +1233,8 @@ if (typeof jQuery === 'undefined') {
 
   function Plugin(option, _relatedTarget) {
     return this.each(function () {
-      var $this   = $(this)
-      var data    = $this.data('bs.modal')
+      var $this = $(this)
+      var data = $this.data('bs.modal')
       var options = $.extend({}, Modal.DEFAULTS, $this.data(), typeof option == 'object' && option)
 
       if (!data) $this.data('bs.modal', (data = new Modal(this, options)))
@@ -1211,7 +1245,7 @@ if (typeof jQuery === 'undefined') {
 
   var old = $.fn.modal
 
-  $.fn.modal             = Plugin
+  $.fn.modal = Plugin
   $.fn.modal.Constructor = Modal
 
 
@@ -1228,10 +1262,13 @@ if (typeof jQuery === 'undefined') {
   // ==============
 
   $(document).on('click.bs.modal.data-api', '[data-toggle="modal"]', function (e) {
-    var $this   = $(this)
-    var href    = $this.attr('href')
-    var $target = $($this.attr('data-target') || (href && href.replace(/.*(?=#[^\s]+$)/, ''))) // strip for ie7
-    var option  = $target.data('bs.modal') ? 'toggle' : $.extend({ remote: !/#/.test(href) && href }, $target.data(), $this.data())
+    var $this = $(this)
+    var href = $this.attr('href')
+    var target = $this.attr('data-target') ||
+      (href && href.replace(/.*(?=#[^\s]+$)/, '')) // strip for ie7
+
+    var $target = $(document).find(target)
+    var option = $target.data('bs.modal') ? 'toggle' : $.extend({ remote: !/#/.test(href) && href }, $target.data(), $this.data())
 
     if ($this.is('a')) e.preventDefault()
 
@@ -1247,18 +1284,148 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: tooltip.js v3.3.6
- * http://getbootstrap.com/javascript/#tooltip
+ * Bootstrap: tooltip.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#tooltip
  * Inspired by the original jQuery.tipsy by Jason Frame
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
-
 +function ($) {
   'use strict';
 
+  var DISALLOWED_ATTRIBUTES = ['sanitize', 'whiteList', 'sanitizeFn']
+
+  var uriAttrs = [
+    'background',
+    'cite',
+    'href',
+    'itemtype',
+    'longdesc',
+    'poster',
+    'src',
+    'xlink:href'
+  ]
+
+  var ARIA_ATTRIBUTE_PATTERN = /^aria-[\w-]*$/i
+
+  var DefaultWhitelist = {
+    // Global attributes allowed on any supplied element below.
+    '*': ['class', 'dir', 'id', 'lang', 'role', ARIA_ATTRIBUTE_PATTERN],
+    a: ['target', 'href', 'title', 'rel'],
+    area: [],
+    b: [],
+    br: [],
+    col: [],
+    code: [],
+    div: [],
+    em: [],
+    hr: [],
+    h1: [],
+    h2: [],
+    h3: [],
+    h4: [],
+    h5: [],
+    h6: [],
+    i: [],
+    img: ['src', 'alt', 'title', 'width', 'height'],
+    li: [],
+    ol: [],
+    p: [],
+    pre: [],
+    s: [],
+    small: [],
+    span: [],
+    sub: [],
+    sup: [],
+    strong: [],
+    u: [],
+    ul: []
+  }
+
+  /**
+   * A pattern that recognizes a commonly useful subset of URLs that are safe.
+   *
+   * Shoutout to Angular 7 https://github.com/angular/angular/blob/7.2.4/packages/core/src/sanitization/url_sanitizer.ts
+   */
+  var SAFE_URL_PATTERN = /^(?:(?:https?|mailto|ftp|tel|file):|[^&:/?#]*(?:[/?#]|$))/gi
+
+  /**
+   * A pattern that matches safe data URLs. Only matches image, video and audio types.
+   *
+   * Shoutout to Angular 7 https://github.com/angular/angular/blob/7.2.4/packages/core/src/sanitization/url_sanitizer.ts
+   */
+  var DATA_URL_PATTERN = /^data:(?:image\/(?:bmp|gif|jpeg|jpg|png|tiff|webp)|video\/(?:mpeg|mp4|ogg|webm)|audio\/(?:mp3|oga|ogg|opus));base64,[a-z0-9+/]+=*$/i
+
+  function allowedAttribute(attr, allowedAttributeList) {
+    var attrName = attr.nodeName.toLowerCase()
+
+    if ($.inArray(attrName, allowedAttributeList) !== -1) {
+      if ($.inArray(attrName, uriAttrs) !== -1) {
+        return Boolean(attr.nodeValue.match(SAFE_URL_PATTERN) || attr.nodeValue.match(DATA_URL_PATTERN))
+      }
+
+      return true
+    }
+
+    var regExp = $(allowedAttributeList).filter(function (index, value) {
+      return value instanceof RegExp
+    })
+
+    // Check if a regular expression validates the attribute.
+    for (var i = 0, l = regExp.length; i < l; i++) {
+      if (attrName.match(regExp[i])) {
+        return true
+      }
+    }
+
+    return false
+  }
+
+  function sanitizeHtml(unsafeHtml, whiteList, sanitizeFn) {
+    if (unsafeHtml.length === 0) {
+      return unsafeHtml
+    }
+
+    if (sanitizeFn && typeof sanitizeFn === 'function') {
+      return sanitizeFn(unsafeHtml)
+    }
+
+    // IE 8 and below don't support createHTMLDocument
+    if (!document.implementation || !document.implementation.createHTMLDocument) {
+      return unsafeHtml
+    }
+
+    var createdDocument = document.implementation.createHTMLDocument('sanitization')
+    createdDocument.body.innerHTML = unsafeHtml
+
+    var whitelistKeys = $.map(whiteList, function (el, i) { return i })
+    var elements = $(createdDocument.body).find('*')
+
+    for (var i = 0, len = elements.length; i < len; i++) {
+      var el = elements[i]
+      var elName = el.nodeName.toLowerCase()
+
+      if ($.inArray(elName, whitelistKeys) === -1) {
+        el.parentNode.removeChild(el)
+
+        continue
+      }
+
+      var attributeList = $.map(el.attributes, function (el) { return el })
+      var whitelistedAttributes = [].concat(whiteList['*'] || [], whiteList[elName] || [])
+
+      for (var j = 0, len2 = attributeList.length; j < len2; j++) {
+        if (!allowedAttribute(attributeList[j], whitelistedAttributes)) {
+          el.removeAttribute(attributeList[j].nodeName)
+        }
+      }
+    }
+
+    return createdDocument.body.innerHTML
+  }
+
   // TOOLTIP PUBLIC CLASS DEFINITION
   // ===============================
 
@@ -1274,7 +1441,7 @@ if (typeof jQuery === 'undefined') {
     this.init('tooltip', element, options)
   }
 
-  Tooltip.VERSION  = '3.3.6'
+  Tooltip.VERSION  = '3.4.1'
 
   Tooltip.TRANSITION_DURATION = 150
 
@@ -1291,7 +1458,10 @@ if (typeof jQuery === 'undefined') {
     viewport: {
       selector: 'body',
       padding: 0
-    }
+    },
+    sanitize : true,
+    sanitizeFn : null,
+    whiteList : DefaultWhitelist
   }
 
   Tooltip.prototype.init = function (type, element, options) {
@@ -1299,7 +1469,7 @@ if (typeof jQuery === 'undefined') {
     this.type      = type
     this.$element  = $(element)
     this.options   = this.getOptions(options)
-    this.$viewport = this.options.viewport && $($.isFunction(this.options.viewport) ? this.options.viewport.call(this, this.$element) : (this.options.viewport.selector || this.options.viewport))
+    this.$viewport = this.options.viewport && $(document).find($.isFunction(this.options.viewport) ? this.options.viewport.call(this, this.$element) : (this.options.viewport.selector || this.options.viewport))
     this.inState   = { click: false, hover: false, focus: false }
 
     if (this.$element[0] instanceof document.constructor && !this.options.selector) {
@@ -1332,7 +1502,15 @@ if (typeof jQuery === 'undefined') {
   }
 
   Tooltip.prototype.getOptions = function (options) {
-    options = $.extend({}, this.getDefaults(), this.$element.data(), options)
+    var dataAttributes = this.$element.data()
+
+    for (var dataAttr in dataAttributes) {
+      if (dataAttributes.hasOwnProperty(dataAttr) && $.inArray(dataAttr, DISALLOWED_ATTRIBUTES) !== -1) {
+        delete dataAttributes[dataAttr]
+      }
+    }
+
+    options = $.extend({}, this.getDefaults(), dataAttributes, options)
 
     if (options.delay && typeof options.delay == 'number') {
       options.delay = {
@@ -1341,6 +1519,10 @@ if (typeof jQuery === 'undefined') {
       }
     }
 
+    if (options.sanitize) {
+      options.template = sanitizeHtml(options.template, options.whiteList, options.sanitizeFn)
+    }
+
     return options
   }
 
@@ -1452,7 +1634,7 @@ if (typeof jQuery === 'undefined') {
         .addClass(placement)
         .data('bs.' + this.type, this)
 
-      this.options.container ? $tip.appendTo(this.options.container) : $tip.insertAfter(this.$element)
+      this.options.container ? $tip.appendTo($(document).find(this.options.container)) : $tip.insertAfter(this.$element)
       this.$element.trigger('inserted.bs.' + this.type)
 
       var pos          = this.getPosition()
@@ -1554,7 +1736,16 @@ if (typeof jQuery === 'undefined') {
     var $tip  = this.tip()
     var title = this.getTitle()
 
-    $tip.find('.tooltip-inner')[this.options.html ? 'html' : 'text'](title)
+    if (this.options.html) {
+      if (this.options.sanitize) {
+        title = sanitizeHtml(title, this.options.whiteList, this.options.sanitizeFn)
+      }
+
+      $tip.find('.tooltip-inner').html(title)
+    } else {
+      $tip.find('.tooltip-inner').text(title)
+    }
+
     $tip.removeClass('fade in top bottom left right')
   }
 
@@ -1565,9 +1756,11 @@ if (typeof jQuery === 'undefined') {
 
     function complete() {
       if (that.hoverState != 'in') $tip.detach()
-      that.$element
-        .removeAttr('aria-describedby')
-        .trigger('hidden.bs.' + that.type)
+      if (that.$element) { // TODO: Check whether guarding this code with this `if` is really necessary.
+        that.$element
+          .removeAttr('aria-describedby')
+          .trigger('hidden.bs.' + that.type)
+      }
       callback && callback()
     }
 
@@ -1610,7 +1803,10 @@ if (typeof jQuery === 'undefined') {
       // width and height are missing in IE8, so compute them manually; see https://github.com/twbs/bootstrap/issues/14093
       elRect = $.extend({}, elRect, { width: elRect.right - elRect.left, height: elRect.bottom - elRect.top })
     }
-    var elOffset  = isBody ? { top: 0, left: 0 } : $element.offset()
+    var isSvg = window.SVGElement && el instanceof window.SVGElement
+    // Avoid using $.offset() on SVGs since it gives incorrect results in jQuery 3.
+    // See https://github.com/twbs/bootstrap/issues/20280
+    var elOffset  = isBody ? { top: 0, left: 0 } : (isSvg ? null : $element.offset())
     var scroll    = { scroll: isBody ? document.documentElement.scrollTop || document.body.scrollTop : $element.scrollTop() }
     var outerDims = isBody ? { width: $(window).width(), height: $(window).height() } : null
 
@@ -1726,9 +1922,13 @@ if (typeof jQuery === 'undefined') {
       that.$tip = null
       that.$arrow = null
       that.$viewport = null
+      that.$element = null
     })
   }
 
+  Tooltip.prototype.sanitizeHtml = function (unsafeHtml) {
+    return sanitizeHtml(unsafeHtml, this.options.whiteList, this.options.sanitizeFn)
+  }
 
   // TOOLTIP PLUGIN DEFINITION
   // =========================
@@ -1762,10 +1962,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: popover.js v3.3.6
- * http://getbootstrap.com/javascript/#popovers
+ * Bootstrap: popover.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#popovers
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -1782,7 +1982,7 @@ if (typeof jQuery === 'undefined') {
 
   if (!$.fn.tooltip) throw new Error('Popover requires tooltip.js')
 
-  Popover.VERSION  = '3.3.6'
+  Popover.VERSION  = '3.4.1'
 
   Popover.DEFAULTS = $.extend({}, $.fn.tooltip.Constructor.DEFAULTS, {
     placement: 'right',
@@ -1808,10 +2008,25 @@ if (typeof jQuery === 'undefined') {
     var title   = this.getTitle()
     var content = this.getContent()
 
-    $tip.find('.popover-title')[this.options.html ? 'html' : 'text'](title)
-    $tip.find('.popover-content').children().detach().end()[ // we use append for html objects to maintain js events
-      this.options.html ? (typeof content == 'string' ? 'html' : 'append') : 'text'
-    ](content)
+    if (this.options.html) {
+      var typeContent = typeof content
+
+      if (this.options.sanitize) {
+        title = this.sanitizeHtml(title)
+
+        if (typeContent === 'string') {
+          content = this.sanitizeHtml(content)
+        }
+      }
+
+      $tip.find('.popover-title').html(title)
+      $tip.find('.popover-content').children().detach().end()[
+        typeContent === 'string' ? 'html' : 'append'
+      ](content)
+    } else {
+      $tip.find('.popover-title').text(title)
+      $tip.find('.popover-content').children().detach().end().text(content)
+    }
 
     $tip.removeClass('fade top bottom left right in')
 
@@ -1830,8 +2045,8 @@ if (typeof jQuery === 'undefined') {
 
     return $e.attr('data-content')
       || (typeof o.content == 'function' ?
-            o.content.call($e[0]) :
-            o.content)
+        o.content.call($e[0]) :
+        o.content)
   }
 
   Popover.prototype.arrow = function () {
@@ -1871,10 +2086,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: scrollspy.js v3.3.6
- * http://getbootstrap.com/javascript/#scrollspy
+ * Bootstrap: scrollspy.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#scrollspy
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -1900,7 +2115,7 @@ if (typeof jQuery === 'undefined') {
     this.process()
   }
 
-  ScrollSpy.VERSION  = '3.3.6'
+  ScrollSpy.VERSION  = '3.4.1'
 
   ScrollSpy.DEFAULTS = {
     offset: 10
@@ -2044,10 +2259,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: tab.js v3.3.6
- * http://getbootstrap.com/javascript/#tabs
+ * Bootstrap: tab.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#tabs
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -2064,7 +2279,7 @@ if (typeof jQuery === 'undefined') {
     // jscs:enable requireDollarBeforejQueryAssignment
   }
 
-  Tab.VERSION = '3.3.6'
+  Tab.VERSION = '3.4.1'
 
   Tab.TRANSITION_DURATION = 150
 
@@ -2093,7 +2308,7 @@ if (typeof jQuery === 'undefined') {
 
     if (showEvent.isDefaultPrevented() || hideEvent.isDefaultPrevented()) return
 
-    var $target = $(selector)
+    var $target = $(document).find(selector)
 
     this.activate($this.closest('li'), $ul)
     this.activate($target, $target.parent(), function () {
@@ -2118,15 +2333,15 @@ if (typeof jQuery === 'undefined') {
       $active
         .removeClass('active')
         .find('> .dropdown-menu > .active')
-          .removeClass('active')
+        .removeClass('active')
         .end()
         .find('[data-toggle="tab"]')
-          .attr('aria-expanded', false)
+        .attr('aria-expanded', false)
 
       element
         .addClass('active')
         .find('[data-toggle="tab"]')
-          .attr('aria-expanded', true)
+        .attr('aria-expanded', true)
 
       if (transition) {
         element[0].offsetWidth // reflow for transition
@@ -2138,10 +2353,10 @@ if (typeof jQuery === 'undefined') {
       if (element.parent('.dropdown-menu').length) {
         element
           .closest('li.dropdown')
-            .addClass('active')
+          .addClass('active')
           .end()
           .find('[data-toggle="tab"]')
-            .attr('aria-expanded', true)
+          .attr('aria-expanded', true)
       }
 
       callback && callback()
@@ -2200,10 +2415,10 @@ if (typeof jQuery === 'undefined') {
 }(jQuery);
 
 /* ========================================================================
- * Bootstrap: affix.js v3.3.6
- * http://getbootstrap.com/javascript/#affix
+ * Bootstrap: affix.js v3.4.1
+ * https://getbootstrap.com/docs/3.4/javascript/#affix
  * ========================================================================
- * Copyright 2011-2015 Twitter, Inc.
+ * Copyright 2011-2019 Twitter, Inc.
  * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE)
  * ======================================================================== */
 
@@ -2217,7 +2432,9 @@ if (typeof jQuery === 'undefined') {
   var Affix = function (element, options) {
     this.options = $.extend({}, Affix.DEFAULTS, options)
 
-    this.$target = $(this.options.target)
+    var target = this.options.target === Affix.DEFAULTS.target ? $(this.options.target) : $(document).find(this.options.target)
+
+    this.$target = target
       .on('scroll.bs.affix.data-api', $.proxy(this.checkPosition, this))
       .on('click.bs.affix.data-api',  $.proxy(this.checkPositionWithEventLoop, this))
 
@@ -2229,7 +2446,7 @@ if (typeof jQuery === 'undefined') {
     this.checkPosition()
   }
 
-  Affix.VERSION  = '3.3.6'
+  Affix.VERSION  = '3.4.1'
 
   Affix.RESET    = 'affix affix-top affix-bottom'
 
diff --git a/bitbake/lib/toaster/toastergui/static/js/bootstrap-3.4.1.min.js b/bitbake/lib/toaster/toastergui/static/js/bootstrap-3.4.1.min.js
new file mode 100644
index 0000000000..eb0a8b410f
--- /dev/null
+++ b/bitbake/lib/toaster/toastergui/static/js/bootstrap-3.4.1.min.js
@@ -0,0 +1,6 @@
+/*!
+ * Bootstrap v3.4.1 (https://getbootstrap.com/)
+ * Copyright 2011-2019 Twitter, Inc.
+ * Licensed under the MIT license
+ */
+if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery");!function(t){"use strict";var e=jQuery.fn.jquery.split(" ")[0].split(".");if(e[0]<2&&e[1]<9||1==e[0]&&9==e[1]&&e[2]<1||3<e[0])throw new Error("Bootstrap's JavaScript requires jQuery version 1.9.1 or higher, but lower than version 4")}(),function(n){"use strict";n.fn.emulateTransitionEnd=function(t){var e=!1,i=this;n(this).one("bsTransitionEnd",function(){e=!0});return setTimeout(function(){e||n(i).trigger(n.support.transition.end)},t),this},n(function(){n.support.transition=function o(){var t=document.createElement("bootstrap"),e={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"};for(var i in e)if(t.style[i]!==undefined)return{end:e[i]};return!1}(),n.support.transition&&(n.event.special.bsTransitionEnd={bindType:n.support.transition.end,delegateType:n.support.transition.end,handle:function(t){if(n(t.target).is(this))return t.handleObj.handler.apply(this,arguments)}})})}(jQuery),function(s){"use strict";var e='[data-dismiss="alert"]',a=function(t){s(t).on("click",e,this.close)};a.VERSION="3.4.1",a.TRANSITION_DURATION=150,a.prototype.close=function(t){var e=s(this),i=e.attr("data-target");i||(i=(i=e.attr("href"))&&i.replace(/.*(?=#[^\s]*$)/,"")),i="#"===i?[]:i;var o=s(document).find(i);function n(){o.detach().trigger("closed.bs.alert").remove()}t&&t.preventDefault(),o.length||(o=e.closest(".alert")),o.trigger(t=s.Event("close.bs.alert")),t.isDefaultPrevented()||(o.removeClass("in"),s.support.transition&&o.hasClass("fade")?o.one("bsTransitionEnd",n).emulateTransitionEnd(a.TRANSITION_DURATION):n())};var t=s.fn.alert;s.fn.alert=function o(i){return this.each(function(){var t=s(this),e=t.data("bs.alert");e||t.data("bs.alert",e=new a(this)),"string"==typeof i&&e[i].call(t)})},s.fn.alert.Constructor=a,s.fn.alert.noConflict=function(){return s.fn.alert=t,this},s(document).on("click.bs.alert.data-api",e,a.prototype.close)}(jQuery),function(s){"use strict";var n=function(t,e){this.$element=s(t),this.options=s.extend({},n.DEFAULTS,e),this.isLoading=!1};function i(o){return this.each(function(){var t=s(this),e=t.data("bs.button"),i="object"==typeof o&&o;e||t.data("bs.button",e=new n(this,i)),"toggle"==o?e.toggle():o&&e.setState(o)})}n.VERSION="3.4.1",n.DEFAULTS={loadingText:"loading..."},n.prototype.setState=function(t){var e="disabled",i=this.$element,o=i.is("input")?"val":"html",n=i.data();t+="Text",null==n.resetText&&i.data("resetText",i[o]()),setTimeout(s.proxy(function(){i[o](null==n[t]?this.options[t]:n[t]),"loadingText"==t?(this.isLoading=!0,i.addClass(e).attr(e,e).prop(e,!0)):this.isLoading&&(this.isLoading=!1,i.removeClass(e).removeAttr(e).prop(e,!1))},this),0)},n.prototype.toggle=function(){var t=!0,e=this.$element.closest('[data-toggle="buttons"]');if(e.length){var i=this.$element.find("input");"radio"==i.prop("type")?(i.prop("checked")&&(t=!1),e.find(".active").removeClass("active"),this.$element.addClass("active")):"checkbox"==i.prop("type")&&(i.prop("checked")!==this.$element.hasClass("active")&&(t=!1),this.$element.toggleClass("active")),i.prop("checked",this.$element.hasClass("active")),t&&i.trigger("change")}else this.$element.attr("aria-pressed",!this.$element.hasClass("active")),this.$element.toggleClass("active")};var t=s.fn.button;s.fn.button=i,s.fn.button.Constructor=n,s.fn.button.noConflict=function(){return s.fn.button=t,this},s(document).on("click.bs.button.data-api",'[data-toggle^="button"]',function(t){var e=s(t.target).closest(".btn");i.call(e,"toggle"),s(t.target).is('input[type="radio"], input[type="checkbox"]')||(t.preventDefault(),e.is("input,button")?e.trigger("focus"):e.find("input:visible,button:visible").first().trigger("focus"))}).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',function(t){s(t.target).closest(".btn").toggleClass("focus",/^focus(in)?$/.test(t.type))})}(jQuery),function(p){"use strict";var c=function(t,e){this.$element=p(t),this.$indicators=this.$element.find(".carousel-indicators"),this.options=e,this.paused=null,this.sliding=null,this.interval=null,this.$active=null,this.$items=null,this.options.keyboard&&this.$element.on("keydown.bs.carousel",p.proxy(this.keydown,this)),"hover"==this.options.pause&&!("ontouchstart"in document.documentElement)&&this.$element.on("mouseenter.bs.carousel",p.proxy(this.pause,this)).on("mouseleave.bs.carousel",p.proxy(this.cycle,this))};function r(n){return this.each(function(){var t=p(this),e=t.data("bs.carousel"),i=p.extend({},c.DEFAULTS,t.data(),"object"==typeof n&&n),o="string"==typeof n?n:i.slide;e||t.data("bs.carousel",e=new c(this,i)),"number"==typeof n?e.to(n):o?e[o]():i.interval&&e.pause().cycle()})}c.VERSION="3.4.1",c.TRANSITION_DURATION=600,c.DEFAULTS={interval:5e3,pause:"hover",wrap:!0,keyboard:!0},c.prototype.keydown=function(t){if(!/input|textarea/i.test(t.target.tagName)){switch(t.which){case 37:this.prev();break;case 39:this.next();break;default:return}t.preventDefault()}},c.prototype.cycle=function(t){return t||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(p.proxy(this.next,this),this.options.interval)),this},c.prototype.getItemIndex=function(t){return this.$items=t.parent().children(".item"),this.$items.index(t||this.$active)},c.prototype.getItemForDirection=function(t,e){var i=this.getItemIndex(e);if(("prev"==t&&0===i||"next"==t&&i==this.$items.length-1)&&!this.options.wrap)return e;var o=(i+("prev"==t?-1:1))%this.$items.length;return this.$items.eq(o)},c.prototype.to=function(t){var e=this,i=this.getItemIndex(this.$active=this.$element.find(".item.active"));if(!(t>this.$items.length-1||t<0))return this.sliding?this.$element.one("slid.bs.carousel",function(){e.to(t)}):i==t?this.pause().cycle():this.slide(i<t?"next":"prev",this.$items.eq(t))},c.prototype.pause=function(t){return t||(this.paused=!0),this.$element.find(".next, .prev").length&&p.support.transition&&(this.$element.trigger(p.support.transition.end),this.cycle(!0)),this.interval=clearInterval(this.interval),this},c.prototype.next=function(){if(!this.sliding)return this.slide("next")},c.prototype.prev=function(){if(!this.sliding)return this.slide("prev")},c.prototype.slide=function(t,e){var i=this.$element.find(".item.active"),o=e||this.getItemForDirection(t,i),n=this.interval,s="next"==t?"left":"right",a=this;if(o.hasClass("active"))return this.sliding=!1;var r=o[0],l=p.Event("slide.bs.carousel",{relatedTarget:r,direction:s});if(this.$element.trigger(l),!l.isDefaultPrevented()){if(this.sliding=!0,n&&this.pause(),this.$indicators.length){this.$indicators.find(".active").removeClass("active");var h=p(this.$indicators.children()[this.getItemIndex(o)]);h&&h.addClass("active")}var d=p.Event("slid.bs.carousel",{relatedTarget:r,direction:s});return p.support.transition&&this.$element.hasClass("slide")?(o.addClass(t),"object"==typeof o&&o.length&&o[0].offsetWidth,i.addClass(s),o.addClass(s),i.one("bsTransitionEnd",function(){o.removeClass([t,s].join(" ")).addClass("active"),i.removeClass(["active",s].join(" ")),a.sliding=!1,setTimeout(function(){a.$element.trigger(d)},0)}).emulateTransitionEnd(c.TRANSITION_DURATION)):(i.removeClass("active"),o.addClass("active"),this.sliding=!1,this.$element.trigger(d)),n&&this.cycle(),this}};var t=p.fn.carousel;p.fn.carousel=r,p.fn.carousel.Constructor=c,p.fn.carousel.noConflict=function(){return p.fn.carousel=t,this};var e=function(t){var e=p(this),i=e.attr("href");i&&(i=i.replace(/.*(?=#[^\s]+$)/,""));var o=e.attr("data-target")||i,n=p(document).find(o);if(n.hasClass("carousel")){var s=p.extend({},n.data(),e.data()),a=e.attr("data-slide-to");a&&(s.interval=!1),r.call(n,s),a&&n.data("bs.carousel").to(a),t.preventDefault()}};p(document).on("click.bs.carousel.data-api","[data-slide]",e).on("click.bs.carousel.data-api","[data-slide-to]",e),p(window).on("load",function(){p('[data-ride="carousel"]').each(function(){var t=p(this);r.call(t,t.data())})})}(jQuery),function(a){"use strict";var r=function(t,e){this.$element=a(t),this.options=a.extend({},r.DEFAULTS,e),this.$trigger=a('[data-toggle="collapse"][href="#'+t.id+'"],[data-toggle="collapse"][data-target="#'+t.id+'"]'),this.transitioning=null,this.options.parent?this.$parent=this.getParent():this.addAriaAndCollapsedClass(this.$element,this.$trigger),this.options.toggle&&this.toggle()};function n(t){var e,i=t.attr("data-target")||(e=t.attr("href"))&&e.replace(/.*(?=#[^\s]+$)/,"");return a(document).find(i)}function l(o){return this.each(function(){var t=a(this),e=t.data("bs.collapse"),i=a.extend({},r.DEFAULTS,t.data(),"object"==typeof o&&o);!e&&i.toggle&&/show|hide/.test(o)&&(i.toggle=!1),e||t.data("bs.collapse",e=new r(this,i)),"string"==typeof o&&e[o]()})}r.VERSION="3.4.1",r.TRANSITION_DURATION=350,r.DEFAULTS={toggle:!0},r.prototype.dimension=function(){return this.$element.hasClass("width")?"width":"height"},r.prototype.show=function(){if(!this.transitioning&&!this.$element.hasClass("in")){var t,e=this.$parent&&this.$parent.children(".panel").children(".in, .collapsing");if(!(e&&e.length&&(t=e.data("bs.collapse"))&&t.transitioning)){var i=a.Event("show.bs.collapse");if(this.$element.trigger(i),!i.isDefaultPrevented()){e&&e.length&&(l.call(e,"hide"),t||e.data("bs.collapse",null));var o=this.dimension();this.$element.removeClass("collapse").addClass("collapsing")[o](0).attr("aria-expanded",!0),this.$trigger.removeClass("collapsed").attr("aria-expanded",!0),this.transitioning=1;var n=function(){this.$element.removeClass("collapsing").addClass("collapse in")[o](""),this.transitioning=0,this.$element.trigger("shown.bs.collapse")};if(!a.support.transition)return n.call(this);var s=a.camelCase(["scroll",o].join("-"));this.$element.one("bsTransitionEnd",a.proxy(n,this)).emulateTransitionEnd(r.TRANSITION_DURATION)[o](this.$element[0][s])}}}},r.prototype.hide=function(){if(!this.transitioning&&this.$element.hasClass("in")){var t=a.Event("hide.bs.collapse");if(this.$element.trigger(t),!t.isDefaultPrevented()){var e=this.dimension();this.$element[e](this.$element[e]())[0].offsetHeight,this.$element.addClass("collapsing").removeClass("collapse in").attr("aria-expanded",!1),this.$trigger.addClass("collapsed").attr("aria-expanded",!1),this.transitioning=1;var i=function(){this.transitioning=0,this.$element.removeClass("collapsing").addClass("collapse").trigger("hidden.bs.collapse")};if(!a.support.transition)return i.call(this);this.$element[e](0).one("bsTransitionEnd",a.proxy(i,this)).emulateTransitionEnd(r.TRANSITION_DURATION)}}},r.prototype.toggle=function(){this[this.$element.hasClass("in")?"hide":"show"]()},r.prototype.getParent=function(){return a(document).find(this.options.parent).find('[data-toggle="collapse"][data-parent="'+this.options.parent+'"]').each(a.proxy(function(t,e){var i=a(e);this.addAriaAndCollapsedClass(n(i),i)},this)).end()},r.prototype.addAriaAndCollapsedClass=function(t,e){var i=t.hasClass("in");t.attr("aria-expanded",i),e.toggleClass("collapsed",!i).attr("aria-expanded",i)};var t=a.fn.collapse;a.fn.collapse=l,a.fn.collapse.Constructor=r,a.fn.collapse.noConflict=function(){return a.fn.collapse=t,this},a(document).on("click.bs.collapse.data-api",'[data-toggle="collapse"]',function(t){var e=a(this);e.attr("data-target")||t.preventDefault();var i=n(e),o=i.data("bs.collapse")?"toggle":e.data();l.call(i,o)})}(jQuery),function(a){"use strict";var r='[data-toggle="dropdown"]',o=function(t){a(t).on("click.bs.dropdown",this.toggle)};function l(t){var e=t.attr("data-target");e||(e=(e=t.attr("href"))&&/#[A-Za-z]/.test(e)&&e.replace(/.*(?=#[^\s]*$)/,""));var i="#"!==e?a(document).find(e):null;return i&&i.length?i:t.parent()}function s(o){o&&3===o.which||(a(".dropdown-backdrop").remove(),a(r).each(function(){var t=a(this),e=l(t),i={relatedTarget:this};e.hasClass("open")&&(o&&"click"==o.type&&/input|textarea/i.test(o.target.tagName)&&a.contains(e[0],o.target)||(e.trigger(o=a.Event("hide.bs.dropdown",i)),o.isDefaultPrevented()||(t.attr("aria-expanded","false"),e.removeClass("open").trigger(a.Event("hidden.bs.dropdown",i)))))}))}o.VERSION="3.4.1",o.prototype.toggle=function(t){var e=a(this);if(!e.is(".disabled, :disabled")){var i=l(e),o=i.hasClass("open");if(s(),!o){"ontouchstart"in document.documentElement&&!i.closest(".navbar-nav").length&&a(document.createElement("div")).addClass("dropdown-backdrop").insertAfter(a(this)).on("click",s);var n={relatedTarget:this};if(i.trigger(t=a.Event("show.bs.dropdown",n)),t.isDefaultPrevented())return;e.trigger("focus").attr("aria-expanded","true"),i.toggleClass("open").trigger(a.Event("shown.bs.dropdown",n))}return!1}},o.prototype.keydown=function(t){if(/(38|40|27|32)/.test(t.which)&&!/input|textarea/i.test(t.target.tagName)){var e=a(this);if(t.preventDefault(),t.stopPropagation(),!e.is(".disabled, :disabled")){var i=l(e),o=i.hasClass("open");if(!o&&27!=t.which||o&&27==t.which)return 27==t.which&&i.find(r).trigger("focus"),e.trigger("click");var n=i.find(".dropdown-menu li:not(.disabled):visible a");if(n.length){var s=n.index(t.target);38==t.which&&0<s&&s--,40==t.which&&s<n.length-1&&s++,~s||(s=0),n.eq(s).trigger("focus")}}}};var t=a.fn.dropdown;a.fn.dropdown=function e(i){return this.each(function(){var t=a(this),e=t.data("bs.dropdown");e||t.data("bs.dropdown",e=new o(this)),"string"==typeof i&&e[i].call(t)})},a.fn.dropdown.Constructor=o,a.fn.dropdown.noConflict=function(){return a.fn.dropdown=t,this},a(document).on("click.bs.dropdown.data-api",s).on("click.bs.dropdown.data-api",".dropdown form",function(t){t.stopPropagation()}).on("click.bs.dropdown.data-api",r,o.prototype.toggle).on("keydown.bs.dropdown.data-api",r,o.prototype.keydown).on("keydown.bs.dropdown.data-api",".dropdown-menu",o.prototype.keydown)}(jQuery),function(a){"use strict";var s=function(t,e){this.options=e,this.$body=a(document.body),this.$element=a(t),this.$dialog=this.$element.find(".modal-dialog"),this.$backdrop=null,this.isShown=null,this.originalBodyPad=null,this.scrollbarWidth=0,this.ignoreBackdropClick=!1,this.fixedContent=".navbar-fixed-top, .navbar-fixed-bottom",this.options.remote&&this.$element.find(".modal-content").load(this.options.remote,a.proxy(function(){this.$element.trigger("loaded.bs.modal")},this))};function r(o,n){return this.each(function(){var t=a(this),e=t.data("bs.modal"),i=a.extend({},s.DEFAULTS,t.data(),"object"==typeof o&&o);e||t.data("bs.modal",e=new s(this,i)),"string"==typeof o?e[o](n):i.show&&e.show(n)})}s.VERSION="3.4.1",s.TRANSITION_DURATION=300,s.BACKDROP_TRANSITION_DURATION=150,s.DEFAULTS={backdrop:!0,keyboard:!0,show:!0},s.prototype.toggle=function(t){return this.isShown?this.hide():this.show(t)},s.prototype.show=function(i){var o=this,t=a.Event("show.bs.modal",{relatedTarget:i});this.$element.trigger(t),this.isShown||t.isDefaultPrevented()||(this.isShown=!0,this.checkScrollbar(),this.setScrollbar(),this.$body.addClass("modal-open"),this.escape(),this.resize(),this.$element.on("click.dismiss.bs.modal",'[data-dismiss="modal"]',a.proxy(this.hide,this)),this.$dialog.on("mousedown.dismiss.bs.modal",function(){o.$element.one("mouseup.dismiss.bs.modal",function(t){a(t.target).is(o.$element)&&(o.ignoreBackdropClick=!0)})}),this.backdrop(function(){var t=a.support.transition&&o.$element.hasClass("fade");o.$element.parent().length||o.$element.appendTo(o.$body),o.$element.show().scrollTop(0),o.adjustDialog(),t&&o.$element[0].offsetWidth,o.$element.addClass("in"),o.enforceFocus();var e=a.Event("shown.bs.modal",{relatedTarget:i});t?o.$dialog.one("bsTransitionEnd",function(){o.$element.trigger("focus").trigger(e)}).emulateTransitionEnd(s.TRANSITION_DURATION):o.$element.trigger("focus").trigger(e)}))},s.prototype.hide=function(t){t&&t.preventDefault(),t=a.Event("hide.bs.modal"),this.$element.trigger(t),this.isShown&&!t.isDefaultPrevented()&&(this.isShown=!1,this.escape(),this.resize(),a(document).off("focusin.bs.modal"),this.$element.removeClass("in").off("click.dismiss.bs.modal").off("mouseup.dismiss.bs.modal"),this.$dialog.off("mousedown.dismiss.bs.modal"),a.support.transition&&this.$element.hasClass("fade")?this.$element.one("bsTransitionEnd",a.proxy(this.hideModal,this)).emulateTransitionEnd(s.TRANSITION_DURATION):this.hideModal())},s.prototype.enforceFocus=function(){a(document).off("focusin.bs.modal").on("focusin.bs.modal",a.proxy(function(t){document===t.target||this.$element[0]===t.target||this.$element.has(t.target).length||this.$element.trigger("focus")},this))},s.prototype.escape=function(){this.isShown&&this.options.keyboard?this.$element.on("keydown.dismiss.bs.modal",a.proxy(function(t){27==t.which&&this.hide()},this)):this.isShown||this.$element.off("keydown.dismiss.bs.modal")},s.prototype.resize=function(){this.isShown?a(window).on("resize.bs.modal",a.proxy(this.handleUpdate,this)):a(window).off("resize.bs.modal")},s.prototype.hideModal=function(){var t=this;this.$element.hide(),this.backdrop(function(){t.$body.removeClass("modal-open"),t.resetAdjustments(),t.resetScrollbar(),t.$element.trigger("hidden.bs.modal")})},s.prototype.removeBackdrop=function(){this.$backdrop&&this.$backdrop.remove(),this.$backdrop=null},s.prototype.backdrop=function(t){var e=this,i=this.$element.hasClass("fade")?"fade":"";if(this.isShown&&this.options.backdrop){var o=a.support.transition&&i;if(this.$backdrop=a(document.createElement("div")).addClass("modal-backdrop "+i).appendTo(this.$body),this.$element.on("click.dismiss.bs.modal",a.proxy(function(t){this.ignoreBackdropClick?this.ignoreBackdropClick=!1:t.target===t.currentTarget&&("static"==this.options.backdrop?this.$element[0].focus():this.hide())},this)),o&&this.$backdrop[0].offsetWidth,this.$backdrop.addClass("in"),!t)return;o?this.$backdrop.one("bsTransitionEnd",t).emulateTransitionEnd(s.BACKDROP_TRANSITION_DURATION):t()}else if(!this.isShown&&this.$backdrop){this.$backdrop.removeClass("in");var n=function(){e.removeBackdrop(),t&&t()};a.support.transition&&this.$element.hasClass("fade")?this.$backdrop.one("bsTransitionEnd",n).emulateTransitionEnd(s.BACKDROP_TRANSITION_DURATION):n()}else t&&t()},s.prototype.handleUpdate=function(){this.adjustDialog()},s.prototype.adjustDialog=function(){var t=this.$element[0].scrollHeight>document.documentElement.clientHeight;this.$element.css({paddingLeft:!this.bodyIsOverflowing&&t?this.scrollbarWidth:"",paddingRight:this.bodyIsOverflowing&&!t?this.scrollbarWidth:""})},s.prototype.resetAdjustments=function(){this.$element.css({paddingLeft:"",paddingRight:""})},s.prototype.checkScrollbar=function(){var t=window.innerWidth;if(!t){var e=document.documentElement.getBoundingClientRect();t=e.right-Math.abs(e.left)}this.bodyIsOverflowing=document.body.clientWidth<t,this.scrollbarWidth=this.measureScrollbar()},s.prototype.setScrollbar=function(){var t=parseInt(this.$body.css("padding-right")||0,10);this.originalBodyPad=document.body.style.paddingRight||"";var n=this.scrollbarWidth;this.bodyIsOverflowing&&(this.$body.css("padding-right",t+n),a(this.fixedContent).each(function(t,e){var i=e.style.paddingRight,o=a(e).css("padding-right");a(e).data("padding-right",i).css("padding-right",parseFloat(o)+n+"px")}))},s.prototype.resetScrollbar=function(){this.$body.css("padding-right",this.originalBodyPad),a(this.fixedContent).each(function(t,e){var i=a(e).data("padding-right");a(e).removeData("padding-right"),e.style.paddingRight=i||""})},s.prototype.measureScrollbar=function(){var t=document.createElement("div");t.className="modal-scrollbar-measure",this.$body.append(t);var e=t.offsetWidth-t.clientWidth;return this.$body[0].removeChild(t),e};var t=a.fn.modal;a.fn.modal=r,a.fn.modal.Constructor=s,a.fn.modal.noConflict=function(){return a.fn.modal=t,this},a(document).on("click.bs.modal.data-api",'[data-toggle="modal"]',function(t){var e=a(this),i=e.attr("href"),o=e.attr("data-target")||i&&i.replace(/.*(?=#[^\s]+$)/,""),n=a(document).find(o),s=n.data("bs.modal")?"toggle":a.extend({remote:!/#/.test(i)&&i},n.data(),e.data());e.is("a")&&t.preventDefault(),n.one("show.bs.modal",function(t){t.isDefaultPrevented()||n.one("hidden.bs.modal",function(){e.is(":visible")&&e.trigger("focus")})}),r.call(n,s,this)})}(jQuery),function(g){"use strict";var o=["sanitize","whiteList","sanitizeFn"],a=["background","cite","href","itemtype","longdesc","poster","src","xlink:href"],t={"*":["class","dir","id","lang","role",/^aria-[\w-]*$/i],a:["target","href","title","rel"],area:[],b:[],br:[],col:[],code:[],div:[],em:[],hr:[],h1:[],h2:[],h3:[],h4:[],h5:[],h6:[],i:[],img:["src","alt","title","width","height"],li:[],ol:[],p:[],pre:[],s:[],small:[],span:[],sub:[],sup:[],strong:[],u:[],ul:[]},r=/^(?:(?:https?|mailto|ftp|tel|file):|[^&:/?#]*(?:[/?#]|$))/gi,l=/^data:(?:image\/(?:bmp|gif|jpeg|jpg|png|tiff|webp)|video\/(?:mpeg|mp4|ogg|webm)|audio\/(?:mp3|oga|ogg|opus));base64,[a-z0-9+/]+=*$/i;function u(t,e){var i=t.nodeName.toLowerCase();if(-1!==g.inArray(i,e))return-1===g.inArray(i,a)||Boolean(t.nodeValue.match(r)||t.nodeValue.match(l));for(var o=g(e).filter(function(t,e){return e instanceof RegExp}),n=0,s=o.length;n<s;n++)if(i.match(o[n]))return!0;return!1}function n(t,e,i){if(0===t.length)return t;if(i&&"function"==typeof i)return i(t);if(!document.implementation||!document.implementation.createHTMLDocument)return t;var o=document.implementation.createHTMLDocument("sanitization");o.body.innerHTML=t;for(var n=g.map(e,function(t,e){return e}),s=g(o.body).find("*"),a=0,r=s.length;a<r;a++){var l=s[a],h=l.nodeName.toLowerCase();if(-1!==g.inArray(h,n))for(var d=g.map(l.attributes,function(t){return t}),p=[].concat(e["*"]||[],e[h]||[]),c=0,f=d.length;c<f;c++)u(d[c],p)||l.removeAttribute(d[c].nodeName);else l.parentNode.removeChild(l)}return o.body.innerHTML}var m=function(t,e){this.type=null,this.options=null,this.enabled=null,this.timeout=null,this.hoverState=null,this.$element=null,this.inState=null,this.init("tooltip",t,e)};m.VERSION="3.4.1",m.TRANSITION_DURATION=150,m.DEFAULTS={animation:!0,placement:"top",selector:!1,template:'<div class="tooltip" role="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>',trigger:"hover focus",title:"",delay:0,html:!1,container:!1,viewport:{selector:"body",padding:0},sanitize:!0,sanitizeFn:null,whiteList:t},m.prototype.init=function(t,e,i){if(this.enabled=!0,this.type=t,this.$element=g(e),this.options=this.getOptions(i),this.$viewport=this.options.viewport&&g(document).find(g.isFunction(this.options.viewport)?this.options.viewport.call(this,this.$element):this.options.viewport.selector||this.options.viewport),this.inState={click:!1,hover:!1,focus:!1},this.$element[0]instanceof document.constructor&&!this.options.selector)throw new Error("`selector` option must be specified when initializing "+this.type+" on the window.document object!");for(var o=this.options.trigger.split(" "),n=o.length;n--;){var s=o[n];if("click"==s)this.$element.on("click."+this.type,this.options.selector,g.proxy(this.toggle,this));else if("manual"!=s){var a="hover"==s?"mouseenter":"focusin",r="hover"==s?"mouseleave":"focusout";this.$element.on(a+"."+this.type,this.options.selector,g.proxy(this.enter,this)),this.$element.on(r+"."+this.type,this.options.selector,g.proxy(this.leave,this))}}this.options.selector?this._options=g.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},m.prototype.getDefaults=function(){return m.DEFAULTS},m.prototype.getOptions=function(t){var e=this.$element.data();for(var i in e)e.hasOwnProperty(i)&&-1!==g.inArray(i,o)&&delete e[i];return(t=g.extend({},this.getDefaults(),e,t)).delay&&"number"==typeof t.delay&&(t.delay={show:t.delay,hide:t.delay}),t.sanitize&&(t.template=n(t.template,t.whiteList,t.sanitizeFn)),t},m.prototype.getDelegateOptions=function(){var i={},o=this.getDefaults();return this._options&&g.each(this._options,function(t,e){o[t]!=e&&(i[t]=e)}),i},m.prototype.enter=function(t){var e=t instanceof this.constructor?t:g(t.currentTarget).data("bs."+this.type);if(e||(e=new this.constructor(t.currentTarget,this.getDelegateOptions()),g(t.currentTarget).data("bs."+this.type,e)),t instanceof g.Event&&(e.inState["focusin"==t.type?"focus":"hover"]=!0),e.tip().hasClass("in")||"in"==e.hoverState)e.hoverState="in";else{if(clearTimeout(e.timeout),e.hoverState="in",!e.options.delay||!e.options.delay.show)return e.show();e.timeout=setTimeout(function(){"in"==e.hoverState&&e.show()},e.options.delay.show)}},m.prototype.isInStateTrue=function(){for(var t in this.inState)if(this.inState[t])return!0;return!1},m.prototype.leave=function(t){var e=t instanceof this.constructor?t:g(t.currentTarget).data("bs."+this.type);if(e||(e=new this.constructor(t.currentTarget,this.getDelegateOptions()),g(t.currentTarget).data("bs."+this.type,e)),t instanceof g.Event&&(e.inState["focusout"==t.type?"focus":"hover"]=!1),!e.isInStateTrue()){if(clearTimeout(e.timeout),e.hoverState="out",!e.options.delay||!e.options.delay.hide)return e.hide();e.timeout=setTimeout(function(){"out"==e.hoverState&&e.hide()},e.options.delay.hide)}},m.prototype.show=function(){var t=g.Event("show.bs."+this.type);if(this.hasContent()&&this.enabled){this.$element.trigger(t);var e=g.contains(this.$element[0].ownerDocument.documentElement,this.$element[0]);if(t.isDefaultPrevented()||!e)return;var i=this,o=this.tip(),n=this.getUID(this.type);this.setContent(),o.attr("id",n),this.$element.attr("aria-describedby",n),this.options.animation&&o.addClass("fade");var s="function"==typeof this.options.placement?this.options.placement.call(this,o[0],this.$element[0]):this.options.placement,a=/\s?auto?\s?/i,r=a.test(s);r&&(s=s.replace(a,"")||"top"),o.detach().css({top:0,left:0,display:"block"}).addClass(s).data("bs."+this.type,this),this.options.container?o.appendTo(g(document).find(this.options.container)):o.insertAfter(this.$element),this.$element.trigger("inserted.bs."+this.type);var l=this.getPosition(),h=o[0].offsetWidth,d=o[0].offsetHeight;if(r){var p=s,c=this.getPosition(this.$viewport);s="bottom"==s&&l.bottom+d>c.bottom?"top":"top"==s&&l.top-d<c.top?"bottom":"right"==s&&l.right+h>c.width?"left":"left"==s&&l.left-h<c.left?"right":s,o.removeClass(p).addClass(s)}var f=this.getCalculatedOffset(s,l,h,d);this.applyPlacement(f,s);var u=function(){var t=i.hoverState;i.$element.trigger("shown.bs."+i.type),i.hoverState=null,"out"==t&&i.leave(i)};g.support.transition&&this.$tip.hasClass("fade")?o.one("bsTransitionEnd",u).emulateTransitionEnd(m.TRANSITION_DURATION):u()}},m.prototype.applyPlacement=function(t,e){var i=this.tip(),o=i[0].offsetWidth,n=i[0].offsetHeight,s=parseInt(i.css("margin-top"),10),a=parseInt(i.css("margin-left"),10);isNaN(s)&&(s=0),isNaN(a)&&(a=0),t.top+=s,t.left+=a,g.offset.setOffset(i[0],g.extend({using:function(t){i.css({top:Math.round(t.top),left:Math.round(t.left)})}},t),0),i.addClass("in");var r=i[0].offsetWidth,l=i[0].offsetHeight;"top"==e&&l!=n&&(t.top=t.top+n-l);var h=this.getViewportAdjustedDelta(e,t,r,l);h.left?t.left+=h.left:t.top+=h.top;var d=/top|bottom/.test(e),p=d?2*h.left-o+r:2*h.top-n+l,c=d?"offsetWidth":"offsetHeight";i.offset(t),this.replaceArrow(p,i[0][c],d)},m.prototype.replaceArrow=function(t,e,i){this.arrow().css(i?"left":"top",50*(1-t/e)+"%").css(i?"top":"left","")},m.prototype.setContent=function(){var t=this.tip(),e=this.getTitle();this.options.html?(this.options.sanitize&&(e=n(e,this.options.whiteList,this.options.sanitizeFn)),t.find(".tooltip-inner").html(e)):t.find(".tooltip-inner").text(e),t.removeClass("fade in top bottom left right")},m.prototype.hide=function(t){var e=this,i=g(this.$tip),o=g.Event("hide.bs."+this.type);function n(){"in"!=e.hoverState&&i.detach(),e.$element&&e.$element.removeAttr("aria-describedby").trigger("hidden.bs."+e.type),t&&t()}if(this.$element.trigger(o),!o.isDefaultPrevented())return i.removeClass("in"),g.support.transition&&i.hasClass("fade")?i.one("bsTransitionEnd",n).emulateTransitionEnd(m.TRANSITION_DURATION):n(),this.hoverState=null,this},m.prototype.fixTitle=function(){var t=this.$element;(t.attr("title")||"string"!=typeof t.attr("data-original-title"))&&t.attr("data-original-title",t.attr("title")||"").attr("title","")},m.prototype.hasContent=function(){return this.getTitle()},m.prototype.getPosition=function(t){var e=(t=t||this.$element)[0],i="BODY"==e.tagName,o=e.getBoundingClientRect();null==o.width&&(o=g.extend({},o,{width:o.right-o.left,height:o.bottom-o.top}));var n=window.SVGElement&&e instanceof window.SVGElement,s=i?{top:0,left:0}:n?null:t.offset(),a={scroll:i?document.documentElement.scrollTop||document.body.scrollTop:t.scrollTop()},r=i?{width:g(window).width(),height:g(window).height()}:null;return g.extend({},o,a,r,s)},m.prototype.getCalculatedOffset=function(t,e,i,o){return"bottom"==t?{top:e.top+e.height,left:e.left+e.width/2-i/2}:"top"==t?{top:e.top-o,left:e.left+e.width/2-i/2}:"left"==t?{top:e.top+e.height/2-o/2,left:e.left-i}:{top:e.top+e.height/2-o/2,left:e.left+e.width}},m.prototype.getViewportAdjustedDelta=function(t,e,i,o){var n={top:0,left:0};if(!this.$viewport)return n;var s=this.options.viewport&&this.options.viewport.padding||0,a=this.getPosition(this.$viewport);if(/right|left/.test(t)){var r=e.top-s-a.scroll,l=e.top+s-a.scroll+o;r<a.top?n.top=a.top-r:l>a.top+a.height&&(n.top=a.top+a.height-l)}else{var h=e.left-s,d=e.left+s+i;h<a.left?n.left=a.left-h:d>a.right&&(n.left=a.left+a.width-d)}return n},m.prototype.getTitle=function(){var t=this.$element,e=this.options;return t.attr("data-original-title")||("function"==typeof e.title?e.title.call(t[0]):e.title)},m.prototype.getUID=function(t){for(;t+=~~(1e6*Math.random()),document.getElementById(t););return t},m.prototype.tip=function(){if(!this.$tip&&(this.$tip=g(this.options.template),1!=this.$tip.length))throw new Error(this.type+" `template` option must consist of exactly 1 top-level element!");return this.$tip},m.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},m.prototype.enable=function(){this.enabled=!0},m.prototype.disable=function(){this.enabled=!1},m.prototype.toggleEnabled=function(){this.enabled=!this.enabled},m.prototype.toggle=function(t){var e=this;t&&((e=g(t.currentTarget).data("bs."+this.type))||(e=new this.constructor(t.currentTarget,this.getDelegateOptions()),g(t.currentTarget).data("bs."+this.type,e))),t?(e.inState.click=!e.inState.click,e.isInStateTrue()?e.enter(e):e.leave(e)):e.tip().hasClass("in")?e.leave(e):e.enter(e)},m.prototype.destroy=function(){var t=this;clearTimeout(this.timeout),this.hide(function(){t.$element.off("."+t.type).removeData("bs."+t.type),t.$tip&&t.$tip.detach(),t.$tip=null,t.$arrow=null,t.$viewport=null,t.$element=null})},m.prototype.sanitizeHtml=function(t){return n(t,this.options.whiteList,this.options.sanitizeFn)};var e=g.fn.tooltip;g.fn.tooltip=function i(o){return this.each(function(){var t=g(this),e=t.data("bs.tooltip"),i="object"==typeof o&&o;!e&&/destroy|hide/.test(o)||(e||t.data("bs.tooltip",e=new m(this,i)),"string"==typeof o&&e[o]())})},g.fn.tooltip.Constructor=m,g.fn.tooltip.noConflict=function(){return g.fn.tooltip=e,this}}(jQuery),function(n){"use strict";var s=function(t,e){this.init("popover",t,e)};if(!n.fn.tooltip)throw new Error("Popover requires tooltip.js");s.VERSION="3.4.1",s.DEFAULTS=n.extend({},n.fn.tooltip.Constructor.DEFAULTS,{placement:"right",trigger:"click",content:"",template:'<div class="popover" role="tooltip"><div class="arrow"></div><h3 class="popover-title"></h3><div class="popover-content"></div></div>'}),((s.prototype=n.extend({},n.fn.tooltip.Constructor.prototype)).constructor=s).prototype.getDefaults=function(){return s.DEFAULTS},s.prototype.setContent=function(){var t=this.tip(),e=this.getTitle(),i=this.getContent();if(this.options.html){var o=typeof i;this.options.sanitize&&(e=this.sanitizeHtml(e),"string"===o&&(i=this.sanitizeHtml(i))),t.find(".popover-title").html(e),t.find(".popover-content").children().detach().end()["string"===o?"html":"append"](i)}else t.find(".popover-title").text(e),t.find(".popover-content").children().detach().end().text(i);t.removeClass("fade top bottom left right in"),t.find(".popover-title").html()||t.find(".popover-title").hide()},s.prototype.hasContent=function(){return this.getTitle()||this.getContent()},s.prototype.getContent=function(){var t=this.$element,e=this.options;return t.attr("data-content")||("function"==typeof e.content?e.content.call(t[0]):e.content)},s.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")};var t=n.fn.popover;n.fn.popover=function e(o){return this.each(function(){var t=n(this),e=t.data("bs.popover"),i="object"==typeof o&&o;!e&&/destroy|hide/.test(o)||(e||t.data("bs.popover",e=new s(this,i)),"string"==typeof o&&e[o]())})},n.fn.popover.Constructor=s,n.fn.popover.noConflict=function(){return n.fn.popover=t,this}}(jQuery),function(s){"use strict";function n(t,e){this.$body=s(document.body),this.$scrollElement=s(t).is(document.body)?s(window):s(t),this.options=s.extend({},n.DEFAULTS,e),this.selector=(this.options.target||"")+" .nav li > a",this.offsets=[],this.targets=[],this.activeTarget=null,this.scrollHeight=0,this.$scrollElement.on("scroll.bs.scrollspy",s.proxy(this.process,this)),this.refresh(),this.process()}function e(o){return this.each(function(){var t=s(this),e=t.data("bs.scrollspy"),i="object"==typeof o&&o;e||t.data("bs.scrollspy",e=new n(this,i)),"string"==typeof o&&e[o]()})}n.VERSION="3.4.1",n.DEFAULTS={offset:10},n.prototype.getScrollHeight=function(){return this.$scrollElement[0].scrollHeight||Math.max(this.$body[0].scrollHeight,document.documentElement.scrollHeight)},n.prototype.refresh=function(){var t=this,o="offset",n=0;this.offsets=[],this.targets=[],this.scrollHeight=this.getScrollHeight(),s.isWindow(this.$scrollElement[0])||(o="position",n=this.$scrollElement.scrollTop()),this.$body.find(this.selector).map(function(){var t=s(this),e=t.data("target")||t.attr("href"),i=/^#./.test(e)&&s(e);return i&&i.length&&i.is(":visible")&&[[i[o]().top+n,e]]||null}).sort(function(t,e){return t[0]-e[0]}).each(function(){t.offsets.push(this[0]),t.targets.push(this[1])})},n.prototype.process=function(){var t,e=this.$scrollElement.scrollTop()+this.options.offset,i=this.getScrollHeight(),o=this.options.offset+i-this.$scrollElement.height(),n=this.offsets,s=this.targets,a=this.activeTarget;if(this.scrollHeight!=i&&this.refresh(),o<=e)return a!=(t=s[s.length-1])&&this.activate(t);if(a&&e<n[0])return this.activeTarget=null,this.clear();for(t=n.length;t--;)a!=s[t]&&e>=n[t]&&(n[t+1]===undefined||e<n[t+1])&&this.activate(s[t])},n.prototype.activate=function(t){this.activeTarget=t,this.clear();var e=this.selector+'[data-target="'+t+'"],'+this.selector+'[href="'+t+'"]',i=s(e).parents("li").addClass("active");i.parent(".dropdown-menu").length&&(i=i.closest("li.dropdown").addClass("active")),i.trigger("activate.bs.scrollspy")},n.prototype.clear=function(){s(this.selector).parentsUntil(this.options.target,".active").removeClass("active")};var t=s.fn.scrollspy;s.fn.scrollspy=e,s.fn.scrollspy.Constructor=n,s.fn.scrollspy.noConflict=function(){return s.fn.scrollspy=t,this},s(window).on("load.bs.scrollspy.data-api",function(){s('[data-spy="scroll"]').each(function(){var t=s(this);e.call(t,t.data())})})}(jQuery),function(r){"use strict";var a=function(t){this.element=r(t)};function e(i){return this.each(function(){var t=r(this),e=t.data("bs.tab");e||t.data("bs.tab",e=new a(this)),"string"==typeof i&&e[i]()})}a.VERSION="3.4.1",a.TRANSITION_DURATION=150,a.prototype.show=function(){var t=this.element,e=t.closest("ul:not(.dropdown-menu)"),i=t.data("target");if(i||(i=(i=t.attr("href"))&&i.replace(/.*(?=#[^\s]*$)/,"")),!t.parent("li").hasClass("active")){var o=e.find(".active:last a"),n=r.Event("hide.bs.tab",{relatedTarget:t[0]}),s=r.Event("show.bs.tab",{relatedTarget:o[0]});if(o.trigger(n),t.trigger(s),!s.isDefaultPrevented()&&!n.isDefaultPrevented()){var a=r(document).find(i);this.activate(t.closest("li"),e),this.activate(a,a.parent(),function(){o.trigger({type:"hidden.bs.tab",relatedTarget:t[0]}),t.trigger({type:"shown.bs.tab",relatedTarget:o[0]})})}}},a.prototype.activate=function(t,e,i){var o=e.find("> .active"),n=i&&r.support.transition&&(o.length&&o.hasClass("fade")||!!e.find("> .fade").length);function s(){o.removeClass("active").find("> .dropdown-menu > .active").removeClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!1),t.addClass("active").find('[data-toggle="tab"]').attr("aria-expanded",!0),n?(t[0].offsetWidth,t.addClass("in")):t.removeClass("fade"),t.parent(".dropdown-menu").length&&t.closest("li.dropdown").addClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!0),i&&i()}o.length&&n?o.one("bsTransitionEnd",s).emulateTransitionEnd(a.TRANSITION_DURATION):s(),o.removeClass("in")};var t=r.fn.tab;r.fn.tab=e,r.fn.tab.Constructor=a,r.fn.tab.noConflict=function(){return r.fn.tab=t,this};var i=function(t){t.preventDefault(),e.call(r(this),"show")};r(document).on("click.bs.tab.data-api",'[data-toggle="tab"]',i).on("click.bs.tab.data-api",'[data-toggle="pill"]',i)}(jQuery),function(l){"use strict";var h=function(t,e){this.options=l.extend({},h.DEFAULTS,e);var i=this.options.target===h.DEFAULTS.target?l(this.options.target):l(document).find(this.options.target);this.$target=i.on("scroll.bs.affix.data-api",l.proxy(this.checkPosition,this)).on("click.bs.affix.data-api",l.proxy(this.checkPositionWithEventLoop,this)),this.$element=l(t),this.affixed=null,this.unpin=null,this.pinnedOffset=null,this.checkPosition()};function i(o){return this.each(function(){var t=l(this),e=t.data("bs.affix"),i="object"==typeof o&&o;e||t.data("bs.affix",e=new h(this,i)),"string"==typeof o&&e[o]()})}h.VERSION="3.4.1",h.RESET="affix affix-top affix-bottom",h.DEFAULTS={offset:0,target:window},h.prototype.getState=function(t,e,i,o){var n=this.$target.scrollTop(),s=this.$element.offset(),a=this.$target.height();if(null!=i&&"top"==this.affixed)return n<i&&"top";if("bottom"==this.affixed)return null!=i?!(n+this.unpin<=s.top)&&"bottom":!(n+a<=t-o)&&"bottom";var r=null==this.affixed,l=r?n:s.top;return null!=i&&n<=i?"top":null!=o&&t-o<=l+(r?a:e)&&"bottom"},h.prototype.getPinnedOffset=function(){if(this.pinnedOffset)return this.pinnedOffset;this.$element.removeClass(h.RESET).addClass("affix");var t=this.$target.scrollTop(),e=this.$element.offset();return this.pinnedOffset=e.top-t},h.prototype.checkPositionWithEventLoop=function(){setTimeout(l.proxy(this.checkPosition,this),1)},h.prototype.checkPosition=function(){if(this.$element.is(":visible")){var t=this.$element.height(),e=this.options.offset,i=e.top,o=e.bottom,n=Math.max(l(document).height(),l(document.body).height());"object"!=typeof e&&(o=i=e),"function"==typeof i&&(i=e.top(this.$element)),"function"==typeof o&&(o=e.bottom(this.$element));var s=this.getState(n,t,i,o);if(this.affixed!=s){null!=this.unpin&&this.$element.css("top","");var a="affix"+(s?"-"+s:""),r=l.Event(a+".bs.affix");if(this.$element.trigger(r),r.isDefaultPrevented())return;this.affixed=s,this.unpin="bottom"==s?this.getPinnedOffset():null,this.$element.removeClass(h.RESET).addClass(a).trigger(a.replace("affix","affixed")+".bs.affix")}"bottom"==s&&this.$element.offset({top:n-t-o})}};var t=l.fn.affix;l.fn.affix=i,l.fn.affix.Constructor=h,l.fn.affix.noConflict=function(){return l.fn.affix=t,this},l(window).on("load",function(){l('[data-spy="affix"]').each(function(){var t=l(this),e=t.data();e.offset=e.offset||{},null!=e.offsetBottom&&(e.offset.bottom=e.offsetBottom),null!=e.offsetTop&&(e.offset.top=e.offsetTop),i.call(t,e)})})}(jQuery);
\ No newline at end of file
diff --git a/bitbake/lib/toaster/toastergui/static/js/bootstrap.min.js b/bitbake/lib/toaster/toastergui/static/js/bootstrap.min.js
deleted file mode 100644
index c4a924160d..0000000000
--- a/bitbake/lib/toaster/toastergui/static/js/bootstrap.min.js
+++ /dev/null
@@ -1,7 +0,0 @@
-/*!
- * Bootstrap v3.3.6 (http://getbootstrap.com)
- * Copyright 2011-2016 Twitter, Inc.
- * Licensed under the MIT license
- */
-if("undefined"==typeof jQuery)throw new Error("Bootstrap's JavaScript requires jQuery");+function(a){"use strict";var b=a.fn.jquery.split(" ")[0].split(".");if(b[0]<2&&b[1]<9||1==b[0]&&9==b[1]&&b[2]<1||b[0]>2)throw new Error("Bootstrap's JavaScript requires jQuery version 1.9.1 or higher, but lower than version 3")}(jQuery),+function(a){"use strict";function b(){var a=document.createElement("bootstrap"),b={WebkitTransition:"webkitTransitionEnd",MozTransition:"transitionend",OTransition:"oTransitionEnd otransitionend",transition:"transitionend"};for(var c in b)if(void 0!==a.style[c])return{end:b[c]};return!1}a.fn.emulateTransitionEnd=function(b){var c=!1,d=this;a(this).one("bsTransitionEnd",function(){c=!0});var e=function(){c||a(d).trigger(a.support.transition.end)};return setTimeout(e,b),this},a(function(){a.support.transition=b(),a.support.transition&&(a.event.special.bsTransitionEnd={bindType:a.support.transition.end,delegateType:a.support.transition.end,handle:function(b){return a(b.target).is(this)?b.handleObj.handler.apply(this,arguments):void 0}})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var c=a(this),e=c.data("bs.alert");e||c.data("bs.alert",e=new d(this)),"string"==typeof b&&e[b].call(c)})}var c='[data-dismiss="alert"]',d=function(b){a(b).on("click",c,this.close)};d.VERSION="3.3.6",d.TRANSITION_DURATION=150,d.prototype.close=function(b){function c(){g.detach().trigger("closed.bs.alert").remove()}var e=a(this),f=e.attr("data-target");f||(f=e.attr("href"),f=f&&f.replace(/.*(?=#[^\s]*$)/,""));var g=a(f);b&&b.preventDefault(),g.length||(g=e.closest(".alert")),g.trigger(b=a.Event("close.bs.alert")),b.isDefaultPrevented()||(g.removeClass("in"),a.support.transition&&g.hasClass("fade")?g.one("bsTransitionEnd",c).emulateTransitionEnd(d.TRANSITION_DURATION):c())};var e=a.fn.alert;a.fn.alert=b,a.fn.alert.Constructor=d,a.fn.alert.noConflict=function(){return a.fn.alert=e,this},a(document).on("click.bs.alert.data-api",c,d.prototype.close)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.button"),f="object"==typeof b&&b;e||d.data("bs.button",e=new c(this,f)),"toggle"==b?e.toggle():b&&e.setState(b)})}var c=function(b,d){this.$element=a(b),this.options=a.extend({},c.DEFAULTS,d),this.isLoading=!1};c.VERSION="3.3.6",c.DEFAULTS={loadingText:"loading..."},c.prototype.setState=function(b){var c="disabled",d=this.$element,e=d.is("input")?"val":"html",f=d.data();b+="Text",null==f.resetText&&d.data("resetText",d[e]()),setTimeout(a.proxy(function(){d[e](null==f[b]?this.options[b]:f[b]),"loadingText"==b?(this.isLoading=!0,d.addClass(c).attr(c,c)):this.isLoading&&(this.isLoading=!1,d.removeClass(c).removeAttr(c))},this),0)},c.prototype.toggle=function(){var a=!0,b=this.$element.closest('[data-toggle="buttons"]');if(b.length){var c=this.$element.find("input");"radio"==c.prop("type")?(c.prop("checked")&&(a=!1),b.find(".active").removeClass("active"),this.$element.addClass("active")):"checkbox"==c.prop("type")&&(c.prop("checked")!==this.$element.hasClass("active")&&(a=!1),this.$element.toggleClass("active")),c.prop("checked",this.$element.hasClass("active")),a&&c.trigger("change")}else this.$element.attr("aria-pressed",!this.$element.hasClass("active")),this.$element.toggleClass("active")};var d=a.fn.button;a.fn.button=b,a.fn.button.Constructor=c,a.fn.button.noConflict=function(){return a.fn.button=d,this},a(document).on("click.bs.button.data-api",'[data-toggle^="button"]',function(c){var d=a(c.target);d.hasClass("btn")||(d=d.closest(".btn")),b.call(d,"toggle"),a(c.target).is('input[type="radio"]')||a(c.target).is('input[type="checkbox"]')||c.preventDefault()}).on("focus.bs.button.data-api blur.bs.button.data-api",'[data-toggle^="button"]',function(b){a(b.target).closest(".btn").toggleClass("focus",/^focus(in)?$/.test(b.type))})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.carousel"),f=a.extend({},c.DEFAULTS,d.data(),"object"==typeof b&&b),g="string"==typeof b?b:f.slide;e||d.data("bs.carousel",e=new c(this,f)),"number"==typeof b?e.to(b):g?e[g]():f.interval&&e.pause().cycle()})}var c=function(b,c){this.$element=a(b),this.$indicators=this.$element.find(".carousel-indicators"),this.options=c,this.paused=null,this.sliding=null,this.interval=null,this.$active=null,this.$items=null,this.options.keyboard&&this.$element.on("keydown.bs.carousel",a.proxy(this.keydown,this)),"hover"==this.options.pause&&!("ontouchstart"in document.documentElement)&&this.$element.on("mouseenter.bs.carousel",a.proxy(this.pause,this)).on("mouseleave.bs.carousel",a.proxy(this.cycle,this))};c.VERSION="3.3.6",c.TRANSITION_DURATION=600,c.DEFAULTS={interval:5e3,pause:"hover",wrap:!0,keyboard:!0},c.prototype.keydown=function(a){if(!/input|textarea/i.test(a.target.tagName)){switch(a.which){case 37:this.prev();break;case 39:this.next();break;default:return}a.preventDefault()}},c.prototype.cycle=function(b){return b||(this.paused=!1),this.interval&&clearInterval(this.interval),this.options.interval&&!this.paused&&(this.interval=setInterval(a.proxy(this.next,this),this.options.interval)),this},c.prototype.getItemIndex=function(a){return this.$items=a.parent().children(".item"),this.$items.index(a||this.$active)},c.prototype.getItemForDirection=function(a,b){var c=this.getItemIndex(b),d="prev"==a&&0===c||"next"==a&&c==this.$items.length-1;if(d&&!this.options.wrap)return b;var e="prev"==a?-1:1,f=(c+e)%this.$items.length;return this.$items.eq(f)},c.prototype.to=function(a){var b=this,c=this.getItemIndex(this.$active=this.$element.find(".item.active"));return a>this.$items.length-1||0>a?void 0:this.sliding?this.$element.one("slid.bs.carousel",function(){b.to(a)}):c==a?this.pause().cycle():this.slide(a>c?"next":"prev",this.$items.eq(a))},c.prototype.pause=function(b){return b||(this.paused=!0),this.$element.find(".next, .prev").length&&a.support.transition&&(this.$element.trigger(a.support.transition.end),this.cycle(!0)),this.interval=clearInterval(this.interval),this},c.prototype.next=function(){return this.sliding?void 0:this.slide("next")},c.prototype.prev=function(){return this.sliding?void 0:this.slide("prev")},c.prototype.slide=function(b,d){var e=this.$element.find(".item.active"),f=d||this.getItemForDirection(b,e),g=this.interval,h="next"==b?"left":"right",i=this;if(f.hasClass("active"))return this.sliding=!1;var j=f[0],k=a.Event("slide.bs.carousel",{relatedTarget:j,direction:h});if(this.$element.trigger(k),!k.isDefaultPrevented()){if(this.sliding=!0,g&&this.pause(),this.$indicators.length){this.$indicators.find(".active").removeClass("active");var l=a(this.$indicators.children()[this.getItemIndex(f)]);l&&l.addClass("active")}var m=a.Event("slid.bs.carousel",{relatedTarget:j,direction:h});return a.support.transition&&this.$element.hasClass("slide")?(f.addClass(b),f[0].offsetWidth,e.addClass(h),f.addClass(h),e.one("bsTransitionEnd",function(){f.removeClass([b,h].join(" ")).addClass("active"),e.removeClass(["active",h].join(" ")),i.sliding=!1,setTimeout(function(){i.$element.trigger(m)},0)}).emulateTransitionEnd(c.TRANSITION_DURATION)):(e.removeClass("active"),f.addClass("active"),this.sliding=!1,this.$element.trigger(m)),g&&this.cycle(),this}};var d=a.fn.carousel;a.fn.carousel=b,a.fn.carousel.Constructor=c,a.fn.carousel.noConflict=function(){return a.fn.carousel=d,this};var e=function(c){var d,e=a(this),f=a(e.attr("data-target")||(d=e.attr("href"))&&d.replace(/.*(?=#[^\s]+$)/,""));if(f.hasClass("carousel")){var g=a.extend({},f.data(),e.data()),h=e.attr("data-slide-to");h&&(g.interval=!1),b.call(f,g),h&&f.data("bs.carousel").to(h),c.preventDefault()}};a(document).on("click.bs.carousel.data-api","[data-slide]",e).on("click.bs.carousel.data-api","[data-slide-to]",e),a(window).on("load",function(){a('[data-ride="carousel"]').each(function(){var c=a(this);b.call(c,c.data())})})}(jQuery),+function(a){"use strict";function b(b){var c,d=b.attr("data-target")||(c=b.attr("href"))&&c.replace(/.*(?=#[^\s]+$)/,"");return a(d)}function c(b){return this.each(function(){var c=a(this),e=c.data("bs.collapse"),f=a.extend({},d.DEFAULTS,c.data(),"object"==typeof b&&b);!e&&f.toggle&&/show|hide/.test(b)&&(f.toggle=!1),e||c.data("bs.collapse",e=new d(this,f)),"string"==typeof b&&e[b]()})}var d=function(b,c){this.$element=a(b),this.options=a.extend({},d.DEFAULTS,c),this.$trigger=a('[data-toggle="collapse"][href="#'+b.id+'"],[data-toggle="collapse"][data-target="#'+b.id+'"]'),this.transitioning=null,this.options.parent?this.$parent=this.getParent():this.addAriaAndCollapsedClass(this.$element,this.$trigger),this.options.toggle&&this.toggle()};d.VERSION="3.3.6",d.TRANSITION_DURATION=350,d.DEFAULTS={toggle:!0},d.prototype.dimension=function(){var a=this.$element.hasClass("width");return a?"width":"height"},d.prototype.show=function(){if(!this.transitioning&&!this.$element.hasClass("in")){var b,e=this.$parent&&this.$parent.children(".panel").children(".in, .collapsing");if(!(e&&e.length&&(b=e.data("bs.collapse"),b&&b.transitioning))){var f=a.Event("show.bs.collapse");if(this.$element.trigger(f),!f.isDefaultPrevented()){e&&e.length&&(c.call(e,"hide"),b||e.data("bs.collapse",null));var g=this.dimension();this.$element.removeClass("collapse").addClass("collapsing")[g](0).attr("aria-expanded",!0),this.$trigger.removeClass("collapsed").attr("aria-expanded",!0),this.transitioning=1;var h=function(){this.$element.removeClass("collapsing").addClass("collapse in")[g](""),this.transitioning=0,this.$element.trigger("shown.bs.collapse")};if(!a.support.transition)return h.call(this);var i=a.camelCase(["scroll",g].join("-"));this.$element.one("bsTransitionEnd",a.proxy(h,this)).emulateTransitionEnd(d.TRANSITION_DURATION)[g](this.$element[0][i])}}}},d.prototype.hide=function(){if(!this.transitioning&&this.$element.hasClass("in")){var b=a.Event("hide.bs.collapse");if(this.$element.trigger(b),!b.isDefaultPrevented()){var c=this.dimension();this.$element[c](this.$element[c]())[0].offsetHeight,this.$element.addClass("collapsing").removeClass("collapse in").attr("aria-expanded",!1),this.$trigger.addClass("collapsed").attr("aria-expanded",!1),this.transitioning=1;var e=function(){this.transitioning=0,this.$element.removeClass("collapsing").addClass("collapse").trigger("hidden.bs.collapse")};return a.support.transition?void this.$element[c](0).one("bsTransitionEnd",a.proxy(e,this)).emulateTransitionEnd(d.TRANSITION_DURATION):e.call(this)}}},d.prototype.toggle=function(){this[this.$element.hasClass("in")?"hide":"show"]()},d.prototype.getParent=function(){return a(this.options.parent).find('[data-toggle="collapse"][data-parent="'+this.options.parent+'"]').each(a.proxy(function(c,d){var e=a(d);this.addAriaAndCollapsedClass(b(e),e)},this)).end()},d.prototype.addAriaAndCollapsedClass=function(a,b){var c=a.hasClass("in");a.attr("aria-expanded",c),b.toggleClass("collapsed",!c).attr("aria-expanded",c)};var e=a.fn.collapse;a.fn.collapse=c,a.fn.collapse.Constructor=d,a.fn.collapse.noConflict=function(){return a.fn.collapse=e,this},a(document).on("click.bs.collapse.data-api",'[data-toggle="collapse"]',function(d){var e=a(this);e.attr("data-target")||d.preventDefault();var f=b(e),g=f.data("bs.collapse"),h=g?"toggle":e.data();c.call(f,h)})}(jQuery),+function(a){"use strict";function b(b){var c=b.attr("data-target");c||(c=b.attr("href"),c=c&&/#[A-Za-z]/.test(c)&&c.replace(/.*(?=#[^\s]*$)/,""));var d=c&&a(c);return d&&d.length?d:b.parent()}function c(c){c&&3===c.which||(a(e).remove(),a(f).each(function(){var d=a(this),e=b(d),f={relatedTarget:this};e.hasClass("open")&&(c&&"click"==c.type&&/input|textarea/i.test(c.target.tagName)&&a.contains(e[0],c.target)||(e.trigger(c=a.Event("hide.bs.dropdown",f)),c.isDefaultPrevented()||(d.attr("aria-expanded","false"),e.removeClass("open").trigger(a.Event("hidden.bs.dropdown",f)))))}))}function d(b){return this.each(function(){var c=a(this),d=c.data("bs.dropdown");d||c.data("bs.dropdown",d=new g(this)),"string"==typeof b&&d[b].call(c)})}var e=".dropdown-backdrop",f='[data-toggle="dropdown"]',g=function(b){a(b).on("click.bs.dropdown",this.toggle)};g.VERSION="3.3.6",g.prototype.toggle=function(d){var e=a(this);if(!e.is(".disabled, :disabled")){var f=b(e),g=f.hasClass("open");if(c(),!g){"ontouchstart"in document.documentElement&&!f.closest(".navbar-nav").length&&a(document.createElement("div")).addClass("dropdown-backdrop").insertAfter(a(this)).on("click",c);var h={relatedTarget:this};if(f.trigger(d=a.Event("show.bs.dropdown",h)),d.isDefaultPrevented())return;e.trigger("focus").attr("aria-expanded","true"),f.toggleClass("open").trigger(a.Event("shown.bs.dropdown",h))}return!1}},g.prototype.keydown=function(c){if(/(38|40|27|32)/.test(c.which)&&!/input|textarea/i.test(c.target.tagName)){var d=a(this);if(c.preventDefault(),c.stopPropagation(),!d.is(".disabled, :disabled")){var e=b(d),g=e.hasClass("open");if(!g&&27!=c.which||g&&27==c.which)return 27==c.which&&e.find(f).trigger("focus"),d.trigger("click");var h=" li:not(.disabled):visible a",i=e.find(".dropdown-menu"+h);if(i.length){var j=i.index(c.target);38==c.which&&j>0&&j--,40==c.which&&j<i.length-1&&j++,~j||(j=0),i.eq(j).trigger("focus")}}}};var h=a.fn.dropdown;a.fn.dropdown=d,a.fn.dropdown.Constructor=g,a.fn.dropdown.noConflict=function(){return a.fn.dropdown=h,this},a(document).on("click.bs.dropdown.data-api",c).on("click.bs.dropdown.data-api",".dropdown form",function(a){a.stopPropagation()}).on("click.bs.dropdown.data-api",f,g.prototype.toggle).on("keydown.bs.dropdown.data-api",f,g.prototype.keydown).on("keydown.bs.dropdown.data-api",".dropdown-menu",g.prototype.keydown)}(jQuery),+function(a){"use strict";function b(b,d){return this.each(function(){var e=a(this),f=e.data("bs.modal"),g=a.extend({},c.DEFAULTS,e.data(),"object"==typeof b&&b);f||e.data("bs.modal",f=new c(this,g)),"string"==typeof b?f[b](d):g.show&&f.show(d)})}var c=function(b,c){this.options=c,this.$body=a(document.body),this.$element=a(b),this.$dialog=this.$element.find(".modal-dialog"),this.$backdrop=null,this.isShown=null,this.originalBodyPad=null,this.scrollbarWidth=0,this.ignoreBackdropClick=!1,this.options.remote&&this.$element.find(".modal-content").load(this.options.remote,a.proxy(function(){this.$element.trigger("loaded.bs.modal")},this))};c.VERSION="3.3.6",c.TRANSITION_DURATION=300,c.BACKDROP_TRANSITION_DURATION=150,c.DEFAULTS={backdrop:!0,keyboard:!0,show:!0},c.prototype.toggle=function(a){return this.isShown?this.hide():this.show(a)},c.prototype.show=function(b){var d=this,e=a.Event("show.bs.modal",{relatedTarget:b});this.$element.trigger(e),this.isShown||e.isDefaultPrevented()||(this.isShown=!0,this.checkScrollbar(),this.setScrollbar(),this.$body.addClass("modal-open"),this.escape(),this.resize(),this.$element.on("click.dismiss.bs.modal",'[data-dismiss="modal"]',a.proxy(this.hide,this)),this.$dialog.on("mousedown.dismiss.bs.modal",function(){d.$element.one("mouseup.dismiss.bs.modal",function(b){a(b.target).is(d.$element)&&(d.ignoreBackdropClick=!0)})}),this.backdrop(function(){var e=a.support.transition&&d.$element.hasClass("fade");d.$element.parent().length||d.$element.appendTo(d.$body),d.$element.show().scrollTop(0),d.adjustDialog(),e&&d.$element[0].offsetWidth,d.$element.addClass("in"),d.enforceFocus();var f=a.Event("shown.bs.modal",{relatedTarget:b});e?d.$dialog.one("bsTransitionEnd",function(){d.$element.trigger("focus").trigger(f)}).emulateTransitionEnd(c.TRANSITION_DURATION):d.$element.trigger("focus").trigger(f)}))},c.prototype.hide=function(b){b&&b.preventDefault(),b=a.Event("hide.bs.modal"),this.$element.trigger(b),this.isShown&&!b.isDefaultPrevented()&&(this.isShown=!1,this.escape(),this.resize(),a(document).off("focusin.bs.modal"),this.$element.removeClass("in").off("click.dismiss.bs.modal").off("mouseup.dismiss.bs.modal"),this.$dialog.off("mousedown.dismiss.bs.modal"),a.support.transition&&this.$element.hasClass("fade")?this.$element.one("bsTransitionEnd",a.proxy(this.hideModal,this)).emulateTransitionEnd(c.TRANSITION_DURATION):this.hideModal())},c.prototype.enforceFocus=function(){a(document).off("focusin.bs.modal").on("focusin.bs.modal",a.proxy(function(a){this.$element[0]===a.target||this.$element.has(a.target).length||this.$element.trigger("focus")},this))},c.prototype.escape=function(){this.isShown&&this.options.keyboard?this.$element.on("keydown.dismiss.bs.modal",a.proxy(function(a){27==a.which&&this.hide()},this)):this.isShown||this.$element.off("keydown.dismiss.bs.modal")},c.prototype.resize=function(){this.isShown?a(window).on("resize.bs.modal",a.proxy(this.handleUpdate,this)):a(window).off("resize.bs.modal")},c.prototype.hideModal=function(){var a=this;this.$element.hide(),this.backdrop(function(){a.$body.removeClass("modal-open"),a.resetAdjustments(),a.resetScrollbar(),a.$element.trigger("hidden.bs.modal")})},c.prototype.removeBackdrop=function(){this.$backdrop&&this.$backdrop.remove(),this.$backdrop=null},c.prototype.backdrop=function(b){var d=this,e=this.$element.hasClass("fade")?"fade":"";if(this.isShown&&this.options.backdrop){var f=a.support.transition&&e;if(this.$backdrop=a(document.createElement("div")).addClass("modal-backdrop "+e).appendTo(this.$body),this.$element.on("click.dismiss.bs.modal",a.proxy(function(a){return this.ignoreBackdropClick?void(this.ignoreBackdropClick=!1):void(a.target===a.currentTarget&&("static"==this.options.backdrop?this.$element[0].focus():this.hide()))},this)),f&&this.$backdrop[0].offsetWidth,this.$backdrop.addClass("in"),!b)return;f?this.$backdrop.one("bsTransitionEnd",b).emulateTransitionEnd(c.BACKDROP_TRANSITION_DURATION):b()}else if(!this.isShown&&this.$backdrop){this.$backdrop.removeClass("in");var g=function(){d.removeBackdrop(),b&&b()};a.support.transition&&this.$element.hasClass("fade")?this.$backdrop.one("bsTransitionEnd",g).emulateTransitionEnd(c.BACKDROP_TRANSITION_DURATION):g()}else b&&b()},c.prototype.handleUpdate=function(){this.adjustDialog()},c.prototype.adjustDialog=function(){var a=this.$element[0].scrollHeight>document.documentElement.clientHeight;this.$element.css({paddingLeft:!this.bodyIsOverflowing&&a?this.scrollbarWidth:"",paddingRight:this.bodyIsOverflowing&&!a?this.scrollbarWidth:""})},c.prototype.resetAdjustments=function(){this.$element.css({paddingLeft:"",paddingRight:""})},c.prototype.checkScrollbar=function(){var a=window.innerWidth;if(!a){var b=document.documentElement.getBoundingClientRect();a=b.right-Math.abs(b.left)}this.bodyIsOverflowing=document.body.clientWidth<a,this.scrollbarWidth=this.measureScrollbar()},c.prototype.setScrollbar=function(){var a=parseInt(this.$body.css("padding-right")||0,10);this.originalBodyPad=document.body.style.paddingRight||"",this.bodyIsOverflowing&&this.$body.css("padding-right",a+this.scrollbarWidth)},c.prototype.resetScrollbar=function(){this.$body.css("padding-right",this.originalBodyPad)},c.prototype.measureScrollbar=function(){var a=document.createElement("div");a.className="modal-scrollbar-measure",this.$body.append(a);var b=a.offsetWidth-a.clientWidth;return this.$body[0].removeChild(a),b};var d=a.fn.modal;a.fn.modal=b,a.fn.modal.Constructor=c,a.fn.modal.noConflict=function(){return a.fn.modal=d,this},a(document).on("click.bs.modal.data-api",'[data-toggle="modal"]',function(c){var d=a(this),e=d.attr("href"),f=a(d.attr("data-target")||e&&e.replace(/.*(?=#[^\s]+$)/,"")),g=f.data("bs.modal")?"toggle":a.extend({remote:!/#/.test(e)&&e},f.data(),d.data());d.is("a")&&c.preventDefault(),f.one("show.bs.modal",function(a){a.isDefaultPrevented()||f.one("hidden.bs.modal",function(){d.is(":visible")&&d.trigger("focus")})}),b.call(f,g,this)})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.tooltip"),f="object"==typeof b&&b;!e&&/destroy|hide/.test(b)||(e||d.data("bs.tooltip",e=new c(this,f)),"string"==typeof b&&e[b]())})}var c=function(a,b){this.type=null,this.options=null,this.enabled=null,this.timeout=null,this.hoverState=null,this.$element=null,this.inState=null,this.init("tooltip",a,b)};c.VERSION="3.3.6",c.TRANSITION_DURATION=150,c.DEFAULTS={animation:!0,placement:"top",selector:!1,template:'<div class="tooltip" role="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>',trigger:"hover focus",title:"",delay:0,html:!1,container:!1,viewport:{selector:"body",padding:0}},c.prototype.init=function(b,c,d){if(this.enabled=!0,this.type=b,this.$element=a(c),this.options=this.getOptions(d),this.$viewport=this.options.viewport&&a(a.isFunction(this.options.viewport)?this.options.viewport.call(this,this.$element):this.options.viewport.selector||this.options.viewport),this.inState={click:!1,hover:!1,focus:!1},this.$element[0]instanceof document.constructor&&!this.options.selector)throw new Error("`selector` option must be specified when initializing "+this.type+" on the window.document object!");for(var e=this.options.trigger.split(" "),f=e.length;f--;){var g=e[f];if("click"==g)this.$element.on("click."+this.type,this.options.selector,a.proxy(this.toggle,this));else if("manual"!=g){var h="hover"==g?"mouseenter":"focusin",i="hover"==g?"mouseleave":"focusout";this.$element.on(h+"."+this.type,this.options.selector,a.proxy(this.enter,this)),this.$element.on(i+"."+this.type,this.options.selector,a.proxy(this.leave,this))}}this.options.selector?this._options=a.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.getOptions=function(b){return b=a.extend({},this.getDefaults(),this.$element.data(),b),b.delay&&"number"==typeof b.delay&&(b.delay={show:b.delay,hide:b.delay}),b},c.prototype.getDelegateOptions=function(){var b={},c=this.getDefaults();return this._options&&a.each(this._options,function(a,d){c[a]!=d&&(b[a]=d)}),b},c.prototype.enter=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusin"==b.type?"focus":"hover"]=!0),c.tip().hasClass("in")||"in"==c.hoverState?void(c.hoverState="in"):(clearTimeout(c.timeout),c.hoverState="in",c.options.delay&&c.options.delay.show?void(c.timeout=setTimeout(function(){"in"==c.hoverState&&c.show()},c.options.delay.show)):c.show())},c.prototype.isInStateTrue=function(){for(var a in this.inState)if(this.inState[a])return!0;return!1},c.prototype.leave=function(b){var c=b instanceof this.constructor?b:a(b.currentTarget).data("bs."+this.type);return c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c)),b instanceof a.Event&&(c.inState["focusout"==b.type?"focus":"hover"]=!1),c.isInStateTrue()?void 0:(clearTimeout(c.timeout),c.hoverState="out",c.options.delay&&c.options.delay.hide?void(c.timeout=setTimeout(function(){"out"==c.hoverState&&c.hide()},c.options.delay.hide)):c.hide())},c.prototype.show=function(){var b=a.Event("show.bs."+this.type);if(this.hasContent()&&this.enabled){this.$element.trigger(b);var d=a.contains(this.$element[0].ownerDocument.documentElement,this.$element[0]);if(b.isDefaultPrevented()||!d)return;var e=this,f=this.tip(),g=this.getUID(this.type);this.setContent(),f.attr("id",g),this.$element.attr("aria-describedby",g),this.options.animation&&f.addClass("fade");var h="function"==typeof this.options.placement?this.options.placement.call(this,f[0],this.$element[0]):this.options.placement,i=/\s?auto?\s?/i,j=i.test(h);j&&(h=h.replace(i,"")||"top"),f.detach().css({top:0,left:0,display:"block"}).addClass(h).data("bs."+this.type,this),this.options.container?f.appendTo(this.options.container):f.insertAfter(this.$element),this.$element.trigger("inserted.bs."+this.type);var k=this.getPosition(),l=f[0].offsetWidth,m=f[0].offsetHeight;if(j){var n=h,o=this.getPosition(this.$viewport);h="bottom"==h&&k.bottom+m>o.bottom?"top":"top"==h&&k.top-m<o.top?"bottom":"right"==h&&k.right+l>o.width?"left":"left"==h&&k.left-l<o.left?"right":h,f.removeClass(n).addClass(h)}var p=this.getCalculatedOffset(h,k,l,m);this.applyPlacement(p,h);var q=function(){var a=e.hoverState;e.$element.trigger("shown.bs."+e.type),e.hoverState=null,"out"==a&&e.leave(e)};a.support.transition&&this.$tip.hasClass("fade")?f.one("bsTransitionEnd",q).emulateTransitionEnd(c.TRANSITION_DURATION):q()}},c.prototype.applyPlacement=function(b,c){var d=this.tip(),e=d[0].offsetWidth,f=d[0].offsetHeight,g=parseInt(d.css("margin-top"),10),h=parseInt(d.css("margin-left"),10);isNaN(g)&&(g=0),isNaN(h)&&(h=0),b.top+=g,b.left+=h,a.offset.setOffset(d[0],a.extend({using:function(a){d.css({top:Math.round(a.top),left:Math.round(a.left)})}},b),0),d.addClass("in");var i=d[0].offsetWidth,j=d[0].offsetHeight;"top"==c&&j!=f&&(b.top=b.top+f-j);var k=this.getViewportAdjustedDelta(c,b,i,j);k.left?b.left+=k.left:b.top+=k.top;var l=/top|bottom/.test(c),m=l?2*k.left-e+i:2*k.top-f+j,n=l?"offsetWidth":"offsetHeight";d.offset(b),this.replaceArrow(m,d[0][n],l)},c.prototype.replaceArrow=function(a,b,c){this.arrow().css(c?"left":"top",50*(1-a/b)+"%").css(c?"top":"left","")},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle();a.find(".tooltip-inner")[this.options.html?"html":"text"](b),a.removeClass("fade in top bottom left right")},c.prototype.hide=function(b){function d(){"in"!=e.hoverState&&f.detach(),e.$element.removeAttr("aria-describedby").trigger("hidden.bs."+e.type),b&&b()}var e=this,f=a(this.$tip),g=a.Event("hide.bs."+this.type);return this.$element.trigger(g),g.isDefaultPrevented()?void 0:(f.removeClass("in"),a.support.transition&&f.hasClass("fade")?f.one("bsTransitionEnd",d).emulateTransitionEnd(c.TRANSITION_DURATION):d(),this.hoverState=null,this)},c.prototype.fixTitle=function(){var a=this.$element;(a.attr("title")||"string"!=typeof a.attr("data-original-title"))&&a.attr("data-original-title",a.attr("title")||"").attr("title","")},c.prototype.hasContent=function(){return this.getTitle()},c.prototype.getPosition=function(b){b=b||this.$element;var c=b[0],d="BODY"==c.tagName,e=c.getBoundingClientRect();null==e.width&&(e=a.extend({},e,{width:e.right-e.left,height:e.bottom-e.top}));var f=d?{top:0,left:0}:b.offset(),g={scroll:d?document.documentElement.scrollTop||document.body.scrollTop:b.scrollTop()},h=d?{width:a(window).width(),height:a(window).height()}:null;return a.extend({},e,g,h,f)},c.prototype.getCalculatedOffset=function(a,b,c,d){return"bottom"==a?{top:b.top+b.height,left:b.left+b.width/2-c/2}:"top"==a?{top:b.top-d,left:b.left+b.width/2-c/2}:"left"==a?{top:b.top+b.height/2-d/2,left:b.left-c}:{top:b.top+b.height/2-d/2,left:b.left+b.width}},c.prototype.getViewportAdjustedDelta=function(a,b,c,d){var e={top:0,left:0};if(!this.$viewport)return e;var f=this.options.viewport&&this.options.viewport.padding||0,g=this.getPosition(this.$viewport);if(/right|left/.test(a)){var h=b.top-f-g.scroll,i=b.top+f-g.scroll+d;h<g.top?e.top=g.top-h:i>g.top+g.height&&(e.top=g.top+g.height-i)}else{var j=b.left-f,k=b.left+f+c;j<g.left?e.left=g.left-j:k>g.right&&(e.left=g.left+g.width-k)}return e},c.prototype.getTitle=function(){var a,b=this.$element,c=this.options;return a=b.attr("data-original-title")||("function"==typeof c.title?c.title.call(b[0]):c.title)},c.prototype.getUID=function(a){do a+=~~(1e6*Math.random());while(document.getElementById(a));return a},c.prototype.tip=function(){if(!this.$tip&&(this.$tip=a(this.options.template),1!=this.$tip.length))throw new Error(this.type+" `template` option must consist of exactly 1 top-level element!");return this.$tip},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},c.prototype.enable=function(){this.enabled=!0},c.prototype.disable=function(){this.enabled=!1},c.prototype.toggleEnabled=function(){this.enabled=!this.enabled},c.prototype.toggle=function(b){var c=this;b&&(c=a(b.currentTarget).data("bs."+this.type),c||(c=new this.constructor(b.currentTarget,this.getDelegateOptions()),a(b.currentTarget).data("bs."+this.type,c))),b?(c.inState.click=!c.inState.click,c.isInStateTrue()?c.enter(c):c.leave(c)):c.tip().hasClass("in")?c.leave(c):c.enter(c)},c.prototype.destroy=function(){var a=this;clearTimeout(this.timeout),this.hide(function(){a.$element.off("."+a.type).removeData("bs."+a.type),a.$tip&&a.$tip.detach(),a.$tip=null,a.$arrow=null,a.$viewport=null})};var d=a.fn.tooltip;a.fn.tooltip=b,a.fn.tooltip.Constructor=c,a.fn.tooltip.noConflict=function(){return a.fn.tooltip=d,this}}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.popover"),f="object"==typeof b&&b;!e&&/destroy|hide/.test(b)||(e||d.data("bs.popover",e=new c(this,f)),"string"==typeof b&&e[b]())})}var c=function(a,b){this.init("popover",a,b)};if(!a.fn.tooltip)throw new Error("Popover requires tooltip.js");c.VERSION="3.3.6",c.DEFAULTS=a.extend({},a.fn.tooltip.Constructor.DEFAULTS,{placement:"right",trigger:"click",content:"",template:'<div class="popover" role="tooltip"><div class="arrow"></div><h3 class="popover-title"></h3><div class="popover-content"></div></div>'}),c.prototype=a.extend({},a.fn.tooltip.Constructor.prototype),c.prototype.constructor=c,c.prototype.getDefaults=function(){return c.DEFAULTS},c.prototype.setContent=function(){var a=this.tip(),b=this.getTitle(),c=this.getContent();a.find(".popover-title")[this.options.html?"html":"text"](b),a.find(".popover-content").children().detach().end()[this.options.html?"string"==typeof c?"html":"append":"text"](c),a.removeClass("fade top bottom left right in"),a.find(".popover-title").html()||a.find(".popover-title").hide()},c.prototype.hasContent=function(){return this.getTitle()||this.getContent()},c.prototype.getContent=function(){var a=this.$element,b=this.options;return a.attr("data-content")||("function"==typeof b.content?b.content.call(a[0]):b.content)},c.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")};var d=a.fn.popover;a.fn.popover=b,a.fn.popover.Constructor=c,a.fn.popover.noConflict=function(){return a.fn.popover=d,this}}(jQuery),+function(a){"use strict";function b(c,d){this.$body=a(document.body),this.$scrollElement=a(a(c).is(document.body)?window:c),this.options=a.extend({},b.DEFAULTS,d),this.selector=(this.options.target||"")+" .nav li > a",this.offsets=[],this.targets=[],this.activeTarget=null,this.scrollHeight=0,this.$scrollElement.on("scroll.bs.scrollspy",a.proxy(this.process,this)),this.refresh(),this.process()}function c(c){return this.each(function(){var d=a(this),e=d.data("bs.scrollspy"),f="object"==typeof c&&c;e||d.data("bs.scrollspy",e=new b(this,f)),"string"==typeof c&&e[c]()})}b.VERSION="3.3.6",b.DEFAULTS={offset:10},b.prototype.getScrollHeight=function(){return this.$scrollElement[0].scrollHeight||Math.max(this.$body[0].scrollHeight,document.documentElement.scrollHeight)},b.prototype.refresh=function(){var b=this,c="offset",d=0;this.offsets=[],this.targets=[],this.scrollHeight=this.getScrollHeight(),a.isWindow(this.$scrollElement[0])||(c="position",d=this.$scrollElement.scrollTop()),this.$body.find(this.selector).map(function(){var b=a(this),e=b.data("target")||b.attr("href"),f=/^#./.test(e)&&a(e);return f&&f.length&&f.is(":visible")&&[[f[c]().top+d,e]]||null}).sort(function(a,b){return a[0]-b[0]}).each(function(){b.offsets.push(this[0]),b.targets.push(this[1])})},b.prototype.process=function(){var a,b=this.$scrollElement.scrollTop()+this.options.offset,c=this.getScrollHeight(),d=this.options.offset+c-this.$scrollElement.height(),e=this.offsets,f=this.targets,g=this.activeTarget;if(this.scrollHeight!=c&&this.refresh(),b>=d)return g!=(a=f[f.length-1])&&this.activate(a);if(g&&b<e[0])return this.activeTarget=null,this.clear();for(a=e.length;a--;)g!=f[a]&&b>=e[a]&&(void 0===e[a+1]||b<e[a+1])&&this.activate(f[a])},b.prototype.activate=function(b){this.activeTarget=b,this.clear();var c=this.selector+'[data-target="'+b+'"],'+this.selector+'[href="'+b+'"]',d=a(c).parents("li").addClass("active");d.parent(".dropdown-menu").length&&(d=d.closest("li.dropdown").addClass("active")),
-d.trigger("activate.bs.scrollspy")},b.prototype.clear=function(){a(this.selector).parentsUntil(this.options.target,".active").removeClass("active")};var d=a.fn.scrollspy;a.fn.scrollspy=c,a.fn.scrollspy.Constructor=b,a.fn.scrollspy.noConflict=function(){return a.fn.scrollspy=d,this},a(window).on("load.bs.scrollspy.data-api",function(){a('[data-spy="scroll"]').each(function(){var b=a(this);c.call(b,b.data())})})}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.tab");e||d.data("bs.tab",e=new c(this)),"string"==typeof b&&e[b]()})}var c=function(b){this.element=a(b)};c.VERSION="3.3.6",c.TRANSITION_DURATION=150,c.prototype.show=function(){var b=this.element,c=b.closest("ul:not(.dropdown-menu)"),d=b.data("target");if(d||(d=b.attr("href"),d=d&&d.replace(/.*(?=#[^\s]*$)/,"")),!b.parent("li").hasClass("active")){var e=c.find(".active:last a"),f=a.Event("hide.bs.tab",{relatedTarget:b[0]}),g=a.Event("show.bs.tab",{relatedTarget:e[0]});if(e.trigger(f),b.trigger(g),!g.isDefaultPrevented()&&!f.isDefaultPrevented()){var h=a(d);this.activate(b.closest("li"),c),this.activate(h,h.parent(),function(){e.trigger({type:"hidden.bs.tab",relatedTarget:b[0]}),b.trigger({type:"shown.bs.tab",relatedTarget:e[0]})})}}},c.prototype.activate=function(b,d,e){function f(){g.removeClass("active").find("> .dropdown-menu > .active").removeClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!1),b.addClass("active").find('[data-toggle="tab"]').attr("aria-expanded",!0),h?(b[0].offsetWidth,b.addClass("in")):b.removeClass("fade"),b.parent(".dropdown-menu").length&&b.closest("li.dropdown").addClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!0),e&&e()}var g=d.find("> .active"),h=e&&a.support.transition&&(g.length&&g.hasClass("fade")||!!d.find("> .fade").length);g.length&&h?g.one("bsTransitionEnd",f).emulateTransitionEnd(c.TRANSITION_DURATION):f(),g.removeClass("in")};var d=a.fn.tab;a.fn.tab=b,a.fn.tab.Constructor=c,a.fn.tab.noConflict=function(){return a.fn.tab=d,this};var e=function(c){c.preventDefault(),b.call(a(this),"show")};a(document).on("click.bs.tab.data-api",'[data-toggle="tab"]',e).on("click.bs.tab.data-api",'[data-toggle="pill"]',e)}(jQuery),+function(a){"use strict";function b(b){return this.each(function(){var d=a(this),e=d.data("bs.affix"),f="object"==typeof b&&b;e||d.data("bs.affix",e=new c(this,f)),"string"==typeof b&&e[b]()})}var c=function(b,d){this.options=a.extend({},c.DEFAULTS,d),this.$target=a(this.options.target).on("scroll.bs.affix.data-api",a.proxy(this.checkPosition,this)).on("click.bs.affix.data-api",a.proxy(this.checkPositionWithEventLoop,this)),this.$element=a(b),this.affixed=null,this.unpin=null,this.pinnedOffset=null,this.checkPosition()};c.VERSION="3.3.6",c.RESET="affix affix-top affix-bottom",c.DEFAULTS={offset:0,target:window},c.prototype.getState=function(a,b,c,d){var e=this.$target.scrollTop(),f=this.$element.offset(),g=this.$target.height();if(null!=c&&"top"==this.affixed)return c>e?"top":!1;if("bottom"==this.affixed)return null!=c?e+this.unpin<=f.top?!1:"bottom":a-d>=e+g?!1:"bottom";var h=null==this.affixed,i=h?e:f.top,j=h?g:b;return null!=c&&c>=e?"top":null!=d&&i+j>=a-d?"bottom":!1},c.prototype.getPinnedOffset=function(){if(this.pinnedOffset)return this.pinnedOffset;this.$element.removeClass(c.RESET).addClass("affix");var a=this.$target.scrollTop(),b=this.$element.offset();return this.pinnedOffset=b.top-a},c.prototype.checkPositionWithEventLoop=function(){setTimeout(a.proxy(this.checkPosition,this),1)},c.prototype.checkPosition=function(){if(this.$element.is(":visible")){var b=this.$element.height(),d=this.options.offset,e=d.top,f=d.bottom,g=Math.max(a(document).height(),a(document.body).height());"object"!=typeof d&&(f=e=d),"function"==typeof e&&(e=d.top(this.$element)),"function"==typeof f&&(f=d.bottom(this.$element));var h=this.getState(g,b,e,f);if(this.affixed!=h){null!=this.unpin&&this.$element.css("top","");var i="affix"+(h?"-"+h:""),j=a.Event(i+".bs.affix");if(this.$element.trigger(j),j.isDefaultPrevented())return;this.affixed=h,this.unpin="bottom"==h?this.getPinnedOffset():null,this.$element.removeClass(c.RESET).addClass(i).trigger(i.replace("affix","affixed")+".bs.affix")}"bottom"==h&&this.$element.offset({top:g-b-f})}};var d=a.fn.affix;a.fn.affix=b,a.fn.affix.Constructor=c,a.fn.affix.noConflict=function(){return a.fn.affix=d,this},a(window).on("load",function(){a('[data-spy="affix"]').each(function(){var c=a(this),d=c.data();d.offset=d.offset||{},null!=d.offsetBottom&&(d.offset.bottom=d.offsetBottom),null!=d.offsetTop&&(d.offset.top=d.offsetTop),b.call(c,d)})})}(jQuery);
\ No newline at end of file
diff --git a/bitbake/lib/toaster/toastergui/static/js/jquery-3.7.1.min.js b/bitbake/lib/toaster/toastergui/static/js/jquery-3.7.1.min.js
new file mode 100644
index 0000000000..7f37b5d991
--- /dev/null
+++ b/bitbake/lib/toaster/toastergui/static/js/jquery-3.7.1.min.js
@@ -0,0 +1,2 @@
+/*! jQuery v3.7.1 | (c) OpenJS Foundation and other contributors | jquery.org/license */
+!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(ie,e){"use strict";var oe=[],r=Object.getPrototypeOf,ae=oe.slice,g=oe.flat?function(e){return oe.flat.call(e)}:function(e){return oe.concat.apply([],e)},s=oe.push,se=oe.indexOf,n={},i=n.toString,ue=n.hasOwnProperty,o=ue.toString,a=o.call(Object),le={},v=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},y=function(e){return null!=e&&e===e.window},C=ie.document,u={type:!0,src:!0,nonce:!0,noModule:!0};function m(e,t,n){var r,i,o=(n=n||C).createElement("script");if(o.text=e,t)for(r in u)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function x(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[i.call(e)]||"object":typeof e}var t="3.7.1",l=/HTML$/i,ce=function(e,t){return new ce.fn.init(e,t)};function c(e){var t=!!e&&"length"in e&&e.length,n=x(e);return!v(e)&&!y(e)&&("array"===n||0===t||"number"==typeof t&&0<t&&t-1 in e)}function fe(e,t){return e.nodeName&&e.nodeName.toLowerCase()===t.toLowerCase()}ce.fn=ce.prototype={jquery:t,constructor:ce,length:0,toArray:function(){return ae.call(this)},get:function(e){return null==e?ae.call(this):e<0?this[e+this.length]:this[e]},pushStack:function(e){var t=ce.merge(this.constructor(),e);return t.prevObject=this,t},each:function(e){return ce.each(this,e)},map:function(n){return this.pushStack(ce.map(this,function(e,t){return n.call(e,t,e)}))},slice:function(){return this.pushStack(ae.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},even:function(){return this.pushStack(ce.grep(this,function(e,t){return(t+1)%2}))},odd:function(){return this.pushStack(ce.grep(this,function(e,t){return t%2}))},eq:function(e){var t=this.length,n=+e+(e<0?t:0);return this.pushStack(0<=n&&n<t?[this[n]]:[])},end:function(){return this.prevObject||this.constructor()},push:s,sort:oe.sort,splice:oe.splice},ce.extend=ce.fn.extend=function(){var e,t,n,r,i,o,a=arguments[0]||{},s=1,u=arguments.length,l=!1;for("boolean"==typeof a&&(l=a,a=arguments[s]||{},s++),"object"==typeof a||v(a)||(a={}),s===u&&(a=this,s--);s<u;s++)if(null!=(e=arguments[s]))for(t in e)r=e[t],"__proto__"!==t&&a!==r&&(l&&r&&(ce.isPlainObject(r)||(i=Array.isArray(r)))?(n=a[t],o=i&&!Array.isArray(n)?[]:i||ce.isPlainObject(n)?n:{},i=!1,a[t]=ce.extend(l,o,r)):void 0!==r&&(a[t]=r));return a},ce.extend({expando:"jQuery"+(t+Math.random()).replace(/\D/g,""),isReady:!0,error:function(e){throw new Error(e)},noop:function(){},isPlainObject:function(e){var t,n;return!(!e||"[object Object]"!==i.call(e))&&(!(t=r(e))||"function"==typeof(n=ue.call(t,"constructor")&&t.constructor)&&o.call(n)===a)},isEmptyObject:function(e){var t;for(t in e)return!1;return!0},globalEval:function(e,t,n){m(e,{nonce:t&&t.nonce},n)},each:function(e,t){var n,r=0;if(c(e)){for(n=e.length;r<n;r++)if(!1===t.call(e[r],r,e[r]))break}else for(r in e)if(!1===t.call(e[r],r,e[r]))break;return e},text:function(e){var t,n="",r=0,i=e.nodeType;if(!i)while(t=e[r++])n+=ce.text(t);return 1===i||11===i?e.textContent:9===i?e.documentElement.textContent:3===i||4===i?e.nodeValue:n},makeArray:function(e,t){var n=t||[];return null!=e&&(c(Object(e))?ce.merge(n,"string"==typeof e?[e]:e):s.call(n,e)),n},inArray:function(e,t,n){return null==t?-1:se.call(t,e,n)},isXMLDoc:function(e){var t=e&&e.namespaceURI,n=e&&(e.ownerDocument||e).documentElement;return!l.test(t||n&&n.nodeName||"HTML")},merge:function(e,t){for(var n=+t.length,r=0,i=e.length;r<n;r++)e[i++]=t[r];return e.length=i,e},grep:function(e,t,n){for(var r=[],i=0,o=e.length,a=!n;i<o;i++)!t(e[i],i)!==a&&r.push(e[i]);return r},map:function(e,t,n){var r,i,o=0,a=[];if(c(e))for(r=e.length;o<r;o++)null!=(i=t(e[o],o,n))&&a.push(i);else for(o in e)null!=(i=t(e[o],o,n))&&a.push(i);return g(a)},guid:1,support:le}),"function"==typeof Symbol&&(ce.fn[Symbol.iterator]=oe[Symbol.iterator]),ce.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(e,t){n["[object "+t+"]"]=t.toLowerCase()});var pe=oe.pop,de=oe.sort,he=oe.splice,ge="[\\x20\\t\\r\\n\\f]",ve=new RegExp("^"+ge+"+|((?:^|[^\\\\])(?:\\\\.)*)"+ge+"+$","g");ce.contains=function(e,t){var n=t&&t.parentNode;return e===n||!(!n||1!==n.nodeType||!(e.contains?e.contains(n):e.compareDocumentPosition&&16&e.compareDocumentPosition(n)))};var f=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\x80-\uFFFF\w-]/g;function p(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e}ce.escapeSelector=function(e){return(e+"").replace(f,p)};var ye=C,me=s;!function(){var e,b,w,o,a,T,r,C,d,i,k=me,S=ce.expando,E=0,n=0,s=W(),c=W(),u=W(),h=W(),l=function(e,t){return e===t&&(a=!0),0},f="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",t="(?:\\\\[\\da-fA-F]{1,6}"+ge+"?|\\\\[^\\r\\n\\f]|[\\w-]|[^\0-\\x7f])+",p="\\["+ge+"*("+t+")(?:"+ge+"*([*^$|!~]?=)"+ge+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+t+"))|)"+ge+"*\\]",g=":("+t+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+p+")*)|.*)\\)|)",v=new RegExp(ge+"+","g"),y=new RegExp("^"+ge+"*,"+ge+"*"),m=new RegExp("^"+ge+"*([>+~]|"+ge+")"+ge+"*"),x=new RegExp(ge+"|>"),j=new RegExp(g),A=new RegExp("^"+t+"$"),D={ID:new RegExp("^#("+t+")"),CLASS:new RegExp("^\\.("+t+")"),TAG:new RegExp("^("+t+"|[*])"),ATTR:new RegExp("^"+p),PSEUDO:new RegExp("^"+g),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+ge+"*(even|odd|(([+-]|)(\\d*)n|)"+ge+"*(?:([+-]|)"+ge+"*(\\d+)|))"+ge+"*\\)|)","i"),bool:new RegExp("^(?:"+f+")$","i"),needsContext:new RegExp("^"+ge+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+ge+"*((?:-\\d)?\\d*)"+ge+"*\\)|)(?=[^-]|$)","i")},N=/^(?:input|select|textarea|button)$/i,q=/^h\d$/i,L=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,H=/[+~]/,O=new RegExp("\\\\[\\da-fA-F]{1,6}"+ge+"?|\\\\([^\\r\\n\\f])","g"),P=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},M=function(){V()},R=J(function(e){return!0===e.disabled&&fe(e,"fieldset")},{dir:"parentNode",next:"legend"});try{k.apply(oe=ae.call(ye.childNodes),ye.childNodes),oe[ye.childNodes.length].nodeType}catch(e){k={apply:function(e,t){me.apply(e,ae.call(t))},call:function(e){me.apply(e,ae.call(arguments,1))}}}function I(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(V(e),e=e||T,C)){if(11!==p&&(u=L.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return k.call(n,a),n}else if(f&&(a=f.getElementById(i))&&I.contains(e,a)&&a.id===i)return k.call(n,a),n}else{if(u[2])return k.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&e.getElementsByClassName)return k.apply(n,e.getElementsByClassName(i)),n}if(!(h[t+" "]||d&&d.test(t))){if(c=t,f=e,1===p&&(x.test(t)||m.test(t))){(f=H.test(t)&&U(e.parentNode)||e)==e&&le.scope||((s=e.getAttribute("id"))?s=ce.escapeSelector(s):e.setAttribute("id",s=S)),o=(l=Y(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+Q(l[o]);c=l.join(",")}try{return k.apply(n,f.querySelectorAll(c)),n}catch(e){h(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return re(t.replace(ve,"$1"),e,n,r)}function W(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function F(e){return e[S]=!0,e}function $(e){var t=T.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function B(t){return function(e){return fe(e,"input")&&e.type===t}}function _(t){return function(e){return(fe(e,"input")||fe(e,"button"))&&e.type===t}}function z(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&R(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function X(a){return F(function(o){return o=+o,F(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function U(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}function V(e){var t,n=e?e.ownerDocument||e:ye;return n!=T&&9===n.nodeType&&n.documentElement&&(r=(T=n).documentElement,C=!ce.isXMLDoc(T),i=r.matches||r.webkitMatchesSelector||r.msMatchesSelector,r.msMatchesSelector&&ye!=T&&(t=T.defaultView)&&t.top!==t&&t.addEventListener("unload",M),le.getById=$(function(e){return r.appendChild(e).id=ce.expando,!T.getElementsByName||!T.getElementsByName(ce.expando).length}),le.disconnectedMatch=$(function(e){return i.call(e,"*")}),le.scope=$(function(){return T.querySelectorAll(":scope")}),le.cssHas=$(function(){try{return T.querySelector(":has(*,:jqfake)"),!1}catch(e){return!0}}),le.getById?(b.filter.ID=function(e){var t=e.replace(O,P);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&C){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(O,P);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&C){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):t.querySelectorAll(e)},b.find.CLASS=function(e,t){if("undefined"!=typeof t.getElementsByClassName&&C)return t.getElementsByClassName(e)},d=[],$(function(e){var t;r.appendChild(e).innerHTML="<a id='"+S+"' href='' disabled='disabled'></a><select id='"+S+"-\r\\' disabled='disabled'><option selected=''></option></select>",e.querySelectorAll("[selected]").length||d.push("\\["+ge+"*(?:value|"+f+")"),e.querySelectorAll("[id~="+S+"-]").length||d.push("~="),e.querySelectorAll("a#"+S+"+*").length||d.push(".#.+[+~]"),e.querySelectorAll(":checked").length||d.push(":checked"),(t=T.createElement("input")).setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),r.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&d.push(":enabled",":disabled"),(t=T.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||d.push("\\["+ge+"*name"+ge+"*="+ge+"*(?:''|\"\")")}),le.cssHas||d.push(":has"),d=d.length&&new RegExp(d.join("|")),l=function(e,t){if(e===t)return a=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!le.sortDetached&&t.compareDocumentPosition(e)===n?e===T||e.ownerDocument==ye&&I.contains(ye,e)?-1:t===T||t.ownerDocument==ye&&I.contains(ye,t)?1:o?se.call(o,e)-se.call(o,t):0:4&n?-1:1)}),T}for(e in I.matches=function(e,t){return I(e,null,null,t)},I.matchesSelector=function(e,t){if(V(e),C&&!h[t+" "]&&(!d||!d.test(t)))try{var n=i.call(e,t);if(n||le.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){h(t,!0)}return 0<I(t,T,null,[e]).length},I.contains=function(e,t){return(e.ownerDocument||e)!=T&&V(e),ce.contains(e,t)},I.attr=function(e,t){(e.ownerDocument||e)!=T&&V(e);var n=b.attrHandle[t.toLowerCase()],r=n&&ue.call(b.attrHandle,t.toLowerCase())?n(e,t,!C):void 0;return void 0!==r?r:e.getAttribute(t)},I.error=function(e){throw new Error("Syntax error, unrecognized expression: "+e)},ce.uniqueSort=function(e){var t,n=[],r=0,i=0;if(a=!le.sortStable,o=!le.sortStable&&ae.call(e,0),de.call(e,l),a){while(t=e[i++])t===e[i]&&(r=n.push(i));while(r--)he.call(e,n[r],1)}return o=null,e},ce.fn.uniqueSort=function(){return this.pushStack(ce.uniqueSort(ae.apply(this)))},(b=ce.expr={cacheLength:50,createPseudo:F,match:D,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(O,P),e[3]=(e[3]||e[4]||e[5]||"").replace(O,P),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||I.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&I.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return D.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&j.test(n)&&(t=Y(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(O,P).toLowerCase();return"*"===e?function(){return!0}:function(e){return fe(e,t)}},CLASS:function(e){var t=s[e+" "];return t||(t=new RegExp("(^|"+ge+")"+e+"("+ge+"|$)"))&&s(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=I.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1<t.indexOf(i):"$="===r?i&&t.slice(-i.length)===i:"~="===r?-1<(" "+t.replace(v," ")+" ").indexOf(i):"|="===r&&(t===i||t.slice(0,i.length+1)===i+"-"))}},CHILD:function(d,e,t,h,g){var v="nth"!==d.slice(0,3),y="last"!==d.slice(-4),m="of-type"===e;return 1===h&&0===g?function(e){return!!e.parentNode}:function(e,t,n){var r,i,o,a,s,u=v!==y?"nextSibling":"previousSibling",l=e.parentNode,c=m&&e.nodeName.toLowerCase(),f=!n&&!m,p=!1;if(l){if(v){while(u){o=e;while(o=o[u])if(m?fe(o,c):1===o.nodeType)return!1;s=u="only"===d&&!s&&"nextSibling"}return!0}if(s=[y?l.firstChild:l.lastChild],y&&f){p=(a=(r=(i=l[S]||(l[S]={}))[d]||[])[0]===E&&r[1])&&r[2],o=a&&l.childNodes[a];while(o=++a&&o&&o[u]||(p=a=0)||s.pop())if(1===o.nodeType&&++p&&o===e){i[d]=[E,a,p];break}}else if(f&&(p=a=(r=(i=e[S]||(e[S]={}))[d]||[])[0]===E&&r[1]),!1===p)while(o=++a&&o&&o[u]||(p=a=0)||s.pop())if((m?fe(o,c):1===o.nodeType)&&++p&&(f&&((i=o[S]||(o[S]={}))[d]=[E,p]),o===e))break;return(p-=g)===h||p%h==0&&0<=p/h}}},PSEUDO:function(e,o){var t,a=b.pseudos[e]||b.setFilters[e.toLowerCase()]||I.error("unsupported pseudo: "+e);return a[S]?a(o):1<a.length?(t=[e,e,"",o],b.setFilters.hasOwnProperty(e.toLowerCase())?F(function(e,t){var n,r=a(e,o),i=r.length;while(i--)e[n=se.call(e,r[i])]=!(t[n]=r[i])}):function(e){return a(e,0,t)}):a}},pseudos:{not:F(function(e){var r=[],i=[],s=ne(e.replace(ve,"$1"));return s[S]?F(function(e,t,n,r){var i,o=s(e,null,r,[]),a=e.length;while(a--)(i=o[a])&&(e[a]=!(t[a]=i))}):function(e,t,n){return r[0]=e,s(r,null,n,i),r[0]=null,!i.pop()}}),has:F(function(t){return function(e){return 0<I(t,e).length}}),contains:F(function(t){return t=t.replace(O,P),function(e){return-1<(e.textContent||ce.text(e)).indexOf(t)}}),lang:F(function(n){return A.test(n||"")||I.error("unsupported lang: "+n),n=n.replace(O,P).toLowerCase(),function(e){var t;do{if(t=C?e.lang:e.getAttribute("xml:lang")||e.getAttribute("lang"))return(t=t.toLowerCase())===n||0===t.indexOf(n+"-")}while((e=e.parentNode)&&1===e.nodeType);return!1}}),target:function(e){var t=ie.location&&ie.location.hash;return t&&t.slice(1)===e.id},root:function(e){return e===r},focus:function(e){return e===function(){try{return T.activeElement}catch(e){}}()&&T.hasFocus()&&!!(e.type||e.href||~e.tabIndex)},enabled:z(!1),disabled:z(!0),checked:function(e){return fe(e,"input")&&!!e.checked||fe(e,"option")&&!!e.selected},selected:function(e){return e.parentNode&&e.parentNode.selectedIndex,!0===e.selected},empty:function(e){for(e=e.firstChild;e;e=e.nextSibling)if(e.nodeType<6)return!1;return!0},parent:function(e){return!b.pseudos.empty(e)},header:function(e){return q.test(e.nodeName)},input:function(e){return N.test(e.nodeName)},button:function(e){return fe(e,"input")&&"button"===e.type||fe(e,"button")},text:function(e){var t;return fe(e,"input")&&"text"===e.type&&(null==(t=e.getAttribute("type"))||"text"===t.toLowerCase())},first:X(function(){return[0]}),last:X(function(e,t){return[t-1]}),eq:X(function(e,t,n){return[n<0?n+t:n]}),even:X(function(e,t){for(var n=0;n<t;n+=2)e.push(n);return e}),odd:X(function(e,t){for(var n=1;n<t;n+=2)e.push(n);return e}),lt:X(function(e,t,n){var r;for(r=n<0?n+t:t<n?t:n;0<=--r;)e.push(r);return e}),gt:X(function(e,t,n){for(var r=n<0?n+t:n;++r<t;)e.push(r);return e})}}).pseudos.nth=b.pseudos.eq,{radio:!0,checkbox:!0,file:!0,password:!0,image:!0})b.pseudos[e]=B(e);for(e in{submit:!0,reset:!0})b.pseudos[e]=_(e);function G(){}function Y(e,t){var n,r,i,o,a,s,u,l=c[e+" "];if(l)return t?0:l.slice(0);a=e,s=[],u=b.preFilter;while(a){for(o in n&&!(r=y.exec(a))||(r&&(a=a.slice(r[0].length)||a),s.push(i=[])),n=!1,(r=m.exec(a))&&(n=r.shift(),i.push({value:n,type:r[0].replace(ve," ")}),a=a.slice(n.length)),b.filter)!(r=D[o].exec(a))||u[o]&&!(r=u[o](r))||(n=r.shift(),i.push({value:n,type:o,matches:r}),a=a.slice(n.length));if(!n)break}return t?a.length:a?I.error(e):c(e,s).slice(0)}function Q(e){for(var t=0,n=e.length,r="";t<n;t++)r+=e[t].value;return r}function J(a,e,t){var s=e.dir,u=e.next,l=u||s,c=t&&"parentNode"===l,f=n++;return e.first?function(e,t,n){while(e=e[s])if(1===e.nodeType||c)return a(e,t,n);return!1}:function(e,t,n){var r,i,o=[E,f];if(n){while(e=e[s])if((1===e.nodeType||c)&&a(e,t,n))return!0}else while(e=e[s])if(1===e.nodeType||c)if(i=e[S]||(e[S]={}),u&&fe(e,u))e=e[s]||e;else{if((r=i[l])&&r[0]===E&&r[1]===f)return o[2]=r[2];if((i[l]=o)[2]=a(e,t,n))return!0}return!1}}function K(i){return 1<i.length?function(e,t,n){var r=i.length;while(r--)if(!i[r](e,t,n))return!1;return!0}:i[0]}function Z(e,t,n,r,i){for(var o,a=[],s=0,u=e.length,l=null!=t;s<u;s++)(o=e[s])&&(n&&!n(o,r,i)||(a.push(o),l&&t.push(s)));return a}function ee(d,h,g,v,y,e){return v&&!v[S]&&(v=ee(v)),y&&!y[S]&&(y=ee(y,e)),F(function(e,t,n,r){var i,o,a,s,u=[],l=[],c=t.length,f=e||function(e,t,n){for(var r=0,i=t.length;r<i;r++)I(e,t[r],n);return n}(h||"*",n.nodeType?[n]:n,[]),p=!d||!e&&h?f:Z(f,u,d,n,r);if(g?g(p,s=y||(e?d:c||v)?[]:t,n,r):s=p,v){i=Z(s,l),v(i,[],n,r),o=i.length;while(o--)(a=i[o])&&(s[l[o]]=!(p[l[o]]=a))}if(e){if(y||d){if(y){i=[],o=s.length;while(o--)(a=s[o])&&i.push(p[o]=a);y(null,s=[],i,r)}o=s.length;while(o--)(a=s[o])&&-1<(i=y?se.call(e,a):u[o])&&(e[i]=!(t[i]=a))}}else s=Z(s===t?s.splice(c,s.length):s),y?y(null,t,s,r):k.apply(t,s)})}function te(e){for(var i,t,n,r=e.length,o=b.relative[e[0].type],a=o||b.relative[" "],s=o?1:0,u=J(function(e){return e===i},a,!0),l=J(function(e){return-1<se.call(i,e)},a,!0),c=[function(e,t,n){var r=!o&&(n||t!=w)||((i=t).nodeType?u(e,t,n):l(e,t,n));return i=null,r}];s<r;s++)if(t=b.relative[e[s].type])c=[J(K(c),t)];else{if((t=b.filter[e[s].type].apply(null,e[s].matches))[S]){for(n=++s;n<r;n++)if(b.relative[e[n].type])break;return ee(1<s&&K(c),1<s&&Q(e.slice(0,s-1).concat({value:" "===e[s-2].type?"*":""})).replace(ve,"$1"),t,s<n&&te(e.slice(s,n)),n<r&&te(e=e.slice(n)),n<r&&Q(e))}c.push(t)}return K(c)}function ne(e,t){var n,v,y,m,x,r,i=[],o=[],a=u[e+" "];if(!a){t||(t=Y(e)),n=t.length;while(n--)(a=te(t[n]))[S]?i.push(a):o.push(a);(a=u(e,(v=o,m=0<(y=i).length,x=0<v.length,r=function(e,t,n,r,i){var o,a,s,u=0,l="0",c=e&&[],f=[],p=w,d=e||x&&b.find.TAG("*",i),h=E+=null==p?1:Math.random()||.1,g=d.length;for(i&&(w=t==T||t||i);l!==g&&null!=(o=d[l]);l++){if(x&&o){a=0,t||o.ownerDocument==T||(V(o),n=!C);while(s=v[a++])if(s(o,t||T,n)){k.call(r,o);break}i&&(E=h)}m&&((o=!s&&o)&&u--,e&&c.push(o))}if(u+=l,m&&l!==u){a=0;while(s=y[a++])s(c,f,t,n);if(e){if(0<u)while(l--)c[l]||f[l]||(f[l]=pe.call(r));f=Z(f)}k.apply(r,f),i&&!e&&0<f.length&&1<u+y.length&&ce.uniqueSort(r)}return i&&(E=h,w=p),c},m?F(r):r))).selector=e}return a}function re(e,t,n,r){var i,o,a,s,u,l="function"==typeof e&&e,c=!r&&Y(e=l.selector||e);if(n=n||[],1===c.length){if(2<(o=c[0]=c[0].slice(0)).length&&"ID"===(a=o[0]).type&&9===t.nodeType&&C&&b.relative[o[1].type]){if(!(t=(b.find.ID(a.matches[0].replace(O,P),t)||[])[0]))return n;l&&(t=t.parentNode),e=e.slice(o.shift().value.length)}i=D.needsContext.test(e)?0:o.length;while(i--){if(a=o[i],b.relative[s=a.type])break;if((u=b.find[s])&&(r=u(a.matches[0].replace(O,P),H.test(o[0].type)&&U(t.parentNode)||t))){if(o.splice(i,1),!(e=r.length&&Q(o)))return k.apply(n,r),n;break}}}return(l||ne(e,c))(r,t,!C,n,!t||H.test(e)&&U(t.parentNode)||t),n}G.prototype=b.filters=b.pseudos,b.setFilters=new G,le.sortStable=S.split("").sort(l).join("")===S,V(),le.sortDetached=$(function(e){return 1&e.compareDocumentPosition(T.createElement("fieldset"))}),ce.find=I,ce.expr[":"]=ce.expr.pseudos,ce.unique=ce.uniqueSort,I.compile=ne,I.select=re,I.setDocument=V,I.tokenize=Y,I.escape=ce.escapeSelector,I.getText=ce.text,I.isXML=ce.isXMLDoc,I.selectors=ce.expr,I.support=ce.support,I.uniqueSort=ce.uniqueSort}();var d=function(e,t,n){var r=[],i=void 0!==n;while((e=e[t])&&9!==e.nodeType)if(1===e.nodeType){if(i&&ce(e).is(n))break;r.push(e)}return r},h=function(e,t){for(var n=[];e;e=e.nextSibling)1===e.nodeType&&e!==t&&n.push(e);return n},b=ce.expr.match.needsContext,w=/^<([a-z][^\/\0>:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function T(e,n,r){return v(n)?ce.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?ce.grep(e,function(e){return e===n!==r}):"string"!=typeof n?ce.grep(e,function(e){return-1<se.call(n,e)!==r}):ce.filter(n,e,r)}ce.filter=function(e,t,n){var r=t[0];return n&&(e=":not("+e+")"),1===t.length&&1===r.nodeType?ce.find.matchesSelector(r,e)?[r]:[]:ce.find.matches(e,ce.grep(t,function(e){return 1===e.nodeType}))},ce.fn.extend({find:function(e){var t,n,r=this.length,i=this;if("string"!=typeof e)return this.pushStack(ce(e).filter(function(){for(t=0;t<r;t++)if(ce.contains(i[t],this))return!0}));for(n=this.pushStack([]),t=0;t<r;t++)ce.find(e,i[t],n);return 1<r?ce.uniqueSort(n):n},filter:function(e){return this.pushStack(T(this,e||[],!1))},not:function(e){return this.pushStack(T(this,e||[],!0))},is:function(e){return!!T(this,"string"==typeof e&&b.test(e)?ce(e):e||[],!1).length}});var k,S=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]+))$/;(ce.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||k,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:S.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof ce?t[0]:t,ce.merge(this,ce.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:C,!0)),w.test(r[1])&&ce.isPlainObject(t))for(r in t)v(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=C.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):v(e)?void 0!==n.ready?n.ready(e):e(ce):ce.makeArray(e,this)}).prototype=ce.fn,k=ce(C);var E=/^(?:parents|prev(?:Until|All))/,j={children:!0,contents:!0,next:!0,prev:!0};function A(e,t){while((e=e[t])&&1!==e.nodeType);return e}ce.fn.extend({has:function(e){var t=ce(e,this),n=t.length;return this.filter(function(){for(var e=0;e<n;e++)if(ce.contains(this,t[e]))return!0})},closest:function(e,t){var n,r=0,i=this.length,o=[],a="string"!=typeof e&&ce(e);if(!b.test(e))for(;r<i;r++)for(n=this[r];n&&n!==t;n=n.parentNode)if(n.nodeType<11&&(a?-1<a.index(n):1===n.nodeType&&ce.find.matchesSelector(n,e))){o.push(n);break}return this.pushStack(1<o.length?ce.uniqueSort(o):o)},index:function(e){return e?"string"==typeof e?se.call(ce(e),this[0]):se.call(this,e.jquery?e[0]:e):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(e,t){return this.pushStack(ce.uniqueSort(ce.merge(this.get(),ce(e,t))))},addBack:function(e){return this.add(null==e?this.prevObject:this.prevObject.filter(e))}}),ce.each({parent:function(e){var t=e.parentNode;return t&&11!==t.nodeType?t:null},parents:function(e){return d(e,"parentNode")},parentsUntil:function(e,t,n){return d(e,"parentNode",n)},next:function(e){return A(e,"nextSibling")},prev:function(e){return A(e,"previousSibling")},nextAll:function(e){return d(e,"nextSibling")},prevAll:function(e){return d(e,"previousSibling")},nextUntil:function(e,t,n){return d(e,"nextSibling",n)},prevUntil:function(e,t,n){return d(e,"previousSibling",n)},siblings:function(e){return h((e.parentNode||{}).firstChild,e)},children:function(e){return h(e.firstChild)},contents:function(e){return null!=e.contentDocument&&r(e.contentDocument)?e.contentDocument:(fe(e,"template")&&(e=e.content||e),ce.merge([],e.childNodes))}},function(r,i){ce.fn[r]=function(e,t){var n=ce.map(this,i,e);return"Until"!==r.slice(-5)&&(t=e),t&&"string"==typeof t&&(n=ce.filter(t,n)),1<this.length&&(j[r]||ce.uniqueSort(n),E.test(r)&&n.reverse()),this.pushStack(n)}});var D=/[^\x20\t\r\n\f]+/g;function N(e){return e}function q(e){throw e}function L(e,t,n,r){var i;try{e&&v(i=e.promise)?i.call(e).done(t).fail(n):e&&v(i=e.then)?i.call(e,t,n):t.apply(void 0,[e].slice(r))}catch(e){n.apply(void 0,[e])}}ce.Callbacks=function(r){var e,n;r="string"==typeof r?(e=r,n={},ce.each(e.match(D)||[],function(e,t){n[t]=!0}),n):ce.extend({},r);var i,t,o,a,s=[],u=[],l=-1,c=function(){for(a=a||r.once,o=i=!0;u.length;l=-1){t=u.shift();while(++l<s.length)!1===s[l].apply(t[0],t[1])&&r.stopOnFalse&&(l=s.length,t=!1)}r.memory||(t=!1),i=!1,a&&(s=t?[]:"")},f={add:function(){return s&&(t&&!i&&(l=s.length-1,u.push(t)),function n(e){ce.each(e,function(e,t){v(t)?r.unique&&f.has(t)||s.push(t):t&&t.length&&"string"!==x(t)&&n(t)})}(arguments),t&&!i&&c()),this},remove:function(){return ce.each(arguments,function(e,t){var n;while(-1<(n=ce.inArray(t,s,n)))s.splice(n,1),n<=l&&l--}),this},has:function(e){return e?-1<ce.inArray(e,s):0<s.length},empty:function(){return s&&(s=[]),this},disable:function(){return a=u=[],s=t="",this},disabled:function(){return!s},lock:function(){return a=u=[],t||i||(s=t=""),this},locked:function(){return!!a},fireWith:function(e,t){return a||(t=[e,(t=t||[]).slice?t.slice():t],u.push(t),i||c()),this},fire:function(){return f.fireWith(this,arguments),this},fired:function(){return!!o}};return f},ce.extend({Deferred:function(e){var o=[["notify","progress",ce.Callbacks("memory"),ce.Callbacks("memory"),2],["resolve","done",ce.Callbacks("once memory"),ce.Callbacks("once memory"),0,"resolved"],["reject","fail",ce.Callbacks("once memory"),ce.Callbacks("once memory"),1,"rejected"]],i="pending",a={state:function(){return i},always:function(){return s.done(arguments).fail(arguments),this},"catch":function(e){return a.then(null,e)},pipe:function(){var i=arguments;return ce.Deferred(function(r){ce.each(o,function(e,t){var n=v(i[t[4]])&&i[t[4]];s[t[1]](function(){var e=n&&n.apply(this,arguments);e&&v(e.promise)?e.promise().progress(r.notify).done(r.resolve).fail(r.reject):r[t[0]+"With"](this,n?[e]:arguments)})}),i=null}).promise()},then:function(t,n,r){var u=0;function l(i,o,a,s){return function(){var n=this,r=arguments,e=function(){var e,t;if(!(i<u)){if((e=a.apply(n,r))===o.promise())throw new TypeError("Thenable self-resolution");t=e&&("object"==typeof e||"function"==typeof e)&&e.then,v(t)?s?t.call(e,l(u,o,N,s),l(u,o,q,s)):(u++,t.call(e,l(u,o,N,s),l(u,o,q,s),l(u,o,N,o.notifyWith))):(a!==N&&(n=void 0,r=[e]),(s||o.resolveWith)(n,r))}},t=s?e:function(){try{e()}catch(e){ce.Deferred.exceptionHook&&ce.Deferred.exceptionHook(e,t.error),u<=i+1&&(a!==q&&(n=void 0,r=[e]),o.rejectWith(n,r))}};i?t():(ce.Deferred.getErrorHook?t.error=ce.Deferred.getErrorHook():ce.Deferred.getStackHook&&(t.error=ce.Deferred.getStackHook()),ie.setTimeout(t))}}return ce.Deferred(function(e){o[0][3].add(l(0,e,v(r)?r:N,e.notifyWith)),o[1][3].add(l(0,e,v(t)?t:N)),o[2][3].add(l(0,e,v(n)?n:q))}).promise()},promise:function(e){return null!=e?ce.extend(e,a):a}},s={};return ce.each(o,function(e,t){var n=t[2],r=t[5];a[t[1]]=n.add,r&&n.add(function(){i=r},o[3-e][2].disable,o[3-e][3].disable,o[0][2].lock,o[0][3].lock),n.add(t[3].fire),s[t[0]]=function(){return s[t[0]+"With"](this===s?void 0:this,arguments),this},s[t[0]+"With"]=n.fireWith}),a.promise(s),e&&e.call(s,s),s},when:function(e){var n=arguments.length,t=n,r=Array(t),i=ae.call(arguments),o=ce.Deferred(),a=function(t){return function(e){r[t]=this,i[t]=1<arguments.length?ae.call(arguments):e,--n||o.resolveWith(r,i)}};if(n<=1&&(L(e,o.done(a(t)).resolve,o.reject,!n),"pending"===o.state()||v(i[t]&&i[t].then)))return o.then();while(t--)L(i[t],a(t),o.reject);return o.promise()}});var H=/^(Eval|Internal|Range|Reference|Syntax|Type|URI)Error$/;ce.Deferred.exceptionHook=function(e,t){ie.console&&ie.console.warn&&e&&H.test(e.name)&&ie.console.warn("jQuery.Deferred exception: "+e.message,e.stack,t)},ce.readyException=function(e){ie.setTimeout(function(){throw e})};var O=ce.Deferred();function P(){C.removeEventListener("DOMContentLoaded",P),ie.removeEventListener("load",P),ce.ready()}ce.fn.ready=function(e){return O.then(e)["catch"](function(e){ce.readyException(e)}),this},ce.extend({isReady:!1,readyWait:1,ready:function(e){(!0===e?--ce.readyWait:ce.isReady)||(ce.isReady=!0)!==e&&0<--ce.readyWait||O.resolveWith(C,[ce])}}),ce.ready.then=O.then,"complete"===C.readyState||"loading"!==C.readyState&&!C.documentElement.doScroll?ie.setTimeout(ce.ready):(C.addEventListener("DOMContentLoaded",P),ie.addEventListener("load",P));var M=function(e,t,n,r,i,o,a){var s=0,u=e.length,l=null==n;if("object"===x(n))for(s in i=!0,n)M(e,t,s,n[s],!0,o,a);else if(void 0!==r&&(i=!0,v(r)||(a=!0),l&&(a?(t.call(e,r),t=null):(l=t,t=function(e,t,n){return l.call(ce(e),n)})),t))for(;s<u;s++)t(e[s],n,a?r:r.call(e[s],s,t(e[s],n)));return i?e:l?t.call(e):u?t(e[0],n):o},R=/^-ms-/,I=/-([a-z])/g;function W(e,t){return t.toUpperCase()}function F(e){return e.replace(R,"ms-").replace(I,W)}var $=function(e){return 1===e.nodeType||9===e.nodeType||!+e.nodeType};function B(){this.expando=ce.expando+B.uid++}B.uid=1,B.prototype={cache:function(e){var t=e[this.expando];return t||(t={},$(e)&&(e.nodeType?e[this.expando]=t:Object.defineProperty(e,this.expando,{value:t,configurable:!0}))),t},set:function(e,t,n){var r,i=this.cache(e);if("string"==typeof t)i[F(t)]=n;else for(r in t)i[F(r)]=t[r];return i},get:function(e,t){return void 0===t?this.cache(e):e[this.expando]&&e[this.expando][F(t)]},access:function(e,t,n){return void 0===t||t&&"string"==typeof t&&void 0===n?this.get(e,t):(this.set(e,t,n),void 0!==n?n:t)},remove:function(e,t){var n,r=e[this.expando];if(void 0!==r){if(void 0!==t){n=(t=Array.isArray(t)?t.map(F):(t=F(t))in r?[t]:t.match(D)||[]).length;while(n--)delete r[t[n]]}(void 0===t||ce.isEmptyObject(r))&&(e.nodeType?e[this.expando]=void 0:delete e[this.expando])}},hasData:function(e){var t=e[this.expando];return void 0!==t&&!ce.isEmptyObject(t)}};var _=new B,z=new B,X=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,U=/[A-Z]/g;function V(e,t,n){var r,i;if(void 0===n&&1===e.nodeType)if(r="data-"+t.replace(U,"-$&").toLowerCase(),"string"==typeof(n=e.getAttribute(r))){try{n="true"===(i=n)||"false"!==i&&("null"===i?null:i===+i+""?+i:X.test(i)?JSON.parse(i):i)}catch(e){}z.set(e,t,n)}else n=void 0;return n}ce.extend({hasData:function(e){return z.hasData(e)||_.hasData(e)},data:function(e,t,n){return z.access(e,t,n)},removeData:function(e,t){z.remove(e,t)},_data:function(e,t,n){return _.access(e,t,n)},_removeData:function(e,t){_.remove(e,t)}}),ce.fn.extend({data:function(n,e){var t,r,i,o=this[0],a=o&&o.attributes;if(void 0===n){if(this.length&&(i=z.get(o),1===o.nodeType&&!_.get(o,"hasDataAttrs"))){t=a.length;while(t--)a[t]&&0===(r=a[t].name).indexOf("data-")&&(r=F(r.slice(5)),V(o,r,i[r]));_.set(o,"hasDataAttrs",!0)}return i}return"object"==typeof n?this.each(function(){z.set(this,n)}):M(this,function(e){var t;if(o&&void 0===e)return void 0!==(t=z.get(o,n))?t:void 0!==(t=V(o,n))?t:void 0;this.each(function(){z.set(this,n,e)})},null,e,1<arguments.length,null,!0)},removeData:function(e){return this.each(function(){z.remove(this,e)})}}),ce.extend({queue:function(e,t,n){var r;if(e)return t=(t||"fx")+"queue",r=_.get(e,t),n&&(!r||Array.isArray(n)?r=_.access(e,t,ce.makeArray(n)):r.push(n)),r||[]},dequeue:function(e,t){t=t||"fx";var n=ce.queue(e,t),r=n.length,i=n.shift(),o=ce._queueHooks(e,t);"inprogress"===i&&(i=n.shift(),r--),i&&("fx"===t&&n.unshift("inprogress"),delete o.stop,i.call(e,function(){ce.dequeue(e,t)},o)),!r&&o&&o.empty.fire()},_queueHooks:function(e,t){var n=t+"queueHooks";return _.get(e,n)||_.access(e,n,{empty:ce.Callbacks("once memory").add(function(){_.remove(e,[t+"queue",n])})})}}),ce.fn.extend({queue:function(t,n){var e=2;return"string"!=typeof t&&(n=t,t="fx",e--),arguments.length<e?ce.queue(this[0],t):void 0===n?this:this.each(function(){var e=ce.queue(this,t,n);ce._queueHooks(this,t),"fx"===t&&"inprogress"!==e[0]&&ce.dequeue(this,t)})},dequeue:function(e){return this.each(function(){ce.dequeue(this,e)})},clearQueue:function(e){return this.queue(e||"fx",[])},promise:function(e,t){var n,r=1,i=ce.Deferred(),o=this,a=this.length,s=function(){--r||i.resolveWith(o,[o])};"string"!=typeof e&&(t=e,e=void 0),e=e||"fx";while(a--)(n=_.get(o[a],e+"queueHooks"))&&n.empty&&(r++,n.empty.add(s));return s(),i.promise(t)}});var G=/[+-]?(?:\d*\.|)\d+(?:[eE][+-]?\d+|)/.source,Y=new RegExp("^(?:([+-])=|)("+G+")([a-z%]*)$","i"),Q=["Top","Right","Bottom","Left"],J=C.documentElement,K=function(e){return ce.contains(e.ownerDocument,e)},Z={composed:!0};J.getRootNode&&(K=function(e){return ce.contains(e.ownerDocument,e)||e.getRootNode(Z)===e.ownerDocument});var ee=function(e,t){return"none"===(e=t||e).style.display||""===e.style.display&&K(e)&&"none"===ce.css(e,"display")};function te(e,t,n,r){var i,o,a=20,s=r?function(){return r.cur()}:function(){return ce.css(e,t,"")},u=s(),l=n&&n[3]||(ce.cssNumber[t]?"":"px"),c=e.nodeType&&(ce.cssNumber[t]||"px"!==l&&+u)&&Y.exec(ce.css(e,t));if(c&&c[3]!==l){u/=2,l=l||c[3],c=+u||1;while(a--)ce.style(e,t,c+l),(1-o)*(1-(o=s()/u||.5))<=0&&(a=0),c/=o;c*=2,ce.style(e,t,c+l),n=n||[]}return n&&(c=+c||+u||0,i=n[1]?c+(n[1]+1)*n[2]:+n[2],r&&(r.unit=l,r.start=c,r.end=i)),i}var ne={};function re(e,t){for(var n,r,i,o,a,s,u,l=[],c=0,f=e.length;c<f;c++)(r=e[c]).style&&(n=r.style.display,t?("none"===n&&(l[c]=_.get(r,"display")||null,l[c]||(r.style.display="")),""===r.style.display&&ee(r)&&(l[c]=(u=a=o=void 0,a=(i=r).ownerDocument,s=i.nodeName,(u=ne[s])||(o=a.body.appendChild(a.createElement(s)),u=ce.css(o,"display"),o.parentNode.removeChild(o),"none"===u&&(u="block"),ne[s]=u)))):"none"!==n&&(l[c]="none",_.set(r,"display",n)));for(c=0;c<f;c++)null!=l[c]&&(e[c].style.display=l[c]);return e}ce.fn.extend({show:function(){return re(this,!0)},hide:function(){return re(this)},toggle:function(e){return"boolean"==typeof e?e?this.show():this.hide():this.each(function(){ee(this)?ce(this).show():ce(this).hide()})}});var xe,be,we=/^(?:checkbox|radio)$/i,Te=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i,Ce=/^$|^module$|\/(?:java|ecma)script/i;xe=C.createDocumentFragment().appendChild(C.createElement("div")),(be=C.createElement("input")).setAttribute("type","radio"),be.setAttribute("checked","checked"),be.setAttribute("name","t"),xe.appendChild(be),le.checkClone=xe.cloneNode(!0).cloneNode(!0).lastChild.checked,xe.innerHTML="<textarea>x</textarea>",le.noCloneChecked=!!xe.cloneNode(!0).lastChild.defaultValue,xe.innerHTML="<option></option>",le.option=!!xe.lastChild;var ke={thead:[1,"<table>","</table>"],col:[2,"<table><colgroup>","</colgroup></table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],_default:[0,"",""]};function Se(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&fe(e,t)?ce.merge([e],n):n}function Ee(e,t){for(var n=0,r=e.length;n<r;n++)_.set(e[n],"globalEval",!t||_.get(t[n],"globalEval"))}ke.tbody=ke.tfoot=ke.colgroup=ke.caption=ke.thead,ke.th=ke.td,le.option||(ke.optgroup=ke.option=[1,"<select multiple='multiple'>","</select>"]);var je=/<|&#?\w+;/;function Ae(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d<h;d++)if((o=e[d])||0===o)if("object"===x(o))ce.merge(p,o.nodeType?[o]:o);else if(je.test(o)){a=a||f.appendChild(t.createElement("div")),s=(Te.exec(o)||["",""])[1].toLowerCase(),u=ke[s]||ke._default,a.innerHTML=u[1]+ce.htmlPrefilter(o)+u[2],c=u[0];while(c--)a=a.lastChild;ce.merge(p,a.childNodes),(a=f.firstChild).textContent=""}else p.push(t.createTextNode(o));f.textContent="",d=0;while(o=p[d++])if(r&&-1<ce.inArray(o,r))i&&i.push(o);else if(l=K(o),a=Se(f.appendChild(o),"script"),l&&Ee(a),n){c=0;while(o=a[c++])Ce.test(o.type||"")&&n.push(o)}return f}var De=/^([^.]*)(?:\.(.+)|)/;function Ne(){return!0}function qe(){return!1}function Le(e,t,n,r,i,o){var a,s;if("object"==typeof t){for(s in"string"!=typeof n&&(r=r||n,n=void 0),t)Le(e,s,n,r,t[s],o);return e}if(null==r&&null==i?(i=n,r=n=void 0):null==i&&("string"==typeof n?(i=r,r=void 0):(i=r,r=n,n=void 0)),!1===i)i=qe;else if(!i)return e;return 1===o&&(a=i,(i=function(e){return ce().off(e),a.apply(this,arguments)}).guid=a.guid||(a.guid=ce.guid++)),e.each(function(){ce.event.add(this,t,i,r,n)})}function He(e,r,t){t?(_.set(e,r,!1),ce.event.add(e,r,{namespace:!1,handler:function(e){var t,n=_.get(this,r);if(1&e.isTrigger&&this[r]){if(n)(ce.event.special[r]||{}).delegateType&&e.stopPropagation();else if(n=ae.call(arguments),_.set(this,r,n),this[r](),t=_.get(this,r),_.set(this,r,!1),n!==t)return e.stopImmediatePropagation(),e.preventDefault(),t}else n&&(_.set(this,r,ce.event.trigger(n[0],n.slice(1),this)),e.stopPropagation(),e.isImmediatePropagationStopped=Ne)}})):void 0===_.get(e,r)&&ce.event.add(e,r,Ne)}ce.event={global:{},add:function(t,e,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=_.get(t);if($(t)){n.handler&&(n=(o=n).handler,i=o.selector),i&&ce.find.matchesSelector(J,i),n.guid||(n.guid=ce.guid++),(u=v.events)||(u=v.events=Object.create(null)),(a=v.handle)||(a=v.handle=function(e){return"undefined"!=typeof ce&&ce.event.triggered!==e.type?ce.event.dispatch.apply(t,arguments):void 0}),l=(e=(e||"").match(D)||[""]).length;while(l--)d=g=(s=De.exec(e[l])||[])[1],h=(s[2]||"").split(".").sort(),d&&(f=ce.event.special[d]||{},d=(i?f.delegateType:f.bindType)||d,f=ce.event.special[d]||{},c=ce.extend({type:d,origType:g,data:r,handler:n,guid:n.guid,selector:i,needsContext:i&&ce.expr.match.needsContext.test(i),namespace:h.join(".")},o),(p=u[d])||((p=u[d]=[]).delegateCount=0,f.setup&&!1!==f.setup.call(t,r,h,a)||t.addEventListener&&t.addEventListener(d,a)),f.add&&(f.add.call(t,c),c.handler.guid||(c.handler.guid=n.guid)),i?p.splice(p.delegateCount++,0,c):p.push(c),ce.event.global[d]=!0)}},remove:function(e,t,n,r,i){var o,a,s,u,l,c,f,p,d,h,g,v=_.hasData(e)&&_.get(e);if(v&&(u=v.events)){l=(t=(t||"").match(D)||[""]).length;while(l--)if(d=g=(s=De.exec(t[l])||[])[1],h=(s[2]||"").split(".").sort(),d){f=ce.event.special[d]||{},p=u[d=(r?f.delegateType:f.bindType)||d]||[],s=s[2]&&new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"),a=o=p.length;while(o--)c=p[o],!i&&g!==c.origType||n&&n.guid!==c.guid||s&&!s.test(c.namespace)||r&&r!==c.selector&&("**"!==r||!c.selector)||(p.splice(o,1),c.selector&&p.delegateCount--,f.remove&&f.remove.call(e,c));a&&!p.length&&(f.teardown&&!1!==f.teardown.call(e,h,v.handle)||ce.removeEvent(e,d,v.handle),delete u[d])}else for(d in u)ce.event.remove(e,d+t[l],n,r,!0);ce.isEmptyObject(u)&&_.remove(e,"handle events")}},dispatch:function(e){var t,n,r,i,o,a,s=new Array(arguments.length),u=ce.event.fix(e),l=(_.get(this,"events")||Object.create(null))[u.type]||[],c=ce.event.special[u.type]||{};for(s[0]=u,t=1;t<arguments.length;t++)s[t]=arguments[t];if(u.delegateTarget=this,!c.preDispatch||!1!==c.preDispatch.call(this,u)){a=ce.event.handlers.call(this,u,l),t=0;while((i=a[t++])&&!u.isPropagationStopped()){u.currentTarget=i.elem,n=0;while((o=i.handlers[n++])&&!u.isImmediatePropagationStopped())u.rnamespace&&!1!==o.namespace&&!u.rnamespace.test(o.namespace)||(u.handleObj=o,u.data=o.data,void 0!==(r=((ce.event.special[o.origType]||{}).handle||o.handler).apply(i.elem,s))&&!1===(u.result=r)&&(u.preventDefault(),u.stopPropagation()))}return c.postDispatch&&c.postDispatch.call(this,u),u.result}},handlers:function(e,t){var n,r,i,o,a,s=[],u=t.delegateCount,l=e.target;if(u&&l.nodeType&&!("click"===e.type&&1<=e.button))for(;l!==this;l=l.parentNode||this)if(1===l.nodeType&&("click"!==e.type||!0!==l.disabled)){for(o=[],a={},n=0;n<u;n++)void 0===a[i=(r=t[n]).selector+" "]&&(a[i]=r.needsContext?-1<ce(i,this).index(l):ce.find(i,this,null,[l]).length),a[i]&&o.push(r);o.length&&s.push({elem:l,handlers:o})}return l=this,u<t.length&&s.push({elem:l,handlers:t.slice(u)}),s},addProp:function(t,e){Object.defineProperty(ce.Event.prototype,t,{enumerable:!0,configurable:!0,get:v(e)?function(){if(this.originalEvent)return e(this.originalEvent)}:function(){if(this.originalEvent)return this.originalEvent[t]},set:function(e){Object.defineProperty(this,t,{enumerable:!0,configurable:!0,writable:!0,value:e})}})},fix:function(e){return e[ce.expando]?e:new ce.Event(e)},special:{load:{noBubble:!0},click:{setup:function(e){var t=this||e;return we.test(t.type)&&t.click&&fe(t,"input")&&He(t,"click",!0),!1},trigger:function(e){var t=this||e;return we.test(t.type)&&t.click&&fe(t,"input")&&He(t,"click"),!0},_default:function(e){var t=e.target;return we.test(t.type)&&t.click&&fe(t,"input")&&_.get(t,"click")||fe(t,"a")}},beforeunload:{postDispatch:function(e){void 0!==e.result&&e.originalEvent&&(e.originalEvent.returnValue=e.result)}}}},ce.removeEvent=function(e,t,n){e.removeEventListener&&e.removeEventListener(t,n)},ce.Event=function(e,t){if(!(this instanceof ce.Event))return new ce.Event(e,t);e&&e.type?(this.originalEvent=e,this.type=e.type,this.isDefaultPrevented=e.defaultPrevented||void 0===e.defaultPrevented&&!1===e.returnValue?Ne:qe,this.target=e.target&&3===e.target.nodeType?e.target.parentNode:e.target,this.currentTarget=e.currentTarget,this.relatedTarget=e.relatedTarget):this.type=e,t&&ce.extend(this,t),this.timeStamp=e&&e.timeStamp||Date.now(),this[ce.expando]=!0},ce.Event.prototype={constructor:ce.Event,isDefaultPrevented:qe,isPropagationStopped:qe,isImmediatePropagationStopped:qe,isSimulated:!1,preventDefault:function(){var e=this.originalEvent;this.isDefaultPrevented=Ne,e&&!this.isSimulated&&e.preventDefault()},stopPropagation:function(){var e=this.originalEvent;this.isPropagationStopped=Ne,e&&!this.isSimulated&&e.stopPropagation()},stopImmediatePropagation:function(){var e=this.originalEvent;this.isImmediatePropagationStopped=Ne,e&&!this.isSimulated&&e.stopImmediatePropagation(),this.stopPropagation()}},ce.each({altKey:!0,bubbles:!0,cancelable:!0,changedTouches:!0,ctrlKey:!0,detail:!0,eventPhase:!0,metaKey:!0,pageX:!0,pageY:!0,shiftKey:!0,view:!0,"char":!0,code:!0,charCode:!0,key:!0,keyCode:!0,button:!0,buttons:!0,clientX:!0,clientY:!0,offsetX:!0,offsetY:!0,pointerId:!0,pointerType:!0,screenX:!0,screenY:!0,targetTouches:!0,toElement:!0,touches:!0,which:!0},ce.event.addProp),ce.each({focus:"focusin",blur:"focusout"},function(r,i){function o(e){if(C.documentMode){var t=_.get(this,"handle"),n=ce.event.fix(e);n.type="focusin"===e.type?"focus":"blur",n.isSimulated=!0,t(e),n.target===n.currentTarget&&t(n)}else ce.event.simulate(i,e.target,ce.event.fix(e))}ce.event.special[r]={setup:function(){var e;if(He(this,r,!0),!C.documentMode)return!1;(e=_.get(this,i))||this.addEventListener(i,o),_.set(this,i,(e||0)+1)},trigger:function(){return He(this,r),!0},teardown:function(){var e;if(!C.documentMode)return!1;(e=_.get(this,i)-1)?_.set(this,i,e):(this.removeEventListener(i,o),_.remove(this,i))},_default:function(e){return _.get(e.target,r)},delegateType:i},ce.event.special[i]={setup:function(){var e=this.ownerDocument||this.document||this,t=C.documentMode?this:e,n=_.get(t,i);n||(C.documentMode?this.addEventListener(i,o):e.addEventListener(r,o,!0)),_.set(t,i,(n||0)+1)},teardown:function(){var e=this.ownerDocument||this.document||this,t=C.documentMode?this:e,n=_.get(t,i)-1;n?_.set(t,i,n):(C.documentMode?this.removeEventListener(i,o):e.removeEventListener(r,o,!0),_.remove(t,i))}}}),ce.each({mouseenter:"mouseover",mouseleave:"mouseout",pointerenter:"pointerover",pointerleave:"pointerout"},function(e,i){ce.event.special[e]={delegateType:i,bindType:i,handle:function(e){var t,n=e.relatedTarget,r=e.handleObj;return n&&(n===this||ce.contains(this,n))||(e.type=r.origType,t=r.handler.apply(this,arguments),e.type=i),t}}}),ce.fn.extend({on:function(e,t,n,r){return Le(this,e,t,n,r)},one:function(e,t,n,r){return Le(this,e,t,n,r,1)},off:function(e,t,n){var r,i;if(e&&e.preventDefault&&e.handleObj)return r=e.handleObj,ce(e.delegateTarget).off(r.namespace?r.origType+"."+r.namespace:r.origType,r.selector,r.handler),this;if("object"==typeof e){for(i in e)this.off(i,t,e[i]);return this}return!1!==t&&"function"!=typeof t||(n=t,t=void 0),!1===n&&(n=qe),this.each(function(){ce.event.remove(this,e,n,t)})}});var Oe=/<script|<style|<link/i,Pe=/checked\s*(?:[^=]|=\s*.checked.)/i,Me=/^\s*<!\[CDATA\[|\]\]>\s*$/g;function Re(e,t){return fe(e,"table")&&fe(11!==t.nodeType?t:t.firstChild,"tr")&&ce(e).children("tbody")[0]||e}function Ie(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function We(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Fe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(_.hasData(e)&&(s=_.get(e).events))for(i in _.remove(t,"handle events"),s)for(n=0,r=s[i].length;n<r;n++)ce.event.add(t,i,s[i][n]);z.hasData(e)&&(o=z.access(e),a=ce.extend({},o),z.set(t,a))}}function $e(n,r,i,o){r=g(r);var e,t,a,s,u,l,c=0,f=n.length,p=f-1,d=r[0],h=v(d);if(h||1<f&&"string"==typeof d&&!le.checkClone&&Pe.test(d))return n.each(function(e){var t=n.eq(e);h&&(r[0]=d.call(this,e,t.html())),$e(t,r,i,o)});if(f&&(t=(e=Ae(r,n[0].ownerDocument,!1,n,o)).firstChild,1===e.childNodes.length&&(e=t),t||o)){for(s=(a=ce.map(Se(e,"script"),Ie)).length;c<f;c++)u=e,c!==p&&(u=ce.clone(u,!0,!0),s&&ce.merge(a,Se(u,"script"))),i.call(n[c],u,c);if(s)for(l=a[a.length-1].ownerDocument,ce.map(a,We),c=0;c<s;c++)u=a[c],Ce.test(u.type||"")&&!_.access(u,"globalEval")&&ce.contains(l,u)&&(u.src&&"module"!==(u.type||"").toLowerCase()?ce._evalUrl&&!u.noModule&&ce._evalUrl(u.src,{nonce:u.nonce||u.getAttribute("nonce")},l):m(u.textContent.replace(Me,""),u,l))}return n}function Be(e,t,n){for(var r,i=t?ce.filter(t,e):e,o=0;null!=(r=i[o]);o++)n||1!==r.nodeType||ce.cleanData(Se(r)),r.parentNode&&(n&&K(r)&&Ee(Se(r,"script")),r.parentNode.removeChild(r));return e}ce.extend({htmlPrefilter:function(e){return e},clone:function(e,t,n){var r,i,o,a,s,u,l,c=e.cloneNode(!0),f=K(e);if(!(le.noCloneChecked||1!==e.nodeType&&11!==e.nodeType||ce.isXMLDoc(e)))for(a=Se(c),r=0,i=(o=Se(e)).length;r<i;r++)s=o[r],u=a[r],void 0,"input"===(l=u.nodeName.toLowerCase())&&we.test(s.type)?u.checked=s.checked:"input"!==l&&"textarea"!==l||(u.defaultValue=s.defaultValue);if(t)if(n)for(o=o||Se(e),a=a||Se(c),r=0,i=o.length;r<i;r++)Fe(o[r],a[r]);else Fe(e,c);return 0<(a=Se(c,"script")).length&&Ee(a,!f&&Se(e,"script")),c},cleanData:function(e){for(var t,n,r,i=ce.event.special,o=0;void 0!==(n=e[o]);o++)if($(n)){if(t=n[_.expando]){if(t.events)for(r in t.events)i[r]?ce.event.remove(n,r):ce.removeEvent(n,r,t.handle);n[_.expando]=void 0}n[z.expando]&&(n[z.expando]=void 0)}}}),ce.fn.extend({detach:function(e){return Be(this,e,!0)},remove:function(e){return Be(this,e)},text:function(e){return M(this,function(e){return void 0===e?ce.text(this):this.empty().each(function(){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||(this.textContent=e)})},null,e,arguments.length)},append:function(){return $e(this,arguments,function(e){1!==this.nodeType&&11!==this.nodeType&&9!==this.nodeType||Re(this,e).appendChild(e)})},prepend:function(){return $e(this,arguments,function(e){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var t=Re(this,e);t.insertBefore(e,t.firstChild)}})},before:function(){return $e(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this)})},after:function(){return $e(this,arguments,function(e){this.parentNode&&this.parentNode.insertBefore(e,this.nextSibling)})},empty:function(){for(var e,t=0;null!=(e=this[t]);t++)1===e.nodeType&&(ce.cleanData(Se(e,!1)),e.textContent="");return this},clone:function(e,t){return e=null!=e&&e,t=null==t?e:t,this.map(function(){return ce.clone(this,e,t)})},html:function(e){return M(this,function(e){var t=this[0]||{},n=0,r=this.length;if(void 0===e&&1===t.nodeType)return t.innerHTML;if("string"==typeof e&&!Oe.test(e)&&!ke[(Te.exec(e)||["",""])[1].toLowerCase()]){e=ce.htmlPrefilter(e);try{for(;n<r;n++)1===(t=this[n]||{}).nodeType&&(ce.cleanData(Se(t,!1)),t.innerHTML=e);t=0}catch(e){}}t&&this.empty().append(e)},null,e,arguments.length)},replaceWith:function(){var n=[];return $e(this,arguments,function(e){var t=this.parentNode;ce.inArray(this,n)<0&&(ce.cleanData(Se(this)),t&&t.replaceChild(e,this))},n)}}),ce.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(e,a){ce.fn[e]=function(e){for(var t,n=[],r=ce(e),i=r.length-1,o=0;o<=i;o++)t=o===i?this:this.clone(!0),ce(r[o])[a](t),s.apply(n,t.get());return this.pushStack(n)}});var _e=new RegExp("^("+G+")(?!px)[a-z%]+$","i"),ze=/^--/,Xe=function(e){var t=e.ownerDocument.defaultView;return t&&t.opener||(t=ie),t.getComputedStyle(e)},Ue=function(e,t,n){var r,i,o={};for(i in t)o[i]=e.style[i],e.style[i]=t[i];for(i in r=n.call(e),t)e.style[i]=o[i];return r},Ve=new RegExp(Q.join("|"),"i");function Ge(e,t,n){var r,i,o,a,s=ze.test(t),u=e.style;return(n=n||Xe(e))&&(a=n.getPropertyValue(t)||n[t],s&&a&&(a=a.replace(ve,"$1")||void 0),""!==a||K(e)||(a=ce.style(e,t)),!le.pixelBoxStyles()&&_e.test(a)&&Ve.test(t)&&(r=u.width,i=u.minWidth,o=u.maxWidth,u.minWidth=u.maxWidth=u.width=a,a=n.width,u.width=r,u.minWidth=i,u.maxWidth=o)),void 0!==a?a+"":a}function Ye(e,t){return{get:function(){if(!e())return(this.get=t).apply(this,arguments);delete this.get}}}!function(){function e(){if(l){u.style.cssText="position:absolute;left:-11111px;width:60px;margin-top:1px;padding:0;border:0",l.style.cssText="position:relative;display:block;box-sizing:border-box;overflow:scroll;margin:auto;border:1px;padding:1px;width:60%;top:1%",J.appendChild(u).appendChild(l);var e=ie.getComputedStyle(l);n="1%"!==e.top,s=12===t(e.marginLeft),l.style.right="60%",o=36===t(e.right),r=36===t(e.width),l.style.position="absolute",i=12===t(l.offsetWidth/3),J.removeChild(u),l=null}}function t(e){return Math.round(parseFloat(e))}var n,r,i,o,a,s,u=C.createElement("div"),l=C.createElement("div");l.style&&(l.style.backgroundClip="content-box",l.cloneNode(!0).style.backgroundClip="",le.clearCloneStyle="content-box"===l.style.backgroundClip,ce.extend(le,{boxSizingReliable:function(){return e(),r},pixelBoxStyles:function(){return e(),o},pixelPosition:function(){return e(),n},reliableMarginLeft:function(){return e(),s},scrollboxSize:function(){return e(),i},reliableTrDimensions:function(){var e,t,n,r;return null==a&&(e=C.createElement("table"),t=C.createElement("tr"),n=C.createElement("div"),e.style.cssText="position:absolute;left:-11111px;border-collapse:separate",t.style.cssText="box-sizing:content-box;border:1px solid",t.style.height="1px",n.style.height="9px",n.style.display="block",J.appendChild(e).appendChild(t).appendChild(n),r=ie.getComputedStyle(t),a=parseInt(r.height,10)+parseInt(r.borderTopWidth,10)+parseInt(r.borderBottomWidth,10)===t.offsetHeight,J.removeChild(e)),a}}))}();var Qe=["Webkit","Moz","ms"],Je=C.createElement("div").style,Ke={};function Ze(e){var t=ce.cssProps[e]||Ke[e];return t||(e in Je?e:Ke[e]=function(e){var t=e[0].toUpperCase()+e.slice(1),n=Qe.length;while(n--)if((e=Qe[n]+t)in Je)return e}(e)||e)}var et=/^(none|table(?!-c[ea]).+)/,tt={position:"absolute",visibility:"hidden",display:"block"},nt={letterSpacing:"0",fontWeight:"400"};function rt(e,t,n){var r=Y.exec(t);return r?Math.max(0,r[2]-(n||0))+(r[3]||"px"):t}function it(e,t,n,r,i,o){var a="width"===t?1:0,s=0,u=0,l=0;if(n===(r?"border":"content"))return 0;for(;a<4;a+=2)"margin"===n&&(l+=ce.css(e,n+Q[a],!0,i)),r?("content"===n&&(u-=ce.css(e,"padding"+Q[a],!0,i)),"margin"!==n&&(u-=ce.css(e,"border"+Q[a]+"Width",!0,i))):(u+=ce.css(e,"padding"+Q[a],!0,i),"padding"!==n?u+=ce.css(e,"border"+Q[a]+"Width",!0,i):s+=ce.css(e,"border"+Q[a]+"Width",!0,i));return!r&&0<=o&&(u+=Math.max(0,Math.ceil(e["offset"+t[0].toUpperCase()+t.slice(1)]-o-u-s-.5))||0),u+l}function ot(e,t,n){var r=Xe(e),i=(!le.boxSizingReliable()||n)&&"border-box"===ce.css(e,"boxSizing",!1,r),o=i,a=Ge(e,t,r),s="offset"+t[0].toUpperCase()+t.slice(1);if(_e.test(a)){if(!n)return a;a="auto"}return(!le.boxSizingReliable()&&i||!le.reliableTrDimensions()&&fe(e,"tr")||"auto"===a||!parseFloat(a)&&"inline"===ce.css(e,"display",!1,r))&&e.getClientRects().length&&(i="border-box"===ce.css(e,"boxSizing",!1,r),(o=s in e)&&(a=e[s])),(a=parseFloat(a)||0)+it(e,t,n||(i?"border":"content"),o,r,a)+"px"}function at(e,t,n,r,i){return new at.prototype.init(e,t,n,r,i)}ce.extend({cssHooks:{opacity:{get:function(e,t){if(t){var n=Ge(e,"opacity");return""===n?"1":n}}}},cssNumber:{animationIterationCount:!0,aspectRatio:!0,borderImageSlice:!0,columnCount:!0,flexGrow:!0,flexShrink:!0,fontWeight:!0,gridArea:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnStart:!0,gridRow:!0,gridRowEnd:!0,gridRowStart:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,scale:!0,widows:!0,zIndex:!0,zoom:!0,fillOpacity:!0,floodOpacity:!0,stopOpacity:!0,strokeMiterlimit:!0,strokeOpacity:!0},cssProps:{},style:function(e,t,n,r){if(e&&3!==e.nodeType&&8!==e.nodeType&&e.style){var i,o,a,s=F(t),u=ze.test(t),l=e.style;if(u||(t=Ze(s)),a=ce.cssHooks[t]||ce.cssHooks[s],void 0===n)return a&&"get"in a&&void 0!==(i=a.get(e,!1,r))?i:l[t];"string"===(o=typeof n)&&(i=Y.exec(n))&&i[1]&&(n=te(e,t,i),o="number"),null!=n&&n==n&&("number"!==o||u||(n+=i&&i[3]||(ce.cssNumber[s]?"":"px")),le.clearCloneStyle||""!==n||0!==t.indexOf("background")||(l[t]="inherit"),a&&"set"in a&&void 0===(n=a.set(e,n,r))||(u?l.setProperty(t,n):l[t]=n))}},css:function(e,t,n,r){var i,o,a,s=F(t);return ze.test(t)||(t=Ze(s)),(a=ce.cssHooks[t]||ce.cssHooks[s])&&"get"in a&&(i=a.get(e,!0,n)),void 0===i&&(i=Ge(e,t,r)),"normal"===i&&t in nt&&(i=nt[t]),""===n||n?(o=parseFloat(i),!0===n||isFinite(o)?o||0:i):i}}),ce.each(["height","width"],function(e,u){ce.cssHooks[u]={get:function(e,t,n){if(t)return!et.test(ce.css(e,"display"))||e.getClientRects().length&&e.getBoundingClientRect().width?ot(e,u,n):Ue(e,tt,function(){return ot(e,u,n)})},set:function(e,t,n){var r,i=Xe(e),o=!le.scrollboxSize()&&"absolute"===i.position,a=(o||n)&&"border-box"===ce.css(e,"boxSizing",!1,i),s=n?it(e,u,n,a,i):0;return a&&o&&(s-=Math.ceil(e["offset"+u[0].toUpperCase()+u.slice(1)]-parseFloat(i[u])-it(e,u,"border",!1,i)-.5)),s&&(r=Y.exec(t))&&"px"!==(r[3]||"px")&&(e.style[u]=t,t=ce.css(e,u)),rt(0,t,s)}}}),ce.cssHooks.marginLeft=Ye(le.reliableMarginLeft,function(e,t){if(t)return(parseFloat(Ge(e,"marginLeft"))||e.getBoundingClientRect().left-Ue(e,{marginLeft:0},function(){return e.getBoundingClientRect().left}))+"px"}),ce.each({margin:"",padding:"",border:"Width"},function(i,o){ce.cssHooks[i+o]={expand:function(e){for(var t=0,n={},r="string"==typeof e?e.split(" "):[e];t<4;t++)n[i+Q[t]+o]=r[t]||r[t-2]||r[0];return n}},"margin"!==i&&(ce.cssHooks[i+o].set=rt)}),ce.fn.extend({css:function(e,t){return M(this,function(e,t,n){var r,i,o={},a=0;if(Array.isArray(t)){for(r=Xe(e),i=t.length;a<i;a++)o[t[a]]=ce.css(e,t[a],!1,r);return o}return void 0!==n?ce.style(e,t,n):ce.css(e,t)},e,t,1<arguments.length)}}),((ce.Tween=at).prototype={constructor:at,init:function(e,t,n,r,i,o){this.elem=e,this.prop=n,this.easing=i||ce.easing._default,this.options=t,this.start=this.now=this.cur(),this.end=r,this.unit=o||(ce.cssNumber[n]?"":"px")},cur:function(){var e=at.propHooks[this.prop];return e&&e.get?e.get(this):at.propHooks._default.get(this)},run:function(e){var t,n=at.propHooks[this.prop];return this.options.duration?this.pos=t=ce.easing[this.easing](e,this.options.duration*e,0,1,this.options.duration):this.pos=t=e,this.now=(this.end-this.start)*t+this.start,this.options.step&&this.options.step.call(this.elem,this.now,this),n&&n.set?n.set(this):at.propHooks._default.set(this),this}}).init.prototype=at.prototype,(at.propHooks={_default:{get:function(e){var t;return 1!==e.elem.nodeType||null!=e.elem[e.prop]&&null==e.elem.style[e.prop]?e.elem[e.prop]:(t=ce.css(e.elem,e.prop,""))&&"auto"!==t?t:0},set:function(e){ce.fx.step[e.prop]?ce.fx.step[e.prop](e):1!==e.elem.nodeType||!ce.cssHooks[e.prop]&&null==e.elem.style[Ze(e.prop)]?e.elem[e.prop]=e.now:ce.style(e.elem,e.prop,e.now+e.unit)}}}).scrollTop=at.propHooks.scrollLeft={set:function(e){e.elem.nodeType&&e.elem.parentNode&&(e.elem[e.prop]=e.now)}},ce.easing={linear:function(e){return e},swing:function(e){return.5-Math.cos(e*Math.PI)/2},_default:"swing"},ce.fx=at.prototype.init,ce.fx.step={};var st,ut,lt,ct,ft=/^(?:toggle|show|hide)$/,pt=/queueHooks$/;function dt(){ut&&(!1===C.hidden&&ie.requestAnimationFrame?ie.requestAnimationFrame(dt):ie.setTimeout(dt,ce.fx.interval),ce.fx.tick())}function ht(){return ie.setTimeout(function(){st=void 0}),st=Date.now()}function gt(e,t){var n,r=0,i={height:e};for(t=t?1:0;r<4;r+=2-t)i["margin"+(n=Q[r])]=i["padding"+n]=e;return t&&(i.opacity=i.width=e),i}function vt(e,t,n){for(var r,i=(yt.tweeners[t]||[]).concat(yt.tweeners["*"]),o=0,a=i.length;o<a;o++)if(r=i[o].call(n,t,e))return r}function yt(o,e,t){var n,a,r=0,i=yt.prefilters.length,s=ce.Deferred().always(function(){delete u.elem}),u=function(){if(a)return!1;for(var e=st||ht(),t=Math.max(0,l.startTime+l.duration-e),n=1-(t/l.duration||0),r=0,i=l.tweens.length;r<i;r++)l.tweens[r].run(n);return s.notifyWith(o,[l,n,t]),n<1&&i?t:(i||s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l]),!1)},l=s.promise({elem:o,props:ce.extend({},e),opts:ce.extend(!0,{specialEasing:{},easing:ce.easing._default},t),originalProperties:e,originalOptions:t,startTime:st||ht(),duration:t.duration,tweens:[],createTween:function(e,t){var n=ce.Tween(o,l.opts,e,t,l.opts.specialEasing[e]||l.opts.easing);return l.tweens.push(n),n},stop:function(e){var t=0,n=e?l.tweens.length:0;if(a)return this;for(a=!0;t<n;t++)l.tweens[t].run(1);return e?(s.notifyWith(o,[l,1,0]),s.resolveWith(o,[l,e])):s.rejectWith(o,[l,e]),this}}),c=l.props;for(!function(e,t){var n,r,i,o,a;for(n in e)if(i=t[r=F(n)],o=e[n],Array.isArray(o)&&(i=o[1],o=e[n]=o[0]),n!==r&&(e[r]=o,delete e[n]),(a=ce.cssHooks[r])&&"expand"in a)for(n in o=a.expand(o),delete e[r],o)n in e||(e[n]=o[n],t[n]=i);else t[r]=i}(c,l.opts.specialEasing);r<i;r++)if(n=yt.prefilters[r].call(l,o,c,l.opts))return v(n.stop)&&(ce._queueHooks(l.elem,l.opts.queue).stop=n.stop.bind(n)),n;return ce.map(c,vt,l),v(l.opts.start)&&l.opts.start.call(o,l),l.progress(l.opts.progress).done(l.opts.done,l.opts.complete).fail(l.opts.fail).always(l.opts.always),ce.fx.timer(ce.extend(u,{elem:o,anim:l,queue:l.opts.queue})),l}ce.Animation=ce.extend(yt,{tweeners:{"*":[function(e,t){var n=this.createTween(e,t);return te(n.elem,e,Y.exec(t),n),n}]},tweener:function(e,t){v(e)?(t=e,e=["*"]):e=e.match(D);for(var n,r=0,i=e.length;r<i;r++)n=e[r],yt.tweeners[n]=yt.tweeners[n]||[],yt.tweeners[n].unshift(t)},prefilters:[function(e,t,n){var r,i,o,a,s,u,l,c,f="width"in t||"height"in t,p=this,d={},h=e.style,g=e.nodeType&&ee(e),v=_.get(e,"fxshow");for(r in n.queue||(null==(a=ce._queueHooks(e,"fx")).unqueued&&(a.unqueued=0,s=a.empty.fire,a.empty.fire=function(){a.unqueued||s()}),a.unqueued++,p.always(function(){p.always(function(){a.unqueued--,ce.queue(e,"fx").length||a.empty.fire()})})),t)if(i=t[r],ft.test(i)){if(delete t[r],o=o||"toggle"===i,i===(g?"hide":"show")){if("show"!==i||!v||void 0===v[r])continue;g=!0}d[r]=v&&v[r]||ce.style(e,r)}if((u=!ce.isEmptyObject(t))||!ce.isEmptyObject(d))for(r in f&&1===e.nodeType&&(n.overflow=[h.overflow,h.overflowX,h.overflowY],null==(l=v&&v.display)&&(l=_.get(e,"display")),"none"===(c=ce.css(e,"display"))&&(l?c=l:(re([e],!0),l=e.style.display||l,c=ce.css(e,"display"),re([e]))),("inline"===c||"inline-block"===c&&null!=l)&&"none"===ce.css(e,"float")&&(u||(p.done(function(){h.display=l}),null==l&&(c=h.display,l="none"===c?"":c)),h.display="inline-block")),n.overflow&&(h.overflow="hidden",p.always(function(){h.overflow=n.overflow[0],h.overflowX=n.overflow[1],h.overflowY=n.overflow[2]})),u=!1,d)u||(v?"hidden"in v&&(g=v.hidden):v=_.access(e,"fxshow",{display:l}),o&&(v.hidden=!g),g&&re([e],!0),p.done(function(){for(r in g||re([e]),_.remove(e,"fxshow"),d)ce.style(e,r,d[r])})),u=vt(g?v[r]:0,r,p),r in v||(v[r]=u.start,g&&(u.end=u.start,u.start=0))}],prefilter:function(e,t){t?yt.prefilters.unshift(e):yt.prefilters.push(e)}}),ce.speed=function(e,t,n){var r=e&&"object"==typeof e?ce.extend({},e):{complete:n||!n&&t||v(e)&&e,duration:e,easing:n&&t||t&&!v(t)&&t};return ce.fx.off?r.duration=0:"number"!=typeof r.duration&&(r.duration in ce.fx.speeds?r.duration=ce.fx.speeds[r.duration]:r.duration=ce.fx.speeds._default),null!=r.queue&&!0!==r.queue||(r.queue="fx"),r.old=r.complete,r.complete=function(){v(r.old)&&r.old.call(this),r.queue&&ce.dequeue(this,r.queue)},r},ce.fn.extend({fadeTo:function(e,t,n,r){return this.filter(ee).css("opacity",0).show().end().animate({opacity:t},e,n,r)},animate:function(t,e,n,r){var i=ce.isEmptyObject(t),o=ce.speed(e,n,r),a=function(){var e=yt(this,ce.extend({},t),o);(i||_.get(this,"finish"))&&e.stop(!0)};return a.finish=a,i||!1===o.queue?this.each(a):this.queue(o.queue,a)},stop:function(i,e,o){var a=function(e){var t=e.stop;delete e.stop,t(o)};return"string"!=typeof i&&(o=e,e=i,i=void 0),e&&this.queue(i||"fx",[]),this.each(function(){var e=!0,t=null!=i&&i+"queueHooks",n=ce.timers,r=_.get(this);if(t)r[t]&&r[t].stop&&a(r[t]);else for(t in r)r[t]&&r[t].stop&&pt.test(t)&&a(r[t]);for(t=n.length;t--;)n[t].elem!==this||null!=i&&n[t].queue!==i||(n[t].anim.stop(o),e=!1,n.splice(t,1));!e&&o||ce.dequeue(this,i)})},finish:function(a){return!1!==a&&(a=a||"fx"),this.each(function(){var e,t=_.get(this),n=t[a+"queue"],r=t[a+"queueHooks"],i=ce.timers,o=n?n.length:0;for(t.finish=!0,ce.queue(this,a,[]),r&&r.stop&&r.stop.call(this,!0),e=i.length;e--;)i[e].elem===this&&i[e].queue===a&&(i[e].anim.stop(!0),i.splice(e,1));for(e=0;e<o;e++)n[e]&&n[e].finish&&n[e].finish.call(this);delete t.finish})}}),ce.each(["toggle","show","hide"],function(e,r){var i=ce.fn[r];ce.fn[r]=function(e,t,n){return null==e||"boolean"==typeof e?i.apply(this,arguments):this.animate(gt(r,!0),e,t,n)}}),ce.each({slideDown:gt("show"),slideUp:gt("hide"),slideToggle:gt("toggle"),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(e,r){ce.fn[e]=function(e,t,n){return this.animate(r,e,t,n)}}),ce.timers=[],ce.fx.tick=function(){var e,t=0,n=ce.timers;for(st=Date.now();t<n.length;t++)(e=n[t])()||n[t]!==e||n.splice(t--,1);n.length||ce.fx.stop(),st=void 0},ce.fx.timer=function(e){ce.timers.push(e),ce.fx.start()},ce.fx.interval=13,ce.fx.start=function(){ut||(ut=!0,dt())},ce.fx.stop=function(){ut=null},ce.fx.speeds={slow:600,fast:200,_default:400},ce.fn.delay=function(r,e){return r=ce.fx&&ce.fx.speeds[r]||r,e=e||"fx",this.queue(e,function(e,t){var n=ie.setTimeout(e,r);t.stop=function(){ie.clearTimeout(n)}})},lt=C.createElement("input"),ct=C.createElement("select").appendChild(C.createElement("option")),lt.type="checkbox",le.checkOn=""!==lt.value,le.optSelected=ct.selected,(lt=C.createElement("input")).value="t",lt.type="radio",le.radioValue="t"===lt.value;var mt,xt=ce.expr.attrHandle;ce.fn.extend({attr:function(e,t){return M(this,ce.attr,e,t,1<arguments.length)},removeAttr:function(e){return this.each(function(){ce.removeAttr(this,e)})}}),ce.extend({attr:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return"undefined"==typeof e.getAttribute?ce.prop(e,t,n):(1===o&&ce.isXMLDoc(e)||(i=ce.attrHooks[t.toLowerCase()]||(ce.expr.match.bool.test(t)?mt:void 0)),void 0!==n?null===n?void ce.removeAttr(e,t):i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:(e.setAttribute(t,n+""),n):i&&"get"in i&&null!==(r=i.get(e,t))?r:null==(r=ce.find.attr(e,t))?void 0:r)},attrHooks:{type:{set:function(e,t){if(!le.radioValue&&"radio"===t&&fe(e,"input")){var n=e.value;return e.setAttribute("type",t),n&&(e.value=n),t}}}},removeAttr:function(e,t){var n,r=0,i=t&&t.match(D);if(i&&1===e.nodeType)while(n=i[r++])e.removeAttribute(n)}}),mt={set:function(e,t,n){return!1===t?ce.removeAttr(e,n):e.setAttribute(n,n),n}},ce.each(ce.expr.match.bool.source.match(/\w+/g),function(e,t){var a=xt[t]||ce.find.attr;xt[t]=function(e,t,n){var r,i,o=t.toLowerCase();return n||(i=xt[o],xt[o]=r,r=null!=a(e,t,n)?o:null,xt[o]=i),r}});var bt=/^(?:input|select|textarea|button)$/i,wt=/^(?:a|area)$/i;function Tt(e){return(e.match(D)||[]).join(" ")}function Ct(e){return e.getAttribute&&e.getAttribute("class")||""}function kt(e){return Array.isArray(e)?e:"string"==typeof e&&e.match(D)||[]}ce.fn.extend({prop:function(e,t){return M(this,ce.prop,e,t,1<arguments.length)},removeProp:function(e){return this.each(function(){delete this[ce.propFix[e]||e]})}}),ce.extend({prop:function(e,t,n){var r,i,o=e.nodeType;if(3!==o&&8!==o&&2!==o)return 1===o&&ce.isXMLDoc(e)||(t=ce.propFix[t]||t,i=ce.propHooks[t]),void 0!==n?i&&"set"in i&&void 0!==(r=i.set(e,n,t))?r:e[t]=n:i&&"get"in i&&null!==(r=i.get(e,t))?r:e[t]},propHooks:{tabIndex:{get:function(e){var t=ce.find.attr(e,"tabindex");return t?parseInt(t,10):bt.test(e.nodeName)||wt.test(e.nodeName)&&e.href?0:-1}}},propFix:{"for":"htmlFor","class":"className"}}),le.optSelected||(ce.propHooks.selected={get:function(e){var t=e.parentNode;return t&&t.parentNode&&t.parentNode.selectedIndex,null},set:function(e){var t=e.parentNode;t&&(t.selectedIndex,t.parentNode&&t.parentNode.selectedIndex)}}),ce.each(["tabIndex","readOnly","maxLength","cellSpacing","cellPadding","rowSpan","colSpan","useMap","frameBorder","contentEditable"],function(){ce.propFix[this.toLowerCase()]=this}),ce.fn.extend({addClass:function(t){var e,n,r,i,o,a;return v(t)?this.each(function(e){ce(this).addClass(t.call(this,e,Ct(this)))}):(e=kt(t)).length?this.each(function(){if(r=Ct(this),n=1===this.nodeType&&" "+Tt(r)+" "){for(o=0;o<e.length;o++)i=e[o],n.indexOf(" "+i+" ")<0&&(n+=i+" ");a=Tt(n),r!==a&&this.setAttribute("class",a)}}):this},removeClass:function(t){var e,n,r,i,o,a;return v(t)?this.each(function(e){ce(this).removeClass(t.call(this,e,Ct(this)))}):arguments.length?(e=kt(t)).length?this.each(function(){if(r=Ct(this),n=1===this.nodeType&&" "+Tt(r)+" "){for(o=0;o<e.length;o++){i=e[o];while(-1<n.indexOf(" "+i+" "))n=n.replace(" "+i+" "," ")}a=Tt(n),r!==a&&this.setAttribute("class",a)}}):this:this.attr("class","")},toggleClass:function(t,n){var e,r,i,o,a=typeof t,s="string"===a||Array.isArray(t);return v(t)?this.each(function(e){ce(this).toggleClass(t.call(this,e,Ct(this),n),n)}):"boolean"==typeof n&&s?n?this.addClass(t):this.removeClass(t):(e=kt(t),this.each(function(){if(s)for(o=ce(this),i=0;i<e.length;i++)r=e[i],o.hasClass(r)?o.removeClass(r):o.addClass(r);else void 0!==t&&"boolean"!==a||((r=Ct(this))&&_.set(this,"__className__",r),this.setAttribute&&this.setAttribute("class",r||!1===t?"":_.get(this,"__className__")||""))}))},hasClass:function(e){var t,n,r=0;t=" "+e+" ";while(n=this[r++])if(1===n.nodeType&&-1<(" "+Tt(Ct(n))+" ").indexOf(t))return!0;return!1}});var St=/\r/g;ce.fn.extend({val:function(n){var r,e,i,t=this[0];return arguments.length?(i=v(n),this.each(function(e){var t;1===this.nodeType&&(null==(t=i?n.call(this,e,ce(this).val()):n)?t="":"number"==typeof t?t+="":Array.isArray(t)&&(t=ce.map(t,function(e){return null==e?"":e+""})),(r=ce.valHooks[this.type]||ce.valHooks[this.nodeName.toLowerCase()])&&"set"in r&&void 0!==r.set(this,t,"value")||(this.value=t))})):t?(r=ce.valHooks[t.type]||ce.valHooks[t.nodeName.toLowerCase()])&&"get"in r&&void 0!==(e=r.get(t,"value"))?e:"string"==typeof(e=t.value)?e.replace(St,""):null==e?"":e:void 0}}),ce.extend({valHooks:{option:{get:function(e){var t=ce.find.attr(e,"value");return null!=t?t:Tt(ce.text(e))}},select:{get:function(e){var t,n,r,i=e.options,o=e.selectedIndex,a="select-one"===e.type,s=a?null:[],u=a?o+1:i.length;for(r=o<0?u:a?o:0;r<u;r++)if(((n=i[r]).selected||r===o)&&!n.disabled&&(!n.parentNode.disabled||!fe(n.parentNode,"optgroup"))){if(t=ce(n).val(),a)return t;s.push(t)}return s},set:function(e,t){var n,r,i=e.options,o=ce.makeArray(t),a=i.length;while(a--)((r=i[a]).selected=-1<ce.inArray(ce.valHooks.option.get(r),o))&&(n=!0);return n||(e.selectedIndex=-1),o}}}}),ce.each(["radio","checkbox"],function(){ce.valHooks[this]={set:function(e,t){if(Array.isArray(t))return e.checked=-1<ce.inArray(ce(e).val(),t)}},le.checkOn||(ce.valHooks[this].get=function(e){return null===e.getAttribute("value")?"on":e.value})});var Et=ie.location,jt={guid:Date.now()},At=/\?/;ce.parseXML=function(e){var t,n;if(!e||"string"!=typeof e)return null;try{t=(new ie.DOMParser).parseFromString(e,"text/xml")}catch(e){}return n=t&&t.getElementsByTagName("parsererror")[0],t&&!n||ce.error("Invalid XML: "+(n?ce.map(n.childNodes,function(e){return e.textContent}).join("\n"):e)),t};var Dt=/^(?:focusinfocus|focusoutblur)$/,Nt=function(e){e.stopPropagation()};ce.extend(ce.event,{trigger:function(e,t,n,r){var i,o,a,s,u,l,c,f,p=[n||C],d=ue.call(e,"type")?e.type:e,h=ue.call(e,"namespace")?e.namespace.split("."):[];if(o=f=a=n=n||C,3!==n.nodeType&&8!==n.nodeType&&!Dt.test(d+ce.event.triggered)&&(-1<d.indexOf(".")&&(d=(h=d.split(".")).shift(),h.sort()),u=d.indexOf(":")<0&&"on"+d,(e=e[ce.expando]?e:new ce.Event(d,"object"==typeof e&&e)).isTrigger=r?2:3,e.namespace=h.join("."),e.rnamespace=e.namespace?new RegExp("(^|\\.)"+h.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,e.result=void 0,e.target||(e.target=n),t=null==t?[e]:ce.makeArray(t,[e]),c=ce.event.special[d]||{},r||!c.trigger||!1!==c.trigger.apply(n,t))){if(!r&&!c.noBubble&&!y(n)){for(s=c.delegateType||d,Dt.test(s+d)||(o=o.parentNode);o;o=o.parentNode)p.push(o),a=o;a===(n.ownerDocument||C)&&p.push(a.defaultView||a.parentWindow||ie)}i=0;while((o=p[i++])&&!e.isPropagationStopped())f=o,e.type=1<i?s:c.bindType||d,(l=(_.get(o,"events")||Object.create(null))[e.type]&&_.get(o,"handle"))&&l.apply(o,t),(l=u&&o[u])&&l.apply&&$(o)&&(e.result=l.apply(o,t),!1===e.result&&e.preventDefault());return e.type=d,r||e.isDefaultPrevented()||c._default&&!1!==c._default.apply(p.pop(),t)||!$(n)||u&&v(n[d])&&!y(n)&&((a=n[u])&&(n[u]=null),ce.event.triggered=d,e.isPropagationStopped()&&f.addEventListener(d,Nt),n[d](),e.isPropagationStopped()&&f.removeEventListener(d,Nt),ce.event.triggered=void 0,a&&(n[u]=a)),e.result}},simulate:function(e,t,n){var r=ce.extend(new ce.Event,n,{type:e,isSimulated:!0});ce.event.trigger(r,null,t)}}),ce.fn.extend({trigger:function(e,t){return this.each(function(){ce.event.trigger(e,t,this)})},triggerHandler:function(e,t){var n=this[0];if(n)return ce.event.trigger(e,t,n,!0)}});var qt=/\[\]$/,Lt=/\r?\n/g,Ht=/^(?:submit|button|image|reset|file)$/i,Ot=/^(?:input|select|textarea|keygen)/i;function Pt(n,e,r,i){var t;if(Array.isArray(e))ce.each(e,function(e,t){r||qt.test(n)?i(n,t):Pt(n+"["+("object"==typeof t&&null!=t?e:"")+"]",t,r,i)});else if(r||"object"!==x(e))i(n,e);else for(t in e)Pt(n+"["+t+"]",e[t],r,i)}ce.param=function(e,t){var n,r=[],i=function(e,t){var n=v(t)?t():t;r[r.length]=encodeURIComponent(e)+"="+encodeURIComponent(null==n?"":n)};if(null==e)return"";if(Array.isArray(e)||e.jquery&&!ce.isPlainObject(e))ce.each(e,function(){i(this.name,this.value)});else for(n in e)Pt(n,e[n],t,i);return r.join("&")},ce.fn.extend({serialize:function(){return ce.param(this.serializeArray())},serializeArray:function(){return this.map(function(){var e=ce.prop(this,"elements");return e?ce.makeArray(e):this}).filter(function(){var e=this.type;return this.name&&!ce(this).is(":disabled")&&Ot.test(this.nodeName)&&!Ht.test(e)&&(this.checked||!we.test(e))}).map(function(e,t){var n=ce(this).val();return null==n?null:Array.isArray(n)?ce.map(n,function(e){return{name:t.name,value:e.replace(Lt,"\r\n")}}):{name:t.name,value:n.replace(Lt,"\r\n")}}).get()}});var Mt=/%20/g,Rt=/#.*$/,It=/([?&])_=[^&]*/,Wt=/^(.*?):[ \t]*([^\r\n]*)$/gm,Ft=/^(?:GET|HEAD)$/,$t=/^\/\//,Bt={},_t={},zt="*/".concat("*"),Xt=C.createElement("a");function Ut(o){return function(e,t){"string"!=typeof e&&(t=e,e="*");var n,r=0,i=e.toLowerCase().match(D)||[];if(v(t))while(n=i[r++])"+"===n[0]?(n=n.slice(1)||"*",(o[n]=o[n]||[]).unshift(t)):(o[n]=o[n]||[]).push(t)}}function Vt(t,i,o,a){var s={},u=t===_t;function l(e){var r;return s[e]=!0,ce.each(t[e]||[],function(e,t){var n=t(i,o,a);return"string"!=typeof n||u||s[n]?u?!(r=n):void 0:(i.dataTypes.unshift(n),l(n),!1)}),r}return l(i.dataTypes[0])||!s["*"]&&l("*")}function Gt(e,t){var n,r,i=ce.ajaxSettings.flatOptions||{};for(n in t)void 0!==t[n]&&((i[n]?e:r||(r={}))[n]=t[n]);return r&&ce.extend(!0,e,r),e}Xt.href=Et.href,ce.extend({active:0,lastModified:{},etag:{},ajaxSettings:{url:Et.href,type:"GET",isLocal:/^(?:about|app|app-storage|.+-extension|file|res|widget):$/.test(Et.protocol),global:!0,processData:!0,async:!0,contentType:"application/x-www-form-urlencoded; charset=UTF-8",accepts:{"*":zt,text:"text/plain",html:"text/html",xml:"application/xml, text/xml",json:"application/json, text/javascript"},contents:{xml:/\bxml\b/,html:/\bhtml/,json:/\bjson\b/},responseFields:{xml:"responseXML",text:"responseText",json:"responseJSON"},converters:{"* text":String,"text html":!0,"text json":JSON.parse,"text xml":ce.parseXML},flatOptions:{url:!0,context:!0}},ajaxSetup:function(e,t){return t?Gt(Gt(e,ce.ajaxSettings),t):Gt(ce.ajaxSettings,e)},ajaxPrefilter:Ut(Bt),ajaxTransport:Ut(_t),ajax:function(e,t){"object"==typeof e&&(t=e,e=void 0),t=t||{};var c,f,p,n,d,r,h,g,i,o,v=ce.ajaxSetup({},t),y=v.context||v,m=v.context&&(y.nodeType||y.jquery)?ce(y):ce.event,x=ce.Deferred(),b=ce.Callbacks("once memory"),w=v.statusCode||{},a={},s={},u="canceled",T={readyState:0,getResponseHeader:function(e){var t;if(h){if(!n){n={};while(t=Wt.exec(p))n[t[1].toLowerCase()+" "]=(n[t[1].toLowerCase()+" "]||[]).concat(t[2])}t=n[e.toLowerCase()+" "]}return null==t?null:t.join(", ")},getAllResponseHeaders:function(){return h?p:null},setRequestHeader:function(e,t){return null==h&&(e=s[e.toLowerCase()]=s[e.toLowerCase()]||e,a[e]=t),this},overrideMimeType:function(e){return null==h&&(v.mimeType=e),this},statusCode:function(e){var t;if(e)if(h)T.always(e[T.status]);else for(t in e)w[t]=[w[t],e[t]];return this},abort:function(e){var t=e||u;return c&&c.abort(t),l(0,t),this}};if(x.promise(T),v.url=((e||v.url||Et.href)+"").replace($t,Et.protocol+"//"),v.type=t.method||t.type||v.method||v.type,v.dataTypes=(v.dataType||"*").toLowerCase().match(D)||[""],null==v.crossDomain){r=C.createElement("a");try{r.href=v.url,r.href=r.href,v.crossDomain=Xt.protocol+"//"+Xt.host!=r.protocol+"//"+r.host}catch(e){v.crossDomain=!0}}if(v.data&&v.processData&&"string"!=typeof v.data&&(v.data=ce.param(v.data,v.traditional)),Vt(Bt,v,t,T),h)return T;for(i in(g=ce.event&&v.global)&&0==ce.active++&&ce.event.trigger("ajaxStart"),v.type=v.type.toUpperCase(),v.hasContent=!Ft.test(v.type),f=v.url.replace(Rt,""),v.hasContent?v.data&&v.processData&&0===(v.contentType||"").indexOf("application/x-www-form-urlencoded")&&(v.data=v.data.replace(Mt,"+")):(o=v.url.slice(f.length),v.data&&(v.processData||"string"==typeof v.data)&&(f+=(At.test(f)?"&":"?")+v.data,delete v.data),!1===v.cache&&(f=f.replace(It,"$1"),o=(At.test(f)?"&":"?")+"_="+jt.guid+++o),v.url=f+o),v.ifModified&&(ce.lastModified[f]&&T.setRequestHeader("If-Modified-Since",ce.lastModified[f]),ce.etag[f]&&T.setRequestHeader("If-None-Match",ce.etag[f])),(v.data&&v.hasContent&&!1!==v.contentType||t.contentType)&&T.setRequestHeader("Content-Type",v.contentType),T.setRequestHeader("Accept",v.dataTypes[0]&&v.accepts[v.dataTypes[0]]?v.accepts[v.dataTypes[0]]+("*"!==v.dataTypes[0]?", "+zt+"; q=0.01":""):v.accepts["*"]),v.headers)T.setRequestHeader(i,v.headers[i]);if(v.beforeSend&&(!1===v.beforeSend.call(y,T,v)||h))return T.abort();if(u="abort",b.add(v.complete),T.done(v.success),T.fail(v.error),c=Vt(_t,v,t,T)){if(T.readyState=1,g&&m.trigger("ajaxSend",[T,v]),h)return T;v.async&&0<v.timeout&&(d=ie.setTimeout(function(){T.abort("timeout")},v.timeout));try{h=!1,c.send(a,l)}catch(e){if(h)throw e;l(-1,e)}}else l(-1,"No Transport");function l(e,t,n,r){var i,o,a,s,u,l=t;h||(h=!0,d&&ie.clearTimeout(d),c=void 0,p=r||"",T.readyState=0<e?4:0,i=200<=e&&e<300||304===e,n&&(s=function(e,t,n){var r,i,o,a,s=e.contents,u=e.dataTypes;while("*"===u[0])u.shift(),void 0===r&&(r=e.mimeType||t.getResponseHeader("Content-Type"));if(r)for(i in s)if(s[i]&&s[i].test(r)){u.unshift(i);break}if(u[0]in n)o=u[0];else{for(i in n){if(!u[0]||e.converters[i+" "+u[0]]){o=i;break}a||(a=i)}o=o||a}if(o)return o!==u[0]&&u.unshift(o),n[o]}(v,T,n)),!i&&-1<ce.inArray("script",v.dataTypes)&&ce.inArray("json",v.dataTypes)<0&&(v.converters["text script"]=function(){}),s=function(e,t,n,r){var i,o,a,s,u,l={},c=e.dataTypes.slice();if(c[1])for(a in e.converters)l[a.toLowerCase()]=e.converters[a];o=c.shift();while(o)if(e.responseFields[o]&&(n[e.responseFields[o]]=t),!u&&r&&e.dataFilter&&(t=e.dataFilter(t,e.dataType)),u=o,o=c.shift())if("*"===o)o=u;else if("*"!==u&&u!==o){if(!(a=l[u+" "+o]||l["* "+o]))for(i in l)if((s=i.split(" "))[1]===o&&(a=l[u+" "+s[0]]||l["* "+s[0]])){!0===a?a=l[i]:!0!==l[i]&&(o=s[0],c.unshift(s[1]));break}if(!0!==a)if(a&&e["throws"])t=a(t);else try{t=a(t)}catch(e){return{state:"parsererror",error:a?e:"No conversion from "+u+" to "+o}}}return{state:"success",data:t}}(v,s,T,i),i?(v.ifModified&&((u=T.getResponseHeader("Last-Modified"))&&(ce.lastModified[f]=u),(u=T.getResponseHeader("etag"))&&(ce.etag[f]=u)),204===e||"HEAD"===v.type?l="nocontent":304===e?l="notmodified":(l=s.state,o=s.data,i=!(a=s.error))):(a=l,!e&&l||(l="error",e<0&&(e=0))),T.status=e,T.statusText=(t||l)+"",i?x.resolveWith(y,[o,l,T]):x.rejectWith(y,[T,l,a]),T.statusCode(w),w=void 0,g&&m.trigger(i?"ajaxSuccess":"ajaxError",[T,v,i?o:a]),b.fireWith(y,[T,l]),g&&(m.trigger("ajaxComplete",[T,v]),--ce.active||ce.event.trigger("ajaxStop")))}return T},getJSON:function(e,t,n){return ce.get(e,t,n,"json")},getScript:function(e,t){return ce.get(e,void 0,t,"script")}}),ce.each(["get","post"],function(e,i){ce[i]=function(e,t,n,r){return v(t)&&(r=r||n,n=t,t=void 0),ce.ajax(ce.extend({url:e,type:i,dataType:r,data:t,success:n},ce.isPlainObject(e)&&e))}}),ce.ajaxPrefilter(function(e){var t;for(t in e.headers)"content-type"===t.toLowerCase()&&(e.contentType=e.headers[t]||"")}),ce._evalUrl=function(e,t,n){return ce.ajax({url:e,type:"GET",dataType:"script",cache:!0,async:!1,global:!1,converters:{"text script":function(){}},dataFilter:function(e){ce.globalEval(e,t,n)}})},ce.fn.extend({wrapAll:function(e){var t;return this[0]&&(v(e)&&(e=e.call(this[0])),t=ce(e,this[0].ownerDocument).eq(0).clone(!0),this[0].parentNode&&t.insertBefore(this[0]),t.map(function(){var e=this;while(e.firstElementChild)e=e.firstElementChild;return e}).append(this)),this},wrapInner:function(n){return v(n)?this.each(function(e){ce(this).wrapInner(n.call(this,e))}):this.each(function(){var e=ce(this),t=e.contents();t.length?t.wrapAll(n):e.append(n)})},wrap:function(t){var n=v(t);return this.each(function(e){ce(this).wrapAll(n?t.call(this,e):t)})},unwrap:function(e){return this.parent(e).not("body").each(function(){ce(this).replaceWith(this.childNodes)}),this}}),ce.expr.pseudos.hidden=function(e){return!ce.expr.pseudos.visible(e)},ce.expr.pseudos.visible=function(e){return!!(e.offsetWidth||e.offsetHeight||e.getClientRects().length)},ce.ajaxSettings.xhr=function(){try{return new ie.XMLHttpRequest}catch(e){}};var Yt={0:200,1223:204},Qt=ce.ajaxSettings.xhr();le.cors=!!Qt&&"withCredentials"in Qt,le.ajax=Qt=!!Qt,ce.ajaxTransport(function(i){var o,a;if(le.cors||Qt&&!i.crossDomain)return{send:function(e,t){var n,r=i.xhr();if(r.open(i.type,i.url,i.async,i.username,i.password),i.xhrFields)for(n in i.xhrFields)r[n]=i.xhrFields[n];for(n in i.mimeType&&r.overrideMimeType&&r.overrideMimeType(i.mimeType),i.crossDomain||e["X-Requested-With"]||(e["X-Requested-With"]="XMLHttpRequest"),e)r.setRequestHeader(n,e[n]);o=function(e){return function(){o&&(o=a=r.onload=r.onerror=r.onabort=r.ontimeout=r.onreadystatechange=null,"abort"===e?r.abort():"error"===e?"number"!=typeof r.status?t(0,"error"):t(r.status,r.statusText):t(Yt[r.status]||r.status,r.statusText,"text"!==(r.responseType||"text")||"string"!=typeof r.responseText?{binary:r.response}:{text:r.responseText},r.getAllResponseHeaders()))}},r.onload=o(),a=r.onerror=r.ontimeout=o("error"),void 0!==r.onabort?r.onabort=a:r.onreadystatechange=function(){4===r.readyState&&ie.setTimeout(function(){o&&a()})},o=o("abort");try{r.send(i.hasContent&&i.data||null)}catch(e){if(o)throw e}},abort:function(){o&&o()}}}),ce.ajaxPrefilter(function(e){e.crossDomain&&(e.contents.script=!1)}),ce.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/\b(?:java|ecma)script\b/},converters:{"text script":function(e){return ce.globalEval(e),e}}}),ce.ajaxPrefilter("script",function(e){void 0===e.cache&&(e.cache=!1),e.crossDomain&&(e.type="GET")}),ce.ajaxTransport("script",function(n){var r,i;if(n.crossDomain||n.scriptAttrs)return{send:function(e,t){r=ce("<script>").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),C.head.appendChild(r[0])},abort:function(){i&&i()}}});var Jt,Kt=[],Zt=/(=)\?(?=&|$)|\?\?/;ce.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Kt.pop()||ce.expando+"_"+jt.guid++;return this[e]=!0,e}}),ce.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Zt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Zt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=v(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Zt,"$1"+r):!1!==e.jsonp&&(e.url+=(At.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||ce.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=ie[r],ie[r]=function(){o=arguments},n.always(function(){void 0===i?ce(ie).removeProp(r):ie[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Kt.push(r)),o&&v(i)&&i(o[0]),o=i=void 0}),"script"}),le.createHTMLDocument=((Jt=C.implementation.createHTMLDocument("").body).innerHTML="<form></form><form></form>",2===Jt.childNodes.length),ce.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(le.createHTMLDocument?((r=(t=C.implementation.createHTMLDocument("")).createElement("base")).href=C.location.href,t.head.appendChild(r)):t=C),o=!n&&[],(i=w.exec(e))?[t.createElement(i[1])]:(i=Ae([e],t,o),o&&o.length&&ce(o).remove(),ce.merge([],i.childNodes)));var r,i,o},ce.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1<s&&(r=Tt(e.slice(s)),e=e.slice(0,s)),v(t)?(n=t,t=void 0):t&&"object"==typeof t&&(i="POST"),0<a.length&&ce.ajax({url:e,type:i||"GET",dataType:"html",data:t}).done(function(e){o=arguments,a.html(r?ce("<div>").append(ce.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},ce.expr.pseudos.animated=function(t){return ce.grep(ce.timers,function(e){return t===e.elem}).length},ce.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=ce.css(e,"position"),c=ce(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=ce.css(e,"top"),u=ce.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),v(t)&&(t=t.call(e,n,ce.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},ce.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){ce.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===ce.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===ce.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=ce(e).offset()).top+=ce.css(e,"borderTopWidth",!0),i.left+=ce.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-ce.css(r,"marginTop",!0),left:t.left-i.left-ce.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===ce.css(e,"position"))e=e.offsetParent;return e||J})}}),ce.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;ce.fn[t]=function(e){return M(this,function(e,t,n){var r;if(y(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),ce.each(["top","left"],function(e,n){ce.cssHooks[n]=Ye(le.pixelPosition,function(e,t){if(t)return t=Ge(e,n),_e.test(t)?ce(e).position()[n]+"px":t})}),ce.each({Height:"height",Width:"width"},function(a,s){ce.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){ce.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return M(this,function(e,t,n){var r;return y(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?ce.css(e,t,i):ce.style(e,t,n,i)},s,n?e:void 0,n)}})}),ce.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){ce.fn[t]=function(e){return this.on(t,e)}}),ce.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.on("mouseenter",e).on("mouseleave",t||e)}}),ce.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){ce.fn[n]=function(e,t){return 0<arguments.length?this.on(n,null,e,t):this.trigger(n)}});var en=/^[\s\uFEFF\xA0]+|([^\s\uFEFF\xA0])[\s\uFEFF\xA0]+$/g;ce.proxy=function(e,t){var n,r,i;if("string"==typeof t&&(n=e[t],t=e,e=n),v(e))return r=ae.call(arguments,2),(i=function(){return e.apply(t||this,r.concat(ae.call(arguments)))}).guid=e.guid=e.guid||ce.guid++,i},ce.holdReady=function(e){e?ce.readyWait++:ce.ready(!0)},ce.isArray=Array.isArray,ce.parseJSON=JSON.parse,ce.nodeName=fe,ce.isFunction=v,ce.isWindow=y,ce.camelCase=F,ce.type=x,ce.now=Date.now,ce.isNumeric=function(e){var t=ce.type(e);return("number"===t||"string"===t)&&!isNaN(e-parseFloat(e))},ce.trim=function(e){return null==e?"":(e+"").replace(en,"$1")},"function"==typeof define&&define.amd&&define("jquery",[],function(){return ce});var tn=ie.jQuery,nn=ie.$;return ce.noConflict=function(e){return ie.$===ce&&(ie.$=nn),e&&ie.jQuery===ce&&(ie.jQuery=tn),ce},"undefined"==typeof e&&(ie.jQuery=ie.$=ce),ce});
diff --git a/bitbake/lib/toaster/toastergui/static/js/jquery-3.7.1.min.map b/bitbake/lib/toaster/toastergui/static/js/jquery-3.7.1.min.map
new file mode 100644
index 0000000000..db38af5893
--- /dev/null
+++ b/bitbake/lib/toaster/toastergui/static/js/jquery-3.7.1.min.map
@@ -0,0 +1 @@
+{"version":3,"sources":["jquery-3.7.1.js"],"names":["global","factory","module","exports","document","w","Error","window","this","noGlobal","arr","getProto","Object","getPrototypeOf","slice","flat","array","call","concat","apply","push","indexOf","class2type","toString","hasOwn","hasOwnProperty","fnToString","ObjectFunctionString","support","isFunction","obj","nodeType","item","isWindow","preservedScriptAttributes","type","src","nonce","noModule","DOMEval","code","node","doc","i","val","script","createElement","text","getAttribute","setAttribute","head","appendChild","parentNode","removeChild","toType","version","rhtmlSuffix","jQuery","selector","context","fn","init","isArrayLike","length","nodeName","elem","name","toLowerCase","prototype","jquery","constructor","toArray","get","num","pushStack","elems","ret","merge","prevObject","each","callback","map","arguments","first","eq","last","even","grep","_elem","odd","len","j","end","sort","splice","extend","options","copy","copyIsArray","clone","target","deep","isPlainObject","Array","isArray","undefined","expando","Math","random","replace","isReady","error","msg","noop","proto","Ctor","isEmptyObject","globalEval","textContent","documentElement","nodeValue","makeArray","results","inArray","isXMLDoc","namespace","namespaceURI","docElem","ownerDocument","test","second","invert","matches","callbackExpect","arg","value","guid","Symbol","iterator","split","_i","pop","whitespace","rtrimCSS","RegExp","contains","a","b","bup","compareDocumentPosition","rcssescape","fcssescape","ch","asCodePoint","charCodeAt","escapeSelector","sel","preferredDoc","pushNative","Expr","outermostContext","sortInput","hasDuplicate","documentIsHTML","rbuggyQSA","dirruns","done","classCache","createCache","tokenCache","compilerCache","nonnativeSelectorCache","sortOrder","booleans","identifier","attributes","pseudos","rwhitespace","rcomma","rleadingCombinator","rdescend","rpseudo","ridentifier","matchExpr","ID","CLASS","TAG","ATTR","PSEUDO","CHILD","bool","needsContext","rinputs","rheader","rquickExpr","rsibling","runescape","funescape","escape","nonHex","high","String","fromCharCode","unloadHandler","setDocument","inDisabledFieldset","addCombinator","disabled","dir","next","childNodes","e","els","find","seed","m","nid","match","groups","newSelector","newContext","exec","getElementById","id","getElementsByTagName","getElementsByClassName","testContext","scope","tokenize","toSelector","join","querySelectorAll","qsaError","removeAttribute","select","keys","cache","key","cacheLength","shift","markFunction","assert","el","createInputPseudo","createButtonPseudo","createDisabledPseudo","isDisabled","createPositionalPseudo","argument","matchIndexes","subWindow","webkitMatchesSelector","msMatchesSelector","defaultView","top","addEventListener","getById","getElementsByName","disconnectedMatch","cssHas","querySelector","filter","attrId","getAttributeNode","tag","className","input","innerHTML","compare","sortDetached","expr","elements","matchesSelector","attr","attrHandle","uniqueSort","duplicates","sortStable","createPseudo","relative",">"," ","+","~","preFilter","excess","unquoted","nodeNameSelector","expectedNodeName","pattern","operator","check","result","what","_argument","simple","forward","ofType","_context","xml","outerCache","nodeIndex","start","parent","useCache","diff","firstChild","lastChild","pseudo","args","setFilters","idx","matched","not","matcher","compile","unmatched","has","lang","elemLang","hash","location","root","focus","activeElement","err","safeActiveElement","hasFocus","href","tabIndex","enabled","checked","selected","selectedIndex","empty","nextSibling","header","button","_matchIndexes","lt","gt","nth","radio","checkbox","file","password","image","submit","reset","parseOnly","tokens","soFar","preFilters","cached","combinator","base","skip","checkNonElements","doneName","oldCache","newCache","elementMatcher","matchers","condense","newUnmatched","mapped","setMatcher","postFilter","postFinder","postSelector","temp","matcherOut","preMap","postMap","preexisting","contexts","multipleContexts","matcherIn","matcherFromTokens","checkContext","leadingRelative","implicitRelative","matchContext","matchAnyContext","elementMatchers","setMatchers","bySet","byElement","superMatcher","outermost","matchedCount","setMatched","contextBackup","dirrunsUnique","token","compiled","filters","unique","getText","isXML","selectors","until","truncate","is","siblings","n","rneedsContext","rsingleTag","winnow","qualifier","self","rootjQuery","parseHTML","ready","rparentsprev","guaranteedUnique","children","contents","prev","sibling","cur","targets","l","closest","index","prevAll","add","addBack","parents","parentsUntil","nextAll","nextUntil","prevUntil","contentDocument","content","reverse","rnothtmlwhite","Identity","v","Thrower","ex","adoptValue","resolve","reject","noValue","method","promise","fail","then","Callbacks","object","_","flag","firing","memory","fired","locked","list","queue","firingIndex","fire","once","stopOnFalse","remove","disable","lock","fireWith","Deferred","func","tuples","state","always","deferred","catch","pipe","fns","newDefer","tuple","returned","progress","notify","onFulfilled","onRejected","onProgress","maxDepth","depth","handler","special","that","mightThrow","TypeError","notifyWith","resolveWith","process","exceptionHook","rejectWith","getErrorHook","getStackHook","setTimeout","stateString","when","singleValue","remaining","resolveContexts","resolveValues","primary","updateFunc","rerrorNames","asyncError","console","warn","message","stack","readyException","readyList","completed","removeEventListener","readyWait","wait","readyState","doScroll","access","chainable","emptyGet","raw","bulk","_key","rmsPrefix","rdashAlpha","fcamelCase","_all","letter","toUpperCase","camelCase","string","acceptData","owner","Data","uid","defineProperty","configurable","set","data","prop","hasData","dataPriv","dataUser","rbrace","rmultiDash","dataAttr","JSON","parse","removeData","_data","_removeData","attrs","dequeue","startLength","hooks","_queueHooks","unshift","stop","setter","clearQueue","tmp","count","defer","pnum","source","rcssNum","cssExpand","isAttached","composed","getRootNode","isHiddenWithinTree","style","display","css","adjustCSS","valueParts","tween","adjusted","scale","maxIterations","currentValue","initial","unit","cssNumber","initialInUnit","defaultDisplayMap","showHide","show","values","body","hide","toggle","div","rcheckableType","rtagName","rscriptType","createDocumentFragment","checkClone","cloneNode","noCloneChecked","defaultValue","option","wrapMap","thead","col","tr","td","_default","getAll","setGlobalEval","refElements","tbody","tfoot","colgroup","caption","th","optgroup","rhtml","buildFragment","scripts","selection","ignored","wrap","attached","fragment","nodes","htmlPrefilter","createTextNode","rtypenamespace","returnTrue","returnFalse","on","types","one","origFn","event","off","leverageNative","isSetup","saved","isTrigger","delegateType","stopPropagation","stopImmediatePropagation","preventDefault","trigger","isImmediatePropagationStopped","handleObjIn","eventHandle","events","t","handleObj","handlers","namespaces","origType","elemData","create","handle","triggered","dispatch","bindType","delegateCount","setup","mappedTypes","origCount","teardown","removeEvent","nativeEvent","handlerQueue","fix","delegateTarget","preDispatch","isPropagationStopped","currentTarget","rnamespace","postDispatch","matchedHandlers","matchedSelectors","addProp","hook","Event","enumerable","originalEvent","writable","load","noBubble","click","beforeunload","returnValue","props","isDefaultPrevented","defaultPrevented","relatedTarget","timeStamp","Date","now","isSimulated","altKey","bubbles","cancelable","changedTouches","ctrlKey","detail","eventPhase","metaKey","pageX","pageY","shiftKey","view","char","charCode","keyCode","buttons","clientX","clientY","offsetX","offsetY","pointerId","pointerType","screenX","screenY","targetTouches","toElement","touches","which","blur","focusMappedHandler","documentMode","simulate","attaches","dataHolder","mouseenter","mouseleave","pointerenter","pointerleave","orig","related","rnoInnerhtml","rchecked","rcleanScript","manipulationTarget","disableScript","restoreScript","cloneCopyEvent","dest","udataOld","udataCur","domManip","collection","hasScripts","iNoClone","valueIsFunction","html","_evalUrl","keepData","cleanData","dataAndEvents","deepDataAndEvents","srcElements","destElements","inPage","detach","append","prepend","insertBefore","before","after","replaceWith","replaceChild","appendTo","prependTo","insertAfter","replaceAll","original","insert","rnumnonpx","rcustomProp","getStyles","opener","getComputedStyle","swap","old","rboxStyle","curCSS","computed","width","minWidth","maxWidth","isCustomProp","getPropertyValue","pixelBoxStyles","addGetHookIf","conditionFn","hookFn","computeStyleTests","container","cssText","divStyle","pixelPositionVal","reliableMarginLeftVal","roundPixelMeasures","marginLeft","right","pixelBoxStylesVal","boxSizingReliableVal","position","scrollboxSizeVal","offsetWidth","measure","round","parseFloat","reliableTrDimensionsVal","backgroundClip","clearCloneStyle","boxSizingReliable","pixelPosition","reliableMarginLeft","scrollboxSize","reliableTrDimensions","table","trChild","trStyle","height","parseInt","borderTopWidth","borderBottomWidth","offsetHeight","cssPrefixes","emptyStyle","vendorProps","finalPropName","final","cssProps","capName","vendorPropName","rdisplayswap","cssShow","visibility","cssNormalTransform","letterSpacing","fontWeight","setPositiveNumber","subtract","max","boxModelAdjustment","dimension","box","isBorderBox","styles","computedVal","extra","delta","marginDelta","ceil","getWidthOrHeight","valueIsBorderBox","offsetProp","getClientRects","Tween","easing","cssHooks","opacity","animationIterationCount","aspectRatio","borderImageSlice","columnCount","flexGrow","flexShrink","gridArea","gridColumn","gridColumnEnd","gridColumnStart","gridRow","gridRowEnd","gridRowStart","lineHeight","order","orphans","widows","zIndex","zoom","fillOpacity","floodOpacity","stopOpacity","strokeMiterlimit","strokeOpacity","origName","setProperty","isFinite","getBoundingClientRect","scrollboxSizeBuggy","left","margin","padding","border","prefix","suffix","expand","expanded","parts","propHooks","run","percent","eased","duration","pos","step","fx","scrollTop","scrollLeft","linear","p","swing","cos","PI","fxNow","inProgress","opt","rfxtypes","rrun","schedule","hidden","requestAnimationFrame","interval","tick","createFxNow","genFx","includeWidth","createTween","animation","Animation","tweeners","properties","stopped","prefilters","currentTime","startTime","tweens","opts","specialEasing","originalProperties","originalOptions","gotoEnd","propFilter","bind","complete","timer","anim","*","tweener","oldfire","propTween","restoreDisplay","isBox","dataShow","unqueued","overflow","overflowX","overflowY","prefilter","speed","speeds","fadeTo","to","animate","optall","doAnimation","finish","stopQueue","timers","cssFn","slideDown","slideUp","slideToggle","fadeIn","fadeOut","fadeToggle","slow","fast","delay","time","timeout","clearTimeout","checkOn","optSelected","radioValue","boolHook","removeAttr","nType","attrHooks","attrNames","getter","lowercaseName","rfocusable","rclickable","stripAndCollapse","getClass","classesToArray","removeProp","propFix","tabindex","for","class","addClass","classNames","curValue","finalValue","removeClass","toggleClass","stateVal","isValidValue","hasClass","rreturn","valHooks","optionSet","rquery","parseXML","parserErrorElem","DOMParser","parseFromString","rfocusMorph","stopPropagationCallback","onlyHandlers","bubbleType","ontype","lastElement","eventPath","parentWindow","triggerHandler","rbracket","rCRLF","rsubmitterTypes","rsubmittable","buildParams","traditional","param","s","valueOrFunction","encodeURIComponent","serialize","serializeArray","r20","rhash","rantiCache","rheaders","rnoContent","rprotocol","transports","allTypes","originAnchor","addToPrefiltersOrTransports","structure","dataTypeExpression","dataType","dataTypes","inspectPrefiltersOrTransports","jqXHR","inspected","seekingTransport","inspect","prefilterOrFactory","dataTypeOrTransport","ajaxExtend","flatOptions","ajaxSettings","active","lastModified","etag","url","isLocal","protocol","processData","async","contentType","accepts","json","responseFields","converters","* text","text html","text json","text xml","ajaxSetup","settings","ajaxPrefilter","ajaxTransport","ajax","transport","cacheURL","responseHeadersString","responseHeaders","timeoutTimer","urlAnchor","fireGlobals","uncached","callbackContext","globalEventContext","completeDeferred","statusCode","requestHeaders","requestHeadersNames","strAbort","getResponseHeader","getAllResponseHeaders","setRequestHeader","overrideMimeType","mimeType","status","abort","statusText","finalText","crossDomain","host","hasContent","ifModified","headers","beforeSend","success","send","nativeStatusText","responses","isSuccess","response","modified","ct","finalDataType","firstDataType","ajaxHandleResponses","conv2","current","conv","dataFilter","throws","ajaxConvert","getJSON","getScript","text script","wrapAll","firstElementChild","wrapInner","htmlIsFunction","unwrap","visible","xhr","XMLHttpRequest","xhrSuccessStatus","0","1223","xhrSupported","cors","errorCallback","open","username","xhrFields","onload","onerror","onabort","ontimeout","onreadystatechange","responseType","responseText","binary","scriptAttrs","charset","scriptCharset","evt","oldCallbacks","rjsonp","jsonp","jsonpCallback","originalSettings","callbackName","overwritten","responseContainer","jsonProp","createHTMLDocument","implementation","keepScripts","parsed","params","animated","offset","setOffset","curPosition","curLeft","curCSSTop","curTop","curOffset","curCSSLeft","curElem","using","rect","win","pageYOffset","pageXOffset","offsetParent","parentOffset","scrollTo","Height","Width","","defaultExtra","funcName","unbind","delegate","undelegate","hover","fnOver","fnOut","rtrim","proxy","holdReady","hold","parseJSON","isNumeric","isNaN","trim","define","amd","_jQuery","_$","$","noConflict"],"mappings":";CAUA,SAAYA,EAAQC,GAEnB,aAEuB,iBAAXC,QAAiD,iBAAnBA,OAAOC,QAShDD,OAAOC,QAAUH,EAAOI,SACvBH,EAASD,GAAQ,GACjB,SAAUK,GACT,IAAMA,EAAED,SACP,MAAM,IAAIE,MAAO,4CAElB,OAAOL,EAASI,IAGlBJ,EAASD,GAtBX,CA0BuB,oBAAXO,OAAyBA,OAASC,KAAM,SAAUD,GAAQE,GAMtE,aAEA,IAAIC,GAAM,GAENC,EAAWC,OAAOC,eAElBC,GAAQJ,GAAII,MAEZC,EAAOL,GAAIK,KAAO,SAAUC,GAC/B,OAAON,GAAIK,KAAKE,KAAMD,IACnB,SAAUA,GACb,OAAON,GAAIQ,OAAOC,MAAO,GAAIH,IAI1BI,EAAOV,GAAIU,KAEXC,GAAUX,GAAIW,QAEdC,EAAa,GAEbC,EAAWD,EAAWC,SAEtBC,GAASF,EAAWG,eAEpBC,EAAaF,GAAOD,SAEpBI,EAAuBD,EAAWT,KAAML,QAExCgB,GAAU,GAEVC,EAAa,SAAqBC,GASpC,MAAsB,mBAARA,GAA8C,iBAAjBA,EAAIC,UAC1B,mBAAbD,EAAIE,MAIVC,EAAW,SAAmBH,GAChC,OAAc,MAAPA,GAAeA,IAAQA,EAAIvB,QAIhCH,EAAWG,GAAOH,SAIjB8B,EAA4B,CAC/BC,MAAM,EACNC,KAAK,EACLC,OAAO,EACPC,UAAU,GAGX,SAASC,EAASC,EAAMC,EAAMC,GAG7B,IAAIC,EAAGC,EACNC,GAHDH,EAAMA,GAAOtC,GAGC0C,cAAe,UAG7B,GADAD,EAAOE,KAAOP,EACTC,EACJ,IAAME,KAAKT,GAYVU,EAAMH,EAAME,IAAOF,EAAKO,cAAgBP,EAAKO,aAAcL,KAE1DE,EAAOI,aAAcN,EAAGC,GAI3BF,EAAIQ,KAAKC,YAAaN,GAASO,WAAWC,YAAaR,GAIzD,SAASS,EAAQxB,GAChB,OAAY,MAAPA,EACGA,EAAM,GAIQ,iBAARA,GAAmC,mBAARA,EACxCR,EAAYC,EAASN,KAAMa,KAAW,gBAC/BA,EAQT,IAAIyB,EAAU,QAEbC,EAAc,SAGdC,GAAS,SAAUC,EAAUC,GAI5B,OAAO,IAAIF,GAAOG,GAAGC,KAAMH,EAAUC,IAmYvC,SAASG,EAAahC,GAMrB,IAAIiC,IAAWjC,GAAO,WAAYA,GAAOA,EAAIiC,OAC5C5B,EAAOmB,EAAQxB,GAEhB,OAAKD,EAAYC,KAASG,EAAUH,KAIpB,UAATK,GAA+B,IAAX4B,GACR,iBAAXA,GAAgC,EAATA,GAAgBA,EAAS,KAAOjC,GAIhE,SAASkC,GAAUC,EAAMC,GAExB,OAAOD,EAAKD,UAAYC,EAAKD,SAASG,gBAAkBD,EAAKC,cApZ9DV,GAAOG,GAAKH,GAAOW,UAAY,CAG9BC,OAAQd,EAERe,YAAab,GAGbM,OAAQ,EAERQ,QAAS,WACR,OAAOzD,GAAMG,KAAMT,OAKpBgE,IAAK,SAAUC,GAGd,OAAY,MAAPA,EACG3D,GAAMG,KAAMT,MAIbiE,EAAM,EAAIjE,KAAMiE,EAAMjE,KAAKuD,QAAWvD,KAAMiE,IAKpDC,UAAW,SAAUC,GAGpB,IAAIC,EAAMnB,GAAOoB,MAAOrE,KAAK8D,cAAeK,GAM5C,OAHAC,EAAIE,WAAatE,KAGVoE,GAIRG,KAAM,SAAUC,GACf,OAAOvB,GAAOsB,KAAMvE,KAAMwE,IAG3BC,IAAK,SAAUD,GACd,OAAOxE,KAAKkE,UAAWjB,GAAOwB,IAAKzE,KAAM,SAAUyD,EAAMtB,GACxD,OAAOqC,EAAS/D,KAAMgD,EAAMtB,EAAGsB,OAIjCnD,MAAO,WACN,OAAON,KAAKkE,UAAW5D,GAAMK,MAAOX,KAAM0E,aAG3CC,MAAO,WACN,OAAO3E,KAAK4E,GAAI,IAGjBC,KAAM,WACL,OAAO7E,KAAK4E,IAAK,IAGlBE,KAAM,WACL,OAAO9E,KAAKkE,UAAWjB,GAAO8B,KAAM/E,KAAM,SAAUgF,EAAO7C,GAC1D,OAASA,EAAI,GAAM,MAIrB8C,IAAK,WACJ,OAAOjF,KAAKkE,UAAWjB,GAAO8B,KAAM/E,KAAM,SAAUgF,EAAO7C,GAC1D,OAAOA,EAAI,MAIbyC,GAAI,SAAUzC,GACb,IAAI+C,EAAMlF,KAAKuD,OACd4B,GAAKhD,GAAMA,EAAI,EAAI+C,EAAM,GAC1B,OAAOlF,KAAKkE,UAAgB,GAALiB,GAAUA,EAAID,EAAM,CAAElF,KAAMmF,IAAQ,KAG5DC,IAAK,WACJ,OAAOpF,KAAKsE,YAActE,KAAK8D,eAKhClD,KAAMA,EACNyE,KAAMnF,GAAImF,KACVC,OAAQpF,GAAIoF,QAGbrC,GAAOsC,OAAStC,GAAOG,GAAGmC,OAAS,WAClC,IAAIC,EAAS9B,EAAM9B,EAAK6D,EAAMC,EAAaC,EAC1CC,EAASlB,UAAW,IAAO,GAC3BvC,EAAI,EACJoB,EAASmB,UAAUnB,OACnBsC,GAAO,EAsBR,IAnBuB,kBAAXD,IACXC,EAAOD,EAGPA,EAASlB,UAAWvC,IAAO,GAC3BA,KAIsB,iBAAXyD,GAAwBvE,EAAYuE,KAC/CA,EAAS,IAILzD,IAAMoB,IACVqC,EAAS5F,KACTmC,KAGOA,EAAIoB,EAAQpB,IAGnB,GAAqC,OAA9BqD,EAAUd,UAAWvC,IAG3B,IAAMuB,KAAQ8B,EACbC,EAAOD,EAAS9B,GAIF,cAATA,GAAwBkC,IAAWH,IAKnCI,GAAQJ,IAAUxC,GAAO6C,cAAeL,KAC1CC,EAAcK,MAAMC,QAASP,MAC/B7D,EAAMgE,EAAQlC,GAIbiC,EADID,IAAgBK,MAAMC,QAASpE,GAC3B,GACI8D,GAAgBzC,GAAO6C,cAAelE,GAG1CA,EAFA,GAIT8D,GAAc,EAGdE,EAAQlC,GAAST,GAAOsC,OAAQM,EAAMF,EAAOF,SAGzBQ,IAATR,IACXG,EAAQlC,GAAS+B,IAOrB,OAAOG,GAGR3C,GAAOsC,OAAQ,CAGdW,QAAS,UAAanD,EAAUoD,KAAKC,UAAWC,QAAS,MAAO,IAGhEC,SAAS,EAETC,MAAO,SAAUC,GAChB,MAAM,IAAI1G,MAAO0G,IAGlBC,KAAM,aAENX,cAAe,SAAUxE,GACxB,IAAIoF,EAAOC,EAIX,SAAMrF,GAAgC,oBAAzBP,EAASN,KAAMa,QAI5BoF,EAAQvG,EAAUmB,KASK,mBADvBqF,EAAO3F,GAAOP,KAAMiG,EAAO,gBAAmBA,EAAM5C,cACf5C,EAAWT,KAAMkG,KAAWxF,IAGlEyF,cAAe,SAAUtF,GACxB,IAAIoC,EAEJ,IAAMA,KAAQpC,EACb,OAAO,EAER,OAAO,GAKRuF,WAAY,SAAU7E,EAAMwD,EAAStD,GACpCH,EAASC,EAAM,CAAEH,MAAO2D,GAAWA,EAAQ3D,OAASK,IAGrDqC,KAAM,SAAUjD,EAAKkD,GACpB,IAAIjB,EAAQpB,EAAI,EAEhB,GAAKmB,EAAahC,IAEjB,IADAiC,EAASjC,EAAIiC,OACLpB,EAAIoB,EAAQpB,IACnB,IAAgD,IAA3CqC,EAAS/D,KAAMa,EAAKa,GAAKA,EAAGb,EAAKa,IACrC,WAIF,IAAMA,KAAKb,EACV,IAAgD,IAA3CkD,EAAS/D,KAAMa,EAAKa,GAAKA,EAAGb,EAAKa,IACrC,MAKH,OAAOb,GAKRiB,KAAM,SAAUkB,GACf,IAAIxB,EACHmC,EAAM,GACNjC,EAAI,EACJZ,EAAWkC,EAAKlC,SAEjB,IAAMA,EAGL,MAAUU,EAAOwB,EAAMtB,KAGtBiC,GAAOnB,GAAOV,KAAMN,GAGtB,OAAkB,IAAbV,GAA+B,KAAbA,EACfkC,EAAKqD,YAEK,IAAbvF,EACGkC,EAAKsD,gBAAgBD,YAEX,IAAbvF,GAA+B,IAAbA,EACfkC,EAAKuD,UAKN5C,GAIR6C,UAAW,SAAU/G,EAAKgH,GACzB,IAAI9C,EAAM8C,GAAW,GAarB,OAXY,MAAPhH,IACCoD,EAAalD,OAAQF,IACzB+C,GAAOoB,MAAOD,EACE,iBAARlE,EACN,CAAEA,GAAQA,GAGZU,EAAKH,KAAM2D,EAAKlE,IAIXkE,GAGR+C,QAAS,SAAU1D,EAAMvD,EAAKiC,GAC7B,OAAc,MAAPjC,GAAe,EAAIW,GAAQJ,KAAMP,EAAKuD,EAAMtB,IAGpDiF,SAAU,SAAU3D,GACnB,IAAI4D,EAAY5D,GAAQA,EAAK6D,aAC5BC,EAAU9D,IAAUA,EAAK+D,eAAiB/D,GAAOsD,gBAIlD,OAAQ/D,EAAYyE,KAAMJ,GAAaE,GAAWA,EAAQ/D,UAAY,SAKvEa,MAAO,SAAUM,EAAO+C,GAKvB,IAJA,IAAIxC,GAAOwC,EAAOnE,OACjB4B,EAAI,EACJhD,EAAIwC,EAAMpB,OAEH4B,EAAID,EAAKC,IAChBR,EAAOxC,KAAQuF,EAAQvC,GAKxB,OAFAR,EAAMpB,OAASpB,EAERwC,GAGRI,KAAM,SAAUZ,EAAOK,EAAUmD,GAShC,IARA,IACCC,EAAU,GACVzF,EAAI,EACJoB,EAASY,EAAMZ,OACfsE,GAAkBF,EAIXxF,EAAIoB,EAAQpB,KACAqC,EAAUL,EAAOhC,GAAKA,KAChB0F,GACxBD,EAAQhH,KAAMuD,EAAOhC,IAIvB,OAAOyF,GAIRnD,IAAK,SAAUN,EAAOK,EAAUsD,GAC/B,IAAIvE,EAAQwE,EACX5F,EAAI,EACJiC,EAAM,GAGP,GAAKd,EAAaa,GAEjB,IADAZ,EAASY,EAAMZ,OACPpB,EAAIoB,EAAQpB,IAGL,OAFd4F,EAAQvD,EAAUL,EAAOhC,GAAKA,EAAG2F,KAGhC1D,EAAIxD,KAAMmH,QAMZ,IAAM5F,KAAKgC,EAGI,OAFd4D,EAAQvD,EAAUL,EAAOhC,GAAKA,EAAG2F,KAGhC1D,EAAIxD,KAAMmH,GAMb,OAAOxH,EAAM6D,IAId4D,KAAM,EAIN5G,QAASA,KAGa,mBAAX6G,SACXhF,GAAOG,GAAI6E,OAAOC,UAAahI,GAAK+H,OAAOC,WAI5CjF,GAAOsB,KAAM,uEAAuE4D,MAAO,KAC1F,SAAUC,EAAI1E,GACb5C,EAAY,WAAa4C,EAAO,KAAQA,EAAKC,gBA0B/C,IAAI0E,GAAMnI,GAAImI,IAGVhD,GAAOnF,GAAImF,KAGXC,GAASpF,GAAIoF,OAGbgD,GAAa,sBAGbC,GAAW,IAAIC,OAClB,IAAMF,GAAa,8BAAgCA,GAAa,KAChE,KAODrF,GAAOwF,SAAW,SAAUC,EAAGC,GAC9B,IAAIC,EAAMD,GAAKA,EAAE/F,WAEjB,OAAO8F,IAAME,MAAWA,GAAwB,IAAjBA,EAAIrH,YAIlCmH,EAAED,SACDC,EAAED,SAAUG,GACZF,EAAEG,yBAA8D,GAAnCH,EAAEG,wBAAyBD,MAS3D,IAAIE,EAAa,+CAEjB,SAASC,EAAYC,EAAIC,GACxB,OAAKA,EAGQ,OAAPD,EACG,SAIDA,EAAG1I,MAAO,GAAI,GAAM,KAAO0I,EAAGE,WAAYF,EAAGzF,OAAS,GAAIxC,SAAU,IAAO,IAI5E,KAAOiI,EAGf/F,GAAOkG,eAAiB,SAAUC,GACjC,OAASA,EAAM,IAAK/C,QAASyC,EAAYC,IAM1C,IAAIM,GAAezJ,EAClB0J,GAAa1I,GAEd,WAEA,IAAIuB,EACHoH,EACAC,EACAC,EACAC,EAIA9J,EACAmH,EACA4C,EACAC,EACAhC,EAPAhH,EAAO0I,GAUPpD,EAAUjD,GAAOiD,QACjB2D,EAAU,EACVC,EAAO,EACPC,EAAaC,IACbC,EAAaD,IACbE,EAAgBF,IAChBG,EAAyBH,IACzBI,EAAY,SAAU1B,EAAGC,GAIxB,OAHKD,IAAMC,IACVe,GAAe,GAET,GAGRW,EAAW,6HAMXC,EAAa,0BAA4BhC,GACxC,0CAGDiC,EAAa,MAAQjC,GAAa,KAAOgC,EAAa,OAAShC,GAG9D,gBAAkBA,GAGlB,2DAA6DgC,EAAa,OAC1EhC,GAAa,OAEdkC,EAAU,KAAOF,EAAa,wFAOAC,EAAa,eAO3CE,EAAc,IAAIjC,OAAQF,GAAa,IAAK,KAE5CoC,EAAS,IAAIlC,OAAQ,IAAMF,GAAa,KAAOA,GAAa,KAC5DqC,EAAqB,IAAInC,OAAQ,IAAMF,GAAa,WAAaA,GAAa,IAC7EA,GAAa,KACdsC,EAAW,IAAIpC,OAAQF,GAAa,MAEpCuC,EAAU,IAAIrC,OAAQgC,GACtBM,EAAc,IAAItC,OAAQ,IAAM8B,EAAa,KAE7CS,EAAY,CACXC,GAAI,IAAIxC,OAAQ,MAAQ8B,EAAa,KACrCW,MAAO,IAAIzC,OAAQ,QAAU8B,EAAa,KAC1CY,IAAK,IAAI1C,OAAQ,KAAO8B,EAAa,SACrCa,KAAM,IAAI3C,OAAQ,IAAM+B,GACxBa,OAAQ,IAAI5C,OAAQ,IAAMgC,GAC1Ba,MAAO,IAAI7C,OACV,yDACCF,GAAa,+BAAiCA,GAAa,cAC3DA,GAAa,aAAeA,GAAa,SAAU,KACrDgD,KAAM,IAAI9C,OAAQ,OAAS6B,EAAW,KAAM,KAI5CkB,aAAc,IAAI/C,OAAQ,IAAMF,GAC/B,mDAAqDA,GACrD,mBAAqBA,GAAa,mBAAoB,MAGxDkD,EAAU,sCACVC,EAAU,SAGVC,EAAa,mCAEbC,EAAW,OAIXC,EAAY,IAAIpD,OAAQ,uBAAyBF,GAChD,uBAAwB,KACzBuD,EAAY,SAAUC,EAAQC,GAC7B,IAAIC,EAAO,KAAOF,EAAOxL,MAAO,GAAM,MAEtC,OAAKyL,IAUEC,EAAO,EACbC,OAAOC,aAAcF,EAAO,OAC5BC,OAAOC,aAAcF,GAAQ,GAAK,MAAe,KAAPA,EAAe,SAO3DG,EAAgB,WACfC,KAGDC,EAAqBC,EACpB,SAAU7I,GACT,OAAyB,IAAlBA,EAAK8I,UAAqB/I,GAAUC,EAAM,aAElD,CAAE+I,IAAK,aAAcC,KAAM,WAa7B,IACC7L,EAAKD,MACFT,GAAMI,GAAMG,KAAM4I,GAAaqD,YACjCrD,GAAaqD,YAMdxM,GAAKmJ,GAAaqD,WAAWnJ,QAAShC,SACrC,MAAQoL,GACT/L,EAAO,CACND,MAAO,SAAUiF,EAAQgH,GACxBtD,GAAW3I,MAAOiF,EAAQtF,GAAMG,KAAMmM,KAEvCnM,KAAM,SAAUmF,GACf0D,GAAW3I,MAAOiF,EAAQtF,GAAMG,KAAMiE,UAAW,MAKpD,SAASmI,EAAM3J,EAAUC,EAAS+D,EAAS4F,GAC1C,IAAIC,EAAG5K,EAAGsB,EAAMuJ,EAAKC,EAAOC,EAAQC,EACnCC,EAAajK,GAAWA,EAAQqE,cAGhCjG,EAAW4B,EAAUA,EAAQ5B,SAAW,EAKzC,GAHA2F,EAAUA,GAAW,GAGI,iBAAbhE,IAA0BA,GACxB,IAAb3B,GAA+B,IAAbA,GAA+B,KAAbA,EAEpC,OAAO2F,EAIR,IAAM4F,IACLV,EAAajJ,GACbA,EAAUA,GAAWvD,EAEhB+J,GAAiB,CAIrB,GAAkB,KAAbpI,IAAqB0L,EAAQvB,EAAW2B,KAAMnK,IAGlD,GAAO6J,EAAIE,EAAO,IAGjB,GAAkB,IAAb1L,EAAiB,CACrB,KAAOkC,EAAON,EAAQmK,eAAgBP,IASrC,OAAO7F,EALP,GAAKzD,EAAK8J,KAAOR,EAEhB,OADAnM,EAAKH,KAAMyG,EAASzD,GACbyD,OAWT,GAAKkG,IAAgB3J,EAAO2J,EAAWE,eAAgBP,KACtDF,EAAKpE,SAAUtF,EAASM,IACxBA,EAAK8J,KAAOR,EAGZ,OADAnM,EAAKH,KAAMyG,EAASzD,GACbyD,MAKH,CAAA,GAAK+F,EAAO,GAElB,OADArM,EAAKD,MAAOuG,EAAS/D,EAAQqK,qBAAsBtK,IAC5CgE,EAGD,IAAO6F,EAAIE,EAAO,KAAS9J,EAAQsK,uBAEzC,OADA7M,EAAKD,MAAOuG,EAAS/D,EAAQsK,uBAAwBV,IAC9C7F,EAKT,KAAMiD,EAAwBjH,EAAW,MACrC0G,GAAcA,EAAUnC,KAAMvE,IAAe,CAYhD,GAVAiK,EAAcjK,EACdkK,EAAajK,EASK,IAAb5B,IACFqJ,EAASnD,KAAMvE,IAAcyH,EAAmBlD,KAAMvE,IAAe,EAGvEkK,EAAazB,EAASlE,KAAMvE,IAAcwK,EAAavK,EAAQP,aAC9DO,IAQkBA,GAAY/B,GAAQuM,SAG/BX,EAAM7J,EAAQX,aAAc,OAClCwK,EAAM/J,GAAOkG,eAAgB6D,GAE7B7J,EAAQV,aAAc,KAAQuK,EAAM9G,IAMtC/D,GADA+K,EAASU,EAAU1K,IACRK,OACX,MAAQpB,IACP+K,EAAQ/K,IAAQ6K,EAAM,IAAMA,EAAM,UAAa,IAC9Ca,EAAYX,EAAQ/K,IAEtBgL,EAAcD,EAAOY,KAAM,KAG5B,IAIC,OAHAlN,EAAKD,MAAOuG,EACXkG,EAAWW,iBAAkBZ,IAEvBjG,EACN,MAAQ8G,GACT7D,EAAwBjH,GAAU,GACjC,QACI8J,IAAQ9G,GACZ/C,EAAQ8K,gBAAiB,QAQ9B,OAAOC,GAAQhL,EAASmD,QAASkC,GAAU,MAAQpF,EAAS+D,EAAS4F,GAStE,SAAS9C,IACR,IAAImE,EAAO,GAaX,OAXA,SAASC,EAAOC,EAAKtG,GASpB,OALKoG,EAAKvN,KAAMyN,EAAM,KAAQ9E,EAAK+E,oBAG3BF,EAAOD,EAAKI,SAEXH,EAAOC,EAAM,KAAQtG,GAShC,SAASyG,EAAcpL,GAEtB,OADAA,EAAI8C,IAAY,EACT9C,EAOR,SAASqL,EAAQrL,GAChB,IAAIsL,EAAK9O,EAAS0C,cAAe,YAEjC,IACC,QAASc,EAAIsL,GACZ,MAAQ/B,GACT,OAAO,EACN,QAGI+B,EAAG9L,YACP8L,EAAG9L,WAAWC,YAAa6L,GAI5BA,EAAK,MAQP,SAASC,EAAmBhN,GAC3B,OAAO,SAAU8B,GAChB,OAAOD,GAAUC,EAAM,UAAaA,EAAK9B,OAASA,GAQpD,SAASiN,EAAoBjN,GAC5B,OAAO,SAAU8B,GAChB,OAASD,GAAUC,EAAM,UAAaD,GAAUC,EAAM,YACrDA,EAAK9B,OAASA,GAQjB,SAASkN,EAAsBtC,GAG9B,OAAO,SAAU9I,GAKhB,MAAK,SAAUA,EASTA,EAAKb,aAAgC,IAAlBa,EAAK8I,SAGvB,UAAW9I,EACV,UAAWA,EAAKb,WACba,EAAKb,WAAW2J,WAAaA,EAE7B9I,EAAK8I,WAAaA,EAMpB9I,EAAKqL,aAAevC,GAG1B9I,EAAKqL,cAAgBvC,GACpBF,EAAoB5I,KAAW8I,EAG3B9I,EAAK8I,WAAaA,EAKd,UAAW9I,GACfA,EAAK8I,WAAaA,GAY5B,SAASwC,EAAwB3L,GAChC,OAAOoL,EAAc,SAAUQ,GAE9B,OADAA,GAAYA,EACLR,EAAc,SAAU1B,EAAMlF,GACpC,IAAIzC,EACH8J,EAAe7L,EAAI,GAAI0J,EAAKvJ,OAAQyL,GACpC7M,EAAI8M,EAAa1L,OAGlB,MAAQpB,IACF2K,EAAQ3H,EAAI8J,EAAc9M,MAC9B2K,EAAM3H,KAASyC,EAASzC,GAAM2H,EAAM3H,SAYzC,SAASuI,EAAavK,GACrB,OAAOA,GAAmD,oBAAjCA,EAAQqK,sBAAwCrK,EAQ1E,SAASiJ,EAAanK,GACrB,IAAIiN,EACHhN,EAAMD,EAAOA,EAAKuF,eAAiBvF,EAAOoH,GAO3C,OAAKnH,GAAOtC,GAA6B,IAAjBsC,EAAIX,UAAmBW,EAAI6E,kBAMnDA,GADAnH,EAAWsC,GACgB6E,gBAC3B4C,GAAkB1G,GAAOmE,SAAUxH,GAInCgI,EAAUb,EAAgBa,SACzBb,EAAgBoI,uBAChBpI,EAAgBqI,kBAOZrI,EAAgBqI,mBAMpB/F,IAAgBzJ,IACdsP,EAAYtP,EAASyP,cAAiBH,EAAUI,MAAQJ,GAG1DA,EAAUK,iBAAkB,SAAUpD,GAOvC/K,GAAQoO,QAAUf,EAAQ,SAAUC,GAEnC,OADA3H,EAAgBpE,YAAa+L,GAAKnB,GAAKtK,GAAOiD,SACtCtG,EAAS6P,oBACf7P,EAAS6P,kBAAmBxM,GAAOiD,SAAU3C,SAMhDnC,GAAQsO,kBAAoBjB,EAAQ,SAAUC,GAC7C,OAAO9G,EAAQnH,KAAMiO,EAAI,OAK1BtN,GAAQuM,MAAQc,EAAQ,WACvB,OAAO7O,EAASmO,iBAAkB,YAYnC3M,GAAQuO,OAASlB,EAAQ,WACxB,IAEC,OADA7O,EAASgQ,cAAe,oBACjB,EACN,MAAQjD,GACT,OAAO,KAKJvL,GAAQoO,SACZjG,EAAKsG,OAAO7E,GAAK,SAAUuC,GAC1B,IAAIuC,EAASvC,EAAGlH,QAASuF,EAAWC,GACpC,OAAO,SAAUpI,GAChB,OAAOA,EAAKjB,aAAc,QAAWsN,IAGvCvG,EAAKsD,KAAK7B,GAAK,SAAUuC,EAAIpK,GAC5B,GAAuC,oBAA3BA,EAAQmK,gBAAkC3D,EAAiB,CACtE,IAAIlG,EAAON,EAAQmK,eAAgBC,GACnC,OAAO9J,EAAO,CAAEA,GAAS,OAI3B8F,EAAKsG,OAAO7E,GAAM,SAAUuC,GAC3B,IAAIuC,EAASvC,EAAGlH,QAASuF,EAAWC,GACpC,OAAO,SAAUpI,GAChB,IAAIxB,EAAwC,oBAA1BwB,EAAKsM,kBACtBtM,EAAKsM,iBAAkB,MACxB,OAAO9N,GAAQA,EAAK8F,QAAU+H,IAMhCvG,EAAKsD,KAAK7B,GAAK,SAAUuC,EAAIpK,GAC5B,GAAuC,oBAA3BA,EAAQmK,gBAAkC3D,EAAiB,CACtE,IAAI1H,EAAME,EAAGgC,EACZV,EAAON,EAAQmK,eAAgBC,GAEhC,GAAK9J,EAAO,CAIX,IADAxB,EAAOwB,EAAKsM,iBAAkB,QACjB9N,EAAK8F,QAAUwF,EAC3B,MAAO,CAAE9J,GAIVU,EAAQhB,EAAQsM,kBAAmBlC,GACnCpL,EAAI,EACJ,MAAUsB,EAAOU,EAAOhC,KAEvB,IADAF,EAAOwB,EAAKsM,iBAAkB,QACjB9N,EAAK8F,QAAUwF,EAC3B,MAAO,CAAE9J,GAKZ,MAAO,MAMV8F,EAAKsD,KAAK3B,IAAM,SAAU8E,EAAK7M,GAC9B,MAA6C,oBAAjCA,EAAQqK,qBACZrK,EAAQqK,qBAAsBwC,GAI9B7M,EAAQ4K,iBAAkBiC,IAKnCzG,EAAKsD,KAAK5B,MAAQ,SAAUgF,EAAW9M,GACtC,GAA+C,oBAAnCA,EAAQsK,wBAA0C9D,EAC7D,OAAOxG,EAAQsK,uBAAwBwC,IASzCrG,EAAY,GAIZ6E,EAAQ,SAAUC,GAEjB,IAAIwB,EAEJnJ,EAAgBpE,YAAa+L,GAAKyB,UACjC,UAAYjK,EAAU,iDACLA,EAAU,oEAKtBwI,EAAGX,iBAAkB,cAAexK,QACzCqG,EAAUhJ,KAAM,MAAQ0H,GAAa,aAAe+B,EAAW,KAI1DqE,EAAGX,iBAAkB,QAAU7H,EAAU,MAAO3C,QACrDqG,EAAUhJ,KAAM,MAMX8N,EAAGX,iBAAkB,KAAO7H,EAAU,MAAO3C,QAClDqG,EAAUhJ,KAAM,YAOX8N,EAAGX,iBAAkB,YAAaxK,QACvCqG,EAAUhJ,KAAM,aAKjBsP,EAAQtQ,EAAS0C,cAAe,UAC1BG,aAAc,OAAQ,UAC5BiM,EAAG/L,YAAauN,GAAQzN,aAAc,OAAQ,KAQ9CsE,EAAgBpE,YAAa+L,GAAKnC,UAAW,EACM,IAA9CmC,EAAGX,iBAAkB,aAAcxK,QACvCqG,EAAUhJ,KAAM,WAAY,cAQ7BsP,EAAQtQ,EAAS0C,cAAe,UAC1BG,aAAc,OAAQ,IAC5BiM,EAAG/L,YAAauN,GACVxB,EAAGX,iBAAkB,aAAcxK,QACxCqG,EAAUhJ,KAAM,MAAQ0H,GAAa,QAAUA,GAAa,KAC3DA,GAAa,kBAIVlH,GAAQuO,QAQb/F,EAAUhJ,KAAM,QAGjBgJ,EAAYA,EAAUrG,QAAU,IAAIiF,OAAQoB,EAAUkE,KAAM,MAM5D1D,EAAY,SAAU1B,EAAGC,GAGxB,GAAKD,IAAMC,EAEV,OADAe,GAAe,EACR,EAIR,IAAI0G,GAAW1H,EAAEG,yBAA2BF,EAAEE,wBAC9C,OAAKuH,IAgBU,GAPfA,GAAY1H,EAAElB,eAAiBkB,KAASC,EAAEnB,eAAiBmB,GAC1DD,EAAEG,wBAAyBF,GAG3B,KAIGvH,GAAQiP,cAAgB1H,EAAEE,wBAAyBH,KAAQ0H,EAOzD1H,IAAM9I,GAAY8I,EAAElB,eAAiB6B,IACzCwD,EAAKpE,SAAUY,GAAcX,IACrB,EAOJC,IAAM/I,GAAY+I,EAAEnB,eAAiB6B,IACzCwD,EAAKpE,SAAUY,GAAcV,GACtB,EAIDc,EACJ5I,GAAQJ,KAAMgJ,EAAWf,GAAM7H,GAAQJ,KAAMgJ,EAAWd,GAC1D,EAGe,EAAVyH,GAAe,EAAI,KAGpBxQ,EAqpBR,IAAMuC,KAlpBN0K,EAAKjF,QAAU,SAAU0I,EAAMC,GAC9B,OAAO1D,EAAMyD,EAAM,KAAM,KAAMC,IAGhC1D,EAAK2D,gBAAkB,SAAU/M,EAAM6M,GAGtC,GAFAlE,EAAa3I,GAERkG,IACHQ,EAAwBmG,EAAO,QAC7B1G,IAAcA,EAAUnC,KAAM6I,IAEjC,IACC,IAAIlM,EAAMwD,EAAQnH,KAAMgD,EAAM6M,GAG9B,GAAKlM,GAAOhD,GAAQsO,mBAIlBjM,EAAK7D,UAAuC,KAA3B6D,EAAK7D,SAAS2B,SAChC,OAAO6C,EAEP,MAAQuI,GACTxC,EAAwBmG,GAAM,GAIhC,OAAuD,EAAhDzD,EAAMyD,EAAM1Q,EAAU,KAAM,CAAE6D,IAASF,QAG/CsJ,EAAKpE,SAAW,SAAUtF,EAASM,GAUlC,OAHON,EAAQqE,eAAiBrE,IAAavD,GAC5CwM,EAAajJ,GAEPF,GAAOwF,SAAUtF,EAASM,IAIlCoJ,EAAK4D,KAAO,SAAUhN,EAAMC,IAOpBD,EAAK+D,eAAiB/D,IAAU7D,GACtCwM,EAAa3I,GAGd,IAAIL,EAAKmG,EAAKmH,WAAYhN,EAAKC,eAG9BvB,EAAMgB,GAAMpC,GAAOP,KAAM8I,EAAKmH,WAAYhN,EAAKC,eAC9CP,EAAIK,EAAMC,GAAOiG,QACjB1D,EAEF,YAAaA,IAAR7D,EACGA,EAGDqB,EAAKjB,aAAckB,IAG3BmJ,EAAKtG,MAAQ,SAAUC,GACtB,MAAM,IAAI1G,MAAO,0CAA4C0G,IAO9DvD,GAAO0N,WAAa,SAAUzJ,GAC7B,IAAIzD,EACHmN,EAAa,GACbzL,EAAI,EACJhD,EAAI,EAWL,GAJAuH,GAAgBtI,GAAQyP,WACxBpH,GAAarI,GAAQyP,YAAcvQ,GAAMG,KAAMyG,EAAS,GACxD7B,GAAK5E,KAAMyG,EAASkD,GAEfV,EAAe,CACnB,MAAUjG,EAAOyD,EAAS/E,KACpBsB,IAASyD,EAAS/E,KACtBgD,EAAIyL,EAAWhQ,KAAMuB,IAGvB,MAAQgD,IACPG,GAAO7E,KAAMyG,EAAS0J,EAAYzL,GAAK,GAQzC,OAFAsE,EAAY,KAELvC,GAGRjE,GAAOG,GAAGuN,WAAa,WACtB,OAAO3Q,KAAKkE,UAAWjB,GAAO0N,WAAYrQ,GAAMK,MAAOX,UAGxDuJ,EAAOtG,GAAOqN,KAAO,CAGpBhC,YAAa,GAEbwC,aAActC,EAEdvB,MAAOlC,EAEP2F,WAAY,GAEZ7D,KAAM,GAENkE,SAAU,CACTC,IAAK,CAAExE,IAAK,aAAc7H,OAAO,GACjCsM,IAAK,CAAEzE,IAAK,cACZ0E,IAAK,CAAE1E,IAAK,kBAAmB7H,OAAO,GACtCwM,IAAK,CAAE3E,IAAK,oBAGb4E,UAAW,CACVjG,KAAM,SAAU8B,GAWf,OAVAA,EAAO,GAAMA,EAAO,GAAI5G,QAASuF,EAAWC,GAG5CoB,EAAO,IAAQA,EAAO,IAAOA,EAAO,IAAOA,EAAO,IAAO,IACvD5G,QAASuF,EAAWC,GAEF,OAAfoB,EAAO,KACXA,EAAO,GAAM,IAAMA,EAAO,GAAM,KAG1BA,EAAM3M,MAAO,EAAG,IAGxB+K,MAAO,SAAU4B,GAkChB,OAtBAA,EAAO,GAAMA,EAAO,GAAItJ,cAEU,QAA7BsJ,EAAO,GAAI3M,MAAO,EAAG,IAGnB2M,EAAO,IACZJ,EAAKtG,MAAO0G,EAAO,IAKpBA,EAAO,KAASA,EAAO,GACtBA,EAAO,IAAQA,EAAO,IAAO,GAC7B,GAAqB,SAAfA,EAAO,IAAiC,QAAfA,EAAO,KAEvCA,EAAO,KAAWA,EAAO,GAAMA,EAAO,IAAwB,QAAfA,EAAO,KAG3CA,EAAO,IAClBJ,EAAKtG,MAAO0G,EAAO,IAGbA,GAGR7B,OAAQ,SAAU6B,GACjB,IAAIoE,EACHC,GAAYrE,EAAO,IAAOA,EAAO,GAElC,OAAKlC,EAAUM,MAAM5D,KAAMwF,EAAO,IAC1B,MAIHA,EAAO,GACXA,EAAO,GAAMA,EAAO,IAAOA,EAAO,IAAO,GAG9BqE,GAAYzG,EAAQpD,KAAM6J,KAGnCD,EAASzD,EAAU0D,GAAU,MAG7BD,EAASC,EAASzQ,QAAS,IAAKyQ,EAAS/N,OAAS8N,GAAWC,EAAS/N,UAGxE0J,EAAO,GAAMA,EAAO,GAAI3M,MAAO,EAAG+Q,GAClCpE,EAAO,GAAMqE,EAAShR,MAAO,EAAG+Q,IAI1BpE,EAAM3M,MAAO,EAAG,MAIzBuP,OAAQ,CAEP3E,IAAK,SAAUqG,GACd,IAAIC,EAAmBD,EAAiBlL,QAASuF,EAAWC,GAAYlI,cACxE,MAA4B,MAArB4N,EACN,WACC,OAAO,GAER,SAAU9N,GACT,OAAOD,GAAUC,EAAM+N,KAI1BvG,MAAO,SAAUgF,GAChB,IAAIwB,EAAU1H,EAAYkG,EAAY,KAEtC,OAAOwB,IACJA,EAAU,IAAIjJ,OAAQ,MAAQF,GAAa,IAAM2H,EAClD,IAAM3H,GAAa,SACpByB,EAAYkG,EAAW,SAAUxM,GAChC,OAAOgO,EAAQhK,KACY,iBAAnBhE,EAAKwM,WAA0BxM,EAAKwM,WACb,oBAAtBxM,EAAKjB,cACXiB,EAAKjB,aAAc,UACpB,OAKL2I,KAAM,SAAUzH,EAAMgO,EAAUC,GAC/B,OAAO,SAAUlO,GAChB,IAAImO,EAAS/E,EAAK4D,KAAMhN,EAAMC,GAE9B,OAAe,MAAVkO,EACgB,OAAbF,GAEFA,IAINE,GAAU,GAEQ,MAAbF,EACGE,IAAWD,EAED,OAAbD,EACGE,IAAWD,EAED,OAAbD,EACGC,GAAqC,IAA5BC,EAAO/Q,QAAS8Q,GAEf,OAAbD,EACGC,IAAoC,EAA3BC,EAAO/Q,QAAS8Q,GAEf,OAAbD,EACGC,GAASC,EAAOtR,OAAQqR,EAAMpO,UAAaoO,EAEjC,OAAbD,GAEkB,GADb,IAAME,EAAOvL,QAASoE,EAAa,KAAQ,KAClD5J,QAAS8Q,GAEM,OAAbD,IACGE,IAAWD,GAASC,EAAOtR,MAAO,EAAGqR,EAAMpO,OAAS,KAAQoO,EAAQ,QAO9EtG,MAAO,SAAU1J,EAAMkQ,EAAMC,EAAWnN,EAAOE,GAC9C,IAAIkN,EAAgC,QAAvBpQ,EAAKrB,MAAO,EAAG,GAC3B0R,EAA+B,SAArBrQ,EAAKrB,OAAQ,GACvB2R,EAAkB,YAATJ,EAEV,OAAiB,IAAVlN,GAAwB,IAATE,EAGrB,SAAUpB,GACT,QAASA,EAAKb,YAGf,SAAUa,EAAMyO,EAAUC,GACzB,IAAI/D,EAAOgE,EAAYnQ,EAAMoQ,EAAWC,EACvC9F,EAAMuF,IAAWC,EAAU,cAAgB,kBAC3CO,EAAS9O,EAAKb,WACdc,EAAOuO,GAAUxO,EAAKD,SAASG,cAC/B6O,GAAYL,IAAQF,EACpBQ,GAAO,EAER,GAAKF,EAAS,CAGb,GAAKR,EAAS,CACb,MAAQvF,EAAM,CACbvK,EAAOwB,EACP,MAAUxB,EAAOA,EAAMuK,GACtB,GAAKyF,EACJzO,GAAUvB,EAAMyB,GACE,IAAlBzB,EAAKV,SAEL,OAAO,EAKT+Q,EAAQ9F,EAAe,SAAT7K,IAAoB2Q,GAAS,cAE5C,OAAO,EAMR,GAHAA,EAAQ,CAAEN,EAAUO,EAAOG,WAAaH,EAAOI,WAG1CX,GAAWQ,EAAW,CAM1BC,GADAJ,GADAjE,GADAgE,EAAaG,EAAQrM,KAAeqM,EAAQrM,GAAY,KACpCvE,IAAU,IACX,KAAQkI,GAAWuE,EAAO,KACzBA,EAAO,GAC3BnM,EAAOoQ,GAAaE,EAAO7F,WAAY2F,GAEvC,MAAUpQ,IAASoQ,GAAapQ,GAAQA,EAAMuK,KAG3CiG,EAAOJ,EAAY,IAAOC,EAAMjK,MAGlC,GAAuB,IAAlBpG,EAAKV,YAAoBkR,GAAQxQ,IAASwB,EAAO,CACrD2O,EAAYzQ,GAAS,CAAEkI,EAASwI,EAAWI,GAC3C,YAgBF,GATKD,IAIJC,EADAJ,GADAjE,GADAgE,EAAa3O,EAAMyC,KAAezC,EAAMyC,GAAY,KAChCvE,IAAU,IACX,KAAQkI,GAAWuE,EAAO,KAMhC,IAATqE,EAGJ,MAAUxQ,IAASoQ,GAAapQ,GAAQA,EAAMuK,KAC3CiG,EAAOJ,EAAY,IAAOC,EAAMjK,MAElC,IAAO4J,EACNzO,GAAUvB,EAAMyB,GACE,IAAlBzB,EAAKV,aACHkR,IAGGD,KACJJ,EAAanQ,EAAMiE,KAChBjE,EAAMiE,GAAY,KACTvE,GAAS,CAAEkI,EAAS4I,IAG5BxQ,IAASwB,GACb,MASL,OADAgP,GAAQ5N,KACQF,GAAW8N,EAAO9N,GAAU,GAAqB,GAAhB8N,EAAO9N,KAK5DyG,OAAQ,SAAUwH,EAAQ5D,GAMzB,IAAI6D,EACHzP,EAAKmG,EAAKiB,QAASoI,IAAYrJ,EAAKuJ,WAAYF,EAAOjP,gBACtDkJ,EAAKtG,MAAO,uBAAyBqM,GAKvC,OAAKxP,EAAI8C,GACD9C,EAAI4L,GAIK,EAAZ5L,EAAGG,QACPsP,EAAO,CAAED,EAAQA,EAAQ,GAAI5D,GACtBzF,EAAKuJ,WAAW7R,eAAgB2R,EAAOjP,eAC7C6K,EAAc,SAAU1B,EAAMlF,GAC7B,IAAImL,EACHC,EAAU5P,EAAI0J,EAAMkC,GACpB7M,EAAI6Q,EAAQzP,OACb,MAAQpB,IAEP2K,EADAiG,EAAMlS,GAAQJ,KAAMqM,EAAMkG,EAAS7Q,OAClByF,EAASmL,GAAQC,EAAS7Q,MAG7C,SAAUsB,GACT,OAAOL,EAAIK,EAAM,EAAGoP,KAIhBzP,IAIToH,QAAS,CAGRyI,IAAKzE,EAAc,SAAUtL,GAK5B,IAAIgN,EAAQ,GACXhJ,EAAU,GACVgM,EAAUC,GAASjQ,EAASmD,QAASkC,GAAU,OAEhD,OAAO2K,EAAShN,GACfsI,EAAc,SAAU1B,EAAMlF,EAASsK,EAAUC,GAChD,IAAI1O,EACH2P,EAAYF,EAASpG,EAAM,KAAMqF,EAAK,IACtChQ,EAAI2K,EAAKvJ,OAGV,MAAQpB,KACAsB,EAAO2P,EAAWjR,MACxB2K,EAAM3K,KAASyF,EAASzF,GAAMsB,MAIjC,SAAUA,EAAMyO,EAAUC,GAOzB,OANAjC,EAAO,GAAMzM,EACbyP,EAAShD,EAAO,KAAMiC,EAAKjL,GAI3BgJ,EAAO,GAAM,MACLhJ,EAAQmB,SAInBgL,IAAK7E,EAAc,SAAUtL,GAC5B,OAAO,SAAUO,GAChB,OAAuC,EAAhCoJ,EAAM3J,EAAUO,GAAOF,UAIhCkF,SAAU+F,EAAc,SAAUjM,GAEjC,OADAA,EAAOA,EAAK8D,QAASuF,EAAWC,GACzB,SAAUpI,GAChB,OAAsE,GAA7DA,EAAKqD,aAAe7D,GAAOV,KAAMkB,IAAS5C,QAAS0B,MAW9D+Q,KAAM9E,EAAc,SAAU8E,GAO7B,OAJMxI,EAAYrD,KAAM6L,GAAQ,KAC/BzG,EAAKtG,MAAO,qBAAuB+M,GAEpCA,EAAOA,EAAKjN,QAASuF,EAAWC,GAAYlI,cACrC,SAAUF,GAChB,IAAI8P,EACJ,GACC,GAAOA,EAAW5J,EACjBlG,EAAK6P,KACL7P,EAAKjB,aAAc,aAAgBiB,EAAKjB,aAAc,QAGtD,OADA+Q,EAAWA,EAAS5P,iBACA2P,GAA2C,IAAnCC,EAAS1S,QAASyS,EAAO,YAE3C7P,EAAOA,EAAKb,aAAkC,IAAlBa,EAAKlC,UAC7C,OAAO,KAKTqE,OAAQ,SAAUnC,GACjB,IAAI+P,EAAOzT,GAAO0T,UAAY1T,GAAO0T,SAASD,KAC9C,OAAOA,GAAQA,EAAKlT,MAAO,KAAQmD,EAAK8J,IAGzCmG,KAAM,SAAUjQ,GACf,OAAOA,IAASsD,GAGjB4M,MAAO,SAAUlQ,GAChB,OAAOA,IA5oCV,WACC,IACC,OAAO7D,EAASgU,cACf,MAAQC,KAyoCQC,IACflU,EAASmU,eACLtQ,EAAK9B,MAAQ8B,EAAKuQ,OAASvQ,EAAKwQ,WAItCC,QAASrF,GAAsB,GAC/BtC,SAAUsC,GAAsB,GAEhCsF,QAAS,SAAU1Q,GAIlB,OAASD,GAAUC,EAAM,YAAeA,EAAK0Q,SAC1C3Q,GAAUC,EAAM,aAAgBA,EAAK2Q,UAGzCA,SAAU,SAAU3Q,GAWnB,OALKA,EAAKb,YAETa,EAAKb,WAAWyR,eAGQ,IAAlB5Q,EAAK2Q,UAIbE,MAAO,SAAU7Q,GAMhB,IAAMA,EAAOA,EAAKiP,WAAYjP,EAAMA,EAAOA,EAAK8Q,YAC/C,GAAK9Q,EAAKlC,SAAW,EACpB,OAAO,EAGT,OAAO,GAGRgR,OAAQ,SAAU9O,GACjB,OAAQ8F,EAAKiB,QAAQ8J,MAAO7Q,IAI7B+Q,OAAQ,SAAU/Q,GACjB,OAAOgI,EAAQhE,KAAMhE,EAAKD,WAG3B0M,MAAO,SAAUzM,GAChB,OAAO+H,EAAQ/D,KAAMhE,EAAKD,WAG3BiR,OAAQ,SAAUhR,GACjB,OAAOD,GAAUC,EAAM,UAA2B,WAAdA,EAAK9B,MACxC6B,GAAUC,EAAM,WAGlBlB,KAAM,SAAUkB,GACf,IAAIgN,EACJ,OAAOjN,GAAUC,EAAM,UAA2B,SAAdA,EAAK9B,OAKI,OAAxC8O,EAAOhN,EAAKjB,aAAc,UACN,SAAvBiO,EAAK9M,gBAIRgB,MAAOoK,EAAwB,WAC9B,MAAO,CAAE,KAGVlK,KAAMkK,EAAwB,SAAU2F,EAAenR,GACtD,MAAO,CAAEA,EAAS,KAGnBqB,GAAImK,EAAwB,SAAU2F,EAAenR,EAAQyL,GAC5D,MAAO,CAAEA,EAAW,EAAIA,EAAWzL,EAASyL,KAG7ClK,KAAMiK,EAAwB,SAAUE,EAAc1L,GAErD,IADA,IAAIpB,EAAI,EACAA,EAAIoB,EAAQpB,GAAK,EACxB8M,EAAarO,KAAMuB,GAEpB,OAAO8M,IAGRhK,IAAK8J,EAAwB,SAAUE,EAAc1L,GAEpD,IADA,IAAIpB,EAAI,EACAA,EAAIoB,EAAQpB,GAAK,EACxB8M,EAAarO,KAAMuB,GAEpB,OAAO8M,IAGR0F,GAAI5F,EAAwB,SAAUE,EAAc1L,EAAQyL,GAC3D,IAAI7M,EAUJ,IAPCA,EADI6M,EAAW,EACXA,EAAWzL,EACOA,EAAXyL,EACPzL,EAEAyL,EAGU,KAAL7M,GACT8M,EAAarO,KAAMuB,GAEpB,OAAO8M,IAGR2F,GAAI7F,EAAwB,SAAUE,EAAc1L,EAAQyL,GAE3D,IADA,IAAI7M,EAAI6M,EAAW,EAAIA,EAAWzL,EAASyL,IACjC7M,EAAIoB,GACb0L,EAAarO,KAAMuB,GAEpB,OAAO8M,OAKLzE,QAAQqK,IAAMtL,EAAKiB,QAAQ5F,GAGrB,CAAEkQ,OAAO,EAAMC,UAAU,EAAMC,MAAM,EAAMC,UAAU,EAAMC,OAAO,GAC5E3L,EAAKiB,QAASrI,GAAMwM,EAAmBxM,GAExC,IAAMA,IAAK,CAAEgT,QAAQ,EAAMC,OAAO,GACjC7L,EAAKiB,QAASrI,GAAMyM,EAAoBzM,GAIzC,SAAS2Q,KAIT,SAASlF,EAAU1K,EAAUmS,GAC5B,IAAIrC,EAAS/F,EAAOqI,EAAQ3T,EAC3B4T,EAAOrI,EAAQsI,EACfC,EAASxL,EAAY/G,EAAW,KAEjC,GAAKuS,EACJ,OAAOJ,EAAY,EAAII,EAAOnV,MAAO,GAGtCiV,EAAQrS,EACRgK,EAAS,GACTsI,EAAajM,EAAK6H,UAElB,MAAQmE,EAAQ,CA2Bf,IAAM5T,KAxBAqR,KAAa/F,EAAQvC,EAAO2C,KAAMkI,MAClCtI,IAGJsI,EAAQA,EAAMjV,MAAO2M,EAAO,GAAI1J,SAAYgS,GAE7CrI,EAAOtM,KAAQ0U,EAAS,KAGzBtC,GAAU,GAGH/F,EAAQtC,EAAmB0C,KAAMkI,MACvCvC,EAAU/F,EAAMsB,QAChB+G,EAAO1U,KAAM,CACZmH,MAAOiL,EAGPrR,KAAMsL,EAAO,GAAI5G,QAASkC,GAAU,OAErCgN,EAAQA,EAAMjV,MAAO0S,EAAQzP,SAIhBgG,EAAKsG,SACX5C,EAAQlC,EAAWpJ,GAAO0L,KAAMkI,KAAgBC,EAAY7T,MAChEsL,EAAQuI,EAAY7T,GAAQsL,MAC9B+F,EAAU/F,EAAMsB,QAChB+G,EAAO1U,KAAM,CACZmH,MAAOiL,EACPrR,KAAMA,EACNiG,QAASqF,IAEVsI,EAAQA,EAAMjV,MAAO0S,EAAQzP,SAI/B,IAAMyP,EACL,MAOF,OAAKqC,EACGE,EAAMhS,OAGPgS,EACN1I,EAAKtG,MAAOrD,GAGZ+G,EAAY/G,EAAUgK,GAAS5M,MAAO,GAGxC,SAASuN,EAAYyH,GAIpB,IAHA,IAAInT,EAAI,EACP+C,EAAMoQ,EAAO/R,OACbL,EAAW,GACJf,EAAI+C,EAAK/C,IAChBe,GAAYoS,EAAQnT,GAAI4F,MAEzB,OAAO7E,EAGR,SAASoJ,EAAe4G,EAASwC,EAAYC,GAC5C,IAAInJ,EAAMkJ,EAAWlJ,IACpBoJ,EAAOF,EAAWjJ,KAClB4B,EAAMuH,GAAQpJ,EACdqJ,EAAmBF,GAAgB,eAARtH,EAC3ByH,EAAWhM,IAEZ,OAAO4L,EAAW/Q,MAGjB,SAAUlB,EAAMN,EAASgP,GACxB,MAAU1O,EAAOA,EAAM+I,GACtB,GAAuB,IAAlB/I,EAAKlC,UAAkBsU,EAC3B,OAAO3C,EAASzP,EAAMN,EAASgP,GAGjC,OAAO,GAIR,SAAU1O,EAAMN,EAASgP,GACxB,IAAI4D,EAAU3D,EACb4D,EAAW,CAAEnM,EAASiM,GAGvB,GAAK3D,GACJ,MAAU1O,EAAOA,EAAM+I,GACtB,IAAuB,IAAlB/I,EAAKlC,UAAkBsU,IACtB3C,EAASzP,EAAMN,EAASgP,GAC5B,OAAO,OAKV,MAAU1O,EAAOA,EAAM+I,GACtB,GAAuB,IAAlB/I,EAAKlC,UAAkBsU,EAG3B,GAFAzD,EAAa3O,EAAMyC,KAAezC,EAAMyC,GAAY,IAE/C0P,GAAQpS,GAAUC,EAAMmS,GAC5BnS,EAAOA,EAAM+I,IAAS/I,MAChB,CAAA,IAAOsS,EAAW3D,EAAY/D,KACpC0H,EAAU,KAAQlM,GAAWkM,EAAU,KAAQD,EAG/C,OAASE,EAAU,GAAMD,EAAU,GAOnC,IAHA3D,EAAY/D,GAAQ2H,GAGH,GAAM9C,EAASzP,EAAMN,EAASgP,GAC9C,OAAO,EAMZ,OAAO,GAIV,SAAS8D,EAAgBC,GACxB,OAAyB,EAAlBA,EAAS3S,OACf,SAAUE,EAAMN,EAASgP,GACxB,IAAIhQ,EAAI+T,EAAS3S,OACjB,MAAQpB,IACP,IAAM+T,EAAU/T,GAAKsB,EAAMN,EAASgP,GACnC,OAAO,EAGT,OAAO,GAER+D,EAAU,GAYZ,SAASC,EAAU/C,EAAW3O,EAAKoL,EAAQ1M,EAASgP,GAOnD,IANA,IAAI1O,EACH2S,EAAe,GACfjU,EAAI,EACJ+C,EAAMkO,EAAU7P,OAChB8S,EAAgB,MAAP5R,EAEFtC,EAAI+C,EAAK/C,KACTsB,EAAO2P,EAAWjR,MAClB0N,IAAUA,EAAQpM,EAAMN,EAASgP,KACtCiE,EAAaxV,KAAM6C,GACd4S,GACJ5R,EAAI7D,KAAMuB,KAMd,OAAOiU,EAGR,SAASE,GAAYlF,EAAWlO,EAAUgQ,EAASqD,EAAYC,EAAYC,GAO1E,OANKF,IAAeA,EAAYrQ,KAC/BqQ,EAAaD,GAAYC,IAErBC,IAAeA,EAAYtQ,KAC/BsQ,EAAaF,GAAYE,EAAYC,IAE/BjI,EAAc,SAAU1B,EAAM5F,EAAS/D,EAASgP,GACtD,IAAIuE,EAAMvU,EAAGsB,EAAMkT,EAClBC,EAAS,GACTC,EAAU,GACVC,EAAc5P,EAAQ3D,OAGtBY,EAAQ2I,GA5CX,SAA2B5J,EAAU6T,EAAU7P,GAG9C,IAFA,IAAI/E,EAAI,EACP+C,EAAM6R,EAASxT,OACRpB,EAAI+C,EAAK/C,IAChB0K,EAAM3J,EAAU6T,EAAU5U,GAAK+E,GAEhC,OAAOA,EAuCJ8P,CAAkB9T,GAAY,IAC7BC,EAAQ5B,SAAW,CAAE4B,GAAYA,EAAS,IAG5C8T,GAAY7F,IAAetE,GAAS5J,EAEnCiB,EADAgS,EAAUhS,EAAOyS,EAAQxF,EAAWjO,EAASgP,GAsB/C,GAnBKe,EAaJA,EAAS+D,EATTN,EAAaH,IAAgB1J,EAAOsE,EAAY0F,GAAeP,GAG9D,GAGArP,EAG+B/D,EAASgP,GAEzCwE,EAAaM,EAITV,EAAa,CACjBG,EAAOP,EAAUQ,EAAYE,GAC7BN,EAAYG,EAAM,GAAIvT,EAASgP,GAG/BhQ,EAAIuU,EAAKnT,OACT,MAAQpB,KACAsB,EAAOiT,EAAMvU,MACnBwU,EAAYE,EAAS1U,MAAW8U,EAAWJ,EAAS1U,IAAQsB,IAK/D,GAAKqJ,GACJ,GAAK0J,GAAcpF,EAAY,CAC9B,GAAKoF,EAAa,CAGjBE,EAAO,GACPvU,EAAIwU,EAAWpT,OACf,MAAQpB,KACAsB,EAAOkT,EAAYxU,KAGzBuU,EAAK9V,KAAQqW,EAAW9U,GAAMsB,GAGhC+S,EAAY,KAAQG,EAAa,GAAMD,EAAMvE,GAI9ChQ,EAAIwU,EAAWpT,OACf,MAAQpB,KACAsB,EAAOkT,EAAYxU,MAC2C,GAAlEuU,EAAOF,EAAa3V,GAAQJ,KAAMqM,EAAMrJ,GAASmT,EAAQzU,MAE3D2K,EAAM4J,KAAYxP,EAASwP,GAASjT,UAOvCkT,EAAaR,EACZQ,IAAezP,EACdyP,EAAWrR,OAAQwR,EAAaH,EAAWpT,QAC3CoT,GAEGH,EACJA,EAAY,KAAMtP,EAASyP,EAAYxE,GAEvCvR,EAAKD,MAAOuG,EAASyP,KAMzB,SAASO,GAAmB5B,GA+B3B,IA9BA,IAAI6B,EAAcjE,EAAS/N,EAC1BD,EAAMoQ,EAAO/R,OACb6T,EAAkB7N,EAAKwH,SAAUuE,EAAQ,GAAI3T,MAC7C0V,EAAmBD,GAAmB7N,EAAKwH,SAAU,KACrD5O,EAAIiV,EAAkB,EAAI,EAG1BE,EAAehL,EAAe,SAAU7I,GACvC,OAAOA,IAAS0T,GACdE,GAAkB,GACrBE,EAAkBjL,EAAe,SAAU7I,GAC1C,OAA6C,EAAtC5C,GAAQJ,KAAM0W,EAAc1T,IACjC4T,GAAkB,GACrBnB,EAAW,CAAE,SAAUzS,EAAMN,EAASgP,GAMrC,IAAI/N,GAASgT,IAAqBjF,GAAOhP,GAAWqG,MACjD2N,EAAehU,GAAU5B,SAC1B+V,EAAc7T,EAAMN,EAASgP,GAC7BoF,EAAiB9T,EAAMN,EAASgP,IAKlC,OADAgF,EAAe,KACR/S,IAGDjC,EAAI+C,EAAK/C,IAChB,GAAO+Q,EAAU3J,EAAKwH,SAAUuE,EAAQnT,GAAIR,MAC3CuU,EAAW,CAAE5J,EAAe2J,EAAgBC,GAAYhD,QAClD,CAIN,IAHAA,EAAU3J,EAAKsG,OAAQyF,EAAQnT,GAAIR,MAAOhB,MAAO,KAAM2U,EAAQnT,GAAIyF,UAGrD1B,GAAY,CAIzB,IADAf,IAAMhD,EACEgD,EAAID,EAAKC,IAChB,GAAKoE,EAAKwH,SAAUuE,EAAQnQ,GAAIxD,MAC/B,MAGF,OAAO2U,GACF,EAAJnU,GAAS8T,EAAgBC,GACrB,EAAJ/T,GAAS0L,EAGRyH,EAAOhV,MAAO,EAAG6B,EAAI,GACnBzB,OAAQ,CAAEqH,MAAgC,MAAzBuN,EAAQnT,EAAI,GAAIR,KAAe,IAAM,MACvD0E,QAASkC,GAAU,MACrB2K,EACA/Q,EAAIgD,GAAK+R,GAAmB5B,EAAOhV,MAAO6B,EAAGgD,IAC7CA,EAAID,GAAOgS,GAAqB5B,EAASA,EAAOhV,MAAO6E,IACvDA,EAAID,GAAO2I,EAAYyH,IAGzBY,EAAStV,KAAMsS,GAIjB,OAAO+C,EAAgBC,GAiIxB,SAAS/C,GAASjQ,EAAU+J,GAC3B,IAAI9K,EA/H8BqV,EAAiBC,EAC/CC,EACHC,EACAC,EA6HAH,EAAc,GACdD,EAAkB,GAClB/B,EAASvL,EAAehH,EAAW,KAEpC,IAAMuS,EAAS,CAGRxI,IACLA,EAAQW,EAAU1K,IAEnBf,EAAI8K,EAAM1J,OACV,MAAQpB,KACPsT,EAASyB,GAAmBjK,EAAO9K,KACtB+D,GACZuR,EAAY7W,KAAM6U,GAElB+B,EAAgB5W,KAAM6U,IAKxBA,EAASvL,EAAehH,GArJSsU,EAsJNA,EArJxBE,EAA6B,GADkBD,EAsJNA,GArJrBlU,OACvBoU,EAAqC,EAAzBH,EAAgBjU,OAC5BqU,EAAe,SAAU9K,EAAM3J,EAASgP,EAAKjL,EAAS2Q,GACrD,IAAIpU,EAAM0B,EAAG+N,EACZ4E,EAAe,EACf3V,EAAI,IACJiR,EAAYtG,GAAQ,GACpBiL,EAAa,GACbC,EAAgBxO,EAGhBrF,EAAQ2I,GAAQ6K,GAAapO,EAAKsD,KAAK3B,IAAK,IAAK2M,GAGjDI,EAAkBpO,GAA4B,MAAjBmO,EAAwB,EAAI7R,KAAKC,UAAY,GAC1ElB,EAAMf,EAAMZ,OAeb,IAbKsU,IAMJrO,EAAmBrG,GAAWvD,GAAYuD,GAAW0U,GAO9C1V,IAAM+C,GAAgC,OAAvBzB,EAAOU,EAAOhC,IAAeA,IAAM,CACzD,GAAKwV,GAAalU,EAAO,CACxB0B,EAAI,EAMEhC,GAAWM,EAAK+D,eAAiB5H,IACtCwM,EAAa3I,GACb0O,GAAOxI,GAER,MAAUuJ,EAAUsE,EAAiBrS,KACpC,GAAK+N,EAASzP,EAAMN,GAAWvD,EAAUuS,GAAQ,CAChDvR,EAAKH,KAAMyG,EAASzD,GACpB,MAGGoU,IACJhO,EAAUoO,GAKPP,KAGGjU,GAAQyP,GAAWzP,IACzBqU,IAIIhL,GACJsG,EAAUxS,KAAM6C,IAgBnB,GATAqU,GAAgB3V,EASXuV,GAASvV,IAAM2V,EAAe,CAClC3S,EAAI,EACJ,MAAU+N,EAAUuE,EAAatS,KAChC+N,EAASE,EAAW2E,EAAY5U,EAASgP,GAG1C,GAAKrF,EAAO,CAGX,GAAoB,EAAfgL,EACJ,MAAQ3V,IACCiR,EAAWjR,IAAO4V,EAAY5V,KACrC4V,EAAY5V,GAAMkG,GAAI5H,KAAMyG,IAM/B6Q,EAAa5B,EAAU4B,GAIxBnX,EAAKD,MAAOuG,EAAS6Q,GAGhBF,IAAc/K,GAA4B,EAApBiL,EAAWxU,QACG,EAAtCuU,EAAeL,EAAYlU,QAE7BN,GAAO0N,WAAYzJ,GAUrB,OALK2Q,IACJhO,EAAUoO,EACVzO,EAAmBwO,GAGb5E,GAGFsE,EACNlJ,EAAcoJ,GACdA,KA8BO1U,SAAWA,EAEnB,OAAOuS,EAYR,SAASvH,GAAQhL,EAAUC,EAAS+D,EAAS4F,GAC5C,IAAI3K,EAAGmT,EAAQ4C,EAAOvW,EAAMkL,EAC3BsL,EAA+B,mBAAbjV,GAA2BA,EAC7C+J,GAASH,GAAQc,EAAY1K,EAAWiV,EAASjV,UAAYA,GAM9D,GAJAgE,EAAUA,GAAW,GAIC,IAAjB+F,EAAM1J,OAAe,CAIzB,GAAqB,GADrB+R,EAASrI,EAAO,GAAMA,EAAO,GAAI3M,MAAO,IAC5BiD,QAA+C,QAA/B2U,EAAQ5C,EAAQ,IAAM3T,MAC3B,IAArBwB,EAAQ5B,UAAkBoI,GAAkBJ,EAAKwH,SAAUuE,EAAQ,GAAI3T,MAAS,CAMjF,KAJAwB,GAAYoG,EAAKsD,KAAK7B,GACrBkN,EAAMtQ,QAAS,GAAIvB,QAASuF,EAAWC,GACvC1I,IACI,IAAM,IAEV,OAAO+D,EAGIiR,IACXhV,EAAUA,EAAQP,YAGnBM,EAAWA,EAAS5C,MAAOgV,EAAO/G,QAAQxG,MAAMxE,QAIjDpB,EAAI4I,EAAUQ,aAAa9D,KAAMvE,GAAa,EAAIoS,EAAO/R,OACzD,MAAQpB,IAAM,CAIb,GAHA+V,EAAQ5C,EAAQnT,GAGXoH,EAAKwH,SAAYpP,EAAOuW,EAAMvW,MAClC,MAED,IAAOkL,EAAOtD,EAAKsD,KAAMlL,MAGjBmL,EAAOD,EACbqL,EAAMtQ,QAAS,GAAIvB,QAASuF,EAAWC,GACvCF,EAASlE,KAAM6N,EAAQ,GAAI3T,OAC1B+L,EAAavK,EAAQP,aAAgBO,IACjC,CAKL,GAFAmS,EAAOhQ,OAAQnD,EAAG,KAClBe,EAAW4J,EAAKvJ,QAAUsK,EAAYyH,IAGrC,OADA1U,EAAKD,MAAOuG,EAAS4F,GACd5F,EAGR,QAeJ,OAPEiR,GAAYhF,GAASjQ,EAAU+J,IAChCH,EACA3J,GACCwG,EACDzC,GACC/D,GAAWwI,EAASlE,KAAMvE,IAAcwK,EAAavK,EAAQP,aAAgBO,GAExE+D,EArlBR4L,EAAWlP,UAAY2F,EAAK6O,QAAU7O,EAAKiB,QAC3CjB,EAAKuJ,WAAa,IAAIA,EA2lBtB1R,GAAQyP,WAAa3K,EAAQiC,MAAO,IAAK9C,KAAM+E,GAAY0D,KAAM,MAAS5H,EAG1EkG,IAIAhL,GAAQiP,aAAe5B,EAAQ,SAAUC,GAGxC,OAA4E,EAArEA,EAAG7F,wBAAyBjJ,EAAS0C,cAAe,eAG5DW,GAAO4J,KAAOA,EAGd5J,GAAOqN,KAAM,KAAQrN,GAAOqN,KAAK9F,QACjCvH,GAAOoV,OAASpV,GAAO0N,WAIvB9D,EAAKsG,QAAUA,GACftG,EAAKqB,OAASA,GACdrB,EAAKT,YAAcA,EACnBS,EAAKe,SAAWA,EAEhBf,EAAKf,OAAS7I,GAAOkG,eACrB0D,EAAKyL,QAAUrV,GAAOV,KACtBsK,EAAK0L,MAAQtV,GAAOmE,SACpByF,EAAK2L,UAAYvV,GAAOqN,KACxBzD,EAAKzL,QAAU6B,GAAO7B,QACtByL,EAAK8D,WAAa1N,GAAO0N,WAniEzB,GA0iEA,IAAInE,EAAM,SAAU/I,EAAM+I,EAAKiM,GAC9B,IAAIzF,EAAU,GACb0F,OAAqBzS,IAAVwS,EAEZ,OAAUhV,EAAOA,EAAM+I,KAA6B,IAAlB/I,EAAKlC,SACtC,GAAuB,IAAlBkC,EAAKlC,SAAiB,CAC1B,GAAKmX,GAAYzV,GAAQQ,GAAOkV,GAAIF,GACnC,MAEDzF,EAAQpS,KAAM6C,GAGhB,OAAOuP,GAIJ4F,EAAW,SAAUC,EAAGpV,GAG3B,IAFA,IAAIuP,EAAU,GAEN6F,EAAGA,EAAIA,EAAEtE,YACI,IAAfsE,EAAEtX,UAAkBsX,IAAMpV,GAC9BuP,EAAQpS,KAAMiY,GAIhB,OAAO7F,GAIJ8F,EAAgB7V,GAAOqN,KAAKrD,MAAM1B,aAElCwN,EAAa,kEAKjB,SAASC,EAAQzI,EAAU0I,EAAWhG,GACrC,OAAK5R,EAAY4X,GACThW,GAAO8B,KAAMwL,EAAU,SAAU9M,EAAMtB,GAC7C,QAAS8W,EAAUxY,KAAMgD,EAAMtB,EAAGsB,KAAWwP,IAK1CgG,EAAU1X,SACP0B,GAAO8B,KAAMwL,EAAU,SAAU9M,GACvC,OAASA,IAASwV,IAAgBhG,IAKV,iBAAdgG,EACJhW,GAAO8B,KAAMwL,EAAU,SAAU9M,GACvC,OAA4C,EAAnC5C,GAAQJ,KAAMwY,EAAWxV,KAAkBwP,IAK/ChQ,GAAO4M,OAAQoJ,EAAW1I,EAAU0C,GAG5ChQ,GAAO4M,OAAS,SAAUS,EAAMnM,EAAO8O,GACtC,IAAIxP,EAAOU,EAAO,GAMlB,OAJK8O,IACJ3C,EAAO,QAAUA,EAAO,KAGH,IAAjBnM,EAAMZ,QAAkC,IAAlBE,EAAKlC,SACxB0B,GAAO4J,KAAK2D,gBAAiB/M,EAAM6M,GAAS,CAAE7M,GAAS,GAGxDR,GAAO4J,KAAKjF,QAAS0I,EAAMrN,GAAO8B,KAAMZ,EAAO,SAAUV,GAC/D,OAAyB,IAAlBA,EAAKlC,aAId0B,GAAOG,GAAGmC,OAAQ,CACjBsH,KAAM,SAAU3J,GACf,IAAIf,EAAGiC,EACNc,EAAMlF,KAAKuD,OACX2V,EAAOlZ,KAER,GAAyB,iBAAbkD,EACX,OAAOlD,KAAKkE,UAAWjB,GAAQC,GAAW2M,OAAQ,WACjD,IAAM1N,EAAI,EAAGA,EAAI+C,EAAK/C,IACrB,GAAKc,GAAOwF,SAAUyQ,EAAM/W,GAAKnC,MAChC,OAAO,KAQX,IAFAoE,EAAMpE,KAAKkE,UAAW,IAEhB/B,EAAI,EAAGA,EAAI+C,EAAK/C,IACrBc,GAAO4J,KAAM3J,EAAUgW,EAAM/W,GAAKiC,GAGnC,OAAa,EAANc,EAAUjC,GAAO0N,WAAYvM,GAAQA,GAE7CyL,OAAQ,SAAU3M,GACjB,OAAOlD,KAAKkE,UAAW8U,EAAQhZ,KAAMkD,GAAY,IAAI,KAEtD+P,IAAK,SAAU/P,GACd,OAAOlD,KAAKkE,UAAW8U,EAAQhZ,KAAMkD,GAAY,IAAI,KAEtDyV,GAAI,SAAUzV,GACb,QAAS8V,EACRhZ,KAIoB,iBAAbkD,GAAyB4V,EAAcrR,KAAMvE,GACnDD,GAAQC,GACRA,GAAY,IACb,GACCK,UASJ,IAAI4V,EAMHzN,EAAa,uCAENzI,GAAOG,GAAGC,KAAO,SAAUH,EAAUC,EAASuQ,GACpD,IAAIzG,EAAOxJ,EAGX,IAAMP,EACL,OAAOlD,KAQR,GAHA0T,EAAOA,GAAQyF,EAGU,iBAAbjW,EAAwB,CAanC,KAPC+J,EALsB,MAAlB/J,EAAU,IACsB,MAApCA,EAAUA,EAASK,OAAS,IACT,GAAnBL,EAASK,OAGD,CAAE,KAAML,EAAU,MAGlBwI,EAAW2B,KAAMnK,MAIV+J,EAAO,IAAQ9J,EA6CxB,OAAMA,GAAWA,EAAQU,QACtBV,GAAWuQ,GAAO7G,KAAM3J,GAK1BlD,KAAK8D,YAAaX,GAAU0J,KAAM3J,GAhDzC,GAAK+J,EAAO,GAAM,CAYjB,GAXA9J,EAAUA,aAAmBF,GAASE,EAAS,GAAMA,EAIrDF,GAAOoB,MAAOrE,KAAMiD,GAAOmW,UAC1BnM,EAAO,GACP9J,GAAWA,EAAQ5B,SAAW4B,EAAQqE,eAAiBrE,EAAUvD,GACjE,IAIImZ,EAAWtR,KAAMwF,EAAO,KAAShK,GAAO6C,cAAe3C,GAC3D,IAAM8J,KAAS9J,EAGT9B,EAAYrB,KAAMiN,IACtBjN,KAAMiN,GAAS9J,EAAS8J,IAIxBjN,KAAKyQ,KAAMxD,EAAO9J,EAAS8J,IAK9B,OAAOjN,KAYP,OARAyD,EAAO7D,EAAS0N,eAAgBL,EAAO,OAKtCjN,KAAM,GAAMyD,EACZzD,KAAKuD,OAAS,GAERvD,KAcH,OAAKkD,EAAS3B,UACpBvB,KAAM,GAAMkD,EACZlD,KAAKuD,OAAS,EACPvD,MAIIqB,EAAY6B,QACD+C,IAAfyN,EAAK2F,MACX3F,EAAK2F,MAAOnW,GAGZA,EAAUD,IAGLA,GAAOgE,UAAW/D,EAAUlD,QAIhC4D,UAAYX,GAAOG,GAGxB+V,EAAalW,GAAQrD,GAGrB,IAAI0Z,EAAe,iCAGlBC,EAAmB,CAClBC,UAAU,EACVC,UAAU,EACVhN,MAAM,EACNiN,MAAM,GAoFR,SAASC,EAASC,EAAKpN,GACtB,OAAUoN,EAAMA,EAAKpN,KAA4B,IAAjBoN,EAAIrY,UACpC,OAAOqY,EAnFR3W,GAAOG,GAAGmC,OAAQ,CACjB8N,IAAK,SAAUzN,GACd,IAAIiU,EAAU5W,GAAQ2C,EAAQ5F,MAC7B8Z,EAAID,EAAQtW,OAEb,OAAOvD,KAAK6P,OAAQ,WAEnB,IADA,IAAI1N,EAAI,EACAA,EAAI2X,EAAG3X,IACd,GAAKc,GAAOwF,SAAUzI,KAAM6Z,EAAS1X,IACpC,OAAO,KAMX4X,QAAS,SAAUvB,EAAWrV,GAC7B,IAAIyW,EACHzX,EAAI,EACJ2X,EAAI9Z,KAAKuD,OACTyP,EAAU,GACV6G,EAA+B,iBAAdrB,GAA0BvV,GAAQuV,GAGpD,IAAMM,EAAcrR,KAAM+Q,GACzB,KAAQrW,EAAI2X,EAAG3X,IACd,IAAMyX,EAAM5Z,KAAMmC,GAAKyX,GAAOA,IAAQzW,EAASyW,EAAMA,EAAIhX,WAGxD,GAAKgX,EAAIrY,SAAW,KAAQsY,GACH,EAAxBA,EAAQG,MAAOJ,GAGE,IAAjBA,EAAIrY,UACH0B,GAAO4J,KAAK2D,gBAAiBoJ,EAAKpB,IAAgB,CAEnDxF,EAAQpS,KAAMgZ,GACd,MAMJ,OAAO5Z,KAAKkE,UAA4B,EAAjB8O,EAAQzP,OAAaN,GAAO0N,WAAYqC,GAAYA,IAI5EgH,MAAO,SAAUvW,GAGhB,OAAMA,EAKe,iBAATA,EACJ5C,GAAQJ,KAAMwC,GAAQQ,GAAQzD,KAAM,IAIrCa,GAAQJ,KAAMT,KAGpByD,EAAKI,OAASJ,EAAM,GAAMA,GAZjBzD,KAAM,IAAOA,KAAM,GAAI4C,WAAe5C,KAAK2E,QAAQsV,UAAU1W,QAAU,GAgBlF2W,IAAK,SAAUhX,EAAUC,GACxB,OAAOnD,KAAKkE,UACXjB,GAAO0N,WACN1N,GAAOoB,MAAOrE,KAAKgE,MAAOf,GAAQC,EAAUC,OAK/CgX,QAAS,SAAUjX,GAClB,OAAOlD,KAAKka,IAAiB,MAAZhX,EAChBlD,KAAKsE,WAAatE,KAAKsE,WAAWuL,OAAQ3M,OAU7CD,GAAOsB,KAAM,CACZgO,OAAQ,SAAU9O,GACjB,IAAI8O,EAAS9O,EAAKb,WAClB,OAAO2P,GAA8B,KAApBA,EAAOhR,SAAkBgR,EAAS,MAEpD6H,QAAS,SAAU3W,GAClB,OAAO+I,EAAK/I,EAAM,eAEnB4W,aAAc,SAAU5W,EAAM2E,EAAIqQ,GACjC,OAAOjM,EAAK/I,EAAM,aAAcgV,IAEjChM,KAAM,SAAUhJ,GACf,OAAOkW,EAASlW,EAAM,gBAEvBiW,KAAM,SAAUjW,GACf,OAAOkW,EAASlW,EAAM,oBAEvB6W,QAAS,SAAU7W,GAClB,OAAO+I,EAAK/I,EAAM,gBAEnBwW,QAAS,SAAUxW,GAClB,OAAO+I,EAAK/I,EAAM,oBAEnB8W,UAAW,SAAU9W,EAAM2E,EAAIqQ,GAC9B,OAAOjM,EAAK/I,EAAM,cAAegV,IAElC+B,UAAW,SAAU/W,EAAM2E,EAAIqQ,GAC9B,OAAOjM,EAAK/I,EAAM,kBAAmBgV,IAEtCG,SAAU,SAAUnV,GACnB,OAAOmV,GAAYnV,EAAKb,YAAc,IAAK8P,WAAYjP,IAExD+V,SAAU,SAAU/V,GACnB,OAAOmV,EAAUnV,EAAKiP,aAEvB+G,SAAU,SAAUhW,GACnB,OAA6B,MAAxBA,EAAKgX,iBAKTta,EAAUsD,EAAKgX,iBAERhX,EAAKgX,iBAMRjX,GAAUC,EAAM,cACpBA,EAAOA,EAAKiX,SAAWjX,GAGjBR,GAAOoB,MAAO,GAAIZ,EAAKiJ,eAE7B,SAAUhJ,EAAMN,GAClBH,GAAOG,GAAIM,GAAS,SAAU+U,EAAOvV,GACpC,IAAI8P,EAAU/P,GAAOwB,IAAKzE,KAAMoD,EAAIqV,GAuBpC,MArB0B,UAArB/U,EAAKpD,OAAQ,KACjB4C,EAAWuV,GAGPvV,GAAgC,iBAAbA,IACvB8P,EAAU/P,GAAO4M,OAAQ3M,EAAU8P,IAGjB,EAAdhT,KAAKuD,SAGHgW,EAAkB7V,IACvBT,GAAO0N,WAAYqC,GAIfsG,EAAa7R,KAAM/D,IACvBsP,EAAQ2H,WAIH3a,KAAKkE,UAAW8O,MAGzB,IAAI4H,EAAgB,oBAsOpB,SAASC,EAAUC,GAClB,OAAOA,EAER,SAASC,EAASC,GACjB,MAAMA,EAGP,SAASC,EAAYlT,EAAOmT,EAASC,EAAQC,GAC5C,IAAIC,EAEJ,IAGMtT,GAAS1G,EAAcga,EAAStT,EAAMuT,SAC1CD,EAAO5a,KAAMsH,GAAQ+B,KAAMoR,GAAUK,KAAMJ,GAGhCpT,GAAS1G,EAAcga,EAAStT,EAAMyT,MACjDH,EAAO5a,KAAMsH,EAAOmT,EAASC,GAQ7BD,EAAQva,WAAOsF,EAAW,CAAE8B,GAAQzH,MAAO8a,IAM3C,MAAQrT,GAIToT,EAAOxa,WAAOsF,EAAW,CAAE8B,KAvO7B9E,GAAOwY,UAAY,SAAUjW,GA9B7B,IAAwBA,EACnBkW,EAiCJlW,EAA6B,iBAAZA,GAlCMA,EAmCPA,EAlCZkW,EAAS,GACbzY,GAAOsB,KAAMiB,EAAQyH,MAAO2N,IAAmB,GAAI,SAAUe,EAAGC,GAC/DF,EAAQE,IAAS,IAEXF,GA+BNzY,GAAOsC,OAAQ,GAAIC,GAEpB,IACCqW,EAGAC,EAGAC,EAGAC,EAGAC,EAAO,GAGPC,EAAQ,GAGRC,GAAe,EAGfC,EAAO,WAQN,IALAJ,EAASA,GAAUxW,EAAQ6W,KAI3BN,EAAQF,GAAS,EACTK,EAAM3Y,OAAQ4Y,GAAe,EAAI,CACxCL,EAASI,EAAM3N,QACf,QAAU4N,EAAcF,EAAK1Y,QAGmC,IAA1D0Y,EAAME,GAAcxb,MAAOmb,EAAQ,GAAKA,EAAQ,KACpDtW,EAAQ8W,cAGRH,EAAcF,EAAK1Y,OACnBuY,GAAS,GAMNtW,EAAQsW,SACbA,GAAS,GAGVD,GAAS,EAGJG,IAIHC,EADIH,EACG,GAIA,KAMV5C,EAAO,CAGNgB,IAAK,WA2BJ,OA1BK+B,IAGCH,IAAWD,IACfM,EAAcF,EAAK1Y,OAAS,EAC5B2Y,EAAMtb,KAAMkb,IAGb,SAAW5B,EAAKrH,GACf5P,GAAOsB,KAAMsO,EAAM,SAAU8I,EAAG7T,GAC1BzG,EAAYyG,GACVtC,EAAQ6S,QAAWa,EAAK7F,IAAKvL,IAClCmU,EAAKrb,KAAMkH,GAEDA,GAAOA,EAAIvE,QAA4B,WAAlBT,EAAQgF,IAGxCoS,EAAKpS,KATR,CAYKpD,WAEAoX,IAAWD,GACfO,KAGKpc,MAIRuc,OAAQ,WAYP,OAXAtZ,GAAOsB,KAAMG,UAAW,SAAUiX,EAAG7T,GACpC,IAAIkS,EACJ,OAA0D,GAAhDA,EAAQ/W,GAAOkE,QAASW,EAAKmU,EAAMjC,IAC5CiC,EAAK3W,OAAQ0U,EAAO,GAGfA,GAASmC,GACbA,MAIInc,MAKRqT,IAAK,SAAUjQ,GACd,OAAOA,GACwB,EAA9BH,GAAOkE,QAAS/D,EAAI6Y,GACN,EAAdA,EAAK1Y,QAIP+Q,MAAO,WAIN,OAHK2H,IACJA,EAAO,IAEDjc,MAMRwc,QAAS,WAGR,OAFAR,EAASE,EAAQ,GACjBD,EAAOH,EAAS,GACT9b,MAERuM,SAAU,WACT,OAAQ0P,GAMTQ,KAAM,WAKL,OAJAT,EAASE,EAAQ,GACXJ,GAAWD,IAChBI,EAAOH,EAAS,IAEV9b,MAERgc,OAAQ,WACP,QAASA,GAIVU,SAAU,SAAUvZ,EAAS0P,GAS5B,OARMmJ,IAELnJ,EAAO,CAAE1P,GADT0P,EAAOA,GAAQ,IACQvS,MAAQuS,EAAKvS,QAAUuS,GAC9CqJ,EAAMtb,KAAMiS,GACNgJ,GACLO,KAGKpc,MAIRoc,KAAM,WAEL,OADAlD,EAAKwD,SAAU1c,KAAM0E,WACd1E,MAIR+b,MAAO,WACN,QAASA,IAIZ,OAAO7C,GA4CRjW,GAAOsC,OAAQ,CAEdoX,SAAU,SAAUC,GACnB,IAAIC,EAAS,CAIX,CAAE,SAAU,WAAY5Z,GAAOwY,UAAW,UACzCxY,GAAOwY,UAAW,UAAY,GAC/B,CAAE,UAAW,OAAQxY,GAAOwY,UAAW,eACtCxY,GAAOwY,UAAW,eAAiB,EAAG,YACvC,CAAE,SAAU,OAAQxY,GAAOwY,UAAW,eACrCxY,GAAOwY,UAAW,eAAiB,EAAG,aAExCqB,EAAQ,UACRxB,EAAU,CACTwB,MAAO,WACN,OAAOA,GAERC,OAAQ,WAEP,OADAC,EAASlT,KAAMpF,WAAY6W,KAAM7W,WAC1B1E,MAERid,QAAS,SAAU7Z,GAClB,OAAOkY,EAAQE,KAAM,KAAMpY,IAI5B8Z,KAAM,WACL,IAAIC,EAAMzY,UAEV,OAAOzB,GAAO0Z,SAAU,SAAUS,GACjCna,GAAOsB,KAAMsY,EAAQ,SAAUzU,EAAIiV,GAGlC,IAAIja,EAAK/B,EAAY8b,EAAKE,EAAO,MAAWF,EAAKE,EAAO,IAKxDL,EAAUK,EAAO,IAAO,WACvB,IAAIC,EAAWla,GAAMA,EAAGzC,MAAOX,KAAM0E,WAChC4Y,GAAYjc,EAAYic,EAAShC,SACrCgC,EAAShC,UACPiC,SAAUH,EAASI,QACnB1T,KAAMsT,EAASlC,SACfK,KAAM6B,EAASjC,QAEjBiC,EAAUC,EAAO,GAAM,QACtBrd,KACAoD,EAAK,CAAEka,GAAa5Y,eAKxByY,EAAM,OACH7B,WAELE,KAAM,SAAUiC,EAAaC,EAAYC,GACxC,IAAIC,EAAW,EACf,SAAS1C,EAAS2C,EAAOb,EAAUc,EAASC,GAC3C,OAAO,WACN,IAAIC,EAAOhe,KACV6S,EAAOnO,UACPuZ,EAAa,WACZ,IAAIX,EAAU9B,EAKd,KAAKqC,EAAQD,GAAb,CAQA,IAJAN,EAAWQ,EAAQnd,MAAOqd,EAAMnL,MAIdmK,EAAS1B,UAC1B,MAAM,IAAI4C,UAAW,4BAOtB1C,EAAO8B,IAKgB,iBAAbA,GACY,mBAAbA,IACRA,EAAS9B,KAGLna,EAAYma,GAGXuC,EACJvC,EAAK/a,KACJ6c,EACApC,EAAS0C,EAAUZ,EAAUnC,EAAUkD,GACvC7C,EAAS0C,EAAUZ,EAAUjC,EAASgD,KAOvCH,IAEApC,EAAK/a,KACJ6c,EACApC,EAAS0C,EAAUZ,EAAUnC,EAAUkD,GACvC7C,EAAS0C,EAAUZ,EAAUjC,EAASgD,GACtC7C,EAAS0C,EAAUZ,EAAUnC,EAC5BmC,EAASmB,eASPL,IAAYjD,IAChBmD,OAAO/X,EACP4M,EAAO,CAAEyK,KAKRS,GAAWf,EAASoB,aAAeJ,EAAMnL,MAK7CwL,EAAUN,EACTE,EACA,WACC,IACCA,IACC,MAAQtR,GAEJ1J,GAAO0Z,SAAS2B,eACpBrb,GAAO0Z,SAAS2B,cAAe3R,EAC9B0R,EAAQ9X,OAMQqX,GAAbC,EAAQ,IAIPC,IAAY/C,IAChBiD,OAAO/X,EACP4M,EAAO,CAAElG,IAGVqQ,EAASuB,WAAYP,EAAMnL,MAS3BgL,EACJQ,KAKKpb,GAAO0Z,SAAS6B,aACpBH,EAAQ9X,MAAQtD,GAAO0Z,SAAS6B,eAMrBvb,GAAO0Z,SAAS8B,eAC3BJ,EAAQ9X,MAAQtD,GAAO0Z,SAAS8B,gBAEjC1e,GAAO2e,WAAYL,KAKtB,OAAOpb,GAAO0Z,SAAU,SAAUS,GAGjCP,EAAQ,GAAK,GAAI3C,IAChBgB,EACC,EACAkC,EACA/b,EAAYsc,GACXA,EACA9C,EACDuC,EAASe,aAKXtB,EAAQ,GAAK,GAAI3C,IAChBgB,EACC,EACAkC,EACA/b,EAAYoc,GACXA,EACA5C,IAKHgC,EAAQ,GAAK,GAAI3C,IAChBgB,EACC,EACAkC,EACA/b,EAAYqc,GACXA,EACA3C,MAGAO,WAKLA,QAAS,SAAUha,GAClB,OAAc,MAAPA,EAAc2B,GAAOsC,OAAQjE,EAAKga,GAAYA,IAGvD0B,EAAW,GAkEZ,OA/DA/Z,GAAOsB,KAAMsY,EAAQ,SAAU1a,EAAGkb,GACjC,IAAIpB,EAAOoB,EAAO,GACjBsB,EAActB,EAAO,GAKtB/B,EAAS+B,EAAO,IAAQpB,EAAK/B,IAGxByE,GACJ1C,EAAK/B,IACJ,WAIC4C,EAAQ6B,GAKT9B,EAAQ,EAAI1a,GAAK,GAAIqa,QAIrBK,EAAQ,EAAI1a,GAAK,GAAIqa,QAGrBK,EAAQ,GAAK,GAAIJ,KAGjBI,EAAQ,GAAK,GAAIJ,MAOnBR,EAAK/B,IAAKmD,EAAO,GAAIjB,MAKrBY,EAAUK,EAAO,IAAQ,WAExB,OADAL,EAAUK,EAAO,GAAM,QAAUrd,OAASgd,OAAW/W,EAAYjG,KAAM0E,WAChE1E,MAMRgd,EAAUK,EAAO,GAAM,QAAWpB,EAAKS,WAIxCpB,EAAQA,QAAS0B,GAGZJ,GACJA,EAAKnc,KAAMuc,EAAUA,GAIfA,GAIR4B,KAAM,SAAUC,GACf,IAGCC,EAAYpa,UAAUnB,OAGtBpB,EAAI2c,EAGJC,EAAkBhZ,MAAO5D,GACzB6c,EAAgB1e,GAAMG,KAAMiE,WAG5Bua,EAAUhc,GAAO0Z,WAGjBuC,EAAa,SAAU/c,GACtB,OAAO,SAAU4F,GAChBgX,EAAiB5c,GAAMnC,KACvBgf,EAAe7c,GAAyB,EAAnBuC,UAAUnB,OAAajD,GAAMG,KAAMiE,WAAcqD,IAC5D+W,GACTG,EAAQb,YAAaW,EAAiBC,KAM1C,GAAKF,GAAa,IACjB7D,EAAY4D,EAAaI,EAAQnV,KAAMoV,EAAY/c,IAAM+Y,QAAS+D,EAAQ9D,QACxE2D,GAGuB,YAApBG,EAAQnC,SACZzb,EAAY2d,EAAe7c,IAAO6c,EAAe7c,GAAIqZ,OAErD,OAAOyD,EAAQzD,OAKjB,MAAQrZ,IACP8Y,EAAY+D,EAAe7c,GAAK+c,EAAY/c,GAAK8c,EAAQ9D,QAG1D,OAAO8D,EAAQ3D,aAOjB,IAAI6D,EAAc,yDAKlBlc,GAAO0Z,SAAS2B,cAAgB,SAAU/X,EAAO6Y,GAI3Crf,GAAOsf,SAAWtf,GAAOsf,QAAQC,MAAQ/Y,GAAS4Y,EAAY1X,KAAMlB,EAAM7C,OAC9E3D,GAAOsf,QAAQC,KAAM,8BAAgC/Y,EAAMgZ,QAC1DhZ,EAAMiZ,MAAOJ,IAOhBnc,GAAOwc,eAAiB,SAAUlZ,GACjCxG,GAAO2e,WAAY,WAClB,MAAMnY,KAQR,IAAImZ,EAAYzc,GAAO0Z,WAkDvB,SAASgD,IACR/f,EAASggB,oBAAqB,mBAAoBD,GAClD5f,GAAO6f,oBAAqB,OAAQD,GACpC1c,GAAOoW,QAnDRpW,GAAOG,GAAGiW,MAAQ,SAAUjW,GAY3B,OAVAsc,EACElE,KAAMpY,GAKN6Z,SAAO,SAAU1W,GACjBtD,GAAOwc,eAAgBlZ,KAGlBvG,MAGRiD,GAAOsC,OAAQ,CAGde,SAAS,EAITuZ,UAAW,EAGXxG,MAAO,SAAUyG,KAGF,IAATA,IAAkB7c,GAAO4c,UAAY5c,GAAOqD,WAKjDrD,GAAOqD,SAAU,KAGZwZ,GAAsC,IAAnB7c,GAAO4c,WAK/BH,EAAUtB,YAAaxe,EAAU,CAAEqD,QAIrCA,GAAOoW,MAAMmC,KAAOkE,EAAUlE,KAaD,aAAxB5b,EAASmgB,YACa,YAAxBngB,EAASmgB,aAA6BngB,EAASmH,gBAAgBiZ,SAGjEjgB,GAAO2e,WAAYzb,GAAOoW,QAK1BzZ,EAAS2P,iBAAkB,mBAAoBoQ,GAG/C5f,GAAOwP,iBAAkB,OAAQoQ,IAQlC,IAAIM,EAAS,SAAU9b,EAAOf,EAAIiL,EAAKtG,EAAOmY,EAAWC,EAAUC,GAClE,IAAIje,EAAI,EACP+C,EAAMf,EAAMZ,OACZ8c,EAAc,MAAPhS,EAGR,GAAuB,WAAlBvL,EAAQuL,GAEZ,IAAMlM,KADN+d,GAAY,EACD7R,EACV4R,EAAQ9b,EAAOf,EAAIjB,EAAGkM,EAAKlM,IAAK,EAAMge,EAAUC,QAI3C,QAAena,IAAV8B,IACXmY,GAAY,EAEN7e,EAAY0G,KACjBqY,GAAM,GAGFC,IAGCD,GACJhd,EAAG3C,KAAM0D,EAAO4D,GAChB3E,EAAK,OAILid,EAAOjd,EACPA,EAAK,SAAUK,EAAM6c,EAAMvY,GAC1B,OAAOsY,EAAK5f,KAAMwC,GAAQQ,GAAQsE,MAKhC3E,GACJ,KAAQjB,EAAI+C,EAAK/C,IAChBiB,EACCe,EAAOhC,GAAKkM,EAAK+R,EAChBrY,EACAA,EAAMtH,KAAM0D,EAAOhC,GAAKA,EAAGiB,EAAIe,EAAOhC,GAAKkM,KAMhD,OAAK6R,EACG/b,EAIHkc,EACGjd,EAAG3C,KAAM0D,GAGVe,EAAM9B,EAAIe,EAAO,GAAKkK,GAAQ8R,GAKlCI,EAAY,QACfC,EAAa,YAGd,SAASC,EAAYC,EAAMC,GAC1B,OAAOA,EAAOC,cAMf,SAASC,EAAWC,GACnB,OAAOA,EAAOza,QAASka,EAAW,OAAQla,QAASma,EAAYC,GAEhE,IAAIM,EAAa,SAAUC,GAQ1B,OAA0B,IAAnBA,EAAMzf,UAAqC,IAAnByf,EAAMzf,YAAsByf,EAAMzf,UAMlE,SAAS0f,IACRjhB,KAAKkG,QAAUjD,GAAOiD,QAAU+a,EAAKC,MAGtCD,EAAKC,IAAM,EAEXD,EAAKrd,UAAY,CAEhBwK,MAAO,SAAU4S,GAGhB,IAAIjZ,EAAQiZ,EAAOhhB,KAAKkG,SA4BxB,OAzBM6B,IACLA,EAAQ,GAKHgZ,EAAYC,KAIXA,EAAMzf,SACVyf,EAAOhhB,KAAKkG,SAAY6B,EAMxB3H,OAAO+gB,eAAgBH,EAAOhhB,KAAKkG,QAAS,CAC3C6B,MAAOA,EACPqZ,cAAc,MAMXrZ,GAERsZ,IAAK,SAAUL,EAAOM,EAAMvZ,GAC3B,IAAIwZ,EACHnT,EAAQpO,KAAKoO,MAAO4S,GAIrB,GAAqB,iBAATM,EACXlT,EAAOyS,EAAWS,IAAWvZ,OAM7B,IAAMwZ,KAAQD,EACblT,EAAOyS,EAAWU,IAAWD,EAAMC,GAGrC,OAAOnT,GAERpK,IAAK,SAAUgd,EAAO3S,GACrB,YAAepI,IAARoI,EACNrO,KAAKoO,MAAO4S,GAGZA,EAAOhhB,KAAKkG,UAAa8a,EAAOhhB,KAAKkG,SAAW2a,EAAWxS,KAE7D4R,OAAQ,SAAUe,EAAO3S,EAAKtG,GAa7B,YAAa9B,IAARoI,GACCA,GAAsB,iBAARA,QAAgCpI,IAAV8B,EAElC/H,KAAKgE,IAAKgd,EAAO3S,IASzBrO,KAAKqhB,IAAKL,EAAO3S,EAAKtG,QAIL9B,IAAV8B,EAAsBA,EAAQsG,IAEtCkO,OAAQ,SAAUyE,EAAO3S,GACxB,IAAIlM,EACHiM,EAAQ4S,EAAOhhB,KAAKkG,SAErB,QAAeD,IAAVmI,EAAL,CAIA,QAAanI,IAARoI,EAAoB,CAkBxBlM,GAXCkM,EAJItI,MAAMC,QAASqI,GAIbA,EAAI5J,IAAKoc,IAEfxS,EAAMwS,EAAWxS,MAIJD,EACZ,CAAEC,GACAA,EAAIpB,MAAO2N,IAAmB,IAG1BrX,OAER,MAAQpB,WACAiM,EAAOC,EAAKlM,UAKR8D,IAARoI,GAAqBpL,GAAO2D,cAAewH,MAM1C4S,EAAMzf,SACVyf,EAAOhhB,KAAKkG,cAAYD,SAEjB+a,EAAOhhB,KAAKkG,YAItBsb,QAAS,SAAUR,GAClB,IAAI5S,EAAQ4S,EAAOhhB,KAAKkG,SACxB,YAAiBD,IAAVmI,IAAwBnL,GAAO2D,cAAewH,KAGvD,IAAIqT,EAAW,IAAIR,EAEfS,EAAW,IAAIT,EAcfU,EAAS,gCACZC,EAAa,SA2Bd,SAASC,EAAUpe,EAAM4K,EAAKiT,GAC7B,IAAI5d,EA1Ba4d,EA8BjB,QAAcrb,IAATqb,GAAwC,IAAlB7d,EAAKlC,SAI/B,GAHAmC,EAAO,QAAU2K,EAAIhI,QAASub,EAAY,OAAQje,cAG7B,iBAFrB2d,EAAO7d,EAAKjB,aAAckB,IAEM,CAC/B,IACC4d,EAnCW,UADGA,EAoCEA,IA/BL,UAATA,IAIS,SAATA,EACG,KAIHA,KAAUA,EAAO,IACbA,EAGJK,EAAOla,KAAM6Z,GACVQ,KAAKC,MAAOT,GAGbA,GAeH,MAAQ3U,IAGV+U,EAASL,IAAK5d,EAAM4K,EAAKiT,QAEzBA,OAAOrb,EAGT,OAAOqb,EAGRre,GAAOsC,OAAQ,CACdic,QAAS,SAAU/d,GAClB,OAAOie,EAASF,QAAS/d,IAAUge,EAASD,QAAS/d,IAGtD6d,KAAM,SAAU7d,EAAMC,EAAM4d,GAC3B,OAAOI,EAASzB,OAAQxc,EAAMC,EAAM4d,IAGrCU,WAAY,SAAUve,EAAMC,GAC3Bge,EAASnF,OAAQ9Y,EAAMC,IAKxBue,MAAO,SAAUxe,EAAMC,EAAM4d,GAC5B,OAAOG,EAASxB,OAAQxc,EAAMC,EAAM4d,IAGrCY,YAAa,SAAUze,EAAMC,GAC5B+d,EAASlF,OAAQ9Y,EAAMC,MAIzBT,GAAOG,GAAGmC,OAAQ,CACjB+b,KAAM,SAAUjT,EAAKtG,GACpB,IAAI5F,EAAGuB,EAAM4d,EACZ7d,EAAOzD,KAAM,GACbmiB,EAAQ1e,GAAQA,EAAK8G,WAGtB,QAAatE,IAARoI,EAAoB,CACxB,GAAKrO,KAAKuD,SACT+d,EAAOI,EAAS1d,IAAKP,GAEE,IAAlBA,EAAKlC,WAAmBkgB,EAASzd,IAAKP,EAAM,iBAAmB,CACnEtB,EAAIggB,EAAM5e,OACV,MAAQpB,IAIFggB,EAAOhgB,IAEsB,KADjCuB,EAAOye,EAAOhgB,GAAIuB,MACR7C,QAAS,WAClB6C,EAAOmd,EAAWnd,EAAKpD,MAAO,IAC9BuhB,EAAUpe,EAAMC,EAAM4d,EAAM5d,KAI/B+d,EAASJ,IAAK5d,EAAM,gBAAgB,GAItC,OAAO6d,EAIR,MAAoB,iBAARjT,EACJrO,KAAKuE,KAAM,WACjBmd,EAASL,IAAKrhB,KAAMqO,KAIf4R,EAAQjgB,KAAM,SAAU+H,GAC9B,IAAIuZ,EAOJ,GAAK7d,QAAkBwC,IAAV8B,EAKZ,YAAc9B,KADdqb,EAAOI,EAAS1d,IAAKP,EAAM4K,IAEnBiT,OAMMrb,KADdqb,EAAOO,EAAUpe,EAAM4K,IAEfiT,OAIR,EAIDthB,KAAKuE,KAAM,WAGVmd,EAASL,IAAKrhB,KAAMqO,EAAKtG,MAExB,KAAMA,EAA0B,EAAnBrD,UAAUnB,OAAY,MAAM,IAG7Cye,WAAY,SAAU3T,GACrB,OAAOrO,KAAKuE,KAAM,WACjBmd,EAASnF,OAAQvc,KAAMqO,QAM1BpL,GAAOsC,OAAQ,CACd2W,MAAO,SAAUzY,EAAM9B,EAAM2f,GAC5B,IAAIpF,EAEJ,GAAKzY,EAYJ,OAXA9B,GAASA,GAAQ,MAAS,QAC1Bua,EAAQuF,EAASzd,IAAKP,EAAM9B,GAGvB2f,KACEpF,GAASnW,MAAMC,QAASsb,GAC7BpF,EAAQuF,EAASxB,OAAQxc,EAAM9B,EAAMsB,GAAOgE,UAAWqa,IAEvDpF,EAAMtb,KAAM0gB,IAGPpF,GAAS,IAIlBkG,QAAS,SAAU3e,EAAM9B,GACxBA,EAAOA,GAAQ,KAEf,IAAIua,EAAQjZ,GAAOiZ,MAAOzY,EAAM9B,GAC/B0gB,EAAcnG,EAAM3Y,OACpBH,EAAK8Y,EAAM3N,QACX+T,EAAQrf,GAAOsf,YAAa9e,EAAM9B,GAMvB,eAAPyB,IACJA,EAAK8Y,EAAM3N,QACX8T,KAGIjf,IAIU,OAATzB,GACJua,EAAMsG,QAAS,qBAITF,EAAMG,KACbrf,EAAG3C,KAAMgD,EApBF,WACNR,GAAOmf,QAAS3e,EAAM9B,IAmBF2gB,KAGhBD,GAAeC,GACpBA,EAAMhO,MAAM8H,QAKdmG,YAAa,SAAU9e,EAAM9B,GAC5B,IAAI0M,EAAM1M,EAAO,aACjB,OAAO8f,EAASzd,IAAKP,EAAM4K,IAASoT,EAASxB,OAAQxc,EAAM4K,EAAK,CAC/DiG,MAAOrR,GAAOwY,UAAW,eAAgBvB,IAAK,WAC7CuH,EAASlF,OAAQ9Y,EAAM,CAAE9B,EAAO,QAAS0M,WAM7CpL,GAAOG,GAAGmC,OAAQ,CACjB2W,MAAO,SAAUva,EAAM2f,GACtB,IAAIoB,EAAS,EAQb,MANqB,iBAAT/gB,IACX2f,EAAO3f,EACPA,EAAO,KACP+gB,KAGIhe,UAAUnB,OAASmf,EAChBzf,GAAOiZ,MAAOlc,KAAM,GAAK2B,QAGjBsE,IAATqb,EACNthB,KACAA,KAAKuE,KAAM,WACV,IAAI2X,EAAQjZ,GAAOiZ,MAAOlc,KAAM2B,EAAM2f,GAGtCre,GAAOsf,YAAaviB,KAAM2B,GAEZ,OAATA,GAAgC,eAAfua,EAAO,IAC5BjZ,GAAOmf,QAASpiB,KAAM2B,MAI1BygB,QAAS,SAAUzgB,GAClB,OAAO3B,KAAKuE,KAAM,WACjBtB,GAAOmf,QAASpiB,KAAM2B,MAGxBghB,WAAY,SAAUhhB,GACrB,OAAO3B,KAAKkc,MAAOva,GAAQ,KAAM,KAKlC2Z,QAAS,SAAU3Z,EAAML,GACxB,IAAIshB,EACHC,EAAQ,EACRC,EAAQ7f,GAAO0Z,WACfpM,EAAWvQ,KACXmC,EAAInC,KAAKuD,OACT2X,EAAU,aACC2H,GACTC,EAAM1E,YAAa7N,EAAU,CAAEA,KAIb,iBAAT5O,IACXL,EAAMK,EACNA,OAAOsE,GAERtE,EAAOA,GAAQ,KAEf,MAAQQ,KACPygB,EAAMnB,EAASzd,IAAKuM,EAAUpO,GAAKR,EAAO,gBAC9BihB,EAAItO,QACfuO,IACAD,EAAItO,MAAM4F,IAAKgB,IAIjB,OADAA,IACO4H,EAAMxH,QAASha,MAGxB,IAAIyhB,EAAO,sCAA0CC,OAEjDC,EAAU,IAAIza,OAAQ,iBAAmBua,EAAO,cAAe,KAG/DG,EAAY,CAAE,MAAO,QAAS,SAAU,QAExCnc,EAAkBnH,EAASmH,gBAI1Boc,EAAa,SAAU1f,GACzB,OAAOR,GAAOwF,SAAUhF,EAAK+D,cAAe/D,IAE7C2f,EAAW,CAAEA,UAAU,GAOnBrc,EAAgBsc,cACpBF,EAAa,SAAU1f,GACtB,OAAOR,GAAOwF,SAAUhF,EAAK+D,cAAe/D,IAC3CA,EAAK4f,YAAaD,KAAe3f,EAAK+D,gBAG1C,IAAI8b,GAAqB,SAAU7f,EAAMiL,GAOvC,MAA8B,UAH9BjL,EAAOiL,GAAMjL,GAGD8f,MAAMC,SACM,KAAvB/f,EAAK8f,MAAMC,SAMXL,EAAY1f,IAEsB,SAAlCR,GAAOwgB,IAAKhgB,EAAM,YAKrB,SAASigB,GAAWjgB,EAAM8d,EAAMoC,EAAYC,GAC3C,IAAIC,EAAUC,EACbC,EAAgB,GAChBC,EAAeJ,EACd,WACC,OAAOA,EAAMhK,OAEd,WACC,OAAO3W,GAAOwgB,IAAKhgB,EAAM8d,EAAM,KAEjC0C,EAAUD,IACVE,EAAOP,GAAcA,EAAY,KAAS1gB,GAAOkhB,UAAW5C,GAAS,GAAK,MAG1E6C,EAAgB3gB,EAAKlC,WAClB0B,GAAOkhB,UAAW5C,IAAmB,OAAT2C,IAAkBD,IAChDhB,EAAQ5V,KAAMpK,GAAOwgB,IAAKhgB,EAAM8d,IAElC,GAAK6C,GAAiBA,EAAe,KAAQF,EAAO,CAInDD,GAAoB,EAGpBC,EAAOA,GAAQE,EAAe,GAG9BA,GAAiBH,GAAW,EAE5B,MAAQF,IAIP9gB,GAAOsgB,MAAO9f,EAAM8d,EAAM6C,EAAgBF,IACnC,EAAIJ,IAAY,GAAMA,EAAQE,IAAiBC,GAAW,MAAW,IAC3EF,EAAgB,GAEjBK,GAAgCN,EAIjCM,GAAgC,EAChCnhB,GAAOsgB,MAAO9f,EAAM8d,EAAM6C,EAAgBF,GAG1CP,EAAaA,GAAc,GAgB5B,OAbKA,IACJS,GAAiBA,IAAkBH,GAAW,EAG9CJ,EAAWF,EAAY,GACtBS,GAAkBT,EAAY,GAAM,GAAMA,EAAY,IACrDA,EAAY,GACTC,IACJA,EAAMM,KAAOA,EACbN,EAAMtR,MAAQ8R,EACdR,EAAMxe,IAAMye,IAGPA,EAIR,IAAIQ,GAAoB,GAyBxB,SAASC,GAAU/T,EAAUgU,GAO5B,IANA,IAAIf,EAAS/f,EAxBcA,EACvBiT,EACHxU,EACAsB,EACAggB,EAqBAgB,EAAS,GACTxK,EAAQ,EACRzW,EAASgN,EAAShN,OAGXyW,EAAQzW,EAAQyW,KACvBvW,EAAO8M,EAAUyJ,IACNuJ,QAIXC,EAAU/f,EAAK8f,MAAMC,QAChBe,GAKa,SAAZf,IACJgB,EAAQxK,GAAUyH,EAASzd,IAAKP,EAAM,YAAe,KAC/C+gB,EAAQxK,KACbvW,EAAK8f,MAAMC,QAAU,KAGK,KAAvB/f,EAAK8f,MAAMC,SAAkBF,GAAoB7f,KACrD+gB,EAAQxK,IA7CVwJ,EAFAthB,EADGwU,OAAAA,EACHxU,GAF0BuB,EAiDaA,GA/C5B+D,cACXhE,EAAWC,EAAKD,UAChBggB,EAAUa,GAAmB7gB,MAM9BkT,EAAOxU,EAAIuiB,KAAK9hB,YAAaT,EAAII,cAAekB,IAChDggB,EAAUvgB,GAAOwgB,IAAK/M,EAAM,WAE5BA,EAAK9T,WAAWC,YAAa6T,GAEZ,SAAZ8M,IACJA,EAAU,SAEXa,GAAmB7gB,GAAaggB,MAkCb,SAAZA,IACJgB,EAAQxK,GAAU,OAGlByH,EAASJ,IAAK5d,EAAM,UAAW+f,KAMlC,IAAMxJ,EAAQ,EAAGA,EAAQzW,EAAQyW,IACR,MAAnBwK,EAAQxK,KACZzJ,EAAUyJ,GAAQuJ,MAAMC,QAAUgB,EAAQxK,IAI5C,OAAOzJ,EAGRtN,GAAOG,GAAGmC,OAAQ,CACjBgf,KAAM,WACL,OAAOD,GAAUtkB,MAAM,IAExB0kB,KAAM,WACL,OAAOJ,GAAUtkB,OAElB2kB,OAAQ,SAAU7H,GACjB,MAAsB,kBAAVA,EACJA,EAAQ9c,KAAKukB,OAASvkB,KAAK0kB,OAG5B1kB,KAAKuE,KAAM,WACZ+e,GAAoBtjB,MACxBiD,GAAQjD,MAAOukB,OAEfthB,GAAQjD,MAAO0kB,YAKnB,IAUEE,GACA1U,GAXE2U,GAAiB,wBAEjBC,GAAW,iCAEXC,GAAc,qCAMhBH,GADchlB,EAASolB,yBACRriB,YAAa/C,EAAS0C,cAAe,SACpD4N,GAAQtQ,EAAS0C,cAAe,UAM3BG,aAAc,OAAQ,SAC5ByN,GAAMzN,aAAc,UAAW,WAC/ByN,GAAMzN,aAAc,OAAQ,KAE5BmiB,GAAIjiB,YAAauN,IAIjB9O,GAAQ6jB,WAAaL,GAAIM,WAAW,GAAOA,WAAW,GAAOvS,UAAUwB,QAIvEyQ,GAAIzU,UAAY,yBAChB/O,GAAQ+jB,iBAAmBP,GAAIM,WAAW,GAAOvS,UAAUyS,aAK3DR,GAAIzU,UAAY,oBAChB/O,GAAQikB,SAAWT,GAAIjS,UAKxB,IAAI2S,GAAU,CAKbC,MAAO,CAAE,EAAG,UAAW,YACvBC,IAAK,CAAE,EAAG,oBAAqB,uBAC/BC,GAAI,CAAE,EAAG,iBAAkB,oBAC3BC,GAAI,CAAE,EAAG,qBAAsB,yBAE/BC,SAAU,CAAE,EAAG,GAAI,KAYpB,SAASC,GAAQziB,EAAS6M,GAIzB,IAAI5L,EAYJ,OATCA,EAD4C,oBAAjCjB,EAAQqK,qBACbrK,EAAQqK,qBAAsBwC,GAAO,KAEI,oBAA7B7M,EAAQ4K,iBACpB5K,EAAQ4K,iBAAkBiC,GAAO,KAGjC,QAGM/J,IAAR+J,GAAqBA,GAAOxM,GAAUL,EAAS6M,GAC5C/M,GAAOoB,MAAO,CAAElB,GAAWiB,GAG5BA,EAKR,SAASyhB,GAAe1hB,EAAO2hB,GAI9B,IAHA,IAAI3jB,EAAI,EACP2X,EAAI3V,EAAMZ,OAEHpB,EAAI2X,EAAG3X,IACdsf,EAASJ,IACRld,EAAOhC,GACP,cACC2jB,GAAerE,EAASzd,IAAK8hB,EAAa3jB,GAAK,eA1CnDmjB,GAAQS,MAAQT,GAAQU,MAAQV,GAAQW,SAAWX,GAAQY,QAAUZ,GAAQC,MAC7ED,GAAQa,GAAKb,GAAQI,GAGftkB,GAAQikB,SACbC,GAAQc,SAAWd,GAAQD,OAAS,CAAE,EAAG,+BAAgC,cA2C1E,IAAIgB,GAAQ,YAEZ,SAASC,GAAeniB,EAAOhB,EAASojB,EAASC,EAAWC,GAO3D,IANA,IAAIhjB,EAAMmf,EAAK5S,EAAK0W,EAAMC,EAAUxhB,EACnCyhB,EAAWzjB,EAAQ6hB,yBACnB6B,EAAQ,GACR1kB,EAAI,EACJ2X,EAAI3V,EAAMZ,OAEHpB,EAAI2X,EAAG3X,IAGd,IAFAsB,EAAOU,EAAOhC,KAEQ,IAATsB,EAGZ,GAAwB,WAAnBX,EAAQW,GAIZR,GAAOoB,MAAOwiB,EAAOpjB,EAAKlC,SAAW,CAAEkC,GAASA,QAG1C,GAAM4iB,GAAM5e,KAAMhE,GAIlB,CACNmf,EAAMA,GAAOgE,EAASjkB,YAAaQ,EAAQb,cAAe,QAG1D0N,GAAQ8U,GAASzX,KAAM5J,IAAU,CAAE,GAAI,KAAQ,GAAIE,cACnD+iB,EAAOpB,GAAStV,IAASsV,GAAQK,SACjC/C,EAAIzS,UAAYuW,EAAM,GAAMzjB,GAAO6jB,cAAerjB,GAASijB,EAAM,GAGjEvhB,EAAIuhB,EAAM,GACV,MAAQvhB,IACPyd,EAAMA,EAAIjQ,UAKX1P,GAAOoB,MAAOwiB,EAAOjE,EAAIlW,aAGzBkW,EAAMgE,EAASlU,YAGX5L,YAAc,QAzBlB+f,EAAMjmB,KAAMuC,EAAQ4jB,eAAgBtjB,IA+BvCmjB,EAAS9f,YAAc,GAEvB3E,EAAI,EACJ,MAAUsB,EAAOojB,EAAO1kB,KAGvB,GAAKqkB,IAAkD,EAArCvjB,GAAOkE,QAAS1D,EAAM+iB,GAClCC,GACJA,EAAQ7lB,KAAM6C,QAgBhB,GAXAkjB,EAAWxD,EAAY1f,GAGvBmf,EAAMgD,GAAQgB,EAASjkB,YAAac,GAAQ,UAGvCkjB,GACJd,GAAejD,GAIX2D,EAAU,CACdphB,EAAI,EACJ,MAAU1B,EAAOmf,EAAKzd,KAChB4f,GAAYtd,KAAMhE,EAAK9B,MAAQ,KACnC4kB,EAAQ3lB,KAAM6C,GAMlB,OAAOmjB,EAIR,IAAII,GAAiB,sBAErB,SAASC,KACR,OAAO,EAGR,SAASC,KACR,OAAO,EAGR,SAASC,GAAI1jB,EAAM2jB,EAAOlkB,EAAUoe,EAAMle,EAAIikB,GAC7C,IAAIC,EAAQ3lB,EAGZ,GAAsB,iBAAVylB,EAAqB,CAShC,IAAMzlB,IANmB,iBAAbuB,IAGXoe,EAAOA,GAAQpe,EACfA,OAAW+C,GAEEmhB,EACbD,GAAI1jB,EAAM9B,EAAMuB,EAAUoe,EAAM8F,EAAOzlB,GAAQ0lB,GAEhD,OAAO5jB,EAsBR,GAnBa,MAAR6d,GAAsB,MAANle,GAGpBA,EAAKF,EACLoe,EAAOpe,OAAW+C,GACD,MAAN7C,IACc,iBAAbF,GAGXE,EAAKke,EACLA,OAAOrb,IAIP7C,EAAKke,EACLA,EAAOpe,EACPA,OAAW+C,KAGD,IAAP7C,EACJA,EAAK8jB,QACC,IAAM9jB,EACZ,OAAOK,EAeR,OAZa,IAAR4jB,IACJC,EAASlkB,GACTA,EAAK,SAAUmkB,GAId,OADAtkB,KAASukB,IAAKD,GACPD,EAAO3mB,MAAOX,KAAM0E,aAIzBsD,KAAOsf,EAAOtf,OAAUsf,EAAOtf,KAAO/E,GAAO+E,SAE1CvE,EAAKc,KAAM,WACjBtB,GAAOskB,MAAMrN,IAAKla,KAAMonB,EAAOhkB,EAAIke,EAAMpe,KA+a3C,SAASukB,GAAgB/Y,EAAI/M,EAAM+lB,GAG5BA,GAQNjG,EAASJ,IAAK3S,EAAI/M,GAAM,GACxBsB,GAAOskB,MAAMrN,IAAKxL,EAAI/M,EAAM,CAC3B0F,WAAW,EACXyW,QAAS,SAAUyJ,GAClB,IAAI3V,EACH+V,EAAQlG,EAASzd,IAAKhE,KAAM2B,GAE7B,GAAyB,EAAlB4lB,EAAMK,WAAmB5nB,KAAM2B,IAGrC,GAAMgmB,GA4BQ1kB,GAAOskB,MAAMxJ,QAASpc,IAAU,IAAKkmB,cAClDN,EAAMO,uBAhBN,GARAH,EAAQrnB,GAAMG,KAAMiE,WACpB+c,EAASJ,IAAKrhB,KAAM2B,EAAMgmB,GAG1B3nB,KAAM2B,KACNiQ,EAAS6P,EAASzd,IAAKhE,KAAM2B,GAC7B8f,EAASJ,IAAKrhB,KAAM2B,GAAM,GAErBgmB,IAAU/V,EAMd,OAHA2V,EAAMQ,2BACNR,EAAMS,iBAECpW,OAeE+V,IAGXlG,EAASJ,IAAKrhB,KAAM2B,EAAMsB,GAAOskB,MAAMU,QACtCN,EAAO,GACPA,EAAMrnB,MAAO,GACbN,OAWDunB,EAAMO,kBACNP,EAAMW,8BAAgCjB,aArENhhB,IAA7Bwb,EAASzd,IAAK0K,EAAI/M,IACtBsB,GAAOskB,MAAMrN,IAAKxL,EAAI/M,EAAMslB,IA5a/BhkB,GAAOskB,MAAQ,CAEd/nB,OAAQ,GAER0a,IAAK,SAAUzW,EAAM2jB,EAAOtJ,EAASwD,EAAMpe,GAE1C,IAAIilB,EAAaC,EAAaxF,EAC7ByF,EAAQC,EAAGC,EACXxK,EAASyK,EAAU7mB,EAAM8mB,EAAYC,EACrCC,EAAWlH,EAASzd,IAAKP,GAG1B,GAAMsd,EAAYtd,GAAlB,CAKKqa,EAAQA,UAEZA,GADAqK,EAAcrK,GACQA,QACtB5a,EAAWilB,EAAYjlB,UAKnBA,GACJD,GAAO4J,KAAK2D,gBAAiBzJ,EAAiB7D,GAIzC4a,EAAQ9V,OACb8V,EAAQ9V,KAAO/E,GAAO+E,SAIfqgB,EAASM,EAASN,UACzBA,EAASM,EAASN,OAASjoB,OAAOwoB,OAAQ,QAEnCR,EAAcO,EAASE,UAC9BT,EAAcO,EAASE,OAAS,SAAUlc,GAIzC,MAAyB,oBAAX1J,IAA0BA,GAAOskB,MAAMuB,YAAcnc,EAAEhL,KACpEsB,GAAOskB,MAAMwB,SAASpoB,MAAO8C,EAAMiB,gBAAcuB,IAMpDqiB,GADAlB,GAAUA,GAAS,IAAKna,MAAO2N,IAAmB,CAAE,KAC1CrX,OACV,MAAQ+kB,IAEP3mB,EAAO+mB,GADP9F,EAAMoE,GAAe3Z,KAAM+Z,EAAOkB,KAAS,IACpB,GACvBG,GAAe7F,EAAK,IAAO,IAAKza,MAAO,KAAM9C,OAGvC1D,IAKNoc,EAAU9a,GAAOskB,MAAMxJ,QAASpc,IAAU,GAG1CA,GAASuB,EAAW6a,EAAQ8J,aAAe9J,EAAQiL,WAAcrnB,EAGjEoc,EAAU9a,GAAOskB,MAAMxJ,QAASpc,IAAU,GAG1C4mB,EAAYtlB,GAAOsC,OAAQ,CAC1B5D,KAAMA,EACN+mB,SAAUA,EACVpH,KAAMA,EACNxD,QAASA,EACT9V,KAAM8V,EAAQ9V,KACd9E,SAAUA,EACVqI,aAAcrI,GAAYD,GAAOqN,KAAKrD,MAAM1B,aAAa9D,KAAMvE,GAC/DmE,UAAWohB,EAAW3a,KAAM,MAC1Bqa,IAGKK,EAAWH,EAAQ1mB,OAC1B6mB,EAAWH,EAAQ1mB,GAAS,IACnBsnB,cAAgB,EAGnBlL,EAAQmL,QACiD,IAA9DnL,EAAQmL,MAAMzoB,KAAMgD,EAAM6d,EAAMmH,EAAYL,IAEvC3kB,EAAK8L,kBACT9L,EAAK8L,iBAAkB5N,EAAMymB,IAK3BrK,EAAQ7D,MACZ6D,EAAQ7D,IAAIzZ,KAAMgD,EAAM8kB,GAElBA,EAAUzK,QAAQ9V,OACvBugB,EAAUzK,QAAQ9V,KAAO8V,EAAQ9V,OAK9B9E,EACJslB,EAASljB,OAAQkjB,EAASS,gBAAiB,EAAGV,GAE9CC,EAAS5nB,KAAM2nB,GAIhBtlB,GAAOskB,MAAM/nB,OAAQmC,IAAS,KAMhC4a,OAAQ,SAAU9Y,EAAM2jB,EAAOtJ,EAAS5a,EAAUimB,GAEjD,IAAIhkB,EAAGikB,EAAWxG,EACjByF,EAAQC,EAAGC,EACXxK,EAASyK,EAAU7mB,EAAM8mB,EAAYC,EACrCC,EAAWlH,EAASD,QAAS/d,IAAUge,EAASzd,IAAKP,GAEtD,GAAMklB,IAAeN,EAASM,EAASN,QAAvC,CAMAC,GADAlB,GAAUA,GAAS,IAAKna,MAAO2N,IAAmB,CAAE,KAC1CrX,OACV,MAAQ+kB,IAMP,GAJA3mB,EAAO+mB,GADP9F,EAAMoE,GAAe3Z,KAAM+Z,EAAOkB,KAAS,IACpB,GACvBG,GAAe7F,EAAK,IAAO,IAAKza,MAAO,KAAM9C,OAGvC1D,EAAN,CAOAoc,EAAU9a,GAAOskB,MAAMxJ,QAASpc,IAAU,GAE1C6mB,EAAWH,EADX1mB,GAASuB,EAAW6a,EAAQ8J,aAAe9J,EAAQiL,WAAcrnB,IACpC,GAC7BihB,EAAMA,EAAK,IACV,IAAIpa,OAAQ,UAAYigB,EAAW3a,KAAM,iBAAoB,WAG9Dsb,EAAYjkB,EAAIqjB,EAASjlB,OACzB,MAAQ4B,IACPojB,EAAYC,EAAUrjB,IAEfgkB,GAAeT,IAAaH,EAAUG,UACzC5K,GAAWA,EAAQ9V,OAASugB,EAAUvgB,MACtC4a,IAAOA,EAAInb,KAAM8gB,EAAUlhB,YAC3BnE,GAAYA,IAAaqlB,EAAUrlB,WACxB,OAAbA,IAAqBqlB,EAAUrlB,YAChCslB,EAASljB,OAAQH,EAAG,GAEfojB,EAAUrlB,UACdslB,EAASS,gBAELlL,EAAQxB,QACZwB,EAAQxB,OAAO9b,KAAMgD,EAAM8kB,IAOzBa,IAAcZ,EAASjlB,SACrBwa,EAAQsL,WACkD,IAA/DtL,EAAQsL,SAAS5oB,KAAMgD,EAAMglB,EAAYE,EAASE,SAElD5lB,GAAOqmB,YAAa7lB,EAAM9B,EAAMgnB,EAASE,eAGnCR,EAAQ1mB,SA1Cf,IAAMA,KAAQ0mB,EACbplB,GAAOskB,MAAMhL,OAAQ9Y,EAAM9B,EAAOylB,EAAOkB,GAAKxK,EAAS5a,GAAU,GA8C/DD,GAAO2D,cAAeyhB,IAC1B5G,EAASlF,OAAQ9Y,EAAM,mBAIzBslB,SAAU,SAAUQ,GAEnB,IAAIpnB,EAAGgD,EAAGf,EAAK4O,EAASuV,EAAWiB,EAClC3W,EAAO,IAAI9M,MAAOrB,UAAUnB,QAG5BgkB,EAAQtkB,GAAOskB,MAAMkC,IAAKF,GAE1Bf,GACC/G,EAASzd,IAAKhE,KAAM,WAAcI,OAAOwoB,OAAQ,OAC/CrB,EAAM5lB,OAAU,GACnBoc,EAAU9a,GAAOskB,MAAMxJ,QAASwJ,EAAM5lB,OAAU,GAKjD,IAFAkR,EAAM,GAAM0U,EAENplB,EAAI,EAAGA,EAAIuC,UAAUnB,OAAQpB,IAClC0Q,EAAM1Q,GAAMuC,UAAWvC,GAMxB,GAHAolB,EAAMmC,eAAiB1pB,MAGlB+d,EAAQ4L,cAA2D,IAA5C5L,EAAQ4L,YAAYlpB,KAAMT,KAAMunB,GAA5D,CAKAiC,EAAevmB,GAAOskB,MAAMiB,SAAS/nB,KAAMT,KAAMunB,EAAOiB,GAGxDrmB,EAAI,EACJ,OAAU6Q,EAAUwW,EAAcrnB,QAAYolB,EAAMqC,uBAAyB,CAC5ErC,EAAMsC,cAAgB7W,EAAQvP,KAE9B0B,EAAI,EACJ,OAAUojB,EAAYvV,EAAQwV,SAAUrjB,QACtCoiB,EAAMW,gCAIDX,EAAMuC,aAAsC,IAAxBvB,EAAUlhB,YACnCkgB,EAAMuC,WAAWriB,KAAM8gB,EAAUlhB,aAEjCkgB,EAAMgB,UAAYA,EAClBhB,EAAMjG,KAAOiH,EAAUjH,UAKVrb,KAHb7B,IAAUnB,GAAOskB,MAAMxJ,QAASwK,EAAUG,WAAc,IAAKG,QAC5DN,EAAUzK,SAAUnd,MAAOqS,EAAQvP,KAAMoP,MAGT,KAAzB0U,EAAM3V,OAASxN,KACrBmjB,EAAMS,iBACNT,EAAMO,oBAYX,OAJK/J,EAAQgM,cACZhM,EAAQgM,aAAatpB,KAAMT,KAAMunB,GAG3BA,EAAM3V,SAGd4W,SAAU,SAAUjB,EAAOiB,GAC1B,IAAIrmB,EAAGomB,EAAWnf,EAAK4gB,EAAiBC,EACvCT,EAAe,GACfP,EAAgBT,EAASS,cACzBrP,EAAM2N,EAAM3hB,OAGb,GAAKqjB,GAIJrP,EAAIrY,YAOc,UAAfgmB,EAAM5lB,MAAoC,GAAhB4lB,EAAM9S,QAEnC,KAAQmF,IAAQ5Z,KAAM4Z,EAAMA,EAAIhX,YAAc5C,KAI7C,GAAsB,IAAjB4Z,EAAIrY,WAAoC,UAAfgmB,EAAM5lB,OAAqC,IAAjBiY,EAAIrN,UAAsB,CAGjF,IAFAyd,EAAkB,GAClBC,EAAmB,GACb9nB,EAAI,EAAGA,EAAI8mB,EAAe9mB,SAME8D,IAA5BgkB,EAFL7gB,GAHAmf,EAAYC,EAAUrmB,IAGNe,SAAW,OAG1B+mB,EAAkB7gB,GAAQmf,EAAUhd,cACC,EAApCtI,GAAQmG,EAAKpJ,MAAOga,MAAOJ,GAC3B3W,GAAO4J,KAAMzD,EAAKpJ,KAAM,KAAM,CAAE4Z,IAAQrW,QAErC0mB,EAAkB7gB,IACtB4gB,EAAgBppB,KAAM2nB,GAGnByB,EAAgBzmB,QACpBimB,EAAa5oB,KAAM,CAAE6C,KAAMmW,EAAK4O,SAAUwB,IAY9C,OALApQ,EAAM5Z,KACDipB,EAAgBT,EAASjlB,QAC7BimB,EAAa5oB,KAAM,CAAE6C,KAAMmW,EAAK4O,SAAUA,EAASloB,MAAO2oB,KAGpDO,GAGRU,QAAS,SAAUxmB,EAAMymB,GACxB/pB,OAAO+gB,eAAgBle,GAAOmnB,MAAMxmB,UAAWF,EAAM,CACpD2mB,YAAY,EACZjJ,cAAc,EAEdpd,IAAK3C,EAAY8oB,GAChB,WACC,GAAKnqB,KAAKsqB,cACT,OAAOH,EAAMnqB,KAAKsqB,gBAGpB,WACC,GAAKtqB,KAAKsqB,cACT,OAAOtqB,KAAKsqB,cAAe5mB,IAI9B2d,IAAK,SAAUtZ,GACd3H,OAAO+gB,eAAgBnhB,KAAM0D,EAAM,CAClC2mB,YAAY,EACZjJ,cAAc,EACdmJ,UAAU,EACVxiB,MAAOA,QAMX0hB,IAAK,SAAUa,GACd,OAAOA,EAAernB,GAAOiD,SAC5BokB,EACA,IAAIrnB,GAAOmnB,MAAOE,IAGpBvM,QAAS,CACRyM,KAAM,CAGLC,UAAU,GAEXC,MAAO,CAGNxB,MAAO,SAAU5H,GAIhB,IAAI5S,EAAK1O,MAAQshB,EAWjB,OARKuD,GAAepd,KAAMiH,EAAG/M,OAC5B+M,EAAGgc,OAASlnB,GAAUkL,EAAI,UAG1B+Y,GAAgB/Y,EAAI,SAAS,IAIvB,GAERuZ,QAAS,SAAU3G,GAIlB,IAAI5S,EAAK1O,MAAQshB,EAUjB,OAPKuD,GAAepd,KAAMiH,EAAG/M,OAC5B+M,EAAGgc,OAASlnB,GAAUkL,EAAI,UAE1B+Y,GAAgB/Y,EAAI,UAId,GAKRiX,SAAU,SAAU4B,GACnB,IAAI3hB,EAAS2hB,EAAM3hB,OACnB,OAAOif,GAAepd,KAAM7B,EAAOjE,OAClCiE,EAAO8kB,OAASlnB,GAAUoC,EAAQ,UAClC6b,EAASzd,IAAK4B,EAAQ,UACtBpC,GAAUoC,EAAQ,OAIrB+kB,aAAc,CACbZ,aAAc,SAAUxC,QAIDthB,IAAjBshB,EAAM3V,QAAwB2V,EAAM+C,gBACxC/C,EAAM+C,cAAcM,YAAcrD,EAAM3V,YA0F7C3O,GAAOqmB,YAAc,SAAU7lB,EAAM9B,EAAMknB,GAGrCplB,EAAKmc,qBACTnc,EAAKmc,oBAAqBje,EAAMknB,IAIlC5lB,GAAOmnB,MAAQ,SAAUxoB,EAAKipB,GAG7B,KAAQ7qB,gBAAgBiD,GAAOmnB,OAC9B,OAAO,IAAInnB,GAAOmnB,MAAOxoB,EAAKipB,GAI1BjpB,GAAOA,EAAID,MACf3B,KAAKsqB,cAAgB1oB,EACrB5B,KAAK2B,KAAOC,EAAID,KAIhB3B,KAAK8qB,mBAAqBlpB,EAAImpB,uBACH9kB,IAAzBrE,EAAImpB,mBAGgB,IAApBnpB,EAAIgpB,YACL3D,GACAC,GAKDlnB,KAAK4F,OAAWhE,EAAIgE,QAAkC,IAAxBhE,EAAIgE,OAAOrE,SACxCK,EAAIgE,OAAOhD,WACXhB,EAAIgE,OAEL5F,KAAK6pB,cAAgBjoB,EAAIioB,cACzB7pB,KAAKgrB,cAAgBppB,EAAIopB,eAIzBhrB,KAAK2B,KAAOC,EAIRipB,GACJ5nB,GAAOsC,OAAQvF,KAAM6qB,GAItB7qB,KAAKirB,UAAYrpB,GAAOA,EAAIqpB,WAAaC,KAAKC,MAG9CnrB,KAAMiD,GAAOiD,UAAY,GAK1BjD,GAAOmnB,MAAMxmB,UAAY,CACxBE,YAAab,GAAOmnB,MACpBU,mBAAoB5D,GACpB0C,qBAAsB1C,GACtBgB,8BAA+BhB,GAC/BkE,aAAa,EAEbpD,eAAgB,WACf,IAAIrb,EAAI3M,KAAKsqB,cAEbtqB,KAAK8qB,mBAAqB7D,GAErBta,IAAM3M,KAAKorB,aACfze,EAAEqb,kBAGJF,gBAAiB,WAChB,IAAInb,EAAI3M,KAAKsqB,cAEbtqB,KAAK4pB,qBAAuB3C,GAEvBta,IAAM3M,KAAKorB,aACfze,EAAEmb,mBAGJC,yBAA0B,WACzB,IAAIpb,EAAI3M,KAAKsqB,cAEbtqB,KAAKkoB,8BAAgCjB,GAEhCta,IAAM3M,KAAKorB,aACfze,EAAEob,2BAGH/nB,KAAK8nB,oBAKP7kB,GAAOsB,KAAM,CACZ8mB,QAAQ,EACRC,SAAS,EACTC,YAAY,EACZC,gBAAgB,EAChBC,SAAS,EACTC,QAAQ,EACRC,YAAY,EACZC,SAAS,EACTC,OAAO,EACPC,OAAO,EACPC,UAAU,EACVC,MAAM,EACNC,QAAQ,EACRjqB,MAAM,EACNkqB,UAAU,EACV7d,KAAK,EACL8d,SAAS,EACT1X,QAAQ,EACR2X,SAAS,EACTC,SAAS,EACTC,SAAS,EACTC,SAAS,EACTC,SAAS,EACTC,WAAW,EACXC,aAAa,EACbC,SAAS,EACTC,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,SAAS,EACTC,OAAO,GACL/pB,GAAOskB,MAAM2C,SAEhBjnB,GAAOsB,KAAM,CAAEoP,MAAO,UAAWsZ,KAAM,YAAc,SAAUtrB,EAAMkmB,GAEpE,SAASqF,EAAoB3D,GAC5B,GAAK3pB,EAASutB,aAAe,CAS5B,IAAItE,EAASpH,EAASzd,IAAKhE,KAAM,UAChCunB,EAAQtkB,GAAOskB,MAAMkC,IAAKF,GAC3BhC,EAAM5lB,KAA4B,YAArB4nB,EAAY5nB,KAAqB,QAAU,OACxD4lB,EAAM6D,aAAc,EAGpBvC,EAAQU,GAMHhC,EAAM3hB,SAAW2hB,EAAMsC,eAK3BhB,EAAQtB,QAMTtkB,GAAOskB,MAAM6F,SAAUvF,EAAc0B,EAAY3jB,OAChD3C,GAAOskB,MAAMkC,IAAKF,IAIrBtmB,GAAOskB,MAAMxJ,QAASpc,GAAS,CAG9BunB,MAAO,WAEN,IAAImE,EAOJ,GAFA5F,GAAgBznB,KAAM2B,GAAM,IAEvB/B,EAASutB,aAcb,OAAO,GARPE,EAAW5L,EAASzd,IAAKhE,KAAM6nB,KAE9B7nB,KAAKuP,iBAAkBsY,EAAcqF,GAEtCzL,EAASJ,IAAKrhB,KAAM6nB,GAAgBwF,GAAY,GAAM,IAOxDpF,QAAS,WAMR,OAHAR,GAAgBznB,KAAM2B,IAGf,GAGR0nB,SAAU,WACT,IAAIgE,EAEJ,IAAKztB,EAASutB,aAWb,OAAO,GAVPE,EAAW5L,EAASzd,IAAKhE,KAAM6nB,GAAiB,GAK/CpG,EAASJ,IAAKrhB,KAAM6nB,EAAcwF,IAHlCrtB,KAAK4f,oBAAqBiI,EAAcqF,GACxCzL,EAASlF,OAAQvc,KAAM6nB,KAa1BlC,SAAU,SAAU4B,GACnB,OAAO9F,EAASzd,IAAKujB,EAAM3hB,OAAQjE,IAGpCkmB,aAAcA,GAef5kB,GAAOskB,MAAMxJ,QAAS8J,GAAiB,CACtCqB,MAAO,WAIN,IAAIhnB,EAAMlC,KAAKwH,eAAiBxH,KAAKJ,UAAYI,KAChDstB,EAAa1tB,EAASutB,aAAentB,KAAOkC,EAC5CmrB,EAAW5L,EAASzd,IAAKspB,EAAYzF,GAMhCwF,IACAztB,EAASutB,aACbntB,KAAKuP,iBAAkBsY,EAAcqF,GAErChrB,EAAIqN,iBAAkB5N,EAAMurB,GAAoB,IAGlDzL,EAASJ,IAAKiM,EAAYzF,GAAgBwF,GAAY,GAAM,IAE7DhE,SAAU,WACT,IAAInnB,EAAMlC,KAAKwH,eAAiBxH,KAAKJ,UAAYI,KAChDstB,EAAa1tB,EAASutB,aAAentB,KAAOkC,EAC5CmrB,EAAW5L,EAASzd,IAAKspB,EAAYzF,GAAiB,EAEjDwF,EAQL5L,EAASJ,IAAKiM,EAAYzF,EAAcwF,IAPnCztB,EAASutB,aACbntB,KAAK4f,oBAAqBiI,EAAcqF,GAExChrB,EAAI0d,oBAAqBje,EAAMurB,GAAoB,GAEpDzL,EAASlF,OAAQ+Q,EAAYzF,QAgBjC5kB,GAAOsB,KAAM,CACZgpB,WAAY,YACZC,WAAY,WACZC,aAAc,cACdC,aAAc,cACZ,SAAUC,EAAMlE,GAClBxmB,GAAOskB,MAAMxJ,QAAS4P,GAAS,CAC9B9F,aAAc4B,EACdT,SAAUS,EAEVZ,OAAQ,SAAUtB,GACjB,IAAInjB,EAEHwpB,EAAUrG,EAAMyD,cAChBzC,EAAYhB,EAAMgB,UASnB,OALMqF,IAAaA,IANT5tB,MAMgCiD,GAAOwF,SANvCzI,KAMyD4tB,MAClErG,EAAM5lB,KAAO4mB,EAAUG,SACvBtkB,EAAMmkB,EAAUzK,QAAQnd,MAAOX,KAAM0E,WACrC6iB,EAAM5lB,KAAO8nB,GAEPrlB,MAKVnB,GAAOG,GAAGmC,OAAQ,CAEjB4hB,GAAI,SAAUC,EAAOlkB,EAAUoe,EAAMle,GACpC,OAAO+jB,GAAInnB,KAAMonB,EAAOlkB,EAAUoe,EAAMle,IAEzCikB,IAAK,SAAUD,EAAOlkB,EAAUoe,EAAMle,GACrC,OAAO+jB,GAAInnB,KAAMonB,EAAOlkB,EAAUoe,EAAMle,EAAI,IAE7CokB,IAAK,SAAUJ,EAAOlkB,EAAUE,GAC/B,IAAImlB,EAAW5mB,EACf,GAAKylB,GAASA,EAAMY,gBAAkBZ,EAAMmB,UAW3C,OARAA,EAAYnB,EAAMmB,UAClBtlB,GAAQmkB,EAAMsC,gBAAiBlC,IAC9Be,EAAUlhB,UACTkhB,EAAUG,SAAW,IAAMH,EAAUlhB,UACrCkhB,EAAUG,SACXH,EAAUrlB,SACVqlB,EAAUzK,SAEJ9d,KAER,GAAsB,iBAAVonB,EAAqB,CAGhC,IAAMzlB,KAAQylB,EACbpnB,KAAKwnB,IAAK7lB,EAAMuB,EAAUkkB,EAAOzlB,IAElC,OAAO3B,KAWR,OATkB,IAAbkD,GAA0C,mBAAbA,IAGjCE,EAAKF,EACLA,OAAW+C,IAEA,IAAP7C,IACJA,EAAK8jB,IAEClnB,KAAKuE,KAAM,WACjBtB,GAAOskB,MAAMhL,OAAQvc,KAAMonB,EAAOhkB,EAAIF,QAMzC,IAKC2qB,GAAe,wBAGfC,GAAW,oCAEXC,GAAe,6BAGhB,SAASC,GAAoBvqB,EAAMiX,GAClC,OAAKlX,GAAUC,EAAM,UACpBD,GAA+B,KAArBkX,EAAQnZ,SAAkBmZ,EAAUA,EAAQhI,WAAY,OAE3DzP,GAAQQ,GAAO+V,SAAU,SAAW,IAGrC/V,EAIR,SAASwqB,GAAexqB,GAEvB,OADAA,EAAK9B,MAAyC,OAAhC8B,EAAKjB,aAAc,SAAsB,IAAMiB,EAAK9B,KAC3D8B,EAER,SAASyqB,GAAezqB,GAOvB,MAN2C,WAApCA,EAAK9B,MAAQ,IAAKrB,MAAO,EAAG,GAClCmD,EAAK9B,KAAO8B,EAAK9B,KAAKrB,MAAO,GAE7BmD,EAAKwK,gBAAiB,QAGhBxK,EAGR,SAAS0qB,GAAgBvsB,EAAKwsB,GAC7B,IAAIjsB,EAAG2X,EAAGnY,EAAgB0sB,EAAUC,EAAUjG,EAE9C,GAAuB,IAAlB+F,EAAK7sB,SAAV,CAKA,GAAKkgB,EAASD,QAAS5f,KAEtBymB,EADW5G,EAASzd,IAAKpC,GACPymB,QAKjB,IAAM1mB,KAFN8f,EAASlF,OAAQ6R,EAAM,iBAET/F,EACb,IAAMlmB,EAAI,EAAG2X,EAAIuO,EAAQ1mB,GAAO4B,OAAQpB,EAAI2X,EAAG3X,IAC9Cc,GAAOskB,MAAMrN,IAAKkU,EAAMzsB,EAAM0mB,EAAQ1mB,GAAQQ,IAO7Cuf,EAASF,QAAS5f,KACtBysB,EAAW3M,EAASzB,OAAQre,GAC5B0sB,EAAWrrB,GAAOsC,OAAQ,GAAI8oB,GAE9B3M,EAASL,IAAK+M,EAAME,KAkBtB,SAASC,GAAUC,EAAY3b,EAAMrO,EAAUiiB,GAG9C5T,EAAOtS,EAAMsS,GAEb,IAAI+T,EAAUjiB,EAAO4hB,EAASkI,EAAYxsB,EAAMC,EAC/CC,EAAI,EACJ2X,EAAI0U,EAAWjrB,OACfmrB,EAAW5U,EAAI,EACf/R,EAAQ8K,EAAM,GACd8b,EAAkBttB,EAAY0G,GAG/B,GAAK4mB,GACG,EAAJ7U,GAA0B,iBAAV/R,IAChB3G,GAAQ6jB,YAAc6I,GAASrmB,KAAMM,GACxC,OAAOymB,EAAWjqB,KAAM,SAAUyV,GACjC,IAAId,EAAOsV,EAAW5pB,GAAIoV,GACrB2U,IACJ9b,EAAM,GAAM9K,EAAMtH,KAAMT,KAAMga,EAAOd,EAAK0V,SAE3CL,GAAUrV,EAAMrG,EAAMrO,EAAUiiB,KAIlC,GAAK3M,IAEJnV,GADAiiB,EAAWN,GAAezT,EAAM2b,EAAY,GAAIhnB,eAAe,EAAOgnB,EAAY/H,IACjE/T,WAEmB,IAA/BkU,EAASla,WAAWnJ,SACxBqjB,EAAWjiB,GAIPA,GAAS8hB,GAAU,CAOvB,IALAgI,GADAlI,EAAUtjB,GAAOwB,IAAKmhB,GAAQgB,EAAU,UAAYqH,KAC/B1qB,OAKbpB,EAAI2X,EAAG3X,IACdF,EAAO2kB,EAEFzkB,IAAMusB,IACVzsB,EAAOgB,GAAO0C,MAAO1D,GAAM,GAAM,GAG5BwsB,GAIJxrB,GAAOoB,MAAOkiB,EAASX,GAAQ3jB,EAAM,YAIvCuC,EAAS/D,KAAM+tB,EAAYrsB,GAAKF,EAAME,GAGvC,GAAKssB,EAOJ,IANAvsB,EAAMqkB,EAASA,EAAQhjB,OAAS,GAAIiE,cAGpCvE,GAAOwB,IAAK8hB,EAAS2H,IAGf/rB,EAAI,EAAGA,EAAIssB,EAAYtsB,IAC5BF,EAAOskB,EAASpkB,GACX4iB,GAAYtd,KAAMxF,EAAKN,MAAQ,MAClC8f,EAASxB,OAAQhe,EAAM,eACxBgB,GAAOwF,SAAUvG,EAAKD,KAEjBA,EAAKL,KAA8C,YAArCK,EAAKN,MAAQ,IAAKgC,cAG/BV,GAAO4rB,WAAa5sB,EAAKH,UAC7BmB,GAAO4rB,SAAU5sB,EAAKL,IAAK,CAC1BC,MAAOI,EAAKJ,OAASI,EAAKO,aAAc,UACtCN,GASJH,EAASE,EAAK6E,YAAYT,QAAS0nB,GAAc,IAAM9rB,EAAMC,IAQnE,OAAOssB,EAGR,SAASjS,GAAQ9Y,EAAMP,EAAU4rB,GAKhC,IAJA,IAAI7sB,EACH4kB,EAAQ3jB,EAAWD,GAAO4M,OAAQ3M,EAAUO,GAASA,EACrDtB,EAAI,EAE4B,OAAvBF,EAAO4kB,EAAO1kB,IAAeA,IAChC2sB,GAA8B,IAAlB7sB,EAAKV,UACtB0B,GAAO8rB,UAAWnJ,GAAQ3jB,IAGtBA,EAAKW,aACJksB,GAAY3L,EAAYlhB,IAC5B4jB,GAAeD,GAAQ3jB,EAAM,WAE9BA,EAAKW,WAAWC,YAAaZ,IAI/B,OAAOwB,EAGRR,GAAOsC,OAAQ,CACduhB,cAAe,SAAU8H,GACxB,OAAOA,GAGRjpB,MAAO,SAAUlC,EAAMurB,EAAeC,GACrC,IAAI9sB,EAAG2X,EAAGoV,EAAaC,EA1INvtB,EAAKwsB,EACnB5qB,EA0IFmC,EAAQlC,EAAKyhB,WAAW,GACxBkK,EAASjM,EAAY1f,GAGtB,KAAMrC,GAAQ+jB,gBAAsC,IAAlB1hB,EAAKlC,UAAoC,KAAlBkC,EAAKlC,UAC3D0B,GAAOmE,SAAU3D,IAOnB,IAHA0rB,EAAevJ,GAAQjgB,GAGjBxD,EAAI,EAAG2X,GAFboV,EAActJ,GAAQniB,IAEOF,OAAQpB,EAAI2X,EAAG3X,IAvJ5BP,EAwJLstB,EAAa/sB,GAxJHisB,EAwJQe,EAAchtB,QAvJzCqB,EAGc,WAHdA,EAAW4qB,EAAK5qB,SAASG,gBAGAkhB,GAAepd,KAAM7F,EAAID,MACrDysB,EAAKja,QAAUvS,EAAIuS,QAGK,UAAb3Q,GAAqC,aAAbA,IACnC4qB,EAAKhJ,aAAexjB,EAAIwjB,cAoJxB,GAAK4J,EACJ,GAAKC,EAIJ,IAHAC,EAAcA,GAAetJ,GAAQniB,GACrC0rB,EAAeA,GAAgBvJ,GAAQjgB,GAEjCxD,EAAI,EAAG2X,EAAIoV,EAAY3rB,OAAQpB,EAAI2X,EAAG3X,IAC3CgsB,GAAgBe,EAAa/sB,GAAKgtB,EAAchtB,SAGjDgsB,GAAgB1qB,EAAMkC,GAWxB,OAL2B,GAD3BwpB,EAAevJ,GAAQjgB,EAAO,WACZpC,QACjBsiB,GAAesJ,GAAeC,GAAUxJ,GAAQniB,EAAM,WAIhDkC,GAGRopB,UAAW,SAAU5qB,GAKpB,IAJA,IAAImd,EAAM7d,EAAM9B,EACfoc,EAAU9a,GAAOskB,MAAMxJ,QACvB5b,EAAI,OAE6B8D,KAAxBxC,EAAOU,EAAOhC,IAAqBA,IAC5C,GAAK4e,EAAYtd,GAAS,CACzB,GAAO6d,EAAO7d,EAAMge,EAASvb,SAAc,CAC1C,GAAKob,EAAK+G,OACT,IAAM1mB,KAAQ2f,EAAK+G,OACbtK,EAASpc,GACbsB,GAAOskB,MAAMhL,OAAQ9Y,EAAM9B,GAI3BsB,GAAOqmB,YAAa7lB,EAAM9B,EAAM2f,EAAKuH,QAOxCplB,EAAMge,EAASvb,cAAYD,EAEvBxC,EAAMie,EAASxb,WAInBzC,EAAMie,EAASxb,cAAYD,OAOhChD,GAAOG,GAAGmC,OAAQ,CACjB8pB,OAAQ,SAAUnsB,GACjB,OAAOqZ,GAAQvc,KAAMkD,GAAU,IAGhCqZ,OAAQ,SAAUrZ,GACjB,OAAOqZ,GAAQvc,KAAMkD,IAGtBX,KAAM,SAAUwF,GACf,OAAOkY,EAAQjgB,KAAM,SAAU+H,GAC9B,YAAiB9B,IAAV8B,EACN9E,GAAOV,KAAMvC,MACbA,KAAKsU,QAAQ/P,KAAM,WACK,IAAlBvE,KAAKuB,UAAoC,KAAlBvB,KAAKuB,UAAqC,IAAlBvB,KAAKuB,WACxDvB,KAAK8G,YAAciB,MAGpB,KAAMA,EAAOrD,UAAUnB,SAG3B+rB,OAAQ,WACP,OAAOf,GAAUvuB,KAAM0E,UAAW,SAAUjB,GACpB,IAAlBzD,KAAKuB,UAAoC,KAAlBvB,KAAKuB,UAAqC,IAAlBvB,KAAKuB,UAC3CysB,GAAoBhuB,KAAMyD,GAChCd,YAAac,MAKvB8rB,QAAS,WACR,OAAOhB,GAAUvuB,KAAM0E,UAAW,SAAUjB,GAC3C,GAAuB,IAAlBzD,KAAKuB,UAAoC,KAAlBvB,KAAKuB,UAAqC,IAAlBvB,KAAKuB,SAAiB,CACzE,IAAIqE,EAASooB,GAAoBhuB,KAAMyD,GACvCmC,EAAO4pB,aAAc/rB,EAAMmC,EAAO8M,gBAKrC+c,OAAQ,WACP,OAAOlB,GAAUvuB,KAAM0E,UAAW,SAAUjB,GACtCzD,KAAK4C,YACT5C,KAAK4C,WAAW4sB,aAAc/rB,EAAMzD,SAKvC0vB,MAAO,WACN,OAAOnB,GAAUvuB,KAAM0E,UAAW,SAAUjB,GACtCzD,KAAK4C,YACT5C,KAAK4C,WAAW4sB,aAAc/rB,EAAMzD,KAAKuU,gBAK5CD,MAAO,WAIN,IAHA,IAAI7Q,EACHtB,EAAI,EAE2B,OAAtBsB,EAAOzD,KAAMmC,IAAeA,IACd,IAAlBsB,EAAKlC,WAGT0B,GAAO8rB,UAAWnJ,GAAQniB,GAAM,IAGhCA,EAAKqD,YAAc,IAIrB,OAAO9G,MAGR2F,MAAO,SAAUqpB,EAAeC,GAI/B,OAHAD,EAAiC,MAAjBA,GAAgCA,EAChDC,EAAyC,MAArBA,EAA4BD,EAAgBC,EAEzDjvB,KAAKyE,IAAK,WAChB,OAAOxB,GAAO0C,MAAO3F,KAAMgvB,EAAeC,MAI5CL,KAAM,SAAU7mB,GACf,OAAOkY,EAAQjgB,KAAM,SAAU+H,GAC9B,IAAItE,EAAOzD,KAAM,IAAO,GACvBmC,EAAI,EACJ2X,EAAI9Z,KAAKuD,OAEV,QAAe0C,IAAV8B,GAAyC,IAAlBtE,EAAKlC,SAChC,OAAOkC,EAAK0M,UAIb,GAAsB,iBAAVpI,IAAuB8lB,GAAapmB,KAAMM,KACpDud,IAAWR,GAASzX,KAAMtF,IAAW,CAAE,GAAI,KAAQ,GAAIpE,eAAkB,CAE1EoE,EAAQ9E,GAAO6jB,cAAe/e,GAE9B,IACC,KAAQ5F,EAAI2X,EAAG3X,IAIS,KAHvBsB,EAAOzD,KAAMmC,IAAO,IAGVZ,WACT0B,GAAO8rB,UAAWnJ,GAAQniB,GAAM,IAChCA,EAAK0M,UAAYpI,GAInBtE,EAAO,EAGN,MAAQkJ,KAGNlJ,GACJzD,KAAKsU,QAAQgb,OAAQvnB,IAEpB,KAAMA,EAAOrD,UAAUnB,SAG3BosB,YAAa,WACZ,IAAIlJ,EAAU,GAGd,OAAO8H,GAAUvuB,KAAM0E,UAAW,SAAUjB,GAC3C,IAAI8O,EAASvS,KAAK4C,WAEbK,GAAOkE,QAASnH,KAAMymB,GAAY,IACtCxjB,GAAO8rB,UAAWnJ,GAAQ5lB,OACrBuS,GACJA,EAAOqd,aAAcnsB,EAAMzD,QAK3BymB,MAILxjB,GAAOsB,KAAM,CACZsrB,SAAU,SACVC,UAAW,UACXN,aAAc,SACdO,YAAa,QACbC,WAAY,eACV,SAAUtsB,EAAMusB,GAClBhtB,GAAOG,GAAIM,GAAS,SAAUR,GAO7B,IANA,IAAIiB,EACHC,EAAM,GACN8rB,EAASjtB,GAAQC,GACjB2B,EAAOqrB,EAAO3sB,OAAS,EACvBpB,EAAI,EAEGA,GAAK0C,EAAM1C,IAClBgC,EAAQhC,IAAM0C,EAAO7E,KAAOA,KAAK2F,OAAO,GACxC1C,GAAQitB,EAAQ/tB,IAAO8tB,GAAY9rB,GAInCvD,EAAKD,MAAOyD,EAAKD,EAAMH,OAGxB,OAAOhE,KAAKkE,UAAWE,MAGzB,IAAI+rB,GAAY,IAAI3nB,OAAQ,KAAOua,EAAO,kBAAmB,KAEzDqN,GAAc,MAGdC,GAAY,SAAU5sB,GAKxB,IAAIuoB,EAAOvoB,EAAK+D,cAAc6H,YAM9B,OAJM2c,GAASA,EAAKsE,SACnBtE,EAAOjsB,IAGDisB,EAAKuE,iBAAkB9sB,IAG5B+sB,GAAO,SAAU/sB,EAAM+B,EAAShB,GACnC,IAAIJ,EAAKV,EACR+sB,EAAM,GAGP,IAAM/sB,KAAQ8B,EACbirB,EAAK/sB,GAASD,EAAK8f,MAAO7f,GAC1BD,EAAK8f,MAAO7f,GAAS8B,EAAS9B,GAM/B,IAAMA,KAHNU,EAAMI,EAAS/D,KAAMgD,GAGP+B,EACb/B,EAAK8f,MAAO7f,GAAS+sB,EAAK/sB,GAG3B,OAAOU,GAIJssB,GAAY,IAAIloB,OAAQ0a,EAAUpV,KAAM,KAAO,KAiJnD,SAAS6iB,GAAQltB,EAAMC,EAAMktB,GAC5B,IAAIC,EAAOC,EAAUC,EAAU3sB,EAC9B4sB,EAAeZ,GAAY3oB,KAAM/D,GAMjC6f,EAAQ9f,EAAK8f,MAoEd,OAlEAqN,EAAWA,GAAYP,GAAW5sB,MAgBjCW,EAAMwsB,EAASK,iBAAkBvtB,IAAUktB,EAAUltB,GAEhDstB,GAAgB5sB,IAkBpBA,EAAMA,EAAIiC,QAASkC,GAAU,YAAUtC,GAG3B,KAAR7B,GAAe+e,EAAY1f,KAC/BW,EAAMnB,GAAOsgB,MAAO9f,EAAMC,KAQrBtC,GAAQ8vB,kBAAoBf,GAAU1oB,KAAMrD,IAASssB,GAAUjpB,KAAM/D,KAG1EmtB,EAAQtN,EAAMsN,MACdC,EAAWvN,EAAMuN,SACjBC,EAAWxN,EAAMwN,SAGjBxN,EAAMuN,SAAWvN,EAAMwN,SAAWxN,EAAMsN,MAAQzsB,EAChDA,EAAMwsB,EAASC,MAGftN,EAAMsN,MAAQA,EACdtN,EAAMuN,SAAWA,EACjBvN,EAAMwN,SAAWA,SAIJ9qB,IAAR7B,EAINA,EAAM,GACNA,EAIF,SAAS+sB,GAAcC,EAAaC,GAGnC,MAAO,CACNrtB,IAAK,WACJ,IAAKotB,IASL,OAASpxB,KAAKgE,IAAMqtB,GAAS1wB,MAAOX,KAAM0E,kBALlC1E,KAAKgE,OA3OhB,WAIC,SAASstB,IAGR,GAAM1M,EAAN,CAIA2M,EAAUhO,MAAMiO,QAAU,+EAE1B5M,EAAIrB,MAAMiO,QACT,4HAGDzqB,EAAgBpE,YAAa4uB,GAAY5uB,YAAaiiB,GAEtD,IAAI6M,EAAW1xB,GAAOwwB,iBAAkB3L,GACxC8M,EAAoC,OAAjBD,EAASniB,IAG5BqiB,EAAsE,KAA9CC,EAAoBH,EAASI,YAIrDjN,EAAIrB,MAAMuO,MAAQ,MAClBC,EAA6D,KAAzCH,EAAoBH,EAASK,OAIjDE,EAAgE,KAAzCJ,EAAoBH,EAASZ,OAMpDjM,EAAIrB,MAAM0O,SAAW,WACrBC,EAAiE,KAA9CN,EAAoBhN,EAAIuN,YAAc,GAEzDprB,EAAgBlE,YAAa0uB,GAI7B3M,EAAM,MAGP,SAASgN,EAAoBQ,GAC5B,OAAOjsB,KAAKksB,MAAOC,WAAYF,IAGhC,IAAIV,EAAkBM,EAAsBE,EAAkBH,EAC7DQ,EAAyBZ,EACzBJ,EAAY3xB,EAAS0C,cAAe,OACpCsiB,EAAMhlB,EAAS0C,cAAe,OAGzBsiB,EAAIrB,QAMVqB,EAAIrB,MAAMiP,eAAiB,cAC3B5N,EAAIM,WAAW,GAAO3B,MAAMiP,eAAiB,GAC7CpxB,GAAQqxB,gBAA+C,gBAA7B7N,EAAIrB,MAAMiP,eAEpCvvB,GAAOsC,OAAQnE,GAAS,CACvBsxB,kBAAmB,WAElB,OADApB,IACOU,GAERd,eAAgB,WAEf,OADAI,IACOS,GAERY,cAAe,WAEd,OADArB,IACOI,GAERkB,mBAAoB,WAEnB,OADAtB,IACOK,GAERkB,cAAe,WAEd,OADAvB,IACOY,GAYRY,qBAAsB,WACrB,IAAIC,EAAOtN,EAAIuN,EAASC,EAmCxB,OAlCgC,MAA3BV,IACJQ,EAAQnzB,EAAS0C,cAAe,SAChCmjB,EAAK7lB,EAAS0C,cAAe,MAC7B0wB,EAAUpzB,EAAS0C,cAAe,OAElCywB,EAAMxP,MAAMiO,QAAU,2DACtB/L,EAAGlC,MAAMiO,QAAU,0CAKnB/L,EAAGlC,MAAM2P,OAAS,MAClBF,EAAQzP,MAAM2P,OAAS,MAQvBF,EAAQzP,MAAMC,QAAU,QAExBzc,EACEpE,YAAaowB,GACbpwB,YAAa8iB,GACb9iB,YAAaqwB,GAEfC,EAAUlzB,GAAOwwB,iBAAkB9K,GACnC8M,EAA4BY,SAAUF,EAAQC,OAAQ,IACrDC,SAAUF,EAAQG,eAAgB,IAClCD,SAAUF,EAAQI,kBAAmB,MAAW5N,EAAG6N,aAEpDvsB,EAAgBlE,YAAakwB,IAEvBR,MAvIV,GAsPA,IAAIgB,GAAc,CAAE,SAAU,MAAO,MACpCC,GAAa5zB,EAAS0C,cAAe,OAAQihB,MAC7CkQ,GAAc,GAkBf,SAASC,GAAehwB,GACvB,IAAIiwB,EAAQ1wB,GAAO2wB,SAAUlwB,IAAU+vB,GAAa/vB,GAEpD,OAAKiwB,IAGAjwB,KAAQ8vB,GACL9vB,EAED+vB,GAAa/vB,GAxBrB,SAAyBA,GAGxB,IAAImwB,EAAUnwB,EAAM,GAAIkd,cAAgBld,EAAKpD,MAAO,GACnD6B,EAAIoxB,GAAYhwB,OAEjB,MAAQpB,IAEP,IADAuB,EAAO6vB,GAAapxB,GAAM0xB,KACbL,GACZ,OAAO9vB,EAeoBowB,CAAgBpwB,IAAUA,GAIxD,IAKCqwB,GAAe,4BACfC,GAAU,CAAE/B,SAAU,WAAYgC,WAAY,SAAUzQ,QAAS,SACjE0Q,GAAqB,CACpBC,cAAe,IACfC,WAAY,OAGd,SAASC,GAAmBrvB,EAAO+C,EAAOusB,GAIzC,IAAI1sB,EAAUqb,EAAQ5V,KAAMtF,GAC5B,OAAOH,EAGNzB,KAAKouB,IAAK,EAAG3sB,EAAS,IAAQ0sB,GAAY,KAAU1sB,EAAS,IAAO,MACpEG,EAGF,SAASysB,GAAoB/wB,EAAMgxB,EAAWC,EAAKC,EAAaC,EAAQC,GACvE,IAAI1yB,EAAkB,UAAdsyB,EAAwB,EAAI,EACnCK,EAAQ,EACRC,EAAQ,EACRC,EAAc,EAGf,GAAKN,KAAUC,EAAc,SAAW,WACvC,OAAO,EAGR,KAAQxyB,EAAI,EAAGA,GAAK,EAKN,WAARuyB,IACJM,GAAe/xB,GAAOwgB,IAAKhgB,EAAMixB,EAAMxR,EAAW/gB,IAAK,EAAMyyB,IAIxDD,GAmBQ,YAARD,IACJK,GAAS9xB,GAAOwgB,IAAKhgB,EAAM,UAAYyf,EAAW/gB,IAAK,EAAMyyB,IAIjD,WAARF,IACJK,GAAS9xB,GAAOwgB,IAAKhgB,EAAM,SAAWyf,EAAW/gB,GAAM,SAAS,EAAMyyB,MAtBvEG,GAAS9xB,GAAOwgB,IAAKhgB,EAAM,UAAYyf,EAAW/gB,IAAK,EAAMyyB,GAGhD,YAARF,EACJK,GAAS9xB,GAAOwgB,IAAKhgB,EAAM,SAAWyf,EAAW/gB,GAAM,SAAS,EAAMyyB,GAItEE,GAAS7xB,GAAOwgB,IAAKhgB,EAAM,SAAWyf,EAAW/gB,GAAM,SAAS,EAAMyyB,IAoCzE,OAhBMD,GAA8B,GAAfE,IAIpBE,GAAS5uB,KAAKouB,IAAK,EAAGpuB,KAAK8uB,KAC1BxxB,EAAM,SAAWgxB,EAAW,GAAI7T,cAAgB6T,EAAUn0B,MAAO,IACjEu0B,EACAE,EACAD,EACA,MAIM,GAGDC,EAAQC,EAGhB,SAASE,GAAkBzxB,EAAMgxB,EAAWK,GAG3C,IAAIF,EAASvE,GAAW5sB,GAKvBkxB,IADmBvzB,GAAQsxB,qBAAuBoC,IAEE,eAAnD7xB,GAAOwgB,IAAKhgB,EAAM,aAAa,EAAOmxB,GACvCO,EAAmBR,EAEnBvyB,EAAMuuB,GAAQltB,EAAMgxB,EAAWG,GAC/BQ,EAAa,SAAWX,EAAW,GAAI7T,cAAgB6T,EAAUn0B,MAAO,GAIzE,GAAK6vB,GAAU1oB,KAAMrF,GAAQ,CAC5B,IAAM0yB,EACL,OAAO1yB,EAERA,EAAM,OAyCP,QAlCQhB,GAAQsxB,qBAAuBiC,IAMrCvzB,GAAQ0xB,wBAA0BtvB,GAAUC,EAAM,OAI3C,SAARrB,IAICkwB,WAAYlwB,IAA0D,WAAjDa,GAAOwgB,IAAKhgB,EAAM,WAAW,EAAOmxB,KAG1DnxB,EAAK4xB,iBAAiB9xB,SAEtBoxB,EAAiE,eAAnD1xB,GAAOwgB,IAAKhgB,EAAM,aAAa,EAAOmxB,IAKpDO,EAAmBC,KAAc3xB,KAEhCrB,EAAMqB,EAAM2xB,MAKdhzB,EAAMkwB,WAAYlwB,IAAS,GAI1BoyB,GACC/wB,EACAgxB,EACAK,IAAWH,EAAc,SAAW,WACpCQ,EACAP,EAGAxyB,GAEE,KAwTL,SAASkzB,GAAO7xB,EAAM+B,EAAS+b,EAAMnc,EAAKmwB,GACzC,OAAO,IAAID,GAAM1xB,UAAUP,KAAMI,EAAM+B,EAAS+b,EAAMnc,EAAKmwB,GAtT5DtyB,GAAOsC,OAAQ,CAIdiwB,SAAU,CACTC,QAAS,CACRzxB,IAAK,SAAUP,EAAMmtB,GACpB,GAAKA,EAAW,CAGf,IAAIxsB,EAAMusB,GAAQltB,EAAM,WACxB,MAAe,KAARW,EAAa,IAAMA,MAO9B+f,UAAW,CACVuR,yBAAyB,EACzBC,aAAa,EACbC,kBAAkB,EAClBC,aAAa,EACbC,UAAU,EACVC,YAAY,EACZ3B,YAAY,EACZ4B,UAAU,EACVC,YAAY,EACZC,eAAe,EACfC,iBAAiB,EACjBC,SAAS,EACTC,YAAY,EACZC,cAAc,EACdC,YAAY,EACZd,SAAS,EACTe,OAAO,EACPC,SAAS,EACT3S,OAAO,EACP4S,QAAQ,EACRC,QAAQ,EACRC,MAAM,EAGNC,aAAa,EACbC,cAAc,EACdC,aAAa,EACbC,kBAAkB,EAClBC,eAAe,GAKhBrD,SAAU,GAGVrQ,MAAO,SAAU9f,EAAMC,EAAMqE,EAAO+sB,GAGnC,GAAMrxB,GAA0B,IAAlBA,EAAKlC,UAAoC,IAAlBkC,EAAKlC,UAAmBkC,EAAK8f,MAAlE,CAKA,IAAInf,EAAKzC,EAAM2gB,EACd4U,EAAWrW,EAAWnd,GACtBstB,EAAeZ,GAAY3oB,KAAM/D,GACjC6f,EAAQ9f,EAAK8f,MAad,GARMyN,IACLttB,EAAOgwB,GAAewD,IAIvB5U,EAAQrf,GAAOuyB,SAAU9xB,IAAUT,GAAOuyB,SAAU0B,QAGrCjxB,IAAV8B,EA0CJ,OAAKua,GAAS,QAASA,QACwBrc,KAA5C7B,EAAMke,EAAMte,IAAKP,GAAM,EAAOqxB,IAEzB1wB,EAIDmf,EAAO7f,GA7CA,YAHd/B,SAAcoG,KAGc3D,EAAM6e,EAAQ5V,KAAMtF,KAAa3D,EAAK,KACjE2D,EAAQ2b,GAAWjgB,EAAMC,EAAMU,GAG/BzC,EAAO,UAIM,MAAToG,GAAiBA,GAAUA,IAOlB,WAATpG,GAAsBqvB,IAC1BjpB,GAAS3D,GAAOA,EAAK,KAASnB,GAAOkhB,UAAW+S,GAAa,GAAK,OAI7D91B,GAAQqxB,iBAA6B,KAAV1qB,GAAiD,IAAjCrE,EAAK7C,QAAS,gBAC9D0iB,EAAO7f,GAAS,WAIX4e,GAAY,QAASA,QACsBrc,KAA9C8B,EAAQua,EAAMjB,IAAK5d,EAAMsE,EAAO+sB,MAE7B9D,EACJzN,EAAM4T,YAAazzB,EAAMqE,GAEzBwb,EAAO7f,GAASqE,MAkBpB0b,IAAK,SAAUhgB,EAAMC,EAAMoxB,EAAOF,GACjC,IAAIxyB,EAAK6B,EAAKqe,EACb4U,EAAWrW,EAAWnd,GA6BvB,OA5BgB0sB,GAAY3oB,KAAM/D,KAMjCA,EAAOgwB,GAAewD,KAIvB5U,EAAQrf,GAAOuyB,SAAU9xB,IAAUT,GAAOuyB,SAAU0B,KAGtC,QAAS5U,IACtBlgB,EAAMkgB,EAAMte,IAAKP,GAAM,EAAMqxB,SAIjB7uB,IAAR7D,IACJA,EAAMuuB,GAAQltB,EAAMC,EAAMkxB,IAId,WAARxyB,GAAoBsB,KAAQwwB,KAChC9xB,EAAM8xB,GAAoBxwB,IAIZ,KAAVoxB,GAAgBA,GACpB7wB,EAAMquB,WAAYlwB,IACD,IAAV0yB,GAAkBsC,SAAUnzB,GAAQA,GAAO,EAAI7B,GAGhDA,KAITa,GAAOsB,KAAM,CAAE,SAAU,SAAW,SAAU6D,EAAIqsB,GACjDxxB,GAAOuyB,SAAUf,GAAc,CAC9BzwB,IAAK,SAAUP,EAAMmtB,EAAUkE,GAC9B,GAAKlE,EAIJ,OAAOmD,GAAatsB,KAAMxE,GAAOwgB,IAAKhgB,EAAM,aAQxCA,EAAK4xB,iBAAiB9xB,QAAWE,EAAK4zB,wBAAwBxG,MAIjEqE,GAAkBzxB,EAAMgxB,EAAWK,GAHnCtE,GAAM/sB,EAAMuwB,GAAS,WACpB,OAAOkB,GAAkBzxB,EAAMgxB,EAAWK,MAM9CzT,IAAK,SAAU5d,EAAMsE,EAAO+sB,GAC3B,IAAIltB,EACHgtB,EAASvE,GAAW5sB,GAIpB6zB,GAAsBl2B,GAAQyxB,iBACT,aAApB+B,EAAO3C,SAIR0C,GADkB2C,GAAsBxC,IAEY,eAAnD7xB,GAAOwgB,IAAKhgB,EAAM,aAAa,EAAOmxB,GACvCN,EAAWQ,EACVN,GACC/wB,EACAgxB,EACAK,EACAH,EACAC,GAED,EAqBF,OAjBKD,GAAe2C,IACnBhD,GAAYnuB,KAAK8uB,KAChBxxB,EAAM,SAAWgxB,EAAW,GAAI7T,cAAgB6T,EAAUn0B,MAAO,IACjEgyB,WAAYsC,EAAQH,IACpBD,GAAoB/wB,EAAMgxB,EAAW,UAAU,EAAOG,GACtD,KAKGN,IAAc1sB,EAAUqb,EAAQ5V,KAAMtF,KACb,QAA3BH,EAAS,IAAO,QAElBnE,EAAK8f,MAAOkR,GAAc1sB,EAC1BA,EAAQ9E,GAAOwgB,IAAKhgB,EAAMgxB,IAGpBJ,GAAmB5wB,EAAMsE,EAAOusB,OAK1CrxB,GAAOuyB,SAAS3D,WAAaV,GAAc/vB,GAAQwxB,mBAClD,SAAUnvB,EAAMmtB,GACf,GAAKA,EACJ,OAAS0B,WAAY3B,GAAQltB,EAAM,gBAClCA,EAAK4zB,wBAAwBE,KAC5B/G,GAAM/sB,EAAM,CAAEouB,WAAY,GAAK,WAC9B,OAAOpuB,EAAK4zB,wBAAwBE,QAEnC,OAMPt0B,GAAOsB,KAAM,CACZizB,OAAQ,GACRC,QAAS,GACTC,OAAQ,SACN,SAAUC,EAAQC,GACpB30B,GAAOuyB,SAAUmC,EAASC,GAAW,CACpCC,OAAQ,SAAU9vB,GAOjB,IANA,IAAI5F,EAAI,EACP21B,EAAW,GAGXC,EAAyB,iBAAVhwB,EAAqBA,EAAMI,MAAO,KAAQ,CAAEJ,GAEpD5F,EAAI,EAAGA,IACd21B,EAAUH,EAASzU,EAAW/gB,GAAMy1B,GACnCG,EAAO51B,IAAO41B,EAAO51B,EAAI,IAAO41B,EAAO,GAGzC,OAAOD,IAIO,WAAXH,IACJ10B,GAAOuyB,SAAUmC,EAASC,GAASvW,IAAMgT,MAI3CpxB,GAAOG,GAAGmC,OAAQ,CACjBke,IAAK,SAAU/f,EAAMqE,GACpB,OAAOkY,EAAQjgB,KAAM,SAAUyD,EAAMC,EAAMqE,GAC1C,IAAI6sB,EAAQ1vB,EACXT,EAAM,GACNtC,EAAI,EAEL,GAAK4D,MAAMC,QAAStC,GAAS,CAI5B,IAHAkxB,EAASvE,GAAW5sB,GACpByB,EAAMxB,EAAKH,OAEHpB,EAAI+C,EAAK/C,IAChBsC,EAAKf,EAAMvB,IAAQc,GAAOwgB,IAAKhgB,EAAMC,EAAMvB,IAAK,EAAOyyB,GAGxD,OAAOnwB,EAGR,YAAiBwB,IAAV8B,EACN9E,GAAOsgB,MAAO9f,EAAMC,EAAMqE,GAC1B9E,GAAOwgB,IAAKhgB,EAAMC,IACjBA,EAAMqE,EAA0B,EAAnBrD,UAAUnB,aAQ5BN,GAAOqyB,MAAQA,IAET1xB,UAAY,CACjBE,YAAawxB,GACbjyB,KAAM,SAAUI,EAAM+B,EAAS+b,EAAMnc,EAAKmwB,EAAQrR,GACjDlkB,KAAKyD,KAAOA,EACZzD,KAAKuhB,KAAOA,EACZvhB,KAAKu1B,OAASA,GAAUtyB,GAAOsyB,OAAO5P,SACtC3lB,KAAKwF,QAAUA,EACfxF,KAAKsS,MAAQtS,KAAKmrB,IAAMnrB,KAAK4Z,MAC7B5Z,KAAKoF,IAAMA,EACXpF,KAAKkkB,KAAOA,IAAUjhB,GAAOkhB,UAAW5C,GAAS,GAAK,OAEvD3H,IAAK,WACJ,IAAI0I,EAAQgT,GAAM0C,UAAWh4B,KAAKuhB,MAElC,OAAOe,GAASA,EAAMte,IACrBse,EAAMte,IAAKhE,MACXs1B,GAAM0C,UAAUrS,SAAS3hB,IAAKhE,OAEhCi4B,IAAK,SAAUC,GACd,IAAIC,EACH7V,EAAQgT,GAAM0C,UAAWh4B,KAAKuhB,MAoB/B,OAlBKvhB,KAAKwF,QAAQ4yB,SACjBp4B,KAAKq4B,IAAMF,EAAQl1B,GAAOsyB,OAAQv1B,KAAKu1B,QACtC2C,EAASl4B,KAAKwF,QAAQ4yB,SAAWF,EAAS,EAAG,EAAGl4B,KAAKwF,QAAQ4yB,UAG9Dp4B,KAAKq4B,IAAMF,EAAQD,EAEpBl4B,KAAKmrB,KAAQnrB,KAAKoF,IAAMpF,KAAKsS,OAAU6lB,EAAQn4B,KAAKsS,MAE/CtS,KAAKwF,QAAQ8yB,MACjBt4B,KAAKwF,QAAQ8yB,KAAK73B,KAAMT,KAAKyD,KAAMzD,KAAKmrB,IAAKnrB,MAGzCsiB,GAASA,EAAMjB,IACnBiB,EAAMjB,IAAKrhB,MAEXs1B,GAAM0C,UAAUrS,SAAStE,IAAKrhB,MAExBA,QAIOqD,KAAKO,UAAY0xB,GAAM1xB,WAEvC0xB,GAAM0C,UAAY,CACjBrS,SAAU,CACT3hB,IAAK,SAAU4f,GACd,IAAIhS,EAIJ,OAA6B,IAAxBgS,EAAMngB,KAAKlC,UACa,MAA5BqiB,EAAMngB,KAAMmgB,EAAMrC,OAAoD,MAAlCqC,EAAMngB,KAAK8f,MAAOK,EAAMrC,MACrDqC,EAAMngB,KAAMmgB,EAAMrC,OAO1B3P,EAAS3O,GAAOwgB,IAAKG,EAAMngB,KAAMmgB,EAAMrC,KAAM,MAGhB,SAAX3P,EAAwBA,EAAJ,GAEvCyP,IAAK,SAAUuC,GAKT3gB,GAAOs1B,GAAGD,KAAM1U,EAAMrC,MAC1Bte,GAAOs1B,GAAGD,KAAM1U,EAAMrC,MAAQqC,GACK,IAAxBA,EAAMngB,KAAKlC,WACtB0B,GAAOuyB,SAAU5R,EAAMrC,OAC6B,MAAnDqC,EAAMngB,KAAK8f,MAAOmQ,GAAe9P,EAAMrC,OAGxCqC,EAAMngB,KAAMmgB,EAAMrC,MAASqC,EAAMuH,IAFjCloB,GAAOsgB,MAAOK,EAAMngB,KAAMmgB,EAAMrC,KAAMqC,EAAMuH,IAAMvH,EAAMM,UAU5CsU,UAAYlD,GAAM0C,UAAUS,WAAa,CACxDpX,IAAK,SAAUuC,GACTA,EAAMngB,KAAKlC,UAAYqiB,EAAMngB,KAAKb,aACtCghB,EAAMngB,KAAMmgB,EAAMrC,MAASqC,EAAMuH,OAKpCloB,GAAOsyB,OAAS,CACfmD,OAAQ,SAAUC,GACjB,OAAOA,GAERC,MAAO,SAAUD,GAChB,MAAO,GAAMxyB,KAAK0yB,IAAKF,EAAIxyB,KAAK2yB,IAAO,GAExCnT,SAAU,SAGX1iB,GAAOs1B,GAAKjD,GAAM1xB,UAAUP,KAG5BJ,GAAOs1B,GAAGD,KAAO,GAKjB,IACCS,GAAOC,GAkrBH9oB,GAEH+oB,GAnrBDC,GAAW,yBACXC,GAAO,cAER,SAASC,KACHJ,MACqB,IAApBp5B,EAASy5B,QAAoBt5B,GAAOu5B,sBACxCv5B,GAAOu5B,sBAAuBF,IAE9Br5B,GAAO2e,WAAY0a,GAAUn2B,GAAOs1B,GAAGgB,UAGxCt2B,GAAOs1B,GAAGiB,QAKZ,SAASC,KAIR,OAHA15B,GAAO2e,WAAY,WAClBqa,QAAQ9yB,IAEA8yB,GAAQ7N,KAAKC,MAIvB,SAASuO,GAAO/3B,EAAMg4B,GACrB,IAAI3M,EACH7qB,EAAI,EACJggB,EAAQ,CAAE+Q,OAAQvxB,GAKnB,IADAg4B,EAAeA,EAAe,EAAI,EAC1Bx3B,EAAI,EAAGA,GAAK,EAAIw3B,EAEvBxX,EAAO,UADP6K,EAAQ9J,EAAW/gB,KACSggB,EAAO,UAAY6K,GAAUrrB,EAO1D,OAJKg4B,IACJxX,EAAMsT,QAAUtT,EAAM0O,MAAQlvB,GAGxBwgB,EAGR,SAASyX,GAAa7xB,EAAOwZ,EAAMsY,GAKlC,IAJA,IAAIjW,EACH4K,GAAesL,GAAUC,SAAUxY,IAAU,IAAK7gB,OAAQo5B,GAAUC,SAAU,MAC9E/f,EAAQ,EACRzW,EAASirB,EAAWjrB,OACbyW,EAAQzW,EAAQyW,IACvB,GAAO4J,EAAQ4K,EAAYxU,GAAQvZ,KAAMo5B,EAAWtY,EAAMxZ,GAGzD,OAAO6b,EAsNV,SAASkW,GAAWr2B,EAAMu2B,EAAYx0B,GACrC,IAAIoM,EACHqoB,EACAjgB,EAAQ,EACRzW,EAASu2B,GAAUI,WAAW32B,OAC9ByZ,EAAW/Z,GAAO0Z,WAAWI,OAAQ,kBAG7Byc,EAAK/1B,OAEb+1B,EAAO,WACN,GAAKS,EACJ,OAAO,EAYR,IAVA,IAAIE,EAAcpB,IAASU,KAC1B3a,EAAY3Y,KAAKouB,IAAK,EAAGsF,EAAUO,UAAYP,EAAUzB,SAAW+B,GAKpEjC,EAAU,GADHpZ,EAAY+a,EAAUzB,UAAY,GAEzCpe,EAAQ,EACRzW,EAASs2B,EAAUQ,OAAO92B,OAEnByW,EAAQzW,EAAQyW,IACvB6f,EAAUQ,OAAQrgB,GAAQie,IAAKC,GAMhC,OAHAlb,EAASmB,WAAY1a,EAAM,CAAEo2B,EAAW3B,EAASpZ,IAG5CoZ,EAAU,GAAK30B,EACZub,GAIFvb,GACLyZ,EAASmB,WAAY1a,EAAM,CAAEo2B,EAAW,EAAG,IAI5C7c,EAASoB,YAAa3a,EAAM,CAAEo2B,KACvB,IAERA,EAAY7c,EAAS1B,QAAS,CAC7B7X,KAAMA,EACNonB,MAAO5nB,GAAOsC,OAAQ,GAAIy0B,GAC1BM,KAAMr3B,GAAOsC,QAAQ,EAAM,CAC1Bg1B,cAAe,GACfhF,OAAQtyB,GAAOsyB,OAAO5P,UACpBngB,GACHg1B,mBAAoBR,EACpBS,gBAAiBj1B,EACjB40B,UAAWrB,IAASU,KACpBrB,SAAU5yB,EAAQ4yB,SAClBiC,OAAQ,GACRT,YAAa,SAAUrY,EAAMnc,GAC5B,IAAIwe,EAAQ3gB,GAAOqyB,MAAO7xB,EAAMo2B,EAAUS,KAAM/Y,EAAMnc,EACrDy0B,EAAUS,KAAKC,cAAehZ,IAAUsY,EAAUS,KAAK/E,QAExD,OADAsE,EAAUQ,OAAOz5B,KAAMgjB,GAChBA,GAERnB,KAAM,SAAUiY,GACf,IAAI1gB,EAAQ,EAIXzW,EAASm3B,EAAUb,EAAUQ,OAAO92B,OAAS,EAC9C,GAAK02B,EACJ,OAAOj6B,KAGR,IADAi6B,GAAU,EACFjgB,EAAQzW,EAAQyW,IACvB6f,EAAUQ,OAAQrgB,GAAQie,IAAK,GAUhC,OANKyC,GACJ1d,EAASmB,WAAY1a,EAAM,CAAEo2B,EAAW,EAAG,IAC3C7c,EAASoB,YAAa3a,EAAM,CAAEo2B,EAAWa,KAEzC1d,EAASuB,WAAY9a,EAAM,CAAEo2B,EAAWa,IAElC16B,QAGT6qB,EAAQgP,EAAUhP,MAInB,KA/HD,SAAqBA,EAAO0P,GAC3B,IAAIvgB,EAAOtW,EAAM6xB,EAAQxtB,EAAOua,EAGhC,IAAMtI,KAAS6Q,EAed,GAbA0K,EAASgF,EADT72B,EAAOmd,EAAW7G,IAElBjS,EAAQ8iB,EAAO7Q,GACVjU,MAAMC,QAAS+B,KACnBwtB,EAASxtB,EAAO,GAChBA,EAAQ8iB,EAAO7Q,GAAUjS,EAAO,IAG5BiS,IAAUtW,IACdmnB,EAAOnnB,GAASqE,SACT8iB,EAAO7Q,KAGfsI,EAAQrf,GAAOuyB,SAAU9xB,KACX,WAAY4e,EAMzB,IAAMtI,KALNjS,EAAQua,EAAMuV,OAAQ9vB,UACf8iB,EAAOnnB,GAICqE,EACNiS,KAAS6Q,IAChBA,EAAO7Q,GAAUjS,EAAOiS,GACxBugB,EAAevgB,GAAUub,QAI3BgF,EAAe72B,GAAS6xB,EA6F1BoF,CAAY9P,EAAOgP,EAAUS,KAAKC,eAE1BvgB,EAAQzW,EAAQyW,IAEvB,GADApI,EAASkoB,GAAUI,WAAYlgB,GAAQvZ,KAAMo5B,EAAWp2B,EAAMonB,EAAOgP,EAAUS,MAM9E,OAJKj5B,EAAYuQ,EAAO6Q,QACvBxf,GAAOsf,YAAasX,EAAUp2B,KAAMo2B,EAAUS,KAAKpe,OAAQuG,KAC1D7Q,EAAO6Q,KAAKmY,KAAMhpB,IAEbA,EAyBT,OArBA3O,GAAOwB,IAAKomB,EAAO+O,GAAaC,GAE3Bx4B,EAAYw4B,EAAUS,KAAKhoB,QAC/BunB,EAAUS,KAAKhoB,MAAM7R,KAAMgD,EAAMo2B,GAIlCA,EACEtc,SAAUsc,EAAUS,KAAK/c,UACzBzT,KAAM+vB,EAAUS,KAAKxwB,KAAM+vB,EAAUS,KAAKO,UAC1Ctf,KAAMse,EAAUS,KAAK/e,MACrBwB,OAAQ8c,EAAUS,KAAKvd,QAEzB9Z,GAAOs1B,GAAGuC,MACT73B,GAAOsC,OAAQi0B,EAAM,CACpB/1B,KAAMA,EACNs3B,KAAMlB,EACN3d,MAAO2d,EAAUS,KAAKpe,SAIjB2d,EAGR52B,GAAO62B,UAAY72B,GAAOsC,OAAQu0B,GAAW,CAE5CC,SAAU,CACTiB,IAAK,CAAE,SAAUzZ,EAAMxZ,GACtB,IAAI6b,EAAQ5jB,KAAK45B,YAAarY,EAAMxZ,GAEpC,OADA2b,GAAWE,EAAMngB,KAAM8d,EAAM0B,EAAQ5V,KAAMtF,GAAS6b,GAC7CA,KAITqX,QAAS,SAAUpQ,EAAOrmB,GACpBnD,EAAYwpB,IAChBrmB,EAAWqmB,EACXA,EAAQ,CAAE,MAEVA,EAAQA,EAAM5d,MAAO2N,GAOtB,IAJA,IAAI2G,EACHvH,EAAQ,EACRzW,EAASsnB,EAAMtnB,OAERyW,EAAQzW,EAAQyW,IACvBuH,EAAOsJ,EAAO7Q,GACd8f,GAAUC,SAAUxY,GAASuY,GAAUC,SAAUxY,IAAU,GAC3DuY,GAAUC,SAAUxY,GAAOiB,QAAShe,IAItC01B,WAAY,CA3Wb,SAA2Bz2B,EAAMonB,EAAOyP,GACvC,IAAI/Y,EAAMxZ,EAAO4c,EAAQrC,EAAO4Y,EAASC,EAAWC,EAAgB5X,EACnE6X,EAAQ,UAAWxQ,GAAS,WAAYA,EACxCkQ,EAAO/6B,KACP2tB,EAAO,GACPpK,EAAQ9f,EAAK8f,MACb8V,EAAS51B,EAAKlC,UAAY+hB,GAAoB7f,GAC9C63B,EAAW7Z,EAASzd,IAAKP,EAAM,UA6BhC,IAAM8d,KA1BA+Y,EAAKpe,QAEa,OADvBoG,EAAQrf,GAAOsf,YAAa9e,EAAM,OACvB83B,WACVjZ,EAAMiZ,SAAW,EACjBL,EAAU5Y,EAAMhO,MAAM8H,KACtBkG,EAAMhO,MAAM8H,KAAO,WACZkG,EAAMiZ,UACXL,MAIH5Y,EAAMiZ,WAENR,EAAKhe,OAAQ,WAGZge,EAAKhe,OAAQ,WACZuF,EAAMiZ,WACAt4B,GAAOiZ,MAAOzY,EAAM,MAAOF,QAChC+e,EAAMhO,MAAM8H,YAOFyO,EAEb,GADA9iB,EAAQ8iB,EAAOtJ,GACV2X,GAASzxB,KAAMM,GAAU,CAG7B,UAFO8iB,EAAOtJ,GACdoD,EAASA,GAAoB,WAAV5c,EACdA,KAAYsxB,EAAS,OAAS,QAAW,CAI7C,GAAe,SAAVtxB,IAAoBuzB,QAAiCr1B,IAArBq1B,EAAU/Z,GAK9C,SAJA8X,GAAS,EAOX1L,EAAMpM,GAAS+Z,GAAYA,EAAU/Z,IAAUte,GAAOsgB,MAAO9f,EAAM8d,GAMrE,IADA4Z,GAAal4B,GAAO2D,cAAeikB,MAChB5nB,GAAO2D,cAAe+mB,GA8DzC,IAAMpM,KAzDD8Z,GAA2B,IAAlB53B,EAAKlC,WAMlB+4B,EAAKkB,SAAW,CAAEjY,EAAMiY,SAAUjY,EAAMkY,UAAWlY,EAAMmY,WAIlC,OADvBN,EAAiBE,GAAYA,EAAS9X,WAErC4X,EAAiB3Z,EAASzd,IAAKP,EAAM,YAGrB,UADjB+f,EAAUvgB,GAAOwgB,IAAKhgB,EAAM,cAEtB23B,EACJ5X,EAAU4X,GAIV9W,GAAU,CAAE7gB,IAAQ,GACpB23B,EAAiB33B,EAAK8f,MAAMC,SAAW4X,EACvC5X,EAAUvgB,GAAOwgB,IAAKhgB,EAAM,WAC5B6gB,GAAU,CAAE7gB,OAKG,WAAZ+f,GAAoC,iBAAZA,GAAgD,MAAlB4X,IACrB,SAAhCn4B,GAAOwgB,IAAKhgB,EAAM,WAGhB03B,IACLJ,EAAKjxB,KAAM,WACVyZ,EAAMC,QAAU4X,IAEM,MAAlBA,IACJ5X,EAAUD,EAAMC,QAChB4X,EAA6B,SAAZ5X,EAAqB,GAAKA,IAG7CD,EAAMC,QAAU,iBAKd8W,EAAKkB,WACTjY,EAAMiY,SAAW,SACjBT,EAAKhe,OAAQ,WACZwG,EAAMiY,SAAWlB,EAAKkB,SAAU,GAChCjY,EAAMkY,UAAYnB,EAAKkB,SAAU,GACjCjY,EAAMmY,UAAYpB,EAAKkB,SAAU,MAKnCL,GAAY,EACExN,EAGPwN,IACAG,EACC,WAAYA,IAChBjC,EAASiC,EAASjC,QAGnBiC,EAAW7Z,EAASxB,OAAQxc,EAAM,SAAU,CAAE+f,QAAS4X,IAInDzW,IACJ2W,EAASjC,QAAUA,GAIfA,GACJ/U,GAAU,CAAE7gB,IAAQ,GAKrBs3B,EAAKjxB,KAAM,WASV,IAAMyX,KAJA8X,GACL/U,GAAU,CAAE7gB,IAEbge,EAASlF,OAAQ9Y,EAAM,UACTkqB,EACb1qB,GAAOsgB,MAAO9f,EAAM8d,EAAMoM,EAAMpM,OAMnC4Z,EAAYvB,GAAaP,EAASiC,EAAU/Z,GAAS,EAAGA,EAAMwZ,GACtDxZ,KAAQ+Z,IACfA,EAAU/Z,GAAS4Z,EAAU7oB,MACxB+mB,IACJ8B,EAAU/1B,IAAM+1B,EAAU7oB,MAC1B6oB,EAAU7oB,MAAQ,MAuMrBqpB,UAAW,SAAUn3B,EAAU+qB,GACzBA,EACJuK,GAAUI,WAAW1X,QAAShe,GAE9Bs1B,GAAUI,WAAWt5B,KAAM4D,MAK9BvB,GAAO24B,MAAQ,SAAUA,EAAOrG,EAAQnyB,GACvC,IAAI61B,EAAM2C,GAA0B,iBAAVA,EAAqB34B,GAAOsC,OAAQ,GAAIq2B,GAAU,CAC3Ef,SAAUz3B,IAAOA,GAAMmyB,GACtBl0B,EAAYu6B,IAAWA,EACxBxD,SAAUwD,EACVrG,OAAQnyB,GAAMmyB,GAAUA,IAAWl0B,EAAYk0B,IAAYA,GAoC5D,OAhCKtyB,GAAOs1B,GAAG/Q,IACdyR,EAAIb,SAAW,EAGc,iBAAjBa,EAAIb,WACVa,EAAIb,YAAYn1B,GAAOs1B,GAAGsD,OAC9B5C,EAAIb,SAAWn1B,GAAOs1B,GAAGsD,OAAQ5C,EAAIb,UAGrCa,EAAIb,SAAWn1B,GAAOs1B,GAAGsD,OAAOlW,UAMjB,MAAbsT,EAAI/c,QAA+B,IAAd+c,EAAI/c,QAC7B+c,EAAI/c,MAAQ,MAIb+c,EAAIxI,IAAMwI,EAAI4B,SAEd5B,EAAI4B,SAAW,WACTx5B,EAAY43B,EAAIxI,MACpBwI,EAAIxI,IAAIhwB,KAAMT,MAGVi5B,EAAI/c,OACRjZ,GAAOmf,QAASpiB,KAAMi5B,EAAI/c,QAIrB+c,GAGRh2B,GAAOG,GAAGmC,OAAQ,CACjBu2B,OAAQ,SAAUF,EAAOG,EAAIxG,EAAQ/wB,GAGpC,OAAOxE,KAAK6P,OAAQyT,IAAqBG,IAAK,UAAW,GAAIc,OAG3Dnf,MAAM42B,QAAS,CAAEvG,QAASsG,GAAMH,EAAOrG,EAAQ/wB,IAElDw3B,QAAS,SAAUza,EAAMqa,EAAOrG,EAAQ/wB,GACvC,IAAI8P,EAAQrR,GAAO2D,cAAe2a,GACjC0a,EAASh5B,GAAO24B,MAAOA,EAAOrG,EAAQ/wB,GACtC03B,EAAc,WAGb,IAAInB,EAAOjB,GAAW95B,KAAMiD,GAAOsC,OAAQ,GAAIgc,GAAQ0a,IAGlD3nB,GAASmN,EAASzd,IAAKhE,KAAM,YACjC+6B,EAAKtY,MAAM,IAMd,OAFAyZ,EAAYC,OAASD,EAEd5nB,IAA0B,IAAjB2nB,EAAO/f,MACtBlc,KAAKuE,KAAM23B,GACXl8B,KAAKkc,MAAO+f,EAAO/f,MAAOggB,IAE5BzZ,KAAM,SAAU9gB,EAAMghB,EAAY+X,GACjC,IAAI0B,EAAY,SAAU9Z,GACzB,IAAIG,EAAOH,EAAMG,YACVH,EAAMG,KACbA,EAAMiY,IAYP,MATqB,iBAAT/4B,IACX+4B,EAAU/X,EACVA,EAAahhB,EACbA,OAAOsE,GAEH0c,GACJ3iB,KAAKkc,MAAOva,GAAQ,KAAM,IAGpB3B,KAAKuE,KAAM,WACjB,IAAI6d,GAAU,EACbpI,EAAgB,MAARrY,GAAgBA,EAAO,aAC/B06B,EAASp5B,GAAOo5B,OAChB/a,EAAOG,EAASzd,IAAKhE,MAEtB,GAAKga,EACCsH,EAAMtH,IAAWsH,EAAMtH,GAAQyI,MACnC2Z,EAAW9a,EAAMtH,SAGlB,IAAMA,KAASsH,EACTA,EAAMtH,IAAWsH,EAAMtH,GAAQyI,MAAQ0W,GAAK1xB,KAAMuS,IACtDoiB,EAAW9a,EAAMtH,IAKpB,IAAMA,EAAQqiB,EAAO94B,OAAQyW,KACvBqiB,EAAQriB,GAAQvW,OAASzD,MACnB,MAAR2B,GAAgB06B,EAAQriB,GAAQkC,QAAUva,IAE5C06B,EAAQriB,GAAQ+gB,KAAKtY,KAAMiY,GAC3BtY,GAAU,EACVia,EAAO/2B,OAAQ0U,EAAO,KAOnBoI,GAAYsY,GAChBz3B,GAAOmf,QAASpiB,KAAM2B,MAIzBw6B,OAAQ,SAAUx6B,GAIjB,OAHc,IAATA,IACJA,EAAOA,GAAQ,MAET3B,KAAKuE,KAAM,WACjB,IAAIyV,EACHsH,EAAOG,EAASzd,IAAKhE,MACrBkc,EAAQoF,EAAM3f,EAAO,SACrB2gB,EAAQhB,EAAM3f,EAAO,cACrB06B,EAASp5B,GAAOo5B,OAChB94B,EAAS2Y,EAAQA,EAAM3Y,OAAS,EAajC,IAVA+d,EAAK6a,QAAS,EAGdl5B,GAAOiZ,MAAOlc,KAAM2B,EAAM,IAErB2gB,GAASA,EAAMG,MACnBH,EAAMG,KAAKhiB,KAAMT,MAAM,GAIlBga,EAAQqiB,EAAO94B,OAAQyW,KACvBqiB,EAAQriB,GAAQvW,OAASzD,MAAQq8B,EAAQriB,GAAQkC,QAAUva,IAC/D06B,EAAQriB,GAAQ+gB,KAAKtY,MAAM,GAC3B4Z,EAAO/2B,OAAQ0U,EAAO,IAKxB,IAAMA,EAAQ,EAAGA,EAAQzW,EAAQyW,IAC3BkC,EAAOlC,IAAWkC,EAAOlC,GAAQmiB,QACrCjgB,EAAOlC,GAAQmiB,OAAO17B,KAAMT,aAKvBshB,EAAK6a,YAKfl5B,GAAOsB,KAAM,CAAE,SAAU,OAAQ,QAAU,SAAU6D,EAAI1E,GACxD,IAAI44B,EAAQr5B,GAAOG,GAAIM,GACvBT,GAAOG,GAAIM,GAAS,SAAUk4B,EAAOrG,EAAQ/wB,GAC5C,OAAgB,MAATo3B,GAAkC,kBAAVA,EAC9BU,EAAM37B,MAAOX,KAAM0E,WACnB1E,KAAKg8B,QAAStC,GAAOh2B,GAAM,GAAQk4B,EAAOrG,EAAQ/wB,MAKrDvB,GAAOsB,KAAM,CACZg4B,UAAW7C,GAAO,QAClB8C,QAAS9C,GAAO,QAChB+C,YAAa/C,GAAO,UACpBgD,OAAQ,CAAEjH,QAAS,QACnBkH,QAAS,CAAElH,QAAS,QACpBmH,WAAY,CAAEnH,QAAS,WACrB,SAAU/xB,EAAMmnB,GAClB5nB,GAAOG,GAAIM,GAAS,SAAUk4B,EAAOrG,EAAQ/wB,GAC5C,OAAOxE,KAAKg8B,QAASnR,EAAO+Q,EAAOrG,EAAQ/wB,MAI7CvB,GAAOo5B,OAAS,GAChBp5B,GAAOs1B,GAAGiB,KAAO,WAChB,IAAIsB,EACH34B,EAAI,EACJk6B,EAASp5B,GAAOo5B,OAIjB,IAFAtD,GAAQ7N,KAAKC,MAELhpB,EAAIk6B,EAAO94B,OAAQpB,KAC1B24B,EAAQuB,EAAQl6B,OAGCk6B,EAAQl6B,KAAQ24B,GAChCuB,EAAO/2B,OAAQnD,IAAK,GAIhBk6B,EAAO94B,QACZN,GAAOs1B,GAAG9V,OAEXsW,QAAQ9yB,GAGThD,GAAOs1B,GAAGuC,MAAQ,SAAUA,GAC3B73B,GAAOo5B,OAAOz7B,KAAMk6B,GACpB73B,GAAOs1B,GAAGjmB,SAGXrP,GAAOs1B,GAAGgB,SAAW,GACrBt2B,GAAOs1B,GAAGjmB,MAAQ,WACZ0mB,KAILA,IAAa,EACbI,OAGDn2B,GAAOs1B,GAAG9V,KAAO,WAChBuW,GAAa,MAGd/1B,GAAOs1B,GAAGsD,OAAS,CAClBgB,KAAM,IACNC,KAAM,IAGNnX,SAAU,KAKX1iB,GAAOG,GAAG25B,MAAQ,SAAUC,EAAMr7B,GAIjC,OAHAq7B,EAAO/5B,GAAOs1B,IAAKt1B,GAAOs1B,GAAGsD,OAAQmB,IAAiBA,EACtDr7B,EAAOA,GAAQ,KAER3B,KAAKkc,MAAOva,EAAM,SAAU8K,EAAM6V,GACxC,IAAI2a,EAAUl9B,GAAO2e,WAAYjS,EAAMuwB,GACvC1a,EAAMG,KAAO,WACZ1iB,GAAOm9B,aAAcD,OAOnB/sB,GAAQtQ,EAAS0C,cAAe,SAEnC22B,GADSr5B,EAAS0C,cAAe,UACpBK,YAAa/C,EAAS0C,cAAe,WAEnD4N,GAAMvO,KAAO,WAIbP,GAAQ+7B,QAA0B,KAAhBjtB,GAAMnI,MAIxB3G,GAAQg8B,YAAcnE,GAAI7kB,UAI1BlE,GAAQtQ,EAAS0C,cAAe,UAC1ByF,MAAQ,IACdmI,GAAMvO,KAAO,QACbP,GAAQi8B,WAA6B,MAAhBntB,GAAMnI,MAI5B,IAAIu1B,GACH5sB,GAAazN,GAAOqN,KAAKI,WAE1BzN,GAAOG,GAAGmC,OAAQ,CACjBkL,KAAM,SAAU/M,EAAMqE,GACrB,OAAOkY,EAAQjgB,KAAMiD,GAAOwN,KAAM/M,EAAMqE,EAA0B,EAAnBrD,UAAUnB,SAG1Dg6B,WAAY,SAAU75B,GACrB,OAAO1D,KAAKuE,KAAM,WACjBtB,GAAOs6B,WAAYv9B,KAAM0D,QAK5BT,GAAOsC,OAAQ,CACdkL,KAAM,SAAUhN,EAAMC,EAAMqE,GAC3B,IAAI3D,EAAKke,EACRkb,EAAQ/5B,EAAKlC,SAGd,GAAe,IAAVi8B,GAAyB,IAAVA,GAAyB,IAAVA,EAKnC,MAAkC,oBAAtB/5B,EAAKjB,aACTS,GAAOse,KAAM9d,EAAMC,EAAMqE,IAKlB,IAAVy1B,GAAgBv6B,GAAOmE,SAAU3D,KACrC6e,EAAQrf,GAAOw6B,UAAW/5B,EAAKC,iBAC5BV,GAAOqN,KAAKrD,MAAM3B,KAAK7D,KAAM/D,GAAS45B,QAAWr3B,SAGtCA,IAAV8B,EACW,OAAVA,OACJ9E,GAAOs6B,WAAY95B,EAAMC,GAIrB4e,GAAS,QAASA,QACuBrc,KAA3C7B,EAAMke,EAAMjB,IAAK5d,EAAMsE,EAAOrE,IACzBU,GAGRX,EAAKhB,aAAciB,EAAMqE,EAAQ,IAC1BA,GAGHua,GAAS,QAASA,GAA+C,QAApCle,EAAMke,EAAMte,IAAKP,EAAMC,IACjDU,EAMM,OAHdA,EAAMnB,GAAO4J,KAAK4D,KAAMhN,EAAMC,SAGTuC,EAAY7B,IAGlCq5B,UAAW,CACV97B,KAAM,CACL0f,IAAK,SAAU5d,EAAMsE,GACpB,IAAM3G,GAAQi8B,YAAwB,UAAVt1B,GAC3BvE,GAAUC,EAAM,SAAY,CAC5B,IAAIrB,EAAMqB,EAAKsE,MAKf,OAJAtE,EAAKhB,aAAc,OAAQsF,GACtB3F,IACJqB,EAAKsE,MAAQ3F,GAEP2F,MAMXw1B,WAAY,SAAU95B,EAAMsE,GAC3B,IAAIrE,EACHvB,EAAI,EAIJu7B,EAAY31B,GAASA,EAAMkF,MAAO2N,GAEnC,GAAK8iB,GAA+B,IAAlBj6B,EAAKlC,SACtB,MAAUmC,EAAOg6B,EAAWv7B,KAC3BsB,EAAKwK,gBAAiBvK,MAO1B45B,GAAW,CACVjc,IAAK,SAAU5d,EAAMsE,EAAOrE,GAQ3B,OAPe,IAAVqE,EAGJ9E,GAAOs6B,WAAY95B,EAAMC,GAEzBD,EAAKhB,aAAciB,EAAMA,GAEnBA,IAITT,GAAOsB,KAAMtB,GAAOqN,KAAKrD,MAAM3B,KAAK0X,OAAO/V,MAAO,QAAU,SAAU7E,EAAI1E,GACzE,IAAIi6B,EAASjtB,GAAYhN,IAAUT,GAAO4J,KAAK4D,KAE/CC,GAAYhN,GAAS,SAAUD,EAAMC,EAAM6U,GAC1C,IAAInU,EAAKykB,EACR+U,EAAgBl6B,EAAKC,cAYtB,OAVM4U,IAGLsQ,EAASnY,GAAYktB,GACrBltB,GAAYktB,GAAkBx5B,EAC9BA,EAAqC,MAA/Bu5B,EAAQl6B,EAAMC,EAAM6U,GACzBqlB,EACA,KACDltB,GAAYktB,GAAkB/U,GAExBzkB,KAOT,IAAIy5B,GAAa,sCAChBC,GAAa,gBAwIb,SAASC,GAAkBh2B,GAE1B,OADaA,EAAMkF,MAAO2N,IAAmB,IAC/B9M,KAAM,KAItB,SAASkwB,GAAUv6B,GAClB,OAAOA,EAAKjB,cAAgBiB,EAAKjB,aAAc,UAAa,GAG7D,SAASy7B,GAAgBl2B,GACxB,OAAKhC,MAAMC,QAAS+B,GACZA,EAEc,iBAAVA,GACJA,EAAMkF,MAAO2N,IAEd,GAvJR3X,GAAOG,GAAGmC,OAAQ,CACjBgc,KAAM,SAAU7d,EAAMqE,GACrB,OAAOkY,EAAQjgB,KAAMiD,GAAOse,KAAM7d,EAAMqE,EAA0B,EAAnBrD,UAAUnB,SAG1D26B,WAAY,SAAUx6B,GACrB,OAAO1D,KAAKuE,KAAM,kBACVvE,KAAMiD,GAAOk7B,QAASz6B,IAAUA,QAK1CT,GAAOsC,OAAQ,CACdgc,KAAM,SAAU9d,EAAMC,EAAMqE,GAC3B,IAAI3D,EAAKke,EACRkb,EAAQ/5B,EAAKlC,SAGd,GAAe,IAAVi8B,GAAyB,IAAVA,GAAyB,IAAVA,EAWnC,OAPe,IAAVA,GAAgBv6B,GAAOmE,SAAU3D,KAGrCC,EAAOT,GAAOk7B,QAASz6B,IAAUA,EACjC4e,EAAQrf,GAAO+0B,UAAWt0B,SAGZuC,IAAV8B,EACCua,GAAS,QAASA,QACuBrc,KAA3C7B,EAAMke,EAAMjB,IAAK5d,EAAMsE,EAAOrE,IACzBU,EAGCX,EAAMC,GAASqE,EAGpBua,GAAS,QAASA,GAA+C,QAApCle,EAAMke,EAAMte,IAAKP,EAAMC,IACjDU,EAGDX,EAAMC,IAGds0B,UAAW,CACV/jB,SAAU,CACTjQ,IAAK,SAAUP,GAMd,IAAI26B,EAAWn7B,GAAO4J,KAAK4D,KAAMhN,EAAM,YAEvC,OAAK26B,EACGjL,SAAUiL,EAAU,IAI3BP,GAAWp2B,KAAMhE,EAAKD,WACtBs6B,GAAWr2B,KAAMhE,EAAKD,WACtBC,EAAKuQ,KAEE,GAGA,KAKXmqB,QAAS,CACRE,MAAO,UACPC,QAAS,eAYLl9B,GAAQg8B,cACbn6B,GAAO+0B,UAAU5jB,SAAW,CAC3BpQ,IAAK,SAAUP,GAId,IAAI8O,EAAS9O,EAAKb,WAIlB,OAHK2P,GAAUA,EAAO3P,YACrB2P,EAAO3P,WAAWyR,cAEZ,MAERgN,IAAK,SAAU5d,GAId,IAAI8O,EAAS9O,EAAKb,WACb2P,IACJA,EAAO8B,cAEF9B,EAAO3P,YACX2P,EAAO3P,WAAWyR,kBAOvBpR,GAAOsB,KAAM,CACZ,WACA,WACA,YACA,cACA,cACA,UACA,UACA,SACA,cACA,mBACE,WACFtB,GAAOk7B,QAASn+B,KAAK2D,eAAkB3D,OA4BxCiD,GAAOG,GAAGmC,OAAQ,CACjBg5B,SAAU,SAAUx2B,GACnB,IAAIy2B,EAAY5kB,EAAK6kB,EAAUxuB,EAAW9N,EAAGu8B,EAE7C,OAAKr9B,EAAY0G,GACT/H,KAAKuE,KAAM,SAAUY,GAC3BlC,GAAQjD,MAAOu+B,SAAUx2B,EAAMtH,KAAMT,KAAMmF,EAAG64B,GAAUh+B,WAI1Dw+B,EAAaP,GAAgBl2B,IAEbxE,OACRvD,KAAKuE,KAAM,WAIjB,GAHAk6B,EAAWT,GAAUh+B,MACrB4Z,EAAwB,IAAlB5Z,KAAKuB,UAAoB,IAAMw8B,GAAkBU,GAAa,IAEzD,CACV,IAAMt8B,EAAI,EAAGA,EAAIq8B,EAAWj7B,OAAQpB,IACnC8N,EAAYuuB,EAAYr8B,GACnByX,EAAI/Y,QAAS,IAAMoP,EAAY,KAAQ,IAC3C2J,GAAO3J,EAAY,KAKrByuB,EAAaX,GAAkBnkB,GAC1B6kB,IAAaC,GACjB1+B,KAAKyC,aAAc,QAASi8B,MAMzB1+B,MAGR2+B,YAAa,SAAU52B,GACtB,IAAIy2B,EAAY5kB,EAAK6kB,EAAUxuB,EAAW9N,EAAGu8B,EAE7C,OAAKr9B,EAAY0G,GACT/H,KAAKuE,KAAM,SAAUY,GAC3BlC,GAAQjD,MAAO2+B,YAAa52B,EAAMtH,KAAMT,KAAMmF,EAAG64B,GAAUh+B,UAIvD0E,UAAUnB,QAIhBi7B,EAAaP,GAAgBl2B,IAEbxE,OACRvD,KAAKuE,KAAM,WAMjB,GALAk6B,EAAWT,GAAUh+B,MAGrB4Z,EAAwB,IAAlB5Z,KAAKuB,UAAoB,IAAMw8B,GAAkBU,GAAa,IAEzD,CACV,IAAMt8B,EAAI,EAAGA,EAAIq8B,EAAWj7B,OAAQpB,IAAM,CACzC8N,EAAYuuB,EAAYr8B,GAGxB,OAAgD,EAAxCyX,EAAI/Y,QAAS,IAAMoP,EAAY,KACtC2J,EAAMA,EAAIvT,QAAS,IAAM4J,EAAY,IAAK,KAK5CyuB,EAAaX,GAAkBnkB,GAC1B6kB,IAAaC,GACjB1+B,KAAKyC,aAAc,QAASi8B,MAMzB1+B,KA/BCA,KAAKyQ,KAAM,QAAS,KAkC7BmuB,YAAa,SAAU72B,EAAO82B,GAC7B,IAAIL,EAAYvuB,EAAW9N,EAAG+W,EAC7BvX,SAAcoG,EACd+2B,EAAwB,WAATn9B,GAAqBoE,MAAMC,QAAS+B,GAEpD,OAAK1G,EAAY0G,GACT/H,KAAKuE,KAAM,SAAUpC,GAC3Bc,GAAQjD,MAAO4+B,YACd72B,EAAMtH,KAAMT,KAAMmC,EAAG67B,GAAUh+B,MAAQ6+B,GACvCA,KAKsB,kBAAbA,GAA0BC,EAC9BD,EAAW7+B,KAAKu+B,SAAUx2B,GAAU/H,KAAK2+B,YAAa52B,IAG9Dy2B,EAAaP,GAAgBl2B,GAEtB/H,KAAKuE,KAAM,WACjB,GAAKu6B,EAKJ,IAFA5lB,EAAOjW,GAAQjD,MAETmC,EAAI,EAAGA,EAAIq8B,EAAWj7B,OAAQpB,IACnC8N,EAAYuuB,EAAYr8B,GAGnB+W,EAAK6lB,SAAU9uB,GACnBiJ,EAAKylB,YAAa1uB,GAElBiJ,EAAKqlB,SAAUtuB,aAKIhK,IAAV8B,GAAgC,YAATpG,KAClCsO,EAAY+tB,GAAUh+B,QAIrByhB,EAASJ,IAAKrhB,KAAM,gBAAiBiQ,GAOjCjQ,KAAKyC,cACTzC,KAAKyC,aAAc,QAClBwN,IAAuB,IAAVlI,EACZ,GACA0Z,EAASzd,IAAKhE,KAAM,kBAAqB,SAO/C++B,SAAU,SAAU77B,GACnB,IAAI+M,EAAWxM,EACdtB,EAAI,EAEL8N,EAAY,IAAM/M,EAAW,IAC7B,MAAUO,EAAOzD,KAAMmC,KACtB,GAAuB,IAAlBsB,EAAKlC,WACoE,GAA3E,IAAMw8B,GAAkBC,GAAUv6B,IAAW,KAAM5C,QAASoP,GAC9D,OAAO,EAIT,OAAO,KAOT,IAAI+uB,GAAU,MAEd/7B,GAAOG,GAAGmC,OAAQ,CACjBnD,IAAK,SAAU2F,GACd,IAAIua,EAAOle,EAAKuqB,EACflrB,EAAOzD,KAAM,GAEd,OAAM0E,UAAUnB,QA0BhBorB,EAAkBttB,EAAY0G,GAEvB/H,KAAKuE,KAAM,SAAUpC,GAC3B,IAAIC,EAEmB,IAAlBpC,KAAKuB,WAWE,OANXa,EADIusB,EACE5mB,EAAMtH,KAAMT,KAAMmC,EAAGc,GAAQjD,MAAOoC,OAEpC2F,GAKN3F,EAAM,GAEoB,iBAARA,EAClBA,GAAO,GAEI2D,MAAMC,QAAS5D,KAC1BA,EAAMa,GAAOwB,IAAKrC,EAAK,SAAU2F,GAChC,OAAgB,MAATA,EAAgB,GAAKA,EAAQ,OAItCua,EAAQrf,GAAOg8B,SAAUj/B,KAAK2B,OAAUsB,GAAOg8B,SAAUj/B,KAAKwD,SAASG,iBAGrD,QAAS2e,QAA+Crc,IAApCqc,EAAMjB,IAAKrhB,KAAMoC,EAAK,WAC3DpC,KAAK+H,MAAQ3F,OAzDTqB,GACJ6e,EAAQrf,GAAOg8B,SAAUx7B,EAAK9B,OAC7BsB,GAAOg8B,SAAUx7B,EAAKD,SAASG,iBAG/B,QAAS2e,QACgCrc,KAAvC7B,EAAMke,EAAMte,IAAKP,EAAM,UAElBW,EAMY,iBAHpBA,EAAMX,EAAKsE,OAIH3D,EAAIiC,QAAS24B,GAAS,IAIhB,MAAP56B,EAAc,GAAKA,OAG3B,KAyCHnB,GAAOsC,OAAQ,CACd05B,SAAU,CACT5Z,OAAQ,CACPrhB,IAAK,SAAUP,GAEd,IAAIrB,EAAMa,GAAO4J,KAAK4D,KAAMhN,EAAM,SAClC,OAAc,MAAPrB,EACNA,EAMA27B,GAAkB96B,GAAOV,KAAMkB,MAGlCyK,OAAQ,CACPlK,IAAK,SAAUP,GACd,IAAIsE,EAAOsd,EAAQljB,EAClBqD,EAAU/B,EAAK+B,QACfwU,EAAQvW,EAAK4Q,cACbgT,EAAoB,eAAd5jB,EAAK9B,KACX6iB,EAAS6C,EAAM,KAAO,GACtBkN,EAAMlN,EAAMrN,EAAQ,EAAIxU,EAAQjC,OAUjC,IAPCpB,EADI6X,EAAQ,EACRua,EAGAlN,EAAMrN,EAAQ,EAIX7X,EAAIoyB,EAAKpyB,IAKhB,KAJAkjB,EAAS7f,EAASrD,IAIJiS,UAAYjS,IAAM6X,KAG7BqL,EAAO9Y,YACL8Y,EAAOziB,WAAW2J,WACnB/I,GAAU6hB,EAAOziB,WAAY,aAAiB,CAMjD,GAHAmF,EAAQ9E,GAAQoiB,GAASjjB,MAGpBilB,EACJ,OAAOtf,EAIRyc,EAAO5jB,KAAMmH,GAIf,OAAOyc,GAGRnD,IAAK,SAAU5d,EAAMsE,GACpB,IAAIm3B,EAAW7Z,EACd7f,EAAU/B,EAAK+B,QACfgf,EAASvhB,GAAOgE,UAAWc,GAC3B5F,EAAIqD,EAAQjC,OAEb,MAAQpB,MACPkjB,EAAS7f,EAASrD,IAINiS,UACuD,EAAlEnR,GAAOkE,QAASlE,GAAOg8B,SAAS5Z,OAAOrhB,IAAKqhB,GAAUb,MAEtD0a,GAAY,GAUd,OAHMA,IACLz7B,EAAK4Q,eAAiB,GAEhBmQ,OAOXvhB,GAAOsB,KAAM,CAAE,QAAS,YAAc,WACrCtB,GAAOg8B,SAAUj/B,MAAS,CACzBqhB,IAAK,SAAU5d,EAAMsE,GACpB,GAAKhC,MAAMC,QAAS+B,GACnB,OAAStE,EAAK0Q,SAA2D,EAAjDlR,GAAOkE,QAASlE,GAAQQ,GAAOrB,MAAO2F,KAI3D3G,GAAQ+7B,UACbl6B,GAAOg8B,SAAUj/B,MAAOgE,IAAM,SAAUP,GACvC,OAAwC,OAAjCA,EAAKjB,aAAc,SAAqB,KAAOiB,EAAKsE,UAS9D,IAAI0L,GAAW1T,GAAO0T,SAElB5R,GAAQ,CAAEmG,KAAMkjB,KAAKC,OAErBgU,GAAS,KAKbl8B,GAAOm8B,SAAW,SAAU9d,GAC3B,IAAInP,EAAKktB,EACT,IAAM/d,GAAwB,iBAATA,EACpB,OAAO,KAKR,IACCnP,GAAM,IAAMpS,GAAOu/B,WAAcC,gBAAiBje,EAAM,YACvD,MAAQ3U,IAYV,OAVA0yB,EAAkBltB,GAAOA,EAAI3E,qBAAsB,eAAiB,GAC9D2E,IAAOktB,GACZp8B,GAAOsD,MAAO,iBACb84B,EACCp8B,GAAOwB,IAAK46B,EAAgB3yB,WAAY,SAAUgC,GACjD,OAAOA,EAAG5H,cACPgH,KAAM,MACVwT,IAGInP,GAIR,IAAIqtB,GAAc,kCACjBC,GAA0B,SAAU9yB,GACnCA,EAAEmb,mBAGJ7kB,GAAOsC,OAAQtC,GAAOskB,MAAO,CAE5BU,QAAS,SAAUV,EAAOjG,EAAM7d,EAAMi8B,GAErC,IAAIv9B,EAAGyX,EAAKgJ,EAAK+c,EAAYC,EAAQ/W,EAAQ9K,EAAS8hB,EACrDC,EAAY,CAAEr8B,GAAQ7D,GACtB+B,EAAOX,GAAOP,KAAM8mB,EAAO,QAAWA,EAAM5lB,KAAO4lB,EACnDkB,EAAaznB,GAAOP,KAAM8mB,EAAO,aAAgBA,EAAMlgB,UAAUc,MAAO,KAAQ,GAKjF,GAHAyR,EAAMimB,EAAcjd,EAAMnf,EAAOA,GAAQ7D,EAGlB,IAAlB6D,EAAKlC,UAAoC,IAAlBkC,EAAKlC,WAK5Bi+B,GAAY/3B,KAAM9F,EAAOsB,GAAOskB,MAAMuB,cAIf,EAAvBnnB,EAAKd,QAAS,OAIlBc,GADA8mB,EAAa9mB,EAAKwG,MAAO,MACPoG,QAClBka,EAAWpjB,QAEZu6B,EAASj+B,EAAKd,QAAS,KAAQ,GAAK,KAAOc,GAG3C4lB,EAAQA,EAAOtkB,GAAOiD,SACrBqhB,EACA,IAAItkB,GAAOmnB,MAAOzoB,EAAuB,iBAAV4lB,GAAsBA,IAGhDK,UAAY8X,EAAe,EAAI,EACrCnY,EAAMlgB,UAAYohB,EAAW3a,KAAM,KACnCyZ,EAAMuC,WAAavC,EAAMlgB,UACxB,IAAImB,OAAQ,UAAYigB,EAAW3a,KAAM,iBAAoB,WAC7D,KAGDyZ,EAAM3V,YAAS3L,EACTshB,EAAM3hB,SACX2hB,EAAM3hB,OAASnC,GAIhB6d,EAAe,MAARA,EACN,CAAEiG,GACFtkB,GAAOgE,UAAWqa,EAAM,CAAEiG,IAG3BxJ,EAAU9a,GAAOskB,MAAMxJ,QAASpc,IAAU,GACpC+9B,IAAgB3hB,EAAQkK,UAAmD,IAAxClK,EAAQkK,QAAQtnB,MAAO8C,EAAM6d,IAAtE,CAMA,IAAMoe,IAAiB3hB,EAAQ0M,WAAahpB,EAAUgC,GAAS,CAM9D,IAJAk8B,EAAa5hB,EAAQ8J,cAAgBlmB,EAC/B69B,GAAY/3B,KAAMk4B,EAAah+B,KACpCiY,EAAMA,EAAIhX,YAEHgX,EAAKA,EAAMA,EAAIhX,WACtBk9B,EAAUl/B,KAAMgZ,GAChBgJ,EAAMhJ,EAIFgJ,KAAUnf,EAAK+D,eAAiB5H,IACpCkgC,EAAUl/B,KAAMgiB,EAAIvT,aAAeuT,EAAImd,cAAgBhgC,IAKzDoC,EAAI,EACJ,OAAUyX,EAAMkmB,EAAW39B,QAAYolB,EAAMqC,uBAC5CiW,EAAcjmB,EACd2N,EAAM5lB,KAAW,EAAJQ,EACZw9B,EACA5hB,EAAQiL,UAAYrnB,GAGrBknB,GAAWpH,EAASzd,IAAK4V,EAAK,WAAcxZ,OAAOwoB,OAAQ,OAAUrB,EAAM5lB,OAC1E8f,EAASzd,IAAK4V,EAAK,YAEnBiP,EAAOloB,MAAOiZ,EAAK0H,IAIpBuH,EAAS+W,GAAUhmB,EAAKgmB,KACT/W,EAAOloB,OAASogB,EAAYnH,KAC1C2N,EAAM3V,OAASiX,EAAOloB,MAAOiZ,EAAK0H,IACZ,IAAjBiG,EAAM3V,QACV2V,EAAMS,kBA8CT,OA1CAT,EAAM5lB,KAAOA,EAGP+9B,GAAiBnY,EAAMuD,sBAEpB/M,EAAQ4H,WACqC,IAApD5H,EAAQ4H,SAAShlB,MAAOm/B,EAAUz3B,MAAOiZ,KACzCP,EAAYtd,IAIPm8B,GAAUv+B,EAAYoC,EAAM9B,MAAaF,EAAUgC,MAGvDmf,EAAMnf,EAAMm8B,MAGXn8B,EAAMm8B,GAAW,MAIlB38B,GAAOskB,MAAMuB,UAAYnnB,EAEpB4lB,EAAMqC,wBACViW,EAAYtwB,iBAAkB5N,EAAM89B,IAGrCh8B,EAAM9B,KAED4lB,EAAMqC,wBACViW,EAAYjgB,oBAAqBje,EAAM89B,IAGxCx8B,GAAOskB,MAAMuB,eAAY7iB,EAEpB2c,IACJnf,EAAMm8B,GAAWhd,IAMd2E,EAAM3V,SAKdwb,SAAU,SAAUzrB,EAAM8B,EAAM8jB,GAC/B,IAAI5a,EAAI1J,GAAOsC,OACd,IAAItC,GAAOmnB,MACX7C,EACA,CACC5lB,KAAMA,EACNypB,aAAa,IAIfnoB,GAAOskB,MAAMU,QAAStb,EAAG,KAAMlJ,MAKjCR,GAAOG,GAAGmC,OAAQ,CAEjB0iB,QAAS,SAAUtmB,EAAM2f,GACxB,OAAOthB,KAAKuE,KAAM,WACjBtB,GAAOskB,MAAMU,QAAStmB,EAAM2f,EAAMthB,SAGpCggC,eAAgB,SAAUr+B,EAAM2f,GAC/B,IAAI7d,EAAOzD,KAAM,GACjB,GAAKyD,EACJ,OAAOR,GAAOskB,MAAMU,QAAStmB,EAAM2f,EAAM7d,GAAM,MAMlD,IACCw8B,GAAW,QACXC,GAAQ,SACRC,GAAkB,wCAClBC,GAAe,qCAEhB,SAASC,GAAa1I,EAAQr2B,EAAKg/B,EAAapmB,GAC/C,IAAIxW,EAEJ,GAAKqC,MAAMC,QAAS1E,GAGnB2B,GAAOsB,KAAMjD,EAAK,SAAUa,EAAG2Y,GACzBwlB,GAAeL,GAASx4B,KAAMkwB,GAGlCzd,EAAKyd,EAAQ7c,GAKbulB,GACC1I,EAAS,KAAqB,iBAAN7c,GAAuB,MAALA,EAAY3Y,EAAI,IAAO,IACjE2Y,EACAwlB,EACApmB,UAKG,GAAMomB,GAAiC,WAAlBx9B,EAAQxB,GAUnC4Y,EAAKyd,EAAQr2B,QAPb,IAAMoC,KAAQpC,EACb++B,GAAa1I,EAAS,IAAMj0B,EAAO,IAAKpC,EAAKoC,GAAQ48B,EAAapmB,GAYrEjX,GAAOs9B,MAAQ,SAAU73B,EAAG43B,GAC3B,IAAI3I,EACH6I,EAAI,GACJtmB,EAAM,SAAU7L,EAAKoyB,GAGpB,IAAI14B,EAAQ1G,EAAYo/B,GACvBA,IACAA,EAEDD,EAAGA,EAAEj9B,QAAWm9B,mBAAoBryB,GAAQ,IAC3CqyB,mBAA6B,MAAT34B,EAAgB,GAAKA,IAG5C,GAAU,MAALW,EACJ,MAAO,GAIR,GAAK3C,MAAMC,QAAS0C,IAASA,EAAE7E,SAAWZ,GAAO6C,cAAe4C,GAG/DzF,GAAOsB,KAAMmE,EAAG,WACfwR,EAAKla,KAAK0D,KAAM1D,KAAK+H,cAOtB,IAAM4vB,KAAUjvB,EACf23B,GAAa1I,EAAQjvB,EAAGivB,GAAU2I,EAAapmB,GAKjD,OAAOsmB,EAAE1yB,KAAM,MAGhB7K,GAAOG,GAAGmC,OAAQ,CACjBo7B,UAAW,WACV,OAAO19B,GAAOs9B,MAAOvgC,KAAK4gC,mBAE3BA,eAAgB,WACf,OAAO5gC,KAAKyE,IAAK,WAGhB,IAAI8L,EAAWtN,GAAOse,KAAMvhB,KAAM,YAClC,OAAOuQ,EAAWtN,GAAOgE,UAAWsJ,GAAavQ,OAC9C6P,OAAQ,WACX,IAAIlO,EAAO3B,KAAK2B,KAGhB,OAAO3B,KAAK0D,OAAST,GAAQjD,MAAO2Y,GAAI,cACvCynB,GAAa34B,KAAMzH,KAAKwD,YAAe28B,GAAgB14B,KAAM9F,KAC3D3B,KAAKmU,UAAY0Q,GAAepd,KAAM9F,MACtC8C,IAAK,SAAU2D,EAAI3E,GACtB,IAAIrB,EAAMa,GAAQjD,MAAOoC,MAEzB,OAAY,MAAPA,EACG,KAGH2D,MAAMC,QAAS5D,GACZa,GAAOwB,IAAKrC,EAAK,SAAUA,GACjC,MAAO,CAAEsB,KAAMD,EAAKC,KAAMqE,MAAO3F,EAAIiE,QAAS65B,GAAO,WAIhD,CAAEx8B,KAAMD,EAAKC,KAAMqE,MAAO3F,EAAIiE,QAAS65B,GAAO,WAClDl8B,SAKN,IACC68B,GAAM,OACNC,GAAQ,OACRC,GAAa,gBACbC,GAAW,6BAIXC,GAAa,iBACbC,GAAY,QAWZhH,GAAa,GAObiH,GAAa,GAGbC,GAAW,KAAK1gC,OAAQ,KAGxB2gC,GAAezhC,EAAS0C,cAAe,KAKxC,SAASg/B,GAA6BC,GAGrC,OAAO,SAAUC,EAAoB5kB,GAED,iBAAvB4kB,IACX5kB,EAAO4kB,EACPA,EAAqB,KAGtB,IAAIC,EACHt/B,EAAI,EACJu/B,EAAYF,EAAmB79B,cAAcsJ,MAAO2N,IAAmB,GAExE,GAAKvZ,EAAYub,GAGhB,MAAU6kB,EAAWC,EAAWv/B,KAGR,MAAlBs/B,EAAU,IACdA,EAAWA,EAASnhC,MAAO,IAAO,KAChCihC,EAAWE,GAAaF,EAAWE,IAAc,IAAKjf,QAAS5F,KAI/D2kB,EAAWE,GAAaF,EAAWE,IAAc,IAAK7gC,KAAMgc,IAQnE,SAAS+kB,GAA+BJ,EAAW/7B,EAASi1B,EAAiBmH,GAE5E,IAAIC,EAAY,GACfC,EAAqBP,IAAcJ,GAEpC,SAASY,EAASN,GACjB,IAAIrtB,EAcJ,OAbAytB,EAAWJ,IAAa,EACxBx+B,GAAOsB,KAAMg9B,EAAWE,IAAc,GAAI,SAAU9lB,EAAGqmB,GACtD,IAAIC,EAAsBD,EAAoBx8B,EAASi1B,EAAiBmH,GACxE,MAAoC,iBAAxBK,GACVH,GAAqBD,EAAWI,GAKtBH,IACD1tB,EAAW6tB,QADf,GAHNz8B,EAAQk8B,UAAUlf,QAASyf,GAC3BF,EAASE,IACF,KAKF7tB,EAGR,OAAO2tB,EAASv8B,EAAQk8B,UAAW,MAAUG,EAAW,MAASE,EAAS,KAM3E,SAASG,GAAYt8B,EAAQhE,GAC5B,IAAIyM,EAAKxI,EACRs8B,EAAcl/B,GAAOm/B,aAAaD,aAAe,GAElD,IAAM9zB,KAAOzM,OACQqE,IAAfrE,EAAKyM,MACP8zB,EAAa9zB,GAAQzI,EAAWC,IAAUA,EAAO,KAAUwI,GAAQzM,EAAKyM,IAO5E,OAJKxI,GACJ5C,GAAOsC,QAAQ,EAAMK,EAAQC,GAGvBD,EA/ERy7B,GAAartB,KAAOP,GAASO,KAgP7B/Q,GAAOsC,OAAQ,CAGd88B,OAAQ,EAGRC,aAAc,GACdC,KAAM,GAENH,aAAc,CACbI,IAAK/uB,GAASO,KACdrS,KAAM,MACN8gC,QAxRgB,4DAwRQh7B,KAAMgM,GAASivB,UACvCljC,QAAQ,EACRmjC,aAAa,EACbC,OAAO,EACPC,YAAa,mDAcbC,QAAS,CACR9H,IAAKoG,GACL7+B,KAAM,aACNqsB,KAAM,YACNzc,IAAK,4BACL4wB,KAAM,qCAGPtpB,SAAU,CACTtH,IAAK,UACLyc,KAAM,SACNmU,KAAM,YAGPC,eAAgB,CACf7wB,IAAK,cACL5P,KAAM,eACNwgC,KAAM,gBAKPE,WAAY,CAGXC,SAAUj3B,OAGVk3B,aAAa,EAGbC,YAAathB,KAAKC,MAGlBshB,WAAYpgC,GAAOm8B,UAOpB+C,YAAa,CACZK,KAAK,EACLr/B,SAAS,IAOXmgC,UAAW,SAAU19B,EAAQ29B,GAC5B,OAAOA,EAGNrB,GAAYA,GAAYt8B,EAAQ3C,GAAOm/B,cAAgBmB,GAGvDrB,GAAYj/B,GAAOm/B,aAAcx8B,IAGnC49B,cAAelC,GAA6BpH,IAC5CuJ,cAAenC,GAA6BH,IAG5CuC,KAAM,SAAUlB,EAAKh9B,GAGA,iBAARg9B,IACXh9B,EAAUg9B,EACVA,OAAMv8B,GAIPT,EAAUA,GAAW,GAErB,IAAIm+B,EAGHC,EAGAC,EACAC,EAGAC,EAGAC,EAGArkB,EAGAskB,EAGA9hC,EAGA+hC,EAGA1D,EAAIv9B,GAAOqgC,UAAW,GAAI99B,GAG1B2+B,EAAkB3D,EAAEr9B,SAAWq9B,EAG/B4D,EAAqB5D,EAAEr9B,UACpBghC,EAAgB5iC,UAAY4iC,EAAgBtgC,QAC9CZ,GAAQkhC,GACRlhC,GAAOskB,MAGRvK,EAAW/Z,GAAO0Z,WAClB0nB,EAAmBphC,GAAOwY,UAAW,eAGrC6oB,EAAa9D,EAAE8D,YAAc,GAG7BC,EAAiB,GACjBC,EAAsB,GAGtBC,EAAW,WAGX7C,EAAQ,CACP7hB,WAAY,EAGZ2kB,kBAAmB,SAAUr2B,GAC5B,IAAIpB,EACJ,GAAK0S,EAAY,CAChB,IAAMmkB,EAAkB,CACvBA,EAAkB,GAClB,MAAU72B,EAAQ+zB,GAAS3zB,KAAMw2B,GAChCC,EAAiB72B,EAAO,GAAItJ,cAAgB,MACzCmgC,EAAiB72B,EAAO,GAAItJ,cAAgB,MAAS,IACrDjD,OAAQuM,EAAO,IAGpBA,EAAQ62B,EAAiBz1B,EAAI1K,cAAgB,KAE9C,OAAgB,MAATsJ,EAAgB,KAAOA,EAAMa,KAAM,OAI3C62B,sBAAuB,WACtB,OAAOhlB,EAAYkkB,EAAwB,MAI5Ce,iBAAkB,SAAUlhC,EAAMqE,GAMjC,OALkB,MAAb4X,IACJjc,EAAO8gC,EAAqB9gC,EAAKC,eAChC6gC,EAAqB9gC,EAAKC,gBAAmBD,EAC9C6gC,EAAgB7gC,GAASqE,GAEnB/H,MAIR6kC,iBAAkB,SAAUljC,GAI3B,OAHkB,MAAbge,IACJ6gB,EAAEsE,SAAWnjC,GAEP3B,MAIRskC,WAAY,SAAU7/B,GACrB,IAAIzC,EACJ,GAAKyC,EACJ,GAAKkb,EAGJiiB,EAAM7kB,OAAQtY,EAAKm9B,EAAMmD,cAIzB,IAAM/iC,KAAQyC,EACb6/B,EAAYtiC,GAAS,CAAEsiC,EAAYtiC,GAAQyC,EAAKzC,IAInD,OAAOhC,MAIRglC,MAAO,SAAUC,GAChB,IAAIC,EAAYD,GAAcR,EAK9B,OAJKd,GACJA,EAAUqB,MAAOE,GAElBp7B,EAAM,EAAGo7B,GACFllC,OAoBV,GAfAgd,EAAS1B,QAASsmB,GAKlBpB,EAAEgC,MAAUA,GAAOhC,EAAEgC,KAAO/uB,GAASO,MAAS,IAC5C3N,QAAS66B,GAAWztB,GAASivB,SAAW,MAG1ClC,EAAE7+B,KAAO6D,EAAQ6V,QAAU7V,EAAQ7D,MAAQ6+B,EAAEnlB,QAAUmlB,EAAE7+B,KAGzD6+B,EAAEkB,WAAclB,EAAEiB,UAAY,KAAM99B,cAAcsJ,MAAO2N,IAAmB,CAAE,IAGxD,MAAjB4lB,EAAE2E,YAAsB,CAC5BnB,EAAYpkC,EAAS0C,cAAe,KAKpC,IACC0hC,EAAUhwB,KAAOwsB,EAAEgC,IAInBwB,EAAUhwB,KAAOgwB,EAAUhwB,KAC3BwsB,EAAE2E,YAAc9D,GAAaqB,SAAW,KAAOrB,GAAa+D,MAC3DpB,EAAUtB,SAAW,KAAOsB,EAAUoB,KACtC,MAAQz4B,GAIT6zB,EAAE2E,aAAc,GAalB,GARK3E,EAAElf,MAAQkf,EAAEmC,aAAiC,iBAAXnC,EAAElf,OACxCkf,EAAElf,KAAOre,GAAOs9B,MAAOC,EAAElf,KAAMkf,EAAEF,cAIlCqB,GAA+BzH,GAAYsG,EAAGh7B,EAASo8B,GAGlDjiB,EACJ,OAAOiiB,EA8ER,IAAMz/B,KAzEN8hC,EAAchhC,GAAOskB,OAASiZ,EAAEhhC,SAGQ,GAApByD,GAAOo/B,UAC1Bp/B,GAAOskB,MAAMU,QAAS,aAIvBuY,EAAE7+B,KAAO6+B,EAAE7+B,KAAKif,cAGhB4f,EAAE6E,YAAcpE,GAAWx5B,KAAM+4B,EAAE7+B,MAKnCiiC,EAAWpD,EAAEgC,IAAIn8B,QAASy6B,GAAO,IAG3BN,EAAE6E,WAwBI7E,EAAElf,MAAQkf,EAAEmC,aACoD,KAAzEnC,EAAEqC,aAAe,IAAKhiC,QAAS,uCACjC2/B,EAAElf,KAAOkf,EAAElf,KAAKjb,QAASw6B,GAAK,OAvB9BqD,EAAW1D,EAAEgC,IAAIliC,MAAOsjC,EAASrgC,QAG5Bi9B,EAAElf,OAAUkf,EAAEmC,aAAiC,iBAAXnC,EAAElf,QAC1CsiB,IAAczE,GAAO13B,KAAMm8B,GAAa,IAAM,KAAQpD,EAAElf,YAGjDkf,EAAElf,OAIO,IAAZkf,EAAEpyB,QACNw1B,EAAWA,EAASv9B,QAAS06B,GAAY,MACzCmD,GAAa/E,GAAO13B,KAAMm8B,GAAa,IAAM,KAAQ,KAAS/hC,GAAMmG,OACnEk8B,GAIF1D,EAAEgC,IAAMoB,EAAWM,GASf1D,EAAE8E,aACDriC,GAAOq/B,aAAcsB,IACzBhC,EAAMgD,iBAAkB,oBAAqB3hC,GAAOq/B,aAAcsB,IAE9D3gC,GAAOs/B,KAAMqB,IACjBhC,EAAMgD,iBAAkB,gBAAiB3hC,GAAOs/B,KAAMqB,MAKnDpD,EAAElf,MAAQkf,EAAE6E,aAAgC,IAAlB7E,EAAEqC,aAAyBr9B,EAAQq9B,cACjEjB,EAAMgD,iBAAkB,eAAgBpE,EAAEqC,aAI3CjB,EAAMgD,iBACL,SACApE,EAAEkB,UAAW,IAAOlB,EAAEsC,QAAStC,EAAEkB,UAAW,IAC3ClB,EAAEsC,QAAStC,EAAEkB,UAAW,KACA,MAArBlB,EAAEkB,UAAW,GAAc,KAAON,GAAW,WAAa,IAC7DZ,EAAEsC,QAAS,MAIFtC,EAAE+E,QACZ3D,EAAMgD,iBAAkBziC,EAAGq+B,EAAE+E,QAASpjC,IAIvC,GAAKq+B,EAAEgF,cAC+C,IAAnDhF,EAAEgF,WAAW/kC,KAAM0jC,EAAiBvC,EAAOpB,IAAiB7gB,GAG9D,OAAOiiB,EAAMoD,QAed,GAXAP,EAAW,QAGXJ,EAAiBnqB,IAAKsmB,EAAE3F,UACxB+G,EAAM93B,KAAM02B,EAAEiF,SACd7D,EAAMrmB,KAAMilB,EAAEj6B,OAGdo9B,EAAYhC,GAA+BR,GAAYX,EAAGh7B,EAASo8B,GAK5D,CASN,GARAA,EAAM7hB,WAAa,EAGdkkB,GACJG,EAAmBnc,QAAS,WAAY,CAAE2Z,EAAOpB,IAI7C7gB,EACJ,OAAOiiB,EAIHpB,EAAEoC,OAAqB,EAAZpC,EAAEvD,UACjB8G,EAAehkC,GAAO2e,WAAY,WACjCkjB,EAAMoD,MAAO,YACXxE,EAAEvD,UAGN,IACCtd,GAAY,EACZgkB,EAAU+B,KAAMnB,EAAgBz6B,GAC/B,MAAQ6C,GAGT,GAAKgT,EACJ,MAAMhT,EAIP7C,GAAO,EAAG6C,SAhCX7C,GAAO,EAAG,gBAqCX,SAASA,EAAMi7B,EAAQY,EAAkBC,EAAWL,GACnD,IAAIM,EAAWJ,EAASl/B,EAAOu/B,EAAUC,EACxCd,EAAaU,EAGThmB,IAILA,GAAY,EAGPokB,GACJhkC,GAAOm9B,aAAc6G,GAKtBJ,OAAY19B,EAGZ49B,EAAwB0B,GAAW,GAGnC3D,EAAM7hB,WAAsB,EAATglB,EAAa,EAAI,EAGpCc,EAAsB,KAAVd,GAAiBA,EAAS,KAAkB,MAAXA,EAGxCa,IACJE,EA7lBJ,SAA8BtF,EAAGoB,EAAOgE,GAEvC,IAAII,EAAIrkC,EAAMskC,EAAeC,EAC5BzsB,EAAW+mB,EAAE/mB,SACbioB,EAAYlB,EAAEkB,UAGf,MAA2B,MAAnBA,EAAW,GAClBA,EAAUnzB,aACEtI,IAAP+/B,IACJA,EAAKxF,EAAEsE,UAAYlD,EAAM8C,kBAAmB,iBAK9C,GAAKsB,EACJ,IAAMrkC,KAAQ8X,EACb,GAAKA,EAAU9X,IAAU8X,EAAU9X,GAAO8F,KAAMu+B,GAAO,CACtDtE,EAAUlf,QAAS7gB,GACnB,MAMH,GAAK+/B,EAAW,KAAOkE,EACtBK,EAAgBvE,EAAW,OACrB,CAGN,IAAM//B,KAAQikC,EAAY,CACzB,IAAMlE,EAAW,IAAOlB,EAAEyC,WAAYthC,EAAO,IAAM+/B,EAAW,IAAQ,CACrEuE,EAAgBtkC,EAChB,MAEKukC,IACLA,EAAgBvkC,GAKlBskC,EAAgBA,GAAiBC,EAMlC,GAAKD,EAIJ,OAHKA,IAAkBvE,EAAW,IACjCA,EAAUlf,QAASyjB,GAEbL,EAAWK,GA0iBLE,CAAqB3F,EAAGoB,EAAOgE,KAIrCC,IACsC,EAA3C5iC,GAAOkE,QAAS,SAAUq5B,EAAEkB,YAC5Bz+B,GAAOkE,QAAS,OAAQq5B,EAAEkB,WAAc,IACxClB,EAAEyC,WAAY,eAAkB,cAIjC6C,EA9iBH,SAAsBtF,EAAGsF,EAAUlE,EAAOiE,GACzC,IAAIO,EAAOC,EAASC,EAAM1jB,EAAKlJ,EAC9BupB,EAAa,GAGbvB,EAAYlB,EAAEkB,UAAUphC,QAGzB,GAAKohC,EAAW,GACf,IAAM4E,KAAQ9F,EAAEyC,WACfA,EAAYqD,EAAK3iC,eAAkB68B,EAAEyC,WAAYqD,GAInDD,EAAU3E,EAAUnzB,QAGpB,MAAQ83B,EAcP,GAZK7F,EAAEwC,eAAgBqD,KACtBzE,EAAOpB,EAAEwC,eAAgBqD,IAAcP,IAIlCpsB,GAAQmsB,GAAarF,EAAE+F,aAC5BT,EAAWtF,EAAE+F,WAAYT,EAAUtF,EAAEiB,WAGtC/nB,EAAO2sB,EACPA,EAAU3E,EAAUnzB,QAKnB,GAAiB,MAAZ83B,EAEJA,EAAU3sB,OAGJ,GAAc,MAATA,GAAgBA,IAAS2sB,EAAU,CAM9C,KAHAC,EAAOrD,EAAYvpB,EAAO,IAAM2sB,IAAapD,EAAY,KAAOoD,IAI/D,IAAMD,KAASnD,EAId,IADArgB,EAAMwjB,EAAMj+B,MAAO,MACT,KAAQk+B,IAGjBC,EAAOrD,EAAYvpB,EAAO,IAAMkJ,EAAK,KACpCqgB,EAAY,KAAOrgB,EAAK,KACb,EAGG,IAAT0jB,EACJA,EAAOrD,EAAYmD,IAGgB,IAAxBnD,EAAYmD,KACvBC,EAAUzjB,EAAK,GACf8e,EAAUlf,QAASI,EAAK,KAEzB,MAOJ,IAAc,IAAT0jB,EAGJ,GAAKA,GAAQ9F,EAAEgG,UACdV,EAAWQ,EAAMR,QAEjB,IACCA,EAAWQ,EAAMR,GAChB,MAAQn5B,GACT,MAAO,CACNmQ,MAAO,cACPvW,MAAO+/B,EAAO35B,EAAI,sBAAwB+M,EAAO,OAAS2sB,IASjE,MAAO,CAAEvpB,MAAO,UAAWwE,KAAMwkB,GAidpBW,CAAajG,EAAGsF,EAAUlE,EAAOiE,GAGvCA,GAGCrF,EAAE8E,cACNS,EAAWnE,EAAM8C,kBAAmB,oBAEnCzhC,GAAOq/B,aAAcsB,GAAamC,IAEnCA,EAAWnE,EAAM8C,kBAAmB,WAEnCzhC,GAAOs/B,KAAMqB,GAAamC,IAKZ,MAAXhB,GAA6B,SAAXvE,EAAE7+B,KACxBsjC,EAAa,YAGS,MAAXF,EACXE,EAAa,eAIbA,EAAaa,EAAShpB,MACtB2oB,EAAUK,EAASxkB,KAEnBukB,IADAt/B,EAAQu/B,EAASv/B,UAMlBA,EAAQ0+B,GACHF,GAAWE,IACfA,EAAa,QACRF,EAAS,IACbA,EAAS,KAMZnD,EAAMmD,OAASA,EACfnD,EAAMqD,YAAeU,GAAoBV,GAAe,GAGnDY,EACJ7oB,EAASoB,YAAa+lB,EAAiB,CAAEsB,EAASR,EAAYrD,IAE9D5kB,EAASuB,WAAY4lB,EAAiB,CAAEvC,EAAOqD,EAAY1+B,IAI5Dq7B,EAAM0C,WAAYA,GAClBA,OAAar+B,EAERg+B,GACJG,EAAmBnc,QAAS4d,EAAY,cAAgB,YACvD,CAAEjE,EAAOpB,EAAGqF,EAAYJ,EAAUl/B,IAIpC89B,EAAiB3nB,SAAUynB,EAAiB,CAAEvC,EAAOqD,IAEhDhB,IACJG,EAAmBnc,QAAS,eAAgB,CAAE2Z,EAAOpB,MAG3Cv9B,GAAOo/B,QAChBp/B,GAAOskB,MAAMU,QAAS,cAKzB,OAAO2Z,GAGR8E,QAAS,SAAUlE,EAAKlhB,EAAM9c,GAC7B,OAAOvB,GAAOe,IAAKw+B,EAAKlhB,EAAM9c,EAAU,SAGzCmiC,UAAW,SAAUnE,EAAKh+B,GACzB,OAAOvB,GAAOe,IAAKw+B,OAAKv8B,EAAWzB,EAAU,aAI/CvB,GAAOsB,KAAM,CAAE,MAAO,QAAU,SAAU6D,EAAIiT,GAC7CpY,GAAQoY,GAAW,SAAUmnB,EAAKlhB,EAAM9c,EAAU7C,GAUjD,OAPKN,EAAYigB,KAChB3f,EAAOA,GAAQ6C,EACfA,EAAW8c,EACXA,OAAOrb,GAIDhD,GAAOygC,KAAMzgC,GAAOsC,OAAQ,CAClCi9B,IAAKA,EACL7gC,KAAM0Z,EACNomB,SAAU9/B,EACV2f,KAAMA,EACNmkB,QAASjhC,GACPvB,GAAO6C,cAAe08B,IAASA,OAIpCv/B,GAAOugC,cAAe,SAAUhD,GAC/B,IAAIr+B,EACJ,IAAMA,KAAKq+B,EAAE+E,QACa,iBAApBpjC,EAAEwB,gBACN68B,EAAEqC,YAAcrC,EAAE+E,QAASpjC,IAAO,MAMrCc,GAAO4rB,SAAW,SAAU2T,EAAKh9B,EAAStD,GACzC,OAAOe,GAAOygC,KAAM,CACnBlB,IAAKA,EAGL7gC,KAAM,MACN8/B,SAAU,SACVrzB,OAAO,EACPw0B,OAAO,EACPpjC,QAAQ,EAKRyjC,WAAY,CACX2D,cAAe,cAEhBL,WAAY,SAAUT,GACrB7iC,GAAO4D,WAAYi/B,EAAUtgC,EAAStD,OAMzCe,GAAOG,GAAGmC,OAAQ,CACjBshC,QAAS,SAAUjY,GAClB,IAAIlI,EAyBJ,OAvBK1mB,KAAM,KACLqB,EAAYutB,KAChBA,EAAOA,EAAKnuB,KAAMT,KAAM,KAIzB0mB,EAAOzjB,GAAQ2rB,EAAM5uB,KAAM,GAAIwH,eAAgB5C,GAAI,GAAIe,OAAO,GAEzD3F,KAAM,GAAI4C,YACd8jB,EAAK8I,aAAcxvB,KAAM,IAG1B0mB,EAAKjiB,IAAK,WACT,IAAIhB,EAAOzD,KAEX,MAAQyD,EAAKqjC,kBACZrjC,EAAOA,EAAKqjC,kBAGb,OAAOrjC,IACJ6rB,OAAQtvB,OAGNA,MAGR+mC,UAAW,SAAUnY,GACpB,OAAKvtB,EAAYutB,GACT5uB,KAAKuE,KAAM,SAAUpC,GAC3Bc,GAAQjD,MAAO+mC,UAAWnY,EAAKnuB,KAAMT,KAAMmC,MAItCnC,KAAKuE,KAAM,WACjB,IAAI2U,EAAOjW,GAAQjD,MAClByZ,EAAWP,EAAKO,WAEZA,EAASlW,OACbkW,EAASotB,QAASjY,GAGlB1V,EAAKoW,OAAQV,MAKhBlI,KAAM,SAAUkI,GACf,IAAIoY,EAAiB3lC,EAAYutB,GAEjC,OAAO5uB,KAAKuE,KAAM,SAAUpC,GAC3Bc,GAAQjD,MAAO6mC,QAASG,EAAiBpY,EAAKnuB,KAAMT,KAAMmC,GAAMysB,MAIlEqY,OAAQ,SAAU/jC,GAIjB,OAHAlD,KAAKuS,OAAQrP,GAAW+P,IAAK,QAAS1O,KAAM,WAC3CtB,GAAQjD,MAAO2vB,YAAa3vB,KAAK0M,cAE3B1M,QAKTiD,GAAOqN,KAAK9F,QAAQ6uB,OAAS,SAAU51B,GACtC,OAAQR,GAAOqN,KAAK9F,QAAQ08B,QAASzjC,IAEtCR,GAAOqN,KAAK9F,QAAQ08B,QAAU,SAAUzjC,GACvC,SAAWA,EAAK0uB,aAAe1uB,EAAK6vB,cAAgB7vB,EAAK4xB,iBAAiB9xB,SAM3EN,GAAOm/B,aAAa+E,IAAM,WACzB,IACC,OAAO,IAAIpnC,GAAOqnC,eACjB,MAAQz6B,MAGX,IAAI06B,GAAmB,CAGrBC,EAAG,IAIHC,KAAM,KAEPC,GAAevkC,GAAOm/B,aAAa+E,MAEpC/lC,GAAQqmC,OAASD,IAAkB,oBAAqBA,GACxDpmC,GAAQsiC,KAAO8D,KAAiBA,GAEhCvkC,GAAOwgC,cAAe,SAAUj+B,GAC/B,IAAIhB,EAAUkjC,EAGd,GAAKtmC,GAAQqmC,MAAQD,KAAiBhiC,EAAQ2/B,YAC7C,MAAO,CACNO,KAAM,SAAUH,EAAS1K,GACxB,IAAI14B,EACHglC,EAAM3hC,EAAQ2hC,MAWf,GATAA,EAAIQ,KACHniC,EAAQ7D,KACR6D,EAAQg9B,IACRh9B,EAAQo9B,MACRp9B,EAAQoiC,SACRpiC,EAAQyP,UAIJzP,EAAQqiC,UACZ,IAAM1lC,KAAKqD,EAAQqiC,UAClBV,EAAKhlC,GAAMqD,EAAQqiC,UAAW1lC,GAmBhC,IAAMA,KAdDqD,EAAQs/B,UAAYqC,EAAItC,kBAC5BsC,EAAItC,iBAAkBr/B,EAAQs/B,UAQzBt/B,EAAQ2/B,aAAgBI,EAAS,sBACtCA,EAAS,oBAAuB,kBAItBA,EACV4B,EAAIvC,iBAAkBziC,EAAGojC,EAASpjC,IAInCqC,EAAW,SAAU7C,GACpB,OAAO,WACD6C,IACJA,EAAWkjC,EAAgBP,EAAIW,OAC9BX,EAAIY,QAAUZ,EAAIa,QAAUb,EAAIc,UAC/Bd,EAAIe,mBAAqB,KAEb,UAATvmC,EACJwlC,EAAInC,QACgB,UAATrjC,EAKgB,iBAAfwlC,EAAIpC,OACflK,EAAU,EAAG,SAEbA,EAGCsM,EAAIpC,OACJoC,EAAIlC,YAINpK,EACCwM,GAAkBF,EAAIpC,SAAYoC,EAAIpC,OACtCoC,EAAIlC,WAK+B,UAAjCkC,EAAIgB,cAAgB,SACM,iBAArBhB,EAAIiB,aACV,CAAEC,OAAQlB,EAAIrB,UACd,CAAEvjC,KAAM4kC,EAAIiB,cACbjB,EAAIxC,4BAQTwC,EAAIW,OAAStjC,IACbkjC,EAAgBP,EAAIY,QAAUZ,EAAIc,UAAYzjC,EAAU,cAKnCyB,IAAhBkhC,EAAIa,QACRb,EAAIa,QAAUN,EAEdP,EAAIe,mBAAqB,WAGA,IAAnBf,EAAIpnB,YAMRhgB,GAAO2e,WAAY,WACbla,GACJkjC,OAQLljC,EAAWA,EAAU,SAErB,IAGC2iC,EAAIzB,KAAMlgC,EAAQ6/B,YAAc7/B,EAAQ8b,MAAQ,MAC/C,MAAQ3U,GAGT,GAAKnI,EACJ,MAAMmI,IAKTq4B,MAAO,WACDxgC,GACJA,QAWLvB,GAAOugC,cAAe,SAAUhD,GAC1BA,EAAE2E,cACN3E,EAAE/mB,SAASpX,QAAS,KAKtBY,GAAOqgC,UAAW,CACjBR,QAAS,CACRzgC,OAAQ,6FAGToX,SAAU,CACTpX,OAAQ,2BAET4gC,WAAY,CACX2D,cAAe,SAAUrkC,GAExB,OADAU,GAAO4D,WAAYtE,GACZA,MAMVU,GAAOugC,cAAe,SAAU,SAAUhD,QACxBv6B,IAAZu6B,EAAEpyB,QACNoyB,EAAEpyB,OAAQ,GAENoyB,EAAE2E,cACN3E,EAAE7+B,KAAO,SAKXsB,GAAOwgC,cAAe,SAAU,SAAUjD,GAIxC,IAAIn+B,EAAQmC,EADb,GAAKg8B,EAAE2E,aAAe3E,EAAE8H,YAEvB,MAAO,CACN5C,KAAM,SAAU/pB,EAAGkf,GAClBx4B,EAASY,GAAQ,YACfwN,KAAM+vB,EAAE8H,aAAe,IACvB/mB,KAAM,CAAEgnB,QAAS/H,EAAEgI,cAAe5mC,IAAK4+B,EAAEgC,MACzCrb,GAAI,aAAc3iB,EAAW,SAAUikC,GACvCpmC,EAAOka,SACP/X,EAAW,KACNikC,GACJ5N,EAAuB,UAAb4N,EAAI9mC,KAAmB,IAAM,IAAK8mC,EAAI9mC,QAKnD/B,EAAS8C,KAAKC,YAAaN,EAAQ,KAEpC2iC,MAAO,WACDxgC,GACJA,QAUL,IAqGKigB,GArGDikB,GAAe,GAClBC,GAAS,oBAGV1lC,GAAOqgC,UAAW,CACjBsF,MAAO,WACPC,cAAe,WACd,IAAIrkC,EAAWkkC,GAAargC,OAAWpF,GAAOiD,QAAU,IAAQrE,GAAMmG,OAEtE,OADAhI,KAAMwE,IAAa,EACZA,KAKTvB,GAAOugC,cAAe,aAAc,SAAUhD,EAAGsI,EAAkBlH,GAElE,IAAImH,EAAcC,EAAaC,EAC9BC,GAAuB,IAAZ1I,EAAEoI,QAAqBD,GAAOlhC,KAAM+4B,EAAEgC,KAChD,MACkB,iBAAXhC,EAAElf,MAE6C,KADnDkf,EAAEqC,aAAe,IACjBhiC,QAAS,sCACX8nC,GAAOlhC,KAAM+4B,EAAElf,OAAU,QAI5B,GAAK4nB,GAAiC,UAArB1I,EAAEkB,UAAW,GA8D7B,OA3DAqH,EAAevI,EAAEqI,cAAgBxnC,EAAYm/B,EAAEqI,eAC9CrI,EAAEqI,gBACFrI,EAAEqI,cAGEK,EACJ1I,EAAG0I,GAAa1I,EAAG0I,GAAW7iC,QAASsiC,GAAQ,KAAOI,IAC/B,IAAZvI,EAAEoI,QACbpI,EAAEgC,MAASrD,GAAO13B,KAAM+4B,EAAEgC,KAAQ,IAAM,KAAQhC,EAAEoI,MAAQ,IAAMG,GAIjEvI,EAAEyC,WAAY,eAAkB,WAI/B,OAHMgG,GACLhmC,GAAOsD,MAAOwiC,EAAe,mBAEvBE,EAAmB,IAI3BzI,EAAEkB,UAAW,GAAM,OAGnBsH,EAAcjpC,GAAQgpC,GACtBhpC,GAAQgpC,GAAiB,WACxBE,EAAoBvkC,WAIrBk9B,EAAM7kB,OAAQ,gBAGQ9W,IAAhB+iC,EACJ/lC,GAAQlD,IAASm+B,WAAY6K,GAI7BhpC,GAAQgpC,GAAiBC,EAIrBxI,EAAGuI,KAGPvI,EAAEqI,cAAgBC,EAAiBD,cAGnCH,GAAa9nC,KAAMmoC,IAIfE,GAAqB5nC,EAAY2nC,IACrCA,EAAaC,EAAmB,IAGjCA,EAAoBD,OAAc/iC,IAI5B,WAYT7E,GAAQ+nC,qBACH1kB,GAAO7kB,EAASwpC,eAAeD,mBAAoB,IAAK1kB,MACvDtU,UAAY,6BACiB,IAA3BsU,GAAK/X,WAAWnJ,QAQxBN,GAAOmW,UAAY,SAAUkI,EAAMne,EAASkmC,GAC3C,MAAqB,iBAAT/nB,EACJ,IAEgB,kBAAZne,IACXkmC,EAAclmC,EACdA,GAAU,GAKLA,IAIA/B,GAAQ+nC,qBAMZxzB,GALAxS,EAAUvD,EAASwpC,eAAeD,mBAAoB,KAKvC7mC,cAAe,SACzB0R,KAAOpU,EAAS6T,SAASO,KAC9B7Q,EAAQT,KAAKC,YAAagT,IAE1BxS,EAAUvD,GAKZ2mB,GAAW8iB,GAAe,IAD1BC,EAASvwB,EAAW1L,KAAMiU,IAKlB,CAAEne,EAAQb,cAAegnC,EAAQ,MAGzCA,EAAShjB,GAAe,CAAEhF,GAAQne,EAASojB,GAEtCA,GAAWA,EAAQhjB,QACvBN,GAAQsjB,GAAUhK,SAGZtZ,GAAOoB,MAAO,GAAIilC,EAAO58B,cAlChC,IAAIiJ,EAAM2zB,EAAQ/iB,GAyCnBtjB,GAAOG,GAAGonB,KAAO,SAAUgY,EAAK+G,EAAQ/kC,GACvC,IAAItB,EAAUvB,EAAMmkC,EACnB5sB,EAAOlZ,KACPwnB,EAAMgb,EAAI3hC,QAAS,KAsDpB,OApDY,EAAP2mB,IACJtkB,EAAW66B,GAAkByE,EAAIliC,MAAOknB,IACxCgb,EAAMA,EAAIliC,MAAO,EAAGknB,IAIhBnmB,EAAYkoC,IAGhB/kC,EAAW+kC,EACXA,OAAStjC,GAGEsjC,GAA4B,iBAAXA,IAC5B5nC,EAAO,QAIW,EAAduX,EAAK3V,QACTN,GAAOygC,KAAM,CACZlB,IAAKA,EAKL7gC,KAAMA,GAAQ,MACd8/B,SAAU,OACVngB,KAAMioB,IACHz/B,KAAM,SAAUs+B,GAGnBtC,EAAWphC,UAEXwU,EAAK0V,KAAM1rB,EAIVD,GAAQ,SAAUqsB,OAAQrsB,GAAOmW,UAAWgvB,IAAiBv7B,KAAM3J,GAGnEklC,KAKErrB,OAAQvY,GAAY,SAAUo9B,EAAOmD,GACxC7rB,EAAK3U,KAAM,WACVC,EAAS7D,MAAOX,KAAM8lC,GAAY,CAAElE,EAAMwG,aAAcrD,EAAQnD,QAK5D5hC,MAMRiD,GAAOqN,KAAK9F,QAAQg/B,SAAW,SAAU/lC,GACxC,OAAOR,GAAO8B,KAAM9B,GAAOo5B,OAAQ,SAAUj5B,GAC5C,OAAOK,IAASL,EAAGK,OAChBF,QAMLN,GAAOwmC,OAAS,CACfC,UAAW,SAAUjmC,EAAM+B,EAASrD,GACnC,IAAIwnC,EAAaC,EAASC,EAAWC,EAAQC,EAAWC,EACvD/X,EAAWhvB,GAAOwgB,IAAKhgB,EAAM,YAC7BwmC,EAAUhnC,GAAQQ,GAClBonB,EAAQ,GAGS,WAAboH,IACJxuB,EAAK8f,MAAM0O,SAAW,YAGvB8X,EAAYE,EAAQR,SACpBI,EAAY5mC,GAAOwgB,IAAKhgB,EAAM,OAC9BumC,EAAa/mC,GAAOwgB,IAAKhgB,EAAM,SACI,aAAbwuB,GAAwC,UAAbA,KACA,GAA9C4X,EAAYG,GAAanpC,QAAS,SAMpCipC,GADAH,EAAcM,EAAQhY,YACD3iB,IACrBs6B,EAAUD,EAAYpS,OAGtBuS,EAASxX,WAAYuX,IAAe,EACpCD,EAAUtX,WAAY0X,IAAgB,GAGlC3oC,EAAYmE,KAGhBA,EAAUA,EAAQ/E,KAAMgD,EAAMtB,EAAGc,GAAOsC,OAAQ,GAAIwkC,KAGjC,MAAfvkC,EAAQ8J,MACZub,EAAMvb,IAAQ9J,EAAQ8J,IAAMy6B,EAAUz6B,IAAQw6B,GAE1B,MAAhBtkC,EAAQ+xB,OACZ1M,EAAM0M,KAAS/xB,EAAQ+xB,KAAOwS,EAAUxS,KAASqS,GAG7C,UAAWpkC,EACfA,EAAQ0kC,MAAMzpC,KAAMgD,EAAMonB,GAG1Bof,EAAQxmB,IAAKoH,KAKhB5nB,GAAOG,GAAGmC,OAAQ,CAGjBkkC,OAAQ,SAAUjkC,GAGjB,GAAKd,UAAUnB,OACd,YAAmB0C,IAAZT,EACNxF,KACAA,KAAKuE,KAAM,SAAUpC,GACpBc,GAAOwmC,OAAOC,UAAW1pC,KAAMwF,EAASrD,KAI3C,IAAIgoC,EAAMC,EACT3mC,EAAOzD,KAAM,GAEd,OAAMyD,EAQAA,EAAK4xB,iBAAiB9xB,QAK5B4mC,EAAO1mC,EAAK4zB,wBACZ+S,EAAM3mC,EAAK+D,cAAc6H,YAClB,CACNC,IAAK66B,EAAK76B,IAAM86B,EAAIC,YACpB9S,KAAM4S,EAAK5S,KAAO6S,EAAIE,cARf,CAAEh7B,IAAK,EAAGioB,KAAM,QATxB,GAuBDtF,SAAU,WACT,GAAMjyB,KAAM,GAAZ,CAIA,IAAIuqC,EAAcd,EAAQvnC,EACzBuB,EAAOzD,KAAM,GACbwqC,EAAe,CAAEl7B,IAAK,EAAGioB,KAAM,GAGhC,GAAwC,UAAnCt0B,GAAOwgB,IAAKhgB,EAAM,YAGtBgmC,EAAShmC,EAAK4zB,4BAER,CACNoS,EAASzpC,KAAKypC,SAIdvnC,EAAMuB,EAAK+D,cACX+iC,EAAe9mC,EAAK8mC,cAAgBroC,EAAI6E,gBACxC,MAAQwjC,IACLA,IAAiBroC,EAAIuiB,MAAQ8lB,IAAiBroC,EAAI6E,kBACT,WAA3C9D,GAAOwgB,IAAK8mB,EAAc,YAE1BA,EAAeA,EAAa3nC,WAExB2nC,GAAgBA,IAAiB9mC,GAAkC,IAA1B8mC,EAAahpC,YAG1DipC,EAAevnC,GAAQsnC,GAAed,UACzBn6B,KAAOrM,GAAOwgB,IAAK8mB,EAAc,kBAAkB,GAChEC,EAAajT,MAAQt0B,GAAOwgB,IAAK8mB,EAAc,mBAAmB,IAKpE,MAAO,CACNj7B,IAAKm6B,EAAOn6B,IAAMk7B,EAAal7B,IAAMrM,GAAOwgB,IAAKhgB,EAAM,aAAa,GACpE8zB,KAAMkS,EAAOlS,KAAOiT,EAAajT,KAAOt0B,GAAOwgB,IAAKhgB,EAAM,cAAc,MAc1E8mC,aAAc,WACb,OAAOvqC,KAAKyE,IAAK,WAChB,IAAI8lC,EAAevqC,KAAKuqC,aAExB,MAAQA,GAA2D,WAA3CtnC,GAAOwgB,IAAK8mB,EAAc,YACjDA,EAAeA,EAAaA,aAG7B,OAAOA,GAAgBxjC,OAM1B9D,GAAOsB,KAAM,CAAEk0B,WAAY,cAAeD,UAAW,eAAiB,SAAUnd,EAAQkG,GACvF,IAAIjS,EAAM,gBAAkBiS,EAE5Bte,GAAOG,GAAIiY,GAAW,SAAUjZ,GAC/B,OAAO6d,EAAQjgB,KAAM,SAAUyD,EAAM4X,EAAQjZ,GAG5C,IAAIgoC,EAOJ,GANK3oC,EAAUgC,GACd2mC,EAAM3mC,EACuB,IAAlBA,EAAKlC,WAChB6oC,EAAM3mC,EAAK4L,kBAGCpJ,IAAR7D,EACJ,OAAOgoC,EAAMA,EAAK7oB,GAAS9d,EAAM4X,GAG7B+uB,EACJA,EAAIK,SACFn7B,EAAY86B,EAAIE,YAAVloC,EACPkN,EAAMlN,EAAMgoC,EAAIC,aAIjB5mC,EAAM4X,GAAWjZ,GAEhBiZ,EAAQjZ,EAAKsC,UAAUnB,WAU5BN,GAAOsB,KAAM,CAAE,MAAO,QAAU,SAAU6D,EAAImZ,GAC7Cte,GAAOuyB,SAAUjU,GAAS4P,GAAc/vB,GAAQuxB,cAC/C,SAAUlvB,EAAMmtB,GACf,GAAKA,EAIJ,OAHAA,EAAWD,GAAQltB,EAAM8d,GAGlB4O,GAAU1oB,KAAMmpB,GACtB3tB,GAAQQ,GAAOwuB,WAAY1Q,GAAS,KACpCqP,MAQL3tB,GAAOsB,KAAM,CAAEmmC,OAAQ,SAAUC,MAAO,SAAW,SAAUjnC,EAAM/B,GAClEsB,GAAOsB,KAAM,CACZkzB,QAAS,QAAU/zB,EACnBgX,QAAS/Y,EACTipC,GAAI,QAAUlnC,GACZ,SAAUmnC,EAAcC,GAG1B7nC,GAAOG,GAAI0nC,GAAa,SAAUtT,EAAQzvB,GACzC,IAAImY,EAAYxb,UAAUnB,SAAYsnC,GAAkC,kBAAXrT,GAC5D1C,EAAQ+V,KAA6B,IAAXrT,IAA6B,IAAVzvB,EAAiB,SAAW,UAE1E,OAAOkY,EAAQjgB,KAAM,SAAUyD,EAAM9B,EAAMoG,GAC1C,IAAI7F,EAEJ,OAAKT,EAAUgC,GAGyB,IAAhCqnC,EAASjqC,QAAS,SACxB4C,EAAM,QAAUC,GAChBD,EAAK7D,SAASmH,gBAAiB,SAAWrD,GAIrB,IAAlBD,EAAKlC,UACTW,EAAMuB,EAAKsD,gBAIJZ,KAAKouB,IACX9wB,EAAKghB,KAAM,SAAW/gB,GAAQxB,EAAK,SAAWwB,GAC9CD,EAAKghB,KAAM,SAAW/gB,GAAQxB,EAAK,SAAWwB,GAC9CxB,EAAK,SAAWwB,UAIDuC,IAAV8B,EAGN9E,GAAOwgB,IAAKhgB,EAAM9B,EAAMmzB,GAGxB7xB,GAAOsgB,MAAO9f,EAAM9B,EAAMoG,EAAO+sB,IAChCnzB,EAAMue,EAAYsX,OAASvxB,EAAWia,QAM5Cjd,GAAOsB,KAAM,CACZ,YACA,WACA,eACA,YACA,cACA,YACE,SAAU6D,EAAIzG,GAChBsB,GAAOG,GAAIzB,GAAS,SAAUyB,GAC7B,OAAOpD,KAAKmnB,GAAIxlB,EAAMyB,MAOxBH,GAAOG,GAAGmC,OAAQ,CAEjBq1B,KAAM,SAAUxT,EAAO9F,EAAMle,GAC5B,OAAOpD,KAAKmnB,GAAIC,EAAO,KAAM9F,EAAMle,IAEpC2nC,OAAQ,SAAU3jB,EAAOhkB,GACxB,OAAOpD,KAAKwnB,IAAKJ,EAAO,KAAMhkB,IAG/B4nC,SAAU,SAAU9nC,EAAUkkB,EAAO9F,EAAMle,GAC1C,OAAOpD,KAAKmnB,GAAIC,EAAOlkB,EAAUoe,EAAMle,IAExC6nC,WAAY,SAAU/nC,EAAUkkB,EAAOhkB,GAGtC,OAA4B,IAArBsB,UAAUnB,OAChBvD,KAAKwnB,IAAKtkB,EAAU,MACpBlD,KAAKwnB,IAAKJ,EAAOlkB,GAAY,KAAME,IAGrC8nC,MAAO,SAAUC,EAAQC,GACxB,OAAOprC,KACLmnB,GAAI,aAAcgkB,GAClBhkB,GAAI,aAAcikB,GAASD,MAI/BloC,GAAOsB,KACN,wLAE4D4D,MAAO,KACnE,SAAUC,EAAI1E,GAGbT,GAAOG,GAAIM,GAAS,SAAU4d,EAAMle,GACnC,OAA0B,EAAnBsB,UAAUnB,OAChBvD,KAAKmnB,GAAIzjB,EAAM,KAAM4d,EAAMle,GAC3BpD,KAAKioB,QAASvkB,MAYlB,IAAI2nC,GAAQ,sDAMZpoC,GAAOqoC,MAAQ,SAAUloC,EAAID,GAC5B,IAAIyf,EAAK/P,EAAMy4B,EAUf,GARwB,iBAAZnoC,IACXyf,EAAMxf,EAAID,GACVA,EAAUC,EACVA,EAAKwf,GAKAvhB,EAAY+B,GAalB,OARAyP,EAAOvS,GAAMG,KAAMiE,UAAW,IAC9B4mC,EAAQ,WACP,OAAOloC,EAAGzC,MAAOwC,GAAWnD,KAAM6S,EAAKnS,OAAQJ,GAAMG,KAAMiE,eAItDsD,KAAO5E,EAAG4E,KAAO5E,EAAG4E,MAAQ/E,GAAO+E,OAElCsjC,GAGRroC,GAAOsoC,UAAY,SAAUC,GACvBA,EACJvoC,GAAO4c,YAEP5c,GAAOoW,OAAO,IAGhBpW,GAAO+C,QAAUD,MAAMC,QACvB/C,GAAOwoC,UAAY3pB,KAAKC,MACxB9e,GAAOO,SAAWA,GAClBP,GAAO5B,WAAaA,EACpB4B,GAAOxB,SAAWA,EAClBwB,GAAO4d,UAAYA,EACnB5d,GAAOtB,KAAOmB,EAEdG,GAAOkoB,IAAMD,KAAKC,IAElBloB,GAAOyoC,UAAY,SAAUpqC,GAK5B,IAAIK,EAAOsB,GAAOtB,KAAML,GACxB,OAAkB,WAATK,GAA8B,WAATA,KAK5BgqC,MAAOrqC,EAAMgxB,WAAYhxB,KAG5B2B,GAAO2oC,KAAO,SAAUrpC,GACvB,OAAe,MAARA,EACN,IACEA,EAAO,IAAK8D,QAASglC,GAAO,OAkBT,mBAAXQ,QAAyBA,OAAOC,KAC3CD,OAAQ,SAAU,GAAI,WACrB,OAAO5oC,KAOT,IAGC8oC,GAAUhsC,GAAOkD,OAGjB+oC,GAAKjsC,GAAOksC,EAwBb,OAtBAhpC,GAAOipC,WAAa,SAAUrmC,GAS7B,OARK9F,GAAOksC,IAAMhpC,KACjBlD,GAAOksC,EAAID,IAGPnmC,GAAQ9F,GAAOkD,SAAWA,KAC9BlD,GAAOkD,OAAS8oC,IAGV9oC,IAMiB,oBAAbhD,IACXF,GAAOkD,OAASlD,GAAOksC,EAAIhpC,IAMrBA","file":"jquery-3.7.1.min.js"}
\ No newline at end of file
diff --git a/bitbake/lib/toaster/toastergui/static/js/jquery.dataTables-1.13.8.min.js b/bitbake/lib/toaster/toastergui/static/js/jquery.dataTables-1.13.8.min.js
new file mode 100644
index 0000000000..b6d9aa8c79
--- /dev/null
+++ b/bitbake/lib/toaster/toastergui/static/js/jquery.dataTables-1.13.8.min.js
@@ -0,0 +1,4 @@
+/*! DataTables 1.13.8
+ * ©2008-2023 SpryMedia Ltd - datatables.net/license
+ */
+!function(n){"use strict";var a;"function"==typeof define&&define.amd?define(["jquery"],function(t){return n(t,window,document)}):"object"==typeof exports?(a=require("jquery"),"undefined"==typeof window?module.exports=function(t,e){return t=t||window,e=e||a(t),n(e,t,t.document)}:module.exports=n(a,window,window.document)):window.DataTable=n(jQuery,window,document)}(function(P,j,v,H){"use strict";function d(t){var e=parseInt(t,10);return!isNaN(e)&&isFinite(t)?e:null}function l(t,e,n){var a=typeof t,r="string"==a;return"number"==a||"bigint"==a||!!h(t)||(e&&r&&(t=$(t,e)),n&&r&&(t=t.replace(q,"")),!isNaN(parseFloat(t))&&isFinite(t))}function a(t,e,n){var a;return!!h(t)||(h(a=t)||"string"==typeof a)&&!!l(t.replace(V,"").replace(/<script/i,""),e,n)||null}function m(t,e,n,a){var r=[],o=0,i=e.length;if(a!==H)for(;o<i;o++)t[e[o]][n]&&r.push(t[e[o]][n][a]);else for(;o<i;o++)r.push(t[e[o]][n]);return r}function f(t,e){var n,a=[];e===H?(e=0,n=t):(n=e,e=t);for(var r=e;r<n;r++)a.push(r);return a}function _(t){for(var e=[],n=0,a=t.length;n<a;n++)t[n]&&e.push(t[n]);return e}function s(t,e){return-1!==this.indexOf(t,e=e===H?0:e)}var p,e,t,w=function(t,v){if(w.factory(t,v))return w;if(this instanceof w)return P(t).DataTable(v);v=t,this.$=function(t,e){return this.api(!0).$(t,e)},this._=function(t,e){return this.api(!0).rows(t,e).data()},this.api=function(t){return new B(t?ge(this[p.iApiIndex]):this)},this.fnAddData=function(t,e){var n=this.api(!0),t=(Array.isArray(t)&&(Array.isArray(t[0])||P.isPlainObject(t[0]))?n.rows:n.row).add(t);return e!==H&&!e||n.draw(),t.flatten().toArray()},this.fnAdjustColumnSizing=function(t){var e=this.api(!0).columns.adjust(),n=e.settings()[0],a=n.oScroll;t===H||t?e.draw(!1):""===a.sX&&""===a.sY||Qt(n)},this.fnClearTable=function(t){var e=this.api(!0).clear();t!==H&&!t||e.draw()},this.fnClose=function(t){this.api(!0).row(t).child.hide()},this.fnDeleteRow=function(t,e,n){var a=this.api(!0),t=a.rows(t),r=t.settings()[0],o=r.aoData[t[0][0]];return t.remove(),e&&e.call(this,r,o),n!==H&&!n||a.draw(),o},this.fnDestroy=function(t){this.api(!0).destroy(t)},this.fnDraw=function(t){this.api(!0).draw(t)},this.fnFilter=function(t,e,n,a,r,o){var i=this.api(!0);(null===e||e===H?i:i.column(e)).search(t,n,a,o),i.draw()},this.fnGetData=function(t,e){var n,a=this.api(!0);return t!==H?(n=t.nodeName?t.nodeName.toLowerCase():"",e!==H||"td"==n||"th"==n?a.cell(t,e).data():a.row(t).data()||null):a.data().toArray()},this.fnGetNodes=function(t){var e=this.api(!0);return t!==H?e.row(t).node():e.rows().nodes().flatten().toArray()},this.fnGetPosition=function(t){var e=this.api(!0),n=t.nodeName.toUpperCase();return"TR"==n?e.row(t).index():"TD"==n||"TH"==n?[(n=e.cell(t).index()).row,n.columnVisible,n.column]:null},this.fnIsOpen=function(t){return this.api(!0).row(t).child.isShown()},this.fnOpen=function(t,e,n){return this.api(!0).row(t).child(e,n).show().child()[0]},this.fnPageChange=function(t,e){t=this.api(!0).page(t);e!==H&&!e||t.draw(!1)},this.fnSetColumnVis=function(t,e,n){t=this.api(!0).column(t).visible(e);n!==H&&!n||t.columns.adjust().draw()},this.fnSettings=function(){return ge(this[p.iApiIndex])},this.fnSort=function(t){this.api(!0).order(t).draw()},this.fnSortListener=function(t,e,n){this.api(!0).order.listener(t,e,n)},this.fnUpdate=function(t,e,n,a,r){var o=this.api(!0);return(n===H||null===n?o.row(e):o.cell(e,n)).data(t),r!==H&&!r||o.columns.adjust(),a!==H&&!a||o.draw(),0},this.fnVersionCheck=p.fnVersionCheck;var e,y=this,D=v===H,_=this.length;for(e in D&&(v={}),this.oApi=this.internal=p.internal,w.ext.internal)e&&(this[e]=$e(e));return this.each(function(){var r=1<_?be({},v,!0):v,o=0,t=this.getAttribute("id"),i=!1,e=w.defaults,l=P(this);if("table"!=this.nodeName.toLowerCase())W(null,0,"Non-table node initialisation ("+this.nodeName+")",2);else{K(e),Q(e.column),C(e,e,!0),C(e.column,e.column,!0),C(e,P.extend(r,l.data()),!0);for(var n=w.settings,o=0,s=n.length;o<s;o++){var a=n[o];if(a.nTable==this||a.nTHead&&a.nTHead.parentNode==this||a.nTFoot&&a.nTFoot.parentNode==this){var u=(r.bRetrieve!==H?r:e).bRetrieve,c=(r.bDestroy!==H?r:e).bDestroy;if(D||u)return a.oInstance;if(c){a.oInstance.fnDestroy();break}return void W(a,0,"Cannot reinitialise DataTable",3)}if(a.sTableId==this.id){n.splice(o,1);break}}null!==t&&""!==t||(t="DataTables_Table_"+w.ext._unique++,this.id=t);var f,d,h=P.extend(!0,{},w.models.oSettings,{sDestroyWidth:l[0].style.width,sInstance:t,sTableId:t}),p=(h.nTable=this,h.oApi=y.internal,h.oInit=r,n.push(h),h.oInstance=1===y.length?y:l.dataTable(),K(r),Z(r.oLanguage),r.aLengthMenu&&!r.iDisplayLength&&(r.iDisplayLength=(Array.isArray(r.aLengthMenu[0])?r.aLengthMenu[0]:r.aLengthMenu)[0]),r=be(P.extend(!0,{},e),r),F(h.oFeatures,r,["bPaginate","bLengthChange","bFilter","bSort","bSortMulti","bInfo","bProcessing","bAutoWidth","bSortClasses","bServerSide","bDeferRender"]),F(h,r,["asStripeClasses","ajax","fnServerData","fnFormatNumber","sServerMethod","aaSorting","aaSortingFixed","aLengthMenu","sPaginationType","sAjaxSource","sAjaxDataProp","iStateDuration","sDom","bSortCellsTop","iTabIndex","fnStateLoadCallback","fnStateSaveCallback","renderer","searchDelay","rowId",["iCookieDuration","iStateDuration"],["oSearch","oPreviousSearch"],["aoSearchCols","aoPreSearchCols"],["iDisplayLength","_iDisplayLength"]]),F(h.oScroll,r,[["sScrollX","sX"],["sScrollXInner","sXInner"],["sScrollY","sY"],["bScrollCollapse","bCollapse"]]),F(h.oLanguage,r,"fnInfoCallback"),L(h,"aoDrawCallback",r.fnDrawCallback,"user"),L(h,"aoServerParams",r.fnServerParams,"user"),L(h,"aoStateSaveParams",r.fnStateSaveParams,"user"),L(h,"aoStateLoadParams",r.fnStateLoadParams,"user"),L(h,"aoStateLoaded",r.fnStateLoaded,"user"),L(h,"aoRowCallback",r.fnRowCallback,"user"),L(h,"aoRowCreatedCallback",r.fnCreatedRow,"user"),L(h,"aoHeaderCallback",r.fnHeaderCallback,"user"),L(h,"aoFooterCallback",r.fnFooterCallback,"user"),L(h,"aoInitComplete",r.fnInitComplete,"user"),L(h,"aoPreDrawCallback",r.fnPreDrawCallback,"user"),h.rowIdFn=A(r.rowId),tt(h),h.oClasses),g=(P.extend(p,w.ext.classes,r.oClasses),l.addClass(p.sTable),h.iInitDisplayStart===H&&(h.iInitDisplayStart=r.iDisplayStart,h._iDisplayStart=r.iDisplayStart),null!==r.iDeferLoading&&(h.bDeferLoading=!0,t=Array.isArray(r.iDeferLoading),h._iRecordsDisplay=t?r.iDeferLoading[0]:r.iDeferLoading,h._iRecordsTotal=t?r.iDeferLoading[1]:r.iDeferLoading),h.oLanguage),t=(P.extend(!0,g,r.oLanguage),g.sUrl?(P.ajax({dataType:"json",url:g.sUrl,success:function(t){C(e.oLanguage,t),Z(t),P.extend(!0,g,t,h.oInit.oLanguage),R(h,null,"i18n",[h]),Jt(h)},error:function(){Jt(h)}}),i=!0):R(h,null,"i18n",[h]),null===r.asStripeClasses&&(h.asStripeClasses=[p.sStripeOdd,p.sStripeEven]),h.asStripeClasses),b=l.children("tbody").find("tr").eq(0),m=(-1!==P.inArray(!0,P.map(t,function(t,e){return b.hasClass(t)}))&&(P("tbody tr",this).removeClass(t.join(" ")),h.asDestroyStripes=t.slice()),[]),t=this.getElementsByTagName("thead");if(0!==t.length&&(wt(h.aoHeader,t[0]),m=Ct(h)),null===r.aoColumns)for(f=[],o=0,s=m.length;o<s;o++)f.push(null);else f=r.aoColumns;for(o=0,s=f.length;o<s;o++)nt(h,m?m[o]:null);st(h,r.aoColumnDefs,f,function(t,e){at(h,t,e)}),b.length&&(d=function(t,e){return null!==t.getAttribute("data-"+e)?e:null},P(b[0]).children("th, td").each(function(t,e){var n,a=h.aoColumns[t];a||W(h,0,"Incorrect column count",18),a.mData===t&&(n=d(e,"sort")||d(e,"order"),e=d(e,"filter")||d(e,"search"),null===n&&null===e||(a.mData={_:t+".display",sort:null!==n?t+".@data-"+n:H,type:null!==n?t+".@data-"+n:H,filter:null!==e?t+".@data-"+e:H},a._isArrayHost=!0,at(h,t)))}));var S=h.oFeatures,t=function(){if(r.aaSorting===H){var t=h.aaSorting;for(o=0,s=t.length;o<s;o++)t[o][1]=h.aoColumns[o].asSorting[0]}ce(h),S.bSort&&L(h,"aoDrawCallback",function(){var t,n;h.bSorted&&(t=I(h),n={},P.each(t,function(t,e){n[e.src]=e.dir}),R(h,null,"order",[h,t,n]),le(h))}),L(h,"aoDrawCallback",function(){(h.bSorted||"ssp"===E(h)||S.bDeferRender)&&ce(h)},"sc");var e=l.children("caption").each(function(){this._captionSide=P(this).css("caption-side")}),n=l.children("thead"),a=(0===n.length&&(n=P("<thead/>").appendTo(l)),h.nTHead=n[0],l.children("tbody")),n=(0===a.length&&(a=P("<tbody/>").insertAfter(n)),h.nTBody=a[0],l.children("tfoot"));if(0===(n=0===n.length&&0<e.length&&(""!==h.oScroll.sX||""!==h.oScroll.sY)?P("<tfoot/>").appendTo(l):n).length||0===n.children().length?l.addClass(p.sNoFooter):0<n.length&&(h.nTFoot=n[0],wt(h.aoFooter,h.nTFoot)),r.aaData)for(o=0;o<r.aaData.length;o++)x(h,r.aaData[o]);else!h.bDeferLoading&&"dom"!=E(h)||ut(h,P(h.nTBody).children("tr"));h.aiDisplay=h.aiDisplayMaster.slice(),!(h.bInitialised=!0)===i&&Jt(h)};L(h,"aoDrawCallback",de,"state_save"),r.bStateSave?(S.bStateSave=!0,he(h,0,t)):t()}}),y=null,this},c={},U=/[\r\n\u2028]/g,V=/<.*?>/g,X=/^\d{2,4}[\.\/\-]\d{1,2}[\.\/\-]\d{1,2}([T ]{1}\d{1,2}[:\.]\d{2}([\.:]\d{2})?)?$/,J=new RegExp("(\\"+["/",".","*","+","?","|","(",")","[","]","{","}","\\","$","^","-"].join("|\\")+")","g"),q=/['\u00A0,$£€¥%\u2009\u202F\u20BD\u20a9\u20BArfkɃΞ]/gi,h=function(t){return!t||!0===t||"-"===t},$=function(t,e){return c[e]||(c[e]=new RegExp(Ot(e),"g")),"string"==typeof t&&"."!==e?t.replace(/\./g,"").replace(c[e],"."):t},N=function(t,e,n){var a=[],r=0,o=t.length;if(n!==H)for(;r<o;r++)t[r]&&t[r][e]&&a.push(t[r][e][n]);else for(;r<o;r++)t[r]&&a.push(t[r][e]);return a},G=function(t){if(!(t.length<2))for(var e=t.slice().sort(),n=e[0],a=1,r=e.length;a<r;a++){if(e[a]===n)return!1;n=e[a]}return!0},z=function(t){if(G(t))return t.slice();var e,n,a,r=[],o=t.length,i=0;t:for(n=0;n<o;n++){for(e=t[n],a=0;a<i;a++)if(r[a]===e)continue t;r.push(e),i++}return r},Y=function(t,e){if(Array.isArray(e))for(var n=0;n<e.length;n++)Y(t,e[n]);else t.push(e);return t};function i(n){var a,r,o={};P.each(n,function(t,e){(a=t.match(/^([^A-Z]+?)([A-Z])/))&&-1!=="a aa ai ao as b fn i m o s ".indexOf(a[1]+" ")&&(r=t.replace(a[0],a[2].toLowerCase()),o[r]=t,"o"===a[1])&&i(n[t])}),n._hungarianMap=o}function C(n,a,r){var o;n._hungarianMap||i(n),P.each(a,function(t,e){(o=n._hungarianMap[t])===H||!r&&a[o]!==H||("o"===o.charAt(0)?(a[o]||(a[o]={}),P.extend(!0,a[o],a[t]),C(n[o],a[o],r)):a[o]=a[t])})}function Z(t){var e,n=w.defaults.oLanguage,a=n.sDecimal;a&&Me(a),t&&(e=t.sZeroRecords,!t.sEmptyTable&&e&&"No data available in table"===n.sEmptyTable&&F(t,t,"sZeroRecords","sEmptyTable"),!t.sLoadingRecords&&e&&"Loading..."===n.sLoadingRecords&&F(t,t,"sZeroRecords","sLoadingRecords"),t.sInfoThousands&&(t.sThousands=t.sInfoThousands),e=t.sDecimal)&&a!==e&&Me(e)}Array.isArray||(Array.isArray=function(t){return"[object Array]"===Object.prototype.toString.call(t)}),Array.prototype.includes||(Array.prototype.includes=s),String.prototype.trim||(String.prototype.trim=function(){return this.replace(/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,"")}),String.prototype.includes||(String.prototype.includes=s),w.util={throttle:function(a,t){var r,o,i=t!==H?t:200;return function(){var t=this,e=+new Date,n=arguments;r&&e<r+i?(clearTimeout(o),o=setTimeout(function(){r=H,a.apply(t,n)},i)):(r=e,a.apply(t,n))}},escapeRegex:function(t){return t.replace(J,"\\$1")},set:function(a){var d;return P.isPlainObject(a)?w.util.set(a._):null===a?function(){}:"function"==typeof a?function(t,e,n){a(t,"set",e,n)}:"string"!=typeof a||-1===a.indexOf(".")&&-1===a.indexOf("[")&&-1===a.indexOf("(")?function(t,e){t[a]=e}:(d=function(t,e,n){for(var a,r,o,i,l=dt(n),n=l[l.length-1],s=0,u=l.length-1;s<u;s++){if("__proto__"===l[s]||"constructor"===l[s])throw new Error("Cannot set prototype values");if(a=l[s].match(ft),r=l[s].match(g),a){if(l[s]=l[s].replace(ft,""),t[l[s]]=[],(a=l.slice()).splice(0,s+1),i=a.join("."),Array.isArray(e))for(var c=0,f=e.length;c<f;c++)d(o={},e[c],i),t[l[s]].push(o);else t[l[s]]=e;return}r&&(l[s]=l[s].replace(g,""),t=t[l[s]](e)),null!==t[l[s]]&&t[l[s]]!==H||(t[l[s]]={}),t=t[l[s]]}n.match(g)?t[n.replace(g,"")](e):t[n.replace(ft,"")]=e},function(t,e){return d(t,e,a)})},get:function(r){var o,d;return P.isPlainObject(r)?(o={},P.each(r,function(t,e){e&&(o[t]=w.util.get(e))}),function(t,e,n,a){var r=o[e]||o._;return r!==H?r(t,e,n,a):t}):null===r?function(t){return t}:"function"==typeof r?function(t,e,n,a){return r(t,e,n,a)}:"string"!=typeof r||-1===r.indexOf(".")&&-1===r.indexOf("[")&&-1===r.indexOf("(")?function(t,e){return t[r]}:(d=function(t,e,n){var a,r,o;if(""!==n)for(var i=dt(n),l=0,s=i.length;l<s;l++){if(f=i[l].match(ft),a=i[l].match(g),f){if(i[l]=i[l].replace(ft,""),""!==i[l]&&(t=t[i[l]]),r=[],i.splice(0,l+1),o=i.join("."),Array.isArray(t))for(var u=0,c=t.length;u<c;u++)r.push(d(t[u],e,o));var f=f[0].substring(1,f[0].length-1);t=""===f?r:r.join(f);break}if(a)i[l]=i[l].replace(g,""),t=t[i[l]]();else{if(null===t||null===t[i[l]])return null;if(t===H||t[i[l]]===H)return H;t=t[i[l]]}}return t},function(t,e){return d(t,e,r)})}};var r=function(t,e,n){t[e]!==H&&(t[n]=t[e])};function K(t){r(t,"ordering","bSort"),r(t,"orderMulti","bSortMulti"),r(t,"orderClasses","bSortClasses"),r(t,"orderCellsTop","bSortCellsTop"),r(t,"order","aaSorting"),r(t,"orderFixed","aaSortingFixed"),r(t,"paging","bPaginate"),r(t,"pagingType","sPaginationType"),r(t,"pageLength","iDisplayLength"),r(t,"searching","bFilter"),"boolean"==typeof t.sScrollX&&(t.sScrollX=t.sScrollX?"100%":""),"boolean"==typeof t.scrollX&&(t.scrollX=t.scrollX?"100%":"");var e=t.aoSearchCols;if(e)for(var n=0,a=e.length;n<a;n++)e[n]&&C(w.models.oSearch,e[n])}function Q(t){r(t,"orderable","bSortable"),r(t,"orderData","aDataSort"),r(t,"orderSequence","asSorting"),r(t,"orderDataType","sortDataType");var e=t.aDataSort;"number"!=typeof e||Array.isArray(e)||(t.aDataSort=[e])}function tt(t){var e,n,a,r;w.__browser||(w.__browser=e={},r=(a=(n=P("<div/>").css({position:"fixed",top:0,left:-1*P(j).scrollLeft(),height:1,width:1,overflow:"hidden"}).append(P("<div/>").css({position:"absolute",top:1,left:1,width:100,overflow:"scroll"}).append(P("<div/>").css({width:"100%",height:10}))).appendTo("body")).children()).children(),e.barWidth=a[0].offsetWidth-a[0].clientWidth,e.bScrollOversize=100===r[0].offsetWidth&&100!==a[0].clientWidth,e.bScrollbarLeft=1!==Math.round(r.offset().left),e.bBounding=!!n[0].getBoundingClientRect().width,n.remove()),P.extend(t.oBrowser,w.__browser),t.oScroll.iBarWidth=w.__browser.barWidth}function et(t,e,n,a,r,o){var i,l=a,s=!1;for(n!==H&&(i=n,s=!0);l!==r;)t.hasOwnProperty(l)&&(i=s?e(i,t[l],l,t):t[l],s=!0,l+=o);return i}function nt(t,e){var n=w.defaults.column,a=t.aoColumns.length,n=P.extend({},w.models.oColumn,n,{nTh:e||v.createElement("th"),sTitle:n.sTitle||(e?e.innerHTML:""),aDataSort:n.aDataSort||[a],mData:n.mData||a,idx:a}),n=(t.aoColumns.push(n),t.aoPreSearchCols);n[a]=P.extend({},w.models.oSearch,n[a]),at(t,a,P(e).data())}function at(t,e,n){function a(t){return"string"==typeof t&&-1!==t.indexOf("@")}var e=t.aoColumns[e],r=t.oClasses,o=P(e.nTh),i=(!e.sWidthOrig&&(e.sWidthOrig=o.attr("width")||null,u=(o.attr("style")||"").match(/width:\s*(\d+[pxem%]+)/))&&(e.sWidthOrig=u[1]),n!==H&&null!==n&&(Q(n),C(w.defaults.column,n,!0),n.mDataProp===H||n.mData||(n.mData=n.mDataProp),n.sType&&(e._sManualType=n.sType),n.className&&!n.sClass&&(n.sClass=n.className),n.sClass&&o.addClass(n.sClass),u=e.sClass,P.extend(e,n),F(e,n,"sWidth","sWidthOrig"),u!==e.sClass&&(e.sClass=u+" "+e.sClass),n.iDataSort!==H&&(e.aDataSort=[n.iDataSort]),F(e,n,"aDataSort"),e.ariaTitle||(e.ariaTitle=o.attr("aria-label"))),e.mData),l=A(i),s=e.mRender?A(e.mRender):null,u=(e._bAttrSrc=P.isPlainObject(i)&&(a(i.sort)||a(i.type)||a(i.filter)),e._setter=null,e.fnGetData=function(t,e,n){var a=l(t,e,H,n);return s&&e?s(a,e,t,n):a},e.fnSetData=function(t,e,n){return b(i)(t,e,n)},"number"==typeof i||e._isArrayHost||(t._rowReadObject=!0),t.oFeatures.bSort||(e.bSortable=!1,o.addClass(r.sSortableNone)),-1!==P.inArray("asc",e.asSorting)),n=-1!==P.inArray("desc",e.asSorting);e.bSortable&&(u||n)?u&&!n?(e.sSortingClass=r.sSortableAsc,e.sSortingClassJUI=r.sSortJUIAscAllowed):!u&&n?(e.sSortingClass=r.sSortableDesc,e.sSortingClassJUI=r.sSortJUIDescAllowed):(e.sSortingClass=r.sSortable,e.sSortingClassJUI=r.sSortJUI):(e.sSortingClass=r.sSortableNone,e.sSortingClassJUI="")}function O(t){if(!1!==t.oFeatures.bAutoWidth){var e=t.aoColumns;ee(t);for(var n=0,a=e.length;n<a;n++)e[n].nTh.style.width=e[n].sWidth}var r=t.oScroll;""===r.sY&&""===r.sX||Qt(t),R(t,null,"column-sizing",[t])}function rt(t,e){t=it(t,"bVisible");return"number"==typeof t[e]?t[e]:null}function ot(t,e){t=it(t,"bVisible"),e=P.inArray(e,t);return-1!==e?e:null}function T(t){var n=0;return P.each(t.aoColumns,function(t,e){e.bVisible&&"none"!==P(e.nTh).css("display")&&n++}),n}function it(t,n){var a=[];return P.map(t.aoColumns,function(t,e){t[n]&&a.push(e)}),a}function lt(t){for(var e,n,a,r,o,i,l,s=t.aoColumns,u=t.aoData,c=w.ext.type.detect,f=0,d=s.length;f<d;f++)if(l=[],!(o=s[f]).sType&&o._sManualType)o.sType=o._sManualType;else if(!o.sType){for(e=0,n=c.length;e<n;e++){for(a=0,r=u.length;a<r&&(l[a]===H&&(l[a]=S(t,a,f,"type")),(i=c[e](l[a],t))||e===c.length-1)&&("html"!==i||h(l[a]));a++);if(i){o.sType=i;break}}o.sType||(o.sType="string")}}function st(t,e,n,a){var r,o,i,l,s=t.aoColumns;if(e)for(r=e.length-1;0<=r;r--)for(var u,c=(u=e[r]).target!==H?u.target:u.targets!==H?u.targets:u.aTargets,f=0,d=(c=Array.isArray(c)?c:[c]).length;f<d;f++)if("number"==typeof c[f]&&0<=c[f]){for(;s.length<=c[f];)nt(t);a(c[f],u)}else if("number"==typeof c[f]&&c[f]<0)a(s.length+c[f],u);else if("string"==typeof c[f])for(i=0,l=s.length;i<l;i++)"_all"!=c[f]&&!P(s[i].nTh).hasClass(c[f])||a(i,u);if(n)for(r=0,o=n.length;r<o;r++)a(r,n[r])}function x(t,e,n,a){for(var r=t.aoData.length,o=P.extend(!0,{},w.models.oRow,{src:n?"dom":"data",idx:r}),i=(o._aData=e,t.aoData.push(o),t.aoColumns),l=0,s=i.length;l<s;l++)i[l].sType=null;t.aiDisplayMaster.push(r);e=t.rowIdFn(e);return e!==H&&(t.aIds[e]=o),!n&&t.oFeatures.bDeferRender||St(t,r,n,a),r}function ut(n,t){var a;return(t=t instanceof P?t:P(t)).map(function(t,e){return a=mt(n,e),x(n,a.data,e,a.cells)})}function S(t,e,n,a){"search"===a?a="filter":"order"===a&&(a="sort");var r=t.iDraw,o=t.aoColumns[n],i=t.aoData[e]._aData,l=o.sDefaultContent,s=o.fnGetData(i,a,{settings:t,row:e,col:n});if(s===H)return t.iDrawError!=r&&null===l&&(W(t,0,"Requested unknown parameter "+("function"==typeof o.mData?"{function}":"'"+o.mData+"'")+" for row "+e+", column "+n,4),t.iDrawError=r),l;if(s!==i&&null!==s||null===l||a===H){if("function"==typeof s)return s.call(i)}else s=l;return null===s&&"display"===a?"":"filter"===a&&(e=w.ext.type.search)[o.sType]?e[o.sType](s):s}function ct(t,e,n,a){var r=t.aoColumns[n],o=t.aoData[e]._aData;r.fnSetData(o,a,{settings:t,row:e,col:n})}var ft=/\[.*?\]$/,g=/\(\)$/;function dt(t){return P.map(t.match(/(\\.|[^\.])+/g)||[""],function(t){return t.replace(/\\\./g,".")})}var A=w.util.get,b=w.util.set;function ht(t){return N(t.aoData,"_aData")}function pt(t){t.aoData.length=0,t.aiDisplayMaster.length=0,t.aiDisplay.length=0,t.aIds={}}function gt(t,e,n){for(var a=-1,r=0,o=t.length;r<o;r++)t[r]==e?a=r:t[r]>e&&t[r]--;-1!=a&&n===H&&t.splice(a,1)}function bt(n,a,t,e){function r(t,e){for(;t.childNodes.length;)t.removeChild(t.firstChild);t.innerHTML=S(n,a,e,"display")}var o,i,l=n.aoData[a];if("dom"!==t&&(t&&"auto"!==t||"dom"!==l.src)){var s=l.anCells;if(s)if(e!==H)r(s[e],e);else for(o=0,i=s.length;o<i;o++)r(s[o],o)}else l._aData=mt(n,l,e,e===H?H:l._aData).data;l._aSortData=null,l._aFilterData=null;var u=n.aoColumns;if(e!==H)u[e].sType=null;else{for(o=0,i=u.length;o<i;o++)u[o].sType=null;vt(n,l)}}function mt(t,e,n,a){function r(t,e){var n;"string"==typeof t&&-1!==(n=t.indexOf("@"))&&(n=t.substring(n+1),b(t)(a,e.getAttribute(n)))}function o(t){n!==H&&n!==f||(l=d[f],s=t.innerHTML.trim(),l&&l._bAttrSrc?(b(l.mData._)(a,s),r(l.mData.sort,t),r(l.mData.type,t),r(l.mData.filter,t)):h?(l._setter||(l._setter=b(l.mData)),l._setter(a,s)):a[f]=s),f++}var i,l,s,u=[],c=e.firstChild,f=0,d=t.aoColumns,h=t._rowReadObject;a=a!==H?a:h?{}:[];if(c)for(;c;)"TD"!=(i=c.nodeName.toUpperCase())&&"TH"!=i||(o(c),u.push(c)),c=c.nextSibling;else for(var p=0,g=(u=e.anCells).length;p<g;p++)o(u[p]);var e=e.firstChild?e:e.nTr;return e&&(e=e.getAttribute("id"))&&b(t.rowId)(a,e),{data:a,cells:u}}function St(t,e,n,a){var r,o,i,l,s,u,c=t.aoData[e],f=c._aData,d=[];if(null===c.nTr){for(r=n||v.createElement("tr"),c.nTr=r,c.anCells=d,r._DT_RowIndex=e,vt(t,c),l=0,s=t.aoColumns.length;l<s;l++)i=t.aoColumns[l],(o=(u=!n)?v.createElement(i.sCellType):a[l])||W(t,0,"Incorrect column count",18),o._DT_CellIndex={row:e,column:l},d.push(o),!u&&(!i.mRender&&i.mData===l||P.isPlainObject(i.mData)&&i.mData._===l+".display")||(o.innerHTML=S(t,e,l,"display")),i.sClass&&(o.className+=" "+i.sClass),i.bVisible&&!n?r.appendChild(o):!i.bVisible&&n&&o.parentNode.removeChild(o),i.fnCreatedCell&&i.fnCreatedCell.call(t.oInstance,o,S(t,e,l),f,e,l);R(t,"aoRowCreatedCallback",null,[r,f,e,d])}}function vt(t,e){var n=e.nTr,a=e._aData;n&&((t=t.rowIdFn(a))&&(n.id=t),a.DT_RowClass&&(t=a.DT_RowClass.split(" "),e.__rowc=e.__rowc?z(e.__rowc.concat(t)):t,P(n).removeClass(e.__rowc.join(" ")).addClass(a.DT_RowClass)),a.DT_RowAttr&&P(n).attr(a.DT_RowAttr),a.DT_RowData)&&P(n).data(a.DT_RowData)}function yt(t){var e,n,a,r=t.nTHead,o=t.nTFoot,i=0===P("th, td",r).length,l=t.oClasses,s=t.aoColumns;for(i&&(n=P("<tr/>").appendTo(r)),c=0,f=s.length;c<f;c++)a=s[c],e=P(a.nTh).addClass(a.sClass),i&&e.appendTo(n),t.oFeatures.bSort&&(e.addClass(a.sSortingClass),!1!==a.bSortable)&&(e.attr("tabindex",t.iTabIndex).attr("aria-controls",t.sTableId),ue(t,a.nTh,c)),a.sTitle!=e[0].innerHTML&&e.html(a.sTitle),ve(t,"header")(t,e,a,l);if(i&&wt(t.aoHeader,r),P(r).children("tr").children("th, td").addClass(l.sHeaderTH),P(o).children("tr").children("th, td").addClass(l.sFooterTH),null!==o)for(var u=t.aoFooter[0],c=0,f=u.length;c<f;c++)(a=s[c])?(a.nTf=u[c].cell,a.sClass&&P(a.nTf).addClass(a.sClass)):W(t,0,"Incorrect column count",18)}function Dt(t,e,n){var a,r,o,i,l,s,u,c,f,d=[],h=[],p=t.aoColumns.length;if(e){for(n===H&&(n=!1),a=0,r=e.length;a<r;a++){for(d[a]=e[a].slice(),d[a].nTr=e[a].nTr,o=p-1;0<=o;o--)t.aoColumns[o].bVisible||n||d[a].splice(o,1);h.push([])}for(a=0,r=d.length;a<r;a++){if(u=d[a].nTr)for(;s=u.firstChild;)u.removeChild(s);for(o=0,i=d[a].length;o<i;o++)if(f=c=1,h[a][o]===H){for(u.appendChild(d[a][o].cell),h[a][o]=1;d[a+c]!==H&&d[a][o].cell==d[a+c][o].cell;)h[a+c][o]=1,c++;for(;d[a][o+f]!==H&&d[a][o].cell==d[a][o+f].cell;){for(l=0;l<c;l++)h[a+l][o+f]=1;f++}P(d[a][o].cell).attr("rowspan",c).attr("colspan",f)}}}}function y(t,e){n="ssp"==E(s=t),(l=s.iInitDisplayStart)!==H&&-1!==l&&(s._iDisplayStart=!n&&l>=s.fnRecordsDisplay()?0:l,s.iInitDisplayStart=-1);var n=R(t,"aoPreDrawCallback","preDraw",[t]);if(-1!==P.inArray(!1,n))D(t,!1);else{var a=[],r=0,o=t.asStripeClasses,i=o.length,l=t.oLanguage,s="ssp"==E(t),u=t.aiDisplay,n=t._iDisplayStart,c=t.fnDisplayEnd();if(t.bDrawing=!0,t.bDeferLoading)t.bDeferLoading=!1,t.iDraw++,D(t,!1);else if(s){if(!t.bDestroying&&!e)return void xt(t)}else t.iDraw++;if(0!==u.length)for(var f=s?t.aoData.length:c,d=s?0:n;d<f;d++){var h,p=u[d],g=t.aoData[p],b=(null===g.nTr&&St(t,p),g.nTr);0!==i&&(h=o[r%i],g._sRowStripe!=h)&&(P(b).removeClass(g._sRowStripe).addClass(h),g._sRowStripe=h),R(t,"aoRowCallback",null,[b,g._aData,r,d,p]),a.push(b),r++}else{e=l.sZeroRecords;1==t.iDraw&&"ajax"==E(t)?e=l.sLoadingRecords:l.sEmptyTable&&0===t.fnRecordsTotal()&&(e=l.sEmptyTable),a[0]=P("<tr/>",{class:i?o[0]:""}).append(P("<td />",{valign:"top",colSpan:T(t),class:t.oClasses.sRowEmpty}).html(e))[0]}R(t,"aoHeaderCallback","header",[P(t.nTHead).children("tr")[0],ht(t),n,c,u]),R(t,"aoFooterCallback","footer",[P(t.nTFoot).children("tr")[0],ht(t),n,c,u]);s=P(t.nTBody);s.children().detach(),s.append(P(a)),R(t,"aoDrawCallback","draw",[t]),t.bSorted=!1,t.bFiltered=!1,t.bDrawing=!1}}function u(t,e){var n=t.oFeatures,a=n.bSort,n=n.bFilter;a&&ie(t),n?Rt(t,t.oPreviousSearch):t.aiDisplay=t.aiDisplayMaster.slice(),!0!==e&&(t._iDisplayStart=0),t._drawHold=e,y(t),t._drawHold=!1}function _t(t){for(var e,n,a,r,o,i,l,s=t.oClasses,u=P(t.nTable),u=P("<div/>").insertBefore(u),c=t.oFeatures,f=P("<div/>",{id:t.sTableId+"_wrapper",class:s.sWrapper+(t.nTFoot?"":" "+s.sNoFooter)}),d=(t.nHolding=u[0],t.nTableWrapper=f[0],t.nTableReinsertBefore=t.nTable.nextSibling,t.sDom.split("")),h=0;h<d.length;h++){if(e=null,"<"==(n=d[h])){if(a=P("<div/>")[0],"'"==(r=d[h+1])||'"'==r){for(o="",i=2;d[h+i]!=r;)o+=d[h+i],i++;"H"==o?o=s.sJUIHeader:"F"==o&&(o=s.sJUIFooter),-1!=o.indexOf(".")?(l=o.split("."),a.id=l[0].substr(1,l[0].length-1),a.className=l[1]):"#"==o.charAt(0)?a.id=o.substr(1,o.length-1):a.className=o,h+=i}f.append(a),f=P(a)}else if(">"==n)f=f.parent();else if("l"==n&&c.bPaginate&&c.bLengthChange)e=Gt(t);else if("f"==n&&c.bFilter)e=Lt(t);else if("r"==n&&c.bProcessing)e=Zt(t);else if("t"==n)e=Kt(t);else if("i"==n&&c.bInfo)e=Ut(t);else if("p"==n&&c.bPaginate)e=zt(t);else if(0!==w.ext.feature.length)for(var p=w.ext.feature,g=0,b=p.length;g<b;g++)if(n==p[g].cFeature){e=p[g].fnInit(t);break}e&&((l=t.aanFeatures)[n]||(l[n]=[]),l[n].push(e),f.append(e))}u.replaceWith(f),t.nHolding=null}function wt(t,e){var n,a,r,o,i,l,s,u,c,f,d=P(e).children("tr");for(t.splice(0,t.length),r=0,l=d.length;r<l;r++)t.push([]);for(r=0,l=d.length;r<l;r++)for(a=(n=d[r]).firstChild;a;){if("TD"==a.nodeName.toUpperCase()||"TH"==a.nodeName.toUpperCase())for(u=(u=+a.getAttribute("colspan"))&&0!=u&&1!=u?u:1,c=(c=+a.getAttribute("rowspan"))&&0!=c&&1!=c?c:1,s=function(t,e,n){for(var a=t[e];a[n];)n++;return n}(t,r,0),f=1==u,i=0;i<u;i++)for(o=0;o<c;o++)t[r+o][s+i]={cell:a,unique:f},t[r+o].nTr=n;a=a.nextSibling}}function Ct(t,e,n){var a=[];n||(n=t.aoHeader,e&&wt(n=[],e));for(var r=0,o=n.length;r<o;r++)for(var i=0,l=n[r].length;i<l;i++)!n[r][i].unique||a[i]&&t.bSortCellsTop||(a[i]=n[r][i].cell);return a}function Tt(r,t,n){function e(t){var e=r.jqXHR?r.jqXHR.status:null;(null===t||"number"==typeof e&&204==e)&&Ft(r,t={},[]),(e=t.error||t.sError)&&W(r,0,e),r.json=t,R(r,null,"xhr",[r,t,r.jqXHR]),n(t)}R(r,"aoServerParams","serverParams",[t]),t&&Array.isArray(t)&&(a={},o=/(.*?)\[\]$/,P.each(t,function(t,e){var n=e.name.match(o);n?(n=n[0],a[n]||(a[n]=[]),a[n].push(e.value)):a[e.name]=e.value}),t=a);var a,o,i,l=r.ajax,s=r.oInstance,u=(P.isPlainObject(l)&&l.data&&(u="function"==typeof(i=l.data)?i(t,r):i,t="function"==typeof i&&u?u:P.extend(!0,t,u),delete l.data),{data:t,success:e,dataType:"json",cache:!1,type:r.sServerMethod,error:function(t,e,n){var a=R(r,null,"xhr",[r,null,r.jqXHR]);-1===P.inArray(!0,a)&&("parsererror"==e?W(r,0,"Invalid JSON response",1):4===t.readyState&&W(r,0,"Ajax error",7)),D(r,!1)}});r.oAjaxData=t,R(r,null,"preXhr",[r,t]),r.fnServerData?r.fnServerData.call(s,r.sAjaxSource,P.map(t,function(t,e){return{name:e,value:t}}),e,r):r.sAjaxSource||"string"==typeof l?r.jqXHR=P.ajax(P.extend(u,{url:l||r.sAjaxSource})):"function"==typeof l?r.jqXHR=l.call(s,t,e,r):(r.jqXHR=P.ajax(P.extend(u,l)),l.data=i)}function xt(e){e.iDraw++,D(e,!0);var n=e._drawHold;Tt(e,At(e),function(t){e._drawHold=n,It(e,t),e._drawHold=!1})}function At(t){for(var e,n,a,r=t.aoColumns,o=r.length,i=t.oFeatures,l=t.oPreviousSearch,s=t.aoPreSearchCols,u=[],c=I(t),f=t._iDisplayStart,d=!1!==i.bPaginate?t._iDisplayLength:-1,h=function(t,e){u.push({name:t,value:e})},p=(h("sEcho",t.iDraw),h("iColumns",o),h("sColumns",N(r,"sName").join(",")),h("iDisplayStart",f),h("iDisplayLength",d),{draw:t.iDraw,columns:[],order:[],start:f,length:d,search:{value:l.sSearch,regex:l.bRegex}}),g=0;g<o;g++)n=r[g],a=s[g],e="function"==typeof n.mData?"function":n.mData,p.columns.push({data:e,name:n.sName,searchable:n.bSearchable,orderable:n.bSortable,search:{value:a.sSearch,regex:a.bRegex}}),h("mDataProp_"+g,e),i.bFilter&&(h("sSearch_"+g,a.sSearch),h("bRegex_"+g,a.bRegex),h("bSearchable_"+g,n.bSearchable)),i.bSort&&h("bSortable_"+g,n.bSortable);i.bFilter&&(h("sSearch",l.sSearch),h("bRegex",l.bRegex)),i.bSort&&(P.each(c,function(t,e){p.order.push({column:e.col,dir:e.dir}),h("iSortCol_"+t,e.col),h("sSortDir_"+t,e.dir)}),h("iSortingCols",c.length));f=w.ext.legacy.ajax;return null===f?t.sAjaxSource?u:p:f?u:p}function It(t,n){function e(t,e){return n[t]!==H?n[t]:n[e]}var a=Ft(t,n),r=e("sEcho","draw"),o=e("iTotalRecords","recordsTotal"),i=e("iTotalDisplayRecords","recordsFiltered");if(r!==H){if(+r<t.iDraw)return;t.iDraw=+r}a=a||[],pt(t),t._iRecordsTotal=parseInt(o,10),t._iRecordsDisplay=parseInt(i,10);for(var l=0,s=a.length;l<s;l++)x(t,a[l]);t.aiDisplay=t.aiDisplayMaster.slice(),y(t,!0),t._bInitComplete||qt(t,n),D(t,!1)}function Ft(t,e,n){t=P.isPlainObject(t.ajax)&&t.ajax.dataSrc!==H?t.ajax.dataSrc:t.sAjaxDataProp;if(!n)return"data"===t?e.aaData||e[t]:""!==t?A(t)(e):e;b(t)(e,n)}function Lt(n){function e(t){i.f;var e=this.value||"";o.return&&"Enter"!==t.key||e!=o.sSearch&&(Rt(n,{sSearch:e,bRegex:o.bRegex,bSmart:o.bSmart,bCaseInsensitive:o.bCaseInsensitive,return:o.return}),n._iDisplayStart=0,y(n))}var t=n.oClasses,a=n.sTableId,r=n.oLanguage,o=n.oPreviousSearch,i=n.aanFeatures,l='<input type="search" class="'+t.sFilterInput+'"/>',s=(s=r.sSearch).match(/_INPUT_/)?s.replace("_INPUT_",l):s+l,l=P("<div/>",{id:i.f?null:a+"_filter",class:t.sFilter}).append(P("<label/>").append(s)),t=null!==n.searchDelay?n.searchDelay:"ssp"===E(n)?400:0,u=P("input",l).val(o.sSearch).attr("placeholder",r.sSearchPlaceholder).on("keyup.DT search.DT input.DT paste.DT cut.DT",t?ne(e,t):e).on("mouseup.DT",function(t){setTimeout(function(){e.call(u[0],t)},10)}).on("keypress.DT",function(t){if(13==t.keyCode)return!1}).attr("aria-controls",a);return P(n.nTable).on("search.dt.DT",function(t,e){if(n===e)try{u[0]!==v.activeElement&&u.val(o.sSearch)}catch(t){}}),l[0]}function Rt(t,e,n){function a(t){o.sSearch=t.sSearch,o.bRegex=t.bRegex,o.bSmart=t.bSmart,o.bCaseInsensitive=t.bCaseInsensitive,o.return=t.return}function r(t){return t.bEscapeRegex!==H?!t.bEscapeRegex:t.bRegex}var o=t.oPreviousSearch,i=t.aoPreSearchCols;if(lt(t),"ssp"!=E(t)){Ht(t,e.sSearch,n,r(e),e.bSmart,e.bCaseInsensitive),a(e);for(var l=0;l<i.length;l++)jt(t,i[l].sSearch,l,r(i[l]),i[l].bSmart,i[l].bCaseInsensitive);Pt(t)}else a(e);t.bFiltered=!0,R(t,null,"search",[t])}function Pt(t){for(var e,n,a=w.ext.search,r=t.aiDisplay,o=0,i=a.length;o<i;o++){for(var l=[],s=0,u=r.length;s<u;s++)n=r[s],e=t.aoData[n],a[o](t,e._aFilterData,n,e._aData,s)&&l.push(n);r.length=0,P.merge(r,l)}}function jt(t,e,n,a,r,o){if(""!==e){for(var i,l=[],s=t.aiDisplay,u=Nt(e,a,r,o),c=0;c<s.length;c++)i=t.aoData[s[c]]._aFilterData[n],u.test(i)&&l.push(s[c]);t.aiDisplay=l}}function Ht(t,e,n,a,r,o){var i,l,s,u=Nt(e,a,r,o),r=t.oPreviousSearch.sSearch,o=t.aiDisplayMaster,c=[];if(0!==w.ext.search.length&&(n=!0),l=Wt(t),e.length<=0)t.aiDisplay=o.slice();else{for((l||n||a||r.length>e.length||0!==e.indexOf(r)||t.bSorted)&&(t.aiDisplay=o.slice()),i=t.aiDisplay,s=0;s<i.length;s++)u.test(t.aoData[i[s]]._sFilterRow)&&c.push(i[s]);t.aiDisplay=c}}function Nt(t,e,n,a){return t=e?t:Ot(t),n&&(t="^(?=.*?"+P.map(t.match(/["\u201C][^"\u201D]+["\u201D]|[^ ]+/g)||[""],function(t){var e;return'"'===t.charAt(0)?t=(e=t.match(/^"(.*)"$/))?e[1]:t:"“"===t.charAt(0)&&(t=(e=t.match(/^\u201C(.*)\u201D$/))?e[1]:t),t.replace('"',"")}).join(")(?=.*?")+").*$"),new RegExp(t,a?"i":"")}var Ot=w.util.escapeRegex,kt=P("<div>")[0],Mt=kt.textContent!==H;function Wt(t){for(var e,n,a,r,o,i=t.aoColumns,l=!1,s=0,u=t.aoData.length;s<u;s++)if(!(o=t.aoData[s])._aFilterData){for(a=[],e=0,n=i.length;e<n;e++)i[e].bSearchable?"string"!=typeof(r=null===(r=S(t,s,e,"filter"))?"":r)&&r.toString&&(r=r.toString()):r="",r.indexOf&&-1!==r.indexOf("&")&&(kt.innerHTML=r,r=Mt?kt.textContent:kt.innerText),r.replace&&(r=r.replace(/[\r\n\u2028]/g,"")),a.push(r);o._aFilterData=a,o._sFilterRow=a.join("  "),l=!0}return l}function Et(t){return{search:t.sSearch,smart:t.bSmart,regex:t.bRegex,caseInsensitive:t.bCaseInsensitive}}function Bt(t){return{sSearch:t.search,bSmart:t.smart,bRegex:t.regex,bCaseInsensitive:t.caseInsensitive}}function Ut(t){var e=t.sTableId,n=t.aanFeatures.i,a=P("<div/>",{class:t.oClasses.sInfo,id:n?null:e+"_info"});return n||(t.aoDrawCallback.push({fn:Vt,sName:"information"}),a.attr("role","status").attr("aria-live","polite"),P(t.nTable).attr("aria-describedby",e+"_info")),a[0]}function Vt(t){var e,n,a,r,o,i,l=t.aanFeatures.i;0!==l.length&&(i=t.oLanguage,e=t._iDisplayStart+1,n=t.fnDisplayEnd(),a=t.fnRecordsTotal(),o=(r=t.fnRecordsDisplay())?i.sInfo:i.sInfoEmpty,r!==a&&(o+=" "+i.sInfoFiltered),o=Xt(t,o+=i.sInfoPostFix),null!==(i=i.fnInfoCallback)&&(o=i.call(t.oInstance,t,e,n,a,r,o)),P(l).html(o))}function Xt(t,e){var n=t.fnFormatNumber,a=t._iDisplayStart+1,r=t._iDisplayLength,o=t.fnRecordsDisplay(),i=-1===r;return e.replace(/_START_/g,n.call(t,a)).replace(/_END_/g,n.call(t,t.fnDisplayEnd())).replace(/_MAX_/g,n.call(t,t.fnRecordsTotal())).replace(/_TOTAL_/g,n.call(t,o)).replace(/_PAGE_/g,n.call(t,i?1:Math.ceil(a/r))).replace(/_PAGES_/g,n.call(t,i?1:Math.ceil(o/r)))}function Jt(n){var a,t,e,r=n.iInitDisplayStart,o=n.aoColumns,i=n.oFeatures,l=n.bDeferLoading;if(n.bInitialised){for(_t(n),yt(n),Dt(n,n.aoHeader),Dt(n,n.aoFooter),D(n,!0),i.bAutoWidth&&ee(n),a=0,t=o.length;a<t;a++)(e=o[a]).sWidth&&(e.nTh.style.width=M(e.sWidth));R(n,null,"preInit",[n]),u(n);i=E(n);"ssp"==i&&!l||("ajax"==i?Tt(n,[],function(t){var e=Ft(n,t);for(a=0;a<e.length;a++)x(n,e[a]);n.iInitDisplayStart=r,u(n),D(n,!1),qt(n,t)}):(D(n,!1),qt(n)))}else setTimeout(function(){Jt(n)},200)}function qt(t,e){t._bInitComplete=!0,(e||t.oInit.aaData)&&O(t),R(t,null,"plugin-init",[t,e]),R(t,"aoInitComplete","init",[t,e])}function $t(t,e){e=parseInt(e,10);t._iDisplayLength=e,Se(t),R(t,null,"length",[t,e])}function Gt(a){for(var t=a.oClasses,e=a.sTableId,n=a.aLengthMenu,r=Array.isArray(n[0]),o=r?n[0]:n,i=r?n[1]:n,l=P("<select/>",{name:e+"_length","aria-controls":e,class:t.sLengthSelect}),s=0,u=o.length;s<u;s++)l[0][s]=new Option("number"==typeof i[s]?a.fnFormatNumber(i[s]):i[s],o[s]);var c=P("<div><label/></div>").addClass(t.sLength);return a.aanFeatures.l||(c[0].id=e+"_length"),c.children().append(a.oLanguage.sLengthMenu.replace("_MENU_",l[0].outerHTML)),P("select",c).val(a._iDisplayLength).on("change.DT",function(t){$t(a,P(this).val()),y(a)}),P(a.nTable).on("length.dt.DT",function(t,e,n){a===e&&P("select",c).val(n)}),c[0]}function zt(t){function c(t){y(t)}var e=t.sPaginationType,f=w.ext.pager[e],d="function"==typeof f,e=P("<div/>").addClass(t.oClasses.sPaging+e)[0],h=t.aanFeatures;return d||f.fnInit(t,e,c),h.p||(e.id=t.sTableId+"_paginate",t.aoDrawCallback.push({fn:function(t){if(d)for(var e=t._iDisplayStart,n=t._iDisplayLength,a=t.fnRecordsDisplay(),r=-1===n,o=r?0:Math.ceil(e/n),i=r?1:Math.ceil(a/n),l=f(o,i),s=0,u=h.p.length;s<u;s++)ve(t,"pageButton")(t,h.p[s],s,l,o,i);else f.fnUpdate(t,c)},sName:"pagination"})),e}function Yt(t,e,n){var a=t._iDisplayStart,r=t._iDisplayLength,o=t.fnRecordsDisplay(),o=(0===o||-1===r?a=0:"number"==typeof e?o<(a=e*r)&&(a=0):"first"==e?a=0:"previous"==e?(a=0<=r?a-r:0)<0&&(a=0):"next"==e?a+r<o&&(a+=r):"last"==e?a=Math.floor((o-1)/r)*r:W(t,0,"Unknown paging action: "+e,5),t._iDisplayStart!==a);return t._iDisplayStart=a,o?(R(t,null,"page",[t]),n&&y(t)):R(t,null,"page-nc",[t]),o}function Zt(t){return P("<div/>",{id:t.aanFeatures.r?null:t.sTableId+"_processing",class:t.oClasses.sProcessing,role:"status"}).html(t.oLanguage.sProcessing).append("<div><div></div><div></div><div></div><div></div></div>").insertBefore(t.nTable)[0]}function D(t,e){t.oFeatures.bProcessing&&P(t.aanFeatures.r).css("display",e?"block":"none"),R(t,null,"processing",[t,e])}function Kt(t){var e,n,a,r,o,i,l,s,u,c,f,d,h=P(t.nTable),p=t.oScroll;return""===p.sX&&""===p.sY?t.nTable:(e=p.sX,n=p.sY,a=t.oClasses,o=(r=h.children("caption")).length?r[0]._captionSide:null,s=P(h[0].cloneNode(!1)),i=P(h[0].cloneNode(!1)),u=function(t){return t?M(t):null},(l=h.children("tfoot")).length||(l=null),s=P(f="<div/>",{class:a.sScrollWrapper}).append(P(f,{class:a.sScrollHead}).css({overflow:"hidden",position:"relative",border:0,width:e?u(e):"100%"}).append(P(f,{class:a.sScrollHeadInner}).css({"box-sizing":"content-box",width:p.sXInner||"100%"}).append(s.removeAttr("id").css("margin-left",0).append("top"===o?r:null).append(h.children("thead"))))).append(P(f,{class:a.sScrollBody}).css({position:"relative",overflow:"auto",width:u(e)}).append(h)),l&&s.append(P(f,{class:a.sScrollFoot}).css({overflow:"hidden",border:0,width:e?u(e):"100%"}).append(P(f,{class:a.sScrollFootInner}).append(i.removeAttr("id").css("margin-left",0).append("bottom"===o?r:null).append(h.children("tfoot"))))),u=s.children(),c=u[0],f=u[1],d=l?u[2]:null,e&&P(f).on("scroll.DT",function(t){var e=this.scrollLeft;c.scrollLeft=e,l&&(d.scrollLeft=e)}),P(f).css("max-height",n),p.bCollapse||P(f).css("height",n),t.nScrollHead=c,t.nScrollBody=f,t.nScrollFoot=d,t.aoDrawCallback.push({fn:Qt,sName:"scrolling"}),s[0])}function Qt(n){function t(t){(t=t.style).paddingTop="0",t.paddingBottom="0",t.borderTopWidth="0",t.borderBottomWidth="0",t.height=0}var e,a,r,o,i,l=n.oScroll,s=l.sX,u=l.sXInner,c=l.sY,l=l.iBarWidth,f=P(n.nScrollHead),d=f[0].style,h=f.children("div"),p=h[0].style,h=h.children("table"),g=n.nScrollBody,b=P(g),m=g.style,S=P(n.nScrollFoot).children("div"),v=S.children("table"),y=P(n.nTHead),D=P(n.nTable),_=D[0],w=_.style,C=n.nTFoot?P(n.nTFoot):null,T=n.oBrowser,x=T.bScrollOversize,A=(N(n.aoColumns,"nTh"),[]),I=[],F=[],L=[],R=g.scrollHeight>g.clientHeight;n.scrollBarVis!==R&&n.scrollBarVis!==H?(n.scrollBarVis=R,O(n)):(n.scrollBarVis=R,D.children("thead, tfoot").remove(),C&&(R=C.clone().prependTo(D),i=C.find("tr"),a=R.find("tr"),R.find("[id]").removeAttr("id")),R=y.clone().prependTo(D),y=y.find("tr"),e=R.find("tr"),R.find("th, td").removeAttr("tabindex"),R.find("[id]").removeAttr("id"),s||(m.width="100%",f[0].style.width="100%"),P.each(Ct(n,R),function(t,e){r=rt(n,t),e.style.width=n.aoColumns[r].sWidth}),C&&k(function(t){t.style.width=""},a),f=D.outerWidth(),""===s?(w.width="100%",x&&(D.find("tbody").height()>g.offsetHeight||"scroll"==b.css("overflow-y"))&&(w.width=M(D.outerWidth()-l)),f=D.outerWidth()):""!==u&&(w.width=M(u),f=D.outerWidth()),k(t,e),k(function(t){var e=j.getComputedStyle?j.getComputedStyle(t).width:M(P(t).width());F.push(t.innerHTML),A.push(e)},e),k(function(t,e){t.style.width=A[e]},y),P(e).css("height",0),C&&(k(t,a),k(function(t){L.push(t.innerHTML),I.push(M(P(t).css("width")))},a),k(function(t,e){t.style.width=I[e]},i),P(a).height(0)),k(function(t,e){t.innerHTML='<div class="dataTables_sizing">'+F[e]+"</div>",t.childNodes[0].style.height="0",t.childNodes[0].style.overflow="hidden",t.style.width=A[e]},e),C&&k(function(t,e){t.innerHTML='<div class="dataTables_sizing">'+L[e]+"</div>",t.childNodes[0].style.height="0",t.childNodes[0].style.overflow="hidden",t.style.width=I[e]},a),Math.round(D.outerWidth())<Math.round(f)?(o=g.scrollHeight>g.offsetHeight||"scroll"==b.css("overflow-y")?f+l:f,x&&(g.scrollHeight>g.offsetHeight||"scroll"==b.css("overflow-y"))&&(w.width=M(o-l)),""!==s&&""===u||W(n,1,"Possible column misalignment",6)):o="100%",m.width=M(o),d.width=M(o),C&&(n.nScrollFoot.style.width=M(o)),c||x&&(m.height=M(_.offsetHeight+l)),R=D.outerWidth(),h[0].style.width=M(R),p.width=M(R),y=D.height()>g.clientHeight||"scroll"==b.css("overflow-y"),p[i="padding"+(T.bScrollbarLeft?"Left":"Right")]=y?l+"px":"0px",C&&(v[0].style.width=M(R),S[0].style.width=M(R),S[0].style[i]=y?l+"px":"0px"),D.children("colgroup").insertBefore(D.children("thead")),b.trigger("scroll"),!n.bSorted&&!n.bFiltered||n._drawHold||(g.scrollTop=0))}function k(t,e,n){for(var a,r,o=0,i=0,l=e.length;i<l;){for(a=e[i].firstChild,r=n?n[i].firstChild:null;a;)1===a.nodeType&&(n?t(a,r,o):t(a,o),o++),a=a.nextSibling,r=n?r.nextSibling:null;i++}}var te=/<.*?>/g;function ee(t){var e,n,a=t.nTable,r=t.aoColumns,o=t.oScroll,i=o.sY,l=o.sX,o=o.sXInner,s=r.length,u=it(t,"bVisible"),c=P("th",t.nTHead),f=a.getAttribute("width"),d=a.parentNode,h=!1,p=t.oBrowser,g=p.bScrollOversize,b=a.style.width,m=(b&&-1!==b.indexOf("%")&&(f=b),ae(N(r,"sWidthOrig"),d));for(_=0;_<u.length;_++)null!==(e=r[u[_]]).sWidth&&(e.sWidth=m[_],h=!0);if(g||!h&&!l&&!i&&s==T(t)&&s==c.length)for(_=0;_<s;_++){var S=rt(t,_);null!==S&&(r[S].sWidth=M(c.eq(_).width()))}else{var b=P(a).clone().css("visibility","hidden").removeAttr("id"),v=(b.find("tbody tr").remove(),P("<tr/>").appendTo(b.find("tbody")));for(b.find("thead, tfoot").remove(),b.append(P(t.nTHead).clone()).append(P(t.nTFoot).clone()),b.find("tfoot th, tfoot td").css("width",""),c=Ct(t,b.find("thead")[0]),_=0;_<u.length;_++)e=r[u[_]],c[_].style.width=null!==e.sWidthOrig&&""!==e.sWidthOrig?M(e.sWidthOrig):"",e.sWidthOrig&&l&&P(c[_]).append(P("<div/>").css({width:e.sWidthOrig,margin:0,padding:0,border:0,height:1}));if(t.aoData.length)for(_=0;_<u.length;_++)e=r[n=u[_]],P(re(t,n)).clone(!1).append(e.sContentPadding).appendTo(v);P("[name]",b).removeAttr("name");for(var y=P("<div/>").css(l||i?{position:"absolute",top:0,left:0,height:1,right:0,overflow:"hidden"}:{}).append(b).appendTo(d),D=(l&&o?b.width(o):l?(b.css("width","auto"),b.removeAttr("width"),b.width()<d.clientWidth&&f&&b.width(d.clientWidth)):i?b.width(d.clientWidth):f&&b.width(f),0),_=0;_<u.length;_++){var w=P(c[_]),C=w.outerWidth()-w.width(),w=p.bBounding?Math.ceil(c[_].getBoundingClientRect().width):w.outerWidth();D+=w,r[u[_]].sWidth=M(w-C)}a.style.width=M(D),y.remove()}f&&(a.style.width=M(f)),!f&&!l||t._reszEvt||(o=function(){P(j).on("resize.DT-"+t.sInstance,ne(function(){O(t)}))},g?setTimeout(o,1e3):o(),t._reszEvt=!0)}var ne=w.util.throttle;function ae(t,e){for(var n=[],a=[],r=0;r<t.length;r++)t[r]?n.push(P("<div/>").css("width",M(t[r])).appendTo(e||v.body)):n.push(null);for(r=0;r<t.length;r++)a.push(n[r]?n[r][0].offsetWidth:null);return P(n).remove(),a}function re(t,e){var n,a=oe(t,e);return a<0?null:(n=t.aoData[a]).nTr?n.anCells[e]:P("<td/>").html(S(t,a,e,"display"))[0]}function oe(t,e){for(var n,a=-1,r=-1,o=0,i=t.aoData.length;o<i;o++)(n=(n=(n=S(t,o,e,"display")+"").replace(te,"")).replace(/&nbsp;/g," ")).length>a&&(a=n.length,r=o);return r}function M(t){return null===t?"0px":"number"==typeof t?t<0?"0px":t+"px":t.match(/\d$/)?t+"px":t}function I(t){function e(t){t.length&&!Array.isArray(t[0])?h.push(t):P.merge(h,t)}var n,a,r,o,i,l,s,u=[],c=t.aoColumns,f=t.aaSortingFixed,d=P.isPlainObject(f),h=[];for(Array.isArray(f)&&e(f),d&&f.pre&&e(f.pre),e(t.aaSorting),d&&f.post&&e(f.post),n=0;n<h.length;n++)for(r=(o=c[s=h[n][a=0]].aDataSort).length;a<r;a++)l=c[i=o[a]].sType||"string",h[n]._idx===H&&(h[n]._idx=P.inArray(h[n][1],c[i].asSorting)),u.push({src:s,col:i,dir:h[n][1],index:h[n]._idx,type:l,formatter:w.ext.type.order[l+"-pre"]});return u}function ie(t){var e,n,a,r,c,f=[],u=w.ext.type.order,d=t.aoData,o=(t.aoColumns,0),i=t.aiDisplayMaster;for(lt(t),e=0,n=(c=I(t)).length;e<n;e++)(r=c[e]).formatter&&o++,fe(t,r.col);if("ssp"!=E(t)&&0!==c.length){for(e=0,a=i.length;e<a;e++)f[i[e]]=e;o===c.length?i.sort(function(t,e){for(var n,a,r,o,i=c.length,l=d[t]._aSortData,s=d[e]._aSortData,u=0;u<i;u++)if(0!=(r=(n=l[(o=c[u]).col])<(a=s[o.col])?-1:a<n?1:0))return"asc"===o.dir?r:-r;return(n=f[t])<(a=f[e])?-1:a<n?1:0}):i.sort(function(t,e){for(var n,a,r,o=c.length,i=d[t]._aSortData,l=d[e]._aSortData,s=0;s<o;s++)if(n=i[(r=c[s]).col],a=l[r.col],0!==(r=(u[r.type+"-"+r.dir]||u["string-"+r.dir])(n,a)))return r;return(n=f[t])<(a=f[e])?-1:a<n?1:0})}t.bSorted=!0}function le(t){for(var e=t.aoColumns,n=I(t),a=t.oLanguage.oAria,r=0,o=e.length;r<o;r++){var i=e[r],l=i.asSorting,s=i.ariaTitle||i.sTitle.replace(/<.*?>/g,""),u=i.nTh;u.removeAttribute("aria-sort"),i=i.bSortable?s+("asc"===(0<n.length&&n[0].col==r&&(u.setAttribute("aria-sort","asc"==n[0].dir?"ascending":"descending"),l[n[0].index+1])||l[0])?a.sSortAscending:a.sSortDescending):s,u.setAttribute("aria-label",i)}}function se(t,e,n,a){function r(t,e){var n=t._idx;return(n=n===H?P.inArray(t[1],s):n)+1<s.length?n+1:e?null:0}var o,i=t.aoColumns[e],l=t.aaSorting,s=i.asSorting;"number"==typeof l[0]&&(l=t.aaSorting=[l]),n&&t.oFeatures.bSortMulti?-1!==(i=P.inArray(e,N(l,"0")))?null===(o=null===(o=r(l[i],!0))&&1===l.length?0:o)?l.splice(i,1):(l[i][1]=s[o],l[i]._idx=o):(l.push([e,s[0],0]),l[l.length-1]._idx=0):l.length&&l[0][0]==e?(o=r(l[0]),l.length=1,l[0][1]=s[o],l[0]._idx=o):(l.length=0,l.push([e,s[0]]),l[0]._idx=0),u(t),"function"==typeof a&&a(t)}function ue(e,t,n,a){var r=e.aoColumns[n];me(t,{},function(t){!1!==r.bSortable&&(e.oFeatures.bProcessing?(D(e,!0),setTimeout(function(){se(e,n,t.shiftKey,a),"ssp"!==E(e)&&D(e,!1)},0)):se(e,n,t.shiftKey,a))})}function ce(t){var e,n,a,r=t.aLastSort,o=t.oClasses.sSortColumn,i=I(t),l=t.oFeatures;if(l.bSort&&l.bSortClasses){for(e=0,n=r.length;e<n;e++)a=r[e].src,P(N(t.aoData,"anCells",a)).removeClass(o+(e<2?e+1:3));for(e=0,n=i.length;e<n;e++)a=i[e].src,P(N(t.aoData,"anCells",a)).addClass(o+(e<2?e+1:3))}t.aLastSort=i}function fe(t,e){for(var n,a,r,o=t.aoColumns[e],i=w.ext.order[o.sSortDataType],l=(i&&(n=i.call(t.oInstance,t,e,ot(t,e))),w.ext.type.order[o.sType+"-pre"]),s=0,u=t.aoData.length;s<u;s++)(a=t.aoData[s])._aSortData||(a._aSortData=[]),a._aSortData[e]&&!i||(r=i?n[s]:S(t,s,e,"sort"),a._aSortData[e]=l?l(r):r)}function de(n){var t;n._bLoadingState||(t={time:+new Date,start:n._iDisplayStart,length:n._iDisplayLength,order:P.extend(!0,[],n.aaSorting),search:Et(n.oPreviousSearch),columns:P.map(n.aoColumns,function(t,e){return{visible:t.bVisible,search:Et(n.aoPreSearchCols[e])}})},n.oSavedState=t,R(n,"aoStateSaveParams","stateSaveParams",[n,t]),n.oFeatures.bStateSave&&!n.bDestroying&&n.fnStateSaveCallback.call(n.oInstance,n,t))}function he(e,t,n){var a;if(e.oFeatures.bStateSave)return(a=e.fnStateLoadCallback.call(e.oInstance,e,function(t){pe(e,t,n)}))!==H&&pe(e,a,n),!0;n()}function pe(n,t,e){var a,r,o=n.aoColumns,i=(n._bLoadingState=!0,n._bInitComplete?new w.Api(n):null);if(t&&t.time){var l=R(n,"aoStateLoadParams","stateLoadParams",[n,t]);if(-1!==P.inArray(!1,l))n._bLoadingState=!1;else{l=n.iStateDuration;if(0<l&&t.time<+new Date-1e3*l)n._bLoadingState=!1;else if(t.columns&&o.length!==t.columns.length)n._bLoadingState=!1;else{if(n.oLoadedState=P.extend(!0,{},t),t.length!==H&&(i?i.page.len(t.length):n._iDisplayLength=t.length),t.start!==H&&(null===i?(n._iDisplayStart=t.start,n.iInitDisplayStart=t.start):Yt(n,t.start/n._iDisplayLength)),t.order!==H&&(n.aaSorting=[],P.each(t.order,function(t,e){n.aaSorting.push(e[0]>=o.length?[0,e[1]]:e)})),t.search!==H&&P.extend(n.oPreviousSearch,Bt(t.search)),t.columns){for(a=0,r=t.columns.length;a<r;a++){var s=t.columns[a];s.visible!==H&&(i?i.column(a).visible(s.visible,!1):o[a].bVisible=s.visible),s.search!==H&&P.extend(n.aoPreSearchCols[a],Bt(s.search))}i&&i.columns.adjust()}n._bLoadingState=!1,R(n,"aoStateLoaded","stateLoaded",[n,t])}}}else n._bLoadingState=!1;e()}function ge(t){var e=w.settings,t=P.inArray(t,N(e,"nTable"));return-1!==t?e[t]:null}function W(t,e,n,a){if(n="DataTables warning: "+(t?"table id="+t.sTableId+" - ":"")+n,a&&(n+=". For more information about this error, please see https://datatables.net/tn/"+a),e)j.console&&console.log&&console.log(n);else{e=w.ext,e=e.sErrMode||e.errMode;if(t&&R(t,null,"error",[t,a,n]),"alert"==e)alert(n);else{if("throw"==e)throw new Error(n);"function"==typeof e&&e(t,a,n)}}}function F(n,a,t,e){Array.isArray(t)?P.each(t,function(t,e){Array.isArray(e)?F(n,a,e[0],e[1]):F(n,a,e)}):(e===H&&(e=t),a[t]!==H&&(n[e]=a[t]))}function be(t,e,n){var a,r;for(r in e)e.hasOwnProperty(r)&&(a=e[r],P.isPlainObject(a)?(P.isPlainObject(t[r])||(t[r]={}),P.extend(!0,t[r],a)):n&&"data"!==r&&"aaData"!==r&&Array.isArray(a)?t[r]=a.slice():t[r]=a);return t}function me(e,t,n){P(e).on("click.DT",t,function(t){P(e).trigger("blur"),n(t)}).on("keypress.DT",t,function(t){13===t.which&&(t.preventDefault(),n(t))}).on("selectstart.DT",function(){return!1})}function L(t,e,n,a){n&&t[e].push({fn:n,sName:a})}function R(n,t,e,a){var r=[];return t&&(r=P.map(n[t].slice().reverse(),function(t,e){return t.fn.apply(n.oInstance,a)})),null!==e&&(t=P.Event(e+".dt"),(e=P(n.nTable)).trigger(t,a),0===e.parents("body").length&&P("body").trigger(t,a),r.push(t.result)),r}function Se(t){var e=t._iDisplayStart,n=t.fnDisplayEnd(),a=t._iDisplayLength;n<=e&&(e=n-a),e-=e%a,t._iDisplayStart=e=-1===a||e<0?0:e}function ve(t,e){var t=t.renderer,n=w.ext.renderer[e];return P.isPlainObject(t)&&t[e]?n[t[e]]||n._:"string"==typeof t&&n[t]||n._}function E(t){return t.oFeatures.bServerSide?"ssp":t.ajax||t.sAjaxSource?"ajax":"dom"}function ye(t,n){var a;return Array.isArray(t)?P.map(t,function(t){return ye(t,n)}):"number"==typeof t?[n[t]]:(a=P.map(n,function(t,e){return t.nTable}),P(a).filter(t).map(function(t){var e=P.inArray(this,a);return n[e]}).toArray())}function De(r,o,t){var e,n;t&&(e=new B(r)).one("draw",function(){t(e.ajax.json())}),"ssp"==E(r)?u(r,o):(D(r,!0),(n=r.jqXHR)&&4!==n.readyState&&n.abort(),Tt(r,[],function(t){pt(r);for(var e=Ft(r,t),n=0,a=e.length;n<a;n++)x(r,e[n]);u(r,o),D(r,!1)}))}function _e(t,e,n,a,r){for(var o,i,l,s,u=[],c=typeof e,f=0,d=(e=e&&"string"!=c&&"function"!=c&&e.length!==H?e:[e]).length;f<d;f++)for(l=0,s=(i=e[f]&&e[f].split&&!e[f].match(/[\[\(:]/)?e[f].split(","):[e[f]]).length;l<s;l++)(o=n("string"==typeof i[l]?i[l].trim():i[l]))&&o.length&&(u=u.concat(o));var h=p.selector[t];if(h.length)for(f=0,d=h.length;f<d;f++)u=h[f](a,r,u);return z(u)}function we(t){return(t=t||{}).filter&&t.search===H&&(t.search=t.filter),P.extend({search:"none",order:"current",page:"all"},t)}function Ce(t){for(var e=0,n=t.length;e<n;e++)if(0<t[e].length)return t[0]=t[e],t[0].length=1,t.length=1,t.context=[t.context[e]],t;return t.length=0,t}function Te(o,t,e,n){function i(t,e){var n;if(Array.isArray(t)||t instanceof P)for(var a=0,r=t.length;a<r;a++)i(t[a],e);else t.nodeName&&"tr"===t.nodeName.toLowerCase()?l.push(t):(n=P("<tr><td></td></tr>").addClass(e),P("td",n).addClass(e).html(t)[0].colSpan=T(o),l.push(n[0]))}var l=[];i(e,n),t._details&&t._details.detach(),t._details=P(l),t._detailsShow&&t._details.insertAfter(t.nTr)}function xe(t,e){var n=t.context;if(n.length&&t.length){var a=n[0].aoData[t[0]];if(a._details){(a._detailsShow=e)?(a._details.insertAfter(a.nTr),P(a.nTr).addClass("dt-hasChild")):(a._details.detach(),P(a.nTr).removeClass("dt-hasChild")),R(n[0],null,"childRow",[e,t.row(t[0])]);var s=n[0],r=new B(s),a=".dt.DT_details",e="draw"+a,t="column-sizing"+a,a="destroy"+a,u=s.aoData;if(r.off(e+" "+t+" "+a),N(u,"_details").length>0){r.on(e,function(t,e){if(s!==e)return;r.rows({page:"current"}).eq(0).each(function(t){var e=u[t];if(e._detailsShow)e._details.insertAfter(e.nTr)})});r.on(t,function(t,e,n,a){if(s!==e)return;var r,o=T(e);for(var i=0,l=u.length;i<l;i++){r=u[i];if(r._details)r._details.each(function(){var t=P(this).children("td");if(t.length==1)t.attr("colspan",o)})}});r.on(a,function(t,e){if(s!==e)return;for(var n=0,a=u.length;n<a;n++)if(u[n]._details)Re(r,n)})}Le(n)}}}function Ae(t,e,n,a,r){for(var o=[],i=0,l=r.length;i<l;i++)o.push(S(t,r[i],e));return o}var Ie=[],o=Array.prototype,B=function(t,e){if(!(this instanceof B))return new B(t,e);function n(t){var e,n,a,r;t=t,a=w.settings,r=P.map(a,function(t,e){return t.nTable}),(t=t?t.nTable&&t.oApi?[t]:t.nodeName&&"table"===t.nodeName.toLowerCase()?-1!==(e=P.inArray(t,r))?[a[e]]:null:t&&"function"==typeof t.settings?t.settings().toArray():("string"==typeof t?n=P(t):t instanceof P&&(n=t),n?n.map(function(t){return-1!==(e=P.inArray(this,r))?a[e]:null}).toArray():void 0):[])&&o.push.apply(o,t)}var o=[];if(Array.isArray(t))for(var a=0,r=t.length;a<r;a++)n(t[a]);else n(t);this.context=z(o),e&&P.merge(this,e),this.selector={rows:null,cols:null,opts:null},B.extend(this,this,Ie)},Fe=(w.Api=B,P.extend(B.prototype,{any:function(){return 0!==this.count()},concat:o.concat,context:[],count:function(){return this.flatten().length},each:function(t){for(var e=0,n=this.length;e<n;e++)t.call(this,this[e],e,this);return this},eq:function(t){var e=this.context;return e.length>t?new B(e[t],this[t]):null},filter:function(t){var e=[];if(o.filter)e=o.filter.call(this,t,this);else for(var n=0,a=this.length;n<a;n++)t.call(this,this[n],n,this)&&e.push(this[n]);return new B(this.context,e)},flatten:function(){var t=[];return new B(this.context,t.concat.apply(t,this.toArray()))},join:o.join,indexOf:o.indexOf||function(t,e){for(var n=e||0,a=this.length;n<a;n++)if(this[n]===t)return n;return-1},iterator:function(t,e,n,a){var r,o,i,l,s,u,c,f,d=[],h=this.context,p=this.selector;for("string"==typeof t&&(a=n,n=e,e=t,t=!1),o=0,i=h.length;o<i;o++){var g=new B(h[o]);if("table"===e)(r=n.call(g,h[o],o))!==H&&d.push(r);else if("columns"===e||"rows"===e)(r=n.call(g,h[o],this[o],o))!==H&&d.push(r);else if("column"===e||"column-rows"===e||"row"===e||"cell"===e)for(c=this[o],"column-rows"===e&&(u=Fe(h[o],p.opts)),l=0,s=c.length;l<s;l++)f=c[l],(r="cell"===e?n.call(g,h[o],f.row,f.column,o,l):n.call(g,h[o],f,o,l,u))!==H&&d.push(r)}return d.length||a?((t=(a=new B(h,t?d.concat.apply([],d):d)).selector).rows=p.rows,t.cols=p.cols,t.opts=p.opts,a):this},lastIndexOf:o.lastIndexOf||function(t,e){return this.indexOf.apply(this.toArray.reverse(),arguments)},length:0,map:function(t){var e=[];if(o.map)e=o.map.call(this,t,this);else for(var n=0,a=this.length;n<a;n++)e.push(t.call(this,this[n],n));return new B(this.context,e)},pluck:function(t){var e=w.util.get(t);return this.map(function(t){return e(t)})},pop:o.pop,push:o.push,reduce:o.reduce||function(t,e){return et(this,t,e,0,this.length,1)},reduceRight:o.reduceRight||function(t,e){return et(this,t,e,this.length-1,-1,-1)},reverse:o.reverse,selector:null,shift:o.shift,slice:function(){return new B(this.context,this)},sort:o.sort,splice:o.splice,toArray:function(){return o.slice.call(this)},to$:function(){return P(this)},toJQuery:function(){return P(this)},unique:function(){return new B(this.context,z(this))},unshift:o.unshift}),B.extend=function(t,e,n){if(n.length&&e&&(e instanceof B||e.__dt_wrapper))for(var a,r=0,o=n.length;r<o;r++)e[(a=n[r]).name]="function"===a.type?function(e,n,a){return function(){var t=n.apply(e,arguments);return B.extend(t,t,a.methodExt),t}}(t,a.val,a):"object"===a.type?{}:a.val,e[a.name].__dt_wrapper=!0,B.extend(t,e[a.name],a.propExt)},B.register=e=function(t,e){if(Array.isArray(t))for(var n=0,a=t.length;n<a;n++)B.register(t[n],e);else for(var r=t.split("."),o=Ie,i=0,l=r.length;i<l;i++){var s,u,c=function(t,e){for(var n=0,a=t.length;n<a;n++)if(t[n].name===e)return t[n];return null}(o,u=(s=-1!==r[i].indexOf("()"))?r[i].replace("()",""):r[i]);c||o.push(c={name:u,val:{},methodExt:[],propExt:[],type:"object"}),i===l-1?(c.val=e,c.type="function"==typeof e?"function":P.isPlainObject(e)?"object":"other"):o=s?c.methodExt:c.propExt}},B.registerPlural=t=function(t,e,n){B.register(t,n),B.register(e,function(){var t=n.apply(this,arguments);return t===this?this:t instanceof B?t.length?Array.isArray(t[0])?new B(t.context,t[0]):t[0]:H:t})},e("tables()",function(t){return t!==H&&null!==t?new B(ye(t,this.context)):this}),e("table()",function(t){var t=this.tables(t),e=t.context;return e.length?new B(e[0]):t}),t("tables().nodes()","table().node()",function(){return this.iterator("table",function(t){return t.nTable},1)}),t("tables().body()","table().body()",function(){return this.iterator("table",function(t){return t.nTBody},1)}),t("tables().header()","table().header()",function(){return this.iterator("table",function(t){return t.nTHead},1)}),t("tables().footer()","table().footer()",function(){return this.iterator("table",function(t){return t.nTFoot},1)}),t("tables().containers()","table().container()",function(){return this.iterator("table",function(t){return t.nTableWrapper},1)}),e("draw()",function(e){return this.iterator("table",function(t){"page"===e?y(t):u(t,!1===(e="string"==typeof e?"full-hold"!==e:e))})}),e("page()",function(e){return e===H?this.page.info().page:this.iterator("table",function(t){Yt(t,e)})}),e("page.info()",function(t){var e,n,a,r,o;return 0===this.context.length?H:(n=(e=this.context[0])._iDisplayStart,a=e.oFeatures.bPaginate?e._iDisplayLength:-1,r=e.fnRecordsDisplay(),{page:(o=-1===a)?0:Math.floor(n/a),pages:o?1:Math.ceil(r/a),start:n,end:e.fnDisplayEnd(),length:a,recordsTotal:e.fnRecordsTotal(),recordsDisplay:r,serverSide:"ssp"===E(e)})}),e("page.len()",function(e){return e===H?0!==this.context.length?this.context[0]._iDisplayLength:H:this.iterator("table",function(t){$t(t,e)})}),e("ajax.json()",function(){var t=this.context;if(0<t.length)return t[0].json}),e("ajax.params()",function(){var t=this.context;if(0<t.length)return t[0].oAjaxData}),e("ajax.reload()",function(e,n){return this.iterator("table",function(t){De(t,!1===n,e)})}),e("ajax.url()",function(e){var t=this.context;return e===H?0===t.length?H:(t=t[0]).ajax?P.isPlainObject(t.ajax)?t.ajax.url:t.ajax:t.sAjaxSource:this.iterator("table",function(t){P.isPlainObject(t.ajax)?t.ajax.url=e:t.ajax=e})}),e("ajax.url().load()",function(e,n){return this.iterator("table",function(t){De(t,!1===n,e)})}),function(t,e){var n,a=[],r=t.aiDisplay,o=t.aiDisplayMaster,i=e.search,l=e.order,e=e.page;if("ssp"==E(t))return"removed"===i?[]:f(0,o.length);if("current"==e)for(u=t._iDisplayStart,c=t.fnDisplayEnd();u<c;u++)a.push(r[u]);else if("current"==l||"applied"==l){if("none"==i)a=o.slice();else if("applied"==i)a=r.slice();else if("removed"==i){for(var s={},u=0,c=r.length;u<c;u++)s[r[u]]=null;a=P.map(o,function(t){return s.hasOwnProperty(t)?null:t})}}else if("index"==l||"original"==l)for(u=0,c=t.aoData.length;u<c;u++)("none"==i||-1===(n=P.inArray(u,r))&&"removed"==i||0<=n&&"applied"==i)&&a.push(u);return a}),Le=(e("rows()",function(e,n){e===H?e="":P.isPlainObject(e)&&(n=e,e=""),n=we(n);var t=this.iterator("table",function(t){return _e("row",e,function(n){var t=d(n),a=r.aoData;if(null!==t&&!o)return[t];if(i=i||Fe(r,o),null!==t&&-1!==P.inArray(t,i))return[t];if(null===n||n===H||""===n)return i;if("function"==typeof n)return P.map(i,function(t){var e=a[t];return n(t,e._aData,e.nTr)?t:null});if(n.nodeName)return t=n._DT_RowIndex,e=n._DT_CellIndex,t!==H?a[t]&&a[t].nTr===n?[t]:[]:e?a[e.row]&&a[e.row].nTr===n.parentNode?[e.row]:[]:(t=P(n).closest("*[data-dt-row]")).length?[t.data("dt-row")]:[];if("string"==typeof n&&"#"===n.charAt(0)){var e=r.aIds[n.replace(/^#/,"")];if(e!==H)return[e.idx]}t=_(m(r.aoData,i,"nTr"));return P(t).filter(n).map(function(){return this._DT_RowIndex}).toArray()},r=t,o=n);var r,o,i},1);return t.selector.rows=e,t.selector.opts=n,t}),e("rows().nodes()",function(){return this.iterator("row",function(t,e){return t.aoData[e].nTr||H},1)}),e("rows().data()",function(){return this.iterator(!0,"rows",function(t,e){return m(t.aoData,e,"_aData")},1)}),t("rows().cache()","row().cache()",function(n){return this.iterator("row",function(t,e){t=t.aoData[e];return"search"===n?t._aFilterData:t._aSortData},1)}),t("rows().invalidate()","row().invalidate()",function(n){return this.iterator("row",function(t,e){bt(t,e,n)})}),t("rows().indexes()","row().index()",function(){return this.iterator("row",function(t,e){return e},1)}),t("rows().ids()","row().id()",function(t){for(var e=[],n=this.context,a=0,r=n.length;a<r;a++)for(var o=0,i=this[a].length;o<i;o++){var l=n[a].rowIdFn(n[a].aoData[this[a][o]]._aData);e.push((!0===t?"#":"")+l)}return new B(n,e)}),t("rows().remove()","row().remove()",function(){var f=this;return this.iterator("row",function(t,e,n){var a,r,o,i,l,s,u=t.aoData,c=u[e];for(u.splice(e,1),a=0,r=u.length;a<r;a++)if(s=(l=u[a]).anCells,null!==l.nTr&&(l.nTr._DT_RowIndex=a),null!==s)for(o=0,i=s.length;o<i;o++)s[o]._DT_CellIndex.row=a;gt(t.aiDisplayMaster,e),gt(t.aiDisplay,e),gt(f[n],e,!1),0<t._iRecordsDisplay&&t._iRecordsDisplay--,Se(t);n=t.rowIdFn(c._aData);n!==H&&delete t.aIds[n]}),this.iterator("table",function(t){for(var e=0,n=t.aoData.length;e<n;e++)t.aoData[e].idx=e}),this}),e("rows.add()",function(o){var t=this.iterator("table",function(t){for(var e,n=[],a=0,r=o.length;a<r;a++)(e=o[a]).nodeName&&"TR"===e.nodeName.toUpperCase()?n.push(ut(t,e)[0]):n.push(x(t,e));return n},1),e=this.rows(-1);return e.pop(),P.merge(e,t),e}),e("row()",function(t,e){return Ce(this.rows(t,e))}),e("row().data()",function(t){var e,n=this.context;return t===H?n.length&&this.length?n[0].aoData[this[0]]._aData:H:((e=n[0].aoData[this[0]])._aData=t,Array.isArray(t)&&e.nTr&&e.nTr.id&&b(n[0].rowId)(t,e.nTr.id),bt(n[0],this[0],"data"),this)}),e("row().node()",function(){var t=this.context;return t.length&&this.length&&t[0].aoData[this[0]].nTr||null}),e("row.add()",function(e){e instanceof P&&e.length&&(e=e[0]);var t=this.iterator("table",function(t){return e.nodeName&&"TR"===e.nodeName.toUpperCase()?ut(t,e)[0]:x(t,e)});return this.row(t[0])}),P(v).on("plugin-init.dt",function(t,e){var n=new B(e),a="on-plugin-init",r="stateSaveParams."+a,o="destroy. "+a,a=(n.on(r,function(t,e,n){for(var a=e.rowIdFn,r=e.aoData,o=[],i=0;i<r.length;i++)r[i]._detailsShow&&o.push("#"+a(r[i]._aData));n.childRows=o}),n.on(o,function(){n.off(r+" "+o)}),n.state.loaded());a&&a.childRows&&n.rows(P.map(a.childRows,function(t){return t.replace(/:/g,"\\:")})).every(function(){R(e,null,"requestChild",[this])})}),w.util.throttle(function(t){de(t[0])},500)),Re=function(t,e){var n=t.context;n.length&&(e=n[0].aoData[e!==H?e:t[0]])&&e._details&&(e._details.remove(),e._detailsShow=H,e._details=H,P(e.nTr).removeClass("dt-hasChild"),Le(n))},Pe="row().child",je=Pe+"()",He=(e(je,function(t,e){var n=this.context;return t===H?n.length&&this.length?n[0].aoData[this[0]]._details:H:(!0===t?this.child.show():!1===t?Re(this):n.length&&this.length&&Te(n[0],n[0].aoData[this[0]],t,e),this)}),e([Pe+".show()",je+".show()"],function(t){return xe(this,!0),this}),e([Pe+".hide()",je+".hide()"],function(){return xe(this,!1),this}),e([Pe+".remove()",je+".remove()"],function(){return Re(this),this}),e(Pe+".isShown()",function(){var t=this.context;return t.length&&this.length&&t[0].aoData[this[0]]._detailsShow||!1}),/^([^:]+):(name|visIdx|visible)$/),Ne=(e("columns()",function(n,a){n===H?n="":P.isPlainObject(n)&&(a=n,n=""),a=we(a);var t=this.iterator("table",function(t){return e=n,l=a,s=(i=t).aoColumns,u=N(s,"sName"),c=N(s,"nTh"),_e("column",e,function(n){var a,t=d(n);if(""===n)return f(s.length);if(null!==t)return[0<=t?t:s.length+t];if("function"==typeof n)return a=Fe(i,l),P.map(s,function(t,e){return n(e,Ae(i,e,0,0,a),c[e])?e:null});var r="string"==typeof n?n.match(He):"";if(r)switch(r[2]){case"visIdx":case"visible":var e,o=parseInt(r[1],10);return o<0?[(e=P.map(s,function(t,e){return t.bVisible?e:null}))[e.length+o]]:[rt(i,o)];case"name":return P.map(u,function(t,e){return t===r[1]?e:null});default:return[]}return n.nodeName&&n._DT_CellIndex?[n._DT_CellIndex.column]:(t=P(c).filter(n).map(function(){return P.inArray(this,c)}).toArray()).length||!n.nodeName?t:(t=P(n).closest("*[data-dt-column]")).length?[t.data("dt-column")]:[]},i,l);var i,e,l,s,u,c},1);return t.selector.cols=n,t.selector.opts=a,t}),t("columns().header()","column().header()",function(t,e){return this.iterator("column",function(t,e){return t.aoColumns[e].nTh},1)}),t("columns().footer()","column().footer()",function(t,e){return this.iterator("column",function(t,e){return t.aoColumns[e].nTf},1)}),t("columns().data()","column().data()",function(){return this.iterator("column-rows",Ae,1)}),t("columns().dataSrc()","column().dataSrc()",function(){return this.iterator("column",function(t,e){return t.aoColumns[e].mData},1)}),t("columns().cache()","column().cache()",function(o){return this.iterator("column-rows",function(t,e,n,a,r){return m(t.aoData,r,"search"===o?"_aFilterData":"_aSortData",e)},1)}),t("columns().nodes()","column().nodes()",function(){return this.iterator("column-rows",function(t,e,n,a,r){return m(t.aoData,r,"anCells",e)},1)}),t("columns().visible()","column().visible()",function(f,n){var e=this,t=this.iterator("column",function(t,e){if(f===H)return t.aoColumns[e].bVisible;var n,a,r=e,e=f,o=t.aoColumns,i=o[r],l=t.aoData;if(e===H)i.bVisible;else if(i.bVisible!==e){if(e)for(var s=P.inArray(!0,N(o,"bVisible"),r+1),u=0,c=l.length;u<c;u++)a=l[u].nTr,n=l[u].anCells,a&&a.insertBefore(n[r],n[s]||null);else P(N(t.aoData,"anCells",r)).detach();i.bVisible=e}});return f!==H&&this.iterator("table",function(t){Dt(t,t.aoHeader),Dt(t,t.aoFooter),t.aiDisplay.length||P(t.nTBody).find("td[colspan]").attr("colspan",T(t)),de(t),e.iterator("column",function(t,e){R(t,null,"column-visibility",[t,e,f,n])}),n!==H&&!n||e.columns.adjust()}),t}),t("columns().indexes()","column().index()",function(n){return this.iterator("column",function(t,e){return"visible"===n?ot(t,e):e},1)}),e("columns.adjust()",function(){return this.iterator("table",function(t){O(t)},1)}),e("column.index()",function(t,e){var n;if(0!==this.context.length)return n=this.context[0],"fromVisible"===t||"toData"===t?rt(n,e):"fromData"===t||"toVisible"===t?ot(n,e):void 0}),e("column()",function(t,e){return Ce(this.columns(t,e))}),e("cells()",function(g,t,b){var a,r,o,i,l,s,e;return P.isPlainObject(g)&&(g.row===H?(b=g,g=null):(b=t,t=null)),P.isPlainObject(t)&&(b=t,t=null),null===t||t===H?this.iterator("table",function(t){return a=t,t=g,e=we(b),f=a.aoData,d=Fe(a,e),n=_(m(f,d,"anCells")),h=P(Y([],n)),p=a.aoColumns.length,_e("cell",t,function(t){var e,n="function"==typeof t;if(null===t||t===H||n){for(o=[],i=0,l=d.length;i<l;i++)for(r=d[i],s=0;s<p;s++)u={row:r,column:s},(!n||(c=f[r],t(u,S(a,r,s),c.anCells?c.anCells[s]:null)))&&o.push(u);return o}return P.isPlainObject(t)?t.column!==H&&t.row!==H&&-1!==P.inArray(t.row,d)?[t]:[]:(e=h.filter(t).map(function(t,e){return{row:e._DT_CellIndex.row,column:e._DT_CellIndex.column}}).toArray()).length||!t.nodeName?e:(c=P(t).closest("*[data-dt-row]")).length?[{row:c.data("dt-row"),column:c.data("dt-column")}]:[]},a,e);var a,e,r,o,i,l,s,u,c,f,d,n,h,p}):(e=b?{page:b.page,order:b.order,search:b.search}:{},a=this.columns(t,e),r=this.rows(g,e),e=this.iterator("table",function(t,e){var n=[];for(o=0,i=r[e].length;o<i;o++)for(l=0,s=a[e].length;l<s;l++)n.push({row:r[e][o],column:a[e][l]});return n},1),e=b&&b.selected?this.cells(e,b):e,P.extend(e.selector,{cols:t,rows:g,opts:b}),e)}),t("cells().nodes()","cell().node()",function(){return this.iterator("cell",function(t,e,n){t=t.aoData[e];return t&&t.anCells?t.anCells[n]:H},1)}),e("cells().data()",function(){return this.iterator("cell",function(t,e,n){return S(t,e,n)},1)}),t("cells().cache()","cell().cache()",function(a){return a="search"===a?"_aFilterData":"_aSortData",this.iterator("cell",function(t,e,n){return t.aoData[e][a][n]},1)}),t("cells().render()","cell().render()",function(a){return this.iterator("cell",function(t,e,n){return S(t,e,n,a)},1)}),t("cells().indexes()","cell().index()",function(){return this.iterator("cell",function(t,e,n){return{row:e,column:n,columnVisible:ot(t,n)}},1)}),t("cells().invalidate()","cell().invalidate()",function(a){return this.iterator("cell",function(t,e,n){bt(t,e,a,n)})}),e("cell()",function(t,e,n){return Ce(this.cells(t,e,n))}),e("cell().data()",function(t){var e=this.context,n=this[0];return t===H?e.length&&n.length?S(e[0],n[0].row,n[0].column):H:(ct(e[0],n[0].row,n[0].column,t),bt(e[0],n[0].row,"data",n[0].column),this)}),e("order()",function(e,t){var n=this.context;return e===H?0!==n.length?n[0].aaSorting:H:("number"==typeof e?e=[[e,t]]:e.length&&!Array.isArray(e[0])&&(e=Array.prototype.slice.call(arguments)),this.iterator("table",function(t){t.aaSorting=e.slice()}))}),e("order.listener()",function(e,n,a){return this.iterator("table",function(t){ue(t,e,n,a)})}),e("order.fixed()",function(e){var t;return e?this.iterator("table",function(t){t.aaSortingFixed=P.extend(!0,{},e)}):(t=(t=this.context).length?t[0].aaSortingFixed:H,Array.isArray(t)?{pre:t}:t)}),e(["columns().order()","column().order()"],function(a){var r=this;return this.iterator("table",function(t,e){var n=[];P.each(r[e],function(t,e){n.push([e,a])}),t.aaSorting=n})}),e("search()",function(e,n,a,r){var t=this.context;return e===H?0!==t.length?t[0].oPreviousSearch.sSearch:H:this.iterator("table",function(t){t.oFeatures.bFilter&&Rt(t,P.extend({},t.oPreviousSearch,{sSearch:e+"",bRegex:null!==n&&n,bSmart:null===a||a,bCaseInsensitive:null===r||r}),1)})}),t("columns().search()","column().search()",function(a,r,o,i){return this.iterator("column",function(t,e){var n=t.aoPreSearchCols;if(a===H)return n[e].sSearch;t.oFeatures.bFilter&&(P.extend(n[e],{sSearch:a+"",bRegex:null!==r&&r,bSmart:null===o||o,bCaseInsensitive:null===i||i}),Rt(t,t.oPreviousSearch,1))})}),e("state()",function(){return this.context.length?this.context[0].oSavedState:null}),e("state.clear()",function(){return this.iterator("table",function(t){t.fnStateSaveCallback.call(t.oInstance,t,{})})}),e("state.loaded()",function(){return this.context.length?this.context[0].oLoadedState:null}),e("state.save()",function(){return this.iterator("table",function(t){de(t)})}),w.use=function(t,e){"lib"===e||t.fn?P=t:"win"==e||t.document?v=(j=t).document:"datetime"!==e&&"DateTime"!==t.type||(w.DateTime=t)},w.factory=function(t,e){var n=!1;return t&&t.document&&(v=(j=t).document),e&&e.fn&&e.fn.jquery&&(P=e,n=!0),n},w.versionCheck=w.fnVersionCheck=function(t){for(var e,n,a=w.version.split("."),r=t.split("."),o=0,i=r.length;o<i;o++)if((e=parseInt(a[o],10)||0)!==(n=parseInt(r[o],10)||0))return n<e;return!0},w.isDataTable=w.fnIsDataTable=function(t){var r=P(t).get(0),o=!1;return t instanceof w.Api||(P.each(w.settings,function(t,e){var n=e.nScrollHead?P("table",e.nScrollHead)[0]:null,a=e.nScrollFoot?P("table",e.nScrollFoot)[0]:null;e.nTable!==r&&n!==r&&a!==r||(o=!0)}),o)},w.tables=w.fnTables=function(e){var t=!1,n=(P.isPlainObject(e)&&(t=e.api,e=e.visible),P.map(w.settings,function(t){if(!e||P(t.nTable).is(":visible"))return t.nTable}));return t?new B(n):n},w.camelToHungarian=C,e("$()",function(t,e){e=this.rows(e).nodes(),e=P(e);return P([].concat(e.filter(t).toArray(),e.find(t).toArray()))}),P.each(["on","one","off"],function(t,n){e(n+"()",function(){var t=Array.prototype.slice.call(arguments),e=(t[0]=P.map(t[0].split(/\s/),function(t){return t.match(/\.dt\b/)?t:t+".dt"}).join(" "),P(this.tables().nodes()));return e[n].apply(e,t),this})}),e("clear()",function(){return this.iterator("table",function(t){pt(t)})}),e("settings()",function(){return new B(this.context,this.context)}),e("init()",function(){var t=this.context;return t.length?t[0].oInit:null}),e("data()",function(){return this.iterator("table",function(t){return N(t.aoData,"_aData")}).flatten()}),e("destroy()",function(c){return c=c||!1,this.iterator("table",function(e){var n,t=e.oClasses,a=e.nTable,r=e.nTBody,o=e.nTHead,i=e.nTFoot,l=P(a),r=P(r),s=P(e.nTableWrapper),u=P.map(e.aoData,function(t){return t.nTr}),i=(e.bDestroying=!0,R(e,"aoDestroyCallback","destroy",[e]),c||new B(e).columns().visible(!0),s.off(".DT").find(":not(tbody *)").off(".DT"),P(j).off(".DT-"+e.sInstance),a!=o.parentNode&&(l.children("thead").detach(),l.append(o)),i&&a!=i.parentNode&&(l.children("tfoot").detach(),l.append(i)),e.aaSorting=[],e.aaSortingFixed=[],ce(e),P(u).removeClass(e.asStripeClasses.join(" ")),P("th, td",o).removeClass(t.sSortable+" "+t.sSortableAsc+" "+t.sSortableDesc+" "+t.sSortableNone),r.children().detach(),r.append(u),e.nTableWrapper.parentNode),o=c?"remove":"detach",u=(l[o](),s[o](),!c&&i&&(i.insertBefore(a,e.nTableReinsertBefore),l.css("width",e.sDestroyWidth).removeClass(t.sTable),n=e.asDestroyStripes.length)&&r.children().each(function(t){P(this).addClass(e.asDestroyStripes[t%n])}),P.inArray(e,w.settings));-1!==u&&w.settings.splice(u,1)})}),P.each(["column","row","cell"],function(t,s){e(s+"s().every()",function(o){var i=this.selector.opts,l=this;return this.iterator(s,function(t,e,n,a,r){o.call(l[s](e,"cell"===s?n:i,"cell"===s?i:H),e,n,a,r)})})}),e("i18n()",function(t,e,n){var a=this.context[0],t=A(t)(a.oLanguage);return t===H&&(t=e),"string"==typeof(t=n!==H&&P.isPlainObject(t)?t[n]!==H?t[n]:t._:t)?t.replace("%d",n):t}),w.version="1.13.8",w.settings=[],w.models={},w.models.oSearch={bCaseInsensitive:!0,sSearch:"",bRegex:!1,bSmart:!0,return:!1},w.models.oRow={nTr:null,anCells:null,_aData:[],_aSortData:null,_aFilterData:null,_sFilterRow:null,_sRowStripe:"",src:null,idx:-1},w.models.oColumn={idx:null,aDataSort:null,asSorting:null,bSearchable:null,bSortable:null,bVisible:null,_sManualType:null,_bAttrSrc:!1,fnCreatedCell:null,fnGetData:null,fnSetData:null,mData:null,mRender:null,nTh:null,nTf:null,sClass:null,sContentPadding:null,sDefaultContent:null,sName:null,sSortDataType:"std",sSortingClass:null,sSortingClassJUI:null,sTitle:null,sType:null,sWidth:null,sWidthOrig:null},w.defaults={aaData:null,aaSorting:[[0,"asc"]],aaSortingFixed:[],ajax:null,aLengthMenu:[10,25,50,100],aoColumns:null,aoColumnDefs:null,aoSearchCols:[],asStripeClasses:null,bAutoWidth:!0,bDeferRender:!1,bDestroy:!1,bFilter:!0,bInfo:!0,bLengthChange:!0,bPaginate:!0,bProcessing:!1,bRetrieve:!1,bScrollCollapse:!1,bServerSide:!1,bSort:!0,bSortMulti:!0,bSortCellsTop:!1,bSortClasses:!0,bStateSave:!1,fnCreatedRow:null,fnDrawCallback:null,fnFooterCallback:null,fnFormatNumber:function(t){return t.toString().replace(/\B(?=(\d{3})+(?!\d))/g,this.oLanguage.sThousands)},fnHeaderCallback:null,fnInfoCallback:null,fnInitComplete:null,fnPreDrawCallback:null,fnRowCallback:null,fnServerData:null,fnServerParams:null,fnStateLoadCallback:function(t){try{return JSON.parse((-1===t.iStateDuration?sessionStorage:localStorage).getItem("DataTables_"+t.sInstance+"_"+location.pathname))}catch(t){return{}}},fnStateLoadParams:null,fnStateLoaded:null,fnStateSaveCallback:function(t,e){try{(-1===t.iStateDuration?sessionStorage:localStorage).setItem("DataTables_"+t.sInstance+"_"+location.pathname,JSON.stringify(e))}catch(t){}},fnStateSaveParams:null,iStateDuration:7200,iDeferLoading:null,iDisplayLength:10,iDisplayStart:0,iTabIndex:0,oClasses:{},oLanguage:{oAria:{sSortAscending:": activate to sort column ascending",sSortDescending:": activate to sort column descending"},oPaginate:{sFirst:"First",sLast:"Last",sNext:"Next",sPrevious:"Previous"},sEmptyTable:"No data available in table",sInfo:"Showing _START_ to _END_ of _TOTAL_ entries",sInfoEmpty:"Showing 0 to 0 of 0 entries",sInfoFiltered:"(filtered from _MAX_ total entries)",sInfoPostFix:"",sDecimal:"",sThousands:",",sLengthMenu:"Show _MENU_ entries",sLoadingRecords:"Loading...",sProcessing:"",sSearch:"Search:",sSearchPlaceholder:"",sUrl:"",sZeroRecords:"No matching records found"},oSearch:P.extend({},w.models.oSearch),sAjaxDataProp:"data",sAjaxSource:null,sDom:"lfrtip",searchDelay:null,sPaginationType:"simple_numbers",sScrollX:"",sScrollXInner:"",sScrollY:"",sServerMethod:"GET",renderer:null,rowId:"DT_RowId"},i(w.defaults),w.defaults.column={aDataSort:null,iDataSort:-1,asSorting:["asc","desc"],bSearchable:!0,bSortable:!0,bVisible:!0,fnCreatedCell:null,mData:null,mRender:null,sCellType:"td",sClass:"",sContentPadding:"",sDefaultContent:null,sName:"",sSortDataType:"std",sTitle:null,sType:null,sWidth:null},i(w.defaults.column),w.models.oSettings={oFeatures:{bAutoWidth:null,bDeferRender:null,bFilter:null,bInfo:null,bLengthChange:null,bPaginate:null,bProcessing:null,bServerSide:null,bSort:null,bSortMulti:null,bSortClasses:null,bStateSave:null},oScroll:{bCollapse:null,iBarWidth:0,sX:null,sXInner:null,sY:null},oLanguage:{fnInfoCallback:null},oBrowser:{bScrollOversize:!1,bScrollbarLeft:!1,bBounding:!1,barWidth:0},ajax:null,aanFeatures:[],aoData:[],aiDisplay:[],aiDisplayMaster:[],aIds:{},aoColumns:[],aoHeader:[],aoFooter:[],oPreviousSearch:{},aoPreSearchCols:[],aaSorting:null,aaSortingFixed:[],asStripeClasses:null,asDestroyStripes:[],sDestroyWidth:0,aoRowCallback:[],aoHeaderCallback:[],aoFooterCallback:[],aoDrawCallback:[],aoRowCreatedCallback:[],aoPreDrawCallback:[],aoInitComplete:[],aoStateSaveParams:[],aoStateLoadParams:[],aoStateLoaded:[],sTableId:"",nTable:null,nTHead:null,nTFoot:null,nTBody:null,nTableWrapper:null,bDeferLoading:!1,bInitialised:!1,aoOpenRows:[],sDom:null,searchDelay:null,sPaginationType:"two_button",iStateDuration:0,aoStateSave:[],aoStateLoad:[],oSavedState:null,oLoadedState:null,sAjaxSource:null,sAjaxDataProp:null,jqXHR:null,json:H,oAjaxData:H,fnServerData:null,aoServerParams:[],sServerMethod:null,fnFormatNumber:null,aLengthMenu:null,iDraw:0,bDrawing:!1,iDrawError:-1,_iDisplayLength:10,_iDisplayStart:0,_iRecordsTotal:0,_iRecordsDisplay:0,oClasses:{},bFiltered:!1,bSorted:!1,bSortCellsTop:null,oInit:null,aoDestroyCallback:[],fnRecordsTotal:function(){return"ssp"==E(this)?+this._iRecordsTotal:this.aiDisplayMaster.length},fnRecordsDisplay:function(){return"ssp"==E(this)?+this._iRecordsDisplay:this.aiDisplay.length},fnDisplayEnd:function(){var t=this._iDisplayLength,e=this._iDisplayStart,n=e+t,a=this.aiDisplay.length,r=this.oFeatures,o=r.bPaginate;return r.bServerSide?!1===o||-1===t?e+a:Math.min(e+t,this._iRecordsDisplay):!o||a<n||-1===t?a:n},oInstance:null,sInstance:null,iTabIndex:0,nScrollHead:null,nScrollFoot:null,aLastSort:[],oPlugins:{},rowIdFn:null,rowId:null},w.ext=p={buttons:{},classes:{},builder:"-source-",errMode:"alert",feature:[],search:[],selector:{cell:[],column:[],row:[]},internal:{},legacy:{ajax:null},pager:{},renderer:{pageButton:{},header:{}},order:{},type:{detect:[],search:{},order:{}},_unique:0,fnVersionCheck:w.fnVersionCheck,iApiIndex:0,oJUIClasses:{},sVersion:w.version},P.extend(p,{afnFiltering:p.search,aTypes:p.type.detect,ofnSearch:p.type.search,oSort:p.type.order,afnSortData:p.order,aoFeatures:p.feature,oApi:p.internal,oStdClasses:p.classes,oPagination:p.pager}),P.extend(w.ext.classes,{sTable:"dataTable",sNoFooter:"no-footer",sPageButton:"paginate_button",sPageButtonActive:"current",sPageButtonDisabled:"disabled",sStripeOdd:"odd",sStripeEven:"even",sRowEmpty:"dataTables_empty",sWrapper:"dataTables_wrapper",sFilter:"dataTables_filter",sInfo:"dataTables_info",sPaging:"dataTables_paginate paging_",sLength:"dataTables_length",sProcessing:"dataTables_processing",sSortAsc:"sorting_asc",sSortDesc:"sorting_desc",sSortable:"sorting",sSortableAsc:"sorting_desc_disabled",sSortableDesc:"sorting_asc_disabled",sSortableNone:"sorting_disabled",sSortColumn:"sorting_",sFilterInput:"",sLengthSelect:"",sScrollWrapper:"dataTables_scroll",sScrollHead:"dataTables_scrollHead",sScrollHeadInner:"dataTables_scrollHeadInner",sScrollBody:"dataTables_scrollBody",sScrollFoot:"dataTables_scrollFoot",sScrollFootInner:"dataTables_scrollFootInner",sHeaderTH:"",sFooterTH:"",sSortJUIAsc:"",sSortJUIDesc:"",sSortJUI:"",sSortJUIAscAllowed:"",sSortJUIDescAllowed:"",sSortJUIWrapper:"",sSortIcon:"",sJUIHeader:"",sJUIFooter:""}),w.ext.pager);function Oe(t,e){var n=[],a=Ne.numbers_length,r=Math.floor(a/2);return e<=a?n=f(0,e):t<=r?((n=f(0,a-2)).push("ellipsis"),n.push(e-1)):((e-1-r<=t?n=f(e-(a-2),e):((n=f(t-r+2,t+r-1)).push("ellipsis"),n.push(e-1),n)).splice(0,0,"ellipsis"),n.splice(0,0,0)),n.DT_el="span",n}P.extend(Ne,{simple:function(t,e){return["previous","next"]},full:function(t,e){return["first","previous","next","last"]},numbers:function(t,e){return[Oe(t,e)]},simple_numbers:function(t,e){return["previous",Oe(t,e),"next"]},full_numbers:function(t,e){return["first","previous",Oe(t,e),"next","last"]},first_last_numbers:function(t,e){return["first",Oe(t,e),"last"]},_numbers:Oe,numbers_length:7}),P.extend(!0,w.ext.renderer,{pageButton:{_:function(u,t,c,e,f,d){function h(t,e){for(var n,a=b.sPageButtonDisabled,r=function(t){Yt(u,t.data.action,!0)},o=0,i=e.length;o<i;o++)if(n=e[o],Array.isArray(n)){var l=P("<"+(n.DT_el||"div")+"/>").appendTo(t);h(l,n)}else{var s=!1;switch(p=null,g=n){case"ellipsis":t.append('<span class="ellipsis">&#x2026;</span>');break;case"first":p=m.sFirst,0===f&&(s=!0);break;case"previous":p=m.sPrevious,0===f&&(s=!0);break;case"next":p=m.sNext,0!==d&&f!==d-1||(s=!0);break;case"last":p=m.sLast,0!==d&&f!==d-1||(s=!0);break;default:p=u.fnFormatNumber(n+1),g=f===n?b.sPageButtonActive:""}null!==p&&(l=u.oInit.pagingTag||"a",s&&(g+=" "+a),me(P("<"+l+">",{class:b.sPageButton+" "+g,"aria-controls":u.sTableId,"aria-disabled":s?"true":null,"aria-label":S[n],role:"link","aria-current":g===b.sPageButtonActive?"page":null,"data-dt-idx":n,tabindex:s?-1:u.iTabIndex,id:0===c&&"string"==typeof n?u.sTableId+"_"+n:null}).html(p).appendTo(t),{action:n},r))}}var p,g,n,b=u.oClasses,m=u.oLanguage.oPaginate,S=u.oLanguage.oAria.paginate||{};try{n=P(t).find(v.activeElement).data("dt-idx")}catch(t){}h(P(t).empty(),e),n!==H&&P(t).find("[data-dt-idx="+n+"]").trigger("focus")}}}),P.extend(w.ext.type.detect,[function(t,e){e=e.oLanguage.sDecimal;return l(t,e)?"num"+e:null},function(t,e){var n;return(!t||t instanceof Date||X.test(t))&&(null!==(n=Date.parse(t))&&!isNaN(n)||h(t))?"date":null},function(t,e){e=e.oLanguage.sDecimal;return l(t,e,!0)?"num-fmt"+e:null},function(t,e){e=e.oLanguage.sDecimal;return a(t,e)?"html-num"+e:null},function(t,e){e=e.oLanguage.sDecimal;return a(t,e,!0)?"html-num-fmt"+e:null},function(t,e){return h(t)||"string"==typeof t&&-1!==t.indexOf("<")?"html":null}]),P.extend(w.ext.type.search,{html:function(t){return h(t)?t:"string"==typeof t?t.replace(U," ").replace(V,""):""},string:function(t){return!h(t)&&"string"==typeof t?t.replace(U," "):t}});function ke(t,e,n,a){var r;return 0===t||t&&"-"!==t?"number"==(r=typeof t)||"bigint"==r?t:+(t=(t=e?$(t,e):t).replace&&(n&&(t=t.replace(n,"")),a)?t.replace(a,""):t):-1/0}function Me(n){P.each({num:function(t){return ke(t,n)},"num-fmt":function(t){return ke(t,n,q)},"html-num":function(t){return ke(t,n,V)},"html-num-fmt":function(t){return ke(t,n,V,q)}},function(t,e){p.type.order[t+n+"-pre"]=e,t.match(/^html\-/)&&(p.type.search[t+n]=p.type.search.html)})}P.extend(p.type.order,{"date-pre":function(t){t=Date.parse(t);return isNaN(t)?-1/0:t},"html-pre":function(t){return h(t)?"":t.replace?t.replace(/<.*?>/g,"").toLowerCase():t+""},"string-pre":function(t){return h(t)?"":"string"==typeof t?t.toLowerCase():t.toString?t.toString():""},"string-asc":function(t,e){return t<e?-1:e<t?1:0},"string-desc":function(t,e){return t<e?1:e<t?-1:0}}),Me(""),P.extend(!0,w.ext.renderer,{header:{_:function(r,o,i,l){P(r.nTable).on("order.dt.DT",function(t,e,n,a){r===e&&(e=i.idx,o.removeClass(l.sSortAsc+" "+l.sSortDesc).addClass("asc"==a[e]?l.sSortAsc:"desc"==a[e]?l.sSortDesc:i.sSortingClass))})},jqueryui:function(r,o,i,l){P("<div/>").addClass(l.sSortJUIWrapper).append(o.contents()).append(P("<span/>").addClass(l.sSortIcon+" "+i.sSortingClassJUI)).appendTo(o),P(r.nTable).on("order.dt.DT",function(t,e,n,a){r===e&&(e=i.idx,o.removeClass(l.sSortAsc+" "+l.sSortDesc).addClass("asc"==a[e]?l.sSortAsc:"desc"==a[e]?l.sSortDesc:i.sSortingClass),o.find("span."+l.sSortIcon).removeClass(l.sSortJUIAsc+" "+l.sSortJUIDesc+" "+l.sSortJUI+" "+l.sSortJUIAscAllowed+" "+l.sSortJUIDescAllowed).addClass("asc"==a[e]?l.sSortJUIAsc:"desc"==a[e]?l.sSortJUIDesc:i.sSortingClassJUI))})}}});function We(t){return"string"==typeof(t=Array.isArray(t)?t.join(","):t)?t.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;"):t}function Ee(t,e,n,a,r){return j.moment?t[e](r):j.luxon?t[n](r):a?t[a](r):t}var Be=!1;function Ue(t,e,n){var a;if(j.moment){if(!(a=j.moment.utc(t,e,n,!0)).isValid())return null}else if(j.luxon){if(!(a=e&&"string"==typeof t?j.luxon.DateTime.fromFormat(t,e):j.luxon.DateTime.fromISO(t)).isValid)return null;a.setLocale(n)}else e?(Be||alert("DataTables warning: Formatted date without Moment.js or Luxon - https://datatables.net/tn/17"),Be=!0):a=new Date(t);return a}function Ve(s){return function(a,r,o,i){0===arguments.length?(o="en",a=r=null):1===arguments.length?(o="en",r=a,a=null):2===arguments.length&&(o=r,r=a,a=null);var l="datetime-"+r;return w.ext.type.order[l]||(w.ext.type.detect.unshift(function(t){return t===l&&l}),w.ext.type.order[l+"-asc"]=function(t,e){t=t.valueOf(),e=e.valueOf();return t===e?0:t<e?-1:1},w.ext.type.order[l+"-desc"]=function(t,e){t=t.valueOf(),e=e.valueOf();return t===e?0:e<t?-1:1}),function(t,e){var n;return null!==t&&t!==H||(t="--now"===i?(n=new Date,new Date(Date.UTC(n.getFullYear(),n.getMonth(),n.getDate(),n.getHours(),n.getMinutes(),n.getSeconds()))):""),"type"===e?l:""===t?"sort"!==e?"":Ue("0000-01-01 00:00:00",null,o):!(null===r||a!==r||"sort"===e||"type"===e||t instanceof Date)||null===(n=Ue(t,a,o))?t:"sort"===e?n:(t=null===r?Ee(n,"toDate","toJSDate","")[s]():Ee(n,"format","toFormat","toISOString",r),"display"===e?We(t):t)}}}var Xe=",",Je=".";if(j.Intl!==H)try{for(var qe=(new Intl.NumberFormat).formatToParts(100000.1),n=0;n<qe.length;n++)"group"===qe[n].type?Xe=qe[n].value:"decimal"===qe[n].type&&(Je=qe[n].value)}catch(t){}function $e(e){return function(){var t=[ge(this[w.ext.iApiIndex])].concat(Array.prototype.slice.call(arguments));return w.ext.internal[e].apply(this,t)}}return w.datetime=function(n,a){var r="datetime-detect-"+n;a=a||"en",w.ext.type.order[r]||(w.ext.type.detect.unshift(function(t){var e=Ue(t,n,a);return!(""!==t&&!e)&&r}),w.ext.type.order[r+"-pre"]=function(t){return Ue(t,n,a)||0})},w.render={date:Ve("toLocaleDateString"),datetime:Ve("toLocaleString"),time:Ve("toLocaleTimeString"),number:function(a,r,o,i,l){return null!==a&&a!==H||(a=Xe),null!==r&&r!==H||(r=Je),{display:function(t){if("number"!=typeof t&&"string"!=typeof t)return t;if(""===t||null===t)return t;var e=t<0?"-":"",n=parseFloat(t);if(isNaN(n))return We(t);n=n.toFixed(o),t=Math.abs(n);n=parseInt(t,10),t=o?r+(t-n).toFixed(o).substring(2):"";return(e=0===n&&0===parseFloat(t)?"":e)+(i||"")+n.toString().replace(/\B(?=(\d{3})+(?!\d))/g,a)+t+(l||"")}}},text:function(){return{display:We,filter:We}}},P.extend(w.ext.internal,{_fnExternApiFunc:$e,_fnBuildAjax:Tt,_fnAjaxUpdate:xt,_fnAjaxParameters:At,_fnAjaxUpdateDraw:It,_fnAjaxDataSrc:Ft,_fnAddColumn:nt,_fnColumnOptions:at,_fnAdjustColumnSizing:O,_fnVisibleToColumnIndex:rt,_fnColumnIndexToVisible:ot,_fnVisbleColumns:T,_fnGetColumns:it,_fnColumnTypes:lt,_fnApplyColumnDefs:st,_fnHungarianMap:i,_fnCamelToHungarian:C,_fnLanguageCompat:Z,_fnBrowserDetect:tt,_fnAddData:x,_fnAddTr:ut,_fnNodeToDataIndex:function(t,e){return e._DT_RowIndex!==H?e._DT_RowIndex:null},_fnNodeToColumnIndex:function(t,e,n){return P.inArray(n,t.aoData[e].anCells)},_fnGetCellData:S,_fnSetCellData:ct,_fnSplitObjNotation:dt,_fnGetObjectDataFn:A,_fnSetObjectDataFn:b,_fnGetDataMaster:ht,_fnClearTable:pt,_fnDeleteIndex:gt,_fnInvalidate:bt,_fnGetRowElements:mt,_fnCreateTr:St,_fnBuildHead:yt,_fnDrawHead:Dt,_fnDraw:y,_fnReDraw:u,_fnAddOptionsHtml:_t,_fnDetectHeader:wt,_fnGetUniqueThs:Ct,_fnFeatureHtmlFilter:Lt,_fnFilterComplete:Rt,_fnFilterCustom:Pt,_fnFilterColumn:jt,_fnFilter:Ht,_fnFilterCreateSearch:Nt,_fnEscapeRegex:Ot,_fnFilterData:Wt,_fnFeatureHtmlInfo:Ut,_fnUpdateInfo:Vt,_fnInfoMacros:Xt,_fnInitialise:Jt,_fnInitComplete:qt,_fnLengthChange:$t,_fnFeatureHtmlLength:Gt,_fnFeatureHtmlPaginate:zt,_fnPageChange:Yt,_fnFeatureHtmlProcessing:Zt,_fnProcessingDisplay:D,_fnFeatureHtmlTable:Kt,_fnScrollDraw:Qt,_fnApplyToChildren:k,_fnCalculateColumnWidths:ee,_fnThrottle:ne,_fnConvertToWidth:ae,_fnGetWidestNode:re,_fnGetMaxLenString:oe,_fnStringToCss:M,_fnSortFlatten:I,_fnSort:ie,_fnSortAria:le,_fnSortListener:se,_fnSortAttachListener:ue,_fnSortingClasses:ce,_fnSortData:fe,_fnSaveState:de,_fnLoadState:he,_fnImplementState:pe,_fnSettingsFromNode:ge,_fnLog:W,_fnMap:F,_fnBindAction:me,_fnCallbackReg:L,_fnCallbackFire:R,_fnLengthOverflow:Se,_fnRenderer:ve,_fnDataSource:E,_fnRowAttributes:vt,_fnExtend:be,_fnCalculateEnd:function(){}}),((P.fn.dataTable=w).$=P).fn.dataTableSettings=w.settings,P.fn.dataTableExt=w.ext,P.fn.DataTable=function(t){return P(this).dataTable(t).api()},P.each(w,function(t,e){P.fn.DataTable[t]=e}),w});
\ No newline at end of file
diff --git a/bitbake/lib/toaster/toastergui/static/js/libtoaster.js b/bitbake/lib/toaster/toastergui/static/js/libtoaster.js
index f2c45c833e..d4ac31234c 100644
--- a/bitbake/lib/toaster/toastergui/static/js/libtoaster.js
+++ b/bitbake/lib/toaster/toastergui/static/js/libtoaster.js
@@ -657,7 +657,7 @@ $(document).ready(function() {
     hljs.initHighlightingOnLoad();
 
     // Prevent invalid links from jumping page scroll
-    $('a[href=#]').click(function() {
+    $('a[href="#"]').click(function() {
         return false;
     });
 
diff --git a/bitbake/lib/toaster/toastergui/static/js/projectpage.js b/bitbake/lib/toaster/toastergui/static/js/projectpage.js
index 506471e091..a3c95810a7 100644
--- a/bitbake/lib/toaster/toastergui/static/js/projectpage.js
+++ b/bitbake/lib/toaster/toastergui/static/js/projectpage.js
@@ -61,7 +61,7 @@ function projectPageInit(ctx) {
         distroChangeInput.val(urlParams.setDistro);
         distroChangeBtn.click();
     } else {
-      updateDistroName(prjInfo.distro.name);
+      updateDistroName(prjInfo.distro?.name);
     }
 
    /* Now we're really ready show the page */
diff --git a/bitbake/lib/toaster/toastergui/templates/base.html b/bitbake/lib/toaster/toastergui/templates/base.html
index 9e19cc33ca..e90be69620 100644
--- a/bitbake/lib/toaster/toastergui/templates/base.html
+++ b/bitbake/lib/toaster/toastergui/templates/base.html
@@ -14,11 +14,11 @@
 
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <meta http-equiv="Content-Type" content="text/html;charset=UTF-8" />
-    <script src="{% static 'js/jquery-2.0.3.min.js' %}">
+    <script src="{% static 'js/jquery-3.7.1.min.js' %}">
     </script>
     <script src="{% static 'js/jquery.cookie.js' %}">
     </script>
-    <script src="{% static 'js/bootstrap.min.js' %}">
+    <script src="{% static 'js/bootstrap-3.4.1.min.js' %}">
     </script>
     <script src="{% static 'js/typeahead.jquery.js' %}">
     </script>
@@ -94,7 +94,7 @@
             </a>
             <a class="brand" href="/">Toaster</a>
             {% if DEBUG %}
-            <span class="glyphicon glyphicon-info-sign" title="<strong>Toaster version information</strong>" data-content="<dl><dt>Git branch</dt><dd>{{TOASTER_BRANCH}}</dd><dt>Git revision</dt><dd>{{TOASTER_REVISION}}</dd></dl>"></i>
+            <span id="toaster-version-info-sign" class="glyphicon glyphicon-info-sign" title="<strong>Toaster version information</strong>" data-content="<dl><dt>Git branch</dt><dd>{{TOASTER_BRANCH}}</dd><dt>Git revision</dt><dd>{{TOASTER_REVISION}}</dd></dl>"></i>
             {% endif %}
           </div>
         </div>
@@ -123,7 +123,7 @@
               {% endif %}
             {% endif %}
               <li id="navbar-docs">
-              <a target="_blank" href="https://www.yoctoproject.org/docs/latest/toaster-manual/toaster-manual.html">
+              <a target="_blank" href="http://docs.yoctoproject.org/toaster-manual/index.html#toaster-user-manual">
                 <i class="glyphicon glyphicon-book"></i>
                 Documentation
               </a>
@@ -132,7 +132,8 @@
             {% if project_enable %}
             <a class="btn btn-default navbar-btn navbar-right" id="new-project-button" href="{% url 'newproject' %}">New project</a>
             {% endif %}
-          </div>
+            <a class="btn btn-default navbar-btn navbar-right" id="import_page" style="margin-right: 5px !important" id="import-cmdline-button" href="{% url 'cmdlines' %}">Import command line builds</a>
+            </div>
       </div>
     </nav>
 
diff --git a/bitbake/lib/toaster/toastergui/templates/base_specific.html b/bitbake/lib/toaster/toastergui/templates/base_specific.html
index e377cadd73..425f7ed73d 100644
--- a/bitbake/lib/toaster/toastergui/templates/base_specific.html
+++ b/bitbake/lib/toaster/toastergui/templates/base_specific.html
@@ -14,11 +14,11 @@
 
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <meta http-equiv="Content-Type" content="text/html;charset=UTF-8" />
-    <script src="{% static 'js/jquery-2.0.3.min.js' %}">
+    <script src="{% static 'js/jquery-3.7.1.min.js' %}">
     </script>
     <script src="{% static 'js/jquery.cookie.js' %}">
     </script>
-    <script src="{% static 'js/bootstrap.min.js' %}">
+    <script src="{% static 'js/bootstrap-3.4.1.min.js' %}">
     </script>
     <script src="{% static 'js/typeahead.jquery.js' %}">
     </script>
diff --git a/bitbake/lib/toaster/toastergui/templates/command_line_builds.html b/bitbake/lib/toaster/toastergui/templates/command_line_builds.html
new file mode 100644
index 0000000000..05db6727e7
--- /dev/null
+++ b/bitbake/lib/toaster/toastergui/templates/command_line_builds.html
@@ -0,0 +1,209 @@
+{% extends "base.html" %}
+{% load projecttags %}
+{% load humanize %}
+{% load static %}
+
+{% block title %} Import Builds from eventlogs - Toaster {% endblock %}
+
+{% block pagecontent %}
+
+<div class="container-fluid">
+    <div id="overlay" class="hide">
+        <div class="spinner">
+            <div class="fa-spin">
+            </div>
+        </div>
+    </div>
+    <div class="row">
+        <div class="col-md-12">
+            <div class="page-header">
+                <div class="row">
+                    <div class="col-md-6">
+                        <h1>Import command line builds</h1>
+                    </div>
+                    {% if import_all %}
+                    <div class="col-md-6">
+                        <button id="import_all" type="button" class="btn btn-primary navbar-btn navbar-right">
+                            <span class="glyphicon glyphicon-upload" style="vertical-align: top;"></span> Import All
+                        </button>
+                    </div>
+                    {% endif %}
+                </div>
+            </div>
+            {% if messages %}
+            <div class="row-fluid" id="empty-state-{{table_name}}">
+                {% for message in messages %}
+                <div class="alert alert-danger">{{message}}</div>
+                {%endfor%}
+            </div>
+            {% endif %}
+            <div class="row">
+                <h4 style="margin-left: 15px;"><strong>Import eventlog file</strong></h4>
+                <form method="POST" enctype="multipart/form-data" action="{% url 'cmdlines' %}" id="form_file">
+                    {% csrf_token %}
+                    <div class="col-md-6" style="padding-left: 20px;">
+                        <div class="row">
+                            <input type="hidden" value="{{dir}}" name="dir">
+                            <div class="col-md-3"> {{ form.eventlog_file}}  </div>
+                        </div>
+                        <div class="row" style="padding-top: 10px;">
+                            <div class="col-md-6">
+                                <button id="file_import" type="submit" disabled="disabled" class="btn btn-default navbar-btn" >
+                                    <span class="glyphicon glyphicon-upload" style="vertical-align: top;"></span> Import
+                                </button>
+                            </div>
+                        </div>
+                    </div>
+                </form>
+            </div>
+
+            <div class="row" style="padding-top: 20px;">
+                <div class="col-md-8 ">
+                    <h4><strong>Eventlogs from existing build directory: </strong>
+                        <a href="#" data-toggle="tooltip" title="{{dir}}">
+                            <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-info-circle" viewBox="0 0 16 16" data-toggle="tooltip">
+                                <path d="M8 15A7 7 0 1 1 8 1a7 7 0 0 1 0 14m0 1A8 8 0 1 0 8 0a8 8 0 0 0 0 16"/>
+                                <path d="m8.93 6.588-2.29.287-.082.38.45.083c.294.07.352.176.288.469l-.738 3.468c-.194.897.105 1.319.808 1.319.545 0 1.178-.252 1.465-.598l.088-.416c-.2.176-.492.246-.686.246-.275 0-.375-.193-.304-.533zM9 4.5a1 1 0 1 1-2 0 1 1 0 0 1 2 0"/>
+                            </svg>
+                        </a>
+                    </h4>
+                    {% if files %}
+                    <div class="table-responsive">
+                        <table class="table col-md-6 table-bordered table-hover" id="eventlog-table" style="border-collapse: collapse;">
+                            <thead>
+                            <tr class="row">
+                                <th scope="col">Name</th>
+                                <th scope="col">Size</th>
+                                <th scope="col">Action</th>
+                            </tr>
+                            </thead>
+                            <tbody>
+                                {% for file in files %}
+                                <tr class="row" style="height: 48px;">
+                                    <th scope="row" class="col-md-4" style="vertical-align: middle;">
+                                        <input type="hidden" value="{{file.name}}" name="{{file.name}}">{{file.name}}
+                                    </th>
+                                    <td class="col-md-4 align-middle" style="vertical-align: middle;">{{file.size|filesizeformat}}</td>
+                                    <td class="col-md-4 align-middle" style="vertical-align: middle;">
+                                        {% if file.imported == True and file.build_id is not None %}
+                                            <a href="{% url 'builddashboard' file.build_id %}">Build Details</a>
+                                        {% elif request.session.file == file.name or request.session.all_builds %}
+                                            <a data-toggle="tooltip" title="Build in progress">
+                                                <span class="glyphicon glyphicon-upload" style="font-size: 18px; color:grey"></span>
+                                            </a>
+                                        {%else%}
+                                            <a onclick="_ajax_update('{{file.name}}', false, '{{dir}}')" data-toggle="tooltip" title="Import File">
+                                                <span class="glyphicon glyphicon-upload" style="font-size: 18px;"></span>
+                                            </a>
+                                        {%endif%}
+                                    </td>
+                                </tr>
+                                {% endfor%}
+                            </tbody>
+                        </table>
+                    </div>
+                    {% else %}
+                    <div class="row-fluid" id="empty-state-{{table_name}}">
+                        <div class="alert alert-info">Sorry - no files found</div>
+                    </div>
+                    {%endif%}
+                </div>
+            </div>
+        </div>
+    </div>
+</div>
+
+<link rel="stylesheet" href="{% static 'css/jquery.dataTables-1.13.8.min.css' %}" type='text/css'/>
+<script src="{% static 'js/jquery.dataTables-1.13.8.min.js' %}"> </script>
+<script>
+
+function _ajax_update(file, all, dir){
+    function getCookie(name) {
+        var cookieValue = null;
+        if (document.cookie && document.cookie !== '') {
+            var cookies = document.cookie.split(';');
+            for (var i = 0; i < cookies.length; i++) {
+                var cookie = jQuery.trim(cookies[i]);
+                // Does this cookie string begin with the name we want?
+                if (cookie.substring(0, name.length + 1) === (name + '=')) {
+                    cookieValue = decodeURIComponent(cookie.substring(name.length + 1));
+                    break;
+                }
+            }
+        }
+    return cookieValue;
+    }
+    var csrftoken = getCookie('csrftoken');
+
+    function csrfSafeMethod(method) {
+        // these HTTP methods do not require CSRF protection
+        return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method));
+    }
+    $.ajaxSetup({
+        beforeSend: function (xhr, settings) {
+            if (!csrfSafeMethod(settings.type) && !this.crossDomain) {
+                xhr.setRequestHeader("X-CSRFToken", csrftoken);
+            }
+        }
+    });
+
+    $.ajax({
+        url:'/toastergui/cmdline/',
+        type: "POST",
+        data: {file: file, all: all, dir: dir},
+        success:function(data){
+            if (data['response']=='building'){
+                location.reload()
+            } else {
+                window.location = '/toastergui/builds/'
+            }
+        },
+        complete:function(data){
+        },
+        error:function (xhr, textStatus, thrownError){
+            console.log('fail');
+        }
+    });
+}
+
+$('#import_all').on('click', function(){
+    _ajax_update("{{files | safe}}", true, "{{dir | safe}}");
+});
+
+
+$('#import_page').hide();
+
+$(function () {
+  $('[data-toggle="tooltip"]').tooltip()
+})
+
+
+$("#id_eventlog_file").change(function(){
+    $('#file_import').prop("disabled", false);
+    $('#file_import').addClass('btn-primary')
+    $('#file_import').removeClass('btn-default')
+})
+
+$(document).ajaxStart(function(){
+    $('#overlay').removeClass('hide');
+    window.setTimeout(
+        function() {
+            window.location = '/toastergui/builds/'
+        }, 10000)
+});
+
+$( "#form_file").on( "submit", function( event ) {
+    $('#overlay').removeClass('hide');
+    window.setTimeout(
+        function() {
+            window.location = '/toastergui/builds/'
+        }, 10000)
+});
+
+$(document).ready( function () {
+    $('#eventlog-table').DataTable({order: [[0, 'desc']], "pageLength": 50});
+});
+
+</script>
+
+{% endblock %}
diff --git a/bitbake/lib/toaster/toastergui/templates/configvars.html b/bitbake/lib/toaster/toastergui/templates/configvars.html
index 33fef9316d..691dace3a2 100644
--- a/bitbake/lib/toaster/toastergui/templates/configvars.html
+++ b/bitbake/lib/toaster/toastergui/templates/configvars.html
@@ -66,7 +66,7 @@
         <td class="description">
             {% if variable.description %}
                 {{variable.description}}
-                <a href="https://www.yoctoproject.org/docs/current/ref-manual/ref-manual.html#var-{{variable.variable_name|variable_parent_name}}" target="_blank">
+                <a href="http://docs.yoctoproject.org/ref-manual/variables.html#term-{{variable.variable_name|variable_parent_name}}" target="_blank">
                 <span class="glyphicon glyphicon-new-window get-info"></span></a>
             {% endif %}
         </td>
diff --git a/bitbake/lib/toaster/toastergui/templates/js-unit-tests.html b/bitbake/lib/toaster/toastergui/templates/js-unit-tests.html
index ca248962f0..41553c4f9d 100644
--- a/bitbake/lib/toaster/toastergui/templates/js-unit-tests.html
+++ b/bitbake/lib/toaster/toastergui/templates/js-unit-tests.html
@@ -11,7 +11,7 @@
 <script src="{% static 'js/layerDepsModal.js' %}"></script>
 <script src="{% static 'js/projectpage.js' %}"></script>
 
-<script src="{% static 'js/bootstrap.min.js' %}"></script>
+<script src="{% static 'js/bootstrap-3.4.1.min.js' %}"></script>
 <script src="{% static 'js/filtersnippet.js' %}"></script>
 <script src="{% static 'js/importlayer.js' %}"></script>
 <script src="{% static 'js/highlight.pack.js' %}"></script>
diff --git a/bitbake/lib/toaster/toastergui/templates/landing.html b/bitbake/lib/toaster/toastergui/templates/landing.html
index bfaaf6fc83..589ee22634 100644
--- a/bitbake/lib/toaster/toastergui/templates/landing.html
+++ b/bitbake/lib/toaster/toastergui/templates/landing.html
@@ -12,10 +12,10 @@
             <div class="col-md-6">
               <h1>This is Toaster</h1>
 
-              <p>A web interface to <a href="https://www.openembedded.org">OpenEmbedded</a> and <a href="https://www.yoctoproject.org/tools-resources/projects/bitbake">BitBake</a>, the <a href="https://www.yoctoproject.org">Yocto Project</a> build system.</p>
+              <p>A web interface to <a href="https://www.openembedded.org">OpenEmbedded</a> and <a href="https://docs.yoctoproject.org/bitbake.html">BitBake</a>, the <a href="https://www.yoctoproject.org">Yocto Project</a> build system.</p>
 
                           <p class="top-air">
-                            <a class="btn btn-info btn-lg" href="https://www.yoctoproject.org/docs/latest/toaster-manual/toaster-manual.html#toaster-manual-setup-and-use">
+                            <a class="btn btn-info btn-lg" href="http://docs.yoctoproject.org/toaster-manual/setup-and-use.html#setting-up-and-using-toaster" style="min-width: 460px;">
                                     Toaster is ready to capture your command line builds
                             </a>
                           </p>
@@ -23,7 +23,7 @@
                           {% if lvs_nos %}
                     {% if project_enable %}
                             <p class="top-air">
-                              <a class="btn btn-primary btn-lg" href="{% url 'newproject' %}">
+                              <a class="btn btn-primary btn-lg" href="{% url 'newproject' %}" style="min-width: 460px;">
                                       Create your first Toaster project to run manage builds
                               </a>
                             </p>
@@ -33,7 +33,7 @@
                   Toaster has no layer information. Without layer information, you cannot run builds. To generate layer information you can:
                   <ul>
                     <li>
-                      <a href="https://www.yoctoproject.org/docs/latest/toaster-manual/toaster-manual.html#layer-source">Configure a layer source</a>
+                      <a href="http://docs.yoctoproject.org/toaster-manual/reference.html#layer-source">Configure a layer source</a>
                     </li>
                     <li>
                                         <a href="{% url 'newproject' %}">Create a project</a>, then import layers
@@ -42,9 +42,15 @@
                 </div>
               {% endif %}
 
+              <p class="top-air">
+                            <a class="btn btn-info btn-lg" href="{% url 'cmdlines' %}" style="min-width: 460px;">
+                                    Import command line event logs from build directory
+                            </a>
+                          </p>
+
               <ul class="list-unstyled lead">
                 <li>
-                  <a href="https://www.yoctoproject.org/docs/latest/toaster-manual/toaster-manual.html">
+                  <a href="http://docs.yoctoproject.org/toaster-manual/index.html#toaster-user-manual">
                     Read the Toaster manual
                   </a>
                 </li>
diff --git a/bitbake/lib/toaster/toastergui/templates/landing_not_managed.html b/bitbake/lib/toaster/toastergui/templates/landing_not_managed.html
deleted file mode 100644
index e7200b8412..0000000000
--- a/bitbake/lib/toaster/toastergui/templates/landing_not_managed.html
+++ /dev/null
@@ -1,34 +0,0 @@
-{% extends "base.html" %}
-
-{% load static %}
-{% load projecttags %}
-{% load humanize %}
-
-{% block title %} Welcome to Toaster {% endblock %}
-
-{% block pagecontent %}
-
-  <div class="container">
-   <div class="row">
-    <!-- Empty - no build module -->
-    <div class="page-header top-air">
-     <h1>
-       This page only works with Toaster in 'Build' mode
-     </h1>
-    </div>
-    <div class="alert alert-info lead">
-     <p">
-     The 'Build' mode allows you to configure and run your Yocto Project builds from Toaster.
-     <ul>
-       <li><a href="https://www.yoctoproject.org/docs/latest/toaster-manual/toaster-manual.html#intro-modes">
-       Read about the 'Build' mode
-       </a></li>
-       <li><a href="/">
-       View your builds
-       </a></li>
-     </ul>
-     </p>
-    </div>
-   </div>
-
-{% endblock %}
diff --git a/bitbake/lib/toaster/toastergui/templates/layerdetails.html b/bitbake/lib/toaster/toastergui/templates/layerdetails.html
index 1e26e31c8b..923ca3bfe4 100644
--- a/bitbake/lib/toaster/toastergui/templates/layerdetails.html
+++ b/bitbake/lib/toaster/toastergui/templates/layerdetails.html
@@ -355,7 +355,7 @@
             {% if layerversion.layer_source == layer_source.TYPE_LAYERINDEX  %}
             <dt>Layer index</dt>
             <dd>
-            <a href="http://layers.openembedded.org/layerindex/branch/{{layerversion.release.name}}/layer/{{layerversion.layer.name}}">Layer index {{layerversion.layer.name}}</a>
+            <a href="https://layers.openembedded.org/layerindex/branch/{{layerversion.release.name}}/layer/{{layerversion.layer.name}}">Layer index {{layerversion.layer.name}}</a>
             </dd>
             {% endif %}
           </dl>
diff --git a/bitbake/lib/toaster/toastergui/templates/mrb_section.html b/bitbake/lib/toaster/toastergui/templates/mrb_section.html
index 98d9fac822..9fc7dfaee4 100644
--- a/bitbake/lib/toaster/toastergui/templates/mrb_section.html
+++ b/bitbake/lib/toaster/toastergui/templates/mrb_section.html
@@ -63,7 +63,7 @@
     <%/if%>
   </div>
 
-  <div data-build-state="<%:state%>">
+  <div class="build-state" data-build-state="<%:state%>">
     <%if state == 'Cloning'%>
       <%include tmpl='#cloning-repos-build-template'/%>
     <%else state == 'Parsing'%>
diff --git a/bitbake/lib/toaster/toastergui/templates/package_built_dependencies.html b/bitbake/lib/toaster/toastergui/templates/package_built_dependencies.html
index a5d5893571..2493954deb 100644
--- a/bitbake/lib/toaster/toastergui/templates/package_built_dependencies.html
+++ b/bitbake/lib/toaster/toastergui/templates/package_built_dependencies.html
@@ -18,7 +18,7 @@
         </ul>
         <div class="tab-content">
             <div class="tab-pane active" id="dependencies">
-            {% ifequal runtime_deps|length 0 %}
+            {% if runtime_deps|length == 0 %}
                 <div class="alert alert-info">
                     <strong>{{package.fullpackagespec}}</strong> has no runtime dependencies.
                 </div>
@@ -54,8 +54,8 @@
                                                 {% endfor %}
                     </tbody>
                 </table>
-            {% endifequal %}
-            {% ifnotequal other_deps|length 0 %}
+            {% endif %}
+            {% if other_deps|length != 0 %}
                 <h3>Other runtime relationships</h3>
                 <table class="table table-bordered table-hover">
                     <thead>
@@ -93,7 +93,7 @@
                         {% endfor %}
                     </tbody>
                 </table>
-                {% endifnotequal %}
+                {% endif %}
             </div> <!-- tab-pane -->
         </div> <!-- tab-content -->
 {% endblock tabcontent %}
diff --git a/bitbake/lib/toaster/toastergui/templates/package_detail_base.html b/bitbake/lib/toaster/toastergui/templates/package_detail_base.html
index 66f8e7f069..a4fcd2aa42 100644
--- a/bitbake/lib/toaster/toastergui/templates/package_detail_base.html
+++ b/bitbake/lib/toaster/toastergui/templates/package_detail_base.html
@@ -127,7 +127,7 @@
                     {% comment %}
                     # Removed per team meeting of 1/29/2014 until
                     # decision on index search algorithm
-                    <a href="http://layers.openembedded.org"  target="_blank">
+                    <a href="https://layers.openembedded.org"  target="_blank">
                     <i class="glyphicon glyphicon-share get-info"></i>
                     </a>
                     {% endcomment %}
diff --git a/bitbake/lib/toaster/toastergui/templates/package_included_dependencies.html b/bitbake/lib/toaster/toastergui/templates/package_included_dependencies.html
index 95e56ded26..1f5ed6d913 100644
--- a/bitbake/lib/toaster/toastergui/templates/package_included_dependencies.html
+++ b/bitbake/lib/toaster/toastergui/templates/package_included_dependencies.html
@@ -14,7 +14,7 @@
     {% include "package_included_tabs.html" with active_tab="dependencies" %}
     <div class="tab-content">
        <div class="tab-pane active" id="dependencies">
-       {% ifnotequal runtime_deps|length 0 %}
+       {% if runtime_deps|length != 0 %}
             <table class="table table-bordered table-hover">
                 <thead>
                     <tr>
@@ -48,9 +48,9 @@
             <div class="alert alert-info">
                 <strong>{{package.fullpackagespec}}</strong> has no runtime dependencies.
             </div>
-        {% endifnotequal %}
+        {% endif %}
 
-        {% ifnotequal other_deps|length 0 %}
+        {% if other_deps|length != 0 %}
             <h3>Other runtime relationships</h3>
             <table class="table table-bordered table-hover">
                 <thead>
@@ -103,7 +103,7 @@
                         {% endfor %}
                 </tbody>
             </table>
-        {% endifnotequal %}
+        {% endif %}
         </div> <!-- end tab-pane -->
     </div> <!-- end tab content -->
     {% endwith %}
diff --git a/bitbake/lib/toaster/toastergui/templates/package_included_reverse_dependencies.html b/bitbake/lib/toaster/toastergui/templates/package_included_reverse_dependencies.html
index fb310c7fc7..dae4549e21 100644
--- a/bitbake/lib/toaster/toastergui/templates/package_included_reverse_dependencies.html
+++ b/bitbake/lib/toaster/toastergui/templates/package_included_reverse_dependencies.html
@@ -15,7 +15,7 @@
     <div class="tab-content">
         <div class="tab-pane active" id="brought-in-by">
 
-        {% ifequal reverse_count 0 %}
+        {% if reverse_count == 0 %}
            <div class="alert alert-info">
                 <strong>{{package.fullpackagespec}}</strong> has no reverse runtime dependencies.
             </div>
@@ -43,7 +43,7 @@
                                         {% endfor %}
                 </tbody>
             </table>
-        {% endifequal %}
+        {% endif %}
         </div> <!-- end tab-pane -->
     </div> <!-- end tab content -->
     {% endwith %}
diff --git a/bitbake/lib/toaster/toastergui/templates/project.html b/bitbake/lib/toaster/toastergui/templates/project.html
index d8ad2c79dc..22239a82fd 100644
--- a/bitbake/lib/toaster/toastergui/templates/project.html
+++ b/bitbake/lib/toaster/toastergui/templates/project.html
@@ -139,7 +139,7 @@
         <ul>
           <li><a href="{% url 'projectlayers' project.id %}">Choose from the layers compatible with this project</a></li>
           <li><a href="{% url 'importlayer' project.id %}">Import a layer</a></li>
-          <li><a href="https://www.yoctoproject.org/docs/current/dev-manual/dev-manual.html#understanding-and-creating-layers" target="_blank">Read about layers in the documentation</a></li>
+          <li><a href="http://docs.yoctoproject.org/dev-manual/common-tasks.html#understanding-and-creating-layers" target="_blank">Read about layers in the documentation</a></li>
           <li>Or type a layer name below</li>
         </ul>
       </div>
diff --git a/bitbake/lib/toaster/toastergui/templates/project_specific.html b/bitbake/lib/toaster/toastergui/templates/project_specific.html
index 42725c0dba..76d45b1b39 100644
--- a/bitbake/lib/toaster/toastergui/templates/project_specific.html
+++ b/bitbake/lib/toaster/toastergui/templates/project_specific.html
@@ -137,7 +137,7 @@
         <ul>
           <li><a href="{% url 'projectlayers' project.id %}">Choose from the layers compatible with this project</a></li>
           <li><a href="{% url 'importlayer' project.id %}">Import a layer</a></li>
-          <li><a href="https://www.yoctoproject.org/docs/current/dev-manual/dev-manual.html#understanding-and-creating-layers" target="_blank">Read about layers in the documentation</a></li>
+          <li><a href="http://docs.yoctoproject.org/dev-manual/common-tasks.html#understanding-and-creating-layers" target="_blank">Read about layers in the documentation</a></li>
           <li>Or type a layer name below</li>
         </ul>
       </div>
diff --git a/bitbake/lib/toaster/toastergui/templates/projectconf.html b/bitbake/lib/toaster/toastergui/templates/projectconf.html
index bd49f1f585..c306835832 100644
--- a/bitbake/lib/toaster/toastergui/templates/projectconf.html
+++ b/bitbake/lib/toaster/toastergui/templates/projectconf.html
@@ -73,7 +73,7 @@
 
     {% if image_install_append_defined %}
     <dt>
-    <span class="js-config-var-name js-config-var-managed-name">IMAGE_INSTALL_append</span>
+    <span class="js-config-var-name js-config-var-managed-name">IMAGE_INSTALL:append</span>
     <span class="glyphicon glyphicon-question-sign get-help" title="Specifies additional packages to install into an image. If your build creates more than one image, the packages will be installed in all of them"></span>
     </dt>
     <dd class="variable-list">
@@ -83,7 +83,7 @@
     <form id="change-image_install-form" class="form-inline" style="display:none;">
       <div class="row">
         <div class="col-md-4">
-          <span class="help-block">To set IMAGE_INSTALL_append to more than one package, type the package names separated by a space.</span>
+          <span class="help-block">To set IMAGE_INSTALL:append to more than one package, type the package names separated by a space.</span>
         </div>
       </div>
       <div class="form-group">
@@ -167,8 +167,8 @@
   {% for fstype in vars_fstypes %}
   <input type="hidden" class="js-checkbox-fstypes-list" value="{{fstype}}">
   {% endfor %}
-  {% for b in vars_blacklist %}
-  <input type="hidden" class="js-config-blacklist-name" value="{{b}}">
+  {% for b in vars_disallowed %}
+  <input type="hidden" class="js-config-disallowed-name" value="{{b}}">
   {% endfor %}
   {% for b in vars_managed %}
   <input type="hidden" class="js-config-managed-name" value="{{b}}">
@@ -201,12 +201,12 @@
           <p>Toaster cannot set any variables that impact 1) the configuration of the build servers,
           or 2) where artifacts produced by the build are stored. Such variables include: </p>
           <p>
-          <code><a href="https://www.yoctoproject.org/docs/current/ref-manual/ref-manual.html#var-BB_DISKMON_DIRS" target="_blank">BB_DISKMON_DIRS</a></code>
-          <code><a href="https://www.yoctoproject.org/docs/current/ref-manual/ref-manual.html#var-BB_NUMBER_THREADS" target="_blank">BB_NUMBER_THREADS</a></code>
+          <code><a href="http://docs.yoctoproject.org/ref-manual/variables.html#term-BB_DISKMON_DIRS" target="_blank">BB_DISKMON_DIRS</a></code>
+          <code><a href="http://docs.yoctoproject.org/ref-manual/variables.html#term-BB_NUMBER_THREADS" target="_blank">BB_NUMBER_THREADS</a></code>
           <code>CVS_PROXY_HOST</code>
           <code>CVS_PROXY_PORT</code>
-          <code><a href="https://www.yoctoproject.org/docs/current/ref-manual/ref-manual.html#var-PARALLEL_MAKE" target="_blank">PARALLEL_MAKE</a></code>
-          <code><a href="https://www.yoctoproject.org/docs/current/ref-manual/ref-manual.html#var-TMPDIR" target="_blank">TMPDIR</a></code></p>
+          <code><a href="http://docs.yoctoproject.org/ref-manual/variables.html#term-PARALLEL_MAKE" target="_blank">PARALLEL_MAKE</a></code>
+          <code><a href="http://docs.yoctoproject.org/ref-manual/variables.html#term-TMPDIR" target="_blank">TMPDIR</a></code></p>
           <p>Plus the following standard shell environment variables:</p>
           <p><code>http_proxy</code> <code>ftp_proxy</code> <code>https_proxy</code> <code>all_proxy</code></p>
         </div>
@@ -238,9 +238,9 @@ function validate_new_variable() {
     }
   }
 
-  var blacklist_configvars = document.getElementsByClassName('js-config-blacklist-name');
-  for (var i = 0, length = blacklist_configvars.length; i < length; i++) {
-    if (blacklist_configvars[i].value.toUpperCase() == variable.toUpperCase()) {
+  var disallowed_configvars = document.getElementsByClassName('js-config-disallowed-name');
+  for (var i = 0, length = disallowed_configvars.length; i < length; i++) {
+    if (disallowed_configvars[i].value.toUpperCase() == variable.toUpperCase()) {
       error_msg = "You cannot edit this variable in Toaster because it is set by the build servers";
     }
   }
@@ -771,10 +771,10 @@ $(document).ready(function() {
 
   {% if image_install_append_defined %}
 
-  // init IMAGE_INSTALL_append trash icon
+  // init IMAGE_INSTALL:append trash icon
   setDeleteTooltip($('#delete-image_install-icon'));
 
-  // change IMAGE_INSTALL_append variable
+  // change IMAGE_INSTALL:append variable
   $('#change-image_install-icon').click(function() {
     // preset the edit value
     var current_val = $("span#image_install").text().trim();
@@ -814,7 +814,7 @@ $(document).ready(function() {
   $('#apply-change-image_install').click(function(){
     // insure these non-empty values have single space prefix
     var value = " " + $('#new-image_install').val().trim();
-    postEditAjaxRequest({"configvarChange" : 'IMAGE_INSTALL_append:'+value});
+    postEditAjaxRequest({"configvarChange" : 'IMAGE_INSTALL:append:'+value});
     $('#image_install').text(value);
     $('#image_install').removeClass('text-muted');
     $("#change-image_install-form").slideUp(function () {
@@ -826,10 +826,10 @@ $(document).ready(function() {
     });
   });
 
-  // delete IMAGE_INSTALL_append variable value
+  // delete IMAGE_INSTALL:append variable value
   $('#delete-image_install-icon').click(function(){
     $(this).tooltip('hide');
-    postEditAjaxRequest({"configvarChange" : 'IMAGE_INSTALL_append:'+''});
+    postEditAjaxRequest({"configvarChange" : 'IMAGE_INSTALL:append:'+''});
     $('#image_install').parent().fadeOut(1000, function(){
       $('#image_install').addClass('text-muted');
       $('#image_install').text('Not set');
@@ -1011,7 +1011,7 @@ $(document).ready(function() {
     $(".save").attr("disabled","disabled");
 
     // Reload page if admin-removed core managed value is manually added back in
-    if (0 <= " DISTRO DL_DIR IMAGE_FSTYPES IMAGE_INSTALL_append PACKAGE_CLASSES SSTATE_DIR ".indexOf( " "+variable+" " )) {
+    if (0 <= " DISTRO DL_DIR IMAGE_FSTYPES IMAGE_INSTALL:append PACKAGE_CLASSES SSTATE_DIR ".indexOf( " "+variable+" " )) {
       // delayed reload to avoid race condition with postEditAjaxRequest
       do_reload=true;
     }
diff --git a/bitbake/lib/toaster/toastergui/templates/recipe.html b/bitbake/lib/toaster/toastergui/templates/recipe.html
index 3f76e656fe..4b5301b548 100644
--- a/bitbake/lib/toaster/toastergui/templates/recipe.html
+++ b/bitbake/lib/toaster/toastergui/templates/recipe.html
@@ -186,9 +186,9 @@
                         <i class="icon-question-sign get-help hover-help" title="{{task.get_outcome_help}}"></i>
                     </td>
                     <td>
-                        {% ifnotequal task.sstate_result task.SSTATE_NA %}
+                        {% if task.sstate_result != task.SSTATE_NA %}
                             {{task.get_sstate_result_display}}
-                        {% endifnotequal %}
+                        {% endif %}
                     </td>
 
                     </tr>
diff --git a/bitbake/lib/toaster/toastergui/templates/target.html b/bitbake/lib/toaster/toastergui/templates/target.html
index 1924a0dad7..d5f60e77a8 100644
--- a/bitbake/lib/toaster/toastergui/templates/target.html
+++ b/bitbake/lib/toaster/toastergui/templates/target.html
@@ -8,11 +8,11 @@
 
 {% block nav-target %}
   {% for t in build.get_sorted_target_list %}
-    {% ifequal target.pk t.pk %}
+    {% if target.pk == t.pk %}
       <li class="active"><a href="{% url 'target' build.pk t.pk %}">{{t.target}}</a><li>
     {% else %}
       <li><a href="{% url 'target' build.pk t.pk %}">{{t.target}}</a><li>
-    {% endifequal %}
+    {% endif %}
   {% endfor %}
 {% endblock %}
 
diff --git a/bitbake/lib/toaster/toastergui/templatetags/projecttags.py b/bitbake/lib/toaster/toastergui/templatetags/projecttags.py
index c432f59a78..aee9bbcd14 100644
--- a/bitbake/lib/toaster/toastergui/templatetags/projecttags.py
+++ b/bitbake/lib/toaster/toastergui/templatetags/projecttags.py
@@ -167,8 +167,8 @@ def check_filter_status(options, filter):
 def variable_parent_name(value):
     """ filter extended variable names to the parent name
     """
-    value=re.sub('_\$.*', '', value)
-    return re.sub('_[a-z].*', '', value)
+    value = re.sub(r'_\$.*', '', value)
+    return re.sub(r'_[a-z].*', '', value)
 
 @register.filter
 def filter_setin_files(file_list, matchstr):
@@ -233,7 +233,6 @@ def filter_sizeovertotal(package_object, total_size):
 
     return '{:.1%}'.format(float(size)/float(total_size))
 
-from django.utils.safestring import mark_safe
 @register.filter
 def format_vpackage_rowclass(size):
     if size == -1:
diff --git a/bitbake/lib/toaster/toastergui/urls.py b/bitbake/lib/toaster/toastergui/urls.py
index d2df4e6048..7f8489d3aa 100644
--- a/bitbake/lib/toaster/toastergui/urls.py
+++ b/bitbake/lib/toaster/toastergui/urls.py
@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-from django.conf.urls import url
+from django.urls import re_path as url
 from django.views.generic import RedirectView
 
 from toastergui import tables
@@ -95,6 +95,7 @@ urlpatterns = [
         # project URLs
         url(r'^newproject/$', views.newproject, name='newproject'),
 
+        url(r'^cmdline/$', views.CommandLineBuilds.as_view(), name='cmdlines'),
         url(r'^projects/$',
             tables.ProjectsTable.as_view(template_name="projects-toastertable.html"),
             name='all-projects'),
@@ -206,8 +207,7 @@ urlpatterns = [
         url(r'^js-unit-tests/$', views.jsunittests, name='js-unit-tests'),
 
         # image customisation functionality
-        url(r'^xhr_customrecipe/(?P<recipe_id>\d+)'
-            '/packages/(?P<package_id>\d+|)$',
+        url(r'^xhr_customrecipe/(?P<recipe_id>\d+)/packages/(?P<package_id>\d+|)$',
             api.XhrCustomRecipePackages.as_view(),
             name='xhr_customrecipe_packages'),
 
diff --git a/bitbake/lib/toaster/toastergui/views.py b/bitbake/lib/toaster/toastergui/views.py
index 9a5e48e3bb..061e6436c8 100644
--- a/bitbake/lib/toaster/toastergui/views.py
+++ b/bitbake/lib/toaster/toastergui/views.py
@@ -6,24 +6,36 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import ast
 import re
+import subprocess
+import sys
+
+import bb.cooker
+from bb.ui import toasterui
+from bb.ui import eventreplay
 
 from django.db.models import F, Q, Sum
 from django.db import IntegrityError
-from django.shortcuts import render, redirect, get_object_or_404
+from django.shortcuts import render, redirect, get_object_or_404, HttpResponseRedirect
 from django.utils.http import urlencode
 from orm.models import Build, Target, Task, Layer, Layer_Version, Recipe
 from orm.models import LogMessage, Variable, Package_Dependency, Package
 from orm.models import Task_Dependency, Package_File
 from orm.models import Target_Installed_Package, Target_File
 from orm.models import TargetKernelFile, TargetSDKFile, Target_Image_File
-from orm.models import BitbakeVersion, CustomImageRecipe
+from orm.models import BitbakeVersion, CustomImageRecipe, EventLogsImports
 
 from django.urls import reverse, resolve
+from django.contrib import messages
+
 from django.core.exceptions import ObjectDoesNotExist
+from django.core.files.storage import FileSystemStorage
+from django.core.files.uploadedfile import InMemoryUploadedFile, TemporaryUploadedFile
 from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
 from django.http import HttpResponseNotFound, JsonResponse
 from django.utils import timezone
+from django.views.generic import TemplateView
 from datetime import timedelta, datetime
 from toastergui.templatetags.projecttags import json as jsonfilter
 from decimal import Decimal
@@ -32,13 +44,20 @@ import os
 from os.path import dirname
 import mimetypes
 
+from toastergui.forms import LoadFileForm
+
+from collections import namedtuple
+
 import logging
 
+from toastermain.logs import log_view_mixin
+
 logger = logging.getLogger("toaster")
 
 # Project creation and managed build enable
 project_enable = ('1' == os.environ.get('TOASTER_BUILDSERVER'))
 is_project_specific = ('1' == os.environ.get('TOASTER_PROJECTSPECIFIC'))
+import_page = False
 
 class MimeTypeFinder(object):
     # setting this to False enables additional non-standard mimetypes
@@ -56,6 +75,7 @@ class MimeTypeFinder(object):
         return guessed_type
 
 # single point to add global values into the context before rendering
+@log_view_mixin
 def toaster_render(request, page, context):
     context['project_enable'] = project_enable
     context['project_specific'] = is_project_specific
@@ -352,7 +372,6 @@ def _get_parameters_values(request, default_count, default_order):
 # set cookies for parameters. this is usefull in case parameters are set
 # manually from the GET values of the link
 def _set_parameters_values(pagesize, orderby, request):
-    from django.urls import resolve
     current_url = resolve(request.path_info).url_name
     request.session['%s_count' % current_url] = pagesize
     request.session['%s_orderby' % current_url] =orderby
@@ -665,20 +684,20 @@ def recipe_packages(request, build_id, recipe_id):
     return response
 
 from django.http import HttpResponse
+@log_view_mixin
 def xhr_dirinfo(request, build_id, target_id):
     top = request.GET.get('start', '/')
     return HttpResponse(_get_dir_entries(build_id, target_id, top), content_type = "application/json")
 
 from django.utils.functional import Promise
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 class LazyEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, Promise):
-            return force_text(obj)
+            return force_str(obj)
         return super(LazyEncoder, self).default(obj)
 
 from toastergui.templatetags.projecttags import filtered_filesizeformat
-import os
 def _get_dir_entries(build_id, target_id, start):
     node_str = {
         Target_File.ITYPE_REGULAR   : '-',
@@ -1404,7 +1423,7 @@ if True:
                     if not os.path.isdir('%s/conf' % request.POST['importdir']):
                         raise BadParameterException("Bad path or missing 'conf' directory (%s)" % request.POST['importdir'])
                     from django.core import management
-                    management.call_command('buildimport', '--command=import', '--name=%s' % request.POST['projectname'], '--path=%s' % request.POST['importdir'], interactive=False)
+                    management.call_command('buildimport', '--command=import', '--name=%s' % request.POST['projectname'], '--path=%s' % request.POST['importdir'])
                     prj = Project.objects.get(name = request.POST['projectname'])
                     prj.merged_attr = True
                     prj.save()
@@ -1606,12 +1625,13 @@ if True:
         # make sure we have a machine set for this project
         ProjectVariable.objects.get_or_create(project=new_project,
                                               name="MACHINE",
-                                              value="qemux86")
+                                              value="qemux86-64")
         context = {'project': new_project}
         return toaster_render(request, "js-unit-tests.html", context)
 
     from django.views.decorators.csrf import csrf_exempt
     @csrf_exempt
+    @log_view_mixin
     def xhr_testreleasechange(request, pid):
         def response(data):
             return HttpResponse(jsonfilter(data),
@@ -1648,6 +1668,7 @@ if True:
         except Exception as e:
             return response({"error": str(e) })
 
+    @log_view_mixin
     def xhr_configvaredit(request, pid):
         try:
             prj = Project.objects.get(id = pid)
@@ -1683,12 +1704,12 @@ if True:
                 t=request.POST['configvarDel'].strip()
                 pt = ProjectVariable.objects.get(pk = int(t)).delete()
 
-            # return all project settings, filter out blacklist and elsewhere-managed variables
-            vars_managed,vars_fstypes,vars_blacklist = get_project_configvars_context()
+            # return all project settings, filter out disallowed and elsewhere-managed variables
+            vars_managed,vars_fstypes,vars_disallowed = get_project_configvars_context()
             configvars_query = ProjectVariable.objects.filter(project_id = pid).all()
             for var in vars_managed:
                 configvars_query = configvars_query.exclude(name = var)
-            for var in vars_blacklist:
+            for var in vars_disallowed:
                 configvars_query = configvars_query.exclude(name = var)
 
             return_data = {
@@ -1708,7 +1729,7 @@ if True:
             except ProjectVariable.DoesNotExist:
                 pass
             try:
-                return_data['image_install_append'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL_append").value,
+                return_data['image_install:append'] = ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL:append").value,
             except ProjectVariable.DoesNotExist:
                 pass
             try:
@@ -1726,6 +1747,7 @@ if True:
             return HttpResponse(json.dumps({"error":str(e) + "\n" + traceback.format_exc()}), content_type = "application/json")
 
 
+    @log_view_mixin
     def customrecipe_download(request, pid, recipe_id):
         recipe = get_object_or_404(CustomImageRecipe, pk=recipe_id)
 
@@ -1781,7 +1803,7 @@ if True:
             'MACHINE', 'BBLAYERS'
         }
 
-        vars_blacklist  = {
+        vars_disallowed  = {
             'PARALLEL_MAKE','BB_NUMBER_THREADS',
             'BB_DISKMON_DIRS','BB_NUMBER_THREADS','CVS_PROXY_HOST','CVS_PROXY_PORT',
             'PARALLEL_MAKE','TMPDIR',
@@ -1790,7 +1812,7 @@ if True:
 
         vars_fstypes = Target_Image_File.SUFFIXES
 
-        return(vars_managed,sorted(vars_fstypes),vars_blacklist)
+        return(vars_managed,sorted(vars_fstypes),vars_disallowed)
 
     def projectconf(request, pid):
 
@@ -1799,12 +1821,12 @@ if True:
         except Project.DoesNotExist:
             return HttpResponseNotFound("<h1>Project id " + pid + " is unavailable</h1>")
 
-        # remove blacklist and externally managed varaibles from this list
-        vars_managed,vars_fstypes,vars_blacklist = get_project_configvars_context()
+        # remove disallowed and externally managed varaibles from this list
+        vars_managed,vars_fstypes,vars_disallowed = get_project_configvars_context()
         configvars = ProjectVariable.objects.filter(project_id = pid).all()
         for var in vars_managed:
             configvars = configvars.exclude(name = var)
-        for var in vars_blacklist:
+        for var in vars_disallowed:
             configvars = configvars.exclude(name = var)
 
         context = {
@@ -1812,7 +1834,7 @@ if True:
             'configvars':       configvars,
             'vars_managed':     vars_managed,
             'vars_fstypes':     vars_fstypes,
-            'vars_blacklist':   vars_blacklist,
+            'vars_disallowed':  vars_disallowed,
         }
 
         try:
@@ -1839,7 +1861,7 @@ if True:
         except ProjectVariable.DoesNotExist:
             pass
         try:
-            context['image_install_append'] =  ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL_append").value
+            context['image_install:append'] =  ProjectVariable.objects.get(project = prj, name = "IMAGE_INSTALL:append").value
             context['image_install_append_defined'] = "1"
         except ProjectVariable.DoesNotExist:
             pass
@@ -1933,3 +1955,163 @@ if True:
         except (ObjectDoesNotExist, IOError):
             return toaster_render(request, "unavailable_artifact.html")
 
+
+class CommandLineBuilds(TemplateView):
+    model = EventLogsImports
+    template_name = 'command_line_builds.html'
+
+    def get_context_data(self, **kwargs):
+        context = super(CommandLineBuilds, self).get_context_data(**kwargs)
+        #get value from BB_DEFAULT_EVENTLOG defined in bitbake.conf
+        eventlog = subprocess.check_output(['bitbake-getvar', 'BB_DEFAULT_EVENTLOG', '--value'])
+        if eventlog:
+            logs_dir = os.path.dirname(eventlog.decode().strip('\n'))
+            files = os.listdir(logs_dir)
+            imported_files = EventLogsImports.objects.all()
+            files_list = []
+
+            # Filter files that end with ".json"
+            event_files = []
+            for file in files:
+                if file.endswith(".json"):
+                    # because BB_DEFAULT_EVENTLOG is a directory, we need to check if the file is a valid eventlog
+                    with open("{}/{}".format(logs_dir, file)) as efile:
+                        content = efile.read()
+                        if 'allvariables' in content:
+                            event_files.append(file)
+
+            #build dict for template using db data
+            for event_file in event_files:
+                if imported_files.filter(name=event_file):
+                    files_list.append({
+                        'name': event_file,
+                        'imported': True,
+                        'build_id': imported_files.filter(name=event_file)[0].build_id,
+                        'size': os.path.getsize("{}/{}".format(logs_dir, event_file))
+                    })
+                else:
+                    files_list.append({
+                        'name': event_file,
+                        'imported': False,
+                        'build_id': None,
+                        'size': os.path.getsize("{}/{}".format(logs_dir, event_file))
+                    })
+                    context['import_all'] = True
+
+            context['files'] = files_list
+            context['dir'] = logs_dir
+        else:
+            context['files'] = []
+            context['dir'] = ''
+
+        # enable session variable
+        if not self.request.session.get('file'):
+            self.request.session['file'] = ""
+
+        context['form'] = LoadFileForm()
+        context['project_enable'] = project_enable
+        return context
+
+    def post(self, request, **kwargs):
+        logs_dir = request.POST.get('dir')
+        all_files =  request.POST.get('all')
+
+        # check if a build is already in progress
+        if Build.objects.filter(outcome=Build.IN_PROGRESS):
+            messages.add_message(
+                self.request,
+                messages.ERROR,
+                "A build is already in progress. Please wait for it to complete before starting a new build."
+            )
+            return JsonResponse({'response': 'building'})
+        imported_files = EventLogsImports.objects.all()
+        try:
+            if all_files == 'true':
+                # use of session variable to deactivate icon for builds in progress
+                request.session['all_builds'] = True
+                request.session.modified = True
+                request.session.save()
+
+                files = ast.literal_eval(request.POST.get('file'))
+                for file in files:
+                    if imported_files.filter(name=file.get('name')).exists():
+                        imported_files.filter(name=file.get('name'))[0].imported = True
+                    else:
+                        with open("{}/{}".format(logs_dir, file.get('name'))) as eventfile:
+                            # load variables from the first line
+                            variables = None
+                            while line := eventfile.readline().strip():
+                                try:
+                                    variables = json.loads(line)['allvariables']
+                                    break
+                                except (KeyError, json.JSONDecodeError):
+                                    continue
+                            if not variables:
+                                raise Exception("File content missing  build variables")
+                            eventfile.seek(0)
+                            params = namedtuple('ConfigParams', ['observe_only'])(True)
+                            player = eventreplay.EventPlayer(eventfile, variables)
+
+                            toasterui.main(player, player, params)
+                        event_log_import = EventLogsImports.objects.create(name=file.get('name'), imported=True)
+                        event_log_import.build_id = Build.objects.last().id
+                        event_log_import.save()
+            else:
+                if self.request.FILES.get('eventlog_file'):
+                    file = self.request.FILES['eventlog_file']
+                else:
+                    file = request.POST.get('file')
+                    # use of session variable to deactivate icon for build in progress
+                    request.session['file'] = file
+                    request.session['all_builds'] = False
+                    request.session.modified = True
+                    request.session.save()
+
+                if imported_files.filter(name=file).exists():
+                    imported_files.filter(name=file)[0].imported = True
+                else:
+                    if isinstance(file, InMemoryUploadedFile) or isinstance(file, TemporaryUploadedFile):
+                        variables = None
+                        while line := file.readline().strip():
+                            try:
+                                variables = json.loads(line)['allvariables']
+                                break
+                            except (KeyError, json.JSONDecodeError):
+                                continue
+                        if not variables:
+                            raise Exception("File content missing  build variables")
+                        file.seek(0)
+                        params = namedtuple('ConfigParams', ['observe_only'])(True)
+                        player = eventreplay.EventPlayer(file, variables)
+                        if not os.path.exists('{}/{}'.format(logs_dir, file.name)):
+                            fs = FileSystemStorage(location=logs_dir)
+                            fs.save(file.name, file)
+                        toasterui.main(player, player, params)
+                    else:
+                        with open("{}/{}".format(logs_dir, file)) as eventfile:
+                            # load variables from the first line
+                            variables = None
+                            while line := eventfile.readline().strip():
+                                try:
+                                    variables = json.loads(line)['allvariables']
+                                    break
+                                except (KeyError, json.JSONDecodeError):
+                                    continue
+                            if not variables:
+                                raise Exception("File content missing  build variables")
+                            eventfile.seek(0)
+                            params = namedtuple('ConfigParams', ['observe_only'])(True)
+                            player = eventreplay.EventPlayer(eventfile, variables)
+                            toasterui.main(player, player, params)
+                    event_log_import = EventLogsImports.objects.create(name=file, imported=True)
+                    event_log_import.build_id = Build.objects.last().id
+                    event_log_import.save()
+                    request.session['file'] = ""
+        except Exception:
+            messages.add_message(
+                self.request,
+                messages.ERROR,
+                "The file content is not in the correct format. Update file content or upload a different file."
+            )
+            return HttpResponseRedirect("/toastergui/cmdline/")
+        return HttpResponseRedirect('/toastergui/builds/')
diff --git a/bitbake/lib/toaster/toastergui/widgets.py b/bitbake/lib/toaster/toastergui/widgets.py
index ceff52942e..b32abf40b3 100644
--- a/bitbake/lib/toaster/toastergui/widgets.py
+++ b/bitbake/lib/toaster/toastergui/widgets.py
@@ -7,6 +7,7 @@
 #
 
 from django.views.generic import View, TemplateView
+from django.utils.decorators import method_decorator
 from django.views.decorators.cache import cache_control
 from django.shortcuts import HttpResponse
 from django.core.cache import cache
@@ -31,6 +32,7 @@ import re
 import os
 
 from toastergui.tablefilter import TableFilterMap
+from toastermain.logs import log_view_mixin
 
 try:
     from urllib import unquote_plus
@@ -63,8 +65,8 @@ class ToasterTable(TemplateView):
         self.default_orderby = ""
 
     # prevent HTTP caching of table data
-    @cache_control(must_revalidate=True,
-                   max_age=0, no_store=True, no_cache=True)
+    @method_decorator(cache_control(must_revalidate=True,
+                   max_age=0, no_store=True, no_cache=True))
     def dispatch(self, *args, **kwargs):
         return super(ToasterTable, self).dispatch(*args, **kwargs)
 
@@ -83,6 +85,7 @@ class ToasterTable(TemplateView):
 
         return context
 
+    @log_view_mixin
     def get(self, request, *args, **kwargs):
         if request.GET.get('format', None) == 'json':
 
@@ -304,6 +307,7 @@ class ToasterTable(TemplateView):
 
         self.setup_columns(**kwargs)
 
+        self.apply_orderby('pk')
         if search:
             self.apply_search(search)
         if filters:
@@ -413,6 +417,7 @@ class ToasterTypeAhead(View):
     def __init__(self, *args, **kwargs):
         super(ToasterTypeAhead, self).__init__()
 
+    @log_view_mixin
     def get(self, request, *args, **kwargs):
         def response(data):
             return HttpResponse(json.dumps(data,
@@ -468,6 +473,7 @@ class MostRecentBuildsView(View):
 
         return False
 
+    @log_view_mixin
     def get(self, request, *args, **kwargs):
         """
         Returns a list of builds in JSON format.
diff --git a/bitbake/lib/toaster/toastermain/logs.py b/bitbake/lib/toaster/toastermain/logs.py
new file mode 100644
index 0000000000..62d871963a
--- /dev/null
+++ b/bitbake/lib/toaster/toastermain/logs.py
@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+import logging
+import json
+from pathlib import Path
+from django.http import HttpRequest
+
+BUILDDIR = Path(os.environ.get('BUILDDIR', '/tmp'))
+
+def log_api_request(request, response, view, logger_name='api'):
+    """Helper function for LogAPIMixin"""
+
+    repjson = {
+        'view': view,
+        'path': request.path,
+        'method': request.method,
+        'status': response.status_code
+    }
+
+    logger = logging.getLogger(logger_name)
+    logger.info(
+        json.dumps(repjson, indent=4, separators=(", ", " : "))
+    )
+
+
+def log_view_mixin(view):
+    def log_view_request(*args, **kwargs):
+        # get request from args else kwargs
+        request = None
+        if len(args) > 0:
+            for req in args:
+                if isinstance(req, HttpRequest):
+                    request = req
+                    break 
+        elif request is None:
+            request = kwargs.get('request')
+
+        response = view(*args, **kwargs)
+        view_name = 'unknown'
+        if hasattr(request, 'resolver_match'):
+            if hasattr(request.resolver_match, 'view_name'):
+                view_name = request.resolver_match.view_name
+
+        log_api_request(
+            request, response, view_name, 'toaster')
+        return response
+    return log_view_request
+
+
+
+class LogAPIMixin:
+    """Logs API requests
+
+    tested with:
+        - APIView
+        - ModelViewSet
+        - ReadOnlyModelViewSet
+        - GenericAPIView
+
+    Note: you can set `view_name` attribute in View to override get_view_name()
+    """
+
+    def get_view_name(self):
+        if hasattr(self, 'view_name'):
+            return self.view_name
+        return super().get_view_name()
+
+    def finalize_response(self, request, response, *args, **kwargs):
+        log_api_request(request, response, self.get_view_name())
+        return super().finalize_response(request, response, *args, **kwargs)
+
+
+LOGGING_SETTINGS = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'filters': {
+        'require_debug_false': {
+            '()': 'django.utils.log.RequireDebugFalse'
+        }
+    },
+    'formatters': {
+        'datetime': {
+            'format': '%(asctime)s %(levelname)s %(message)s'
+        },
+        'verbose': {
+            'format': '{levelname} {asctime} {module} {name}.{funcName} {process:d} {thread:d} {message}',
+            'datefmt': "%d/%b/%Y %H:%M:%S",
+            'style': '{',
+        },
+        'api': {
+            'format': '\n{levelname} {asctime} {name}.{funcName}:\n{message}',
+            'style': '{'
+        }
+    },
+    'handlers': {
+        'mail_admins': {
+            'level': 'ERROR',
+            'filters': ['require_debug_false'],
+            'class': 'django.utils.log.AdminEmailHandler'
+        },
+        'console': {
+            'level': 'DEBUG',
+            'class': 'logging.StreamHandler',
+            'formatter': 'datetime',
+        },
+        'file_django': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': BUILDDIR / 'toaster_logs/django.log',
+            'when': 'D',  # interval type
+            'interval': 1,  # defaults to 1
+            'backupCount': 10,  # how many files to keep
+            'formatter': 'verbose',
+        },
+        'file_api': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': BUILDDIR / 'toaster_logs/api.log',
+            'when': 'D',
+            'interval': 1,
+            'backupCount': 10,
+            'formatter': 'verbose',
+        },
+        'file_toaster': {
+            'level': 'INFO',
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'filename': BUILDDIR / 'toaster_logs/web.log',
+            'when': 'D',
+            'interval': 1,
+            'backupCount': 10,
+            'formatter': 'verbose',
+        },
+    },
+    'loggers': {
+        'django.request': {
+            'handlers': ['file_django', 'console'],
+            'level': 'WARN',
+            'propagate': True,
+        },
+        'django': {
+            'handlers': ['file_django', 'console'],
+            'level': 'WARNING',
+            'propogate': True,
+        },
+        'toaster': {
+            'handlers': ['file_toaster'],
+            'level': 'INFO',
+            'propagate': False,
+        },
+        'api': {
+            'handlers': ['file_api'],
+            'level': 'INFO',
+            'propagate': False,
+        }
+    }
+}
diff --git a/bitbake/lib/toaster/toastermain/management/commands/buildimport.py b/bitbake/lib/toaster/toastermain/management/commands/buildimport.py
index 59da6ff7ac..f7139aa041 100644
--- a/bitbake/lib/toaster/toastermain/management/commands/buildimport.py
+++ b/bitbake/lib/toaster/toastermain/management/commands/buildimport.py
@@ -451,7 +451,7 @@ class Command(BaseCommand):
             # Catch vars relevant to Toaster (in case no Toaster section)
             self.update_project_vars(project,'DISTRO')
             self.update_project_vars(project,'MACHINE')
-            self.update_project_vars(project,'IMAGE_INSTALL_append')
+            self.update_project_vars(project,'IMAGE_INSTALL:append')
             self.update_project_vars(project,'IMAGE_FSTYPES')
             self.update_project_vars(project,'PACKAGE_CLASSES')
             # These vars are typically only assigned by Toaster
@@ -545,7 +545,7 @@ class Command(BaseCommand):
             # Find the directory's release, and promote to default_release if local paths
             release = self.find_import_release(layers_list,lv_dict,default_release)
             # create project, SANITY: reuse any project of same name
-            project = Project.objects.create_project(project_name,release,project)
+            project = Project.objects.create_project(project_name,release,project, imported=True)
             # Apply any new layers or variables
             self.apply_conf_variables(project,layers_list,lv_dict,release)
             # WORKAROUND: since we now derive the release, redirect 'newproject_specific' to 'project_specific'
diff --git a/bitbake/lib/toaster/toastermain/management/commands/checksocket.py b/bitbake/lib/toaster/toastermain/management/commands/checksocket.py
index 811fd5d516..b2c002da7a 100644
--- a/bitbake/lib/toaster/toastermain/management/commands/checksocket.py
+++ b/bitbake/lib/toaster/toastermain/management/commands/checksocket.py
@@ -13,7 +13,7 @@ import errno
 import socket
 
 from django.core.management.base import BaseCommand, CommandError
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 
 DEFAULT_ADDRPORT = "0.0.0.0:8000"
 
@@ -51,7 +51,7 @@ class Command(BaseCommand):
             if hasattr(err, 'errno') and err.errno in errors:
                 errtext = errors[err.errno]
             else:
-                errtext = force_text(err)
+                errtext = force_str(err)
             raise CommandError(errtext)
 
         self.stdout.write("OK")
diff --git a/bitbake/lib/toaster/toastermain/settings.py b/bitbake/lib/toaster/toastermain/settings.py
index a4b370c8d4..d2a449627f 100644
--- a/bitbake/lib/toaster/toastermain/settings.py
+++ b/bitbake/lib/toaster/toastermain/settings.py
@@ -9,6 +9,8 @@
 # Django settings for Toaster project.
 
 import os
+from pathlib import Path
+from toastermain.logs import LOGGING_SETTINGS
 
 DEBUG = True
 
@@ -39,6 +41,9 @@ DATABASES = {
     }
 }
 
+# New in Django 3.2
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
 # Needed when Using sqlite especially to add a longer timeout for waiting
 # for the database lock to be  released
 # https://docs.djangoproject.com/en/1.6/ref/databases/#database-is-locked-errors
@@ -84,14 +89,17 @@ else:
                 from pytz.exceptions import UnknownTimeZoneError
                 try:
                     if pytz.timezone(zonename) is not None:
-                        zonefilelist[hashlib.md5(open(filepath, 'rb').read()).hexdigest()] = zonename
+                        with open(filepath, 'rb') as f:
+                            zonefilelist[hashlib.md5(f.read()).hexdigest()] = zonename
                 except UnknownTimeZoneError as ValueError:
                     # we expect timezone failures here, just move over
                     pass
             except ImportError:
-                zonefilelist[hashlib.md5(open(filepath, 'rb').read()).hexdigest()] = zonename
+                with open(filepath, 'rb') as f:
+                    zonefilelist[hashlib.md5(f.read()).hexdigest()] = zonename
 
-    TIME_ZONE = zonefilelist[hashlib.md5(open('/etc/localtime', 'rb').read()).hexdigest()]
+    with open('/etc/localtime', 'rb') as f:
+        TIME_ZONE = zonefilelist[hashlib.md5(f.read()).hexdigest()]
 
 # Language code for this installation. All choices can be found here:
 # http://www.i18nguy.com/unicode/language-identifiers.html
@@ -103,10 +111,6 @@ SITE_ID = 1
 # to load the internationalization machinery.
 USE_I18N = True
 
-# If you set this to False, Django will not format dates, numbers and
-# calendars according to the current locale.
-USE_L10N = True
-
 # If you set this to False, Django will not use timezone-aware datetimes.
 USE_TZ = True
 
@@ -147,6 +151,8 @@ STATICFILES_FINDERS = (
 # Make this unique, and don't share it with anybody.
 SECRET_KEY = 'NOT_SUITABLE_FOR_HOSTED_DEPLOYMENT'
 
+TMPDIR = os.environ.get('TOASTER_DJANGO_TMPDIR', '/tmp')
+
 class InvalidString(str):
     def __mod__(self, other):
         from django.template.base import TemplateSyntaxError
@@ -183,7 +189,13 @@ TEMPLATES = [
                 'django.template.loaders.app_directories.Loader',
                 #'django.template.loaders.eggs.Loader',
             ],
-            'string_if_invalid': InvalidString("%s"),
+            # https://docs.djangoproject.com/en/4.2/ref/templates/api/#how-invalid-variables-are-handled
+            # Generally, string_if_invalid should only be enabled in order to debug
+            # a specific template problem, then cleared once debugging is complete.
+            # If you assign a value other than '' to string_if_invalid,
+            # you will experience rendering problems with these templates and sites.
+            #  'string_if_invalid': InvalidString("%s"),
+            'string_if_invalid': "",
             'debug': DEBUG,
         },
     },
@@ -207,7 +219,7 @@ CACHES = {
     #        },
            'default': {
                'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
-               'LOCATION': '/tmp/toaster_cache_%d' % os.getuid(),
+               'LOCATION': '%s/toaster_cache_%d' % (TMPDIR, os.getuid()),
                'TIMEOUT': 1,
             }
           }
@@ -239,6 +251,9 @@ INSTALLED_APPS = (
     'django.contrib.humanize',
     'bldcollector',
     'toastermain',
+
+    # 3rd-lib
+    "log_viewer",
 )
 
 
@@ -283,7 +298,6 @@ SOUTH_TESTS_MIGRATE = False
 
 # We automatically detect and install applications here if
 # they have a 'models.py' or 'views.py' file
-import os
 currentdir = os.path.dirname(__file__)
 for t in os.walk(os.path.dirname(currentdir)):
     modulename = os.path.basename(t[0])
@@ -299,43 +313,21 @@ for t in os.walk(os.path.dirname(currentdir)):
 # the site admins on every HTTP 500 error when DEBUG=False.
 # See http://docs.djangoproject.com/en/dev/topics/logging for
 # more details on how to customize your logging configuration.
-LOGGING = {
-    'version': 1,
-    'disable_existing_loggers': False,
-    'filters': {
-        'require_debug_false': {
-            '()': 'django.utils.log.RequireDebugFalse'
-        }
-    },
-    'formatters': {
-        'datetime': {
-            'format': '%(asctime)s %(levelname)s %(message)s'
-        }
-    },
-    'handlers': {
-        'mail_admins': {
-            'level': 'ERROR',
-            'filters': ['require_debug_false'],
-            'class': 'django.utils.log.AdminEmailHandler'
-        },
-        'console': {
-            'level': 'DEBUG',
-            'class': 'logging.StreamHandler',
-            'formatter': 'datetime',
-        }
-    },
-    'loggers': {
-        'toaster' : {
-            'handlers': ['console'],
-            'level': 'DEBUG',
-        },
-        'django.request': {
-            'handlers': ['console'],
-            'level': 'WARN',
-            'propagate': True,
-        },
-    }
-}
+LOGGING = LOGGING_SETTINGS
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BUILDDIR = os.environ.get("BUILDDIR", TMPDIR)
+
+# LOG VIEWER
+# https://pypi.org/project/django-log-viewer/
+LOG_VIEWER_FILES_PATTERN = '*.log*'
+LOG_VIEWER_FILES_DIR = os.path.join(BUILDDIR, "toaster_logs/")
+LOG_VIEWER_PAGE_LENGTH = 25      # total log lines per-page
+LOG_VIEWER_MAX_READ_LINES = 100000  # total log lines will be read
+LOG_VIEWER_PATTERNS = ['INFO', 'DEBUG', 'WARNING', 'ERROR', 'CRITICAL']
+
+# Optionally you can set the next variables in order to customize the admin:
+LOG_VIEWER_FILE_LIST_TITLE = "Logs list"
 
 if DEBUG and SQL_DEBUG:
     LOGGING['loggers']['django.db.backends'] = {
diff --git a/bitbake/lib/toaster/toastermain/settings_test.py b/bitbake/lib/toaster/toastermain/settings_test.py
index 6538d9e453..74def2d240 100644
--- a/bitbake/lib/toaster/toastermain/settings_test.py
+++ b/bitbake/lib/toaster/toastermain/settings_test.py
@@ -19,10 +19,10 @@ TEMPLATE_DEBUG = DEBUG
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': '/tmp/toaster-test-db.sqlite',
+        'NAME': '%s/toaster-test-db.sqlite' % TMPDIR,
         'TEST': {
             'ENGINE': 'django.db.backends.sqlite3',
-            'NAME': '/tmp/toaster-test-db.sqlite',
+            'NAME': '%s/toaster-test-db.sqlite' % TMPDIR,
         }
     }
 }
diff --git a/bitbake/lib/toaster/toastermain/urls.py b/bitbake/lib/toaster/toastermain/urls.py
index 5fb520b384..3be46fcf0c 100644
--- a/bitbake/lib/toaster/toastermain/urls.py
+++ b/bitbake/lib/toaster/toastermain/urls.py
@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
-from django.conf.urls import include, url
+from django.urls import re_path as url, include
 from django.views.generic import RedirectView, TemplateView
 from django.views.decorators.cache import never_cache
 import bldcollector.views
@@ -28,6 +28,8 @@ urlpatterns = [
     # url(r'^admin/doc/', include('django.contrib.admindocs.urls')),
 
 
+    url(r'^logs/', include('log_viewer.urls')),
+
     # This is here to maintain backward compatibility and will be deprecated
     # in the future.
     url(r'^orm/eventfile$', bldcollector.views.eventfile),
diff --git a/bitbake/lib/toaster/tox.ini b/bitbake/lib/toaster/tox.ini
new file mode 100644
index 0000000000..1516a527ae
--- /dev/null
+++ b/bitbake/lib/toaster/tox.ini
@@ -0,0 +1,24 @@
+[tox]
+envlist = py38, py39, py310, py311, py312
+skipsdist = True
+toxworkdir = {env:TOX_WORKDIR:.tox}
+passenv = *
+
+[testenv]
+passenv =
+    SSTATE_DIR
+    DL_DIR
+    TOASTER_DJANGO_TMPDIR
+setenv =
+    DJANGO_SETTINGS_MODULE=toastermain.settings_test
+    TOASTER_BUILDSERVER=1
+    BUILDDIR = {env:BUILDDIR}
+    EVENTREPLAY_DIR = {env:EVENTREPLAY_DIR:BUILDDIR}
+commands =
+    python3 {toxinidir}/manage.py test tests.db tests.commands tests.builds tests.browser tests.functional tests.views
+deps = 
+    -r {toxinidir}/../../toaster-requirements.txt
+    -r {toxinidir}/tests/toaster-tests-requirements.txt
+
+[testenv:chrome]
+commands={[testenv]commands} --splinter-webdriver=chrome
\ No newline at end of file
diff --git a/bitbake/toaster-requirements.txt b/bitbake/toaster-requirements.txt
index 735b614546..d8e48b7f3a 100644
--- a/bitbake/toaster-requirements.txt
+++ b/bitbake/toaster-requirements.txt
@@ -1,3 +1,4 @@
-Django>2.2,<2.3
+Django>4.2,<4.3
 beautifulsoup4>=4.4.0
 pytz
+django-log-viewer==1.1.7