12 files changed, 190 insertions, 46 deletions

diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake
index 382983e087..40b5d895c1 100755
--- a/bitbake/bin/bitbake
+++ b/bitbake/bin/bitbake
@@ -27,7 +27,7 @@ from bb.main import bitbake_main, BitBakeConfigParameters, BBMainException
 
 bb.utils.check_system_locale()
 
-__version__ = "2.9.0"
+__version__ = "2.15.1"
 
 if __name__ == "__main__":
     if __version__ != bb.__version__:
diff --git a/bitbake/bin/bitbake-config-build b/bitbake/bin/bitbake-config-build
new file mode 120000
index 0000000000..11e6df80c4
--- /dev/null
+++ b/bitbake/bin/bitbake-config-build
@@ -0,0 +1 @@
+bitbake-layers
\ No newline at end of file
diff --git a/bitbake/bin/bitbake-diffsigs b/bitbake/bin/bitbake-diffsigs
index 8202c78623..9d6cb8c944 100755
--- a/bitbake/bin/bitbake-diffsigs
+++ b/bitbake/bin/bitbake-diffsigs
@@ -72,16 +72,17 @@ def find_siginfo_task(bbhandler, pn, taskname, sig1=None, sig2=None):
         elif sig2 not in sigfiles:
             logger.error('No sigdata files found matching %s %s with signature %s' % (pn, taskname, sig2))
             sys.exit(1)
+
+        latestfiles = [sigfiles[sig1]['path'], sigfiles[sig2]['path']]
     else:
         sigfiles = find_siginfo(bbhandler, pn, taskname)
         latestsigs = sorted(sigfiles.keys(), key=lambda h: sigfiles[h]['time'])[-2:]
         if not latestsigs:
             logger.error('No sigdata files found matching %s %s' % (pn, taskname))
             sys.exit(1)
-        sig1 = latestsigs[0]
-        sig2 = latestsigs[1]
-
-    latestfiles = [sigfiles[sig1]['path'], sigfiles[sig2]['path']]
+        latestfiles = [sigfiles[latestsigs[0]]['path']]
+        if len(latestsigs) > 1:
+            latestfiles.append(sigfiles[latestsigs[1]]['path'])
 
     return latestfiles
 
diff --git a/bitbake/bin/bitbake-getvar b/bitbake/bin/bitbake-getvar
index 8901f99ae2..378fb13572 100755
--- a/bitbake/bin/bitbake-getvar
+++ b/bitbake/bin/bitbake-getvar
@@ -10,12 +10,14 @@ import io
 import os
 import sys
 import warnings
+import logging
 warnings.simplefilter("default")
 
 bindir = os.path.dirname(__file__)
 topdir = os.path.dirname(bindir)
 sys.path[0:0] = [os.path.join(topdir, 'lib')]
 
+import bb.providers
 import bb.tinfoil
 
 if __name__ == "__main__":
@@ -37,13 +39,22 @@ if __name__ == "__main__":
             sys.exit("--flag only makes sense with --value")
 
     quiet = args.quiet or args.value
+    if quiet:
+        logger = logging.getLogger("BitBake")
+        logger.setLevel(logging.WARNING)
+
     with bb.tinfoil.Tinfoil(tracking=True, setup_logging=not quiet) as tinfoil:
         if args.recipe:
             tinfoil.prepare(quiet=3 if quiet else 2)
-            d = tinfoil.parse_recipe(args.recipe)
+            try:
+                d = tinfoil.parse_recipe(args.recipe)
+            except bb.providers.NoProvider as e:
+                sys.exit(str(e))
         else:
             tinfoil.prepare(quiet=2, config_only=True)
-            d = tinfoil.config_data
+            # Expand keys and run anonymous functions to get identical result to
+            # "bitbake -e"
+            d = tinfoil.finalizeData()
 
         value = None
         if args.flag:
diff --git a/bitbake/bin/bitbake-hashclient b/bitbake/bin/bitbake-hashclient
index 610787ed2b..b8755c5797 100755
--- a/bitbake/bin/bitbake-hashclient
+++ b/bitbake/bin/bitbake-hashclient
@@ -16,6 +16,8 @@ import time
 import warnings
 import netrc
 import json
+import statistics
+import textwrap
 warnings.simplefilter("default")
 
 try:
@@ -81,6 +83,7 @@ def main():
             nonlocal found_hashes
             nonlocal missed_hashes
             nonlocal max_time
+            nonlocal times
 
             with hashserv.create_client(args.address) as client:
                 for i in range(args.requests):
@@ -98,29 +101,41 @@ def main():
                         else:
                             missed_hashes += 1
 
-                        max_time = max(elapsed, max_time)
+                        times.append(elapsed)
                         pbar.update()
 
         max_time = 0
         found_hashes = 0
         missed_hashes = 0
         lock = threading.Lock()
-        total_requests = args.clients * args.requests
+        times = []
         start_time = time.perf_counter()
-        with ProgressBar(total=total_requests) as pbar:
+        with ProgressBar(total=args.clients * args.requests) as pbar:
             threads = [threading.Thread(target=thread_main, args=(pbar, lock), daemon=False) for _ in range(args.clients)]
             for t in threads:
                 t.start()
 
             for t in threads:
                 t.join()
+        total_elapsed = time.perf_counter() - start_time
 
-        elapsed = time.perf_counter() - start_time
         with lock:
-            print("%d requests in %.1fs. %.1f requests per second" % (total_requests, elapsed, total_requests / elapsed))
-            print("Average request time %.8fs" % (elapsed / total_requests))
-            print("Max request time was %.8fs" % max_time)
-            print("Found %d hashes, missed %d" % (found_hashes, missed_hashes))
+            mean = statistics.mean(times)
+            median = statistics.median(times)
+            stddev = statistics.pstdev(times)
+
+            print(f"Number of clients:    {args.clients}")
+            print(f"Requests per client:  {args.requests}")
+            print(f"Number of requests:   {len(times)}")
+            print(f"Total elapsed time:   {total_elapsed:.3f}s")
+            print(f"Total request rate:   {len(times)/total_elapsed:.3f} req/s")
+            print(f"Average request time: {mean:.3f}s")
+            print(f"Median request time:  {median:.3f}s")
+            print(f"Request time std dev: {stddev:.3f}s")
+            print(f"Maximum request time: {max(times):.3f}s")
+            print(f"Minimum request time: {min(times):.3f}s")
+            print(f"Hashes found:         {found_hashes}")
+            print(f"Hashes missed:        {missed_hashes}")
 
         if args.report:
             with ProgressBar(total=args.requests) as pbar:
@@ -212,6 +227,27 @@ def main():
         print("New hashes marked: %d" % result["count"])
         return 0
 
+    def handle_gc_mark_stream(args, client):
+        stdin = (l.strip() for l in sys.stdin)
+        marked_hashes = 0
+
+        try:
+            result = client.gc_mark_stream(args.mark, stdin)
+            marked_hashes = result["count"]
+        except ConnectionError:
+            logger.warning(
+                "Server doesn't seem to support `gc-mark-stream`. Sending "
+                "hashes sequentially using `gc-mark` API."
+            )
+            for line in stdin:
+                pairs = line.split()
+                condition = dict(zip(pairs[::2], pairs[1::2]))
+                result = client.gc_mark(args.mark, condition)
+                marked_hashes += result["count"]
+
+        print("New hashes marked: %d" % marked_hashes)
+        return 0
+
     def handle_gc_sweep(args, client):
         result = client.gc_sweep(args.mark)
         print("Removed %d rows" % result["count"])
@@ -225,7 +261,45 @@ def main():
         print("true" if result else "false")
         return 0
 
-    parser = argparse.ArgumentParser(description='Hash Equivalence Client')
+    def handle_ping(args, client):
+        times = []
+        for i in range(1, args.count + 1):
+            if not args.quiet:
+                print(f"Ping {i} of {args.count}... ", end="")
+            start_time = time.perf_counter()
+            client.ping()
+            elapsed = time.perf_counter() - start_time
+            times.append(elapsed)
+            if not args.quiet:
+                print(f"{elapsed:.3f}s")
+
+        mean = statistics.mean(times)
+        median = statistics.median(times)
+        std_dev = statistics.pstdev(times)
+
+        if not args.quiet:
+            print("------------------------")
+        print(f"Number of pings:         {len(times)}")
+        print(f"Average round trip time: {mean:.3f}s")
+        print(f"Median round trip time:  {median:.3f}s")
+        print(f"Round trip time std dev: {std_dev:.3f}s")
+        print(f"Min time is:             {min(times):.3f}s")
+        print(f"Max time is:             {max(times):.3f}s")
+        return 0
+
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description='Hash Equivalence Client',
+        epilog=textwrap.dedent(
+            """
+            Possible ADDRESS options are:
+                unix://PATH         Connect to UNIX domain socket at PATH
+                ws://HOST[:PORT]    Connect to websocket at HOST:PORT (default port is 80)
+                wss://HOST[:PORT]   Connect to secure websocket at HOST:PORT (default port is 443)
+                HOST:PORT           Connect to TCP server at HOST:PORT
+            """
+        ),
+    )
     parser.add_argument('--address', default=DEFAULT_ADDRESS, help='Server address (default "%(default)s")')
     parser.add_argument('--log', default='WARNING', help='Set logging level')
     parser.add_argument('--login', '-l', metavar="USERNAME", help="Authenticate as USERNAME")
@@ -313,6 +387,16 @@ def main():
                              help="Keep entries in table where KEY == VALUE")
     gc_mark_parser.set_defaults(func=handle_gc_mark)
 
+    gc_mark_parser_stream = subparsers.add_parser(
+        'gc-mark-stream',
+        help=(
+            "Mark multiple hashes to be retained for garbage collection. Input should be provided via stdin, "
+            "with each line formatted as key-value pairs separated by spaces, for example 'column1 foo column2 bar'."
+        )
+    )
+    gc_mark_parser_stream.add_argument("mark", help="Mark for this garbage collection operation")
+    gc_mark_parser_stream.set_defaults(func=handle_gc_mark_stream)
+
     gc_sweep_parser = subparsers.add_parser('gc-sweep', help="Perform garbage collection and delete any entries that are not marked")
     gc_sweep_parser.add_argument("mark", help="Mark for this garbage collection operation")
     gc_sweep_parser.set_defaults(func=handle_gc_sweep)
@@ -322,6 +406,11 @@ def main():
     unihash_exists_parser.add_argument("unihash", help="Unihash to check")
     unihash_exists_parser.set_defaults(func=handle_unihash_exists)
 
+    ping_parser = subparsers.add_parser('ping', help="Ping server")
+    ping_parser.add_argument("-n", "--count", type=int, help="Number of pings. Default is %(default)s", default=10)
+    ping_parser.add_argument("-q", "--quiet", action="store_true", help="Don't print each ping; only print results")
+    ping_parser.set_defaults(func=handle_ping)
+
     args = parser.parse_args()
 
     logger = logging.getLogger('hashserv')
diff --git a/bitbake/bin/bitbake-hashserv b/bitbake/bin/bitbake-hashserv
index 4bfb7abfbc..01503736b9 100755
--- a/bitbake/bin/bitbake-hashserv
+++ b/bitbake/bin/bitbake-hashserv
@@ -125,6 +125,11 @@ The following permissions are supported by the server:
         default=os.environ.get("HASHSERVER_ADMIN_PASSWORD", None),
         help="Create default admin user with password ADMIN_PASSWORD ($HASHSERVER_ADMIN_PASSWORD)",
     )
+    parser.add_argument(
+        "--reuseport",
+        action="store_true",
+        help="Enable SO_REUSEPORT, allowing multiple servers to bind to the same port for load balancing",
+    )
 
     args = parser.parse_args()
 
@@ -132,7 +137,9 @@ The following permissions are supported by the server:
 
     level = getattr(logging, args.log.upper(), None)
     if not isinstance(level, int):
-        raise ValueError("Invalid log level: %s (Try ERROR/WARNING/INFO/DEBUG)" % args.log)
+        raise ValueError(
+            "Invalid log level: %s (Try ERROR/WARNING/INFO/DEBUG)" % args.log
+        )
 
     logger.setLevel(level)
     console = logging.StreamHandler()
@@ -155,6 +162,7 @@ The following permissions are supported by the server:
         anon_perms=anon_perms,
         admin_username=args.admin_user,
         admin_password=args.admin_password,
+        reuseport=args.reuseport,
     )
     server.serve_forever()
     return 0
diff --git a/bitbake/bin/bitbake-layers b/bitbake/bin/bitbake-layers
index d4b1d1aaf2..341ecbcd97 100755
--- a/bitbake/bin/bitbake-layers
+++ b/bitbake/bin/bitbake-layers
@@ -18,13 +18,14 @@ import warnings
 warnings.simplefilter("default")
 
 bindir = os.path.dirname(__file__)
+toolname = os.path.basename(__file__).split(".")[0]
 topdir = os.path.dirname(bindir)
 sys.path[0:0] = [os.path.join(topdir, 'lib')]
 
 import bb.tinfoil
 import bb.msg
 
-logger = bb.msg.logger_create('bitbake-layers', sys.stdout)
+logger = bb.msg.logger_create(toolname, sys.stdout)
 
 def main():
     parser = argparse.ArgumentParser(
@@ -33,7 +34,7 @@ def main():
         add_help=False)
     parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
     parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
-    parser.add_argument('-F', '--force', help='Force add without recipe parse verification', action='store_true')
+    parser.add_argument('-F', '--force', help='Forced execution: can be specified multiple times. -F will force add without recipe parse verification and -FF will additionally force the run withput layer parsing.', action='count', default=0)
     parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR')
 
     global_args, unparsed_args = parser.parse_known_args()
@@ -57,18 +58,23 @@ def main():
                          level=logger.getEffectiveLevel())
 
     plugins = []
-    tinfoil = bb.tinfoil.Tinfoil(tracking=True)
-    tinfoil.logger.setLevel(logger.getEffectiveLevel())
-    try:
-        tinfoil.prepare(True)
-        for path in ([topdir] +
-                    tinfoil.config_data.getVar('BBPATH').split(':')):
-            pluginpath = os.path.join(path, 'lib', 'bblayers')
+    with bb.tinfoil.Tinfoil(tracking=True) as tinfoil:
+        tinfoil.logger.setLevel(logger.getEffectiveLevel())
+
+        if global_args.force > 1:
+            bbpaths = []
+        else:
+            tinfoil.prepare(True)
+            bbpaths = tinfoil.config_data.getVar('BBPATH').split(':')
+
+        for path in ([topdir] + bbpaths):
+            pluginbasepath = {"bitbake-layers":'bblayers', 'bitbake-config-build':'bbconfigbuild'}[toolname]
+            pluginpath = os.path.join(path, 'lib', pluginbasepath)
             bb.utils.load_plugins(logger, plugins, pluginpath)
 
         registered = False
         for plugin in plugins:
-            if hasattr(plugin, 'tinfoil_init'):
+            if hasattr(plugin, 'tinfoil_init') and global_args.force <= 1:
                 plugin.tinfoil_init(tinfoil)
             if hasattr(plugin, 'register_commands'):
                 registered = True
@@ -86,8 +92,6 @@ def main():
             tinfoil.config_data.enableTracking()
 
         return args.func(args)
-    finally:
-        tinfoil.shutdown()
 
 
 if __name__ == "__main__":
diff --git a/bitbake/bin/bitbake-prserv b/bitbake/bin/bitbake-prserv
index ad0a069401..3992e84eab 100755
--- a/bitbake/bin/bitbake-prserv
+++ b/bitbake/bin/bitbake-prserv
@@ -16,11 +16,18 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "lib
 import prserv
 import prserv.serv
 
-VERSION = "1.1.0"
+VERSION = "2.0.0"
 
 PRHOST_DEFAULT="0.0.0.0"
 PRPORT_DEFAULT=8585
 
+def init_logger(logfile, loglevel):
+    numeric_level = getattr(logging, loglevel.upper(), None)
+    if not isinstance(numeric_level, int):
+        raise ValueError("Invalid log level: %s" % loglevel)
+    FORMAT = "%(asctime)-15s %(message)s"
+    logging.basicConfig(level=numeric_level, filename=logfile, format=FORMAT)
+
 def main():
     parser = argparse.ArgumentParser(
         description="BitBake PR Server. Version=%s" % VERSION,
@@ -70,12 +77,25 @@ def main():
         action="store_true",
         help="open database in read-only mode",
     )
+    parser.add_argument(
+        "-u",
+        "--upstream",
+        default=os.environ.get("PRSERV_UPSTREAM", None),
+        help="Upstream PR service (host:port)",
+    )
 
     args = parser.parse_args()
-    prserv.init_logger(os.path.abspath(args.log), args.loglevel)
+    init_logger(os.path.abspath(args.log), args.loglevel)
 
     if args.start:
-        ret=prserv.serv.start_daemon(args.file, args.host, args.port, os.path.abspath(args.log), args.read_only)
+        ret=prserv.serv.start_daemon(
+            args.file,
+            args.host,
+            args.port,
+            os.path.abspath(args.log),
+            args.read_only,
+            args.upstream
+        )
     elif args.stop:
         ret=prserv.serv.stop_daemon(args.host, args.port)
     else:
diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest
index f25f23b1ae..1b7a783fdc 100755
--- a/bitbake/bin/bitbake-selftest
+++ b/bitbake/bin/bitbake-selftest
@@ -15,6 +15,7 @@ import unittest
 try:
     import bb
     import hashserv
+    import prserv
     import layerindexlib
 except RuntimeError as exc:
     sys.exit(str(exc))
@@ -27,12 +28,12 @@ tests = ["bb.tests.codeparser",
          "bb.tests.event",
          "bb.tests.fetch",
          "bb.tests.parse",
-         "bb.tests.persist_data",
          "bb.tests.runqueue",
          "bb.tests.siggen",
          "bb.tests.utils",
          "bb.tests.compression",
          "hashserv.tests",
+         "prserv.tests",
          "layerindexlib.tests.layerindexobj",
          "layerindexlib.tests.restapi",
          "layerindexlib.tests.cooker"]
diff --git a/bitbake/bin/bitbake-server b/bitbake/bin/bitbake-server
index 454a3919aa..a559109e3f 100755
--- a/bitbake/bin/bitbake-server
+++ b/bitbake/bin/bitbake-server
@@ -9,6 +9,7 @@ import os
 import sys
 import warnings
 warnings.simplefilter("default")
+warnings.filterwarnings("ignore", category=DeprecationWarning, message=".*use.of.fork.*may.lead.to.deadlocks.in.the.child.*")
 import logging
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 
@@ -38,9 +39,9 @@ if xmlrpcinterface[0] == "None":
 with open('/dev/null', 'r') as si:
     os.dup2(si.fileno(), sys.stdin.fileno())
 
-so = open(logfile, 'a+')
-os.dup2(so.fileno(), sys.stdout.fileno())
-os.dup2(so.fileno(), sys.stderr.fileno())
+with open(logfile, 'a+') as so:
+    os.dup2(so.fileno(), sys.stdout.fileno())
+    os.dup2(so.fileno(), sys.stderr.fileno())
 
 # Have stdout and stderr be the same so log output matches chronologically
 # and there aren't two seperate buffers
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker
index e8073f2ac3..d2b146a6a9 100755
--- a/bitbake/bin/bitbake-worker
+++ b/bitbake/bin/bitbake-worker
@@ -9,6 +9,7 @@ import os
 import sys
 import warnings
 warnings.simplefilter("default")
+warnings.filterwarnings("ignore", category=DeprecationWarning, message=".*use.of.fork.*may.lead.to.deadlocks.in.the.child.*")
 sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib'))
 from bb import fetch2
 import logging
@@ -21,9 +22,14 @@ import traceback
 import queue
 import shlex
 import subprocess
+import fcntl
 from multiprocessing import Lock
 from threading import Thread
 
+# Remove when we have a minimum of python 3.10
+if not hasattr(fcntl, 'F_SETPIPE_SZ'):
+    fcntl.F_SETPIPE_SZ = 1031
+
 bb.utils.check_system_locale()
 
 # Users shouldn't be running this code directly
@@ -44,7 +50,6 @@ if sys.argv[1].startswith("decafbadbad"):
 # updates to log files for use with tail
 try:
     if sys.stdout.name == '<stdout>':
-        import fcntl
         fl = fcntl.fcntl(sys.stdout.fileno(), fcntl.F_GETFL)
         fl |= os.O_SYNC 
         fcntl.fcntl(sys.stdout.fileno(), fcntl.F_SETFL, fl)
@@ -56,6 +61,12 @@ logger = logging.getLogger("BitBake")
 
 worker_pipe = sys.stdout.fileno()
 bb.utils.nonblockingfd(worker_pipe)
+# Try to make the pipe buffers larger as it is much more efficient. If we can't
+# e.g. out of buffer space (/proc/sys/fs/pipe-user-pages-soft) then just pass over.
+try:
+    fcntl.fcntl(worker_pipe, fcntl.F_SETPIPE_SZ, 512 * 1024)
+except:
+    pass
 # Need to guard against multiprocessing being used in child processes
 # and multiple processes trying to write to the parent at the same time
 worker_pipe_lock = None
@@ -105,7 +116,7 @@ def worker_flush(worker_queue):
                 if not worker_queue.empty():
                     worker_queue_int.extend(worker_queue.get())
                 written = os.write(worker_pipe, worker_queue_int)
-                worker_queue_int = worker_queue_int[written:]
+                del worker_queue_int[0:written]
             except (IOError, OSError) as e:
                 if e.errno != errno.EAGAIN and e.errno != errno.EPIPE:
                     raise
@@ -171,11 +182,8 @@ def fork_off_task(cfg, data, databuilder, workerdata, extraconfigdata, runtask):
     elif workerdata["umask"]:
         umask = workerdata["umask"]
     if umask:
-        # umask might come in as a number or text string..
-        try:
-             umask = int(umask, 8)
-        except TypeError:
-             pass
+        # Convert to a python numeric value as it could be a string
+        umask = bb.utils.to_filemode(umask)
 
     dry_run = cfg.dry_run or runtask['dry_run']
 
@@ -357,7 +365,7 @@ class runQueueWorkerPipe():
     def read(self):
         start = len(self.queue)
         try:
-            self.queue.extend(self.input.read(102400) or b"")
+            self.queue.extend(self.input.read(512*1024) or b"")
         except (OSError, IOError) as e:
             if e.errno != errno.EAGAIN:
                 raise
diff --git a/bitbake/bin/git-make-shallow b/bitbake/bin/git-make-shallow
index 9de557c10e..e6c180b4d6 100755
--- a/bitbake/bin/git-make-shallow
+++ b/bitbake/bin/git-make-shallow
@@ -115,8 +115,8 @@ def filter_refs(refs):
     all_refs = get_all_refs()
     to_remove = set(all_refs) - set(refs)
     if to_remove:
-        check_output(['xargs', '-0', '-n', '1'] + git_cmd + ['update-ref', '-d', '--no-deref'],
-                     input=''.join(l + '\0' for l in to_remove))
+        check_output(git_cmd + ['update-ref', '--no-deref', '--stdin', '-z'],
+                     input=''.join('delete ' + l + '\0\0' for l in to_remove))
 
 
 def follow_history_intersections(revisions, refs):