8 files changed, 1363 insertions, 8 deletions

diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest
index d45c2d406d..fb7c57dd83 100755
--- a/bitbake/bin/bitbake-selftest
+++ b/bitbake/bin/bitbake-selftest
@@ -29,6 +29,7 @@ tests = ["bb.tests.codeparser",
          "bb.tests.fetch",
          "bb.tests.parse",
          "bb.tests.runqueue",
+         "bb.tests.setup",
          "bb.tests.siggen",
          "bb.tests.utils",
          "bb.tests.compression",
diff --git a/bitbake/bin/bitbake-setup b/bitbake/bin/bitbake-setup
new file mode 100755
index 0000000000..38bb8099fd
--- /dev/null
+++ b/bitbake/bin/bitbake-setup
@@ -0,0 +1,860 @@
+#!/usr/bin/env python3
+
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import logging
+import os
+import sys
+import argparse
+import warnings
+import json
+import shutil
+import time
+import stat
+import tempfile
+import configparser
+import datetime
+import glob
+import subprocess
+
+default_registry = os.path.normpath(os.path.dirname(__file__) + "/../default-registry")
+
+bindir = os.path.abspath(os.path.dirname(__file__))
+sys.path[0:0] = [os.path.join(os.path.dirname(bindir), 'lib')]
+
+import bb.msg
+import bb.process
+
+logger = bb.msg.logger_create('bitbake-setup', sys.stdout)
+
+def cache_dir(top_dir):
+    return os.path.join(top_dir, '.bitbake-setup-cache')
+
+def init_bb_cache(top_dir, settings, args):
+    dldir = settings["default"]["dl-dir"]
+    bb_cachedir = os.path.join(cache_dir(top_dir), 'bitbake-cache')
+
+    d = bb.data.init()
+    d.setVar("DL_DIR", dldir)
+    d.setVar("BB_CACHEDIR", bb_cachedir)
+    d.setVar("__BBSRCREV_SEEN", "1")
+    if args.no_network:
+        d.setVar("BB_SRCREV_POLICY", "cache")
+    bb.fetch.fetcher_init(d)
+    return d
+
+def save_bb_cache():
+    bb.fetch2.fetcher_parse_save()
+    bb.fetch2.fetcher_parse_done()
+
+def get_config_name(config):
+    suffix = '.conf.json'
+    config_file = os.path.basename(config)
+    if config_file.endswith(suffix):
+        return config_file[:-len(suffix)]
+    else:
+        raise Exception("Config file {} does not end with {}, please rename the file.".format(config, suffix))
+
+def write_config(config, config_dir):
+    with open(os.path.join(config_dir, "config-upstream.json"),'w') as s:
+        json.dump(config, s, sort_keys=True, indent=4)
+
+def commit_config(config_dir):
+    bb.process.run("git -C {} add .".format(config_dir))
+    bb.process.run("git -C {} commit --no-verify -a -m 'Configuration at {}'".format(config_dir, time.asctime()))
+
+def _write_layer_list(dest, repodirs):
+    layers = []
+    for r in repodirs:
+        for root, dirs, files in os.walk(os.path.join(dest,r)):
+            if os.path.basename(root) == 'conf' and 'layer.conf' in files:
+                layers.append(os.path.relpath(os.path.dirname(root), dest))
+    layers_f = os.path.join(dest, ".oe-layers.json")
+    with open(layers_f, 'w') as f:
+        json.dump({"version":"1.0","layers":layers}, f, sort_keys=True, indent=4)
+
+def checkout_layers(layers, layerdir, d):
+    repodirs = []
+    oesetupbuild = None
+    print("Fetching layer/tool repositories into {}".format(layerdir))
+    for r_name in layers:
+        r_data = layers[r_name]
+        repodir = r_data["path"]
+        repodirs.append(repodir)
+
+        r_remote = r_data['git-remote']
+        rev = r_remote['rev']
+        branch = r_remote.get('branch', None)
+        remotes = r_remote['remotes']
+
+        for remote in remotes:
+            type,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"])
+            fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params))
+            print("    {}".format(r_name))
+            if branch:
+                fetcher = bb.fetch.Fetch(["{};protocol={};rev={};branch={};destsuffix={}".format(fetchuri,type,rev,branch,repodir)], d)
+            else:
+                fetcher = bb.fetch.Fetch(["{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir)], d)
+            do_fetch(fetcher, layerdir)
+
+        if os.path.exists(os.path.join(layerdir, repodir, 'scripts/oe-setup-build')):
+            oesetupbuild = os.path.join(layerdir, repodir, 'scripts/oe-setup-build')
+            oeinitbuildenvdir = os.path.join(layerdir, repodir)
+
+    print("        ")
+    _write_layer_list(layerdir, repodirs)
+
+    if oesetupbuild:
+        links = {'setup-build': oesetupbuild, 'oe-scripts': os.path.dirname(oesetupbuild), 'oe-init-build-env-dir': oeinitbuildenvdir}
+        for l,t in links.items():
+            symlink = os.path.join(layerdir, l)
+            if os.path.lexists(symlink):
+                os.remove(symlink)
+            os.symlink(os.path.relpath(t,layerdir),symlink)
+
+def setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir):
+    def _setup_build_conf(layers, build_conf_dir):
+        os.makedirs(build_conf_dir)
+        layers_s = []
+        for l in layers:
+            if l.startswith("{THISDIR}/"):
+                if thisdir:
+                    l = l.format(THISDIR=thisdir)
+                else:
+                    raise Exception("Configuration is using {THISDIR} to specify " \
+                    "a layer path relative to itself. This can be done only " \
+                    "when the configuration is specified by its path on local " \
+                    "disk, not when it's in a registry or is fetched over http.")
+            if not os.path.isabs(l):
+                l = os.path.join(layerdir, l)
+            layers_s.append("  {} \\".format(l))
+        layers_s = "\n".join(layers_s)
+        bblayers_conf = """BBLAYERS ?= " \\
+{}
+  "
+""".format(layers_s)
+        with open(os.path.join(build_conf_dir, "bblayers.conf"), 'w') as f:
+            f.write(bblayers_conf)
+
+        local_conf = """#
+# This file is intended for local configuration tweaks.
+#
+# If you would like to publish and share changes made to this file,
+# it is recommended to put them into a distro config, or to create
+# layer fragments from changes made here.
+#
+"""
+        with open(os.path.join(build_conf_dir, "local.conf"), 'w') as f:
+            f.write(local_conf)
+
+        with open(os.path.join(build_conf_dir, "templateconf.cfg"), 'w') as f:
+            f.write("")
+
+        with open(os.path.join(build_conf_dir, "conf-summary.txt"), 'w') as f:
+            f.write(bitbake_config["description"] + "\n")
+
+        with open(os.path.join(build_conf_dir, "conf-notes.txt"), 'w') as f:
+            f.write("")
+
+    def _make_init_build_env(builddir, oeinitbuildenvdir):
+        builddir = os.path.realpath(builddir)
+        cmd = "cd {}\nset {}\n. ./oe-init-build-env\n".format(oeinitbuildenvdir, builddir)
+        initbuild_in_builddir = os.path.join(builddir, 'init-build-env')
+
+        with open(initbuild_in_builddir, 'w') as f:
+            f.write("# init-build-env wrapper created by bitbake-setup\n")
+            f.write(cmd + '\n')
+
+    def _prepend_passthrough_to_init_build_env(builddir):
+        env = bitbake_config.get("bb-env-passthrough-additions")
+        if not env:
+            return
+
+        initbuild_in_builddir = os.path.join(builddir, 'init-build-env')
+        with open(initbuild_in_builddir) as f:
+            content = f.read()
+
+        joined = " \\\n".join(env)
+        env = "export BB_ENV_PASSTHROUGH_ADDITIONS=\" \\\n"
+        env += "${BB_ENV_PASSTHROUGH_ADDITIONS} \\\n"
+        env += joined
+        env += '"'
+
+        with open(initbuild_in_builddir, 'w') as f:
+            f.write("# environment passthrough added by bitbake-setup\n")
+            f.write(env + '\n')
+            f.write('\n')
+            f.write(content)
+
+    bitbake_builddir = os.path.join(builddir, "build")
+    print("Setting up bitbake configuration in\n    {}\n".format(bitbake_builddir))
+
+    template = bitbake_config.get("oe-template")
+    layers = bitbake_config.get("bb-layers")
+    if not template and not layers:
+        print("Bitbake configuration does not contain a reference to an OpenEmbedded build template via 'oe-template' or a list of layers via 'bb-layers'; please use oe-setup-build, oe-init-build-env or another mechanism manually to complete the setup.")
+        return
+    oesetupbuild = os.path.join(layerdir, 'setup-build')
+    if template and not os.path.exists(oesetupbuild):
+        raise Exception("Cannot complete setting up a bitbake build directory from OpenEmbedded template '{}' as oe-setup-build was not found in any layers; please use oe-init-build-env manually.".format(template))
+
+    bitbake_confdir = os.path.join(bitbake_builddir, 'conf')
+    backup_bitbake_confdir = bitbake_confdir + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
+    if os.path.exists(bitbake_confdir):
+        os.rename(bitbake_confdir, backup_bitbake_confdir)
+
+    if layers:
+        _setup_build_conf(layers, bitbake_confdir)
+
+    if template:
+        bb.process.run("{} setup -c {} -b {} --no-shell".format(oesetupbuild, template, bitbake_builddir))
+    else:
+        oeinitbuildenvdir = os.path.join(layerdir, 'oe-init-build-env-dir')
+        if not os.path.exists(os.path.join(oeinitbuildenvdir, "oe-init-build-env")):
+            print("Could not find oe-init-build-env in any of the layers; please use another mechanism to initialize the bitbake environment")
+            return
+        _make_init_build_env(bitbake_builddir, os.path.realpath(oeinitbuildenvdir))
+
+    _prepend_passthrough_to_init_build_env(bitbake_builddir)
+
+    siteconf_symlink = os.path.join(bitbake_confdir, "site.conf")
+    siteconf = os.path.normpath(os.path.join(builddir, '..', "site.conf"))
+    if os.path.lexists(siteconf_symlink):
+        os.remove(symlink)
+    os.symlink(os.path.relpath(siteconf, bitbake_confdir) ,siteconf_symlink)
+
+
+    init_script = os.path.join(bitbake_builddir, "init-build-env")
+    shell = "bash"
+    fragments = bitbake_config.get("oe-fragments", []) + sorted(bitbake_config.get("oe-fragment-choices",{}).values())
+    if fragments:
+        bb.process.run("{} -c '. {} && bitbake-config-build enable-fragment {}'".format(shell, init_script, " ".join(fragments)))
+
+    if os.path.exists(backup_bitbake_confdir):
+        bitbake_config_diff = get_diff(backup_bitbake_confdir, bitbake_confdir)
+        if bitbake_config_diff:
+            print("Existing bitbake configuration directory renamed to {}".format(backup_bitbake_confdir))
+            print("The bitbake configuration has changed:")
+            print(bitbake_config_diff)
+        else:
+            shutil.rmtree(backup_bitbake_confdir)
+
+    print("This bitbake configuration provides:\n    {}\n".format(bitbake_config["description"]))
+
+    readme = """{}\n\nAdditional information is in {} and {}\n
+Source the environment using '. {}' to run builds from the command line.
+The bitbake configuration files (local.conf, bblayers.conf and more) can be found in {}/conf
+""".format(
+        bitbake_config["description"],
+        os.path.join(bitbake_builddir,'conf/conf-summary.txt'),
+        os.path.join(bitbake_builddir,'conf/conf-notes.txt'),
+        init_script,
+        bitbake_builddir
+        )
+    readme_file = os.path.join(bitbake_builddir, "README")
+    with open(readme_file, 'w') as f:
+        f.write(readme)
+    print("Usage instructions and additional information are in\n     {}\n".format(readme_file))
+    print("The bitbake configuration files (local.conf, bblayers.conf and more) can be found in\n    {}/conf\n".format(bitbake_builddir))
+    print("To run builds, source the environment using\n    . {}".format(init_script))
+
+def get_registry_config(registry_path, id):
+    for root, dirs, files in os.walk(registry_path):
+        for f in files:
+            if f.endswith('.conf.json') and id == get_config_name(f):
+                return os.path.join(root, f)
+    raise Exception("Unable to find {} in available configurations; use 'list' sub-command to see what is available".format(id))
+
+def update_build(config, confdir, builddir, layerdir, d):
+    layer_config = config["data"]["sources"]
+    layer_overrides = config["source-overrides"]["sources"]
+    for k,v in layer_overrides.items():
+        if k in layer_config:
+            layer_config[k]["git-remote"] = v["git-remote"]
+    checkout_layers(layer_config, layerdir, d)
+    bitbake_config = config["bitbake-config"]
+    thisdir = os.path.dirname(config["path"]) if config["type"] == 'local' else None
+    setup_bitbake_build(bitbake_config, layerdir, builddir, thisdir)
+
+def int_input(allowed_values):
+    n = None
+    while n is None:
+        try:
+            n = int(input())
+        except ValueError:
+            print('Not a valid number, please try again:')
+            continue
+        if n not in allowed_values:
+            print('Number {} not one of {}, please try again:'.format(n, allowed_values))
+            n = None
+    return n
+
+def flatten_bitbake_configs(configs):
+    def merge_configs(c1,c2):
+        c_merged = {}
+        for k,v in c2.items():
+            if k not in c1.keys():
+                c_merged[k] = v
+        for k,v in c1.items():
+            if k not in c2.keys():
+                c_merged[k] = v
+            else:
+               c_merged[k] = c1[k] + c2[k]
+        del c_merged['configurations']
+        return c_merged
+
+    flattened_configs = []
+    for c in configs:
+        if 'configurations' not in c:
+            flattened_configs.append(c)
+        else:
+            for sub_c in flatten_bitbake_configs(c['configurations']):
+                flattened_configs.append(merge_configs(c, sub_c))
+    return flattened_configs
+
+def choose_bitbake_config(configs, parameters, non_interactive):
+    flattened_configs = flatten_bitbake_configs(configs)
+    configs_dict = {i["name"]:i for i in flattened_configs}
+
+    if parameters:
+        config_id = parameters[0]
+        if config_id not in configs_dict:
+            raise Exception("Bitbake configuration {} not found; replace with one of {}".format(config_id, configs_dict))
+        return configs_dict[config_id]
+
+    enumerated_configs = list(enumerate(flattened_configs))
+    if len(enumerated_configs) == 1:
+        only_config = flattened_configs[0]
+        print("\nSelecting the only available bitbake configuration {}".format(only_config["name"]))
+        return only_config
+
+    if non_interactive:
+        raise Exception("Unable to choose from bitbake configurations in non-interactive mode: {}".format(configs_dict))
+
+    print("\nAvailable bitbake configurations:")
+    for n, config_data in enumerated_configs:
+        print("{}. {}\t{}".format(n, config_data["name"], config_data["description"]))
+    print("\nPlease select one of the above bitbake configurations by its number:")
+    config_n = int_input([i[0] for i in enumerated_configs])
+    return flattened_configs[config_n]
+
+def choose_config(configs, non_interactive):
+    not_expired_configs = [k for k in configs.keys() if not has_expired(configs[k].get("expires", None))]
+    config_list = list(enumerate(not_expired_configs))
+    if len(config_list) == 1:
+        only_config = config_list[0][1]
+        print("\nSelecting the only available configuration {}\n".format(only_config))
+        return only_config
+
+    if non_interactive:
+        raise Exception("Unable to choose from configurations in non-interactive mode: {}".format(not_expired_configs))
+
+    print("\nAvailable configurations:")
+    for n, config_name in config_list:
+        config_data = configs[config_name]
+        expiry_date = config_data.get("expires", None)
+        config_desc = config_data["description"]
+        if expiry_date:
+           print("{}. {}\t{} (supported until {})".format(n, config_name, config_desc, expiry_date))
+        else:
+           print("{}. {}\t{}".format(n, config_name, config_desc))
+    print("\nPlease select one of the above configurations by its number:")
+    config_n = int_input([i[0] for i in config_list])
+    return config_list[config_n][1]
+
+def choose_fragments(possibilities, parameters, non_interactive, skip_selection):
+    choices = {}
+    for k,v in possibilities.items():
+        if skip_selection and k in skip_selection:
+            print("Skipping a selection of {}, as requested on command line. The resulting bitbake configuration may require further manual adjustments.".format(k))
+            continue
+        choice = [o for o in v["options"] if o in parameters]
+        if len(choice) > 1:
+            raise Exception("Options specified on command line do not allow a single selection from possibilities {}, please remove one or more from {}".format(v["options"], parameters))
+        if len(choice) == 1:
+            choices[k] = choice[0]
+            continue
+
+        if non_interactive:
+            raise Exception("Unable to choose from options in non-interactive mode: {}".format(v["options"]))
+
+        print("\n" + v["description"] + ":")
+        options_enumerated = list(enumerate(v["options"]))
+        for n,o in options_enumerated:
+            print("{}. {}".format(n, o))
+        print("\nPlease select one of the above options by its number:")
+        option_n = int_input([i[0] for i in options_enumerated])
+        choices[k] = options_enumerated[option_n][1]
+    return choices
+
+def obtain_config(top_dir, settings, args, source_overrides, d):
+    if args.config:
+        config_id = args.config[0]
+        config_parameters = args.config[1:]
+        if os.path.exists(config_id):
+            print("Reading configuration from local file\n    {}".format(config_id))
+            upstream_config = {'type':'local',
+                               'path':os.path.abspath(config_id),
+                               'name':get_config_name(config_id),
+                               'data':json.load(open(config_id))
+                               }
+        elif config_id.startswith("http://") or config_id.startswith("https://"):
+            print("Reading configuration from network URI\n    {}".format(config_id))
+            import urllib.request
+            with urllib.request.urlopen(config_id) as f:
+              upstream_config = {'type':'network','uri':config_id,'name':get_config_name(config_id),'data':json.load(f)}
+        else:
+            print("Looking up config {} in configuration registry".format(config_id))
+            registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
+            registry_configs = list_registry(registry_path, with_expired=True)
+            if config_id not in registry_configs:
+                raise Exception("Config {} not found in configuration registry, re-run 'init' without parameters to choose from available configurations.".format(config_id))
+            upstream_config = {'type':'registry','registry':settings["default"]["registry"],'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}
+        expiry_date = upstream_config['data'].get("expires", None)
+        if has_expired(expiry_date):
+            print("This configuration is no longer supported after {}. Please consider changing to a supported configuration.".format(expiry_date))
+    else:
+        registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
+        registry_configs = list_registry(registry_path, with_expired=True)
+        config_id = choose_config(registry_configs, args.non_interactive)
+        config_parameters = []
+        upstream_config = {'type':'registry','registry':settings["default"]["registry"],'name':config_id,'data':json.load(open(get_registry_config(registry_path,config_id)))}
+
+    upstream_config['bitbake-config'] = choose_bitbake_config(upstream_config['data']['bitbake-setup']['configurations'], config_parameters, args.non_interactive)
+    upstream_config['bitbake-config']['oe-fragment-choices'] = choose_fragments(upstream_config['bitbake-config'].get('oe-fragments-one-of',{}), config_parameters[1:], args.non_interactive, args.skip_selection)
+    upstream_config['non-interactive-cmdline-options'] = [config_id, upstream_config['bitbake-config']['name']] + sorted(upstream_config['bitbake-config']['oe-fragment-choices'].values())
+    upstream_config['source-overrides'] = source_overrides
+    upstream_config['skip-selection'] = args.skip_selection
+    return upstream_config
+
+def init_config(top_dir, settings, args, d):
+    stdout = sys.stdout
+    def handle_task_progress(event, d):
+        rate = event.rate if event.rate else ''
+        progress = event.progress if event.progress > 0 else 0
+        print("{}% {}                ".format(progress, rate), file=stdout, end='\r')
+
+    create_siteconf(top_dir, args.non_interactive)
+    source_overrides = json.load(open(args.source_overrides)) if args.source_overrides else {'sources':{}}
+    upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)
+    print("\nRun 'bitbake-setup init --non-interactive {}' to select this configuration non-interactively.\n".format(" ".join(upstream_config['non-interactive-cmdline-options'])))
+
+    builddir = os.path.join(os.path.abspath(top_dir), args.build_dir_name or "{}-{}".format(upstream_config['name']," ".join(upstream_config['non-interactive-cmdline-options'][1:]).replace(" ","-").replace("/","_")))
+    if os.path.exists(os.path.join(builddir, "layers")):
+        print(f"Build already initialized in:\n    {builddir}\nUse 'bitbake-setup status' to check if it needs to be updated, or 'bitbake-setup update' to perform the update.\nIf you would like to start over and re-initialize a build in this directory, remove it, and run 'bitbake-setup init' again.")
+        return
+
+    print("Initializing a build in\n    {}".format(builddir))
+    if not args.non_interactive:
+        y_or_n = input('Continue? (y/N): ')
+        if y_or_n != 'y':
+            exit()
+        print()
+
+    os.makedirs(builddir, exist_ok=True)
+
+    confdir = os.path.join(builddir, "config")
+    layerdir = os.path.join(builddir, "layers")
+
+    os.makedirs(confdir)
+    os.makedirs(layerdir)
+
+    bb.process.run("git -C {} init -b main".format(confdir))
+    # Make sure commiting doesn't fail if no default git user is configured on the machine
+    bb.process.run("git -C {} config user.name bitbake-setup".format(confdir))
+    bb.process.run("git -C {} config user.email bitbake-setup@not.set".format(confdir))
+    bb.process.run("git -C {} commit --no-verify --allow-empty -m 'Initial commit'".format(confdir))
+
+    bb.event.register("bb.build.TaskProgress", handle_task_progress, data=d)
+
+    write_config(upstream_config, confdir)
+    commit_config(confdir)
+    update_build(upstream_config, confdir, builddir, layerdir, d)
+
+    bb.event.remove("bb.build.TaskProgress", None)
+
+def get_diff(file1, file2):
+    try:
+        bb.process.run('diff -uNr {} {}'.format(file1, file2))
+    except bb.process.ExecutionError as e:
+        if e.exitcode == 1:
+            return e.stdout
+        else:
+            raise e
+    return None
+
+def are_layers_changed(layers, layerdir, d):
+    changed = False
+    for r_name in layers:
+        r_data = layers[r_name]
+        repodir = r_data["path"]
+
+        r_remote = r_data['git-remote']
+        rev = r_remote['rev']
+        branch = r_remote.get('branch', None)
+        remotes = r_remote['remotes']
+
+        for remote in remotes:
+            type,host,path,user,pswd,params = bb.fetch.decodeurl(remotes[remote]["uri"])
+            fetchuri = bb.fetch.encodeurl(('git',host,path,user,pswd,params))
+            if branch:
+                fetcher = bb.fetch.FetchData("{};protocol={};rev={};branch={};destsuffix={}".format(fetchuri,type,rev,branch,repodir), d)
+            else:
+                fetcher = bb.fetch.FetchData("{};protocol={};rev={};nobranch=1;destsuffix={}".format(fetchuri,type,rev,repodir), d)
+            upstream_revision = fetcher.method.latest_revision(fetcher, d, 'default')
+            rev_parse_result = bb.process.run('git -C {} rev-parse HEAD'.format(os.path.join(layerdir, repodir)))
+            local_revision = rev_parse_result[0].strip()
+            if upstream_revision != local_revision:
+                changed = True
+                print('Layer repository {} checked out into {} updated revision {} from {} to {}'.format(remotes[remote]["uri"], os.path.join(layerdir, repodir), rev, local_revision, upstream_revision))
+
+    return changed
+
+def build_status(top_dir, settings, args, d, update=False):
+    builddir = args.build_dir
+
+    confdir = os.path.join(builddir, "config")
+    layerdir = os.path.join(builddir, "layers")
+
+    current_upstream_config = json.load(open(os.path.join(confdir, "config-upstream.json")))
+
+    args.config = current_upstream_config['non-interactive-cmdline-options']
+    args.non_interactive = True
+    args.skip_selection = current_upstream_config['skip-selection']
+    source_overrides = current_upstream_config["source-overrides"]
+    new_upstream_config = obtain_config(top_dir, settings, args, source_overrides, d)
+
+    write_config(new_upstream_config, confdir)
+    config_diff = bb.process.run('git -C {} diff'.format(confdir))[0]
+
+    if config_diff:
+        print('\nConfiguration in {} has changed:\n{}'.format(builddir, config_diff))
+        if update:
+            commit_config(confdir)
+            update_build(new_upstream_config, confdir, builddir, layerdir, d)
+        else:
+            bb.process.run('git -C {} restore config-upstream.json'.format(confdir))
+        return
+
+    if are_layers_changed(current_upstream_config["data"]["sources"], layerdir, d):
+        if update:
+            update_build(current_upstream_config, confdir, builddir, layerdir, d)
+        return
+
+    print("\nConfiguration in {} has not changed.".format(builddir))
+
+def build_update(top_dir, settings, args, d):
+    build_status(top_dir, settings, args, d, update=True)
+
+def do_fetch(fetcher, dir):
+    # git fetcher simply dumps git output to stdout; in bitbake context that is redirected to temp/log.do_fetch
+    # and we need to set up smth similar here
+    fetchlogdir = os.path.join(dir, 'logs')
+    os.makedirs(fetchlogdir, exist_ok=True)
+    fetchlog = os.path.join(fetchlogdir, 'fetch_log.{}'.format(datetime.datetime.now().strftime("%Y%m%d%H%M%S")))
+    with open(fetchlog, 'a') as f:
+        oldstdout = sys.stdout
+        sys.stdout = f
+        fetcher.download()
+        fetcher.unpack(dir)
+        sys.stdout = oldstdout
+
+def update_registry(registry, cachedir, d):
+    registrydir = 'configurations'
+    if registry.startswith("."):
+        full_registrydir = os.path.join(os.getcwd(), registry, registrydir)
+    elif registry.startswith("/"):
+        full_registrydir = os.path.join(registry, registrydir)
+    else:
+        full_registrydir = os.path.join(cachedir, registrydir)
+        print("Fetching configuration registry\n    {}\ninto\n    {}".format(registry, full_registrydir))
+        fetcher = bb.fetch.Fetch(["{};destsuffix={}".format(registry, registrydir)], d)
+        do_fetch(fetcher, cachedir)
+    return full_registrydir
+
+def has_expired(expiry_date):
+    if expiry_date:
+        return datetime.datetime.now() > datetime.datetime.fromisoformat(expiry_date)
+    return False
+
+def list_registry(registry_path, with_expired):
+    json_data = {}
+
+    for root, dirs, files in os.walk(registry_path):
+        for f in files:
+            if f.endswith('.conf.json'):
+                config_name = get_config_name(f)
+                config_data = json.load(open(os.path.join(root, f)))
+                config_desc = config_data["description"]
+                expiry_date = config_data.get("expires", None)
+                if expiry_date:
+                    if with_expired or not has_expired(expiry_date):
+                        json_data[config_name] = {"description": config_desc, "expires": expiry_date}
+                else:
+                    json_data[config_name] = {"description": config_desc}
+    return json_data
+
+def list_configs(top_dir, settings, args, d):
+    registry_path = update_registry(settings["default"]["registry"], cache_dir(top_dir), d)
+    json_data = list_registry(registry_path, args.with_expired)
+    print("\nAvailable configurations:")
+    for config_name, config_data in json_data.items():
+        expiry_date = config_data.get("expires", None)
+        config_desc = config_data["description"]
+        if expiry_date:
+           if args.with_expired or not has_expired(expiry_date):
+               print("{}\t{} (supported until {})".format(config_name, config_desc, expiry_date))
+        else:
+           print("{}\t{}".format(config_name, config_desc))
+    print("\nRun 'init' with one of the above configuration identifiers to set up a build.")
+
+    if args.write_json:
+        with open(args.write_json, 'w') as f:
+            json.dump(json_data, f, sort_keys=True, indent=4)
+        print("Available configurations written into {}".format(args.write_json))
+
+def install_buildtools(top_dir, settings, args, d):
+    buildtools_install_dir = os.path.join(args.build_dir, 'buildtools')
+    if os.path.exists(buildtools_install_dir):
+        if not args.force:
+            print("Buildtools are already installed in {}.".format(buildtools_install_dir))
+            env_scripts = glob.glob(os.path.join(buildtools_install_dir, 'environment-setup-*'))
+            if env_scripts:
+                print("If you wish to use them, you need to source the environment setup script e.g.")
+                for s in env_scripts:
+                    print("$ . {}".format(s))
+            print("You can also re-run bitbake-setup install-buildtools with --force option to force a reinstallation.")
+            return
+        shutil.rmtree(buildtools_install_dir)
+
+    install_buildtools = os.path.join(args.build_dir, 'layers/oe-scripts/install-buildtools')
+    buildtools_download_dir = os.path.join(args.build_dir, 'buildtools-downloads/{}'.format(time.strftime("%Y%m%d%H%M%S")))
+    print("Buildtools archive is downloaded into {} and its content installed into {}".format(buildtools_download_dir, buildtools_install_dir))
+    subprocess.check_call("{} -d {} --downloads-directory {}".format(install_buildtools, buildtools_install_dir, buildtools_download_dir), shell=True)
+
+def default_settings_path(top_dir):
+    return os.path.join(top_dir, 'settings.conf')
+
+def create_siteconf(top_dir, non_interactive=True):
+    siteconfpath = os.path.join(top_dir, 'site.conf')
+    print('A common site.conf file will be created, please edit or replace before running builds\n    {}\n'.format(siteconfpath))
+    if not non_interactive:
+        y_or_n = input('Proceed? (y/N): ')
+        if y_or_n != 'y':
+            exit()
+
+    os.makedirs(os.path.dirname(top_dir), exist_ok=True)
+    if os.path.exists(siteconfpath):
+        backup_siteconf = siteconfpath + "-backup.{}".format(time.strftime("%Y%m%d%H%M%S"))
+        os.rename(siteconfpath, backup_siteconf)
+        print("Previous settings are in {}".format(backup_siteconf))
+    with open(siteconfpath, 'w') as siteconffile:
+        siteconffile.write('# This file is intended for build host-specific bitbake settings\n')
+
+def global_settings_path(args):
+    return os.path.abspath(args.global_settings) if args.global_settings else os.path.join(os.path.expanduser('~'), '.config', 'bitbake-setup', 'settings.conf')
+
+def load_settings(settings_path):
+    settings = configparser.ConfigParser()
+    if os.path.exists(settings_path):
+        print('Loading settings from\n    {}\n'.format(settings_path))
+        settings.read_file(open(settings_path))
+    return settings
+
+def change_setting(top_dir, args):
+    if vars(args)['global']:
+        settings_path = global_settings_path(args)
+    else:
+        settings_path = default_settings_path(top_dir)
+    settings = load_settings(settings_path)
+
+    if args.subcommand == 'set':
+        if args.section not in settings.keys():
+            settings[args.section] = {}
+        settings[args.section][args.setting] = args.value
+        print(f"From section '{args.section}' the setting '{args.setting}' was changed to '{args.value}'")
+    if args.subcommand == 'unset':
+        if args.section in settings.keys() and args.setting in settings[args.section].keys():
+            del settings[args.section][args.setting]
+            print(f"From section '{args.section}' the setting '{args.setting}' has been removed")
+
+    os.makedirs(os.path.dirname(settings_path), exist_ok=True)
+    with open(settings_path, 'w') as settingsfile:
+        settings.write(settingsfile)
+    print(f"Settings written to {settings_path}")
+
+def list_settings(all_settings):
+    for section, section_settings in all_settings.items():
+        for key, value in section_settings.items():
+            print("{} {} {}".format(section, key, value))
+
+def settings_func(top_dir, all_settings, args):
+    if args.subcommand == 'list':
+        list_settings(all_settings)
+    elif args.subcommand == 'set' or args.subcommand == 'unset':
+        change_setting(top_dir, args)
+
+def get_build_dir_via_bbpath():
+    bbpath = os.environ.get('BBPATH')
+    if bbpath:
+        bitbake_dir = os.path.normpath(bbpath.split(':')[0])
+        if os.path.exists(os.path.join(bitbake_dir,'init-build-env')):
+            build_dir = os.path.dirname(bitbake_dir)
+            return build_dir
+    return None
+
+def get_top_dir(args, settings):
+    build_dir_via_bbpath = get_build_dir_via_bbpath()
+    if build_dir_via_bbpath:
+        top_dir = os.path.dirname(build_dir_via_bbpath)
+        if os.path.exists(default_settings_path(top_dir)):
+            return top_dir
+
+    if hasattr(args, 'build_dir'):
+        top_dir = os.path.dirname(os.path.normpath(args.build_dir))
+        return top_dir
+
+    top_dir_prefix = settings['default']['top-dir-prefix']
+    top_dir_name = settings['default']['top-dir-name']
+    return os.path.join(top_dir_prefix, top_dir_name)
+
+def merge_settings(builtin_settings, global_settings, local_settings, cmdline_settings):
+    all_settings = builtin_settings
+
+    for s in (global_settings, local_settings):
+        for section, section_settings in s.items():
+            for setting, value in section_settings.items():
+                all_settings[section][setting] = value
+
+    for (section, setting, value) in cmdline_settings:
+        all_settings[section][setting] = value
+
+    return all_settings
+
+def main():
+    def add_build_dir_arg(parser):
+        build_dir = get_build_dir_via_bbpath()
+        if build_dir:
+            parser.add_argument('--build-dir', default=build_dir, help="Path to the build, default is %(default)s via BBPATH")
+        else:
+            parser.add_argument('--build-dir', required=True, help="Path to the build")
+
+    parser = argparse.ArgumentParser(
+        description="BitBake setup utility. Run with 'init' argument to get started.",
+        epilog="Use %(prog)s <subcommand> --help to get help on a specific command"
+        )
+    parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
+    parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
+    parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR')
+    parser.add_argument('--no-network', action='store_true', help='Do not check whether configuration repositories and layer repositories have been updated; use only the local cache.')
+    parser.add_argument('--global-settings', action='store', metavar='PATH', help='Path to the global settings file.')
+    parser.add_argument('--setting', default=[], action='append', dest='cmdline_settings',
+                        nargs=3, metavar=('SECTION', 'SETTING', 'VALUE'),
+                        help='Modify a setting (for this bitbake-setup invocation only), for example "--setting default top-dir-prefix /path/to/top/dir".')
+
+    subparsers = parser.add_subparsers()
+
+    parser_list = subparsers.add_parser('list', help='List available configurations')
+    parser_list.add_argument('--with-expired', action='store_true', help='List also configurations that are no longer supported due to reaching their end-of-life dates.')
+    parser_list.add_argument('--write-json', action='store', help='Write available configurations into a json file so they can be programmatically processed.')
+    parser_list.set_defaults(func=list_configs)
+
+    parser_init = subparsers.add_parser('init', help='Select a configuration and initialize a build from it')
+    parser_init.add_argument('config', nargs='*', help="path/URL/id to a configuration file (use 'list' command to get available ids), followed by configuration options. Bitbake-setup will ask to choose from available choices if command line doesn't completely specify them.")
+    parser_init.add_argument('--non-interactive', action='store_true', help='Do not ask to interactively choose from available options; if bitbake-setup cannot make a decision it will stop with a failure.')
+    parser_init.add_argument('--source-overrides', action='store', help='Override sources information (repositories/revisions) with values from a local json file.')
+    parser_init.add_argument('--build-dir-name', action='store', help='A custom build directory name under the top directory.')
+    parser_init.add_argument('--skip-selection', action='append', help='Do not select and set an option/fragment from available choices; the resulting bitbake configuration may be incomplete.')
+    parser_init.set_defaults(func=init_config)
+
+    parser_status = subparsers.add_parser('status', help='Check if the build needs to be synchronized with configuration')
+    add_build_dir_arg(parser_status)
+    parser_status.set_defaults(func=build_status)
+
+    parser_update = subparsers.add_parser('update', help='Update a build to be in sync with configuration')
+    add_build_dir_arg(parser_update)
+    parser_update.set_defaults(func=build_update)
+
+    parser_install_buildtools = subparsers.add_parser('install-buildtools', help='Install buildtools which can help fulfil missing or incorrect dependencies on the host machine')
+    add_build_dir_arg(parser_install_buildtools)
+    parser_install_buildtools.add_argument('--force', action='store_true', help='Force a reinstall of buildtools over the previous installation.')
+    parser_install_buildtools.set_defaults(func=install_buildtools)
+
+    parser_settings_arg_global = argparse.ArgumentParser(add_help=False)
+    parser_settings_arg_global.add_argument('--global', action='store_true', help="Modify the setting in a global settings file, rather than one specific to a top directory")
+
+    parser_settings = subparsers.add_parser('settings', parents=[parser_settings_arg_global],
+                                            help='List current settings, or set or unset a setting in a settings file (e.g. the default prefix and name of the top directory, the location of build configuration registry, downloads directory and other settings specific to a top directory)')
+    parser_settings.set_defaults(func=settings_func)
+
+    subparser_settings = parser_settings.add_subparsers(dest="subcommand", required=True, help="The action to perform on the settings file")
+
+    parser_settings_list = subparser_settings.add_parser('list',
+                                                         help="List all settings with their values")
+
+    parser_settings_set = subparser_settings.add_parser('set', parents=[parser_settings_arg_global],
+                                                        help="In a Section, set a setting to a certain value")
+    parser_settings_set.add_argument("section", metavar="<section>", help="Section in a settings file, typically 'default'")
+    parser_settings_set.add_argument("setting", metavar="<setting>", help="Name of a setting")
+    parser_settings_set.add_argument("value", metavar="<value>", help="The setting value")
+
+    parser_settings_unset = subparser_settings.add_parser('unset', parents=[parser_settings_arg_global],
+                                                          help="Unset a setting, e.g. 'bitbake-setup settings unset default registry' would revert to the registry setting in a global settings file")
+    parser_settings_unset.add_argument("section", metavar="<section>", help="Section in a settings file, typically 'default'")
+    parser_settings_unset.add_argument("setting", metavar="<setting>", help="The setting to remove")
+
+    args = parser.parse_args()
+
+    logging.basicConfig(stream=sys.stdout)
+    if args.debug:
+        logger.setLevel(logging.DEBUG)
+    elif args.quiet:
+        logger.setLevel(logging.ERROR)
+
+    # Need to re-run logger_create with color argument
+    # (will be the same logger since it has the same name)
+    bb.msg.logger_create('bitbake-setup', output=sys.stdout,
+                         color=args.color,
+                         level=logger.getEffectiveLevel())
+
+    if 'func' in args:
+        if hasattr(args, 'build_dir'):
+            if not os.path.exists(os.path.join(args.build_dir,'build', 'init-build-env')):
+                print("Not a valid build directory: build/init-build-env does not exist in {}".format(args.build_dir))
+                return
+
+        if not hasattr(args, 'non_interactive'):
+            args.non_interactive = True
+
+        builtin_settings = {}
+        builtin_settings['default'] = {
+                         'top-dir-prefix':os.path.expanduser('~'),
+                         'top-dir-name':'bitbake-builds',
+                         'registry':default_registry,
+                         }
+
+        global_settings = load_settings(global_settings_path(args))
+        top_dir = get_top_dir(args, merge_settings(builtin_settings, global_settings, {}, args.cmdline_settings))
+
+        # This cannot be set with the rest of the builtin settings as top_dir needs to be determined first
+        builtin_settings['default']['dl-dir'] = os.path.join(top_dir, '.bitbake-setup-downloads')
+
+        topdir_settings = load_settings(default_settings_path(top_dir))
+        all_settings = merge_settings(builtin_settings, global_settings, topdir_settings, args.cmdline_settings)
+
+        if args.func == settings_func:
+            settings_func(top_dir, all_settings, args)
+            return
+
+        print('Bitbake-setup is using {} as top directory ("bitbake-setup settings --help" shows how to change it).\n'.format(top_dir, global_settings_path(args)))
+
+        d = init_bb_cache(top_dir, all_settings, args)
+        args.func(top_dir, all_settings, args, d)
+        save_bb_cache()
+    else:
+        from argparse import Namespace
+        parser.print_help()
+
+main()
diff --git a/bitbake/default-registry/configurations/oe-nodistro.conf.json b/bitbake/default-registry/configurations/oe-nodistro.conf.json
new file mode 100644
index 0000000000..7619738b1d
--- /dev/null
+++ b/bitbake/default-registry/configurations/oe-nodistro.conf.json
@@ -0,0 +1,57 @@
+{
+    "description": "OpenEmbedded - 'nodistro' basic configuration",
+    "sources": {
+        "bitbake": {
+            "git-remote": {
+                "remotes": {
+                    "origin": {
+                        "uri": "git://git.openembedded.org/bitbake;protocol=https"
+                    }
+                },
+                "branch": "master",
+                "rev": "master"
+            },
+            "path": "bitbake"
+        },
+        "openembedded-core": {
+            "git-remote": {
+                "remotes": {
+                    "origin": {
+                        "uri": "git://git.openembedded.org/openembedded-core;protocol=https"
+                    }
+                },
+                "branch": "master",
+                "rev": "master"
+            },
+            "path": "openembedded-core"
+        },
+        "yocto-docs": {
+            "git-remote": {
+                "remotes": {
+                    "origin": {
+                        "uri": "git://git.yoctoproject.org/yocto-docs;protocol=https"
+                    }
+                },
+                "branch": "master",
+                "rev": "master"
+            },
+            "path": "yocto-docs"
+        }
+    },
+    "bitbake-setup": {
+        "configurations": [
+        {
+            "name": "nodistro",
+            "description": "OpenEmbedded 'nodistro'",
+            "bb-layers": ["openembedded-core/meta"],
+            "oe-fragments-one-of": {
+                "machine": {
+                    "description": "Target machines",
+                    "options" : ["machine/qemux86-64", "machine/qemuarm64", "machine/qemuriscv64"]
+                }
+            }
+        }
+        ]
+    },
+    "version": "1.0"
+}
diff --git a/bitbake/default-registry/configurations/poky-master.conf.json b/bitbake/default-registry/configurations/poky-master.conf.json
new file mode 100644
index 0000000000..60531ba024
--- /dev/null
+++ b/bitbake/default-registry/configurations/poky-master.conf.json
@@ -0,0 +1,82 @@
+{
+    "description": "Poky - The Yocto Project testing distribution configurations and hardware test platforms",
+    "sources": {
+        "bitbake": {
+            "git-remote": {
+                "remotes": {
+                    "origin": {
+                        "uri": "git://git.openembedded.org/bitbake;protocol=https"
+                    }
+                },
+                "branch": "master",
+                "rev": "master"
+            },
+            "path": "bitbake"
+        },
+        "openembedded-core": {
+            "git-remote": {
+                "remotes": {
+                    "origin": {
+                        "uri": "git://git.openembedded.org/openembedded-core;protocol=https"
+                    }
+                },
+                "branch": "master",
+                "rev": "master"
+            },
+            "path": "openembedded-core"
+        },
+        "meta-yocto": {
+            "git-remote": {
+                "remotes": {
+                    "origin": {
+                        "uri": "git://git.yoctoproject.org/meta-yocto;protocol=https"
+                    }
+                },
+                "branch": "master",
+                "rev": "master"
+            },
+            "path": "meta-yocto"
+        },
+        "yocto-docs": {
+            "git-remote": {
+                "remotes": {
+                    "origin": {
+                        "uri": "git://git.yoctoproject.org/yocto-docs;protocol=https"
+                    }
+                },
+                "branch": "master",
+                "rev": "master"
+            },
+            "path": "yocto-docs"
+        }
+    },
+    "bitbake-setup": {
+        "configurations": [
+        {
+            "bb-layers": ["openembedded-core/meta","meta-yocto/meta-yocto-bsp","meta-yocto/meta-poky"],
+            "oe-fragments-one-of": {
+                "machine": {
+                    "description": "Target machines",
+                    "options" : ["machine/qemux86-64", "machine/qemuarm64", "machine/qemuriscv64", "machine/genericarm64", "machine/genericx86-64"]
+                },
+                "distro": {
+                    "description": "Distribution configuration variants",
+                    "options" : ["distro/poky", "distro/poky-altcfg", "distro/poky-tiny"]
+                }
+            },
+            "configurations": [
+            {
+                "name": "poky",
+                "description": "Poky - The Yocto Project testing distribution"
+            },
+            {
+                "name": "poky-with-sstate",
+                "description": "Poky - The Yocto Project testing distribution with internet sstate acceleration. Use with caution as it requires a completely robust local network with sufficient bandwidth.",
+                "oe-fragments": ["core/yocto/sstate-mirror-cdn"]
+            }
+            ]
+        }
+        ]
+    },
+    "version": "1.0"
+}
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index fb87368f17..03f262ac16 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -320,7 +320,10 @@ class BBCooker:
                     except ImportError as e:
                         bb.fatal(""""Unable to use hash equivalence server at '%s' due to missing or incorrect python module:
 %s
-Please install the needed module on the build host, or use an environment containing it (e.g a pip venv or OpenEmbedded's buildtools tarball).
+Please install the needed module on the build host, or use an environment containing it:
+ - if you are using bitbake-setup, run 'bitbake-setup install-buildtools'
+ - openembedded-core layer contains 'scripts/install-buildtools' that can also be used
+ - or set up pip venv
 You can also remove the BB_HASHSERVE_UPSTREAM setting, but this may result in significantly longer build times as bitbake will be unable to reuse prebuilt sstate artefacts."""
                                  % (upstream, repr(e)))
                     except ConnectionError as e:
diff --git a/bitbake/lib/bb/tests/setup.py b/bitbake/lib/bb/tests/setup.py
new file mode 100644
index 0000000000..e320cdf56f
--- /dev/null
+++ b/bitbake/lib/bb/tests/setup.py
@@ -0,0 +1,358 @@
+#
+# Copyright BitBake Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+from bb.tests.fetch import FetcherTest
+import json
+
+class BitbakeSetupTest(FetcherTest):
+    def setUp(self):
+        super(BitbakeSetupTest, self).setUp()
+
+        self.registrypath = os.path.join(self.tempdir, "bitbake-setup-configurations")
+
+        os.makedirs(self.registrypath)
+        self.git_init(cwd=self.registrypath)
+        self.git('commit --allow-empty -m "Initial commit"', cwd=self.registrypath)
+
+        self.testrepopath = os.path.join(self.tempdir, "test-repo")
+        os.makedirs(self.testrepopath)
+        self.git_init(cwd=self.testrepopath)
+        self.git('commit --allow-empty -m "Initial commit"', cwd=self.testrepopath)
+
+        oeinitbuildenv = """BBPATH=$1
+export BBPATH
+PATH={}:$PATH
+""".format(os.path.join(self.testrepopath, 'scripts'))
+        self.add_file_to_testrepo('oe-init-build-env',oeinitbuildenv, script=True)
+
+        oesetupbuild = """#!/usr/bin/env python3
+import getopt
+import sys
+import os
+import shutil
+opts, args = getopt.getopt(sys.argv[2:], "c:b:", ["no-shell"])
+for option, value in opts:
+    if option == '-c':
+        template = value
+    if option == '-b':
+        builddir = value
+confdir = os.path.join(builddir, 'conf')
+os.makedirs(confdir, exist_ok=True)
+with open(os.path.join(confdir, 'conf-summary.txt'), 'w') as f:
+    f.write(template)
+shutil.copy(os.path.join(os.path.dirname(__file__), 'test-repo/test-file'), confdir)
+with open(os.path.join(builddir, 'init-build-env'), 'w') as f:
+    f.write("BBPATH={}\\nexport BBPATH\\nPATH={}:$PATH".format(builddir, os.path.join(os.path.dirname(__file__), 'test-repo/scripts')))
+"""
+        self.add_file_to_testrepo('scripts/oe-setup-build', oesetupbuild, script=True)
+
+        installbuildtools = """#!/usr/bin/env python3
+import getopt
+import sys
+import os
+
+opts, args = getopt.getopt(sys.argv[1:], "d:", ["downloads-directory="])
+for option, value in opts:
+    if option == '-d':
+        installdir = value
+
+print("Buildtools installed into {}".format(installdir))
+os.makedirs(installdir)
+"""
+        self.add_file_to_testrepo('scripts/install-buildtools', installbuildtools, script=True)
+
+        bitbakeconfigbuild = """#!/usr/bin/env python3
+import os
+import sys
+confdir = os.path.join(os.environ['BBPATH'], 'conf')
+fragment = sys.argv[2]
+with open(os.path.join(confdir, fragment), 'w') as f:
+    f.write('')
+"""
+        self.add_file_to_testrepo('scripts/bitbake-config-build', bitbakeconfigbuild, script=True)
+
+        sometargetexecutable_template = """#!/usr/bin/env python3
+import os
+print("This is {}")
+print("BBPATH is {{}}".format(os.environ["BBPATH"]))
+"""
+        for e_name in ("some-target-executable-1", "some-target-executable-2"):
+            sometargetexecutable = sometargetexecutable_template.format(e_name)
+            self.add_file_to_testrepo('scripts/{}'.format(e_name), sometargetexecutable, script=True)
+
+    def runbbsetup(self, cmd):
+        bbsetup = os.path.abspath(os.path.dirname(__file__) +  "/../../../bin/bitbake-setup")
+        return bb.process.run("{} --global-settings {} {}".format(bbsetup, os.path.join(self.tempdir, 'global-config'), cmd))
+
+    def add_json_config_to_registry(self, name, rev, branch):
+        config = """
+{
+    "sources": {
+        "test-repo": {
+            "git-remote": {
+                "remotes": {
+                    "origin": {
+                        "uri": "file://%s"
+                    }
+                },
+                "branch": "%s",
+                "rev": "%s"
+            },
+            "path": "test-repo"
+        }
+    },
+    "description": "Test configuration",
+    "bitbake-setup": {
+        "configurations": [
+            {
+                "name": "gadget",
+                "description": "Gadget build configuration",
+                "oe-template": "test-configuration-gadget",
+                "oe-fragments": ["test-fragment-1"]
+            },
+            {
+                "name": "gizmo",
+                "description": "Gizmo build configuration",
+                "oe-template": "test-configuration-gizmo",
+                "oe-fragments": ["test-fragment-2"]
+            },
+            {
+                "name": "gizmo-env-passthrough",
+                "description": "Gizmo build configuration with environment-passthrough",
+                "bb-layers": ["layerC","layerD/meta-layer"],
+                "oe-fragments": ["test-fragment-1"],
+                "bb-env-passthrough-additions": [
+                    "BUILD_ID",
+                    "BUILD_DATE",
+                    "BUILD_SERVER"
+                ]
+            },
+            {
+                "name": "gizmo-no-fragment",
+                "description": "Gizmo no-fragment template-only build configuration",
+                "oe-template": "test-configuration-gizmo"
+            },
+            {
+                "name": "gadget-notemplate",
+                "description": "Gadget notemplate build configuration",
+                "bb-layers": ["layerA","layerB/meta-layer"],
+                "oe-fragments": ["test-fragment-1"]
+            },
+            {
+                "name": "gizmo-notemplate",
+                "description": "Gizmo notemplate build configuration",
+                "bb-layers": ["layerC","layerD/meta-layer"],
+                "oe-fragments": ["test-fragment-2"]
+            },
+            {
+                "name": "gizmo-notemplate-with-thisdir",
+                "description": "Gizmo notemplate build configuration using THISDIR",
+                "bb-layers": ["layerC","layerD/meta-layer","{THISDIR}/layerE/meta-layer"],
+                "oe-fragments": ["test-fragment-2"]
+            }
+        ]
+    },
+    "version": "1.0"
+}
+""" % (self.testrepopath, branch, rev)
+        os.makedirs(os.path.join(self.registrypath, os.path.dirname(name)), exist_ok=True)
+        with open(os.path.join(self.registrypath, name), 'w') as f:
+            f.write(config)
+        self.git('add {}'.format(name), cwd=self.registrypath)
+        self.git('commit -m "Adding {}"'.format(name), cwd=self.registrypath)
+        return json.loads(config)
+
+    def add_file_to_testrepo(self, name, content, script=False):
+        fullname = os.path.join(self.testrepopath, name)
+        os.makedirs(os.path.join(self.testrepopath, os.path.dirname(name)), exist_ok=True)
+        with open(fullname, 'w') as f:
+            f.write(content)
+        if script:
+            import stat
+            st = os.stat(fullname)
+            os.chmod(fullname, st.st_mode | stat.S_IEXEC)
+        self.git('add {}'.format(name), cwd=self.testrepopath)
+        self.git('commit -m "Adding {}"'.format(name), cwd=self.testrepopath)
+
+    def check_builddir_files(self, buildpath, test_file_content, json_config):
+        with open(os.path.join(buildpath, 'layers', 'test-repo', 'test-file')) as f:
+            self.assertEqual(f.read(), test_file_content)
+        bitbake_config = json_config["bitbake-config"]
+        bb_build_path = os.path.join(buildpath, 'build')
+        bb_conf_path = os.path.join(bb_build_path, 'conf')
+        self.assertTrue(os.path.exists(os.path.join(bb_build_path, 'init-build-env')))
+
+        if "oe-template" in bitbake_config:
+            with open(os.path.join(bb_conf_path, 'conf-summary.txt')) as f:
+                self.assertEqual(f.read(), bitbake_config["oe-template"])
+            with open(os.path.join(bb_conf_path, 'test-file')) as f:
+                self.assertEqual(f.read(), test_file_content)
+        else:
+            with open(os.path.join(bb_conf_path, 'conf-summary.txt')) as f:
+                self.assertIn(bitbake_config["description"], f.read())
+            with open(os.path.join(bb_conf_path, 'bblayers.conf')) as f:
+                bblayers = f.read()
+                for l in bitbake_config["bb-layers"]:
+                    if l.startswith('{THISDIR}/'):
+                        thisdir_layer = os.path.join(
+                            os.path.dirname(json_config["path"]),
+                            l.removeprefix("{THISDIR}/"),
+                        )
+                        self.assertIn(thisdir_layer, bblayers)
+                    else:
+                        self.assertIn(os.path.join(buildpath, "layers", l), bblayers)
+
+        if 'oe-fragment' in bitbake_config.keys():
+            for f in bitbake_config["oe-fragments"]:
+                self.assertTrue(os.path.exists(os.path.join(bb_conf_path, f)))
+
+        if 'bb-environment-passthrough' in bitbake_config.keys():
+            with open(os.path.join(bb_build_path, 'init-build-env'), 'r') as f:
+                init_build_env = f.read()
+            self.assertTrue('BB_ENV_PASSTHROUGH_ADDITIONS' in init_build_env)
+            self.assertTrue('BUILD_ID' in init_build_env)
+            self.assertTrue('BUILD_DATE' in init_build_env)
+            self.assertTrue('BUILD_SERVER' in init_build_env)
+            # a more throrough test could be to initialize a bitbake build-env, export FOO to the shell environment, set the env-passthrough on it and finally check against 'bitbake-getvar FOO'
+
+
+    def test_setup(self):
+        # unset BBPATH to ensure tests run in isolation from the existing bitbake environment
+        import os
+        if 'BBPATH' in os.environ:
+            del os.environ['BBPATH']
+
+        # check that no arguments works
+        self.runbbsetup("")
+
+        # check that --help works
+        self.runbbsetup("--help")
+
+        # set up global location for top-dir-prefix
+        out = self.runbbsetup("settings set --global default top-dir-prefix {}".format(self.tempdir))
+        settings_path = "{}/global-config".format(self.tempdir)
+        self.assertIn(settings_path, out[0])
+        self.assertIn("From section 'default' the setting 'top-dir-prefix' was changed to", out[0])
+        self.assertIn("Settings written to".format(settings_path), out[0])
+        out = self.runbbsetup("settings set --global default dl-dir {}".format(os.path.join(self.tempdir, 'downloads')))
+        self.assertIn("From section 'default' the setting 'dl-dir' was changed to", out[0])
+        self.assertIn("Settings written to".format(settings_path), out[0])
+
+        # check that writing settings works and then adjust them to point to
+        # test registry repo
+        out = self.runbbsetup("settings set default registry 'git://{};protocol=file;branch=master;rev=master'".format(self.registrypath))
+        settings_path = "{}/bitbake-builds/settings.conf".format(self.tempdir)
+        self.assertIn(settings_path, out[0])
+        self.assertIn("From section 'default' the setting 'registry' was changed to", out[0])
+        self.assertIn("Settings written to".format(settings_path), out[0])
+
+        # check that listing settings works
+        out = self.runbbsetup("settings list")
+        self.assertIn("default top-dir-prefix {}".format(self.tempdir), out[0])
+        self.assertIn("default dl-dir {}".format(os.path.join(self.tempdir, 'downloads')), out[0])
+        self.assertIn("default registry {}".format('git://{};protocol=file;branch=master;rev=master'.format(self.registrypath)), out[0])
+
+        # check that 'list' produces correct output with no configs, one config and two configs
+        out = self.runbbsetup("list")
+        self.assertNotIn("test-config-1", out[0])
+        self.assertNotIn("test-config-2", out[0])
+
+        json_1 = self.add_json_config_to_registry('test-config-1.conf.json', 'master', 'master')
+        out = self.runbbsetup("list")
+        self.assertIn("test-config-1", out[0])
+        self.assertNotIn("test-config-2", out[0])
+
+        json_2 = self.add_json_config_to_registry('config-2/test-config-2.conf.json', 'master', 'master')
+        out = self.runbbsetup("list --write-json={}".format(os.path.join(self.tempdir, "test-configs.json")))
+        self.assertIn("test-config-1", out[0])
+        self.assertIn("test-config-2", out[0])
+        with open(os.path.join(self.tempdir, "test-configs.json")) as f:
+            json_configs = json.load(f)
+        self.assertIn("test-config-1", json_configs)
+        self.assertIn("test-config-2", json_configs)
+
+        # check that init/status/update work
+        # (the latter two should do nothing and say that config hasn't changed)
+        test_file_content = 'initial\n'
+        self.add_file_to_testrepo('test-file', test_file_content)
+
+        # test-config-1 is tested as a registry config, test-config-2 as a local file
+        test_configurations = {'test-config-1': {'cmdline': 'test-config-1',
+                                                 'buildconfigs':('gadget','gizmo',
+                                                                 'gizmo-env-passthrough',
+                                                                 'gizmo-no-fragment',
+                                                                 'gadget-notemplate','gizmo-notemplate')},
+                               'test-config-2': {'cmdline': os.path.join(self.registrypath,'config-2/test-config-2.conf.json'),
+                                                 'buildconfigs': ('gadget','gizmo',
+                                                                  'gizmo-env-passthrough',
+                                                                  'gizmo-no-fragment',
+                                                                  'gadget-notemplate','gizmo-notemplate',
+                                                                  'gizmo-notemplate-with-thisdir')}
+                               }
+        for cf, v in test_configurations.items():
+            for c in v['buildconfigs']:
+                out = self.runbbsetup("init --non-interactive {} {}".format(v['cmdline'], c))
+                buildpath = os.path.join(self.tempdir, 'bitbake-builds', '{}-{}'.format(cf, c))
+                with open(os.path.join(buildpath, 'config', "config-upstream.json")) as f:
+                    config_upstream = json.load(f)
+                self.check_builddir_files(buildpath, test_file_content, config_upstream)
+                os.environ['BBPATH'] = os.path.join(buildpath, 'build')
+                out = self.runbbsetup("status")
+                self.assertIn("Configuration in {} has not changed".format(buildpath), out[0])
+                out = self.runbbsetup("update")
+                self.assertIn("Configuration in {} has not changed".format(buildpath), out[0])
+
+        # install buildtools
+        out = self.runbbsetup("install-buildtools")
+        self.assertIn("Buildtools installed into", out[0])
+        self.assertTrue(os.path.exists(os.path.join(buildpath, 'buildtools')))
+
+        # change a file in the test layer repo, make a new commit and
+        # test that status/update correctly report the change and update the config
+        prev_test_file_content = test_file_content
+        test_file_content = 'modified\n'
+        self.add_file_to_testrepo('test-file', test_file_content)
+        for c in ('gadget', 'gizmo',
+                  'gizmo-env-passthrough',
+                  'gizmo-no-fragment',
+                  'gadget-notemplate', 'gizmo-notemplate'):
+            buildpath = os.path.join(self.tempdir, 'bitbake-builds', 'test-config-1-{}'.format(c))
+            os.environ['BBPATH'] = os.path.join(buildpath, 'build')
+            out = self.runbbsetup("status")
+            self.assertIn("Layer repository file://{} checked out into {}/layers/test-repo updated revision master from".format(self.testrepopath, buildpath), out[0])
+            out = self.runbbsetup("update")
+            if c in ('gadget', 'gizmo'):
+                self.assertIn("Existing bitbake configuration directory renamed to {}/build/conf-backup.".format(buildpath), out[0])
+                self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0])
+            with open(os.path.join(buildpath, 'config', "config-upstream.json")) as f:
+                config_upstream = json.load(f)
+            self.check_builddir_files(buildpath, test_file_content, config_upstream)
+
+        # make a new branch in the test layer repo, change a file on that branch,
+        # make a new commit, update the top level json config to refer to that branch,
+        # and test that status/update correctly report the change and update the config
+        prev_test_file_content = test_file_content
+        test_file_content = 'modified-in-branch\n'
+        branch = "another-branch"
+        self.git('checkout -b {}'.format(branch), cwd=self.testrepopath)
+        self.add_file_to_testrepo('test-file', test_file_content)
+        json_1 = self.add_json_config_to_registry('test-config-1.conf.json', branch, branch)
+        for c in ('gadget', 'gizmo',
+                  'gizmo-env-passthrough',
+                  'gizmo-no-fragment',
+                  'gadget-notemplate', 'gizmo-notemplate'):
+            buildpath = os.path.join(self.tempdir, 'bitbake-builds', 'test-config-1-{}'.format(c))
+            os.environ['BBPATH'] = os.path.join(buildpath, 'build')
+            out = self.runbbsetup("status")
+            self.assertIn("Configuration in {} has changed:".format(buildpath), out[0])
+            self.assertIn('-                    "rev": "master"\n+                    "rev": "another-branch"', out[0])
+            out = self.runbbsetup("update")
+            if c in ('gadget', 'gizmo'):
+                self.assertIn("Existing bitbake configuration directory renamed to {}/build/conf-backup.".format(buildpath), out[0])
+                self.assertIn('-{}+{}'.format(prev_test_file_content, test_file_content), out[0])
+            with open(os.path.join(buildpath, 'config', "config-upstream.json")) as f:
+                config_upstream = json.load(f)
+            self.check_builddir_files(buildpath, test_file_content, config_upstream)
diff --git a/bitbake/lib/bb/ui/knotty.py b/bitbake/lib/bb/ui/knotty.py
index 492ea20763..00258c80ff 100644
--- a/bitbake/lib/bb/ui/knotty.py
+++ b/bitbake/lib/bb/ui/knotty.py
@@ -169,7 +169,6 @@ class TerminalFilter(object):
         self.stdinbackup = None
         self.interactive = sys.stdout.isatty()
         self.footer_present = False
-        self.lastpids = []
         self.lasttime = time.time()
         self.quiet = quiet
 
@@ -254,7 +253,6 @@ class TerminalFilter(object):
             return
         activetasks = self.helper.running_tasks
         failedtasks = self.helper.failed_tasks
-        runningpids = self.helper.running_pids
         currenttime = time.time()
         deltatime = currenttime - self.lasttime
 
@@ -283,7 +281,7 @@ class TerminalFilter(object):
         self._footer_buf.seek(0)
 
         tasks = []
-        for t in runningpids:
+        for t in activetasks.keys():
             start_time = activetasks[t].get("starttime", None)
             if start_time:
                 msg = "%s - %s (pid %s)" % (activetasks[t]["title"], self.elapsed(currenttime - start_time), activetasks[t]["pid"])
@@ -358,7 +356,6 @@ class TerminalFilter(object):
                     content = "%s: %s" % (tasknum, task)
                     print(content, file=self._footer_buf)
                 lines = lines + self.getlines(content)
-        self.lastpids = runningpids[:]
         self.lastcount = self.helper.tasknumber_current
 
         # Clear footer and Print buffer.
diff --git a/bitbake/lib/bb/ui/uihelper.py b/bitbake/lib/bb/ui/uihelper.py
index e6983bd559..a223632471 100644
--- a/bitbake/lib/bb/ui/uihelper.py
+++ b/bitbake/lib/bb/ui/uihelper.py
@@ -13,7 +13,6 @@ class BBUIHelper:
         self.needUpdate = False
         self.running_tasks = {}
         # Running PIDs preserves the order tasks were executed in
-        self.running_pids = []
         self.failed_tasks = []
         self.pidmap = {}
         self.tasknumber_current = 0
@@ -23,7 +22,6 @@ class BBUIHelper:
         # PIDs are a bad idea as they can be reused before we process all UI events.
         # We maintain a 'fuzzy' match for TaskProgress since there is no other way to match
         def removetid(pid, tid):
-            self.running_pids.remove(tid)
             del self.running_tasks[tid]
             if self.pidmap[pid] == tid:
                 del self.pidmap[pid]
@@ -35,7 +33,6 @@ class BBUIHelper:
                 self.running_tasks[tid] = { 'title' : "mc:%s:%s %s" % (event._mc, event._package, event._task), 'starttime' : time.time(), 'pid' : event.pid }
             else:
                 self.running_tasks[tid] = { 'title' : "%s %s" % (event._package, event._task), 'starttime' : time.time(), 'pid' : event.pid }
-            self.running_pids.append(tid)
             self.pidmap[event.pid] = tid
             self.needUpdate = True
         elif isinstance(event, bb.build.TaskSucceeded):