7 files changed, 180 insertions, 9 deletions

diff --git a/meta/classes-recipe/go-mod-update-modules.bbclass b/meta/classes-recipe/go-mod-update-modules.bbclass
new file mode 100644
index 0000000000..5fccd0bb0d
--- /dev/null
+++ b/meta/classes-recipe/go-mod-update-modules.bbclass
@@ -0,0 +1,152 @@
+addtask do_update_modules after do_configure
+do_update_modules[nostamp] = "1"
+do_update_modules[network] = "1"
+
+# This class maintains two files, BPN-go-mods.inc and BPN-licenses.inc.
+#
+# -go-mods.inc will append SRC_URI with all of the Go modules that are
+# dependencies of this recipe.
+#
+# -licenses.inc will append LICENSE and LIC_FILES_CHKSUM with the found licenses
+# in the modules.
+#
+# These files are machine-generated and should not be modified.
+
+python do_update_modules() {
+    import subprocess, tempfile, json, re, urllib.parse
+    from oe.license import tidy_licenses
+    from oe.license_finder import find_licenses
+
+    def unescape_path(path):
+        """Unescape capital letters using exclamation points."""
+        return re.sub(r'!([a-z])', lambda m: m.group(1).upper(), path)
+
+    def fold_uri(uri):
+        """Fold URI for sorting shorter module paths before longer."""
+        return uri.replace(';', ' ').replace('/', '!')
+
+    def parse_existing_licenses():
+        hashes = {}
+        for url in d.getVar("LIC_FILES_CHKSUM").split():
+            (method, host, path, user, pswd, parm) = bb.fetch.decodeurl(url)
+            if "spdx" in parm and parm["spdx"] != "Unknown":
+                hashes[parm["md5"]] = urllib.parse.unquote_plus(parm["spdx"])
+        return hashes
+
+    bpn = d.getVar("BPN")
+    thisdir = d.getVar("THISDIR")
+    s_dir = d.getVar("S")
+
+    with tempfile.TemporaryDirectory(prefix='go-mod-') as mod_cache_dir:
+        notice = """
+# This file has been generated by go-mod-update-modules.bbclass
+#
+# Do not modify it by hand, as the contents will be replaced when
+# running the update-modules task.
+
+"""
+
+        env = dict(os.environ, GOMODCACHE=mod_cache_dir)
+
+        source = d.expand("${UNPACKDIR}/${GO_SRCURI_DESTSUFFIX}")
+        output = subprocess.check_output(("go", "mod", "edit", "-json"), cwd=source, env=env, text=True)
+        go_mod = json.loads(output)
+
+        output = subprocess.check_output(("go", "list", "-json=Dir,Module", "-deps", f"{go_mod['Module']['Path']}/..."), cwd=source, env=env, text=True)
+
+        #
+        # Licenses
+        #
+
+        # load hashes from the existing licenses.inc
+        extra_hashes = parse_existing_licenses()
+
+        # The output of this isn't actually valid JSON, but a series of dicts.
+        # Wrap in [] and join the dicts with ,
+        # Very frustrating that the json parser in python can't repeatedly
+        # parse from a stream.
+        pkgs = json.loads('[' + output.replace('}\n{', '},\n{') + ']')
+        # Collect licenses for the dependencies.
+        licenses = set()
+        lic_files_chksum = []
+        lic_files = {}
+
+        for pkg in pkgs:
+            mod = pkg.get('Module', None)
+            if not mod or mod.get('Main', False):
+                continue
+
+            mod_dir = mod['Dir']
+
+            if not mod_dir.startswith(mod_cache_dir):
+                continue
+
+            path = os.path.relpath(mod_dir, mod_cache_dir)
+
+            for license_name, license_file, license_md5 in find_licenses(mod['Dir'], d, first_only=True, extra_hashes=extra_hashes):
+                lic_files[os.path.join(path, license_file)] = (license_name, license_md5)
+
+        for lic_file in lic_files:
+            license_name, license_md5 = lic_files[lic_file]
+            if license_name == "Unknown":
+                bb.warn(f"Unknown license: {lic_file} {license_md5}")
+
+            licenses.add(lic_files[lic_file][0])
+            lic_files_chksum.append(
+                f'file://pkg/mod/{lic_file};md5={license_md5};spdx={urllib.parse.quote_plus(license_name)}')
+
+        licenses_filename = os.path.join(thisdir, f"{bpn}-licenses.inc")
+        with open(licenses_filename, "w") as f:
+            f.write(notice)
+            f.write(f'LICENSE += "& {" & ".join(tidy_licenses(licenses))}"\n\n')
+            f.write('LIC_FILES_CHKSUM += "\\\n')
+            for lic in sorted(lic_files_chksum, key=fold_uri):
+                f.write('    ' + lic + ' \\\n')
+            f.write('"\n')
+
+        #
+        # Sources
+        #
+
+        # Collect the module cache files downloaded by the go list command as
+        # the go list command knows best what the go list command needs and it
+        # needs more files in the module cache than the go install command as
+        # it doesn't do the dependency pruning mentioned in the Go module
+        # reference, https://go.dev/ref/mod, for go 1.17 or higher.
+        src_uris = []
+        downloaddir = os.path.join(mod_cache_dir, 'cache', 'download')
+        for dirpath, _, filenames in os.walk(downloaddir):
+            # We want to process files under @v directories
+            path, base = os.path.split(os.path.relpath(dirpath, downloaddir))
+            if base != '@v':
+                continue
+
+            path = unescape_path(path)
+            zipver = None
+            for name in filenames:
+                ver, ext = os.path.splitext(name)
+                if ext == '.zip':
+                    chksum = bb.utils.sha256_file(os.path.join(dirpath, name))
+                    src_uris.append(f'gomod://{path};version={ver};sha256sum={chksum}')
+                    zipver = ver
+                    break
+            for name in filenames:
+                ver, ext = os.path.splitext(name)
+                if ext == '.mod' and ver != zipver:
+                    chksum = bb.utils.sha256_file(os.path.join(dirpath, name))
+                    src_uris.append(f'gomod://{path};version={ver};mod=1;sha256sum={chksum}')
+
+
+        go_mods_filename = os.path.join(thisdir, f"{bpn}-go-mods.inc")
+        with open(go_mods_filename, "w") as f:
+            f.write(notice)
+            f.write('SRC_URI += "\\\n')
+            for uri in sorted(src_uris, key=fold_uri):
+                f.write('    ' + uri + ' \\\n')
+            f.write('"\n')
+
+        subprocess.check_output(("go", "clean", "-modcache"), cwd=source, env=env, text=True)
+}
+
+# This doesn't work as we need to wipe the inc files first so we don't try looking for LICENSE files that don't yet exist
+# RECIPE_UPGRADE_EXTRA_TASKS += "do_update_modules"
diff --git a/meta/classes-recipe/go-mod.bbclass b/meta/classes-recipe/go-mod.bbclass
index 93ae72235f..a15dda8f0e 100644
--- a/meta/classes-recipe/go-mod.bbclass
+++ b/meta/classes-recipe/go-mod.bbclass
@@ -23,7 +23,7 @@ GOBUILDFLAGS:append = " -modcacherw"
 inherit go
 
 export GOMODCACHE = "${S}/pkg/mod"
-GO_MOD_CACHE_DIR = "${@os.path.relpath(d.getVar('GOMODCACHE'), d.getVar('WORKDIR'))}"
+GO_MOD_CACHE_DIR = "${@os.path.relpath(d.getVar('GOMODCACHE'), d.getVar('UNPACKDIR'))}"
 do_unpack[cleandirs] += "${GOMODCACHE}"
 
 GO_WORKDIR ?= "${GO_IMPORT}"
diff --git a/meta/classes-recipe/image-live.bbclass b/meta/classes-recipe/image-live.bbclass
index d2e95ef51c..c3054be630 100644
--- a/meta/classes-recipe/image-live.bbclass
+++ b/meta/classes-recipe/image-live.bbclass
@@ -147,7 +147,10 @@ build_iso() {
                 isohybrid_args="-u"
         fi
 
-        isohybrid $isohybrid_args ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso
+        # EFI only does not need isohybrid
+        if [ "${PCBIOS}" = "1" ]  || [ "${EFI}" != "1" ]; then
+                isohybrid $isohybrid_args ${IMGDEPLOYDIR}/${IMAGE_NAME}.iso
+        fi
 }
 
 build_fat_img() {
diff --git a/meta/classes-recipe/image_types_wic.bbclass b/meta/classes-recipe/image_types_wic.bbclass
index 740ed946f8..6180874a4c 100644
--- a/meta/classes-recipe/image_types_wic.bbclass
+++ b/meta/classes-recipe/image_types_wic.bbclass
@@ -57,6 +57,16 @@ def wks_search(files, search_path):
             if searched:
                 return searched
 
+def wks_checksums(files, search_path):
+    ret = ""
+    for f in files:
+        found, hist = bb.utils.which(search_path, f, history=True)
+        ret = ret + " " + " ".join(h + ":False" for h in hist[:-1])
+        if found:
+            ret = ret + " " + found + ":True"
+    return ret
+
+
 WIC_CREATE_EXTRA_ARGS ?= ""
 
 IMAGE_CMD:wic () {
@@ -98,7 +108,7 @@ do_image_wic[cleandirs] = "${WORKDIR}/build-wic"
 
 # Rebuild when the wks file or vars in WICVARS change
 USING_WIC = "${@bb.utils.contains_any('IMAGE_FSTYPES', 'wic ' + ' '.join('wic.%s' % c for c in '${CONVERSIONTYPES}'.split()), '1', '', d)}"
-WKS_FILE_CHECKSUM = "${@'${WKS_FULL_PATH}:%s' % os.path.exists('${WKS_FULL_PATH}') if '${USING_WIC}' else ''}"
+WKS_FILE_CHECKSUM = "${@wks_checksums(d.getVar('WKS_FILES').split(), d.getVar('WKS_SEARCH_PATH')) if '${USING_WIC}' else ''}"
 do_image_wic[file-checksums] += "${WKS_FILE_CHECKSUM}"
 do_image_wic[depends] += "${@' '.join('%s-native:do_populate_sysroot' % r for r in ('parted', 'gptfdisk', 'dosfstools', 'mtools'))}"
 
diff --git a/meta/classes-recipe/rust-target-config.bbclass b/meta/classes-recipe/rust-target-config.bbclass
index 906a5083d7..cac6e90a9e 100644
--- a/meta/classes-recipe/rust-target-config.bbclass
+++ b/meta/classes-recipe/rust-target-config.bbclass
@@ -171,7 +171,7 @@ MAX_ATOMIC_WIDTH[armv7-eabi] = "64"
 FEATURES[armv7-eabi] = "+v7,+vfp2,+thumb2"
 
 ## aarch64-unknown-linux-{gnu, musl}
-DATA_LAYOUT[aarch64] = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
+DATA_LAYOUT[aarch64] = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128-Fn32"
 TARGET_ENDIAN[aarch64] = "little"
 TARGET_POINTER_WIDTH[aarch64] = "64"
 TARGET_C_INT_WIDTH[aarch64] = "32"
diff --git a/meta/classes-recipe/testsdk.bbclass b/meta/classes-recipe/testsdk.bbclass
index 59d2834c99..b1c4fa67e6 100644
--- a/meta/classes-recipe/testsdk.bbclass
+++ b/meta/classes-recipe/testsdk.bbclass
@@ -19,6 +19,7 @@ TESTSDK_SUITES ?= ""
 
 TESTSDK_CLASS_NAME ?= "oeqa.sdk.testsdk.TestSDK"
 TESTSDKEXT_CLASS_NAME ?= "oeqa.sdkext.testsdk.TestSDKExt"
+TESTSDK_CASE_DIRS ?= "sdk"
 
 def import_and_run(name, d):
     import importlib
diff --git a/meta/classes-recipe/uboot-config.bbclass b/meta/classes-recipe/uboot-config.bbclass
index f44605cb6a..bc20913f73 100644
--- a/meta/classes-recipe/uboot-config.bbclass
+++ b/meta/classes-recipe/uboot-config.bbclass
@@ -22,12 +22,17 @@ def removesuffix(s, suffix):
 UBOOT_ENTRYPOINT ?= "0x20008000"
 UBOOT_LOADADDRESS ?= "${UBOOT_ENTRYPOINT}"
 
+# When naming the files we install/deploy, the package version and revision
+# are part of the filename.  Create a single variable to represent this and
+# allow it to be customized if desired.
+UBOOT_VERSION ?= "${PV}-${PR}"
+
 # Some versions of u-boot use .bin and others use .img.  By default use .bin
 # but enable individual recipes to change this value.
 UBOOT_SUFFIX ??= "bin"
 UBOOT_BINARY ?= "u-boot.${UBOOT_SUFFIX}"
 UBOOT_BINARYNAME ?= "${@os.path.splitext(d.getVar("UBOOT_BINARY"))[0]}"
-UBOOT_IMAGE ?= "${UBOOT_BINARYNAME}-${MACHINE}-${PV}-${PR}.${UBOOT_SUFFIX}"
+UBOOT_IMAGE ?= "${UBOOT_BINARYNAME}-${MACHINE}-${UBOOT_VERSION}.${UBOOT_SUFFIX}"
 UBOOT_SYMLINK ?= "${UBOOT_BINARYNAME}-${MACHINE}.${UBOOT_SUFFIX}"
 UBOOT_MAKE_TARGET ?= "all"
 
@@ -36,7 +41,7 @@ UBOOT_MAKE_TARGET ?= "all"
 # purposes.
 UBOOT_ELF ?= ""
 UBOOT_ELF_SUFFIX ?= "elf"
-UBOOT_ELF_IMAGE ?= "u-boot-${MACHINE}-${PV}-${PR}.${UBOOT_ELF_SUFFIX}"
+UBOOT_ELF_IMAGE ?= "u-boot-${MACHINE}-${UBOOT_VERSION}.${UBOOT_ELF_SUFFIX}"
 UBOOT_ELF_BINARY ?= "u-boot.${UBOOT_ELF_SUFFIX}"
 UBOOT_ELF_SYMLINK ?= "u-boot-${MACHINE}.${UBOOT_ELF_SUFFIX}"
 
@@ -49,7 +54,7 @@ SPL_BINARY ?= ""
 SPL_DELIMITER  ?= "${@'.' if d.getVar("SPL_SUFFIX") else ''}"
 SPL_BINARYFILE ?= "${@os.path.basename(d.getVar("SPL_BINARY"))}"
 SPL_BINARYNAME ?= "${@removesuffix(d.getVar("SPL_BINARYFILE"), "." + d.getVar("SPL_SUFFIX"))}"
-SPL_IMAGE ?= "${SPL_BINARYNAME}-${MACHINE}-${PV}-${PR}${SPL_DELIMITER}${SPL_SUFFIX}"
+SPL_IMAGE ?= "${SPL_BINARYNAME}-${MACHINE}-${UBOOT_VERSION}${SPL_DELIMITER}${SPL_SUFFIX}"
 SPL_SYMLINK ?= "${SPL_BINARYNAME}-${MACHINE}${SPL_DELIMITER}${SPL_SUFFIX}"
 
 # Additional environment variables or a script can be installed alongside
@@ -62,14 +67,14 @@ UBOOT_ENV ?= ""
 UBOOT_ENV_SRC_SUFFIX ?= "cmd"
 UBOOT_ENV_SRC ?= "${UBOOT_ENV}.${UBOOT_ENV_SRC_SUFFIX}"
 UBOOT_ENV_BINARY ?= "${UBOOT_ENV}.${UBOOT_ENV_SUFFIX}"
-UBOOT_ENV_IMAGE ?= "${UBOOT_ENV}-${MACHINE}-${PV}-${PR}.${UBOOT_ENV_SUFFIX}"
+UBOOT_ENV_IMAGE ?= "${UBOOT_ENV}-${MACHINE}-${UBOOT_VERSION}.${UBOOT_ENV_SUFFIX}"
 UBOOT_ENV_SYMLINK ?= "${UBOOT_ENV}-${MACHINE}.${UBOOT_ENV_SUFFIX}"
 
 # U-Boot EXTLINUX variables. U-Boot searches for /boot/extlinux/extlinux.conf
 # to find EXTLINUX conf file.
 UBOOT_EXTLINUX_INSTALL_DIR ?= "/boot/extlinux"
 UBOOT_EXTLINUX_CONF_NAME ?= "extlinux.conf"
-UBOOT_EXTLINUX_SYMLINK ?= "${UBOOT_EXTLINUX_CONF_NAME}-${MACHINE}-${PR}"
+UBOOT_EXTLINUX_SYMLINK ?= "${UBOOT_EXTLINUX_CONF_NAME}-${MACHINE}-${UBOOT_VERSION}"
 
 # Options for the device tree compiler passed to mkimage '-D' feature:
 UBOOT_MKIMAGE_DTCOPTS ??= ""