9 files changed, 179 insertions, 61 deletions

diff --git a/meta/lib/oe/__init__.py b/meta/lib/oe/__init__.py
index dd094a874a..73de774266 100644
--- a/meta/lib/oe/__init__.py
+++ b/meta/lib/oe/__init__.py
@@ -12,4 +12,4 @@ __path__ = extend_path(__path__, __name__)
 BBIMPORTS = ["qa", "data", "path", "utils", "types", "package", "packagedata", \
              "packagegroup", "sstatesig", "lsb", "cachedpath", "license", "qemu", \
              "reproducible", "rust", "buildcfg", "go", "spdx30_tasks", "spdx_common", \
-             "cve_check"]
+             "cve_check", "tune"]
diff --git a/meta/lib/oe/go.py b/meta/lib/oe/go.py
index dfd957d157..4559dc63b2 100644
--- a/meta/lib/oe/go.py
+++ b/meta/lib/oe/go.py
@@ -7,6 +7,10 @@
 import re
 
 def map_arch(a):
+    """
+    Map our architecture names to Go's GOARCH names.
+    See https://github.com/golang/go/blob/master/src/internal/syslist/syslist.go for the complete list.
+    """
     if re.match('i.86', a):
         return '386'
     elif a == 'x86_64':
@@ -31,4 +35,4 @@ def map_arch(a):
         return 'riscv64'
     elif a == 'loongarch64':
         return 'loong64'
-    return ''
+    raise KeyError(f"Cannot map architecture {a}")
diff --git a/meta/lib/oe/license.py b/meta/lib/oe/license.py
index 6f882c3812..6e55fa1e7f 100644
--- a/meta/lib/oe/license.py
+++ b/meta/lib/oe/license.py
@@ -462,3 +462,18 @@ def skip_incompatible_package_licenses(d, pkgs):
             skipped_pkgs[pkg] = incompatible_lic
 
     return skipped_pkgs
+
+def tidy_licenses(value):
+    """
+    Flat, split and sort licenses.
+    """
+    from oe.license import flattened_licenses
+
+    def _choose(a, b):
+        str_a, str_b  = sorted((" & ".join(a), " & ".join(b)), key=str.casefold)
+        return ["(%s | %s)" % (str_a, str_b)]
+
+    if not isinstance(value, str):
+        value = " & ".join(value)
+
+    return sorted(list(set(flattened_licenses(value, _choose))), key=str.casefold)
diff --git a/meta/lib/oe/reproducible.py b/meta/lib/oe/reproducible.py
index cdb38d5aa4..0270024a83 100644
--- a/meta/lib/oe/reproducible.py
+++ b/meta/lib/oe/reproducible.py
@@ -75,10 +75,11 @@ def get_source_date_epoch_from_known_files(d, sourcedir):
     return source_date_epoch
 
 def find_git_folder(d, sourcedir):
-    # First guess: UNPACKDIR/git
+    # First guess: UNPACKDIR/BB_GIT_DEFAULT_DESTSUFFIX
     # This is the default git fetcher unpack path
     unpackdir = d.getVar('UNPACKDIR')
-    gitpath = os.path.join(unpackdir, "git/.git")
+    default_destsuffix = d.getVar('BB_GIT_DEFAULT_DESTSUFFIX')
+    gitpath = os.path.join(unpackdir, default_destsuffix, ".git")
     if os.path.isdir(gitpath):
         return gitpath
 
diff --git a/meta/lib/oe/rust.py b/meta/lib/oe/rust.py
index 185553eeeb..1dc9cf150d 100644
--- a/meta/lib/oe/rust.py
+++ b/meta/lib/oe/rust.py
@@ -8,6 +8,4 @@
 def arch_to_rust_arch(arch):
     if arch == "ppc64le":
         return "powerpc64le"
-    if arch in ('riscv32', 'riscv64'):
-        return arch + 'gc'
     return arch
diff --git a/meta/lib/oe/spdx30_tasks.py b/meta/lib/oe/spdx30_tasks.py
index 61d7ba45e3..c352dab152 100644
--- a/meta/lib/oe/spdx30_tasks.py
+++ b/meta/lib/oe/spdx30_tasks.py
@@ -156,6 +156,11 @@ def add_package_files(
         bb.note(f"Skip {topdir}")
         return spdx_files
 
+    check_compiled_sources = d.getVar("SPDX_INCLUDE_COMPILED_SOURCES") == "1"
+    if check_compiled_sources:
+        compiled_sources, types = oe.spdx_common.get_compiled_sources(d)
+        bb.debug(1, f"Total compiled files: {len(compiled_sources)}")
+
     for subdir, dirs, files in os.walk(topdir, onerror=walk_error):
         dirs[:] = [d for d in dirs if d not in ignore_dirs]
         if subdir == str(topdir):
@@ -171,6 +176,11 @@ def add_package_files(
             filename = str(filepath.relative_to(topdir))
             file_purposes = get_purposes(filepath)
 
+            # Check if file is compiled
+            if check_compiled_sources:
+                if not oe.spdx_common.is_compiled_source(filename, compiled_sources, types):
+                    continue
+
             spdx_file = objset.new_file(
                 get_spdxid(file_counter),
                 filename,
@@ -542,7 +552,7 @@ def create_spdx(d):
     )
     build_objset.new_relationship(
         source_files,
-        oe.spdx30.RelationshipType.hasConcludedLicense,
+        oe.spdx30.RelationshipType.hasDeclaredLicense,
         [oe.sbom30.get_element_link_id(recipe_spdx_license)],
     )
 
@@ -714,24 +724,23 @@ def create_spdx(d):
                             impact_statement=description,
                         )
 
-                        if detail in (
-                            "ignored",
-                            "cpe-incorrect",
-                            "disputed",
-                            "upstream-wontfix",
-                        ):
-                            # VEX doesn't have justifications for this
-                            pass
-                        elif detail in (
-                            "not-applicable-config",
-                            "not-applicable-platform",
-                        ):
-                            for v in spdx_vex:
-                                v.security_justificationType = (
-                                    oe.spdx30.security_VexJustificationType.vulnerableCodeNotPresent
+                        vex_just_type = d.getVarFlag(
+                            "CVE_CHECK_VEX_JUSTIFICATION", detail
+                        )
+                        if vex_just_type:
+                            if (
+                                vex_just_type
+                                not in oe.spdx30.security_VexJustificationType.NAMED_INDIVIDUALS
+                            ):
+                                bb.fatal(
+                                    f"Unknown vex justification '{vex_just_type}', detail '{detail}', for ignored {cve}"
                                 )
-                        else:
-                            bb.fatal(f"Unknown detail '{detail}' for ignored {cve}")
+
+                            for v in spdx_vex:
+                                v.security_justificationType = oe.spdx30.security_VexJustificationType.NAMED_INDIVIDUALS[
+                                    vex_just_type
+                                ]
+
                     elif status == "Unknown":
                         bb.note(f"Skipping {cve} with status 'Unknown'")
                     else:
diff --git a/meta/lib/oe/spdx_common.py b/meta/lib/oe/spdx_common.py
index 4caefc7673..c2dec65563 100644
--- a/meta/lib/oe/spdx_common.py
+++ b/meta/lib/oe/spdx_common.py
@@ -242,3 +242,44 @@ def fetch_data_to_uri(fd, name):
         uri = uri + "@" + fd.revision
 
     return uri
+
+def is_compiled_source (filename, compiled_sources, types):
+    """
+    Check if the file is a compiled file
+    """
+    import os
+    # If we don't have compiled source, we assume all are compiled.
+    if not compiled_sources:
+        return True
+
+    # We return always true if the file type is not in the list of compiled files.
+    # Some files in the source directory are not compiled, for example, Makefiles,
+    # but also python .py file. We need to include them in the SPDX.
+    basename = os.path.basename(filename)
+    ext = basename.partition(".")[2]
+    if ext not in types:
+        return True
+    # Check that the file is in the list
+    return filename in compiled_sources
+
+def get_compiled_sources(d):
+    """
+    Get list of compiled sources from debug information and normalize the paths
+    """
+    import itertools
+    source_info = oe.package.read_debugsources_info(d)
+    if not source_info:
+        bb.debug(1, "Do not have debugsources.list. Skipping")
+        return [], []
+
+    # Sources are not split now in SPDX, so we aggregate them
+    sources = set(itertools.chain.from_iterable(source_info.values()))
+    # Check extensions of files
+    types = set()
+    for src in sources:
+        basename = os.path.basename(src)
+        ext = basename.partition(".")[2]
+        if ext not in types and ext:
+            types.add(ext)
+    bb.debug(1, f"Num of sources: {len(sources)} and types: {len(types)} {str(types)}")
+    return sources, types
diff --git a/meta/lib/oe/tune.py b/meta/lib/oe/tune.py
new file mode 100644
index 0000000000..7fda19430d
--- /dev/null
+++ b/meta/lib/oe/tune.py
@@ -0,0 +1,81 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+# riscv_isa_to_tune(isa)
+#
+# Automatically translate a RISC-V ISA string to TUNE_FEATURES
+#
+# Abbreviations, such as rv32g -> rv32imaffd_zicsr_zifencei are supported.
+#
+# Profiles, such as rva22u64, are NOT supported, you must use ISA strings.
+#
+def riscv_isa_to_tune(isa):
+    _isa = isa.lower()
+
+    feature = []
+    iter = 0
+
+    # rv or riscv
+    if _isa[iter:].startswith('rv'):
+        feature.append('rv')
+        iter = iter + 2
+    elif _isa[iter:].startswith('riscv'):
+        feature.append('rv')
+        iter = iter + 5
+    else:
+        # Not a risc-v ISA!
+        return _isa
+
+    while (_isa[iter:]):
+        # Skip _ and whitespace
+        if _isa[iter] == '_' or _isa[iter].isspace():
+            iter = iter + 1
+            continue
+
+        # Length, just capture numbers here
+        if _isa[iter].isdigit():
+            iter_end = iter
+            while iter_end < len(_isa) and _isa[iter_end].isdigit():
+                iter_end = iter_end + 1
+
+            feature.append(_isa[iter:iter_end])
+            iter = iter_end
+            continue
+
+        # Typically i, e or g is next, followed by extensions.
+        # Extensions are single character, except for Z, Ss, Sh, Sm, Sv, and X
+
+        # If the extension starts with 'Z', 'S' or 'X' use the name until the next _, whitespace or end
+        if _isa[iter] in ['z', 's', 'x']:
+            ext_type = _isa[iter]
+            iter_end = iter + 1
+
+            # Multicharacter extension, these are supposed to have a _ before the next multicharacter extension
+            # See 37.4 and 37.5:
+            # 37.4: Underscores "_" may be used to separate ISA extensions...
+            # 37.5: All multi-letter extensions ... must be separated from other multi-letter extensions by an underscore...
+            # Some extensions permit only alphabetic characters, while others allow alphanumeric chartacters
+            while iter_end < len(_isa) and _isa[iter_end] != "_" and not _isa[iter_end].isspace():
+                iter_end = iter_end + 1
+
+            feature.append(_isa[iter:iter_end])
+            iter = iter_end
+            continue
+
+        # 'g' is special, it's an abbreviation for imafd_zicsr_zifencei
+        # When expanding the abbreviation, any additional letters must appear before the _z* extensions
+        if _isa[iter] == 'g':
+            _isa = 'imafd' + _isa[iter+1:] + '_zicsr_zifencei'
+            iter = 0
+            continue
+
+        feature.append(_isa[iter])
+        iter = iter + 1
+        continue
+
+    # Eliminate duplicates, but preserve the order
+    feature = list(dict.fromkeys(feature))
+    return ' '.join(feature)
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index a11db5f3cd..779c5e593f 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -415,60 +415,29 @@ def format_pkg_list(pkg_dict, ret_format=None, pkgdata_dir=None):
     return output_str
 
 
-# Helper function to get the host compiler version
-# Do not assume the compiler is gcc
-def get_host_compiler_version(d, taskcontextonly=False):
+# Helper function to get the host gcc version
+def get_host_gcc_version(d, taskcontextonly=False):
     import re, subprocess
 
     if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1':
         return
 
-    compiler = d.getVar("BUILD_CC")
-    # Get rid of ccache since it is not present when parsing.
-    if compiler.startswith('ccache '):
-        compiler = compiler[7:]
     try:
         env = os.environ.copy()
         # datastore PATH does not contain session PATH as set by environment-setup-...
         # this breaks the install-buildtools use-case
         # env["PATH"] = d.getVar("PATH")
-        output = subprocess.check_output("%s --version" % compiler, \
+        output = subprocess.check_output("gcc --version", \
                     shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8")
     except subprocess.CalledProcessError as e:
-        bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8")))
+        bb.fatal("Error running gcc --version: %s" % (e.output.decode("utf-8")))
 
     match = re.match(r".* (\d+\.\d+)\.\d+.*", output.split('\n')[0])
     if not match:
-        bb.fatal("Can't get compiler version from %s --version output" % compiler)
+        bb.fatal("Can't get compiler version from gcc --version output")
 
     version = match.group(1)
-    return compiler, version
-
-
-def host_gcc_version(d, taskcontextonly=False):
-    import re, subprocess
-
-    if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1':
-        return
-
-    compiler = d.getVar("BUILD_CC")
-    # Get rid of ccache since it is not present when parsing.
-    if compiler.startswith('ccache '):
-        compiler = compiler[7:]
-    try:
-        env = os.environ.copy()
-        env["PATH"] = d.getVar("PATH")
-        output = subprocess.check_output("%s --version" % compiler, \
-                    shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8")
-    except subprocess.CalledProcessError as e:
-        bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8")))
-
-    match = re.match(r".* (\d+\.\d+)\.\d+.*", output.split('\n')[0])
-    if not match:
-        bb.fatal("Can't get compiler version from %s --version output" % compiler)
-
-    version = match.group(1)
-    return "-%s" % version if version in ("4.8", "4.9") else ""
+    return version
 
 @bb.parse.vardepsexclude("DEFAULTTUNE_MULTILIB_ORIGINAL", "OVERRIDES")
 def get_multilib_datastore(variant, d):