9 files changed, 204 insertions, 11 deletions

diff --git a/meta/lib/oe/bootfiles.py b/meta/lib/oe/bootfiles.py
index 155fe742db..7ee148c4e2 100644
--- a/meta/lib/oe/bootfiles.py
+++ b/meta/lib/oe/bootfiles.py
@@ -10,7 +10,7 @@
 # Returns a list of tuples with (original filepath relative to
 # deploy_dir, desired filepath renaming)
 #
-# Heavily inspired of bootimg-partition.py
+# Heavily inspired of bootimg_partition.py
 #
 def get_boot_files(deploy_dir, boot_files):
     import re
diff --git a/meta/lib/oe/license_finder.py b/meta/lib/oe/license_finder.py
new file mode 100644
index 0000000000..16f5d7c94c
--- /dev/null
+++ b/meta/lib/oe/license_finder.py
@@ -0,0 +1,179 @@
+#
+# Copyright OpenEmbedded Contributors
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import fnmatch
+import hashlib
+import logging
+import os
+import re
+
+import bb
+import bb.utils
+
+logger = logging.getLogger("BitBake.OE.LicenseFinder")
+
+def _load_hash_csv(d):
+    """
+    Load a mapping of (checksum: license name) from all files/license-hashes.csv
+    files that can be found in the available layers.
+    """
+    import csv
+    md5sums = {}
+
+    # Read license md5sums from csv file
+    for path in d.getVar('BBPATH').split(':'):
+        csv_path = os.path.join(path, 'files', 'license-hashes.csv')
+        if os.path.isfile(csv_path):
+            with open(csv_path, newline='') as csv_file:
+                reader = csv.DictReader(csv_file, delimiter=',', fieldnames=['md5sum', 'license'])
+                for row in reader:
+                    md5sums[row['md5sum']] = row['license']
+
+    return md5sums
+
+
+def _crunch_known_licenses(d):
+    """
+    Calculate the MD5 checksums for the original and "crunched" versions of all
+    known licenses.
+    """
+    md5sums = {}
+
+    lic_dirs = [d.getVar('COMMON_LICENSE_DIR')] + (d.getVar('LICENSE_PATH') or "").split()
+    for lic_dir in lic_dirs:
+        for fn in os.listdir(lic_dir):
+            path = os.path.join(lic_dir, fn)
+            # Hash the exact contents
+            md5value = bb.utils.md5_file(path)
+            md5sums[md5value] = fn
+            # Also hash a "crunched" version
+            md5value = _crunch_license(path)
+            md5sums[md5value] = fn
+
+    return md5sums
+
+
+def _crunch_license(licfile):
+    '''
+    Remove non-material text from a license file and then calculate its
+    md5sum. This works well for licenses that contain a copyright statement,
+    but is also a useful way to handle people's insistence upon reformatting
+    the license text slightly (with no material difference to the text of the
+    license).
+    '''
+
+    import oe.utils
+
+    # Note: these are carefully constructed!
+    license_title_re = re.compile(r'^#*\(? *(This is )?([Tt]he )?.{0,15} ?[Ll]icen[sc]e( \(.{1,10}\))?\)?[:\.]? ?#*$')
+    license_statement_re = re.compile(r'^((This (project|software)|.{1,10}) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$')
+    copyright_re = re.compile(r'^ *[#\*]* *(Modified work |MIT LICENSED )?Copyright ?(\([cC]\))? .*$')
+    disclaimer_re = re.compile(r'^ *\*? ?All [Rr]ights [Rr]eserved\.$')
+    email_re = re.compile(r'^.*<[\w\.-]*@[\w\.\-]*>$')
+    header_re = re.compile(r'^(\/\**!?)? ?[\-=\*]* ?(\*\/)?$')
+    tag_re = re.compile(r'^ *@?\(?([Ll]icense|MIT)\)?$')
+    url_re = re.compile(r'^ *[#\*]* *https?:\/\/[\w\.\/\-]+$')
+
+    lictext = []
+    with open(licfile, 'r', errors='surrogateescape') as f:
+        for line in f:
+            # Drop opening statements
+            if copyright_re.match(line):
+                continue
+            elif disclaimer_re.match(line):
+                continue
+            elif email_re.match(line):
+                continue
+            elif header_re.match(line):
+                continue
+            elif tag_re.match(line):
+                continue
+            elif url_re.match(line):
+                continue
+            elif license_title_re.match(line):
+                continue
+            elif license_statement_re.match(line):
+                continue
+            # Strip comment symbols
+            line = line.replace('*', '') \
+                       .replace('#', '')
+            # Unify spelling
+            line = line.replace('sub-license', 'sublicense')
+            # Squash spaces
+            line = oe.utils.squashspaces(line.strip())
+            # Replace smart quotes, double quotes and backticks with single quotes
+            line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'')
+            # Unify brackets
+            line = line.replace("{", "[").replace("}", "]")
+            if line:
+                lictext.append(line)
+
+    m = hashlib.md5()
+    try:
+        m.update(' '.join(lictext).encode('utf-8'))
+        md5val = m.hexdigest()
+    except UnicodeEncodeError:
+        md5val = None
+    return md5val
+
+
+def find_license_files(srctree, first_only=False):
+    """
+    Search srctree for files that look like they could be licenses.
+    If first_only is True, only return the first file found.
+    """
+    licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*', 'e[dp]l-v10']
+    skip_extensions = (".html", ".js", ".json", ".svg", ".ts", ".go", ".sh")
+    licfiles = []
+    for root, dirs, files in os.walk(srctree):
+        # Sort files so that LICENSE is before LICENSE.subcomponent, which is
+        # meaningful if first_only is set.
+        for fn in sorted(files):
+            if fn.endswith(skip_extensions):
+                continue
+            for spec in licspecs:
+                if fnmatch.fnmatch(fn, spec):
+                    fullpath = os.path.join(root, fn)
+                    if not fullpath in licfiles:
+                        licfiles.append(fullpath)
+                        if first_only:
+                            return licfiles
+
+    return licfiles
+
+
+def match_licenses(licfiles, srctree, d, extra_hashes={}):
+    md5sums = {}
+    md5sums.update(_load_hash_csv(d))
+    md5sums.update(_crunch_known_licenses(d))
+    md5sums.update(extra_hashes)
+
+    licenses = []
+    for licfile in sorted(licfiles):
+        resolved_licfile = d.expand(licfile)
+        md5value = bb.utils.md5_file(resolved_licfile)
+        license = md5sums.get(md5value, None)
+        if not license:
+            crunched_md5 = _crunch_license(resolved_licfile)
+            license = md5sums.get(crunched_md5, None)
+            if not license:
+                license = 'Unknown'
+                logger.info("Please add the following line for '%s' to a 'license-hashes.csv' " \
+                    "and replace `Unknown` with the license:\n" \
+                    "%s,Unknown" % (os.path.relpath(licfile, srctree + "/.."), md5value))
+
+        licenses.append((license, os.path.relpath(licfile, srctree), md5value))
+
+    return licenses
+
+
+def find_licenses(srctree, d, first_only=False, extra_hashes={}):
+    licfiles = find_license_files(srctree, first_only)
+    licenses = match_licenses(licfiles, srctree, d, extra_hashes)
+
+    # FIXME should we grab at least one source file with a license header and add that too?
+
+    return licenses
diff --git a/meta/lib/oe/package.py b/meta/lib/oe/package.py
index 60392cbced..ce69151e5d 100644
--- a/meta/lib/oe/package.py
+++ b/meta/lib/oe/package.py
@@ -16,6 +16,7 @@ import mmap
 import subprocess
 import shutil
 
+import bb.parse
 import oe.cachedpath
 
 def runstrip(file, elftype, strip, extra_strip_sections=''):
@@ -1049,6 +1050,7 @@ def copydebugsources(debugsrcdir, sources, d):
             if os.path.exists(p) and not os.listdir(p):
                 os.rmdir(p)
 
+@bb.parse.vardepsexclude("BB_NUMBER_THREADS")
 def save_debugsources_info(debugsrcdir, sources_raw, d):
     import json
     import bb.compress.zstd
@@ -1081,6 +1083,7 @@ def save_debugsources_info(debugsrcdir, sources_raw, d):
         with bb.compress.zstd.open(debugsources_file, "wt", encoding="utf-8", num_threads=num_threads) as f:
             json.dump(sources_dict, f, sort_keys=True)
 
+@bb.parse.vardepsexclude("BB_NUMBER_THREADS")
 def read_debugsources_info(d):
     import json
     import bb.compress.zstd
diff --git a/meta/lib/oe/packagedata.py b/meta/lib/oe/packagedata.py
index 2d1d6ddeb7..b6a10a930a 100644
--- a/meta/lib/oe/packagedata.py
+++ b/meta/lib/oe/packagedata.py
@@ -7,6 +7,7 @@
 import codecs
 import os
 import json
+import bb.parse
 import bb.compress.zstd
 import oe.path
 
@@ -64,6 +65,7 @@ def read_subpkgdata_dict(pkg, d):
         ret[newvar] = subd[var]
     return ret
 
+@bb.parse.vardepsexclude("BB_NUMBER_THREADS")
 def read_subpkgdata_extended(pkg, d):
     import json
     import bb.compress.zstd
@@ -182,6 +184,7 @@ def runtime_mapping_rename(varname, pkg, d):
 
     #bb.note("%s after: %s" % (varname, d.getVar(varname)))
 
+@bb.parse.vardepsexclude("BB_NUMBER_THREADS")
 def emit_pkgdata(pkgfiles, d):
     def process_postinst_on_target(pkg, mlprefix):
         pkgval = d.getVar('PKG:%s' % pkg)
diff --git a/meta/lib/oe/path.py b/meta/lib/oe/path.py
index 5d21cdcbdf..a1efe97d88 100644
--- a/meta/lib/oe/path.py
+++ b/meta/lib/oe/path.py
@@ -10,6 +10,8 @@ import shutil
 import subprocess
 import os.path
 
+import bb.parse
+
 def join(*paths):
     """Like os.path.join but doesn't treat absolute RHS specially"""
     return os.path.normpath("/".join(paths))
@@ -77,6 +79,7 @@ def replace_absolute_symlinks(basedir, d):
             os.remove(path)
             os.symlink(base, path)
 
+@bb.parse.vardepsexclude("TOPDIR")
 def format_display(path, metadata):
     """ Prepare a path for display to the user. """
     rel = relative(metadata.getVar("TOPDIR"), path)
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index 826549948e..ef687f5d41 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-only
 #
+import bb.parse
 import bb.siggen
 import bb.runqueue
 import oe
@@ -493,6 +494,7 @@ def sstate_get_manifest_filename(task, d):
         d2.setVar("SSTATE_MANMACH", extrainf)
     return (d2.expand("${SSTATE_MANFILEPREFIX}.%s" % task), d2)
 
+@bb.parse.vardepsexclude("BBEXTENDCURR", "BBEXTENDVARIANT", "OVERRIDES", "PACKAGE_EXTRA_ARCHS")
 def find_sstate_manifest(taskdata, taskdata2, taskname, d, multilibcache):
     d2 = d
     variant = ''
diff --git a/meta/lib/oe/utils.py b/meta/lib/oe/utils.py
index d272dd2b8d..a11db5f3cd 100644
--- a/meta/lib/oe/utils.py
+++ b/meta/lib/oe/utils.py
@@ -9,6 +9,8 @@ import multiprocessing
 import traceback
 import errno
 
+import bb.parse
+
 def read_file(filename):
     try:
         f = open( filename, "r" )
@@ -265,6 +267,7 @@ def execute_pre_post_process(d, cmds):
         bb.note("Executing %s ..." % cmd)
         bb.build.exec_func(cmd, d)
 
+@bb.parse.vardepsexclude("BB_NUMBER_THREADS")
 def get_bb_number_threads(d):
     return int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
 
@@ -467,7 +470,7 @@ def host_gcc_version(d, taskcontextonly=False):
     version = match.group(1)
     return "-%s" % version if version in ("4.8", "4.9") else ""
 
-
+@bb.parse.vardepsexclude("DEFAULTTUNE_MULTILIB_ORIGINAL", "OVERRIDES")
 def get_multilib_datastore(variant, d):
     localdata = bb.data.createCopy(d)
     if variant:
diff --git a/meta/lib/oeqa/selftest/cases/fitimage.py b/meta/lib/oeqa/selftest/cases/fitimage.py
index be291e4b0f..ca4724d1ae 100644
--- a/meta/lib/oeqa/selftest/cases/fitimage.py
+++ b/meta/lib/oeqa/selftest/cases/fitimage.py
@@ -786,7 +786,7 @@ FIT_CONF_PREFIX = "foo-"
         EXPECTED_COMP = ["ti,am335x-bone-green", "ti,am335x-bone-black", "ti,am335x-bone", "ti,am33xx"]
 
         config = """
-DISTRO="poky"
+DISTRO = "poky"
 MACHINE = "beaglebone-yocto"
 """
         self.write_config(config)
@@ -918,7 +918,7 @@ UBOOT_MKIMAGE_SIGN_ARGS = "-c 'a smart comment'"
         bb_vars = self._fit_get_bb_vars()
 
         # Ensure new keys are generated and FIT_GENERATE_KEYS = "1" is tested
-        bitbake("kernel-signing-keys-native -c cleansstate")
+        bitbake("kernel-signing-keys-native -c compile -f")
 
         self._test_fitimage(bb_vars)
 
@@ -938,7 +938,7 @@ UBOOT_MKIMAGE_SIGN_ARGS = "-c 'a smart comment'"
         """
 
         config = """
-DISTRO="poky"
+DISTRO = "poky"
 MACHINE = "beaglebone-yocto"
 INITRAMFS_IMAGE = "core-image-minimal-initramfs"
 INITRAMFS_SCRIPTS = ""
@@ -992,7 +992,7 @@ FIT_HASH_ALG = "sha256"
         """
 
         config = """
-DISTRO="poky"
+DISTRO = "poky"
 MACHINE = "beaglebone-yocto"
 INITRAMFS_IMAGE_BUNDLE = "1"
 INITRAMFS_IMAGE = "core-image-minimal-initramfs"
diff --git a/meta/lib/oeqa/selftest/cases/wic.py b/meta/lib/oeqa/selftest/cases/wic.py
index 1a67b6df51..f45608172f 100644
--- a/meta/lib/oeqa/selftest/cases/wic.py
+++ b/meta/lib/oeqa/selftest/cases/wic.py
@@ -153,7 +153,7 @@ class Wic(WicTestCase):
         # create a temporary file for the WKS content
         with NamedTemporaryFile("w", suffix=".wks") as wks:
             wks.write(
-                'part --source bootimg-efi '
+                'part --source bootimg_efi '
                 '--sourceparams="loader=grub-efi,install-kernel-into-boot-dir=false" '
                 '--label boot --active\n'
             )
@@ -186,7 +186,7 @@ class Wic(WicTestCase):
         # create a temporary file for the WKS content
         with NamedTemporaryFile("w", suffix=".wks") as wks:
             wks.write(
-                'part --source bootimg-efi '
+                'part --source bootimg_efi '
                 '--sourceparams="loader=grub-efi,install-kernel-into-boot-dir=true" '
                 '--label boot --active\n'
             )
@@ -1358,7 +1358,7 @@ class Wic2(WicTestCase):
     def test_biosplusefi_plugin(self):
         """Test biosplusefi plugin"""
         # Wic generation below may fail depending on the order of the unittests
-        # This is because bootimg-pcbios (that bootimg-biosplusefi uses) generate its MBR inside STAGING_DATADIR directory
+        # This is because bootimg_pcbios (that bootimg_biosplusefi uses) generate its MBR inside STAGING_DATADIR directory
         #    which may or may not exists depending on what was built already
         # If an image hasn't been built yet, directory ${STAGING_DATADIR}/syslinux won't exists and _get_bootimg_dir()
         #   will raise with "Couldn't find correct bootimg_dir"
@@ -1370,7 +1370,7 @@ class Wic2(WicTestCase):
 
         img = 'core-image-minimal'
         with NamedTemporaryFile("w", suffix=".wks") as wks:
-            wks.writelines(['part /boot --active --source bootimg-biosplusefi --sourceparams="loader=grub-efi"\n',
+            wks.writelines(['part /boot --active --source bootimg_biosplusefi --sourceparams="loader=grub-efi"\n',
                             'part / --source rootfs --fstype=ext4 --align 1024 --use-uuid\n'\
                             'bootloader --timeout=0 --append="console=ttyS0,115200n8"\n'])
             wks.flush()
@@ -1390,7 +1390,7 @@ class Wic2(WicTestCase):
 
         img = 'core-image-minimal'
         with NamedTemporaryFile("w", suffix=".wks") as wks:
-            wks.writelines(['part /boot --source bootimg-efi --sourceparams="loader=uefi-kernel"\n'
+            wks.writelines(['part /boot --source bootimg_efi --sourceparams="loader=uefi-kernel"\n'
                             'part / --source rootfs --fstype=ext4 --align 1024 --use-uuid\n'\
                             'bootloader --timeout=0 --append="console=ttyS0,115200n8"\n'])
             wks.flush()