1 files changed, 22 insertions, 231 deletions

diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 8e9ff38db6..ef0ba974a9 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -18,6 +18,8 @@ from urllib.parse import urlparse, urldefrag, urlsplit
 import hashlib
 import bb.fetch2
 logger = logging.getLogger('recipetool')
+from oe.license import tidy_licenses
+from oe.license_finder import find_licenses
 
 tinfoil = None
 plugins = None
@@ -528,7 +530,7 @@ def create_recipe(args):
         if ftmpdir and args.keep_temp:
             logger.info('Fetch temp directory is %s' % ftmpdir)
 
-        dirlist = scriptutils.filter_src_subdirs(srctree)
+        dirlist = os.listdir(srctree)
         logger.debug('Directory listing (excluding filtered out):\n  %s' % '\n  '.join(dirlist))
         if len(dirlist) == 1:
             singleitem = os.path.join(srctree, dirlist[0])
@@ -637,7 +639,6 @@ def create_recipe(args):
                     if len(splitline) > 1:
                         if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]):
                             srcuri = reformat_git_uri(splitline[1]) + ';branch=master'
-                            srcsubdir = 'git'
                             break
 
     if args.src_subdir:
@@ -735,7 +736,7 @@ def create_recipe(args):
     if srcsubdir and not args.binary:
         # (for binary packages we explicitly specify subdir= when fetching to
         # match the default value of S, so we don't need to set it in that case)
-        lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
+        lines_before.append('S = "${UNPACKDIR}/%s"' % srcsubdir)
         lines_before.append('')
 
     if pkgarch:
@@ -764,6 +765,7 @@ def create_recipe(args):
     extrafiles = extravalues.pop('extrafiles', {})
     extra_pn = extravalues.pop('PN', None)
     extra_pv = extravalues.pop('PV', None)
+    run_tasks = extravalues.pop('run_tasks', "").split()
 
     if extra_pv and not realpv:
         realpv = extra_pv
@@ -824,7 +826,8 @@ def create_recipe(args):
             extraoutdir = os.path.join(os.path.dirname(outfile), pn)
         bb.utils.mkdirhier(extraoutdir)
         for destfn, extrafile in extrafiles.items():
-            shutil.move(extrafile, os.path.join(extraoutdir, destfn))
+            fn = destfn.format(pn=pn, pv=realpv)
+            shutil.move(extrafile, os.path.join(extraoutdir, fn))
 
     lines = lines_before
     lines_before = []
@@ -839,7 +842,7 @@ def create_recipe(args):
                 line = line.replace(realpv, '${PV}')
             if pn:
                 line = line.replace(pn, '${BPN}')
-            if line == 'S = "${WORKDIR}/${BPN}-${PV}"':
+            if line == 'S = "${UNPACKDIR}/${BPN}-${PV}"' or 'tmp-recipetool-' in line:
                 skipblank = True
                 continue
         elif line.startswith('SRC_URI = '):
@@ -917,6 +920,10 @@ def create_recipe(args):
         log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool)
         tinfoil.modified_files()
 
+    for task in run_tasks:
+        logger.info("Running task %s" % task)
+        tinfoil.build_file_sync(outfile, task)
+
     if tempsrc:
         if args.keep_temp:
             logger.info('Preserving temporary directory %s' % tempsrc)
@@ -944,23 +951,13 @@ def fixup_license(value):
         return '(' + value + ')'
     return value
 
-def tidy_licenses(value):
-    """Flat, split and sort licenses"""
-    from oe.license import flattened_licenses
-    def _choose(a, b):
-        str_a, str_b  = sorted((" & ".join(a), " & ".join(b)), key=str.casefold)
-        return ["(%s | %s)" % (str_a, str_b)]
-    if not isinstance(value, str):
-        value = " & ".join(value)
-    return sorted(list(set(flattened_licenses(value, _choose))), key=str.casefold)
-
 def handle_license_vars(srctree, lines_before, handled, extravalues, d):
     lichandled = [x for x in handled if x[0] == 'license']
     if lichandled:
         # Someone else has already handled the license vars, just return their value
         return lichandled[0][1]
 
-    licvalues = guess_license(srctree, d)
+    licvalues = find_licenses(srctree, d)
     licenses = []
     lic_files_chksum = []
     lic_unknown = []
@@ -1040,222 +1037,9 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
     handled.append(('license', licvalues))
     return licvalues
 
-def get_license_md5sums(d, static_only=False, linenumbers=False):
-    import bb.utils
-    import csv
-    md5sums = {}
-    if not static_only and not linenumbers:
-        # Gather md5sums of license files in common license dir
-        commonlicdir = d.getVar('COMMON_LICENSE_DIR')
-        for fn in os.listdir(commonlicdir):
-            md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn))
-            md5sums[md5value] = fn
-
-    # The following were extracted from common values in various recipes
-    # (double checking the license against the license file itself, not just
-    # the LICENSE value in the recipe)
-
-    # Read license md5sums from csv file
-    scripts_path = os.path.dirname(os.path.realpath(__file__))
-    for path in (d.getVar('BBPATH').split(':')
-                + [os.path.join(scripts_path, '..', '..')]):
-        csv_path = os.path.join(path, 'lib', 'recipetool', 'licenses.csv')
-        if os.path.isfile(csv_path):
-            with open(csv_path, newline='') as csv_file:
-                fieldnames = ['md5sum', 'license', 'beginline', 'endline', 'md5']
-                reader = csv.DictReader(csv_file, delimiter=',', fieldnames=fieldnames)
-                for row in reader:
-                    if linenumbers:
-                        md5sums[row['md5sum']] = (
-                            row['license'], row['beginline'], row['endline'], row['md5'])
-                    else:
-                        md5sums[row['md5sum']] = row['license']
-
-    return md5sums
-
-def crunch_known_licenses(d):
-    '''
-    Calculate the MD5 checksums for the crunched versions of all common
-    licenses. Also add additional known checksums.
-    '''
-    
-    crunched_md5sums = {}
-
-    # common licenses
-    crunched_md5sums['ad4e9d34a2e966dfe9837f18de03266d'] = 'GFDL-1.1-only'
-    crunched_md5sums['d014fb11a34eb67dc717fdcfc97e60ed'] = 'GFDL-1.2-only'
-    crunched_md5sums['e020ca655b06c112def28e597ab844f1'] = 'GFDL-1.3-only'
-
-    # The following two were gleaned from the "forever" npm package
-    crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC'
-    # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt
-    crunched_md5sums['50fab24ce589d69af8964fdbfe414c60'] = 'BSD-2-Clause'
-    # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE
-    crunched_md5sums['88a4355858a1433fea99fae34a44da88'] = 'GPL-2.0-only'
-    # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt
-    crunched_md5sums['063b5c3ebb5f3aa4c85a2ed18a31fbe7'] = 'GPL-2.0-only'
-    # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1
-    crunched_md5sums['7f5202f4d44ed15dcd4915f5210417d8'] = 'LGPL-2.1-only'
-    # unixODBC-2.3.4 COPYING
-    crunched_md5sums['3debde09238a8c8e1f6a847e1ec9055b'] = 'LGPL-2.1-only'
-    # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3
-    crunched_md5sums['f90c613c51aa35da4d79dd55fc724ceb'] = 'LGPL-3.0-only'
-    # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/epl-v10
-    crunched_md5sums['efe2cb9a35826992b9df68224e3c2628'] = 'EPL-1.0'
-
-    # https://raw.githubusercontent.com/jquery/esprima/3.1.3/LICENSE.BSD
-    crunched_md5sums['80fa7b56a28e8c902e6af194003220a5'] = 'BSD-2-Clause'
-    # https://raw.githubusercontent.com/npm/npm-install-checks/master/LICENSE
-    crunched_md5sums['e659f77bfd9002659e112d0d3d59b2c1'] = 'BSD-2-Clause'
-    # https://raw.githubusercontent.com/silverwind/default-gateway/4.2.0/LICENSE
-    crunched_md5sums['4c641f2d995c47f5cb08bdb4b5b6ea05'] = 'BSD-2-Clause'
-    # https://raw.githubusercontent.com/tad-lispy/node-damerau-levenshtein/v1.0.5/LICENSE
-    crunched_md5sums['2b8c039b2b9a25f0feb4410c4542d346'] = 'BSD-2-Clause'
-    # https://raw.githubusercontent.com/terser/terser/v3.17.0/LICENSE
-    crunched_md5sums['8bd23871802951c9ad63855151204c2c'] = 'BSD-2-Clause'
-    # https://raw.githubusercontent.com/alexei/sprintf.js/1.0.3/LICENSE
-    crunched_md5sums['008c22318c8ea65928bf730ddd0273e3'] = 'BSD-3-Clause'
-    # https://raw.githubusercontent.com/Caligatio/jsSHA/v3.2.0/LICENSE
-    crunched_md5sums['0e46634a01bfef056892949acaea85b1'] = 'BSD-3-Clause'
-    # https://raw.githubusercontent.com/d3/d3-path/v1.0.9/LICENSE
-    crunched_md5sums['b5f72aef53d3b2b432702c30b0215666'] = 'BSD-3-Clause'
-    # https://raw.githubusercontent.com/feross/ieee754/v1.1.13/LICENSE
-    crunched_md5sums['a39327c997c20da0937955192d86232d'] = 'BSD-3-Clause'
-    # https://raw.githubusercontent.com/joyent/node-extsprintf/v1.3.0/LICENSE
-    crunched_md5sums['721f23a96ff4161ca3a5f071bbe18108'] = 'MIT'
-    # https://raw.githubusercontent.com/pvorb/clone/v0.2.0/LICENSE
-    crunched_md5sums['b376d29a53c9573006b9970709231431'] = 'MIT'
-    # https://raw.githubusercontent.com/andris9/encoding/v0.1.12/LICENSE
-    crunched_md5sums['85d8a977ee9d7c5ab4ac03c9b95431c4'] = 'MIT-0'
-    # https://raw.githubusercontent.com/faye/websocket-driver-node/0.7.3/LICENSE.md
-    crunched_md5sums['b66384e7137e41a9b1904ef4d39703b6'] = 'Apache-2.0'
-    # https://raw.githubusercontent.com/less/less.js/v4.1.1/LICENSE
-    crunched_md5sums['b27575459e02221ccef97ec0bfd457ae'] = 'Apache-2.0'
-    # https://raw.githubusercontent.com/microsoft/TypeScript/v3.5.3/LICENSE.txt
-    crunched_md5sums['a54a1a6a39e7f9dbb4a23a42f5c7fd1c'] = 'Apache-2.0'
-    # https://raw.githubusercontent.com/request/request/v2.87.0/LICENSE
-    crunched_md5sums['1034431802e57486b393d00c5d262b8a'] = 'Apache-2.0'
-    # https://raw.githubusercontent.com/dchest/tweetnacl-js/v0.14.5/LICENSE
-    crunched_md5sums['75605e6bdd564791ab698fca65c94a4f'] = 'Unlicense'
-    # https://raw.githubusercontent.com/stackgl/gl-mat3/v2.0.0/LICENSE.md
-    crunched_md5sums['75512892d6f59dddb6d1c7e191957e9c'] = 'Zlib'
-
-    commonlicdir = d.getVar('COMMON_LICENSE_DIR')
-    for fn in sorted(os.listdir(commonlicdir)):
-        md5value, lictext = crunch_license(os.path.join(commonlicdir, fn))
-        if md5value not in crunched_md5sums:
-            crunched_md5sums[md5value] = fn
-        elif fn != crunched_md5sums[md5value]:
-            bb.debug(2, "crunched_md5sums['%s'] is already set to '%s' rather than '%s'" % (md5value, crunched_md5sums[md5value], fn))
-        else:
-            bb.debug(2, "crunched_md5sums['%s'] is already set to '%s'" % (md5value, crunched_md5sums[md5value]))
-
-    return crunched_md5sums
-
-def crunch_license(licfile):
-    '''
-    Remove non-material text from a license file and then calculate its
-    md5sum. This works well for licenses that contain a copyright statement,
-    but is also a useful way to handle people's insistence upon reformatting
-    the license text slightly (with no material difference to the text of the
-    license).
-    '''
-
-    import oe.utils
-
-    # Note: these are carefully constructed!
-    license_title_re = re.compile(r'^#*\(? *(This is )?([Tt]he )?.{0,15} ?[Ll]icen[sc]e( \(.{1,10}\))?\)?[:\.]? ?#*$')
-    license_statement_re = re.compile(r'^((This (project|software)|.{1,10}) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$')
-    copyright_re = re.compile(r'^ *[#\*]* *(Modified work |MIT LICENSED )?Copyright ?(\([cC]\))? .*$')
-    disclaimer_re = re.compile(r'^ *\*? ?All [Rr]ights [Rr]eserved\.$')
-    email_re = re.compile(r'^.*<[\w\.-]*@[\w\.\-]*>$')
-    header_re = re.compile(r'^(\/\**!?)? ?[\-=\*]* ?(\*\/)?$')
-    tag_re = re.compile(r'^ *@?\(?([Ll]icense|MIT)\)?$')
-    url_re = re.compile(r'^ *[#\*]* *https?:\/\/[\w\.\/\-]+$')
-
-    lictext = []
-    with open(licfile, 'r', errors='surrogateescape') as f:
-        for line in f:
-            # Drop opening statements
-            if copyright_re.match(line):
-                continue
-            elif disclaimer_re.match(line):
-                continue
-            elif email_re.match(line):
-                continue
-            elif header_re.match(line):
-                continue
-            elif tag_re.match(line):
-                continue
-            elif url_re.match(line):
-                continue
-            elif license_title_re.match(line):
-                continue
-            elif license_statement_re.match(line):
-                continue
-            # Strip comment symbols
-            line = line.replace('*', '') \
-                       .replace('#', '')
-            # Unify spelling
-            line = line.replace('sub-license', 'sublicense')
-            # Squash spaces
-            line = oe.utils.squashspaces(line.strip())
-            # Replace smart quotes, double quotes and backticks with single quotes
-            line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'')
-            # Unify brackets
-            line = line.replace("{", "[").replace("}", "]")
-            if line:
-                lictext.append(line)
-
-    m = hashlib.md5()
-    try:
-        m.update(' '.join(lictext).encode('utf-8'))
-        md5val = m.hexdigest()
-    except UnicodeEncodeError:
-        md5val = None
-        lictext = ''
-    return md5val, lictext
-
-def guess_license(srctree, d):
-    import bb
-    md5sums = get_license_md5sums(d)
-
-    crunched_md5sums = crunch_known_licenses(d)
-
-    licenses = []
-    licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*', 'e[dp]l-v10']
-    skip_extensions = (".html", ".js", ".json", ".svg", ".ts", ".go")
-    licfiles = []
-    for root, dirs, files in os.walk(srctree):
-        for fn in files:
-            if fn.endswith(skip_extensions):
-                continue
-            for spec in licspecs:
-                if fnmatch.fnmatch(fn, spec):
-                    fullpath = os.path.join(root, fn)
-                    if not fullpath in licfiles:
-                        licfiles.append(fullpath)
-    for licfile in sorted(licfiles):
-        md5value = bb.utils.md5_file(licfile)
-        license = md5sums.get(md5value, None)
-        if not license:
-            crunched_md5, lictext = crunch_license(licfile)
-            license = crunched_md5sums.get(crunched_md5, None)
-            if lictext and not license:
-                license = 'Unknown'
-                logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
-                    "and replace `Unknown` with the license:\n" \
-                    "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
-        if license:
-            licenses.append((license, os.path.relpath(licfile, srctree), md5value))
-
-    # FIXME should we grab at least one source file with a license header and add that too?
-
-    return licenses
-
 def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn='${PN}'):
     """
-    Given a list of (license, path, md5sum) as returned by guess_license(),
+    Given a list of (license, path, md5sum) as returned by match_licenses(),
     a dict of package name to path mappings, write out a set of
     package-specific LICENSE values.
     """
@@ -1284,6 +1068,14 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn
         outlicenses[pkgname] = licenses
     return outlicenses
 
+def generate_common_licenses_chksums(common_licenses, d):
+    lic_files_chksums = []
+    for license in tidy_licenses(common_licenses):
+        licfile = '${COMMON_LICENSE_DIR}/' + license
+        md5value = bb.utils.md5_file(d.expand(licfile))
+        lic_files_chksums.append('file://%s;md5=%s' % (licfile, md5value))
+    return lic_files_chksums
+
 def read_pkgconfig_provides(d):
     pkgdatadir = d.getVar('PKGDATA_DIR')
     pkgmap = {}
@@ -1418,4 +1210,3 @@ def register_commands(subparsers):
     parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS)
     parser_create.add_argument('--mirrors', action="store_true", help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by default).')
     parser_create.set_defaults(func=create_recipe)
-