8 files changed, 1169 insertions, 760 deletions
diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py
index 88ed8c5f01..041d79f162 100644
--- a/scripts/lib/recipetool/append.py
+++ b/scripts/lib/recipetool/append.py
@@ -18,6 +18,7 @@ import shutil
 import scriptutils
 import errno
 from collections import defaultdict
+import difflib
 logger = logging.getLogger('recipetool')
@@ -100,7 +101,7 @@ def determine_file_source(targetpath, rd):
    import oe.recipeutils
    # See if it's in do_install for the recipe
-    workdir = rd.getVar('WORKDIR')
+    unpackdir = rd.getVar('UNPACKDIR')
    src_uri = rd.getVar('SRC_URI')
    srcfile = ''
    modpatches = []
@@ -112,9 +113,9 @@ def determine_file_source(targetpath, rd):
        if not srcpath.startswith('/'):
            # Handle non-absolute path
            srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath))
-        if srcpath.startswith(workdir):
+        if srcpath.startswith(unpackdir):
            # OK, now we have the source file name, look for it in SRC_URI
-            workdirfile = os.path.relpath(srcpath, workdir)
+            workdirfile = os.path.relpath(srcpath, unpackdir)
            # FIXME this is where we ought to have some code in the fetcher, because this is naive
            for item in src_uri.split():
                localpath = bb.fetch2.localpath(item, rd)
@@ -299,7 +300,10 @@ def appendfile(args):
                if st.st_mode & stat.S_IXUSR:
                    perms = '0755'
            install = {args.newfile: (args.targetpath, perms)}
-        oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine)
+        if sourcepath:
+            sourcepath = os.path.basename(sourcepath)
+        oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: {'newname' : sourcepath}}, install, wildcardver=args.wildcard_version, machine=args.machine)
+        tinfoil.modified_files()
        return 0
    else:
        if alternative_pns:
@@ -313,7 +317,7 @@ def appendsrc(args, files, rd, extralines=None):
    import oe.recipeutils
    srcdir = rd.getVar('S')
-    workdir = rd.getVar('WORKDIR')
+    unpackdir = rd.getVar('UNPACKDIR')
    import bb.fetch
    simplified = {}
@@ -327,35 +331,57 @@ def appendsrc(args, files, rd, extralines=None):
    copyfiles = {}
    extralines = extralines or []
+    params = []
    for newfile, srcfile in files.items():
        src_destdir = os.path.dirname(srcfile)
        if not args.use_workdir:
            if rd.getVar('S') == rd.getVar('STAGING_KERNEL_DIR'):
-                srcdir = os.path.join(workdir, 'git')
+                srcdir = os.path.join(unpackdir, rd.getVar('BB_GIT_DEFAULT_DESTSUFFIX'))
                if not bb.data.inherits_class('kernel-yocto', rd):
-                    logger.warning('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git')
+                    logger.warning('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${UNPACKDIR}/${BB_GIT_DEFAULT_DESTSUFFIX}')
-            src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir)
+            src_destdir = os.path.join(os.path.relpath(srcdir, unpackdir), src_destdir)
        src_destdir = os.path.normpath(src_destdir)
-        source_uri = 'file://{0}'.format(os.path.basename(srcfile))
        if src_destdir and src_destdir != '.':
-            source_uri += ';subdir={0}'.format(src_destdir)
+            params.append({'subdir': src_destdir})
-        simple = bb.fetch.URI(source_uri)
-        simple.params = {}
-        simple_str = str(simple)
-        if simple_str in simplified:
-            existing = simplified[simple_str]
-            if source_uri != existing:
-                logger.warning('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing))
-            else:
-                logger.warning('{0!r} is already in SRC_URI, not adding'.format(source_uri))
        else:
-            extralines.append('SRC_URI += {0}'.format(source_uri))
+            params.append({})
-        copyfiles[newfile] = srcfile
+        copyfiles[newfile] = {'newname' : os.path.basename(srcfile)}
-    oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines)
+    dry_run_output = None
+    dry_run_outdir = None
+    if args.dry_run:
+        import tempfile
+        dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
+        dry_run_outdir = dry_run_output.name
+    appendfile, _ = oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines, params=params,
+                                                   redirect_output=dry_run_outdir, update_original_recipe=args.update_recipe)
+    if not appendfile:
+        return
+    if args.dry_run:
+        output = ''
+        appendfilename = os.path.basename(appendfile)
+        newappendfile = appendfile
+        if appendfile and os.path.exists(appendfile):
+            with open(appendfile, 'r') as f:
+                oldlines = f.readlines()
+        else:
+            appendfile = '/dev/null'
+            oldlines = []
+        with open(os.path.join(dry_run_outdir, appendfilename), 'r') as f:
+            newlines = f.readlines()
+        diff = difflib.unified_diff(oldlines, newlines, appendfile, newappendfile)
+        difflines = list(diff)
+        if difflines:
+            output += ''.join(difflines)
+        if output:
+            logger.info('Diff of changed files:\n%s' % output)
+        else:
+            logger.info('No changed files')
+    tinfoil.modified_files()
 def appendsrcfiles(parser, args):
    recipedata = _parse_recipe(args.recipe, tinfoil)
@@ -435,6 +461,8 @@ def register_commands(subparsers):
                                   help='Create/update a bbappend to add or replace source files',
                                   description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.')
    parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path)
+    parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true')
+    parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true')
    parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path)
    parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True)
@@ -442,6 +470,8 @@ def register_commands(subparsers):
                                   parents=[common_src],
                                   help='Create/update a bbappend to add or replace a source file',
                                   description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.')
+    parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true')
+    parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true')
    parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path)
    parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path)
    parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True)
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 4f6e01c639..ef0ba974a9 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -18,6 +18,8 @@ from urllib.parse import urlparse, urldefrag, urlsplit
 import hashlib
 import bb.fetch2
 logger = logging.getLogger('recipetool')
+from oe.license import tidy_licenses
+from oe.license_finder import find_licenses
 tinfoil = None
 plugins = None
@@ -389,9 +391,6 @@ def reformat_git_uri(uri):
                parms.update({('protocol', 'ssh')})
        elif (scheme == "http" or scheme == 'https' or scheme == 'ssh') and not ('protocol' in parms):
            parms.update({('protocol', scheme)})
-        # We assume 'master' branch if not set
-        if not 'branch' in parms:
-            parms.update({('branch', 'master')})
        # Always append 'git://'
        fUrl = bb.fetch2.encodeurl(('git', host, path, user, pswd, parms))
        return fUrl
@@ -426,6 +425,36 @@ def create_recipe(args):
    storeTagName = ''
    pv_srcpv = False
+    handled = []
+    classes = []
+    # Find all plugins that want to register handlers
+    logger.debug('Loading recipe handlers')
+    raw_handlers = []
+    for plugin in plugins:
+        if hasattr(plugin, 'register_recipe_handlers'):
+            plugin.register_recipe_handlers(raw_handlers)
+    # Sort handlers by priority
+    handlers = []
+    for i, handler in enumerate(raw_handlers):
+        if isinstance(handler, tuple):
+            handlers.append((handler[0], handler[1], i))
+        else:
+            handlers.append((handler, 0, i))
+    handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
+    for handler, priority, _ in handlers:
+        logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
+        setattr(handler, '_devtool', args.devtool)
+    handlers = [item[0] for item in handlers]
+    fetchuri = None
+    for handler in handlers:
+        if hasattr(handler, 'process_url'):
+            ret = handler.process_url(args, classes, handled, extravalues)
+            if 'url' in handled and ret:
+                fetchuri = ret
+                break
    if os.path.isfile(source):
        source = 'file://%s' % os.path.abspath(source)
@@ -434,11 +463,12 @@ def create_recipe(args):
        if re.match(r'https?://github.com/[^/]+/[^/]+/archive/.+(\.tar\..*|\.zip)$', source):
            logger.warning('github archive files are not guaranteed to be stable and may be re-generated over time. If the latter occurs, the checksums will likely change and the recipe will fail at do_fetch. It is recommended that you point to an actual commit or tag in the repository instead (using the repository URL in conjunction with the -S/--srcrev option).')
        # Fetch a URL
-        fetchuri = reformat_git_uri(urldefrag(source)[0])
+        if not fetchuri:
+            fetchuri = reformat_git_uri(urldefrag(source)[0])
        if args.binary:
            # Assume the archive contains the directory structure verbatim
            # so we need to extract to a subdirectory
-            fetchuri += ';subdir=${BP}'
+            fetchuri += ';subdir=${BPN}'
        srcuri = fetchuri
        rev_re = re.compile(';rev=([^;]+)')
        res = rev_re.search(srcuri)
@@ -481,6 +511,9 @@ def create_recipe(args):
            storeTagName = params['tag']
            params['nobranch'] = '1'
            del params['tag']
+        # Assume 'master' branch if not set
+        if scheme in ['git', 'gitsm'] and 'branch' not in params and 'nobranch' not in params:
+            params['branch'] = 'master'
        fetchuri = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params))
        tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
@@ -497,7 +530,7 @@ def create_recipe(args):
        if ftmpdir and args.keep_temp:
            logger.info('Fetch temp directory is %s' % ftmpdir)
-        dirlist = scriptutils.filter_src_subdirs(srctree)
+        dirlist = os.listdir(srctree)
        logger.debug('Directory listing (excluding filtered out):\n  %s' % '\n  '.join(dirlist))
        if len(dirlist) == 1:
            singleitem = os.path.join(srctree, dirlist[0])
@@ -530,10 +563,9 @@ def create_recipe(args):
            # Remove HEAD reference point and drop remote prefix
            get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')]
            if 'master' in get_branch:
-                # If it is master, we do not need to append 'branch=master' as this is default.
                # Even with the case where get_branch has multiple objects, if 'master' is one
                # of them, we should default take from 'master'
-                srcbranch = ''
+                srcbranch = 'master'
            elif len(get_branch) == 1:
                # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch'
                srcbranch = get_branch[0]
@@ -546,8 +578,8 @@ def create_recipe(args):
        # Since we might have a value in srcbranch, we need to
        # recontruct the srcuri to include 'branch' in params.
        scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(srcuri)
-        if srcbranch:
+        if scheme in ['git', 'gitsm']:
-            params['branch'] = srcbranch
+            params['branch'] = srcbranch or 'master'
        if storeTagName and scheme in ['git', 'gitsm']:
            # Check srcrev using tag and check validity of the tag
@@ -606,8 +638,7 @@ def create_recipe(args):
                    splitline = line.split()
                    if len(splitline) > 1:
                        if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]):
-                            srcuri = reformat_git_uri(splitline[1])
+                            srcuri = reformat_git_uri(splitline[1]) + ';branch=master'
-                            srcsubdir = 'git'
                            break
    if args.src_subdir:
@@ -639,8 +670,6 @@ def create_recipe(args):
    # We'll come back and replace this later in handle_license_vars()
    lines_before.append('##LICENSE_PLACEHOLDER##')
-    handled = []
-    classes = []
    # FIXME This is kind of a hack, we probably ought to be using bitbake to do this
    pn = None
@@ -678,8 +707,10 @@ def create_recipe(args):
    if not srcuri:
        lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
    lines_before.append('SRC_URI = "%s"' % srcuri)
+    shown_checksums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST]
    for key, value in sorted(checksums.items()):
-        lines_before.append('SRC_URI[%s] = "%s"' % (key, value))
+        if key in shown_checksums:
+            lines_before.append('SRC_URI[%s] = "%s"' % (key, value))
    if srcuri and supports_srcrev(srcuri):
        lines_before.append('')
        lines_before.append('# Modify these as desired')
@@ -691,7 +722,7 @@ def create_recipe(args):
            srcpvprefix = 'svnr'
        else:
            srcpvprefix = scheme
-        lines_before.append('PV = "%s+%s${SRCPV}"' % (realpv or '1.0', srcpvprefix))
+        lines_before.append('PV = "%s+%s"' % (realpv or '1.0', srcpvprefix))
        pv_srcpv = True
        if not args.autorev and srcrev == '${AUTOREV}':
            if os.path.exists(os.path.join(srctree, '.git')):
@@ -705,7 +736,7 @@ def create_recipe(args):
    if srcsubdir and not args.binary:
        # (for binary packages we explicitly specify subdir= when fetching to
        # match the default value of S, so we don't need to set it in that case)
-        lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
+        lines_before.append('S = "${UNPACKDIR}/%s"' % srcsubdir)
        lines_before.append('')
    if pkgarch:
@@ -719,25 +750,6 @@ def create_recipe(args):
    if args.npm_dev:
        extravalues['NPM_INSTALL_DEV'] = 1
-    # Find all plugins that want to register handlers
-    logger.debug('Loading recipe handlers')
-    raw_handlers = []
-    for plugin in plugins:
-        if hasattr(plugin, 'register_recipe_handlers'):
-            plugin.register_recipe_handlers(raw_handlers)
-    # Sort handlers by priority
-    handlers = []
-    for i, handler in enumerate(raw_handlers):
-        if isinstance(handler, tuple):
-            handlers.append((handler[0], handler[1], i))
-        else:
-            handlers.append((handler, 0, i))
-    handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
-    for handler, priority, _ in handlers:
-        logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
-        setattr(handler, '_devtool', args.devtool)
-    handlers = [item[0] for item in handlers]
    # Apply the handlers
    if args.binary:
        classes.append('bin_package')
@@ -746,9 +758,14 @@ def create_recipe(args):
    for handler in handlers:
        handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
+    # native and nativesdk classes are special and must be inherited last
+    # If present, put them at the end of the classes list
+    classes.sort(key=lambda c: c in ("native", "nativesdk"))
    extrafiles = extravalues.pop('extrafiles', {})
    extra_pn = extravalues.pop('PN', None)
    extra_pv = extravalues.pop('PV', None)
+    run_tasks = extravalues.pop('run_tasks', "").split()
    if extra_pv and not realpv:
        realpv = extra_pv
@@ -809,7 +826,8 @@ def create_recipe(args):
            extraoutdir = os.path.join(os.path.dirname(outfile), pn)
        bb.utils.mkdirhier(extraoutdir)
        for destfn, extrafile in extrafiles.items():
-            shutil.move(extrafile, os.path.join(extraoutdir, destfn))
+            fn = destfn.format(pn=pn, pv=realpv)
+            shutil.move(extrafile, os.path.join(extraoutdir, fn))
    lines = lines_before
    lines_before = []
@@ -824,7 +842,7 @@ def create_recipe(args):
                line = line.replace(realpv, '${PV}')
            if pn:
                line = line.replace(pn, '${BPN}')
-            if line == 'S = "${WORKDIR}/${BPN}-${PV}"':
+            if line == 'S = "${UNPACKDIR}/${BPN}-${PV}"' or 'tmp-recipetool-' in line:
                skipblank = True
                continue
        elif line.startswith('SRC_URI = '):
@@ -870,8 +888,10 @@ def create_recipe(args):
        outlines.append('')
    outlines.extend(lines_after)
+    outlines = [ line.rstrip('\n') +"\n" for line in outlines]
    if extravalues:
-        _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False)
+        _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=True)
    if args.extract_to:
        scriptutils.git_convert_standalone_clone(srctree)
@@ -887,7 +907,7 @@ def create_recipe(args):
        log_info_cond('Source extracted to %s' % args.extract_to, args.devtool)
    if outfile == '-':
-        sys.stdout.write('\n'.join(outlines) + '\n')
+        sys.stdout.write(''.join(outlines) + '\n')
    else:
        with open(outfile, 'w') as f:
            lastline = None
@@ -895,9 +915,14 @@ def create_recipe(args):
                if not lastline and not line:
                    # Skip extra blank lines
                    continue
-                f.write('%s\n' % line)
+                f.write('%s' % line)
                lastline = line
        log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool)
+        tinfoil.modified_files()
+    for task in run_tasks:
+        logger.info("Running task %s" % task)
+        tinfoil.build_file_sync(outfile, task)
    if tempsrc:
        if args.keep_temp:
@@ -920,23 +945,32 @@ def split_value(value):
    else:
        return value
+def fixup_license(value):
+    # Ensure licenses with OR starts and ends with brackets
+    if '|' in value:
+        return '(' + value + ')'
+    return value
 def handle_license_vars(srctree, lines_before, handled, extravalues, d):
    lichandled = [x for x in handled if x[0] == 'license']
    if lichandled:
        # Someone else has already handled the license vars, just return their value
        return lichandled[0][1]
-    licvalues = guess_license(srctree, d)
+    licvalues = find_licenses(srctree, d)
    licenses = []
    lic_files_chksum = []
    lic_unknown = []
    lines = []
    if licvalues:
        for licvalue in licvalues:
-            if not licvalue[0] in licenses:
+            license = licvalue[0]
-                licenses.append(licvalue[0])
+            lics = tidy_licenses(fixup_license(license))
+            lics = [lic for lic in lics if lic not in licenses]
+            if len(lics):
+                licenses.extend(lics)
            lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
-            if licvalue[0] == 'Unknown':
+            if license == 'Unknown':
                lic_unknown.append(licvalue[1])
        if lic_unknown:
            lines.append('#')
@@ -945,9 +979,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
            for licfile in lic_unknown:
                lines.append('#   %s' % licfile)
-    extra_license = split_value(extravalues.pop('LICENSE', []))
+    extra_license = tidy_licenses(extravalues.pop('LICENSE', ''))
-    if '&' in extra_license:
-        extra_license.remove('&')
    if extra_license:
        if licenses == ['Unknown']:
            licenses = extra_license
@@ -988,7 +1020,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
        lines.append('# instead of &. If there is any doubt, check the accompanying documentation')
        lines.append('# to determine which situation is applicable.')
-    lines.append('LICENSE = "%s"' % ' & '.join(licenses))
+    lines.append('LICENSE = "%s"' % ' & '.join(sorted(licenses, key=str.casefold)))
    lines.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n                    '.join(lic_files_chksum))
    lines.append('')
@@ -1005,228 +1037,15 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
    handled.append(('license', licvalues))
    return licvalues
-def get_license_md5sums(d, static_only=False, linenumbers=False):
+def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn='${PN}'):
-    import bb.utils
-    import csv
-    md5sums = {}
-    if not static_only and not linenumbers:
-        # Gather md5sums of license files in common license dir
-        commonlicdir = d.getVar('COMMON_LICENSE_DIR')
-        for fn in os.listdir(commonlicdir):
-            md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn))
-            md5sums[md5value] = fn
-    # The following were extracted from common values in various recipes
-    # (double checking the license against the license file itself, not just
-    # the LICENSE value in the recipe)
-    # Read license md5sums from csv file
-    scripts_path = os.path.dirname(os.path.realpath(__file__))
-    for path in (d.getVar('BBPATH').split(':')
-                + [os.path.join(scripts_path, '..', '..')]):
-        csv_path = os.path.join(path, 'lib', 'recipetool', 'licenses.csv')
-        if os.path.isfile(csv_path):
-            with open(csv_path, newline='') as csv_file:
-                fieldnames = ['md5sum', 'license', 'beginline', 'endline', 'md5']
-                reader = csv.DictReader(csv_file, delimiter=',', fieldnames=fieldnames)
-                for row in reader:
-                    if linenumbers:
-                        md5sums[row['md5sum']] = (
-                            row['license'], row['beginline'], row['endline'], row['md5'])
-                    else:
-                        md5sums[row['md5sum']] = row['license']
-    return md5sums
-def crunch_license(licfile):
-    '''
-    Remove non-material text from a license file and then check
-    its md5sum against a known list. This works well for licenses
-    which contain a copyright statement, but is also a useful way
-    to handle people's insistence upon reformatting the license text
-    slightly (with no material difference to the text of the
-    license).
-    '''
-    import oe.utils
-    # Note: these are carefully constructed!
-    license_title_re = re.compile(r'^#*\(? *(This is )?([Tt]he )?.{0,15} ?[Ll]icen[sc]e( \(.{1,10}\))?\)?[:\.]? ?#*$')
-    license_statement_re = re.compile(r'^((This (project|software)|.{1,10}) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$')
-    copyright_re = re.compile('^ *[#\*]* *(Modified work |MIT LICENSED )?Copyright ?(\([cC]\))? .*$')
-    disclaimer_re = re.compile('^ *\*? ?All [Rr]ights [Rr]eserved\.$')
-    email_re = re.compile('^.*<[\w\.-]*@[\w\.\-]*>$')
-    header_re = re.compile('^(\/\**!?)? ?[\-=\*]* ?(\*\/)?$')
-    tag_re = re.compile('^ *@?\(?([Ll]icense|MIT)\)?$')
-    url_re = re.compile('^ *[#\*]* *https?:\/\/[\w\.\/\-]+$')
-    crunched_md5sums = {}
-    # common licenses
-    crunched_md5sums['89f3bf322f30a1dcfe952e09945842f0'] = 'Apache-2.0'
-    crunched_md5sums['13b6fe3075f8f42f2270a748965bf3a1'] = 'BSD-0-Clause'
-    crunched_md5sums['ba87a7d7c20719c8df4b8beed9b78c43'] = 'BSD-2-Clause'
-    crunched_md5sums['7f8892c03b72de419c27be4ebfa253f8'] = 'BSD-3-Clause'
-    crunched_md5sums['21128c0790b23a8a9f9e260d5f6b3619'] = 'BSL-1.0'
-    crunched_md5sums['975742a59ae1b8abdea63a97121f49f4'] = 'EDL-1.0'
-    crunched_md5sums['5322cee4433d84fb3aafc9e253116447'] = 'EPL-1.0'
-    crunched_md5sums['6922352e87de080f42419bed93063754'] = 'EPL-2.0'
-    crunched_md5sums['793475baa22295cae1d3d4046a3a0ceb'] = 'GPL-2.0-only'
-    crunched_md5sums['ff9047f969b02c20f0559470df5cb433'] = 'GPL-2.0-or-later'
-    crunched_md5sums['ea6de5453fcadf534df246e6cdafadcd'] = 'GPL-3.0-only'
-    crunched_md5sums['b419257d4d153a6fde92ddf96acf5b67'] = 'GPL-3.0-or-later'
-    crunched_md5sums['228737f4c49d3ee75b8fb3706b090b84'] = 'ISC'
-    crunched_md5sums['c6a782e826ca4e85bf7f8b89435a677d'] = 'LGPL-2.0-only'
-    crunched_md5sums['32d8f758a066752f0db09bd7624b8090'] = 'LGPL-2.0-or-later'
-    crunched_md5sums['4820937eb198b4f84c52217ed230be33'] = 'LGPL-2.1-only'
-    crunched_md5sums['db13fe9f3a13af7adab2dc7a76f9e44a'] = 'LGPL-2.1-or-later'
-    crunched_md5sums['d7a0f2e4e0950e837ac3eabf5bd1d246'] = 'LGPL-3.0-only'
-    crunched_md5sums['abbf328e2b434f9153351f06b9f79d02'] = 'LGPL-3.0-or-later'
-    crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT'
-    crunched_md5sums['b218b0e94290b9b818c4be67c8e1cc82'] = 'MIT-0'
-    crunched_md5sums['ddc18131d6748374f0f35a621c245b49'] = 'Unlicense'
-    crunched_md5sums['51f9570ff32571fc0a443102285c5e33'] = 'WTFPL'
-    # The following two were gleaned from the "forever" npm package
-    crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC'
-    # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt
-    crunched_md5sums['50fab24ce589d69af8964fdbfe414c60'] = 'BSD-2-Clause'
-    # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE
-    crunched_md5sums['88a4355858a1433fea99fae34a44da88'] = 'GPLv2'
-    # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt
-    crunched_md5sums['063b5c3ebb5f3aa4c85a2ed18a31fbe7'] = 'GPLv2'
-    # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1
-    crunched_md5sums['7f5202f4d44ed15dcd4915f5210417d8'] = 'LGPLv2.1'
-    # unixODBC-2.3.4 COPYING
-    crunched_md5sums['3debde09238a8c8e1f6a847e1ec9055b'] = 'LGPLv2.1'
-    # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3
-    crunched_md5sums['f90c613c51aa35da4d79dd55fc724ceb'] = 'LGPLv3'
-    # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/epl-v10
-    crunched_md5sums['efe2cb9a35826992b9df68224e3c2628'] = 'EPL-1.0'
-    # https://raw.githubusercontent.com/jquery/esprima/3.1.3/LICENSE.BSD
-    crunched_md5sums['80fa7b56a28e8c902e6af194003220a5'] = 'BSD-2-Clause'
-    # https://raw.githubusercontent.com/npm/npm-install-checks/master/LICENSE
-    crunched_md5sums['e659f77bfd9002659e112d0d3d59b2c1'] = 'BSD-2-Clause'
-    # https://raw.githubusercontent.com/silverwind/default-gateway/4.2.0/LICENSE
-    crunched_md5sums['4c641f2d995c47f5cb08bdb4b5b6ea05'] = 'BSD-2-Clause'
-    # https://raw.githubusercontent.com/tad-lispy/node-damerau-levenshtein/v1.0.5/LICENSE
-    crunched_md5sums['2b8c039b2b9a25f0feb4410c4542d346'] = 'BSD-2-Clause'
-    # https://raw.githubusercontent.com/terser/terser/v3.17.0/LICENSE
-    crunched_md5sums['8bd23871802951c9ad63855151204c2c'] = 'BSD-2-Clause'
-    # https://raw.githubusercontent.com/alexei/sprintf.js/1.0.3/LICENSE
-    crunched_md5sums['008c22318c8ea65928bf730ddd0273e3'] = 'BSD-3-Clause'
-    # https://raw.githubusercontent.com/Caligatio/jsSHA/v3.2.0/LICENSE
-    crunched_md5sums['0e46634a01bfef056892949acaea85b1'] = 'BSD-3-Clause'
-    # https://raw.githubusercontent.com/d3/d3-path/v1.0.9/LICENSE
-    crunched_md5sums['b5f72aef53d3b2b432702c30b0215666'] = 'BSD-3-Clause'
-    # https://raw.githubusercontent.com/feross/ieee754/v1.1.13/LICENSE
-    crunched_md5sums['a39327c997c20da0937955192d86232d'] = 'BSD-3-Clause'
-    # https://raw.githubusercontent.com/joyent/node-extsprintf/v1.3.0/LICENSE
-    crunched_md5sums['721f23a96ff4161ca3a5f071bbe18108'] = 'MIT'
-    # https://raw.githubusercontent.com/pvorb/clone/v0.2.0/LICENSE
-    crunched_md5sums['b376d29a53c9573006b9970709231431'] = 'MIT'
-    # https://raw.githubusercontent.com/andris9/encoding/v0.1.12/LICENSE
-    crunched_md5sums['85d8a977ee9d7c5ab4ac03c9b95431c4'] = 'MIT-0'
-    # https://raw.githubusercontent.com/faye/websocket-driver-node/0.7.3/LICENSE.md
-    crunched_md5sums['b66384e7137e41a9b1904ef4d39703b6'] = 'Apache-2.0'
-    # https://raw.githubusercontent.com/less/less.js/v4.1.1/LICENSE
-    crunched_md5sums['b27575459e02221ccef97ec0bfd457ae'] = 'Apache-2.0'
-    # https://raw.githubusercontent.com/microsoft/TypeScript/v3.5.3/LICENSE.txt
-    crunched_md5sums['a54a1a6a39e7f9dbb4a23a42f5c7fd1c'] = 'Apache-2.0'
-    # https://raw.githubusercontent.com/request/request/v2.87.0/LICENSE
-    crunched_md5sums['1034431802e57486b393d00c5d262b8a'] = 'Apache-2.0'
-    # https://raw.githubusercontent.com/dchest/tweetnacl-js/v0.14.5/LICENSE
-    crunched_md5sums['75605e6bdd564791ab698fca65c94a4f'] = 'Unlicense'
-    # https://raw.githubusercontent.com/stackgl/gl-mat3/v2.0.0/LICENSE.md
-    crunched_md5sums['75512892d6f59dddb6d1c7e191957e9c'] = 'Zlib'
-    lictext = []
-    with open(licfile, 'r', errors='surrogateescape') as f:
-        for line in f:
-            # Drop opening statements
-            if copyright_re.match(line):
-                continue
-            elif disclaimer_re.match(line):
-                continue
-            elif email_re.match(line):
-                continue
-            elif header_re.match(line):
-                continue
-            elif tag_re.match(line):
-                continue
-            elif url_re.match(line):
-                continue
-            elif license_title_re.match(line):
-                continue
-            elif license_statement_re.match(line):
-                continue
-            # Strip comment symbols
-            line = line.replace('*', '') \
-                       .replace('#', '')
-            # Unify spelling
-            line = line.replace('sub-license', 'sublicense')
-            # Squash spaces
-            line = oe.utils.squashspaces(line.strip())
-            # Replace smart quotes, double quotes and backticks with single quotes
-            line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'')
-            # Unify brackets
-            line = line.replace("{", "[").replace("}", "]")
-            if line:
-                lictext.append(line)
-    m = hashlib.md5()
-    try:
-        m.update(' '.join(lictext).encode('utf-8'))
-        md5val = m.hexdigest()
-    except UnicodeEncodeError:
-        md5val = None
-        lictext = ''
-    license = crunched_md5sums.get(md5val, None)
-    return license, md5val, lictext
-def guess_license(srctree, d):
-    import bb
-    md5sums = get_license_md5sums(d)
-    licenses = []
-    licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*', 'e[dp]l-v10']
-    skip_extensions = (".html", ".js", ".json", ".svg", ".ts")
-    licfiles = []
-    for root, dirs, files in os.walk(srctree):
-        for fn in files:
-            if fn.endswith(skip_extensions):
-                continue
-            for spec in licspecs:
-                if fnmatch.fnmatch(fn, spec):
-                    fullpath = os.path.join(root, fn)
-                    if not fullpath in licfiles:
-                        licfiles.append(fullpath)
-    for licfile in licfiles:
-        md5value = bb.utils.md5_file(licfile)
-        license = md5sums.get(md5value, None)
-        if not license:
-            license, crunched_md5, lictext = crunch_license(licfile)
-            if lictext and not license:
-                license = 'Unknown'
-                logger.info("Please add the following line for '%s' to a 'lib/recipetool/licenses.csv' " \
-                    "and replace `Unknown` with the license:\n" \
-                    "%s,Unknown" % (os.path.relpath(licfile, srctree), md5value))
-        if license:
-            licenses.append((license, os.path.relpath(licfile, srctree), md5value))
-    # FIXME should we grab at least one source file with a license header and add that too?
-    return licenses
-def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=[], pn='${PN}'):
    """
-    Given a list of (license, path, md5sum) as returned by guess_license(),
+    Given a list of (license, path, md5sum) as returned by match_licenses(),
    a dict of package name to path mappings, write out a set of
    package-specific LICENSE values.
    """
    pkglicenses = {pn: []}
    for license, licpath, _ in licvalues:
+        license = fixup_license(license)
        for pkgname, pkgpath in packages.items():
            if licpath.startswith(pkgpath + '/'):
                if pkgname in pkglicenses:
@@ -1239,13 +1058,24 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=[], pn='
            pkglicenses[pn].append(license)
    outlicenses = {}
    for pkgname in packages:
-        license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown'
+        # Assume AND operator between license files
-        if license == 'Unknown' and pkgname in fallback_licenses:
+        license = ' & '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown'
+        if license == 'Unknown' and fallback_licenses and pkgname in fallback_licenses:
            license = fallback_licenses[pkgname]
+        licenses = tidy_licenses(license)
+        license = ' & '.join(licenses)
        outlines.append('LICENSE:%s = "%s"' % (pkgname, license))
-        outlicenses[pkgname] = license.split()
+        outlicenses[pkgname] = licenses
    return outlicenses
+def generate_common_licenses_chksums(common_licenses, d):
+    lic_files_chksums = []
+    for license in tidy_licenses(common_licenses):
+        licfile = '${COMMON_LICENSE_DIR}/' + license
+        md5value = bb.utils.md5_file(d.expand(licfile))
+        lic_files_chksums.append('file://%s;md5=%s' % (licfile, md5value))
+    return lic_files_chksums
 def read_pkgconfig_provides(d):
    pkgdatadir = d.getVar('PKGDATA_DIR')
    pkgmap = {}
@@ -1376,7 +1206,7 @@ def register_commands(subparsers):
    parser_create.add_argument('-B', '--srcbranch', help='Branch in source repository if fetching from an SCM such as git (default master)')
    parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
    parser_create.add_argument('--npm-dev', action="store_true", help='For npm, also fetch devDependencies')
+    parser_create.add_argument('--no-pypi', action="store_true", help='Do not inherit pypi class')
    parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS)
    parser_create.add_argument('--mirrors', action="store_true", help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by default).')
    parser_create.set_defaults(func=create_recipe)
diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py
index 5015634476..ec9d510e23 100644
--- a/scripts/lib/recipetool/create_buildsys.py
+++ b/scripts/lib/recipetool/create_buildsys.py
@@ -5,9 +5,9 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
+import os
 import re
 import logging
-import glob
 from recipetool.create import RecipeHandler, validate_pv
 logger = logging.getLogger('recipetool')
@@ -137,15 +137,15 @@ class CmakeRecipeHandler(RecipeHandler):
        deps = []
        unmappedpkgs = []
-        proj_re = re.compile('project\s*\(([^)]*)\)', re.IGNORECASE)
+        proj_re = re.compile(r'project\s*\(([^)]*)\)', re.IGNORECASE)
-        pkgcm_re = re.compile('pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE)
+        pkgcm_re = re.compile(r'pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE)
-        pkgsm_re = re.compile('pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE)
+        pkgsm_re = re.compile(r'pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE)
-        findpackage_re = re.compile('find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE)
+        findpackage_re = re.compile(r'find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE)
-        findlibrary_re = re.compile('find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*')
+        findlibrary_re = re.compile(r'find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*')
-        checklib_re = re.compile('check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE)
+        checklib_re = re.compile(r'check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE)
-        include_re = re.compile('include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE)
+        include_re = re.compile(r'include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE)
-        subdir_re = re.compile('add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE)
+        subdir_re = re.compile(r'add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE)
-        dep_re = re.compile('([^ ><=]+)( *[<>=]+ *[^ ><=]+)?')
+        dep_re = re.compile(r'([^ ><=]+)( *[<>=]+ *[^ ><=]+)?')
        def find_cmake_package(pkg):
            RecipeHandler.load_devel_filemap(tinfoil.config_data)
@@ -423,16 +423,16 @@ class AutotoolsRecipeHandler(RecipeHandler):
                'makeinfo': 'texinfo',
                }
-        pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
+        pkg_re = re.compile(r'PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
-        pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
+        pkgce_re = re.compile(r'PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
-        lib_re = re.compile('AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*')
+        lib_re = re.compile(r'AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*')
-        libx_re = re.compile('AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*')
+        libx_re = re.compile(r'AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*')
-        progs_re = re.compile('_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
+        progs_re = re.compile(r'_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
-        dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?')
+        dep_re = re.compile(r'([^ ><=]+)( [<>=]+ [^ ><=]+)?')
-        ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*')
+        ac_init_re = re.compile(r'AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*')
-        am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*')
+        am_init_re = re.compile(r'AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*')
-        define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)')
+        define_re = re.compile(r'\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)')
-        version_re = re.compile('([0-9.]+)')
+        version_re = re.compile(r'([0-9.]+)')
        defines = {}
        def subst_defines(value):
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py
index 0b6b042ed1..a807dafae5 100644
--- a/scripts/lib/recipetool/create_buildsys_python.py
+++ b/scripts/lib/recipetool/create_buildsys_python.py
@@ -8,9 +8,9 @@
 import ast
 import codecs
 import collections
-import distutils.command.build_py
+import setuptools.command.build_py
 import email
-import imp
+import importlib
 import glob
 import itertools
 import logging
@@ -18,7 +18,11 @@ import os
 import re
 import sys
 import subprocess
+import json
+import urllib.request
 from recipetool.create import RecipeHandler
+from urllib.parse import urldefrag
+from recipetool.create import determine_from_url
 logger = logging.getLogger('recipetool')
@@ -37,7 +41,334 @@ class PythonRecipeHandler(RecipeHandler):
    assume_provided = ['builtins', 'os.path']
    # Assumes that the host python3 builtin_module_names is sane for target too
    assume_provided = assume_provided + list(sys.builtin_module_names)
+    excluded_fields = []
+    classifier_license_map = {
+        'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
+        'License :: OSI Approved :: Apache Software License': 'Apache',
+        'License :: OSI Approved :: Apple Public Source License': 'APSL',
+        'License :: OSI Approved :: Artistic License': 'Artistic',
+        'License :: OSI Approved :: Attribution Assurance License': 'AAL',
+        'License :: OSI Approved :: BSD License': 'BSD-3-Clause',
+        'License :: OSI Approved :: Boost Software License 1.0 (BSL-1.0)': 'BSL-1.0',
+        'License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1)': 'CECILL-2.1',
+        'License :: OSI Approved :: Common Development and Distribution License 1.0 (CDDL-1.0)': 'CDDL-1.0',
+        'License :: OSI Approved :: Common Public License': 'CPL',
+        'License :: OSI Approved :: Eclipse Public License 1.0 (EPL-1.0)': 'EPL-1.0',
+        'License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0)': 'EPL-2.0',
+        'License :: OSI Approved :: Eiffel Forum License': 'EFL',
+        'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0',
+        'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1',
+        'License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)': 'EUPL-1.2',
+        'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0-only',
+        'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0-or-later',
+        'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL',
+        'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL',
+        'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0-only',
+        'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0-or-later',
+        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0-only',
+        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0-or-later',
+        'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0-only',
+        'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0-or-later',
+        'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0-only',
+        'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0-or-later',
+        'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL',
+        'License :: OSI Approved :: Historical Permission Notice and Disclaimer (HPND)': 'HPND',
+        'License :: OSI Approved :: IBM Public License': 'IPL',
+        'License :: OSI Approved :: ISC License (ISCL)': 'ISC',
+        'License :: OSI Approved :: Intel Open Source License': 'Intel',
+        'License :: OSI Approved :: Jabber Open Source License': 'Jabber',
+        'License :: OSI Approved :: MIT License': 'MIT',
+        'License :: OSI Approved :: MIT No Attribution License (MIT-0)': 'MIT-0',
+        'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL',
+        'License :: OSI Approved :: MirOS License (MirOS)': 'MirOS',
+        'License :: OSI Approved :: Motosoto License': 'Motosoto',
+        'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0',
+        'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1',
+        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0',
+        'License :: OSI Approved :: Nethack General Public License': 'NGPL',
+        'License :: OSI Approved :: Nokia Open Source License': 'Nokia',
+        'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL',
+        'License :: OSI Approved :: Open Software License 3.0 (OSL-3.0)': 'OSL-3.0',
+        'License :: OSI Approved :: PostgreSQL License': 'PostgreSQL',
+        'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python',
+        'License :: OSI Approved :: Python Software Foundation License': 'PSF-2.0',
+        'License :: OSI Approved :: Qt Public License (QPL)': 'QPL',
+        'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL',
+        'License :: OSI Approved :: SIL Open Font License 1.1 (OFL-1.1)': 'OFL-1.1',
+        'License :: OSI Approved :: Sleepycat License': 'Sleepycat',
+        'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': 'SISSL',
+        'License :: OSI Approved :: Sun Public License': 'SPL',
+        'License :: OSI Approved :: The Unlicense (Unlicense)': 'Unlicense',
+        'License :: OSI Approved :: Universal Permissive License (UPL)': 'UPL-1.0',
+        'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA',
+        'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0',
+        'License :: OSI Approved :: W3C License': 'W3C',
+        'License :: OSI Approved :: X.Net License': 'Xnet',
+        'License :: OSI Approved :: Zope Public License': 'ZPL',
+        'License :: OSI Approved :: zlib/libpng License': 'Zlib',
+        'License :: Other/Proprietary License': 'Proprietary',
+        'License :: Public Domain': 'PD',
+    }
+    def __init__(self):
+        pass
+    def process_url(self, args, classes, handled, extravalues):
+        """
+        Convert any pypi url https://pypi.org/project/<package>/<version> into https://files.pythonhosted.org/packages/source/...
+        which corresponds to the archive location, and add pypi class
+        """
+        if 'url' in handled:
+            return None
+        fetch_uri = None
+        source = args.source
+        required_version = args.version if args.version else None
+        match = re.match(r'https?://pypi.org/project/([^/]+)(?:/([^/]+))?/?$', urldefrag(source)[0])
+        if match:
+            package = match.group(1)
+            version = match.group(2) if match.group(2) else required_version
+            json_url = f"https://pypi.org/pypi/%s/json" % package
+            response = urllib.request.urlopen(json_url)
+            if response.status == 200:
+                data = json.loads(response.read())
+                if not version:
+                    # grab latest version
+                    version = data["info"]["version"]
+                pypi_package = data["info"]["name"]
+                for release in reversed(data["releases"][version]):
+                    if release["packagetype"] == "sdist":
+                        fetch_uri = release["url"]
+                        break
+            else:
+                logger.warning("Cannot handle pypi url %s: cannot fetch package information using %s", source, json_url)
+                return None
+        else:
+            match = re.match(r'^https?://files.pythonhosted.org/packages.*/(.*)-.*$', source)
+            if match:
+                fetch_uri = source
+                pypi_package = match.group(1)
+                _, version = determine_from_url(fetch_uri)
+        if match and not args.no_pypi:
+            if required_version and version != required_version:
+                raise Exception("Version specified using --version/-V (%s) and version specified in the url (%s) do not match" % (required_version, version))
+            # This is optionnal if BPN looks like "python-<pypi_package>" or "python3-<pypi_package>" (see pypi.bbclass)
+            # but at this point we cannot know because because user can specify the output name of the recipe on the command line
+            extravalues["PYPI_PACKAGE"] = pypi_package
+            # If the tarball extension is not 'tar.gz' (default value in pypi.bblcass) whe should set PYPI_PACKAGE_EXT in the recipe
+            pypi_package_ext = re.match(r'.*%s-%s\.(.*)$' % (pypi_package, version), fetch_uri)
+            if pypi_package_ext:
+                pypi_package_ext = pypi_package_ext.group(1)
+                if pypi_package_ext != "tar.gz":
+                    extravalues["PYPI_PACKAGE_EXT"] = pypi_package_ext
+            # Pypi class will handle S and SRC_URI variables, so remove them
+            # TODO: allow oe.recipeutils.patch_recipe_lines() to accept regexp so we can simplify the following to:
+            # extravalues['SRC_URI(?:\[.*?\])?'] = None
+            extravalues['S'] = None
+            extravalues['SRC_URI'] = None
+            classes.append('pypi')
+        handled.append('url')
+        return fetch_uri
+    def handle_classifier_license(self, classifiers, existing_licenses=""):
+        licenses = []
+        for classifier in classifiers:
+            if classifier in self.classifier_license_map:
+                license = self.classifier_license_map[classifier]
+                if license == 'Apache' and 'Apache-2.0' in existing_licenses:
+                    license = 'Apache-2.0'
+                elif license == 'GPL':
+                    if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
+                        license = 'GPL-2.0'
+                    elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
+                        license = 'GPL-3.0'
+                elif license == 'LGPL':
+                    if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
+                        license = 'LGPL-2.1'
+                    elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
+                        license = 'LGPL-2.0'
+                    elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
+                        license = 'LGPL-3.0'
+                licenses.append(license)
+        if licenses:
+            return ' & '.join(licenses)
+        return None
+    def map_info_to_bbvar(self, info, extravalues):
+        # Map PKG-INFO & setup.py fields to bitbake variables
+        for field, values in info.items():
+            if field in self.excluded_fields:
+                continue
+            if field not in self.bbvar_map:
+                continue
+            if isinstance(values, str):
+                value = values
+            else:
+                value = ' '.join(str(v) for v in values if v)
+            bbvar = self.bbvar_map[field]
+            if bbvar == "PN":
+                # by convention python recipes start with "python3-"
+                if not value.startswith('python'):
+                    value = 'python3-' + value
+            if bbvar not in extravalues and value:
+                extravalues[bbvar] = value
+    def apply_info_replacements(self, info):
+        if not self.replacements:
+            return
+        for variable, search, replace in self.replacements:
+            if variable not in info:
+                continue
+            def replace_value(search, replace, value):
+                if replace is None:
+                    if re.search(search, value):
+                        return None
+                else:
+                    new_value = re.sub(search, replace, value)
+                    if value != new_value:
+                        return new_value
+                return value
+            value = info[variable]
+            if isinstance(value, str):
+                new_value = replace_value(search, replace, value)
+                if new_value is None:
+                    del info[variable]
+                elif new_value != value:
+                    info[variable] = new_value
+            elif hasattr(value, 'items'):
+                for dkey, dvalue in list(value.items()):
+                    new_list = []
+                    for pos, a_value in enumerate(dvalue):
+                        new_value = replace_value(search, replace, a_value)
+                        if new_value is not None and new_value != value:
+                            new_list.append(new_value)
+                    if value != new_list:
+                        value[dkey] = new_list
+            else:
+                new_list = []
+                for pos, a_value in enumerate(value):
+                    new_value = replace_value(search, replace, a_value)
+                    if new_value is not None and new_value != value:
+                        new_list.append(new_value)
+                if value != new_list:
+                    info[variable] = new_list
+    def scan_python_dependencies(self, paths):
+        deps = set()
+        try:
+            dep_output = self.run_command(['pythondeps', '-d'] + paths)
+        except (OSError, subprocess.CalledProcessError):
+            pass
+        else:
+            for line in dep_output.splitlines():
+                line = line.rstrip()
+                dep, filename = line.split('\t', 1)
+                if filename.endswith('/setup.py'):
+                    continue
+                deps.add(dep)
+        try:
+            provides_output = self.run_command(['pythondeps', '-p'] + paths)
+        except (OSError, subprocess.CalledProcessError):
+            pass
+        else:
+            provides_lines = (l.rstrip() for l in provides_output.splitlines())
+            provides = set(l for l in provides_lines if l and l != 'setup')
+            deps -= provides
+        return deps
+    def parse_pkgdata_for_python_packages(self):
+        pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
+        ldata = tinfoil.config_data.createCopy()
+        bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True)
+        python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
+        dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
+        python_dirs = [python_sitedir + os.sep,
+                       os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
+                       os.path.dirname(python_sitedir) + os.sep]
+        packages = {}
+        for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
+            files_info = None
+            with open(pkgdatafile, 'r') as f:
+                for line in f.readlines():
+                    field, value = line.split(': ', 1)
+                    if field.startswith('FILES_INFO'):
+                        files_info = ast.literal_eval(value)
+                        break
+                else:
+                    continue
+            for fn in files_info:
+                for suffix in importlib.machinery.all_suffixes():
+                    if fn.endswith(suffix):
+                        break
+                else:
+                    continue
+                if fn.startswith(dynload_dir + os.sep):
+                    if '/.debug/' in fn:
+                        continue
+                    base = os.path.basename(fn)
+                    provided = base.split('.', 1)[0]
+                    packages[provided] = os.path.basename(pkgdatafile)
+                    continue
+                for python_dir in python_dirs:
+                    if fn.startswith(python_dir):
+                        relpath = fn[len(python_dir):]
+                        relstart, _, relremaining = relpath.partition(os.sep)
+                        if relstart.endswith('.egg'):
+                            relpath = relremaining
+                        base, _ = os.path.splitext(relpath)
+                        if '/.debug/' in base:
+                            continue
+                        if os.path.basename(base) == '__init__':
+                            base = os.path.dirname(base)
+                        base = base.replace(os.sep + os.sep, os.sep)
+                        provided = base.replace(os.sep, '.')
+                        packages[provided] = os.path.basename(pkgdatafile)
+        return packages
+    @classmethod
+    def run_command(cls, cmd, **popenargs):
+        if 'stderr' not in popenargs:
+            popenargs['stderr'] = subprocess.STDOUT
+        try:
+            return subprocess.check_output(cmd, **popenargs).decode('utf-8')
+        except OSError as exc:
+            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
+            raise
+        except subprocess.CalledProcessError as exc:
+            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
+            raise
+class PythonSetupPyRecipeHandler(PythonRecipeHandler):
    bbvar_map = {
        'Name': 'PN',
        'Version': 'PV',
@@ -75,6 +406,7 @@ class PythonRecipeHandler(RecipeHandler):
        'Supported-Platform',
    ]
    setuparg_multi_line_values = ['Description']
    replacements = [
        ('License', r' +$', ''),
        ('License', r'^ +', ''),
@@ -95,71 +427,161 @@ class PythonRecipeHandler(RecipeHandler):
        ('Install-requires', r'\[[^\]]+\]$', ''),
    ]
-    classifier_license_map = {
-        'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
-        'License :: OSI Approved :: Apache Software License': 'Apache',
-        'License :: OSI Approved :: Apple Public Source License': 'APSL',
-        'License :: OSI Approved :: Artistic License': 'Artistic',
-        'License :: OSI Approved :: Attribution Assurance License': 'AAL',
-        'License :: OSI Approved :: BSD License': 'BSD-3-Clause',
-        'License :: OSI Approved :: Common Public License': 'CPL',
-        'License :: OSI Approved :: Eiffel Forum License': 'EFL',
-        'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0',
-        'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1',
-        'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0+',
-        'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0',
-        'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL',
-        'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL',
-        'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0',
-        'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0+',
-        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0',
-        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0+',
-        'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0',
-        'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0+',
-        'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0',
-        'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0+',
-        'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL',
-        'License :: OSI Approved :: IBM Public License': 'IPL',
-        'License :: OSI Approved :: ISC License (ISCL)': 'ISC',
-        'License :: OSI Approved :: Intel Open Source License': 'Intel',
-        'License :: OSI Approved :: Jabber Open Source License': 'Jabber',
-        'License :: OSI Approved :: MIT License': 'MIT',
-        'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL',
-        'License :: OSI Approved :: Motosoto License': 'Motosoto',
-        'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0',
-        'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1',
-        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0',
-        'License :: OSI Approved :: Nethack General Public License': 'NGPL',
-        'License :: OSI Approved :: Nokia Open Source License': 'Nokia',
-        'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL',
-        'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python',
-        'License :: OSI Approved :: Python Software Foundation License': 'PSF',
-        'License :: OSI Approved :: Qt Public License (QPL)': 'QPL',
-        'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL',
-        'License :: OSI Approved :: Sleepycat License': 'Sleepycat',
-        'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': '--  Sun Industry Standards Source License (SISSL)',
-        'License :: OSI Approved :: Sun Public License': 'SPL',
-        'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA',
-        'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0',
-        'License :: OSI Approved :: W3C License': 'W3C',
-        'License :: OSI Approved :: X.Net License': 'Xnet',
-        'License :: OSI Approved :: Zope Public License': 'ZPL',
-        'License :: OSI Approved :: zlib/libpng License': 'Zlib',
-    }
    def __init__(self):
        pass
+    def parse_setup_py(self, setupscript='./setup.py'):
+        with codecs.open(setupscript) as f:
+            info, imported_modules, non_literals, extensions = gather_setup_info(f)
+        def _map(key):
+            key = key.replace('_', '-')
+            key = key[0].upper() + key[1:]
+            if key in self.setup_parse_map:
+                key = self.setup_parse_map[key]
+            return key
+        # Naive mapping of setup() arguments to PKG-INFO field names
+        for d in [info, non_literals]:
+            for key, value in list(d.items()):
+                if key is None:
+                    continue
+                new_key = _map(key)
+                if new_key != key:
+                    del d[key]
+                    d[new_key] = value
+        return info, 'setuptools' in imported_modules, non_literals, extensions
+    def get_setup_args_info(self, setupscript='./setup.py'):
+        cmd = ['python3', setupscript]
+        info = {}
+        keys = set(self.bbvar_map.keys())
+        keys |= set(self.setuparg_list_fields)
+        keys |= set(self.setuparg_multi_line_values)
+        grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
+        for index, keys in grouped_keys:
+            if index == (True, False):
+                # Splitlines output for each arg as a list value
+                for key in keys:
+                    arg = self.setuparg_map.get(key, key.lower())
+                    try:
+                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
+                    except (OSError, subprocess.CalledProcessError):
+                        pass
+                    else:
+                        info[key] = [l.rstrip() for l in arg_info.splitlines()]
+            elif index == (False, True):
+                # Entire output for each arg
+                for key in keys:
+                    arg = self.setuparg_map.get(key, key.lower())
+                    try:
+                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
+                    except (OSError, subprocess.CalledProcessError):
+                        pass
+                    else:
+                        info[key] = arg_info
+            else:
+                info.update(self.get_setup_byline(list(keys), setupscript))
+        return info
+    def get_setup_byline(self, fields, setupscript='./setup.py'):
+        info = {}
+        cmd = ['python3', setupscript]
+        cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
+        try:
+            info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
+        except (OSError, subprocess.CalledProcessError):
+            pass
+        else:
+            if len(fields) != len(info_lines):
+                logger.error('Mismatch between setup.py output lines and number of fields')
+                sys.exit(1)
+            for lineno, line in enumerate(info_lines):
+                line = line.rstrip()
+                info[fields[lineno]] = line
+        return info
+    def get_pkginfo(self, pkginfo_fn):
+        msg = email.message_from_file(open(pkginfo_fn, 'r'))
+        msginfo = {}
+        for field in msg.keys():
+            values = msg.get_all(field)
+            if len(values) == 1:
+                msginfo[field] = values[0]
+            else:
+                msginfo[field] = values
+        return msginfo
+    def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
+        if 'Package-dir' in setup_info:
+            package_dir = setup_info['Package-dir']
+        else:
+            package_dir = {}
+        dist = setuptools.Distribution()
+        class PackageDir(setuptools.command.build_py.build_py):
+            def __init__(self, package_dir):
+                self.package_dir = package_dir
+                self.dist = dist
+                super().__init__(self.dist)
+        pd = PackageDir(package_dir)
+        to_scan = []
+        if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
+            if 'Py-modules' in setup_info:
+                for module in setup_info['Py-modules']:
+                    try:
+                        package, module = module.rsplit('.', 1)
+                    except ValueError:
+                        package, module = '.', module
+                    module_path = os.path.join(pd.get_package_dir(package), module + '.py')
+                    to_scan.append(module_path)
+            if 'Packages' in setup_info:
+                for package in setup_info['Packages']:
+                    to_scan.append(pd.get_package_dir(package))
+            if 'Scripts' in setup_info:
+                to_scan.extend(setup_info['Scripts'])
+        else:
+            logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
+        if not to_scan:
+            to_scan = ['.']
+        logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
+        provided_packages = self.parse_pkgdata_for_python_packages()
+        scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
+        mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
+        for dep in scanned_deps:
+            mapped = provided_packages.get(dep)
+            if mapped:
+                logger.debug('Mapped %s to %s' % (dep, mapped))
+                mapped_deps.add(mapped)
+            else:
+                logger.debug('Could not map %s' % dep)
+                unmapped_deps.add(dep)
+        return mapped_deps, unmapped_deps
    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
        if 'buildsystem' in handled:
            return False
+        logger.debug("Trying setup.py parser")
        # Check for non-zero size setup.py files
        setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py'])
        for fn in setupfiles:
            if os.path.getsize(fn):
                break
        else:
+            logger.debug("No setup.py found")
            return False
        # setup.py is always parsed to get at certain required information, such as
@@ -193,6 +615,18 @@ class PythonRecipeHandler(RecipeHandler):
                            continue
                        if line.startswith('['):
+                            # PACKAGECONFIG must not contain expressions or whitespace
+                            line = line.replace(" ", "")
+                            line = line.replace(':', "")
+                            line = line.replace('.', "-dot-")
+                            line = line.replace('"', "")
+                            line = line.replace('<', "-smaller-")
+                            line = line.replace('>', "-bigger-")
+                            line = line.replace('_', "-")
+                            line = line.replace('(', "")
+                            line = line.replace(')', "")
+                            line = line.replace('!', "-not-")
+                            line = line.replace('=', "-equals-")
                            current_feature = line[1:-1]
                        elif current_feature:
                            extras_req[current_feature].append(line)
@@ -226,51 +660,16 @@ class PythonRecipeHandler(RecipeHandler):
        if license_str:
            for i, line in enumerate(lines_before):
-                if line.startswith('LICENSE = '):
+                if line.startswith('##LICENSE_PLACEHOLDER##'):
                    lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
                    break
        if 'Classifier' in info:
-            existing_licenses = info.get('License', '')
+            license = self.handle_classifier_license(info['Classifier'], info.get('License', ''))
-            licenses = []
+            if license:
-            for classifier in info['Classifier']:
+                info['License'] = license
-                if classifier in self.classifier_license_map:
-                    license = self.classifier_license_map[classifier]
-                    if license == 'Apache' and 'Apache-2.0' in existing_licenses:
-                        license = 'Apache-2.0'
-                    elif license == 'GPL':
-                        if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
-                            license = 'GPL-2.0'
-                        elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
-                            license = 'GPL-3.0'
-                    elif license == 'LGPL':
-                        if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
-                            license = 'LGPL-2.1'
-                        elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
-                            license = 'LGPL-2.0'
-                        elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
-                            license = 'LGPL-3.0'
-                    licenses.append(license)
-            if licenses:
-                info['License'] = ' & '.join(licenses)
-        # Map PKG-INFO & setup.py fields to bitbake variables
+        self.map_info_to_bbvar(info, extravalues)
-        for field, values in info.items():
-            if field in self.excluded_fields:
-                continue
-            if field not in self.bbvar_map:
-                continue
-            if isinstance(values, str):
-                value = values
-            else:
-                value = ' '.join(str(v) for v in values if v)
-            bbvar = self.bbvar_map[field]
-            if bbvar not in extravalues and value:
-                extravalues[bbvar] = value
        mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
@@ -281,6 +680,7 @@ class PythonRecipeHandler(RecipeHandler):
                lines_after.append('# The following configs & dependencies are from setuptools extras_require.')
                lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.')
                lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
+                lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.')
                lines_after.append('#')
                lines_after.append('# Uncomment this line to enable all the optional features.')
                lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req)))
@@ -326,275 +726,283 @@ class PythonRecipeHandler(RecipeHandler):
        handled.append('buildsystem')
-    def get_pkginfo(self, pkginfo_fn):
+class PythonPyprojectTomlRecipeHandler(PythonRecipeHandler):
-        msg = email.message_from_file(open(pkginfo_fn, 'r'))
+    """Base class to support PEP517 and PEP518
-        msginfo = {}
-        for field in msg.keys():
+    PEP517 https://peps.python.org/pep-0517/#source-trees
-            values = msg.get_all(field)
+    PEP518 https://peps.python.org/pep-0518/#build-system-table
-            if len(values) == 1:
+    """
-                msginfo[field] = values[0]
+    # bitbake currently supports the 4 following backends
-            else:
+    build_backend_map = {
-                msginfo[field] = values
+        "setuptools.build_meta": "python_setuptools_build_meta",
-        return msginfo
+        "poetry.core.masonry.api": "python_poetry_core",
+        "flit_core.buildapi": "python_flit_core",
+        "hatchling.build": "python_hatchling",
+        "maturin": "python_maturin",
+        "mesonpy": "python_mesonpy",
+    }
-    def parse_setup_py(self, setupscript='./setup.py'):
+    # setuptools.build_meta and flit declare project metadata into the "project" section of pyproject.toml
-        with codecs.open(setupscript) as f:
+    # according to PEP-621: https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#declaring-project-metadata
-            info, imported_modules, non_literals, extensions = gather_setup_info(f)
+    # while poetry uses the "tool.poetry" section according to its official documentation: https://python-poetry.org/docs/pyproject/
+    # keys from "project" and "tool.poetry" sections are almost the same except for the  HOMEPAGE which is "homepage" for tool.poetry
+    # and "Homepage" for "project" section. So keep both
+    bbvar_map = {
+        "name": "PN",
+        "version": "PV",
+        "Homepage": "HOMEPAGE",
+        "homepage": "HOMEPAGE",
+        "description": "SUMMARY",
+        "license": "LICENSE",
+        "dependencies": "RDEPENDS:${PN}",
+        "requires": "DEPENDS",
+    }
-        def _map(key):
+    replacements = [
-            key = key.replace('_', '-')
+        ("license", r" +$", ""),
-            key = key[0].upper() + key[1:]
+        ("license", r"^ +", ""),
-            if key in self.setup_parse_map:
+        ("license", r" ", "-"),
-                key = self.setup_parse_map[key]
+        ("license", r"^GNU-", ""),
-            return key
+        ("license", r"-[Ll]icen[cs]e(,?-[Vv]ersion)?", ""),
+        ("license", r"^UNKNOWN$", ""),
+        # Remove currently unhandled version numbers from these variables
+        ("requires", r"\[[^\]]+\]$", ""),
+        ("requires", r"^([^><= ]+).*", r"\1"),
+        ("dependencies", r"\[[^\]]+\]$", ""),
+        ("dependencies", r"^([^><= ]+).*", r"\1"),
+    ]
-        # Naive mapping of setup() arguments to PKG-INFO field names
+    excluded_native_pkgdeps = [
-        for d in [info, non_literals]:
+        # already provided by python_setuptools_build_meta.bbclass
-            for key, value in list(d.items()):
+        "python3-setuptools-native",
-                if key is None:
+        "python3-wheel-native",
-                    continue
+        # already provided by python_poetry_core.bbclass
-                new_key = _map(key)
+        "python3-poetry-core-native",
-                if new_key != key:
+        # already provided by python_flit_core.bbclass
-                    del d[key]
+        "python3-flit-core-native",
-                    d[new_key] = value
+        # already provided by python_mesonpy
+        "python3-meson-python-native",
+    ]
-        return info, 'setuptools' in imported_modules, non_literals, extensions
+    # add here a list of known and often used packages and the corresponding bitbake package
+    known_deps_map = {
+        "setuptools": "python3-setuptools",
+        "wheel": "python3-wheel",
+        "poetry-core": "python3-poetry-core",
+        "flit_core": "python3-flit-core",
+        "setuptools-scm": "python3-setuptools-scm",
+        "hatchling": "python3-hatchling",
+        "hatch-vcs": "python3-hatch-vcs",
+        "meson-python" : "python3-meson-python",
+    }
-    def get_setup_args_info(self, setupscript='./setup.py'):
+    def __init__(self):
-        cmd = ['python3', setupscript]
+        pass
-        info = {}
-        keys = set(self.bbvar_map.keys())
-        keys |= set(self.setuparg_list_fields)
-        keys |= set(self.setuparg_multi_line_values)
-        grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
-        for index, keys in grouped_keys:
-            if index == (True, False):
-                # Splitlines output for each arg as a list value
-                for key in keys:
-                    arg = self.setuparg_map.get(key, key.lower())
-                    try:
-                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
-                    except (OSError, subprocess.CalledProcessError):
-                        pass
-                    else:
-                        info[key] = [l.rstrip() for l in arg_info.splitlines()]
-            elif index == (False, True):
-                # Entire output for each arg
-                for key in keys:
-                    arg = self.setuparg_map.get(key, key.lower())
-                    try:
-                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
-                    except (OSError, subprocess.CalledProcessError):
-                        pass
-                    else:
-                        info[key] = arg_info
-            else:
-                info.update(self.get_setup_byline(list(keys), setupscript))
-        return info
-    def get_setup_byline(self, fields, setupscript='./setup.py'):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
        info = {}
+        metadata = {}
-        cmd = ['python3', setupscript]
+        if 'buildsystem' in handled:
-        cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
+            return False
-        try:
-            info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
-        except (OSError, subprocess.CalledProcessError):
-            pass
-        else:
-            if len(fields) != len(info_lines):
-                logger.error('Mismatch between setup.py output lines and number of fields')
-                sys.exit(1)
-            for lineno, line in enumerate(info_lines):
-                line = line.rstrip()
-                info[fields[lineno]] = line
-        return info
-    def apply_info_replacements(self, info):
-        for variable, search, replace in self.replacements:
-            if variable not in info:
-                continue
-            def replace_value(search, replace, value):
-                if replace is None:
-                    if re.search(search, value):
-                        return None
-                else:
-                    new_value = re.sub(search, replace, value)
-                    if value != new_value:
-                        return new_value
-                return value
-            value = info[variable]
-            if isinstance(value, str):
-                new_value = replace_value(search, replace, value)
-                if new_value is None:
-                    del info[variable]
-                elif new_value != value:
-                    info[variable] = new_value
-            elif hasattr(value, 'items'):
-                for dkey, dvalue in list(value.items()):
-                    new_list = []
-                    for pos, a_value in enumerate(dvalue):
-                        new_value = replace_value(search, replace, a_value)
-                        if new_value is not None and new_value != value:
-                            new_list.append(new_value)
-                    if value != new_list:
-                        value[dkey] = new_list
-            else:
-                new_list = []
-                for pos, a_value in enumerate(value):
-                    new_value = replace_value(search, replace, a_value)
-                    if new_value is not None and new_value != value:
-                        new_list.append(new_value)
-                if value != new_list:
-                    info[variable] = new_list
-    def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
-        if 'Package-dir' in setup_info:
-            package_dir = setup_info['Package-dir']
-        else:
-            package_dir = {}
-        class PackageDir(distutils.command.build_py.build_py):
-            def __init__(self, package_dir):
-                self.package_dir = package_dir
-        pd = PackageDir(package_dir)
-        to_scan = []
-        if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
-            if 'Py-modules' in setup_info:
-                for module in setup_info['Py-modules']:
-                    try:
-                        package, module = module.rsplit('.', 1)
-                    except ValueError:
-                        package, module = '.', module
-                    module_path = os.path.join(pd.get_package_dir(package), module + '.py')
-                    to_scan.append(module_path)
-            if 'Packages' in setup_info:
+        logger.debug("Trying pyproject.toml parser")
-                for package in setup_info['Packages']:
-                    to_scan.append(pd.get_package_dir(package))
-            if 'Scripts' in setup_info:
+        # Check for non-zero size setup.py files
-                to_scan.extend(setup_info['Scripts'])
+        setupfiles = RecipeHandler.checkfiles(srctree, ["pyproject.toml"])
+        for fn in setupfiles:
+            if os.path.getsize(fn):
+                break
        else:
-            logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
+            logger.debug("No pyproject.toml found")
+            return False
-        if not to_scan:
-            to_scan = ['.']
-        logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
-        provided_packages = self.parse_pkgdata_for_python_packages()
+        setupscript = os.path.join(srctree, "pyproject.toml")
-        scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
-        mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
-        for dep in scanned_deps:
-            mapped = provided_packages.get(dep)
-            if mapped:
-                logger.debug('Mapped %s to %s' % (dep, mapped))
-                mapped_deps.add(mapped)
-            else:
-                logger.debug('Could not map %s' % dep)
-                unmapped_deps.add(dep)
-        return mapped_deps, unmapped_deps
-    def scan_python_dependencies(self, paths):
-        deps = set()
        try:
-            dep_output = self.run_command(['pythondeps', '-d'] + paths)
+            try:
-        except (OSError, subprocess.CalledProcessError):
+                import tomllib
-            pass
+            except ImportError:
-        else:
+                try:
-            for line in dep_output.splitlines():
+                    import tomli as tomllib
-                line = line.rstrip()
+                except ImportError:
-                dep, filename = line.split('\t', 1)
+                    logger.error("Neither 'tomllib' nor 'tomli' could be imported, cannot scan pyproject.toml.")
-                if filename.endswith('/setup.py'):
+                    return False
-                    continue
-                deps.add(dep)
+            try:
+                with open(setupscript, "rb") as f:
+                    config = tomllib.load(f)
+            except Exception:
+                logger.exception("Failed to parse pyproject.toml")
+                return False
+            build_backend = config["build-system"]["build-backend"]
+            if build_backend in self.build_backend_map:
+                classes.append(self.build_backend_map[build_backend])
+            else:
+                logger.error(
+                    "Unsupported build-backend: %s, cannot use pyproject.toml. Will try to use legacy setup.py"
+                    % build_backend
+                )
+                return False
-        try:
+            licfile = ""
-            provides_output = self.run_command(['pythondeps', '-p'] + paths)
-        except (OSError, subprocess.CalledProcessError):
-            pass
-        else:
-            provides_lines = (l.rstrip() for l in provides_output.splitlines())
-            provides = set(l for l in provides_lines if l and l != 'setup')
-            deps -= provides
-        return deps
+            if build_backend == "poetry.core.masonry.api":
+                if "tool" in config and "poetry" in config["tool"]:
+                    metadata = config["tool"]["poetry"]
+            else:
+                if "project" in config:
+                    metadata = config["project"]
+            if metadata:
+                for field, values in metadata.items():
+                    if field == "license":
+                        # For setuptools.build_meta and flit, licence is a table
+                        # but for poetry licence is a string
+                        # for hatchling, both table (jsonschema) and string (iniconfig) have been used
+                        if build_backend == "poetry.core.masonry.api":
+                            value = values
+                        else:
+                            value = values.get("text", "")
+                            if not value:
+                                licfile = values.get("file", "")
+                                continue
+                    elif field == "dependencies" and build_backend == "poetry.core.masonry.api":
+                        # For poetry backend, "dependencies" section looks like:
+                        # [tool.poetry.dependencies]
+                        # requests = "^2.13.0"
+                        # requests = { version = "^2.13.0", source = "private" }
+                        # See https://python-poetry.org/docs/master/pyproject/#dependencies-and-dependency-groups for more details
+                        # This class doesn't handle versions anyway, so we just get the dependencies name here and construct a list
+                        value = []
+                        for k in values.keys():
+                            value.append(k)
+                    elif isinstance(values, dict):
+                        for k, v in values.items():
+                            info[k] = v
+                        continue
+                    else:
+                        value = values
-    def parse_pkgdata_for_python_packages(self):
+                    info[field] = value
-        suffixes = [t[0] for t in imp.get_suffixes()]
-        pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
-        ldata = tinfoil.config_data.createCopy()
+            # Grab the license value before applying replacements
-        bb.parse.handle('classes/python3-dir.bbclass', ldata, True)
+            license_str = info.get("license", "").strip()
-        python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
-        dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
+            if license_str:
-        python_dirs = [python_sitedir + os.sep,
+                for i, line in enumerate(lines_before):
-                       os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
+                    if line.startswith("##LICENSE_PLACEHOLDER##"):
-                       os.path.dirname(python_sitedir) + os.sep]
+                        lines_before.insert(
-        packages = {}
+                            i, "# NOTE: License in pyproject.toml is: %s" % license_str
-        for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
+                        )
-            files_info = None
-            with open(pkgdatafile, 'r') as f:
-                for line in f.readlines():
-                    field, value = line.split(': ', 1)
-                    if field.startswith('FILES_INFO'):
-                        files_info = ast.literal_eval(value)
                        break
-                else:
-                    continue
-            for fn in files_info:
+            info["requires"] = config["build-system"]["requires"]
-                for suffix in suffixes:
-                    if fn.endswith(suffix):
+            self.apply_info_replacements(info)
-                        break
-                else:
+            if "classifiers" in info:
-                    continue
+                license = self.handle_classifier_license(
+                    info["classifiers"], info.get("license", "")
+                )
+                if license:
+                    if licfile:
+                        lines = []
+                        md5value = bb.utils.md5_file(os.path.join(srctree, licfile))
+                        lines.append('LICENSE = "%s"' % license)
+                        lines.append(
+                            'LIC_FILES_CHKSUM = "file://%s;md5=%s"'
+                            % (licfile, md5value)
+                        )
+                        lines.append("")
+                        # Replace the placeholder so we get the values in the right place in the recipe file
+                        try:
+                            pos = lines_before.index("##LICENSE_PLACEHOLDER##")
+                        except ValueError:
+                            pos = -1
+                        if pos == -1:
+                            lines_before.extend(lines)
+                        else:
+                            lines_before[pos : pos + 1] = lines
-                if fn.startswith(dynload_dir + os.sep):
+                        handled.append(("license", [license, licfile, md5value]))
-                    if '/.debug/' in fn:
+                    else:
-                        continue
+                        info["license"] = license
-                    base = os.path.basename(fn)
-                    provided = base.split('.', 1)[0]
-                    packages[provided] = os.path.basename(pkgdatafile)
-                    continue
-                for python_dir in python_dirs:
+            provided_packages = self.parse_pkgdata_for_python_packages()
-                    if fn.startswith(python_dir):
+            provided_packages.update(self.known_deps_map)
-                        relpath = fn[len(python_dir):]
+            native_mapped_deps, native_unmapped_deps = set(), set()
-                        relstart, _, relremaining = relpath.partition(os.sep)
+            mapped_deps, unmapped_deps = set(), set()
-                        if relstart.endswith('.egg'):
-                            relpath = relremaining
-                        base, _ = os.path.splitext(relpath)
-                        if '/.debug/' in base:
+            if "requires" in info:
-                            continue
+                for require in info["requires"]:
-                        if os.path.basename(base) == '__init__':
+                    mapped = provided_packages.get(require)
-                            base = os.path.dirname(base)
-                        base = base.replace(os.sep + os.sep, os.sep)
-                        provided = base.replace(os.sep, '.')
-                        packages[provided] = os.path.basename(pkgdatafile)
-        return packages
-    @classmethod
+                    if mapped:
-    def run_command(cls, cmd, **popenargs):
+                        logger.debug("Mapped %s to %s" % (require, mapped))
-        if 'stderr' not in popenargs:
+                        native_mapped_deps.add(mapped)
-            popenargs['stderr'] = subprocess.STDOUT
+                    else:
-        try:
+                        logger.debug("Could not map %s" % require)
-            return subprocess.check_output(cmd, **popenargs).decode('utf-8')
+                        native_unmapped_deps.add(require)
-        except OSError as exc:
-            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
+                info.pop("requires")
-            raise
-        except subprocess.CalledProcessError as exc:
+                if native_mapped_deps != set():
-            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
+                    native_mapped_deps = {
-            raise
+                        item + "-native" for item in native_mapped_deps
+                    }
+                    native_mapped_deps -= set(self.excluded_native_pkgdeps)
+                    if native_mapped_deps != set():
+                        info["requires"] = " ".join(sorted(native_mapped_deps))
+                if native_unmapped_deps:
+                    lines_after.append("")
+                    lines_after.append(
+                        "# WARNING: We were unable to map the following python package/module"
+                    )
+                    lines_after.append(
+                        "# dependencies to the bitbake packages which include them:"
+                    )
+                    lines_after.extend(
+                        "#    {}".format(d) for d in sorted(native_unmapped_deps)
+                    )
+            if "dependencies" in info:
+                for dependency in info["dependencies"]:
+                    mapped = provided_packages.get(dependency)
+                    if mapped:
+                        logger.debug("Mapped %s to %s" % (dependency, mapped))
+                        mapped_deps.add(mapped)
+                    else:
+                        logger.debug("Could not map %s" % dependency)
+                        unmapped_deps.add(dependency)
+                info.pop("dependencies")
+                if mapped_deps != set():
+                    if mapped_deps != set():
+                        info["dependencies"] = " ".join(sorted(mapped_deps))
+                if unmapped_deps:
+                    lines_after.append("")
+                    lines_after.append(
+                        "# WARNING: We were unable to map the following python package/module"
+                    )
+                    lines_after.append(
+                        "# runtime dependencies to the bitbake packages which include them:"
+                    )
+                    lines_after.extend(
+                        "#    {}".format(d) for d in sorted(unmapped_deps)
+                    )
+            self.map_info_to_bbvar(info, extravalues)
+            handled.append("buildsystem")
+        except Exception:
+            logger.exception("Failed to correctly handle pyproject.toml, falling back to another method")
+            return False
 def gather_setup_info(fileobj):
@@ -710,5 +1118,7 @@ def has_non_literals(value):
 def register_recipe_handlers(handlers):
-    # We need to make sure this is ahead of the makefile fallback handler
+    # We need to make sure these are ahead of the makefile fallback handler
-    handlers.append((PythonRecipeHandler(), 70))
+    # and the pyproject.toml handler ahead of the setup.py handler
+    handlers.append((PythonPyprojectTomlRecipeHandler(), 75))
+    handlers.append((PythonSetupPyRecipeHandler(), 70))
diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
new file mode 100644
index 0000000000..4b1fa39d13
--- /dev/null
+++ b/scripts/lib/recipetool/create_go.py
@@ -0,0 +1,174 @@
+# Recipe creation tool - go support plugin
+#
+# The code is based on golang internals. See the afftected
+# methods for further reference and information.
+#
+# Copyright (C) 2023 Weidmueller GmbH & Co KG
+# Author: Lukas Funke <lukas.funke@weidmueller.com>
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+from recipetool.create import RecipeHandler, handle_license_vars
+import bb.utils
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+import tempfile
+logger = logging.getLogger('recipetool')
+tinfoil = None
+def tinfoil_init(instance):
+    global tinfoil
+    tinfoil = instance
+class GoRecipeHandler(RecipeHandler):
+    """Class to handle the go recipe creation"""
+    @staticmethod
+    def __ensure_go():
+        """Check if the 'go' command is available in the recipes"""
+        recipe = "go-native"
+        if not tinfoil.recipes_parsed:
+            tinfoil.parse_recipes()
+        try:
+            rd = tinfoil.parse_recipe(recipe)
+        except bb.providers.NoProvider:
+            bb.error(
+                "Nothing provides '%s' which is required for the build" % (recipe))
+            bb.note(
+                "You will likely need to add a layer that provides '%s'" % (recipe))
+            return None
+        bindir = rd.getVar('STAGING_BINDIR_NATIVE')
+        gopath = os.path.join(bindir, 'go')
+        if not os.path.exists(gopath):
+            tinfoil.build_targets(recipe, 'addto_recipe_sysroot')
+            if not os.path.exists(gopath):
+                logger.error(
+                    '%s required to process specified source, but %s did not seem to populate it' % 'go', recipe)
+                return None
+        return bindir
+    def process(self, srctree, classes, lines_before,
+                lines_after, handled, extravalues):
+        if 'buildsystem' in handled:
+            return False
+        files = RecipeHandler.checkfiles(srctree, ['go.mod'])
+        if not files:
+            return False
+        go_bindir = self.__ensure_go()
+        if not go_bindir:
+            sys.exit(14)
+        handled.append('buildsystem')
+        classes.append("go-mod")
+        # Use go-mod-update-modules to set the full SRC_URI and LICENSE
+        classes.append("go-mod-update-modules")
+        extravalues["run_tasks"] = "update_modules"
+        with tempfile.TemporaryDirectory(prefix="go-mod-") as tmp_mod_dir:
+            env = dict(os.environ)
+            env["PATH"] += f":{go_bindir}"
+            env['GOMODCACHE'] = tmp_mod_dir
+            stdout = subprocess.check_output(["go", "mod", "edit", "-json"], cwd=srctree, env=env, text=True)
+            go_mod = json.loads(stdout)
+            go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path'])
+            localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
+            extravalues.setdefault('extrafiles', {})
+            # Write the stub ${BPN}-licenses.inc and ${BPN}-go-mods.inc files
+            basename = "{pn}-licenses.inc"
+            filename = os.path.join(localfilesdir, basename)
+            with open(filename, "w") as f:
+                f.write("# FROM RECIPETOOL\n")
+            extravalues['extrafiles'][f"../{basename}"] = filename
+            basename = "{pn}-go-mods.inc"
+            filename = os.path.join(localfilesdir, basename)
+            with open(filename, "w") as f:
+                f.write("# FROM RECIPETOOL\n")
+            extravalues['extrafiles'][f"../{basename}"] = filename
+            # Do generic license handling
+            d = bb.data.createCopy(tinfoil.config_data)
+            handle_license_vars(srctree, lines_before, handled, extravalues, d)
+            self.__rewrite_lic_vars(lines_before)
+            self.__rewrite_src_uri(lines_before)
+            lines_before.append('require ${BPN}-licenses.inc')
+            lines_before.append('require ${BPN}-go-mods.inc')
+            lines_before.append(f'GO_IMPORT = "{go_import}"')
+    def __update_lines_before(self, updated, newlines, lines_before):
+        if updated:
+            del lines_before[:]
+            for line in newlines:
+                # Hack to avoid newlines that edit_metadata inserts
+                if line.endswith('\n'):
+                    line = line[:-1]
+                lines_before.append(line)
+        return updated
+    def __rewrite_lic_vars(self, lines_before):
+        def varfunc(varname, origvalue, op, newlines):
+            import urllib.parse
+            if varname == 'LIC_FILES_CHKSUM':
+                new_licenses = []
+                licenses = origvalue.split('\\')
+                for license in licenses:
+                    if not license:
+                        logger.warning("No license file was detected for the main module!")
+                        # the license list of the main recipe must be empty
+                        # this can happen for example in case of CLOSED license
+                        # Fall through to complete recipe generation
+                        continue
+                    license = license.strip()
+                    uri, chksum = license.split(';', 1)
+                    url = urllib.parse.urlparse(uri)
+                    new_uri = os.path.join(
+                        url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum
+                    new_licenses.append(new_uri)
+                return new_licenses, None, -1, True
+            return origvalue, None, 0, True
+        updated, newlines = bb.utils.edit_metadata(
+            lines_before, ['LIC_FILES_CHKSUM'], varfunc)
+        return self.__update_lines_before(updated, newlines, lines_before)
+    def __rewrite_src_uri(self, lines_before):
+        def varfunc(varname, origvalue, op, newlines):
+            if varname == 'SRC_URI':
+                src_uri = ['git://${GO_IMPORT};protocol=https;nobranch=1;destsuffix=${GO_SRCURI_DESTSUFFIX}']
+                return src_uri, None, -1, True
+            return origvalue, None, 0, True
+        updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
+        return self.__update_lines_before(updated, newlines, lines_before)
+def register_recipe_handlers(handlers):
+    handlers.append((GoRecipeHandler(), 60))
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
index 3394a89970..8c4cdd5234 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -13,10 +13,11 @@ import sys
 import tempfile
 import bb
 from bb.fetch2.npm import NpmEnvironment
+from bb.fetch2.npm import npm_package
 from bb.fetch2.npmsw import foreach_dependencies
+from oe.license_finder import match_licenses, find_license_files
 from recipetool.create import RecipeHandler
-from recipetool.create import get_license_md5sums
+from recipetool.create import generate_common_licenses_chksums
-from recipetool.create import guess_license
 from recipetool.create import split_pkg_licenses
 logger = logging.getLogger('recipetool')
@@ -31,15 +32,6 @@ class NpmRecipeHandler(RecipeHandler):
    """Class to handle the npm recipe creation"""
    @staticmethod
-    def _npm_name(name):
-        """Generate a Yocto friendly npm name"""
-        name = re.sub("/", "-", name)
-        name = name.lower()
-        name = re.sub(r"[^\-a-z0-9]", "", name)
-        name = name.strip("-")
-        return name
-    @staticmethod
    def _get_registry(lines):
        """Get the registry value from the 'npm://registry' url"""
        registry = None
@@ -120,41 +112,71 @@ class NpmRecipeHandler(RecipeHandler):
        """Return the extra license files and the list of packages"""
        licfiles = []
        packages = {}
+        # Licenses from package.json will point to COMMON_LICENSE_DIR so we need
+        # to associate them explicitely to packages for split_pkg_licenses()
+        fallback_licenses = dict()
+        def _find_package_licenses(destdir):
+            """Either find license files, or use package.json metadata"""
+            def _get_licenses_from_package_json(package_json):
+                with open(os.path.join(srctree, package_json), "r") as f:
+                    data = json.load(f)
+                    if "license" in data:
+                        licenses = data["license"].split(" ")
+                        licenses = [license.strip("()") for license in licenses if license != "OR" and license != "AND"]
+                        return [], licenses
+                    else:
+                        return [package_json], None
-        # Handle the parent package
-        packages["${PN}"] = ""
-        def _licfiles_append_fallback_readme_files(destdir):
-            """Append README files as fallback to license files if a license files is missing"""
-            fallback = True
-            readmes = []
            basedir = os.path.join(srctree, destdir)
-            for fn in os.listdir(basedir):
+            licfiles = find_license_files(basedir)
-                upper = fn.upper()
+            if len(licfiles) > 0:
-                if upper.startswith("README"):
+                return licfiles, None
-                    fullpath = os.path.join(basedir, fn)
+            else:
-                    readmes.append(fullpath)
+                # A license wasn't found in the package directory, so we'll use the package.json metadata
-                if upper.startswith("COPYING") or "LICENCE" in upper or "LICENSE" in upper:
+                pkg_json = os.path.join(basedir, "package.json")
-                    fallback = False
+                return _get_licenses_from_package_json(pkg_json)
-            if fallback:
-                for readme in readmes:
+        def _get_package_licenses(destdir, package):
-                    licfiles.append(os.path.relpath(readme, srctree))
+            (package_licfiles, package_licenses) = _find_package_licenses(destdir)
+            if package_licfiles:
+                licfiles.extend(package_licfiles)
+            else:
+                fallback_licenses[package] = package_licenses
        # Handle the dependencies
-        def _handle_dependency(name, params, deptree):
+        def _handle_dependency(name, params, destdir):
-            suffix = "-".join([self._npm_name(dep) for dep in deptree])
+            deptree = destdir.split('node_modules/')
-            destdirs = [os.path.join("node_modules", dep) for dep in deptree]
+            suffix = "-".join([npm_package(dep) for dep in deptree])
-            destdir = os.path.join(*destdirs)
+            packages["${PN}" + suffix] = destdir
-            packages["${PN}-" + suffix] = destdir
+            _get_package_licenses(destdir, "${PN}" + suffix)
-            _licfiles_append_fallback_readme_files(destdir)
        with open(shrinkwrap_file, "r") as f:
            shrinkwrap = json.load(f)
        foreach_dependencies(shrinkwrap, _handle_dependency, dev)
-        return licfiles, packages
+        # Handle the parent package
+        packages["${PN}"] = ""
+        _get_package_licenses(srctree, "${PN}")
+        return licfiles, packages, fallback_licenses
+    
+    # Handle the peer dependencies   
+    def _handle_peer_dependency(self, shrinkwrap_file):
+        """Check if package has peer dependencies and show warning if it is the case"""
+        with open(shrinkwrap_file, "r") as f:
+            shrinkwrap = json.load(f)
+        
+        packages = shrinkwrap.get("packages", {})
+        peer_deps = packages.get("", {}).get("peerDependencies", {})
+        
+        for peer_dep in peer_deps:
+            peer_dep_yocto_name = npm_package(peer_dep)
+            bb.warn(peer_dep + " is a peer dependencie of the actual package. " + 
+            "Please add this peer dependencie to the RDEPENDS variable as %s and generate its recipe with devtool"
+            % peer_dep_yocto_name)
    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
        """Handle the npm recipe creation"""
@@ -173,7 +195,7 @@ class NpmRecipeHandler(RecipeHandler):
        if "name" not in data or "version" not in data:
            return False
-        extravalues["PN"] = self._npm_name(data["name"])
+        extravalues["PN"] = npm_package(data["name"])
        extravalues["PV"] = data["version"]
        if "description" in data:
@@ -242,7 +264,7 @@ class NpmRecipeHandler(RecipeHandler):
            value = origvalue.replace("version=" + data["version"], "version=${PV}")
            value = value.replace("version=latest", "version=${PV}")
            values = [line.strip() for line in value.strip('\n').splitlines()]
-            if "dependencies" in shrinkwrap:
+            if "dependencies" in shrinkwrap.get("packages", {}).get("", {}):
                values.append(url_recipe)
            return values, None, 4, False
@@ -258,40 +280,19 @@ class NpmRecipeHandler(RecipeHandler):
        fetcher.unpack(srctree)
        bb.note("Handling licences ...")
-        (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev)
+        (licfiles, packages, fallback_licenses) = self._handle_licenses(srctree, shrinkwrap_file, dev)
+        licvalues = match_licenses(licfiles, srctree, d)
-        def _guess_odd_license(licfiles):
+        split_pkg_licenses(licvalues, packages, lines_after, fallback_licenses)
-            import bb
+        fallback_licenses_flat = [license for sublist in fallback_licenses.values() for license in sublist]
+        extravalues["LIC_FILES_CHKSUM"] = generate_common_licenses_chksums(fallback_licenses_flat, d)
-            md5sums = get_license_md5sums(d, linenumbers=True)
+        extravalues["LICENSE"] = fallback_licenses_flat
-            chksums = []
-            licenses = []
-            for licfile in licfiles:
-                f = os.path.join(srctree, licfile)
-                md5value = bb.utils.md5_file(f)
-                (license, beginline, endline, md5) = md5sums.get(md5value,
-                    (None, "", "", ""))
-                if not license:
-                    license = "Unknown"
-                    logger.info("Please add the following line for '%s' to a "
-                        "'lib/recipetool/licenses.csv' and replace `Unknown`, "
-                        "`X`, `Y` and `MD5` with the license, begin line, "
-                        "end line and partial MD5 checksum:\n" \
-                        "%s,Unknown,X,Y,MD5" % (licfile, md5value))
-                chksums.append("file://%s%s%s;md5=%s" % (licfile,
-                    ";beginline=%s" % (beginline) if beginline else "",
-                    ";endline=%s" % (endline) if endline else "",
-                    md5 if md5 else md5value))
-                licenses.append((license, licfile, md5value))
-            return (licenses, chksums)
-        (licenses, extravalues["LIC_FILES_CHKSUM"]) = _guess_odd_license(licfiles)
-        split_pkg_licenses([*licenses, *guess_license(srctree, d)], packages, lines_after)
        classes.append("npm")
        handled.append("buildsystem")
+        # Check if package has peer dependencies and inform the user
+        self._handle_peer_dependency(shrinkwrap_file)
        return True
 def register_recipe_handlers(handlers):
diff --git a/scripts/lib/recipetool/licenses.csv b/scripts/lib/recipetool/licenses.csv
deleted file mode 100644
index 0d3fb0607b..0000000000
--- a/scripts/lib/recipetool/licenses.csv
+++ /dev/null
@@ -1,37 +0,0 @@
-0636e73ff0215e8d672dc4c32c317bb3,GPLv2
-12f884d2ae1ff87c09e5b7ccc2c4ca7e,GPLv2
-18810669f13b87348459e611d31ab760,GPLv2
-252890d9eee26aab7b432e8b8a616475,LGPLv2
-2d5025d4aa3495befef8f17206a5b0a1,LGPLv2.1
-3214f080875748938ba060314b4f727d,LGPLv2
-385c55653886acac3821999a3ccd17b3,Artistic-1.0 | GPL-2.0
-393a5ca445f6965873eca0259a17f833,GPLv2
-3b83ef96387f14655fc854ddc3c6bd57,Apache-2.0
-3bf50002aefd002f49e7bb854063f7e7,LGPLv2
-4325afd396febcb659c36b49533135d4,GPLv2
-4fbd65380cdd255951079008b364516c,LGPLv2.1
-54c7042be62e169199200bc6477f04d1,BSD-3-Clause
-55ca817ccb7d5b5b66355690e9abc605,LGPLv2
-59530bdf33659b29e73d4adb9f9f6552,GPLv2
-5f30f0716dfdd0d91eb439ebec522ec2,LGPLv2
-6a6a8e020838b23406c81b19c1d46df6,LGPLv3
-751419260aa954499f7abaabaa882bbe,GPLv2
-7fbc338309ac38fefcd64b04bb903e34,LGPLv2.1
-8ca43cbc842c2336e835926c2166c28b,GPLv2
-94d55d512a9ba36caa9b7df079bae19f,GPLv2
-9ac2e7cff1ddaf48b6eab6028f23ef88,GPLv2
-9f604d8a4f8e74f4f5140845a21b6674,LGPLv2
-a6f89e2100d9b6cdffcea4f398e37343,LGPLv2.1
-b234ee4d69f5fce4486a80fdaf4a4263,GPLv2
-bbb461211a33b134d42ed5ee802b37ff,LGPLv2.1
-bfe1f75d606912a4111c90743d6c7325,MPL-1.1
-c93c0550bd3173f4504b2cbd8991e50b,GPLv2
-d32239bcb673463ab874e80d47fae504,GPLv3
-d7810fab7487fb0aad327b76f1be7cd7,GPLv2
-d8045f3b8f929c1cb29a1e3fd737b499,LGPLv2.1
-db979804f025cf55aabec7129cb671ed,LGPLv2
-eb723b61539feef013de476e68b5c50a,GPLv2
-ebb5c50ab7cab4baeffba14977030c07,GPLv2
-f27defe1e96c2e1ecd4e0c9be8967949,GPLv3
-fad9b3332be894bab9bc501572864b29,LGPLv2.1
-fbc093901857fcd118f065f900982c24,LGPLv2.1
diff --git a/scripts/lib/recipetool/setvar.py b/scripts/lib/recipetool/setvar.py
index f8e2ee75fb..b5ad335cae 100644
--- a/scripts/lib/recipetool/setvar.py
+++ b/scripts/lib/recipetool/setvar.py
@@ -49,6 +49,7 @@ def setvar(args):
        for patch in patches:
            for line in patch:
                sys.stdout.write(line)
+    tinfoil.modified_files()
    return 0