8 files changed, 1189 insertions, 637 deletions

diff --git a/scripts/lib/recipetool/append.py b/scripts/lib/recipetool/append.py
index e9d52bb67b..041d79f162 100644
--- a/scripts/lib/recipetool/append.py
+++ b/scripts/lib/recipetool/append.py
@@ -18,6 +18,7 @@ import shutil
 import scriptutils
 import errno
 from collections import defaultdict
+import difflib
 
 logger = logging.getLogger('recipetool')
 
@@ -49,7 +50,7 @@ def find_target_file(targetpath, d, pkglist=None):
                       '/etc/group': '/etc/group should be managed through the useradd and extrausers classes',
                       '/etc/shadow': '/etc/shadow should be managed through the useradd and extrausers classes',
                       '/etc/gshadow': '/etc/gshadow should be managed through the useradd and extrausers classes',
-                      '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname_pn-base-files = "value" in configuration',}
+                      '${sysconfdir}/hostname': '${sysconfdir}/hostname contents should be set by setting hostname:pn-base-files = "value" in configuration',}
 
     for pthspec, message in invalidtargets.items():
         if fnmatch.fnmatchcase(targetpath, d.expand(pthspec)):
@@ -72,15 +73,15 @@ def find_target_file(targetpath, d, pkglist=None):
                 # This does assume that PN comes before other values, but that's a fairly safe assumption
                 for line in f:
                     if line.startswith('PN:'):
-                        pn = line.split(':', 1)[1].strip()
-                    elif line.startswith('FILES_INFO:'):
-                        val = line.split(':', 1)[1].strip()
+                        pn = line.split(': ', 1)[1].strip()
+                    elif line.startswith('FILES_INFO'):
+                        val = line.split(': ', 1)[1].strip()
                         dictval = json.loads(val)
                         for fullpth in dictval.keys():
                             if fnmatch.fnmatchcase(fullpth, targetpath):
                                 recipes[targetpath].append(pn)
-                    elif line.startswith('pkg_preinst_') or line.startswith('pkg_postinst_'):
-                        scriptval = line.split(':', 1)[1].strip().encode('utf-8').decode('unicode_escape')
+                    elif line.startswith('pkg_preinst:') or line.startswith('pkg_postinst:'):
+                        scriptval = line.split(': ', 1)[1].strip().encode('utf-8').decode('unicode_escape')
                         if 'update-alternatives --install %s ' % targetpath in scriptval:
                             recipes[targetpath].append('?%s' % pn)
                         elif targetpath_re.search(scriptval):
@@ -100,7 +101,7 @@ def determine_file_source(targetpath, rd):
     import oe.recipeutils
 
     # See if it's in do_install for the recipe
-    workdir = rd.getVar('WORKDIR')
+    unpackdir = rd.getVar('UNPACKDIR')
     src_uri = rd.getVar('SRC_URI')
     srcfile = ''
     modpatches = []
@@ -112,9 +113,9 @@ def determine_file_source(targetpath, rd):
         if not srcpath.startswith('/'):
             # Handle non-absolute path
             srcpath = os.path.abspath(os.path.join(rd.getVarFlag('do_install', 'dirs').split()[-1], srcpath))
-        if srcpath.startswith(workdir):
+        if srcpath.startswith(unpackdir):
             # OK, now we have the source file name, look for it in SRC_URI
-            workdirfile = os.path.relpath(srcpath, workdir)
+            workdirfile = os.path.relpath(srcpath, unpackdir)
             # FIXME this is where we ought to have some code in the fetcher, because this is naive
             for item in src_uri.split():
                 localpath = bb.fetch2.localpath(item, rd)
@@ -299,7 +300,10 @@ def appendfile(args):
                 if st.st_mode & stat.S_IXUSR:
                     perms = '0755'
             install = {args.newfile: (args.targetpath, perms)}
-        oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: sourcepath}, install, wildcardver=args.wildcard_version, machine=args.machine)
+        if sourcepath:
+            sourcepath = os.path.basename(sourcepath)
+        oe.recipeutils.bbappend_recipe(rd, args.destlayer, {args.newfile: {'newname' : sourcepath}}, install, wildcardver=args.wildcard_version, machine=args.machine)
+        tinfoil.modified_files()
         return 0
     else:
         if alternative_pns:
@@ -313,7 +317,7 @@ def appendsrc(args, files, rd, extralines=None):
     import oe.recipeutils
 
     srcdir = rd.getVar('S')
-    workdir = rd.getVar('WORKDIR')
+    unpackdir = rd.getVar('UNPACKDIR')
 
     import bb.fetch
     simplified = {}
@@ -327,35 +331,57 @@ def appendsrc(args, files, rd, extralines=None):
 
     copyfiles = {}
     extralines = extralines or []
+    params = []
     for newfile, srcfile in files.items():
         src_destdir = os.path.dirname(srcfile)
         if not args.use_workdir:
             if rd.getVar('S') == rd.getVar('STAGING_KERNEL_DIR'):
-                srcdir = os.path.join(workdir, 'git')
+                srcdir = os.path.join(unpackdir, rd.getVar('BB_GIT_DEFAULT_DESTSUFFIX'))
                 if not bb.data.inherits_class('kernel-yocto', rd):
-                    logger.warning('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${WORKDIR}/git')
-            src_destdir = os.path.join(os.path.relpath(srcdir, workdir), src_destdir)
+                    logger.warning('S == STAGING_KERNEL_DIR and non-kernel-yocto, unable to determine path to srcdir, defaulting to ${UNPACKDIR}/${BB_GIT_DEFAULT_DESTSUFFIX}')
+            src_destdir = os.path.join(os.path.relpath(srcdir, unpackdir), src_destdir)
         src_destdir = os.path.normpath(src_destdir)
 
-        source_uri = 'file://{0}'.format(os.path.basename(srcfile))
         if src_destdir and src_destdir != '.':
-            source_uri += ';subdir={0}'.format(src_destdir)
-
-        simple = bb.fetch.URI(source_uri)
-        simple.params = {}
-        simple_str = str(simple)
-        if simple_str in simplified:
-            existing = simplified[simple_str]
-            if source_uri != existing:
-                logger.warning('{0!r} is already in SRC_URI, with different parameters: {1!r}, not adding'.format(source_uri, existing))
-            else:
-                logger.warning('{0!r} is already in SRC_URI, not adding'.format(source_uri))
+            params.append({'subdir': src_destdir})
         else:
-            extralines.append('SRC_URI += {0}'.format(source_uri))
-        copyfiles[newfile] = srcfile
-
-    oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines)
-
+            params.append({})
+
+        copyfiles[newfile] = {'newname' : os.path.basename(srcfile)}
+
+    dry_run_output = None
+    dry_run_outdir = None
+    if args.dry_run:
+        import tempfile
+        dry_run_output = tempfile.TemporaryDirectory(prefix='devtool')
+        dry_run_outdir = dry_run_output.name
+
+    appendfile, _ = oe.recipeutils.bbappend_recipe(rd, args.destlayer, copyfiles, None, wildcardver=args.wildcard_version, machine=args.machine, extralines=extralines, params=params,
+                                                   redirect_output=dry_run_outdir, update_original_recipe=args.update_recipe)
+    if not appendfile:
+        return
+    if args.dry_run:
+        output = ''
+        appendfilename = os.path.basename(appendfile)
+        newappendfile = appendfile
+        if appendfile and os.path.exists(appendfile):
+            with open(appendfile, 'r') as f:
+                oldlines = f.readlines()
+        else:
+            appendfile = '/dev/null'
+            oldlines = []
+
+        with open(os.path.join(dry_run_outdir, appendfilename), 'r') as f:
+            newlines = f.readlines()
+        diff = difflib.unified_diff(oldlines, newlines, appendfile, newappendfile)
+        difflines = list(diff)
+        if difflines:
+            output += ''.join(difflines)
+        if output:
+            logger.info('Diff of changed files:\n%s' % output)
+        else:
+            logger.info('No changed files')
+    tinfoil.modified_files()
 
 def appendsrcfiles(parser, args):
     recipedata = _parse_recipe(args.recipe, tinfoil)
@@ -435,6 +461,8 @@ def register_commands(subparsers):
                                    help='Create/update a bbappend to add or replace source files',
                                    description='Creates a bbappend (or updates an existing one) to add or replace the specified file in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify multiple files with a destination directory, so cannot specify the destination filename. See the `appendsrcfile` command for the other behavior.')
     parser.add_argument('-D', '--destdir', help='Destination directory (relative to S or WORKDIR, defaults to ".")', default='', type=destination_path)
+    parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true')
+    parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true')
     parser.add_argument('files', nargs='+', metavar='FILE', help='File(s) to be added to the recipe sources (WORKDIR or S)', type=existing_path)
     parser.set_defaults(func=lambda a: appendsrcfiles(parser, a), parserecipes=True)
 
@@ -442,6 +470,8 @@ def register_commands(subparsers):
                                    parents=[common_src],
                                    help='Create/update a bbappend to add or replace a source file',
                                    description='Creates a bbappend (or updates an existing one) to add or replace the specified files in the recipe sources, either those in WORKDIR or those in the source tree. This command lets you specify the destination filename, not just destination directory, but only works for one file. See the `appendsrcfiles` command for the other behavior.')
+    parser.add_argument('-u', '--update-recipe', help='Update recipe instead of creating (or updating) a bbapend file. DESTLAYER must contains the recipe to update', action='store_true')
+    parser.add_argument('-n', '--dry-run', help='Dry run mode', action='store_true')
     parser.add_argument('file', metavar='FILE', help='File to be added to the recipe sources (WORKDIR or S)', type=existing_path)
     parser.add_argument('destfile', metavar='DESTFILE', nargs='?', help='Destination path (relative to S or WORKDIR, optional)', type=destination_path)
     parser.set_defaults(func=lambda a: appendsrcfile(parser, a), parserecipes=True)
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 566c75369a..ef0ba974a9 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -18,6 +18,8 @@ from urllib.parse import urlparse, urldefrag, urlsplit
 import hashlib
 import bb.fetch2
 logger = logging.getLogger('recipetool')
+from oe.license import tidy_licenses
+from oe.license_finder import find_licenses
 
 tinfoil = None
 plugins = None
@@ -115,8 +117,8 @@ class RecipeHandler(object):
                 for line in f:
                     if line.startswith('PN:'):
                         pn = line.split(':', 1)[-1].strip()
-                    elif line.startswith('FILES_INFO:'):
-                        val = line.split(':', 1)[1].strip()
+                    elif line.startswith('FILES_INFO:%s:' % pkg):
+                        val = line.split(': ', 1)[1].strip()
                         dictval = json.loads(val)
                         for fullpth in sorted(dictval):
                             if fullpth.startswith(includedir) and fullpth.endswith('.h'):
@@ -366,7 +368,7 @@ def supports_srcrev(uri):
 def reformat_git_uri(uri):
     '''Convert any http[s]://....git URI into git://...;protocol=http[s]'''
     checkuri = uri.split(';', 1)[0]
-    if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://github.com/[^/]+/[^/]+/?$', checkuri):
+    if checkuri.endswith('.git') or '/git/' in checkuri or re.match('https?://git(hub|lab).com/[^/]+/[^/]+/?$', checkuri):
         # Appends scheme if the scheme is missing
         if not '://' in uri:
             uri = 'git://' + uri
@@ -423,6 +425,36 @@ def create_recipe(args):
     storeTagName = ''
     pv_srcpv = False
 
+    handled = []
+    classes = []
+
+    # Find all plugins that want to register handlers
+    logger.debug('Loading recipe handlers')
+    raw_handlers = []
+    for plugin in plugins:
+        if hasattr(plugin, 'register_recipe_handlers'):
+            plugin.register_recipe_handlers(raw_handlers)
+    # Sort handlers by priority
+    handlers = []
+    for i, handler in enumerate(raw_handlers):
+        if isinstance(handler, tuple):
+            handlers.append((handler[0], handler[1], i))
+        else:
+            handlers.append((handler, 0, i))
+    handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
+    for handler, priority, _ in handlers:
+        logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
+        setattr(handler, '_devtool', args.devtool)
+    handlers = [item[0] for item in handlers]
+
+    fetchuri = None
+    for handler in handlers:
+        if hasattr(handler, 'process_url'):
+            ret = handler.process_url(args, classes, handled, extravalues)
+            if 'url' in handled and ret:
+                fetchuri = ret
+                break
+
     if os.path.isfile(source):
         source = 'file://%s' % os.path.abspath(source)
 
@@ -431,11 +463,12 @@ def create_recipe(args):
         if re.match(r'https?://github.com/[^/]+/[^/]+/archive/.+(\.tar\..*|\.zip)$', source):
             logger.warning('github archive files are not guaranteed to be stable and may be re-generated over time. If the latter occurs, the checksums will likely change and the recipe will fail at do_fetch. It is recommended that you point to an actual commit or tag in the repository instead (using the repository URL in conjunction with the -S/--srcrev option).')
         # Fetch a URL
-        fetchuri = reformat_git_uri(urldefrag(source)[0])
+        if not fetchuri:
+            fetchuri = reformat_git_uri(urldefrag(source)[0])
         if args.binary:
             # Assume the archive contains the directory structure verbatim
             # so we need to extract to a subdirectory
-            fetchuri += ';subdir=${BP}'
+            fetchuri += ';subdir=${BPN}'
         srcuri = fetchuri
         rev_re = re.compile(';rev=([^;]+)')
         res = rev_re.search(srcuri)
@@ -478,6 +511,9 @@ def create_recipe(args):
             storeTagName = params['tag']
             params['nobranch'] = '1'
             del params['tag']
+        # Assume 'master' branch if not set
+        if scheme in ['git', 'gitsm'] and 'branch' not in params and 'nobranch' not in params:
+            params['branch'] = 'master'
         fetchuri = bb.fetch2.encodeurl((scheme, network, path, user, passwd, params))
 
         tmpparent = tinfoil.config_data.getVar('BASE_WORKDIR')
@@ -494,7 +530,7 @@ def create_recipe(args):
         if ftmpdir and args.keep_temp:
             logger.info('Fetch temp directory is %s' % ftmpdir)
 
-        dirlist = scriptutils.filter_src_subdirs(srctree)
+        dirlist = os.listdir(srctree)
         logger.debug('Directory listing (excluding filtered out):\n  %s' % '\n  '.join(dirlist))
         if len(dirlist) == 1:
             singleitem = os.path.join(srctree, dirlist[0])
@@ -527,10 +563,9 @@ def create_recipe(args):
             # Remove HEAD reference point and drop remote prefix
             get_branch = [x.split('/', 1)[1] for x in get_branch if not x.startswith('origin/HEAD')]
             if 'master' in get_branch:
-                # If it is master, we do not need to append 'branch=master' as this is default.
                 # Even with the case where get_branch has multiple objects, if 'master' is one
                 # of them, we should default take from 'master'
-                srcbranch = ''
+                srcbranch = 'master'
             elif len(get_branch) == 1:
                 # If 'master' isn't in get_branch and get_branch contains only ONE object, then store result into 'srcbranch'
                 srcbranch = get_branch[0]
@@ -543,8 +578,8 @@ def create_recipe(args):
         # Since we might have a value in srcbranch, we need to
         # recontruct the srcuri to include 'branch' in params.
         scheme, network, path, user, passwd, params = bb.fetch2.decodeurl(srcuri)
-        if srcbranch:
-            params['branch'] = srcbranch
+        if scheme in ['git', 'gitsm']:
+            params['branch'] = srcbranch or 'master'
 
         if storeTagName and scheme in ['git', 'gitsm']:
             # Check srcrev using tag and check validity of the tag
@@ -603,8 +638,7 @@ def create_recipe(args):
                     splitline = line.split()
                     if len(splitline) > 1:
                         if splitline[0] == 'origin' and scriptutils.is_src_url(splitline[1]):
-                            srcuri = reformat_git_uri(splitline[1])
-                            srcsubdir = 'git'
+                            srcuri = reformat_git_uri(splitline[1]) + ';branch=master'
                             break
 
     if args.src_subdir:
@@ -636,8 +670,6 @@ def create_recipe(args):
     # We'll come back and replace this later in handle_license_vars()
     lines_before.append('##LICENSE_PLACEHOLDER##')
 
-    handled = []
-    classes = []
 
     # FIXME This is kind of a hack, we probably ought to be using bitbake to do this
     pn = None
@@ -675,8 +707,10 @@ def create_recipe(args):
     if not srcuri:
         lines_before.append('# No information for SRC_URI yet (only an external source tree was specified)')
     lines_before.append('SRC_URI = "%s"' % srcuri)
+    shown_checksums = ["%ssum" % s for s in bb.fetch2.SHOWN_CHECKSUM_LIST]
     for key, value in sorted(checksums.items()):
-        lines_before.append('SRC_URI[%s] = "%s"' % (key, value))
+        if key in shown_checksums:
+            lines_before.append('SRC_URI[%s] = "%s"' % (key, value))
     if srcuri and supports_srcrev(srcuri):
         lines_before.append('')
         lines_before.append('# Modify these as desired')
@@ -688,7 +722,7 @@ def create_recipe(args):
             srcpvprefix = 'svnr'
         else:
             srcpvprefix = scheme
-        lines_before.append('PV = "%s+%s${SRCPV}"' % (realpv or '1.0', srcpvprefix))
+        lines_before.append('PV = "%s+%s"' % (realpv or '1.0', srcpvprefix))
         pv_srcpv = True
         if not args.autorev and srcrev == '${AUTOREV}':
             if os.path.exists(os.path.join(srctree, '.git')):
@@ -702,7 +736,7 @@ def create_recipe(args):
     if srcsubdir and not args.binary:
         # (for binary packages we explicitly specify subdir= when fetching to
         # match the default value of S, so we don't need to set it in that case)
-        lines_before.append('S = "${WORKDIR}/%s"' % srcsubdir)
+        lines_before.append('S = "${UNPACKDIR}/%s"' % srcsubdir)
         lines_before.append('')
 
     if pkgarch:
@@ -710,31 +744,12 @@ def create_recipe(args):
         lines_after.append('')
 
     if args.binary:
-        lines_after.append('INSANE_SKIP_${PN} += "already-stripped"')
+        lines_after.append('INSANE_SKIP:${PN} += "already-stripped"')
         lines_after.append('')
 
     if args.npm_dev:
         extravalues['NPM_INSTALL_DEV'] = 1
 
-    # Find all plugins that want to register handlers
-    logger.debug('Loading recipe handlers')
-    raw_handlers = []
-    for plugin in plugins:
-        if hasattr(plugin, 'register_recipe_handlers'):
-            plugin.register_recipe_handlers(raw_handlers)
-    # Sort handlers by priority
-    handlers = []
-    for i, handler in enumerate(raw_handlers):
-        if isinstance(handler, tuple):
-            handlers.append((handler[0], handler[1], i))
-        else:
-            handlers.append((handler, 0, i))
-    handlers.sort(key=lambda item: (item[1], -item[2]), reverse=True)
-    for handler, priority, _ in handlers:
-        logger.debug('Handler: %s (priority %d)' % (handler.__class__.__name__, priority))
-        setattr(handler, '_devtool', args.devtool)
-    handlers = [item[0] for item in handlers]
-
     # Apply the handlers
     if args.binary:
         classes.append('bin_package')
@@ -743,9 +758,14 @@ def create_recipe(args):
     for handler in handlers:
         handler.process(srctree_use, classes, lines_before, lines_after, handled, extravalues)
 
+    # native and nativesdk classes are special and must be inherited last
+    # If present, put them at the end of the classes list
+    classes.sort(key=lambda c: c in ("native", "nativesdk"))
+
     extrafiles = extravalues.pop('extrafiles', {})
     extra_pn = extravalues.pop('PN', None)
     extra_pv = extravalues.pop('PV', None)
+    run_tasks = extravalues.pop('run_tasks', "").split()
 
     if extra_pv and not realpv:
         realpv = extra_pv
@@ -806,7 +826,8 @@ def create_recipe(args):
             extraoutdir = os.path.join(os.path.dirname(outfile), pn)
         bb.utils.mkdirhier(extraoutdir)
         for destfn, extrafile in extrafiles.items():
-            shutil.move(extrafile, os.path.join(extraoutdir, destfn))
+            fn = destfn.format(pn=pn, pv=realpv)
+            shutil.move(extrafile, os.path.join(extraoutdir, fn))
 
     lines = lines_before
     lines_before = []
@@ -821,7 +842,7 @@ def create_recipe(args):
                 line = line.replace(realpv, '${PV}')
             if pn:
                 line = line.replace(pn, '${BPN}')
-            if line == 'S = "${WORKDIR}/${BPN}-${PV}"':
+            if line == 'S = "${UNPACKDIR}/${BPN}-${PV}"' or 'tmp-recipetool-' in line:
                 skipblank = True
                 continue
         elif line.startswith('SRC_URI = '):
@@ -867,8 +888,10 @@ def create_recipe(args):
         outlines.append('')
     outlines.extend(lines_after)
 
+    outlines = [ line.rstrip('\n') +"\n" for line in outlines]
+
     if extravalues:
-        _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=False)
+        _, outlines = oe.recipeutils.patch_recipe_lines(outlines, extravalues, trailing_newline=True)
 
     if args.extract_to:
         scriptutils.git_convert_standalone_clone(srctree)
@@ -884,7 +907,7 @@ def create_recipe(args):
         log_info_cond('Source extracted to %s' % args.extract_to, args.devtool)
 
     if outfile == '-':
-        sys.stdout.write('\n'.join(outlines) + '\n')
+        sys.stdout.write(''.join(outlines) + '\n')
     else:
         with open(outfile, 'w') as f:
             lastline = None
@@ -892,9 +915,14 @@ def create_recipe(args):
                 if not lastline and not line:
                     # Skip extra blank lines
                     continue
-                f.write('%s\n' % line)
+                f.write('%s' % line)
                 lastline = line
         log_info_cond('Recipe %s has been created; further editing may be required to make it fully functional' % outfile, args.devtool)
+        tinfoil.modified_files()
+
+    for task in run_tasks:
+        logger.info("Running task %s" % task)
+        tinfoil.build_file_sync(outfile, task)
 
     if tempsrc:
         if args.keep_temp:
@@ -917,23 +945,32 @@ def split_value(value):
     else:
         return value
 
+def fixup_license(value):
+    # Ensure licenses with OR starts and ends with brackets
+    if '|' in value:
+        return '(' + value + ')'
+    return value
+
 def handle_license_vars(srctree, lines_before, handled, extravalues, d):
     lichandled = [x for x in handled if x[0] == 'license']
     if lichandled:
         # Someone else has already handled the license vars, just return their value
         return lichandled[0][1]
 
-    licvalues = guess_license(srctree, d)
+    licvalues = find_licenses(srctree, d)
     licenses = []
     lic_files_chksum = []
     lic_unknown = []
     lines = []
     if licvalues:
         for licvalue in licvalues:
-            if not licvalue[0] in licenses:
-                licenses.append(licvalue[0])
+            license = licvalue[0]
+            lics = tidy_licenses(fixup_license(license))
+            lics = [lic for lic in lics if lic not in licenses]
+            if len(lics):
+                licenses.extend(lics)
             lic_files_chksum.append('file://%s;md5=%s' % (licvalue[1], licvalue[2]))
-            if licvalue[0] == 'Unknown':
+            if license == 'Unknown':
                 lic_unknown.append(licvalue[1])
         if lic_unknown:
             lines.append('#')
@@ -942,9 +979,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
             for licfile in lic_unknown:
                 lines.append('#   %s' % licfile)
 
-    extra_license = split_value(extravalues.pop('LICENSE', []))
-    if '&' in extra_license:
-        extra_license.remove('&')
+    extra_license = tidy_licenses(extravalues.pop('LICENSE', ''))
     if extra_license:
         if licenses == ['Unknown']:
             licenses = extra_license
@@ -985,7 +1020,7 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
         lines.append('# instead of &. If there is any doubt, check the accompanying documentation')
         lines.append('# to determine which situation is applicable.')
 
-    lines.append('LICENSE = "%s"' % ' & '.join(licenses))
+    lines.append('LICENSE = "%s"' % ' & '.join(sorted(licenses, key=str.casefold)))
     lines.append('LIC_FILES_CHKSUM = "%s"' % ' \\\n                    '.join(lic_files_chksum))
     lines.append('')
 
@@ -1002,166 +1037,15 @@ def handle_license_vars(srctree, lines_before, handled, extravalues, d):
     handled.append(('license', licvalues))
     return licvalues
 
-def get_license_md5sums(d, static_only=False):
-    import bb.utils
-    md5sums = {}
-    if not static_only:
-        # Gather md5sums of license files in common license dir
-        commonlicdir = d.getVar('COMMON_LICENSE_DIR')
-        for fn in os.listdir(commonlicdir):
-            md5value = bb.utils.md5_file(os.path.join(commonlicdir, fn))
-            md5sums[md5value] = fn
-    # The following were extracted from common values in various recipes
-    # (double checking the license against the license file itself, not just
-    # the LICENSE value in the recipe)
-    md5sums['94d55d512a9ba36caa9b7df079bae19f'] = 'GPLv2'
-    md5sums['b234ee4d69f5fce4486a80fdaf4a4263'] = 'GPLv2'
-    md5sums['59530bdf33659b29e73d4adb9f9f6552'] = 'GPLv2'
-    md5sums['0636e73ff0215e8d672dc4c32c317bb3'] = 'GPLv2'
-    md5sums['eb723b61539feef013de476e68b5c50a'] = 'GPLv2'
-    md5sums['751419260aa954499f7abaabaa882bbe'] = 'GPLv2'
-    md5sums['393a5ca445f6965873eca0259a17f833'] = 'GPLv2'
-    md5sums['12f884d2ae1ff87c09e5b7ccc2c4ca7e'] = 'GPLv2'
-    md5sums['8ca43cbc842c2336e835926c2166c28b'] = 'GPLv2'
-    md5sums['ebb5c50ab7cab4baeffba14977030c07'] = 'GPLv2'
-    md5sums['c93c0550bd3173f4504b2cbd8991e50b'] = 'GPLv2'
-    md5sums['9ac2e7cff1ddaf48b6eab6028f23ef88'] = 'GPLv2'
-    md5sums['4325afd396febcb659c36b49533135d4'] = 'GPLv2'
-    md5sums['18810669f13b87348459e611d31ab760'] = 'GPLv2'
-    md5sums['d7810fab7487fb0aad327b76f1be7cd7'] = 'GPLv2' # the Linux kernel's COPYING file
-    md5sums['bbb461211a33b134d42ed5ee802b37ff'] = 'LGPLv2.1'
-    md5sums['7fbc338309ac38fefcd64b04bb903e34'] = 'LGPLv2.1'
-    md5sums['4fbd65380cdd255951079008b364516c'] = 'LGPLv2.1'
-    md5sums['2d5025d4aa3495befef8f17206a5b0a1'] = 'LGPLv2.1'
-    md5sums['fbc093901857fcd118f065f900982c24'] = 'LGPLv2.1'
-    md5sums['a6f89e2100d9b6cdffcea4f398e37343'] = 'LGPLv2.1'
-    md5sums['d8045f3b8f929c1cb29a1e3fd737b499'] = 'LGPLv2.1'
-    md5sums['fad9b3332be894bab9bc501572864b29'] = 'LGPLv2.1'
-    md5sums['3bf50002aefd002f49e7bb854063f7e7'] = 'LGPLv2'
-    md5sums['9f604d8a4f8e74f4f5140845a21b6674'] = 'LGPLv2'
-    md5sums['5f30f0716dfdd0d91eb439ebec522ec2'] = 'LGPLv2'
-    md5sums['55ca817ccb7d5b5b66355690e9abc605'] = 'LGPLv2'
-    md5sums['252890d9eee26aab7b432e8b8a616475'] = 'LGPLv2'
-    md5sums['3214f080875748938ba060314b4f727d'] = 'LGPLv2'
-    md5sums['db979804f025cf55aabec7129cb671ed'] = 'LGPLv2'
-    md5sums['d32239bcb673463ab874e80d47fae504'] = 'GPLv3'
-    md5sums['f27defe1e96c2e1ecd4e0c9be8967949'] = 'GPLv3'
-    md5sums['6a6a8e020838b23406c81b19c1d46df6'] = 'LGPLv3'
-    md5sums['3b83ef96387f14655fc854ddc3c6bd57'] = 'Apache-2.0'
-    md5sums['385c55653886acac3821999a3ccd17b3'] = 'Artistic-1.0 | GPL-2.0' # some perl modules
-    md5sums['54c7042be62e169199200bc6477f04d1'] = 'BSD-3-Clause'
-    md5sums['bfe1f75d606912a4111c90743d6c7325'] = 'MPL-1.1'
-    return md5sums
-
-def crunch_license(licfile):
-    '''
-    Remove non-material text from a license file and then check
-    its md5sum against a known list. This works well for licenses
-    which contain a copyright statement, but is also a useful way
-    to handle people's insistence upon reformatting the license text
-    slightly (with no material difference to the text of the
-    license).
-    '''
-
-    import oe.utils
-
-    # Note: these are carefully constructed!
-    license_title_re = re.compile(r'^\(?(#+ *)?(The )?.{1,10} [Ll]icen[sc]e( \(.{1,10}\))?\)?:?$')
-    license_statement_re = re.compile(r'^(This (project|software) is( free software)? (released|licen[sc]ed)|(Released|Licen[cs]ed)) under the .{1,10} [Ll]icen[sc]e:?$')
-    copyright_re = re.compile('^(#+)? *Copyright .*$')
-
-    crunched_md5sums = {}
-    # The following two were gleaned from the "forever" npm package
-    crunched_md5sums['0a97f8e4cbaf889d6fa51f84b89a79f6'] = 'ISC'
-    crunched_md5sums['eecf6429523cbc9693547cf2db790b5c'] = 'MIT'
-    # https://github.com/vasi/pixz/blob/master/LICENSE
-    crunched_md5sums['2f03392b40bbe663597b5bd3cc5ebdb9'] = 'BSD-2-Clause'
-    # https://github.com/waffle-gl/waffle/blob/master/LICENSE.txt
-    crunched_md5sums['e72e5dfef0b1a4ca8a3d26a60587db66'] = 'BSD-2-Clause'
-    # https://github.com/spigwitmer/fakeds1963s/blob/master/LICENSE
-    crunched_md5sums['8be76ac6d191671f347ee4916baa637e'] = 'GPLv2'
-    # https://github.com/datto/dattobd/blob/master/COPYING
-    # http://git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/GPLv2.TXT
-    crunched_md5sums['1d65c5ad4bf6489f85f4812bf08ae73d'] = 'GPLv2'
-    # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt
-    # http://git.neil.brown.name/?p=mdadm.git;a=blob;f=COPYING;h=d159169d1050894d3ea3b98e1c965c4058208fe1;hb=HEAD
-    crunched_md5sums['fb530f66a7a89ce920f0e912b5b66d4b'] = 'GPLv2'
-    # https://github.com/gkos/nrf24/blob/master/COPYING
-    crunched_md5sums['7b6aaa4daeafdfa6ed5443fd2684581b'] = 'GPLv2'
-    # https://github.com/josch09/resetusb/blob/master/COPYING
-    crunched_md5sums['8b8ac1d631a4d220342e83bcf1a1fbc3'] = 'GPLv3'
-    # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv2.1
-    crunched_md5sums['2ea316ed973ae176e502e2297b574bb3'] = 'LGPLv2.1'
-    # unixODBC-2.3.4 COPYING
-    crunched_md5sums['1daebd9491d1e8426900b4fa5a422814'] = 'LGPLv2.1'
-    # https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3
-    crunched_md5sums['2ebfb3bb49b9a48a075cc1425e7f4129'] = 'LGPLv3'
-    # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/epl-v10
-    crunched_md5sums['efe2cb9a35826992b9df68224e3c2628'] = 'EPL-1.0'
-    # https://raw.githubusercontent.com/eclipse/mosquitto/v1.4.14/edl-v10
-    crunched_md5sums['0a9c78c0a398d1bbce4a166757d60387'] = 'EDL-1.0'
-    lictext = []
-    with open(licfile, 'r', errors='surrogateescape') as f:
-        for line in f:
-            # Drop opening statements
-            if copyright_re.match(line):
-                continue
-            elif license_title_re.match(line):
-                continue
-            elif license_statement_re.match(line):
-                continue
-            # Squash spaces, and replace smart quotes, double quotes
-            # and backticks with single quotes
-            line = oe.utils.squashspaces(line.strip())
-            line = line.replace(u"\u2018", "'").replace(u"\u2019", "'").replace(u"\u201c","'").replace(u"\u201d", "'").replace('"', '\'').replace('`', '\'')
-            if line:
-                lictext.append(line)
-
-    m = hashlib.md5()
-    try:
-        m.update(' '.join(lictext).encode('utf-8'))
-        md5val = m.hexdigest()
-    except UnicodeEncodeError:
-        md5val = None
-        lictext = ''
-    license = crunched_md5sums.get(md5val, None)
-    return license, md5val, lictext
-
-def guess_license(srctree, d):
-    import bb
-    md5sums = get_license_md5sums(d)
-
-    licenses = []
-    licspecs = ['*LICEN[CS]E*', 'COPYING*', '*[Ll]icense*', 'LEGAL*', '[Ll]egal*', '*GPL*', 'README.lic*', 'COPYRIGHT*', '[Cc]opyright*', 'e[dp]l-v10']
-    licfiles = []
-    for root, dirs, files in os.walk(srctree):
-        for fn in files:
-            for spec in licspecs:
-                if fnmatch.fnmatch(fn, spec):
-                    fullpath = os.path.join(root, fn)
-                    if not fullpath in licfiles:
-                        licfiles.append(fullpath)
-    for licfile in licfiles:
-        md5value = bb.utils.md5_file(licfile)
-        license = md5sums.get(md5value, None)
-        if not license:
-            license, crunched_md5, lictext = crunch_license(licfile)
-            if not license:
-                license = 'Unknown'
-        licenses.append((license, os.path.relpath(licfile, srctree), md5value))
-
-    # FIXME should we grab at least one source file with a license header and add that too?
-
-    return licenses
-
 def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn='${PN}'):
     """
-    Given a list of (license, path, md5sum) as returned by guess_license(),
+    Given a list of (license, path, md5sum) as returned by match_licenses(),
     a dict of package name to path mappings, write out a set of
     package-specific LICENSE values.
     """
     pkglicenses = {pn: []}
     for license, licpath, _ in licvalues:
+        license = fixup_license(license)
         for pkgname, pkgpath in packages.items():
             if licpath.startswith(pkgpath + '/'):
                 if pkgname in pkglicenses:
@@ -1174,13 +1058,24 @@ def split_pkg_licenses(licvalues, packages, outlines, fallback_licenses=None, pn
             pkglicenses[pn].append(license)
     outlicenses = {}
     for pkgname in packages:
-        license = ' '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown'
-        if license == 'Unknown' and pkgname in fallback_licenses:
+        # Assume AND operator between license files
+        license = ' & '.join(list(set(pkglicenses.get(pkgname, ['Unknown'])))) or 'Unknown'
+        if license == 'Unknown' and fallback_licenses and pkgname in fallback_licenses:
             license = fallback_licenses[pkgname]
-        outlines.append('LICENSE_%s = "%s"' % (pkgname, license))
-        outlicenses[pkgname] = license.split()
+        licenses = tidy_licenses(license)
+        license = ' & '.join(licenses)
+        outlines.append('LICENSE:%s = "%s"' % (pkgname, license))
+        outlicenses[pkgname] = licenses
     return outlicenses
 
+def generate_common_licenses_chksums(common_licenses, d):
+    lic_files_chksums = []
+    for license in tidy_licenses(common_licenses):
+        licfile = '${COMMON_LICENSE_DIR}/' + license
+        md5value = bb.utils.md5_file(d.expand(licfile))
+        lic_files_chksums.append('file://%s;md5=%s' % (licfile, md5value))
+    return lic_files_chksums
+
 def read_pkgconfig_provides(d):
     pkgdatadir = d.getVar('PKGDATA_DIR')
     pkgmap = {}
@@ -1311,7 +1206,7 @@ def register_commands(subparsers):
     parser_create.add_argument('-B', '--srcbranch', help='Branch in source repository if fetching from an SCM such as git (default master)')
     parser_create.add_argument('--keep-temp', action="store_true", help='Keep temporary directory (for debugging)')
     parser_create.add_argument('--npm-dev', action="store_true", help='For npm, also fetch devDependencies')
+    parser_create.add_argument('--no-pypi', action="store_true", help='Do not inherit pypi class')
     parser_create.add_argument('--devtool', action="store_true", help=argparse.SUPPRESS)
     parser_create.add_argument('--mirrors', action="store_true", help='Enable PREMIRRORS and MIRRORS for source tree fetching (disabled by default).')
     parser_create.set_defaults(func=create_recipe)
-
diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py
index 35a97c9345..ec9d510e23 100644
--- a/scripts/lib/recipetool/create_buildsys.py
+++ b/scripts/lib/recipetool/create_buildsys.py
@@ -5,9 +5,9 @@
 # SPDX-License-Identifier: GPL-2.0-only
 #
 
+import os
 import re
 import logging
-import glob
 from recipetool.create import RecipeHandler, validate_pv
 
 logger = logging.getLogger('recipetool')
@@ -137,15 +137,15 @@ class CmakeRecipeHandler(RecipeHandler):
         deps = []
         unmappedpkgs = []
 
-        proj_re = re.compile('project\s*\(([^)]*)\)', re.IGNORECASE)
-        pkgcm_re = re.compile('pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE)
-        pkgsm_re = re.compile('pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE)
-        findpackage_re = re.compile('find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE)
-        findlibrary_re = re.compile('find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*')
-        checklib_re = re.compile('check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE)
-        include_re = re.compile('include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE)
-        subdir_re = re.compile('add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE)
-        dep_re = re.compile('([^ ><=]+)( *[<>=]+ *[^ ><=]+)?')
+        proj_re = re.compile(r'project\s*\(([^)]*)\)', re.IGNORECASE)
+        pkgcm_re = re.compile(r'pkg_check_modules\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?\s+([^)\s]+)\s*\)', re.IGNORECASE)
+        pkgsm_re = re.compile(r'pkg_search_module\s*\(\s*[a-zA-Z0-9-_]+\s*(REQUIRED)?((\s+[^)\s]+)+)\s*\)', re.IGNORECASE)
+        findpackage_re = re.compile(r'find_package\s*\(\s*([a-zA-Z0-9-_]+)\s*.*', re.IGNORECASE)
+        findlibrary_re = re.compile(r'find_library\s*\(\s*[a-zA-Z0-9-_]+\s*(NAMES\s+)?([a-zA-Z0-9-_ ]+)\s*.*')
+        checklib_re = re.compile(r'check_library_exists\s*\(\s*([^\s)]+)\s*.*', re.IGNORECASE)
+        include_re = re.compile(r'include\s*\(\s*([^)\s]*)\s*\)', re.IGNORECASE)
+        subdir_re = re.compile(r'add_subdirectory\s*\(\s*([^)\s]*)\s*([^)\s]*)\s*\)', re.IGNORECASE)
+        dep_re = re.compile(r'([^ ><=]+)( *[<>=]+ *[^ ><=]+)?')
 
         def find_cmake_package(pkg):
             RecipeHandler.load_devel_filemap(tinfoil.config_data)
@@ -423,16 +423,16 @@ class AutotoolsRecipeHandler(RecipeHandler):
                 'makeinfo': 'texinfo',
                 }
 
-        pkg_re = re.compile('PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
-        pkgce_re = re.compile('PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
-        lib_re = re.compile('AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*')
-        libx_re = re.compile('AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*')
-        progs_re = re.compile('_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
-        dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?')
-        ac_init_re = re.compile('AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*')
-        am_init_re = re.compile('AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*')
-        define_re = re.compile('\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)')
-        version_re = re.compile('([0-9.]+)')
+        pkg_re = re.compile(r'PKG_CHECK_MODULES\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
+        pkgce_re = re.compile(r'PKG_CHECK_EXISTS\(\s*\[?([^,\]]*)\]?[),].*')
+        lib_re = re.compile(r'AC_CHECK_LIB\(\s*\[?([^,\]]*)\]?,.*')
+        libx_re = re.compile(r'AX_CHECK_LIBRARY\(\s*\[?[^,\]]*\]?,\s*\[?([^,\]]*)\]?,\s*\[?([a-zA-Z0-9-]*)\]?,.*')
+        progs_re = re.compile(r'_PROGS?\(\s*\[?[a-zA-Z0-9_]*\]?,\s*\[?([^,\]]*)\]?[),].*')
+        dep_re = re.compile(r'([^ ><=]+)( [<>=]+ [^ ><=]+)?')
+        ac_init_re = re.compile(r'AC_INIT\(\s*([^,]+),\s*([^,]+)[,)].*')
+        am_init_re = re.compile(r'AM_INIT_AUTOMAKE\(\s*([^,]+),\s*([^,]+)[,)].*')
+        define_re = re.compile(r'\s*(m4_)?define\(\s*([^,]+),\s*([^,]+)\)')
+        version_re = re.compile(r'([0-9.]+)')
 
         defines = {}
         def subst_defines(value):
@@ -545,7 +545,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
                 deps.append('zlib')
             elif keyword in ('AX_CHECK_OPENSSL', 'AX_LIB_CRYPTO'):
                 deps.append('openssl')
-            elif keyword == 'AX_LIB_CURL':
+            elif keyword in ('AX_LIB_CURL', 'LIBCURL_CHECK_CONFIG'):
                 deps.append('curl')
             elif keyword == 'AX_LIB_BEECRYPT':
                 deps.append('beecrypt')
@@ -624,6 +624,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
                     'AX_CHECK_OPENSSL',
                     'AX_LIB_CRYPTO',
                     'AX_LIB_CURL',
+                    'LIBCURL_CHECK_CONFIG',
                     'AX_LIB_BEECRYPT',
                     'AX_LIB_EXPAT',
                     'AX_LIB_GCRYPT',
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py
index adfa377956..a807dafae5 100644
--- a/scripts/lib/recipetool/create_buildsys_python.py
+++ b/scripts/lib/recipetool/create_buildsys_python.py
@@ -8,9 +8,9 @@
 import ast
 import codecs
 import collections
-import distutils.command.build_py
+import setuptools.command.build_py
 import email
-import imp
+import importlib
 import glob
 import itertools
 import logging
@@ -18,7 +18,11 @@ import os
 import re
 import sys
 import subprocess
+import json
+import urllib.request
 from recipetool.create import RecipeHandler
+from urllib.parse import urldefrag
+from recipetool.create import determine_from_url
 
 logger = logging.getLogger('recipetool')
 
@@ -37,7 +41,334 @@ class PythonRecipeHandler(RecipeHandler):
     assume_provided = ['builtins', 'os.path']
     # Assumes that the host python3 builtin_module_names is sane for target too
     assume_provided = assume_provided + list(sys.builtin_module_names)
+    excluded_fields = []
 
+
+    classifier_license_map = {
+        'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
+        'License :: OSI Approved :: Apache Software License': 'Apache',
+        'License :: OSI Approved :: Apple Public Source License': 'APSL',
+        'License :: OSI Approved :: Artistic License': 'Artistic',
+        'License :: OSI Approved :: Attribution Assurance License': 'AAL',
+        'License :: OSI Approved :: BSD License': 'BSD-3-Clause',
+        'License :: OSI Approved :: Boost Software License 1.0 (BSL-1.0)': 'BSL-1.0',
+        'License :: OSI Approved :: CEA CNRS Inria Logiciel Libre License, version 2.1 (CeCILL-2.1)': 'CECILL-2.1',
+        'License :: OSI Approved :: Common Development and Distribution License 1.0 (CDDL-1.0)': 'CDDL-1.0',
+        'License :: OSI Approved :: Common Public License': 'CPL',
+        'License :: OSI Approved :: Eclipse Public License 1.0 (EPL-1.0)': 'EPL-1.0',
+        'License :: OSI Approved :: Eclipse Public License 2.0 (EPL-2.0)': 'EPL-2.0',
+        'License :: OSI Approved :: Eiffel Forum License': 'EFL',
+        'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0',
+        'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1',
+        'License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)': 'EUPL-1.2',
+        'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0-only',
+        'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0-or-later',
+        'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL',
+        'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL',
+        'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0-only',
+        'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0-or-later',
+        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0-only',
+        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0-or-later',
+        'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0-only',
+        'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0-or-later',
+        'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0-only',
+        'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0-or-later',
+        'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL',
+        'License :: OSI Approved :: Historical Permission Notice and Disclaimer (HPND)': 'HPND',
+        'License :: OSI Approved :: IBM Public License': 'IPL',
+        'License :: OSI Approved :: ISC License (ISCL)': 'ISC',
+        'License :: OSI Approved :: Intel Open Source License': 'Intel',
+        'License :: OSI Approved :: Jabber Open Source License': 'Jabber',
+        'License :: OSI Approved :: MIT License': 'MIT',
+        'License :: OSI Approved :: MIT No Attribution License (MIT-0)': 'MIT-0',
+        'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL',
+        'License :: OSI Approved :: MirOS License (MirOS)': 'MirOS',
+        'License :: OSI Approved :: Motosoto License': 'Motosoto',
+        'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0',
+        'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1',
+        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0',
+        'License :: OSI Approved :: Nethack General Public License': 'NGPL',
+        'License :: OSI Approved :: Nokia Open Source License': 'Nokia',
+        'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL',
+        'License :: OSI Approved :: Open Software License 3.0 (OSL-3.0)': 'OSL-3.0',
+        'License :: OSI Approved :: PostgreSQL License': 'PostgreSQL',
+        'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python',
+        'License :: OSI Approved :: Python Software Foundation License': 'PSF-2.0',
+        'License :: OSI Approved :: Qt Public License (QPL)': 'QPL',
+        'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL',
+        'License :: OSI Approved :: SIL Open Font License 1.1 (OFL-1.1)': 'OFL-1.1',
+        'License :: OSI Approved :: Sleepycat License': 'Sleepycat',
+        'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': 'SISSL',
+        'License :: OSI Approved :: Sun Public License': 'SPL',
+        'License :: OSI Approved :: The Unlicense (Unlicense)': 'Unlicense',
+        'License :: OSI Approved :: Universal Permissive License (UPL)': 'UPL-1.0',
+        'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA',
+        'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0',
+        'License :: OSI Approved :: W3C License': 'W3C',
+        'License :: OSI Approved :: X.Net License': 'Xnet',
+        'License :: OSI Approved :: Zope Public License': 'ZPL',
+        'License :: OSI Approved :: zlib/libpng License': 'Zlib',
+        'License :: Other/Proprietary License': 'Proprietary',
+        'License :: Public Domain': 'PD',
+    }
+
+    def __init__(self):
+        pass
+
+    def process_url(self, args, classes, handled, extravalues):
+        """
+        Convert any pypi url https://pypi.org/project/<package>/<version> into https://files.pythonhosted.org/packages/source/...
+        which corresponds to the archive location, and add pypi class
+        """
+
+        if 'url' in handled:
+            return None
+
+        fetch_uri = None
+        source = args.source
+        required_version = args.version if args.version else None
+        match = re.match(r'https?://pypi.org/project/([^/]+)(?:/([^/]+))?/?$', urldefrag(source)[0])
+        if match:
+            package = match.group(1)
+            version = match.group(2) if match.group(2) else required_version
+
+            json_url = f"https://pypi.org/pypi/%s/json" % package
+            response = urllib.request.urlopen(json_url)
+            if response.status == 200:
+                data = json.loads(response.read())
+                if not version:
+                    # grab latest version
+                    version = data["info"]["version"]
+                pypi_package = data["info"]["name"]
+                for release in reversed(data["releases"][version]):
+                    if release["packagetype"] == "sdist":
+                        fetch_uri = release["url"]
+                        break
+            else:
+                logger.warning("Cannot handle pypi url %s: cannot fetch package information using %s", source, json_url)
+                return None
+        else:
+            match = re.match(r'^https?://files.pythonhosted.org/packages.*/(.*)-.*$', source)
+            if match:
+                fetch_uri = source
+                pypi_package = match.group(1)
+                _, version = determine_from_url(fetch_uri)
+
+        if match and not args.no_pypi:
+            if required_version and version != required_version:
+                raise Exception("Version specified using --version/-V (%s) and version specified in the url (%s) do not match" % (required_version, version))
+            # This is optionnal if BPN looks like "python-<pypi_package>" or "python3-<pypi_package>" (see pypi.bbclass)
+            # but at this point we cannot know because because user can specify the output name of the recipe on the command line
+            extravalues["PYPI_PACKAGE"] = pypi_package
+            # If the tarball extension is not 'tar.gz' (default value in pypi.bblcass) whe should set PYPI_PACKAGE_EXT in the recipe
+            pypi_package_ext = re.match(r'.*%s-%s\.(.*)$' % (pypi_package, version), fetch_uri)
+            if pypi_package_ext:
+                pypi_package_ext = pypi_package_ext.group(1)
+                if pypi_package_ext != "tar.gz":
+                    extravalues["PYPI_PACKAGE_EXT"] = pypi_package_ext
+
+            # Pypi class will handle S and SRC_URI variables, so remove them
+            # TODO: allow oe.recipeutils.patch_recipe_lines() to accept regexp so we can simplify the following to:
+            # extravalues['SRC_URI(?:\[.*?\])?'] = None
+            extravalues['S'] = None
+            extravalues['SRC_URI'] = None
+
+            classes.append('pypi')
+
+        handled.append('url')
+        return fetch_uri
+
+    def handle_classifier_license(self, classifiers, existing_licenses=""):
+
+        licenses = []
+        for classifier in classifiers:
+            if classifier in self.classifier_license_map:
+                license = self.classifier_license_map[classifier]
+                if license == 'Apache' and 'Apache-2.0' in existing_licenses:
+                    license = 'Apache-2.0'
+                elif license == 'GPL':
+                    if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
+                        license = 'GPL-2.0'
+                    elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
+                        license = 'GPL-3.0'
+                elif license == 'LGPL':
+                    if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
+                        license = 'LGPL-2.1'
+                    elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
+                        license = 'LGPL-2.0'
+                    elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
+                        license = 'LGPL-3.0'
+                licenses.append(license)
+
+        if licenses:
+            return ' & '.join(licenses)
+
+        return None
+
+    def map_info_to_bbvar(self, info, extravalues):
+
+        # Map PKG-INFO & setup.py fields to bitbake variables
+        for field, values in info.items():
+            if field in self.excluded_fields:
+                continue
+
+            if field not in self.bbvar_map:
+                continue
+
+            if isinstance(values, str):
+                value = values
+            else:
+                value = ' '.join(str(v) for v in values if v)
+
+            bbvar = self.bbvar_map[field]
+            if bbvar == "PN":
+                # by convention python recipes start with "python3-"
+                if not value.startswith('python'):
+                    value = 'python3-' + value
+
+            if bbvar not in extravalues and value:
+                extravalues[bbvar] = value
+
+    def apply_info_replacements(self, info):
+        if not self.replacements:
+            return
+
+        for variable, search, replace in self.replacements:
+            if variable not in info:
+                continue
+
+            def replace_value(search, replace, value):
+                if replace is None:
+                    if re.search(search, value):
+                        return None
+                else:
+                    new_value = re.sub(search, replace, value)
+                    if value != new_value:
+                        return new_value
+                return value
+
+            value = info[variable]
+            if isinstance(value, str):
+                new_value = replace_value(search, replace, value)
+                if new_value is None:
+                    del info[variable]
+                elif new_value != value:
+                    info[variable] = new_value
+            elif hasattr(value, 'items'):
+                for dkey, dvalue in list(value.items()):
+                    new_list = []
+                    for pos, a_value in enumerate(dvalue):
+                        new_value = replace_value(search, replace, a_value)
+                        if new_value is not None and new_value != value:
+                            new_list.append(new_value)
+
+                    if value != new_list:
+                        value[dkey] = new_list
+            else:
+                new_list = []
+                for pos, a_value in enumerate(value):
+                    new_value = replace_value(search, replace, a_value)
+                    if new_value is not None and new_value != value:
+                        new_list.append(new_value)
+
+                if value != new_list:
+                    info[variable] = new_list
+
+
+    def scan_python_dependencies(self, paths):
+        deps = set()
+        try:
+            dep_output = self.run_command(['pythondeps', '-d'] + paths)
+        except (OSError, subprocess.CalledProcessError):
+            pass
+        else:
+            for line in dep_output.splitlines():
+                line = line.rstrip()
+                dep, filename = line.split('\t', 1)
+                if filename.endswith('/setup.py'):
+                    continue
+                deps.add(dep)
+
+        try:
+            provides_output = self.run_command(['pythondeps', '-p'] + paths)
+        except (OSError, subprocess.CalledProcessError):
+            pass
+        else:
+            provides_lines = (l.rstrip() for l in provides_output.splitlines())
+            provides = set(l for l in provides_lines if l and l != 'setup')
+            deps -= provides
+
+        return deps
+
+    def parse_pkgdata_for_python_packages(self):
+        pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
+
+        ldata = tinfoil.config_data.createCopy()
+        bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True)
+        python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
+
+        dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
+        python_dirs = [python_sitedir + os.sep,
+                       os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
+                       os.path.dirname(python_sitedir) + os.sep]
+        packages = {}
+        for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
+            files_info = None
+            with open(pkgdatafile, 'r') as f:
+                for line in f.readlines():
+                    field, value = line.split(': ', 1)
+                    if field.startswith('FILES_INFO'):
+                        files_info = ast.literal_eval(value)
+                        break
+                else:
+                    continue
+
+            for fn in files_info:
+                for suffix in importlib.machinery.all_suffixes():
+                    if fn.endswith(suffix):
+                        break
+                else:
+                    continue
+
+                if fn.startswith(dynload_dir + os.sep):
+                    if '/.debug/' in fn:
+                        continue
+                    base = os.path.basename(fn)
+                    provided = base.split('.', 1)[0]
+                    packages[provided] = os.path.basename(pkgdatafile)
+                    continue
+
+                for python_dir in python_dirs:
+                    if fn.startswith(python_dir):
+                        relpath = fn[len(python_dir):]
+                        relstart, _, relremaining = relpath.partition(os.sep)
+                        if relstart.endswith('.egg'):
+                            relpath = relremaining
+                        base, _ = os.path.splitext(relpath)
+
+                        if '/.debug/' in base:
+                            continue
+                        if os.path.basename(base) == '__init__':
+                            base = os.path.dirname(base)
+                        base = base.replace(os.sep + os.sep, os.sep)
+                        provided = base.replace(os.sep, '.')
+                        packages[provided] = os.path.basename(pkgdatafile)
+        return packages
+
+    @classmethod
+    def run_command(cls, cmd, **popenargs):
+        if 'stderr' not in popenargs:
+            popenargs['stderr'] = subprocess.STDOUT
+        try:
+            return subprocess.check_output(cmd, **popenargs).decode('utf-8')
+        except OSError as exc:
+            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
+            raise
+        except subprocess.CalledProcessError as exc:
+            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
+            raise
+
+class PythonSetupPyRecipeHandler(PythonRecipeHandler):
     bbvar_map = {
         'Name': 'PN',
         'Version': 'PV',
@@ -45,9 +376,9 @@ class PythonRecipeHandler(RecipeHandler):
         'Summary': 'SUMMARY',
         'Description': 'DESCRIPTION',
         'License': 'LICENSE',
-        'Requires': 'RDEPENDS_${PN}',
-        'Provides': 'RPROVIDES_${PN}',
-        'Obsoletes': 'RREPLACES_${PN}',
+        'Requires': 'RDEPENDS:${PN}',
+        'Provides': 'RPROVIDES:${PN}',
+        'Obsoletes': 'RREPLACES:${PN}',
     }
     # PN/PV are already set by recipetool core & desc can be extremely long
     excluded_fields = [
@@ -75,6 +406,7 @@ class PythonRecipeHandler(RecipeHandler):
         'Supported-Platform',
     ]
     setuparg_multi_line_values = ['Description']
+
     replacements = [
         ('License', r' +$', ''),
         ('License', r'^ +', ''),
@@ -95,71 +427,161 @@ class PythonRecipeHandler(RecipeHandler):
         ('Install-requires', r'\[[^\]]+\]$', ''),
     ]
 
-    classifier_license_map = {
-        'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
-        'License :: OSI Approved :: Apache Software License': 'Apache',
-        'License :: OSI Approved :: Apple Public Source License': 'APSL',
-        'License :: OSI Approved :: Artistic License': 'Artistic',
-        'License :: OSI Approved :: Attribution Assurance License': 'AAL',
-        'License :: OSI Approved :: BSD License': 'BSD',
-        'License :: OSI Approved :: Common Public License': 'CPL',
-        'License :: OSI Approved :: Eiffel Forum License': 'EFL',
-        'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0',
-        'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1',
-        'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0+',
-        'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0',
-        'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL',
-        'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL',
-        'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0',
-        'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0+',
-        'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0',
-        'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0+',
-        'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0',
-        'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0+',
-        'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0',
-        'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0+',
-        'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL',
-        'License :: OSI Approved :: IBM Public License': 'IPL',
-        'License :: OSI Approved :: ISC License (ISCL)': 'ISC',
-        'License :: OSI Approved :: Intel Open Source License': 'Intel',
-        'License :: OSI Approved :: Jabber Open Source License': 'Jabber',
-        'License :: OSI Approved :: MIT License': 'MIT',
-        'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL',
-        'License :: OSI Approved :: Motosoto License': 'Motosoto',
-        'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0',
-        'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1',
-        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0',
-        'License :: OSI Approved :: Nethack General Public License': 'NGPL',
-        'License :: OSI Approved :: Nokia Open Source License': 'Nokia',
-        'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL',
-        'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python',
-        'License :: OSI Approved :: Python Software Foundation License': 'PSF',
-        'License :: OSI Approved :: Qt Public License (QPL)': 'QPL',
-        'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL',
-        'License :: OSI Approved :: Sleepycat License': 'Sleepycat',
-        'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': '--  Sun Industry Standards Source License (SISSL)',
-        'License :: OSI Approved :: Sun Public License': 'SPL',
-        'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA',
-        'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0',
-        'License :: OSI Approved :: W3C License': 'W3C',
-        'License :: OSI Approved :: X.Net License': 'Xnet',
-        'License :: OSI Approved :: Zope Public License': 'ZPL',
-        'License :: OSI Approved :: zlib/libpng License': 'Zlib',
-    }
-
     def __init__(self):
         pass
 
+    def parse_setup_py(self, setupscript='./setup.py'):
+        with codecs.open(setupscript) as f:
+            info, imported_modules, non_literals, extensions = gather_setup_info(f)
+
+        def _map(key):
+            key = key.replace('_', '-')
+            key = key[0].upper() + key[1:]
+            if key in self.setup_parse_map:
+                key = self.setup_parse_map[key]
+            return key
+
+        # Naive mapping of setup() arguments to PKG-INFO field names
+        for d in [info, non_literals]:
+            for key, value in list(d.items()):
+                if key is None:
+                    continue
+                new_key = _map(key)
+                if new_key != key:
+                    del d[key]
+                    d[new_key] = value
+
+        return info, 'setuptools' in imported_modules, non_literals, extensions
+
+    def get_setup_args_info(self, setupscript='./setup.py'):
+        cmd = ['python3', setupscript]
+        info = {}
+        keys = set(self.bbvar_map.keys())
+        keys |= set(self.setuparg_list_fields)
+        keys |= set(self.setuparg_multi_line_values)
+        grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
+        for index, keys in grouped_keys:
+            if index == (True, False):
+                # Splitlines output for each arg as a list value
+                for key in keys:
+                    arg = self.setuparg_map.get(key, key.lower())
+                    try:
+                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
+                    except (OSError, subprocess.CalledProcessError):
+                        pass
+                    else:
+                        info[key] = [l.rstrip() for l in arg_info.splitlines()]
+            elif index == (False, True):
+                # Entire output for each arg
+                for key in keys:
+                    arg = self.setuparg_map.get(key, key.lower())
+                    try:
+                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
+                    except (OSError, subprocess.CalledProcessError):
+                        pass
+                    else:
+                        info[key] = arg_info
+            else:
+                info.update(self.get_setup_byline(list(keys), setupscript))
+        return info
+
+    def get_setup_byline(self, fields, setupscript='./setup.py'):
+        info = {}
+
+        cmd = ['python3', setupscript]
+        cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
+        try:
+            info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
+        except (OSError, subprocess.CalledProcessError):
+            pass
+        else:
+            if len(fields) != len(info_lines):
+                logger.error('Mismatch between setup.py output lines and number of fields')
+                sys.exit(1)
+
+            for lineno, line in enumerate(info_lines):
+                line = line.rstrip()
+                info[fields[lineno]] = line
+        return info
+
+    def get_pkginfo(self, pkginfo_fn):
+        msg = email.message_from_file(open(pkginfo_fn, 'r'))
+        msginfo = {}
+        for field in msg.keys():
+            values = msg.get_all(field)
+            if len(values) == 1:
+                msginfo[field] = values[0]
+            else:
+                msginfo[field] = values
+        return msginfo
+
+    def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
+        if 'Package-dir' in setup_info:
+            package_dir = setup_info['Package-dir']
+        else:
+            package_dir = {}
+
+        dist = setuptools.Distribution()
+
+        class PackageDir(setuptools.command.build_py.build_py):
+            def __init__(self, package_dir):
+                self.package_dir = package_dir
+                self.dist = dist
+                super().__init__(self.dist)
+
+        pd = PackageDir(package_dir)
+        to_scan = []
+        if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
+            if 'Py-modules' in setup_info:
+                for module in setup_info['Py-modules']:
+                    try:
+                        package, module = module.rsplit('.', 1)
+                    except ValueError:
+                        package, module = '.', module
+                    module_path = os.path.join(pd.get_package_dir(package), module + '.py')
+                    to_scan.append(module_path)
+
+            if 'Packages' in setup_info:
+                for package in setup_info['Packages']:
+                    to_scan.append(pd.get_package_dir(package))
+
+            if 'Scripts' in setup_info:
+                to_scan.extend(setup_info['Scripts'])
+        else:
+            logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
+
+        if not to_scan:
+            to_scan = ['.']
+
+        logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
+
+        provided_packages = self.parse_pkgdata_for_python_packages()
+        scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
+        mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
+        for dep in scanned_deps:
+            mapped = provided_packages.get(dep)
+            if mapped:
+                logger.debug('Mapped %s to %s' % (dep, mapped))
+                mapped_deps.add(mapped)
+            else:
+                logger.debug('Could not map %s' % dep)
+                unmapped_deps.add(dep)
+        return mapped_deps, unmapped_deps
+
     def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+
         if 'buildsystem' in handled:
             return False
 
+        logger.debug("Trying setup.py parser")
+
         # Check for non-zero size setup.py files
         setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py'])
         for fn in setupfiles:
             if os.path.getsize(fn):
                 break
         else:
+            logger.debug("No setup.py found")
             return False
 
         # setup.py is always parsed to get at certain required information, such as
@@ -193,6 +615,18 @@ class PythonRecipeHandler(RecipeHandler):
                             continue
 
                         if line.startswith('['):
+                            # PACKAGECONFIG must not contain expressions or whitespace
+                            line = line.replace(" ", "")
+                            line = line.replace(':', "")
+                            line = line.replace('.', "-dot-")
+                            line = line.replace('"', "")
+                            line = line.replace('<', "-smaller-")
+                            line = line.replace('>', "-bigger-")
+                            line = line.replace('_', "-")
+                            line = line.replace('(', "")
+                            line = line.replace(')', "")
+                            line = line.replace('!', "-not-")
+                            line = line.replace('=', "-equals-")
                             current_feature = line[1:-1]
                         elif current_feature:
                             extras_req[current_feature].append(line)
@@ -226,51 +660,16 @@ class PythonRecipeHandler(RecipeHandler):
 
         if license_str:
             for i, line in enumerate(lines_before):
-                if line.startswith('LICENSE = '):
+                if line.startswith('##LICENSE_PLACEHOLDER##'):
                     lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
                     break
 
         if 'Classifier' in info:
-            existing_licenses = info.get('License', '')
-            licenses = []
-            for classifier in info['Classifier']:
-                if classifier in self.classifier_license_map:
-                    license = self.classifier_license_map[classifier]
-                    if license == 'Apache' and 'Apache-2.0' in existing_licenses:
-                        license = 'Apache-2.0'
-                    elif license == 'GPL':
-                        if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
-                            license = 'GPL-2.0'
-                        elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
-                            license = 'GPL-3.0'
-                    elif license == 'LGPL':
-                        if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
-                            license = 'LGPL-2.1'
-                        elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
-                            license = 'LGPL-2.0'
-                        elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
-                            license = 'LGPL-3.0'
-                    licenses.append(license)
-
-            if licenses:
-                info['License'] = ' & '.join(licenses)
+            license = self.handle_classifier_license(info['Classifier'], info.get('License', ''))
+            if license:
+                info['License'] = license
 
-        # Map PKG-INFO & setup.py fields to bitbake variables
-        for field, values in info.items():
-            if field in self.excluded_fields:
-                continue
-
-            if field not in self.bbvar_map:
-                continue
-
-            if isinstance(values, str):
-                value = values
-            else:
-                value = ' '.join(str(v) for v in values if v)
-
-            bbvar = self.bbvar_map[field]
-            if bbvar not in extravalues and value:
-                extravalues[bbvar] = value
+        self.map_info_to_bbvar(info, extravalues)
 
         mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
 
@@ -281,6 +680,7 @@ class PythonRecipeHandler(RecipeHandler):
                 lines_after.append('# The following configs & dependencies are from setuptools extras_require.')
                 lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.')
                 lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
+                lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.')
                 lines_after.append('#')
                 lines_after.append('# Uncomment this line to enable all the optional features.')
                 lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req)))
@@ -301,7 +701,7 @@ class PythonRecipeHandler(RecipeHandler):
                 inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
                 lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
                 lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
-                lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
+                lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
 
         if mapped_deps:
             name = info.get('Name')
@@ -313,7 +713,7 @@ class PythonRecipeHandler(RecipeHandler):
                 lines_after.append('')
             lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the')
             lines_after.append('# python sources, and might not be 100% accurate.')
-            lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
+            lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
 
         unmapped_deps -= set(extensions)
         unmapped_deps -= set(self.assume_provided)
@@ -326,275 +726,283 @@ class PythonRecipeHandler(RecipeHandler):
 
         handled.append('buildsystem')
 
-    def get_pkginfo(self, pkginfo_fn):
-        msg = email.message_from_file(open(pkginfo_fn, 'r'))
-        msginfo = {}
-        for field in msg.keys():
-            values = msg.get_all(field)
-            if len(values) == 1:
-                msginfo[field] = values[0]
-            else:
-                msginfo[field] = values
-        return msginfo
+class PythonPyprojectTomlRecipeHandler(PythonRecipeHandler):
+    """Base class to support PEP517 and PEP518
+
+    PEP517 https://peps.python.org/pep-0517/#source-trees
+    PEP518 https://peps.python.org/pep-0518/#build-system-table
+    """
+    # bitbake currently supports the 4 following backends
+    build_backend_map = {
+        "setuptools.build_meta": "python_setuptools_build_meta",
+        "poetry.core.masonry.api": "python_poetry_core",
+        "flit_core.buildapi": "python_flit_core",
+        "hatchling.build": "python_hatchling",
+        "maturin": "python_maturin",
+        "mesonpy": "python_mesonpy",
+    }
 
-    def parse_setup_py(self, setupscript='./setup.py'):
-        with codecs.open(setupscript) as f:
-            info, imported_modules, non_literals, extensions = gather_setup_info(f)
+    # setuptools.build_meta and flit declare project metadata into the "project" section of pyproject.toml
+    # according to PEP-621: https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#declaring-project-metadata
+    # while poetry uses the "tool.poetry" section according to its official documentation: https://python-poetry.org/docs/pyproject/
+    # keys from "project" and "tool.poetry" sections are almost the same except for the  HOMEPAGE which is "homepage" for tool.poetry
+    # and "Homepage" for "project" section. So keep both
+    bbvar_map = {
+        "name": "PN",
+        "version": "PV",
+        "Homepage": "HOMEPAGE",
+        "homepage": "HOMEPAGE",
+        "description": "SUMMARY",
+        "license": "LICENSE",
+        "dependencies": "RDEPENDS:${PN}",
+        "requires": "DEPENDS",
+    }
 
-        def _map(key):
-            key = key.replace('_', '-')
-            key = key[0].upper() + key[1:]
-            if key in self.setup_parse_map:
-                key = self.setup_parse_map[key]
-            return key
+    replacements = [
+        ("license", r" +$", ""),
+        ("license", r"^ +", ""),
+        ("license", r" ", "-"),
+        ("license", r"^GNU-", ""),
+        ("license", r"-[Ll]icen[cs]e(,?-[Vv]ersion)?", ""),
+        ("license", r"^UNKNOWN$", ""),
+        # Remove currently unhandled version numbers from these variables
+        ("requires", r"\[[^\]]+\]$", ""),
+        ("requires", r"^([^><= ]+).*", r"\1"),
+        ("dependencies", r"\[[^\]]+\]$", ""),
+        ("dependencies", r"^([^><= ]+).*", r"\1"),
+    ]
 
-        # Naive mapping of setup() arguments to PKG-INFO field names
-        for d in [info, non_literals]:
-            for key, value in list(d.items()):
-                if key is None:
-                    continue
-                new_key = _map(key)
-                if new_key != key:
-                    del d[key]
-                    d[new_key] = value
+    excluded_native_pkgdeps = [
+        # already provided by python_setuptools_build_meta.bbclass
+        "python3-setuptools-native",
+        "python3-wheel-native",
+        # already provided by python_poetry_core.bbclass
+        "python3-poetry-core-native",
+        # already provided by python_flit_core.bbclass
+        "python3-flit-core-native",
+        # already provided by python_mesonpy
+        "python3-meson-python-native",
+    ]
 
-        return info, 'setuptools' in imported_modules, non_literals, extensions
+    # add here a list of known and often used packages and the corresponding bitbake package
+    known_deps_map = {
+        "setuptools": "python3-setuptools",
+        "wheel": "python3-wheel",
+        "poetry-core": "python3-poetry-core",
+        "flit_core": "python3-flit-core",
+        "setuptools-scm": "python3-setuptools-scm",
+        "hatchling": "python3-hatchling",
+        "hatch-vcs": "python3-hatch-vcs",
+        "meson-python" : "python3-meson-python",
+    }
 
-    def get_setup_args_info(self, setupscript='./setup.py'):
-        cmd = ['python3', setupscript]
-        info = {}
-        keys = set(self.bbvar_map.keys())
-        keys |= set(self.setuparg_list_fields)
-        keys |= set(self.setuparg_multi_line_values)
-        grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
-        for index, keys in grouped_keys:
-            if index == (True, False):
-                # Splitlines output for each arg as a list value
-                for key in keys:
-                    arg = self.setuparg_map.get(key, key.lower())
-                    try:
-                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
-                    except (OSError, subprocess.CalledProcessError):
-                        pass
-                    else:
-                        info[key] = [l.rstrip() for l in arg_info.splitlines()]
-            elif index == (False, True):
-                # Entire output for each arg
-                for key in keys:
-                    arg = self.setuparg_map.get(key, key.lower())
-                    try:
-                        arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
-                    except (OSError, subprocess.CalledProcessError):
-                        pass
-                    else:
-                        info[key] = arg_info
-            else:
-                info.update(self.get_setup_byline(list(keys), setupscript))
-        return info
+    def __init__(self):
+        pass
 
-    def get_setup_byline(self, fields, setupscript='./setup.py'):
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         info = {}
+        metadata = {}
 
-        cmd = ['python3', setupscript]
-        cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
-        try:
-            info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
-        except (OSError, subprocess.CalledProcessError):
-            pass
-        else:
-            if len(fields) != len(info_lines):
-                logger.error('Mismatch between setup.py output lines and number of fields')
-                sys.exit(1)
-
-            for lineno, line in enumerate(info_lines):
-                line = line.rstrip()
-                info[fields[lineno]] = line
-        return info
-
-    def apply_info_replacements(self, info):
-        for variable, search, replace in self.replacements:
-            if variable not in info:
-                continue
-
-            def replace_value(search, replace, value):
-                if replace is None:
-                    if re.search(search, value):
-                        return None
-                else:
-                    new_value = re.sub(search, replace, value)
-                    if value != new_value:
-                        return new_value
-                return value
-
-            value = info[variable]
-            if isinstance(value, str):
-                new_value = replace_value(search, replace, value)
-                if new_value is None:
-                    del info[variable]
-                elif new_value != value:
-                    info[variable] = new_value
-            elif hasattr(value, 'items'):
-                for dkey, dvalue in list(value.items()):
-                    new_list = []
-                    for pos, a_value in enumerate(dvalue):
-                        new_value = replace_value(search, replace, a_value)
-                        if new_value is not None and new_value != value:
-                            new_list.append(new_value)
-
-                    if value != new_list:
-                        value[dkey] = new_list
-            else:
-                new_list = []
-                for pos, a_value in enumerate(value):
-                    new_value = replace_value(search, replace, a_value)
-                    if new_value is not None and new_value != value:
-                        new_list.append(new_value)
-
-                if value != new_list:
-                    info[variable] = new_list
-
-    def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
-        if 'Package-dir' in setup_info:
-            package_dir = setup_info['Package-dir']
-        else:
-            package_dir = {}
-
-        class PackageDir(distutils.command.build_py.build_py):
-            def __init__(self, package_dir):
-                self.package_dir = package_dir
-
-        pd = PackageDir(package_dir)
-        to_scan = []
-        if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
-            if 'Py-modules' in setup_info:
-                for module in setup_info['Py-modules']:
-                    try:
-                        package, module = module.rsplit('.', 1)
-                    except ValueError:
-                        package, module = '.', module
-                    module_path = os.path.join(pd.get_package_dir(package), module + '.py')
-                    to_scan.append(module_path)
+        if 'buildsystem' in handled:
+            return False
 
-            if 'Packages' in setup_info:
-                for package in setup_info['Packages']:
-                    to_scan.append(pd.get_package_dir(package))
+        logger.debug("Trying pyproject.toml parser")
 
-            if 'Scripts' in setup_info:
-                to_scan.extend(setup_info['Scripts'])
+        # Check for non-zero size setup.py files
+        setupfiles = RecipeHandler.checkfiles(srctree, ["pyproject.toml"])
+        for fn in setupfiles:
+            if os.path.getsize(fn):
+                break
         else:
-            logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
-
-        if not to_scan:
-            to_scan = ['.']
-
-        logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
+            logger.debug("No pyproject.toml found")
+            return False
 
-        provided_packages = self.parse_pkgdata_for_python_packages()
-        scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
-        mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
-        for dep in scanned_deps:
-            mapped = provided_packages.get(dep)
-            if mapped:
-                logger.debug('Mapped %s to %s' % (dep, mapped))
-                mapped_deps.add(mapped)
-            else:
-                logger.debug('Could not map %s' % dep)
-                unmapped_deps.add(dep)
-        return mapped_deps, unmapped_deps
+        setupscript = os.path.join(srctree, "pyproject.toml")
 
-    def scan_python_dependencies(self, paths):
-        deps = set()
         try:
-            dep_output = self.run_command(['pythondeps', '-d'] + paths)
-        except (OSError, subprocess.CalledProcessError):
-            pass
-        else:
-            for line in dep_output.splitlines():
-                line = line.rstrip()
-                dep, filename = line.split('\t', 1)
-                if filename.endswith('/setup.py'):
-                    continue
-                deps.add(dep)
+            try:
+                import tomllib
+            except ImportError:
+                try:
+                    import tomli as tomllib
+                except ImportError:
+                    logger.error("Neither 'tomllib' nor 'tomli' could be imported, cannot scan pyproject.toml.")
+                    return False
+
+            try:
+                with open(setupscript, "rb") as f:
+                    config = tomllib.load(f)
+            except Exception:
+                logger.exception("Failed to parse pyproject.toml")
+                return False
+
+            build_backend = config["build-system"]["build-backend"]
+            if build_backend in self.build_backend_map:
+                classes.append(self.build_backend_map[build_backend])
+            else:
+                logger.error(
+                    "Unsupported build-backend: %s, cannot use pyproject.toml. Will try to use legacy setup.py"
+                    % build_backend
+                )
+                return False
 
-        try:
-            provides_output = self.run_command(['pythondeps', '-p'] + paths)
-        except (OSError, subprocess.CalledProcessError):
-            pass
-        else:
-            provides_lines = (l.rstrip() for l in provides_output.splitlines())
-            provides = set(l for l in provides_lines if l and l != 'setup')
-            deps -= provides
+            licfile = ""
 
-        return deps
+            if build_backend == "poetry.core.masonry.api":
+                if "tool" in config and "poetry" in config["tool"]:
+                    metadata = config["tool"]["poetry"]
+            else:
+                if "project" in config:
+                    metadata = config["project"]
+
+            if metadata:
+                for field, values in metadata.items():
+                    if field == "license":
+                        # For setuptools.build_meta and flit, licence is a table
+                        # but for poetry licence is a string
+                        # for hatchling, both table (jsonschema) and string (iniconfig) have been used
+                        if build_backend == "poetry.core.masonry.api":
+                            value = values
+                        else:
+                            value = values.get("text", "")
+                            if not value:
+                                licfile = values.get("file", "")
+                                continue
+                    elif field == "dependencies" and build_backend == "poetry.core.masonry.api":
+                        # For poetry backend, "dependencies" section looks like:
+                        # [tool.poetry.dependencies]
+                        # requests = "^2.13.0"
+                        # requests = { version = "^2.13.0", source = "private" }
+                        # See https://python-poetry.org/docs/master/pyproject/#dependencies-and-dependency-groups for more details
+                        # This class doesn't handle versions anyway, so we just get the dependencies name here and construct a list
+                        value = []
+                        for k in values.keys():
+                            value.append(k)
+                    elif isinstance(values, dict):
+                        for k, v in values.items():
+                            info[k] = v
+                        continue
+                    else:
+                        value = values
 
-    def parse_pkgdata_for_python_packages(self):
-        suffixes = [t[0] for t in imp.get_suffixes()]
-        pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
+                    info[field] = value
 
-        ldata = tinfoil.config_data.createCopy()
-        bb.parse.handle('classes/python3-dir.bbclass', ldata, True)
-        python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
+            # Grab the license value before applying replacements
+            license_str = info.get("license", "").strip()
 
-        dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
-        python_dirs = [python_sitedir + os.sep,
-                       os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
-                       os.path.dirname(python_sitedir) + os.sep]
-        packages = {}
-        for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
-            files_info = None
-            with open(pkgdatafile, 'r') as f:
-                for line in f.readlines():
-                    field, value = line.split(': ', 1)
-                    if field == 'FILES_INFO':
-                        files_info = ast.literal_eval(value)
+            if license_str:
+                for i, line in enumerate(lines_before):
+                    if line.startswith("##LICENSE_PLACEHOLDER##"):
+                        lines_before.insert(
+                            i, "# NOTE: License in pyproject.toml is: %s" % license_str
+                        )
                         break
-                else:
-                    continue
 
-            for fn in files_info:
-                for suffix in suffixes:
-                    if fn.endswith(suffix):
-                        break
-                else:
-                    continue
+            info["requires"] = config["build-system"]["requires"]
+
+            self.apply_info_replacements(info)
+
+            if "classifiers" in info:
+                license = self.handle_classifier_license(
+                    info["classifiers"], info.get("license", "")
+                )
+                if license:
+                    if licfile:
+                        lines = []
+                        md5value = bb.utils.md5_file(os.path.join(srctree, licfile))
+                        lines.append('LICENSE = "%s"' % license)
+                        lines.append(
+                            'LIC_FILES_CHKSUM = "file://%s;md5=%s"'
+                            % (licfile, md5value)
+                        )
+                        lines.append("")
+
+                        # Replace the placeholder so we get the values in the right place in the recipe file
+                        try:
+                            pos = lines_before.index("##LICENSE_PLACEHOLDER##")
+                        except ValueError:
+                            pos = -1
+                        if pos == -1:
+                            lines_before.extend(lines)
+                        else:
+                            lines_before[pos : pos + 1] = lines
 
-                if fn.startswith(dynload_dir + os.sep):
-                    if '/.debug/' in fn:
-                        continue
-                    base = os.path.basename(fn)
-                    provided = base.split('.', 1)[0]
-                    packages[provided] = os.path.basename(pkgdatafile)
-                    continue
+                        handled.append(("license", [license, licfile, md5value]))
+                    else:
+                        info["license"] = license
 
-                for python_dir in python_dirs:
-                    if fn.startswith(python_dir):
-                        relpath = fn[len(python_dir):]
-                        relstart, _, relremaining = relpath.partition(os.sep)
-                        if relstart.endswith('.egg'):
-                            relpath = relremaining
-                        base, _ = os.path.splitext(relpath)
+            provided_packages = self.parse_pkgdata_for_python_packages()
+            provided_packages.update(self.known_deps_map)
+            native_mapped_deps, native_unmapped_deps = set(), set()
+            mapped_deps, unmapped_deps = set(), set()
 
-                        if '/.debug/' in base:
-                            continue
-                        if os.path.basename(base) == '__init__':
-                            base = os.path.dirname(base)
-                        base = base.replace(os.sep + os.sep, os.sep)
-                        provided = base.replace(os.sep, '.')
-                        packages[provided] = os.path.basename(pkgdatafile)
-        return packages
+            if "requires" in info:
+                for require in info["requires"]:
+                    mapped = provided_packages.get(require)
 
-    @classmethod
-    def run_command(cls, cmd, **popenargs):
-        if 'stderr' not in popenargs:
-            popenargs['stderr'] = subprocess.STDOUT
-        try:
-            return subprocess.check_output(cmd, **popenargs).decode('utf-8')
-        except OSError as exc:
-            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
-            raise
-        except subprocess.CalledProcessError as exc:
-            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
-            raise
+                    if mapped:
+                        logger.debug("Mapped %s to %s" % (require, mapped))
+                        native_mapped_deps.add(mapped)
+                    else:
+                        logger.debug("Could not map %s" % require)
+                        native_unmapped_deps.add(require)
+
+                info.pop("requires")
+
+                if native_mapped_deps != set():
+                    native_mapped_deps = {
+                        item + "-native" for item in native_mapped_deps
+                    }
+                    native_mapped_deps -= set(self.excluded_native_pkgdeps)
+                    if native_mapped_deps != set():
+                        info["requires"] = " ".join(sorted(native_mapped_deps))
+
+                if native_unmapped_deps:
+                    lines_after.append("")
+                    lines_after.append(
+                        "# WARNING: We were unable to map the following python package/module"
+                    )
+                    lines_after.append(
+                        "# dependencies to the bitbake packages which include them:"
+                    )
+                    lines_after.extend(
+                        "#    {}".format(d) for d in sorted(native_unmapped_deps)
+                    )
+
+            if "dependencies" in info:
+                for dependency in info["dependencies"]:
+                    mapped = provided_packages.get(dependency)
+                    if mapped:
+                        logger.debug("Mapped %s to %s" % (dependency, mapped))
+                        mapped_deps.add(mapped)
+                    else:
+                        logger.debug("Could not map %s" % dependency)
+                        unmapped_deps.add(dependency)
+
+                info.pop("dependencies")
+
+                if mapped_deps != set():
+                    if mapped_deps != set():
+                        info["dependencies"] = " ".join(sorted(mapped_deps))
+
+                if unmapped_deps:
+                    lines_after.append("")
+                    lines_after.append(
+                        "# WARNING: We were unable to map the following python package/module"
+                    )
+                    lines_after.append(
+                        "# runtime dependencies to the bitbake packages which include them:"
+                    )
+                    lines_after.extend(
+                        "#    {}".format(d) for d in sorted(unmapped_deps)
+                    )
+
+            self.map_info_to_bbvar(info, extravalues)
+
+            handled.append("buildsystem")
+        except Exception:
+            logger.exception("Failed to correctly handle pyproject.toml, falling back to another method")
+            return False
 
 
 def gather_setup_info(fileobj):
@@ -710,5 +1118,7 @@ def has_non_literals(value):
 
 
 def register_recipe_handlers(handlers):
-    # We need to make sure this is ahead of the makefile fallback handler
-    handlers.append((PythonRecipeHandler(), 70))
+    # We need to make sure these are ahead of the makefile fallback handler
+    # and the pyproject.toml handler ahead of the setup.py handler
+    handlers.append((PythonPyprojectTomlRecipeHandler(), 75))
+    handlers.append((PythonSetupPyRecipeHandler(), 70))
diff --git a/scripts/lib/recipetool/create_go.py b/scripts/lib/recipetool/create_go.py
new file mode 100644
index 0000000000..4b1fa39d13
--- /dev/null
+++ b/scripts/lib/recipetool/create_go.py
@@ -0,0 +1,174 @@
+# Recipe creation tool - go support plugin
+#
+# The code is based on golang internals. See the afftected
+# methods for further reference and information.
+#
+# Copyright (C) 2023 Weidmueller GmbH & Co KG
+# Author: Lukas Funke <lukas.funke@weidmueller.com>
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+
+from recipetool.create import RecipeHandler, handle_license_vars
+
+import bb.utils
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+import tempfile
+
+
+logger = logging.getLogger('recipetool')
+
+tinfoil = None
+
+
+def tinfoil_init(instance):
+    global tinfoil
+    tinfoil = instance
+
+
+
+class GoRecipeHandler(RecipeHandler):
+    """Class to handle the go recipe creation"""
+
+    @staticmethod
+    def __ensure_go():
+        """Check if the 'go' command is available in the recipes"""
+        recipe = "go-native"
+        if not tinfoil.recipes_parsed:
+            tinfoil.parse_recipes()
+        try:
+            rd = tinfoil.parse_recipe(recipe)
+        except bb.providers.NoProvider:
+            bb.error(
+                "Nothing provides '%s' which is required for the build" % (recipe))
+            bb.note(
+                "You will likely need to add a layer that provides '%s'" % (recipe))
+            return None
+
+        bindir = rd.getVar('STAGING_BINDIR_NATIVE')
+        gopath = os.path.join(bindir, 'go')
+
+        if not os.path.exists(gopath):
+            tinfoil.build_targets(recipe, 'addto_recipe_sysroot')
+
+            if not os.path.exists(gopath):
+                logger.error(
+                    '%s required to process specified source, but %s did not seem to populate it' % 'go', recipe)
+                return None
+
+        return bindir
+
+    def process(self, srctree, classes, lines_before,
+                lines_after, handled, extravalues):
+
+        if 'buildsystem' in handled:
+            return False
+
+        files = RecipeHandler.checkfiles(srctree, ['go.mod'])
+        if not files:
+            return False
+
+        go_bindir = self.__ensure_go()
+        if not go_bindir:
+            sys.exit(14)
+
+        handled.append('buildsystem')
+        classes.append("go-mod")
+
+        # Use go-mod-update-modules to set the full SRC_URI and LICENSE
+        classes.append("go-mod-update-modules")
+        extravalues["run_tasks"] = "update_modules"
+
+        with tempfile.TemporaryDirectory(prefix="go-mod-") as tmp_mod_dir:
+            env = dict(os.environ)
+            env["PATH"] += f":{go_bindir}"
+            env['GOMODCACHE'] = tmp_mod_dir
+
+            stdout = subprocess.check_output(["go", "mod", "edit", "-json"], cwd=srctree, env=env, text=True)
+            go_mod = json.loads(stdout)
+            go_import = re.sub(r'/v([0-9]+)$', '', go_mod['Module']['Path'])
+
+            localfilesdir = tempfile.mkdtemp(prefix='recipetool-go-')
+            extravalues.setdefault('extrafiles', {})
+
+            # Write the stub ${BPN}-licenses.inc and ${BPN}-go-mods.inc files
+            basename = "{pn}-licenses.inc"
+            filename = os.path.join(localfilesdir, basename)
+            with open(filename, "w") as f:
+                f.write("# FROM RECIPETOOL\n")
+            extravalues['extrafiles'][f"../{basename}"] = filename
+
+            basename = "{pn}-go-mods.inc"
+            filename = os.path.join(localfilesdir, basename)
+            with open(filename, "w") as f:
+                f.write("# FROM RECIPETOOL\n")
+            extravalues['extrafiles'][f"../{basename}"] = filename
+
+            # Do generic license handling
+            d = bb.data.createCopy(tinfoil.config_data)
+            handle_license_vars(srctree, lines_before, handled, extravalues, d)
+            self.__rewrite_lic_vars(lines_before)
+
+            self.__rewrite_src_uri(lines_before)
+
+            lines_before.append('require ${BPN}-licenses.inc')
+            lines_before.append('require ${BPN}-go-mods.inc')
+            lines_before.append(f'GO_IMPORT = "{go_import}"')
+
+    def __update_lines_before(self, updated, newlines, lines_before):
+        if updated:
+            del lines_before[:]
+            for line in newlines:
+                # Hack to avoid newlines that edit_metadata inserts
+                if line.endswith('\n'):
+                    line = line[:-1]
+                lines_before.append(line)
+        return updated
+
+    def __rewrite_lic_vars(self, lines_before):
+        def varfunc(varname, origvalue, op, newlines):
+            import urllib.parse
+            if varname == 'LIC_FILES_CHKSUM':
+                new_licenses = []
+                licenses = origvalue.split('\\')
+                for license in licenses:
+                    if not license:
+                        logger.warning("No license file was detected for the main module!")
+                        # the license list of the main recipe must be empty
+                        # this can happen for example in case of CLOSED license
+                        # Fall through to complete recipe generation
+                        continue
+                    license = license.strip()
+                    uri, chksum = license.split(';', 1)
+                    url = urllib.parse.urlparse(uri)
+                    new_uri = os.path.join(
+                        url.scheme + "://", "src", "${GO_IMPORT}", url.netloc + url.path) + ";" + chksum
+                    new_licenses.append(new_uri)
+
+                return new_licenses, None, -1, True
+            return origvalue, None, 0, True
+
+        updated, newlines = bb.utils.edit_metadata(
+            lines_before, ['LIC_FILES_CHKSUM'], varfunc)
+        return self.__update_lines_before(updated, newlines, lines_before)
+
+    def __rewrite_src_uri(self, lines_before):
+
+        def varfunc(varname, origvalue, op, newlines):
+            if varname == 'SRC_URI':
+                src_uri = ['git://${GO_IMPORT};protocol=https;nobranch=1;destsuffix=${GO_SRCURI_DESTSUFFIX}']
+                return src_uri, None, -1, True
+            return origvalue, None, 0, True
+
+        updated, newlines = bb.utils.edit_metadata(lines_before, ['SRC_URI'], varfunc)
+        return self.__update_lines_before(updated, newlines, lines_before)
+
+
+def register_recipe_handlers(handlers):
+    handlers.append((GoRecipeHandler(), 60))
diff --git a/scripts/lib/recipetool/create_kmod.py b/scripts/lib/recipetool/create_kmod.py
index 85b5c48e53..cc00106961 100644
--- a/scripts/lib/recipetool/create_kmod.py
+++ b/scripts/lib/recipetool/create_kmod.py
@@ -113,7 +113,7 @@ class KernelModuleRecipeHandler(RecipeHandler):
                 kdirpath, _ = check_target(compile_lines, install=False)
 
             if manual_install or not install_lines:
-                lines_after.append('EXTRA_OEMAKE_append_task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"')
+                lines_after.append('EXTRA_OEMAKE:append:task-install = " -C ${STAGING_KERNEL_DIR} M=${S}"')
             elif install_target and install_target != 'modules_install':
                 lines_after.append('MODULES_INSTALL_TARGET = "install"')
 
diff --git a/scripts/lib/recipetool/create_npm.py b/scripts/lib/recipetool/create_npm.py
index 2bcae91dfa..8c4cdd5234 100644
--- a/scripts/lib/recipetool/create_npm.py
+++ b/scripts/lib/recipetool/create_npm.py
@@ -6,16 +6,20 @@
 """Recipe creation tool - npm module support plugin"""
 
 import json
+import logging
 import os
 import re
 import sys
 import tempfile
 import bb
 from bb.fetch2.npm import NpmEnvironment
+from bb.fetch2.npm import npm_package
 from bb.fetch2.npmsw import foreach_dependencies
+from oe.license_finder import match_licenses, find_license_files
 from recipetool.create import RecipeHandler
-from recipetool.create import guess_license
+from recipetool.create import generate_common_licenses_chksums
 from recipetool.create import split_pkg_licenses
+logger = logging.getLogger('recipetool')
 
 TINFOIL = None
 
@@ -28,15 +32,6 @@ class NpmRecipeHandler(RecipeHandler):
     """Class to handle the npm recipe creation"""
 
     @staticmethod
-    def _npm_name(name):
-        """Generate a Yocto friendly npm name"""
-        name = re.sub("/", "-", name)
-        name = name.lower()
-        name = re.sub(r"[^\-a-z0-9]", "", name)
-        name = name.strip("-")
-        return name
-
-    @staticmethod
     def _get_registry(lines):
         """Get the registry value from the 'npm://registry' url"""
         registry = None
@@ -117,31 +112,71 @@ class NpmRecipeHandler(RecipeHandler):
         """Return the extra license files and the list of packages"""
         licfiles = []
         packages = {}
+        # Licenses from package.json will point to COMMON_LICENSE_DIR so we need
+        # to associate them explicitely to packages for split_pkg_licenses()
+        fallback_licenses = dict()
+
+        def _find_package_licenses(destdir):
+            """Either find license files, or use package.json metadata"""
+            def _get_licenses_from_package_json(package_json):
+                with open(os.path.join(srctree, package_json), "r") as f:
+                    data = json.load(f)
+                    if "license" in data:
+                        licenses = data["license"].split(" ")
+                        licenses = [license.strip("()") for license in licenses if license != "OR" and license != "AND"]
+                        return [], licenses
+                    else:
+                        return [package_json], None
+
+            basedir = os.path.join(srctree, destdir)
+            licfiles = find_license_files(basedir)
+            if len(licfiles) > 0:
+                return licfiles, None
+            else:
+                # A license wasn't found in the package directory, so we'll use the package.json metadata
+                pkg_json = os.path.join(basedir, "package.json")
+                return _get_licenses_from_package_json(pkg_json)
+
+        def _get_package_licenses(destdir, package):
+            (package_licfiles, package_licenses) = _find_package_licenses(destdir)
+            if package_licfiles:
+                licfiles.extend(package_licfiles)
+            else:
+                fallback_licenses[package] = package_licenses
 
-        def _licfiles_append(licfile):
-            """Append 'licfile' to the license files list"""
-            licfilepath = os.path.join(srctree, licfile)
-            licmd5 = bb.utils.md5_file(licfilepath)
-            licfiles.append("file://%s;md5=%s" % (licfile, licmd5))
+        # Handle the dependencies
+        def _handle_dependency(name, params, destdir):
+            deptree = destdir.split('node_modules/')
+            suffix = "-".join([npm_package(dep) for dep in deptree])
+            packages["${PN}" + suffix] = destdir
+            _get_package_licenses(destdir, "${PN}" + suffix)
+
+        with open(shrinkwrap_file, "r") as f:
+            shrinkwrap = json.load(f)
+        foreach_dependencies(shrinkwrap, _handle_dependency, dev)
 
         # Handle the parent package
-        _licfiles_append("package.json")
         packages["${PN}"] = ""
+        _get_package_licenses(srctree, "${PN}")
 
-        # Handle the dependencies
-        def _handle_dependency(name, params, deptree):
-            suffix = "-".join([self._npm_name(dep) for dep in deptree])
-            destdirs = [os.path.join("node_modules", dep) for dep in deptree]
-            destdir = os.path.join(*destdirs)
-            _licfiles_append(os.path.join(destdir, "package.json"))
-            packages["${PN}-" + suffix] = destdir
-
+        return licfiles, packages, fallback_licenses
+    
+    # Handle the peer dependencies   
+    def _handle_peer_dependency(self, shrinkwrap_file):
+        """Check if package has peer dependencies and show warning if it is the case"""
         with open(shrinkwrap_file, "r") as f:
             shrinkwrap = json.load(f)
+        
+        packages = shrinkwrap.get("packages", {})
+        peer_deps = packages.get("", {}).get("peerDependencies", {})
+        
+        for peer_dep in peer_deps:
+            peer_dep_yocto_name = npm_package(peer_dep)
+            bb.warn(peer_dep + " is a peer dependencie of the actual package. " + 
+            "Please add this peer dependencie to the RDEPENDS variable as %s and generate its recipe with devtool"
+            % peer_dep_yocto_name)
 
-        foreach_dependencies(shrinkwrap, _handle_dependency, dev)
 
-        return licfiles, packages
 
     def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
         """Handle the npm recipe creation"""
@@ -160,7 +195,7 @@ class NpmRecipeHandler(RecipeHandler):
         if "name" not in data or "version" not in data:
             return False
 
-        extravalues["PN"] = self._npm_name(data["name"])
+        extravalues["PN"] = npm_package(data["name"])
         extravalues["PV"] = data["version"]
 
         if "description" in data:
@@ -229,7 +264,7 @@ class NpmRecipeHandler(RecipeHandler):
             value = origvalue.replace("version=" + data["version"], "version=${PV}")
             value = value.replace("version=latest", "version=${PV}")
             values = [line.strip() for line in value.strip('\n').splitlines()]
-            if "dependencies" in shrinkwrap:
+            if "dependencies" in shrinkwrap.get("packages", {}).get("", {}):
                 values.append(url_recipe)
             return values, None, 4, False
 
@@ -245,13 +280,19 @@ class NpmRecipeHandler(RecipeHandler):
         fetcher.unpack(srctree)
 
         bb.note("Handling licences ...")
-        (licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev)
-        extravalues["LIC_FILES_CHKSUM"] = licfiles
-        split_pkg_licenses(guess_license(srctree, d), packages, lines_after, [])
+        (licfiles, packages, fallback_licenses) = self._handle_licenses(srctree, shrinkwrap_file, dev)
+        licvalues = match_licenses(licfiles, srctree, d)
+        split_pkg_licenses(licvalues, packages, lines_after, fallback_licenses)
+        fallback_licenses_flat = [license for sublist in fallback_licenses.values() for license in sublist]
+        extravalues["LIC_FILES_CHKSUM"] = generate_common_licenses_chksums(fallback_licenses_flat, d)
+        extravalues["LICENSE"] = fallback_licenses_flat
 
         classes.append("npm")
         handled.append("buildsystem")
 
+        # Check if package has peer dependencies and inform the user
+        self._handle_peer_dependency(shrinkwrap_file)
+
         return True
 
 def register_recipe_handlers(handlers):
diff --git a/scripts/lib/recipetool/setvar.py b/scripts/lib/recipetool/setvar.py
index f8e2ee75fb..b5ad335cae 100644
--- a/scripts/lib/recipetool/setvar.py
+++ b/scripts/lib/recipetool/setvar.py
@@ -49,6 +49,7 @@ def setvar(args):
         for patch in patches:
             for line in patch:
                 sys.stdout.write(line)
+    tinfoil.modified_files()
     return 0