recipetool/create_buildsys_python: refactor code for futur PEP517 addition

In order to prepare the support for pyproject.toml (PEP517 [1]) enabled
projects, refactor the code and move setup.py specific code into a
specific class in order to allow sharing the PythonRecipeHandler class

No functionnal changes expected

[1]: https://peps.python.org/pep-0517/#source-tree

(From OE-Core rev: 2281e93347da4129062cfb40710df03c87c63168)

Signed-off-by: Julien Stephan <jstephan@baylibre.com>
Signed-off-by: Luca Ceresoli <luca.ceresoli@bootlin.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 371 insertions, 349 deletions

diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py
index 502e1dfbc3..69f6f5ca51 100644
--- a/scripts/lib/recipetool/create_buildsys_python.py
+++ b/scripts/lib/recipetool/create_buildsys_python.py
@@ -37,63 +37,8 @@ class PythonRecipeHandler(RecipeHandler):
     assume_provided = ['builtins', 'os.path']
     # Assumes that the host python3 builtin_module_names is sane for target too
     assume_provided = assume_provided + list(sys.builtin_module_names)
+    excluded_fields = []
 
-    bbvar_map = {
-        'Name': 'PN',
-        'Version': 'PV',
-        'Home-page': 'HOMEPAGE',
-        'Summary': 'SUMMARY',
-        'Description': 'DESCRIPTION',
-        'License': 'LICENSE',
-        'Requires': 'RDEPENDS:${PN}',
-        'Provides': 'RPROVIDES:${PN}',
-        'Obsoletes': 'RREPLACES:${PN}',
-    }
-    # PN/PV are already set by recipetool core & desc can be extremely long
-    excluded_fields = [
-        'Description',
-    ]
-    setup_parse_map = {
-        'Url': 'Home-page',
-        'Classifiers': 'Classifier',
-        'Description': 'Summary',
-    }
-    setuparg_map = {
-        'Home-page': 'url',
-        'Classifier': 'classifiers',
-        'Summary': 'description',
-        'Description': 'long-description',
-    }
-    # Values which are lists, used by the setup.py argument based metadata
-    # extraction method, to determine how to process the setup.py output.
-    setuparg_list_fields = [
-        'Classifier',
-        'Requires',
-        'Provides',
-        'Obsoletes',
-        'Platform',
-        'Supported-Platform',
-    ]
-    setuparg_multi_line_values = ['Description']
-    replacements = [
-        ('License', r' +$', ''),
-        ('License', r'^ +', ''),
-        ('License', r' ', '-'),
-        ('License', r'^GNU-', ''),
-        ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''),
-        ('License', r'^UNKNOWN$', ''),
-
-        # Remove currently unhandled version numbers from these variables
-        ('Requires', r' *\([^)]*\)', ''),
-        ('Provides', r' *\([^)]*\)', ''),
-        ('Obsoletes', r' *\([^)]*\)', ''),
-        ('Install-requires', r'^([^><= ]+).*', r'\1'),
-        ('Extras-require', r'^([^><= ]+).*', r'\1'),
-        ('Tests-require', r'^([^><= ]+).*', r'\1'),
-
-        # Remove unhandled dependency on particular features (e.g. foo[PDF])
-        ('Install-requires', r'\[[^\]]+\]$', ''),
-    ]
 
     classifier_license_map = {
         'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
@@ -166,122 +111,34 @@ class PythonRecipeHandler(RecipeHandler):
     def __init__(self):
         pass
 
-    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
-        if 'buildsystem' in handled:
-            return False
-
-        # Check for non-zero size setup.py files
-        setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py'])
-        for fn in setupfiles:
-            if os.path.getsize(fn):
-                break
-        else:
-            return False
-
-        # setup.py is always parsed to get at certain required information, such as
-        # distutils vs setuptools
-        #
-        # If egg info is available, we use it for both its PKG-INFO metadata
-        # and for its requires.txt for install_requires.
-        # If PKG-INFO is available but no egg info is, we use that for metadata in preference to
-        # the parsed setup.py, but use the install_requires info from the
-        # parsed setup.py.
-
-        setupscript = os.path.join(srctree, 'setup.py')
-        try:
-            setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript)
-        except Exception:
-            logger.exception("Failed to parse setup.py")
-            setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], []
-
-        egginfo = glob.glob(os.path.join(srctree, '*.egg-info'))
-        if egginfo:
-            info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO'))
-            requires_txt = os.path.join(egginfo[0], 'requires.txt')
-            if os.path.exists(requires_txt):
-                with codecs.open(requires_txt) as f:
-                    inst_req = []
-                    extras_req = collections.defaultdict(list)
-                    current_feature = None
-                    for line in f.readlines():
-                        line = line.rstrip()
-                        if not line:
-                            continue
-
-                        if line.startswith('['):
-                            # PACKAGECONFIG must not contain expressions or whitespace
-                            line = line.replace(" ", "")
-                            line = line.replace(':', "")
-                            line = line.replace('.', "-dot-")
-                            line = line.replace('"', "")
-                            line = line.replace('<', "-smaller-")
-                            line = line.replace('>', "-bigger-")
-                            line = line.replace('_', "-")
-                            line = line.replace('(', "")
-                            line = line.replace(')', "")
-                            line = line.replace('!', "-not-")
-                            line = line.replace('=', "-equals-")
-                            current_feature = line[1:-1]
-                        elif current_feature:
-                            extras_req[current_feature].append(line)
-                        else:
-                            inst_req.append(line)
-                    info['Install-requires'] = inst_req
-                    info['Extras-require'] = extras_req
-        elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']):
-            info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO'))
-
-            if setup_info:
-                if 'Install-requires' in setup_info:
-                    info['Install-requires'] = setup_info['Install-requires']
-                if 'Extras-require' in setup_info:
-                    info['Extras-require'] = setup_info['Extras-require']
-        else:
-            if setup_info:
-                info = setup_info
-            else:
-                info = self.get_setup_args_info(setupscript)
-
-        # Grab the license value before applying replacements
-        license_str = info.get('License', '').strip()
-
-        self.apply_info_replacements(info)
-
-        if uses_setuptools:
-            classes.append('setuptools3')
-        else:
-            classes.append('distutils3')
-
-        if license_str:
-            for i, line in enumerate(lines_before):
-                if line.startswith('##LICENSE_PLACEHOLDER##'):
-                    lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
-                    break
-
-        if 'Classifier' in info:
-            existing_licenses = info.get('License', '')
-            licenses = []
-            for classifier in info['Classifier']:
-                if classifier in self.classifier_license_map:
-                    license = self.classifier_license_map[classifier]
-                    if license == 'Apache' and 'Apache-2.0' in existing_licenses:
-                        license = 'Apache-2.0'
-                    elif license == 'GPL':
-                        if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
-                            license = 'GPL-2.0'
-                        elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
-                            license = 'GPL-3.0'
-                    elif license == 'LGPL':
-                        if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
-                            license = 'LGPL-2.1'
-                        elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
-                            license = 'LGPL-2.0'
-                        elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
-                            license = 'LGPL-3.0'
-                    licenses.append(license)
-
-            if licenses:
-                info['License'] = ' & '.join(licenses)
+    def handle_classifier_license(self, classifiers, existing_licenses=""):
+
+        licenses = []
+        for classifier in classifiers:
+            if classifier in self.classifier_license_map:
+                license = self.classifier_license_map[classifier]
+                if license == 'Apache' and 'Apache-2.0' in existing_licenses:
+                    license = 'Apache-2.0'
+                elif license == 'GPL':
+                    if 'GPL-2.0' in existing_licenses or 'GPLv2' in existing_licenses:
+                        license = 'GPL-2.0'
+                    elif 'GPL-3.0' in existing_licenses or 'GPLv3' in existing_licenses:
+                        license = 'GPL-3.0'
+                elif license == 'LGPL':
+                    if 'LGPL-2.1' in existing_licenses or 'LGPLv2.1' in existing_licenses:
+                        license = 'LGPL-2.1'
+                    elif 'LGPL-2.0' in existing_licenses or 'LGPLv2' in existing_licenses:
+                        license = 'LGPL-2.0'
+                    elif 'LGPL-3.0' in existing_licenses or 'LGPLv3' in existing_licenses:
+                        license = 'LGPL-3.0'
+                licenses.append(license)
+
+        if licenses:
+            return ' & '.join(licenses)
+
+        return None
+
+    def map_info_to_bbvar(self, info, extravalues):
 
         # Map PKG-INFO & setup.py fields to bitbake variables
         for field, values in info.items():
@@ -305,71 +162,206 @@ class PythonRecipeHandler(RecipeHandler):
             if bbvar not in extravalues and value:
                 extravalues[bbvar] = value
 
-        mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
+    def apply_info_replacements(self, info):
+        if not self.replacements:
+            return
 
-        extras_req = set()
-        if 'Extras-require' in info:
-            extras_req = info['Extras-require']
-            if extras_req:
-                lines_after.append('# The following configs & dependencies are from setuptools extras_require.')
-                lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.')
-                lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
-                lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.')
-                lines_after.append('#')
-                lines_after.append('# Uncomment this line to enable all the optional features.')
-                lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req)))
-                for feature, feature_reqs in extras_req.items():
-                    unmapped_deps.difference_update(feature_reqs)
+        for variable, search, replace in self.replacements:
+            if variable not in info:
+                continue
 
-                    feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs))
-                    lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps)))
+            def replace_value(search, replace, value):
+                if replace is None:
+                    if re.search(search, value):
+                        return None
+                else:
+                    new_value = re.sub(search, replace, value)
+                    if value != new_value:
+                        return new_value
+                return value
 
-        inst_reqs = set()
-        if 'Install-requires' in info:
-            if extras_req:
-                lines_after.append('')
-            inst_reqs = info['Install-requires']
-            if inst_reqs:
-                unmapped_deps.difference_update(inst_reqs)
+            value = info[variable]
+            if isinstance(value, str):
+                new_value = replace_value(search, replace, value)
+                if new_value is None:
+                    del info[variable]
+                elif new_value != value:
+                    info[variable] = new_value
+            elif hasattr(value, 'items'):
+                for dkey, dvalue in list(value.items()):
+                    new_list = []
+                    for pos, a_value in enumerate(dvalue):
+                        new_value = replace_value(search, replace, a_value)
+                        if new_value is not None and new_value != value:
+                            new_list.append(new_value)
 
-                inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
-                lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
-                lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
-                lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
+                    if value != new_list:
+                        value[dkey] = new_list
+            else:
+                new_list = []
+                for pos, a_value in enumerate(value):
+                    new_value = replace_value(search, replace, a_value)
+                    if new_value is not None and new_value != value:
+                        new_list.append(new_value)
 
-        if mapped_deps:
-            name = info.get('Name')
-            if name and name[0] in mapped_deps:
-                # Attempt to avoid self-reference
-                mapped_deps.remove(name[0])
-            mapped_deps -= set(self.excluded_pkgdeps)
-            if inst_reqs or extras_req:
-                lines_after.append('')
-            lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the')
-            lines_after.append('# python sources, and might not be 100% accurate.')
-            lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
+                if value != new_list:
+                    info[variable] = new_list
 
-        unmapped_deps -= set(extensions)
-        unmapped_deps -= set(self.assume_provided)
-        if unmapped_deps:
-            if mapped_deps:
-                lines_after.append('')
-            lines_after.append('# WARNING: We were unable to map the following python package/module')
-            lines_after.append('# dependencies to the bitbake packages which include them:')
-            lines_after.extend('#    {}'.format(d) for d in sorted(unmapped_deps))
 
-        handled.append('buildsystem')
+    def scan_python_dependencies(self, paths):
+        deps = set()
+        try:
+            dep_output = self.run_command(['pythondeps', '-d'] + paths)
+        except (OSError, subprocess.CalledProcessError):
+            pass
+        else:
+            for line in dep_output.splitlines():
+                line = line.rstrip()
+                dep, filename = line.split('\t', 1)
+                if filename.endswith('/setup.py'):
+                    continue
+                deps.add(dep)
 
-    def get_pkginfo(self, pkginfo_fn):
-        msg = email.message_from_file(open(pkginfo_fn, 'r'))
-        msginfo = {}
-        for field in msg.keys():
-            values = msg.get_all(field)
-            if len(values) == 1:
-                msginfo[field] = values[0]
-            else:
-                msginfo[field] = values
-        return msginfo
+        try:
+            provides_output = self.run_command(['pythondeps', '-p'] + paths)
+        except (OSError, subprocess.CalledProcessError):
+            pass
+        else:
+            provides_lines = (l.rstrip() for l in provides_output.splitlines())
+            provides = set(l for l in provides_lines if l and l != 'setup')
+            deps -= provides
+
+        return deps
+
+    def parse_pkgdata_for_python_packages(self):
+        pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
+
+        ldata = tinfoil.config_data.createCopy()
+        bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True)
+        python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
+
+        dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
+        python_dirs = [python_sitedir + os.sep,
+                       os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
+                       os.path.dirname(python_sitedir) + os.sep]
+        packages = {}
+        for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
+            files_info = None
+            with open(pkgdatafile, 'r') as f:
+                for line in f.readlines():
+                    field, value = line.split(': ', 1)
+                    if field.startswith('FILES_INFO'):
+                        files_info = ast.literal_eval(value)
+                        break
+                else:
+                    continue
+
+            for fn in files_info:
+                for suffix in importlib.machinery.all_suffixes():
+                    if fn.endswith(suffix):
+                        break
+                else:
+                    continue
+
+                if fn.startswith(dynload_dir + os.sep):
+                    if '/.debug/' in fn:
+                        continue
+                    base = os.path.basename(fn)
+                    provided = base.split('.', 1)[0]
+                    packages[provided] = os.path.basename(pkgdatafile)
+                    continue
+
+                for python_dir in python_dirs:
+                    if fn.startswith(python_dir):
+                        relpath = fn[len(python_dir):]
+                        relstart, _, relremaining = relpath.partition(os.sep)
+                        if relstart.endswith('.egg'):
+                            relpath = relremaining
+                        base, _ = os.path.splitext(relpath)
+
+                        if '/.debug/' in base:
+                            continue
+                        if os.path.basename(base) == '__init__':
+                            base = os.path.dirname(base)
+                        base = base.replace(os.sep + os.sep, os.sep)
+                        provided = base.replace(os.sep, '.')
+                        packages[provided] = os.path.basename(pkgdatafile)
+        return packages
+
+    @classmethod
+    def run_command(cls, cmd, **popenargs):
+        if 'stderr' not in popenargs:
+            popenargs['stderr'] = subprocess.STDOUT
+        try:
+            return subprocess.check_output(cmd, **popenargs).decode('utf-8')
+        except OSError as exc:
+            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
+            raise
+        except subprocess.CalledProcessError as exc:
+            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
+            raise
+
+class PythonSetupPyRecipeHandler(PythonRecipeHandler):
+    bbvar_map = {
+        'Name': 'PN',
+        'Version': 'PV',
+        'Home-page': 'HOMEPAGE',
+        'Summary': 'SUMMARY',
+        'Description': 'DESCRIPTION',
+        'License': 'LICENSE',
+        'Requires': 'RDEPENDS:${PN}',
+        'Provides': 'RPROVIDES:${PN}',
+        'Obsoletes': 'RREPLACES:${PN}',
+    }
+    # PN/PV are already set by recipetool core & desc can be extremely long
+    excluded_fields = [
+        'Description',
+    ]
+    setup_parse_map = {
+        'Url': 'Home-page',
+        'Classifiers': 'Classifier',
+        'Description': 'Summary',
+    }
+    setuparg_map = {
+        'Home-page': 'url',
+        'Classifier': 'classifiers',
+        'Summary': 'description',
+        'Description': 'long-description',
+    }
+    # Values which are lists, used by the setup.py argument based metadata
+    # extraction method, to determine how to process the setup.py output.
+    setuparg_list_fields = [
+        'Classifier',
+        'Requires',
+        'Provides',
+        'Obsoletes',
+        'Platform',
+        'Supported-Platform',
+    ]
+    setuparg_multi_line_values = ['Description']
+
+    replacements = [
+        ('License', r' +$', ''),
+        ('License', r'^ +', ''),
+        ('License', r' ', '-'),
+        ('License', r'^GNU-', ''),
+        ('License', r'-[Ll]icen[cs]e(,?-[Vv]ersion)?', ''),
+        ('License', r'^UNKNOWN$', ''),
+
+        # Remove currently unhandled version numbers from these variables
+        ('Requires', r' *\([^)]*\)', ''),
+        ('Provides', r' *\([^)]*\)', ''),
+        ('Obsoletes', r' *\([^)]*\)', ''),
+        ('Install-requires', r'^([^><= ]+).*', r'\1'),
+        ('Extras-require', r'^([^><= ]+).*', r'\1'),
+        ('Tests-require', r'^([^><= ]+).*', r'\1'),
+
+        # Remove unhandled dependency on particular features (e.g. foo[PDF])
+        ('Install-requires', r'\[[^\]]+\]$', ''),
+    ]
+
+    def __init__(self):
+        pass
 
     def parse_setup_py(self, setupscript='./setup.py'):
         with codecs.open(setupscript) as f:
@@ -445,47 +437,16 @@ class PythonRecipeHandler(RecipeHandler):
                 info[fields[lineno]] = line
         return info
 
-    def apply_info_replacements(self, info):
-        for variable, search, replace in self.replacements:
-            if variable not in info:
-                continue
-
-            def replace_value(search, replace, value):
-                if replace is None:
-                    if re.search(search, value):
-                        return None
-                else:
-                    new_value = re.sub(search, replace, value)
-                    if value != new_value:
-                        return new_value
-                return value
-
-            value = info[variable]
-            if isinstance(value, str):
-                new_value = replace_value(search, replace, value)
-                if new_value is None:
-                    del info[variable]
-                elif new_value != value:
-                    info[variable] = new_value
-            elif hasattr(value, 'items'):
-                for dkey, dvalue in list(value.items()):
-                    new_list = []
-                    for pos, a_value in enumerate(dvalue):
-                        new_value = replace_value(search, replace, a_value)
-                        if new_value is not None and new_value != value:
-                            new_list.append(new_value)
-
-                    if value != new_list:
-                        value[dkey] = new_list
+    def get_pkginfo(self, pkginfo_fn):
+        msg = email.message_from_file(open(pkginfo_fn, 'r'))
+        msginfo = {}
+        for field in msg.keys():
+            values = msg.get_all(field)
+            if len(values) == 1:
+                msginfo[field] = values[0]
             else:
-                new_list = []
-                for pos, a_value in enumerate(value):
-                    new_value = replace_value(search, replace, a_value)
-                    if new_value is not None and new_value != value:
-                        new_list.append(new_value)
-
-                if value != new_list:
-                    info[variable] = new_list
+                msginfo[field] = values
+        return msginfo
 
     def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
         if 'Package-dir' in setup_info:
@@ -540,99 +501,160 @@ class PythonRecipeHandler(RecipeHandler):
                 unmapped_deps.add(dep)
         return mapped_deps, unmapped_deps
 
-    def scan_python_dependencies(self, paths):
-        deps = set()
-        try:
-            dep_output = self.run_command(['pythondeps', '-d'] + paths)
-        except (OSError, subprocess.CalledProcessError):
-            pass
+    def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
+
+        if 'buildsystem' in handled:
+            return False
+
+        # Check for non-zero size setup.py files
+        setupfiles = RecipeHandler.checkfiles(srctree, ['setup.py'])
+        for fn in setupfiles:
+            if os.path.getsize(fn):
+                break
         else:
-            for line in dep_output.splitlines():
-                line = line.rstrip()
-                dep, filename = line.split('\t', 1)
-                if filename.endswith('/setup.py'):
-                    continue
-                deps.add(dep)
+            return False
 
+        # setup.py is always parsed to get at certain required information, such as
+        # distutils vs setuptools
+        #
+        # If egg info is available, we use it for both its PKG-INFO metadata
+        # and for its requires.txt for install_requires.
+        # If PKG-INFO is available but no egg info is, we use that for metadata in preference to
+        # the parsed setup.py, but use the install_requires info from the
+        # parsed setup.py.
+
+        setupscript = os.path.join(srctree, 'setup.py')
         try:
-            provides_output = self.run_command(['pythondeps', '-p'] + paths)
-        except (OSError, subprocess.CalledProcessError):
-            pass
+            setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript)
+        except Exception:
+            logger.exception("Failed to parse setup.py")
+            setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], []
+
+        egginfo = glob.glob(os.path.join(srctree, '*.egg-info'))
+        if egginfo:
+            info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO'))
+            requires_txt = os.path.join(egginfo[0], 'requires.txt')
+            if os.path.exists(requires_txt):
+                with codecs.open(requires_txt) as f:
+                    inst_req = []
+                    extras_req = collections.defaultdict(list)
+                    current_feature = None
+                    for line in f.readlines():
+                        line = line.rstrip()
+                        if not line:
+                            continue
+
+                        if line.startswith('['):
+                            # PACKAGECONFIG must not contain expressions or whitespace
+                            line = line.replace(" ", "")
+                            line = line.replace(':', "")
+                            line = line.replace('.', "-dot-")
+                            line = line.replace('"', "")
+                            line = line.replace('<', "-smaller-")
+                            line = line.replace('>', "-bigger-")
+                            line = line.replace('_', "-")
+                            line = line.replace('(', "")
+                            line = line.replace(')', "")
+                            line = line.replace('!', "-not-")
+                            line = line.replace('=', "-equals-")
+                            current_feature = line[1:-1]
+                        elif current_feature:
+                            extras_req[current_feature].append(line)
+                        else:
+                            inst_req.append(line)
+                    info['Install-requires'] = inst_req
+                    info['Extras-require'] = extras_req
+        elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']):
+            info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO'))
+
+            if setup_info:
+                if 'Install-requires' in setup_info:
+                    info['Install-requires'] = setup_info['Install-requires']
+                if 'Extras-require' in setup_info:
+                    info['Extras-require'] = setup_info['Extras-require']
         else:
-            provides_lines = (l.rstrip() for l in provides_output.splitlines())
-            provides = set(l for l in provides_lines if l and l != 'setup')
-            deps -= provides
+            if setup_info:
+                info = setup_info
+            else:
+                info = self.get_setup_args_info(setupscript)
 
-        return deps
+        # Grab the license value before applying replacements
+        license_str = info.get('License', '').strip()
 
-    def parse_pkgdata_for_python_packages(self):
-        pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR')
+        self.apply_info_replacements(info)
 
-        ldata = tinfoil.config_data.createCopy()
-        bb.parse.handle('classes-recipe/python3-dir.bbclass', ldata, True)
-        python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR')
+        if uses_setuptools:
+            classes.append('setuptools3')
+        else:
+            classes.append('distutils3')
 
-        dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
-        python_dirs = [python_sitedir + os.sep,
-                       os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
-                       os.path.dirname(python_sitedir) + os.sep]
-        packages = {}
-        for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
-            files_info = None
-            with open(pkgdatafile, 'r') as f:
-                for line in f.readlines():
-                    field, value = line.split(': ', 1)
-                    if field.startswith('FILES_INFO'):
-                        files_info = ast.literal_eval(value)
-                        break
-                else:
-                    continue
+        if license_str:
+            for i, line in enumerate(lines_before):
+                if line.startswith('##LICENSE_PLACEHOLDER##'):
+                    lines_before.insert(i, '# NOTE: License in setup.py/PKGINFO is: %s' % license_str)
+                    break
 
-            for fn in files_info:
-                for suffix in importlib.machinery.all_suffixes():
-                    if fn.endswith(suffix):
-                        break
-                else:
-                    continue
+        if 'Classifier' in info:
+            license = self.handle_classifier_license(info['Classifier'], info.get('License', ''))
+            if license:
+                info['License'] = license
 
-                if fn.startswith(dynload_dir + os.sep):
-                    if '/.debug/' in fn:
-                        continue
-                    base = os.path.basename(fn)
-                    provided = base.split('.', 1)[0]
-                    packages[provided] = os.path.basename(pkgdatafile)
-                    continue
+        self.map_info_to_bbvar(info, extravalues)
 
-                for python_dir in python_dirs:
-                    if fn.startswith(python_dir):
-                        relpath = fn[len(python_dir):]
-                        relstart, _, relremaining = relpath.partition(os.sep)
-                        if relstart.endswith('.egg'):
-                            relpath = relremaining
-                        base, _ = os.path.splitext(relpath)
+        mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
 
-                        if '/.debug/' in base:
-                            continue
-                        if os.path.basename(base) == '__init__':
-                            base = os.path.dirname(base)
-                        base = base.replace(os.sep + os.sep, os.sep)
-                        provided = base.replace(os.sep, '.')
-                        packages[provided] = os.path.basename(pkgdatafile)
-        return packages
+        extras_req = set()
+        if 'Extras-require' in info:
+            extras_req = info['Extras-require']
+            if extras_req:
+                lines_after.append('# The following configs & dependencies are from setuptools extras_require.')
+                lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.')
+                lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
+                lines_after.append('# The configs are might not correct, since PACKAGECONFIG does not support expressions as may used in requires.txt - they are just replaced by text.')
+                lines_after.append('#')
+                lines_after.append('# Uncomment this line to enable all the optional features.')
+                lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req)))
+                for feature, feature_reqs in extras_req.items():
+                    unmapped_deps.difference_update(feature_reqs)
 
-    @classmethod
-    def run_command(cls, cmd, **popenargs):
-        if 'stderr' not in popenargs:
-            popenargs['stderr'] = subprocess.STDOUT
-        try:
-            return subprocess.check_output(cmd, **popenargs).decode('utf-8')
-        except OSError as exc:
-            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
-            raise
-        except subprocess.CalledProcessError as exc:
-            logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
-            raise
+                    feature_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(feature_reqs))
+                    lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps)))
 
+        inst_reqs = set()
+        if 'Install-requires' in info:
+            if extras_req:
+                lines_after.append('')
+            inst_reqs = info['Install-requires']
+            if inst_reqs:
+                unmapped_deps.difference_update(inst_reqs)
+
+                inst_req_deps = ('python3-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
+                lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
+                lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
+                lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
+
+        if mapped_deps:
+            name = info.get('Name')
+            if name and name[0] in mapped_deps:
+                # Attempt to avoid self-reference
+                mapped_deps.remove(name[0])
+            mapped_deps -= set(self.excluded_pkgdeps)
+            if inst_reqs or extras_req:
+                lines_after.append('')
+            lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the')
+            lines_after.append('# python sources, and might not be 100% accurate.')
+            lines_after.append('RDEPENDS:${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
+
+        unmapped_deps -= set(extensions)
+        unmapped_deps -= set(self.assume_provided)
+        if unmapped_deps:
+            if mapped_deps:
+                lines_after.append('')
+            lines_after.append('# WARNING: We were unable to map the following python package/module')
+            lines_after.append('# dependencies to the bitbake packages which include them:')
+            lines_after.extend('#    {}'.format(d) for d in sorted(unmapped_deps))
+
+        handled.append('buildsystem')
 
 def gather_setup_info(fileobj):
     parsed = ast.parse(fileobj.read(), fileobj.name)
@@ -748,4 +770,4 @@ def has_non_literals(value):
 
 def register_recipe_handlers(handlers):
     # We need to make sure this is ahead of the makefile fallback handler
-    handlers.append((PythonRecipeHandler(), 70))
+    handlers.append((PythonSetupPyRecipeHandler(), 70))