diff options
Diffstat (limited to 'scripts/lib/recipetool/create_buildsys_python.py')
-rw-r--r-- | scripts/lib/recipetool/create_buildsys_python.py | 160 |
1 files changed, 160 insertions, 0 deletions
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py index 9e4e1ebd4c..f4f4212559 100644 --- a/scripts/lib/recipetool/create_buildsys_python.py +++ b/scripts/lib/recipetool/create_buildsys_python.py | |||
@@ -41,6 +41,13 @@ def tinfoil_init(instance): | |||
41 | 41 | ||
42 | 42 | ||
43 | class PythonRecipeHandler(RecipeHandler): | 43 | class PythonRecipeHandler(RecipeHandler): |
44 | base_pkgdeps = ['python-core'] | ||
45 | excluded_pkgdeps = ['python-dbg'] | ||
46 | # os.path is provided by python-core | ||
47 | assume_provided = ['builtins', 'os.path'] | ||
48 | # Assumes that the host python builtin_module_names is sane for target too | ||
49 | assume_provided = assume_provided + list(sys.builtin_module_names) | ||
50 | |||
44 | bbvar_map = { | 51 | bbvar_map = { |
45 | 'Name': 'PN', | 52 | 'Name': 'PN', |
46 | 'Version': 'PV', | 53 | 'Version': 'PV', |
@@ -273,6 +280,8 @@ class PythonRecipeHandler(RecipeHandler): | |||
273 | mdinfo.append('{} = "{}"'.format(k, v)) | 280 | mdinfo.append('{} = "{}"'.format(k, v)) |
274 | lines_before[src_uri_line-1:src_uri_line-1] = mdinfo | 281 | lines_before[src_uri_line-1:src_uri_line-1] = mdinfo |
275 | 282 | ||
283 | mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) | ||
284 | |||
276 | extras_req = set() | 285 | extras_req = set() |
277 | if 'Extras-require' in info: | 286 | if 'Extras-require' in info: |
278 | extras_req = info['Extras-require'] | 287 | extras_req = info['Extras-require'] |
@@ -284,6 +293,8 @@ class PythonRecipeHandler(RecipeHandler): | |||
284 | lines_after.append('# Uncomment this line to enable all the optional features.') | 293 | lines_after.append('# Uncomment this line to enable all the optional features.') |
285 | lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req.iterkeys()))) | 294 | lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req.iterkeys()))) |
286 | for feature, feature_reqs in extras_req.iteritems(): | 295 | for feature, feature_reqs in extras_req.iteritems(): |
296 | unmapped_deps.difference_update(feature_reqs) | ||
297 | |||
287 | feature_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) | 298 | feature_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) |
288 | lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) | 299 | lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) |
289 | 300 | ||
@@ -293,11 +304,34 @@ class PythonRecipeHandler(RecipeHandler): | |||
293 | lines_after.append('') | 304 | lines_after.append('') |
294 | inst_reqs = info['Install-requires'] | 305 | inst_reqs = info['Install-requires'] |
295 | if inst_reqs: | 306 | if inst_reqs: |
307 | unmapped_deps.difference_update(inst_reqs) | ||
308 | |||
296 | inst_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) | 309 | inst_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) |
297 | lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') | 310 | lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') |
298 | lines_after.append('# upstream names may not correspond exactly to bitbake package names.') | 311 | lines_after.append('# upstream names may not correspond exactly to bitbake package names.') |
299 | lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps))) | 312 | lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps))) |
300 | 313 | ||
314 | if mapped_deps: | ||
315 | name = info.get('Name') | ||
316 | if name and name[0] in mapped_deps: | ||
317 | # Attempt to avoid self-reference | ||
318 | mapped_deps.remove(name[0]) | ||
319 | mapped_deps -= set(self.excluded_pkgdeps) | ||
320 | if inst_reqs or extras_req: | ||
321 | lines_after.append('') | ||
322 | lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') | ||
323 | lines_after.append('# python sources, and might not be 100% accurate.') | ||
324 | lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) | ||
325 | |||
326 | unmapped_deps -= set(extensions) | ||
327 | unmapped_deps -= set(self.assume_provided) | ||
328 | if unmapped_deps: | ||
329 | if mapped_deps: | ||
330 | lines_after.append('') | ||
331 | lines_after.append('# WARNING: We were unable to map the following python package/module') | ||
332 | lines_after.append('# dependencies to the bitbake packages which include them:') | ||
333 | lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps)) | ||
334 | |||
301 | handled.append('buildsystem') | 335 | handled.append('buildsystem') |
302 | 336 | ||
303 | def get_pkginfo(self, pkginfo_fn): | 337 | def get_pkginfo(self, pkginfo_fn): |
@@ -425,6 +459,132 @@ class PythonRecipeHandler(RecipeHandler): | |||
425 | if value != new_list: | 459 | if value != new_list: |
426 | info[variable] = new_list | 460 | info[variable] = new_list |
427 | 461 | ||
462 | def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals): | ||
463 | if 'Package-dir' in setup_info: | ||
464 | package_dir = setup_info['Package-dir'] | ||
465 | else: | ||
466 | package_dir = {} | ||
467 | |||
468 | class PackageDir(distutils.command.build_py.build_py): | ||
469 | def __init__(self, package_dir): | ||
470 | self.package_dir = package_dir | ||
471 | |||
472 | pd = PackageDir(package_dir) | ||
473 | to_scan = [] | ||
474 | if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']): | ||
475 | if 'Py-modules' in setup_info: | ||
476 | for module in setup_info['Py-modules']: | ||
477 | try: | ||
478 | package, module = module.rsplit('.', 1) | ||
479 | except ValueError: | ||
480 | package, module = '.', module | ||
481 | module_path = os.path.join(pd.get_package_dir(package), module + '.py') | ||
482 | to_scan.append(module_path) | ||
483 | |||
484 | if 'Packages' in setup_info: | ||
485 | for package in setup_info['Packages']: | ||
486 | to_scan.append(pd.get_package_dir(package)) | ||
487 | |||
488 | if 'Scripts' in setup_info: | ||
489 | to_scan.extend(setup_info['Scripts']) | ||
490 | else: | ||
491 | logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.") | ||
492 | |||
493 | if not to_scan: | ||
494 | to_scan = ['.'] | ||
495 | |||
496 | logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan)) | ||
497 | |||
498 | provided_packages = self.parse_pkgdata_for_python_packages() | ||
499 | scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan]) | ||
500 | mapped_deps, unmapped_deps = set(self.base_pkgdeps), set() | ||
501 | for dep in scanned_deps: | ||
502 | mapped = provided_packages.get(dep) | ||
503 | if mapped: | ||
504 | mapped_deps.add(mapped) | ||
505 | else: | ||
506 | unmapped_deps.add(dep) | ||
507 | return mapped_deps, unmapped_deps | ||
508 | |||
509 | def scan_python_dependencies(self, paths): | ||
510 | deps = set() | ||
511 | try: | ||
512 | dep_output = self.run_command(['pythondeps', '-d'] + paths) | ||
513 | except (OSError, subprocess.CalledProcessError): | ||
514 | pass | ||
515 | else: | ||
516 | for line in dep_output.splitlines(): | ||
517 | line = line.rstrip() | ||
518 | dep, filename = line.split('\t', 1) | ||
519 | if filename.endswith('/setup.py'): | ||
520 | continue | ||
521 | deps.add(dep) | ||
522 | |||
523 | try: | ||
524 | provides_output = self.run_command(['pythondeps', '-p'] + paths) | ||
525 | except (OSError, subprocess.CalledProcessError): | ||
526 | pass | ||
527 | else: | ||
528 | provides_lines = (l.rstrip() for l in provides_output.splitlines()) | ||
529 | provides = set(l for l in provides_lines if l and l != 'setup') | ||
530 | deps -= provides | ||
531 | |||
532 | return deps | ||
533 | |||
534 | def parse_pkgdata_for_python_packages(self): | ||
535 | suffixes = [t[0] for t in imp.get_suffixes()] | ||
536 | pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) | ||
537 | |||
538 | ldata = tinfoil.config_data.createCopy() | ||
539 | bb.parse.handle('classes/python-dir.bbclass', ldata, True) | ||
540 | python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR', True) | ||
541 | |||
542 | dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') | ||
543 | python_dirs = [python_sitedir + os.sep, | ||
544 | os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, | ||
545 | os.path.dirname(python_sitedir) + os.sep] | ||
546 | packages = {} | ||
547 | for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): | ||
548 | files_info = None | ||
549 | with open(pkgdatafile, 'r') as f: | ||
550 | for line in f.readlines(): | ||
551 | field, value = line.split(': ', 1) | ||
552 | if field == 'FILES_INFO': | ||
553 | files_info = ast.literal_eval(value) | ||
554 | break | ||
555 | else: | ||
556 | continue | ||
557 | |||
558 | for fn in files_info.iterkeys(): | ||
559 | for suffix in suffixes: | ||
560 | if fn.endswith(suffix): | ||
561 | break | ||
562 | else: | ||
563 | continue | ||
564 | |||
565 | if fn.startswith(dynload_dir + os.sep): | ||
566 | base = os.path.basename(fn) | ||
567 | provided = base.split('.', 1)[0] | ||
568 | packages[provided] = os.path.basename(pkgdatafile) | ||
569 | continue | ||
570 | |||
571 | for python_dir in python_dirs: | ||
572 | if fn.startswith(python_dir): | ||
573 | relpath = fn[len(python_dir):] | ||
574 | relstart, _, relremaining = relpath.partition(os.sep) | ||
575 | if relstart.endswith('.egg'): | ||
576 | relpath = relremaining | ||
577 | base, _ = os.path.splitext(relpath) | ||
578 | |||
579 | if '/.debug/' in base: | ||
580 | continue | ||
581 | if os.path.basename(base) == '__init__': | ||
582 | base = os.path.dirname(base) | ||
583 | base = base.replace(os.sep + os.sep, os.sep) | ||
584 | provided = base.replace(os.sep, '.') | ||
585 | packages[provided] = os.path.basename(pkgdatafile) | ||
586 | return packages | ||
587 | |||
428 | @classmethod | 588 | @classmethod |
429 | def run_command(cls, cmd, **popenargs): | 589 | def run_command(cls, cmd, **popenargs): |
430 | if 'stderr' not in popenargs: | 590 | if 'stderr' not in popenargs: |