summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorChristopher Larson <kergoth@gmail.com>2015-01-19 11:52:31 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-01-23 11:36:28 +0000
commit1b7b58ac97aafc2c88038bfb1f2301c4013f6761 (patch)
tree1e8a8ca8b148832c912643cb3a49fb9c0127c9c5
parente490d79fb740319b35227eb66968be852a781036 (diff)
downloadpoky-1b7b58ac97aafc2c88038bfb1f2301c4013f6761.tar.gz
recipetool: add python dependency scanning support
This uses a standalone python script named `pythondeps` which now lives in scripts. It supports scanning for provided packages and imported modules/packages, the latter via the python ast. It's not perfect, and obviously conditional imports and try/except import blocks are handled naively, listing all the imports even if they aren't all used at once, but it gives the user a solid starting point for the recipe. Currently `python_dir` from setup.py isn't being handled in an ideal way. This is easily seen when testing the python-async package. There, the root of the project is the async package, so the root has __init__.py and friends, and the python provides scanning currently just assumes the basedir of that dir is the package name in this case, which is not correct. Forthcoming patches will resolve this. (From OE-Core rev: cb093aca3b78f130dc7da820a8710342a12d1231) Signed-off-by: Christopher Larson <kergoth@gmail.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--scripts/lib/recipetool/create_buildsys_python.py160
-rwxr-xr-xscripts/pythondeps250
2 files changed, 410 insertions, 0 deletions
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py
index 9e4e1ebd4c..f4f4212559 100644
--- a/scripts/lib/recipetool/create_buildsys_python.py
+++ b/scripts/lib/recipetool/create_buildsys_python.py
@@ -41,6 +41,13 @@ def tinfoil_init(instance):
41 41
42 42
43class PythonRecipeHandler(RecipeHandler): 43class PythonRecipeHandler(RecipeHandler):
44 base_pkgdeps = ['python-core']
45 excluded_pkgdeps = ['python-dbg']
46 # os.path is provided by python-core
47 assume_provided = ['builtins', 'os.path']
48 # Assumes that the host python builtin_module_names is sane for target too
49 assume_provided = assume_provided + list(sys.builtin_module_names)
50
44 bbvar_map = { 51 bbvar_map = {
45 'Name': 'PN', 52 'Name': 'PN',
46 'Version': 'PV', 53 'Version': 'PV',
@@ -273,6 +280,8 @@ class PythonRecipeHandler(RecipeHandler):
273 mdinfo.append('{} = "{}"'.format(k, v)) 280 mdinfo.append('{} = "{}"'.format(k, v))
274 lines_before[src_uri_line-1:src_uri_line-1] = mdinfo 281 lines_before[src_uri_line-1:src_uri_line-1] = mdinfo
275 282
283 mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals)
284
276 extras_req = set() 285 extras_req = set()
277 if 'Extras-require' in info: 286 if 'Extras-require' in info:
278 extras_req = info['Extras-require'] 287 extras_req = info['Extras-require']
@@ -284,6 +293,8 @@ class PythonRecipeHandler(RecipeHandler):
284 lines_after.append('# Uncomment this line to enable all the optional features.') 293 lines_after.append('# Uncomment this line to enable all the optional features.')
285 lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req.iterkeys()))) 294 lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req.iterkeys())))
286 for feature, feature_reqs in extras_req.iteritems(): 295 for feature, feature_reqs in extras_req.iteritems():
296 unmapped_deps.difference_update(feature_reqs)
297
287 feature_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) 298 feature_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(feature_reqs))
288 lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) 299 lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps)))
289 300
@@ -293,11 +304,34 @@ class PythonRecipeHandler(RecipeHandler):
293 lines_after.append('') 304 lines_after.append('')
294 inst_reqs = info['Install-requires'] 305 inst_reqs = info['Install-requires']
295 if inst_reqs: 306 if inst_reqs:
307 unmapped_deps.difference_update(inst_reqs)
308
296 inst_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) 309 inst_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
297 lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') 310 lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
298 lines_after.append('# upstream names may not correspond exactly to bitbake package names.') 311 lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
299 lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps))) 312 lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
300 313
314 if mapped_deps:
315 name = info.get('Name')
316 if name and name[0] in mapped_deps:
317 # Attempt to avoid self-reference
318 mapped_deps.remove(name[0])
319 mapped_deps -= set(self.excluded_pkgdeps)
320 if inst_reqs or extras_req:
321 lines_after.append('')
322 lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the')
323 lines_after.append('# python sources, and might not be 100% accurate.')
324 lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps))))
325
326 unmapped_deps -= set(extensions)
327 unmapped_deps -= set(self.assume_provided)
328 if unmapped_deps:
329 if mapped_deps:
330 lines_after.append('')
331 lines_after.append('# WARNING: We were unable to map the following python package/module')
332 lines_after.append('# dependencies to the bitbake packages which include them:')
333 lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps))
334
301 handled.append('buildsystem') 335 handled.append('buildsystem')
302 336
303 def get_pkginfo(self, pkginfo_fn): 337 def get_pkginfo(self, pkginfo_fn):
@@ -425,6 +459,132 @@ class PythonRecipeHandler(RecipeHandler):
425 if value != new_list: 459 if value != new_list:
426 info[variable] = new_list 460 info[variable] = new_list
427 461
462 def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals):
463 if 'Package-dir' in setup_info:
464 package_dir = setup_info['Package-dir']
465 else:
466 package_dir = {}
467
468 class PackageDir(distutils.command.build_py.build_py):
469 def __init__(self, package_dir):
470 self.package_dir = package_dir
471
472 pd = PackageDir(package_dir)
473 to_scan = []
474 if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']):
475 if 'Py-modules' in setup_info:
476 for module in setup_info['Py-modules']:
477 try:
478 package, module = module.rsplit('.', 1)
479 except ValueError:
480 package, module = '.', module
481 module_path = os.path.join(pd.get_package_dir(package), module + '.py')
482 to_scan.append(module_path)
483
484 if 'Packages' in setup_info:
485 for package in setup_info['Packages']:
486 to_scan.append(pd.get_package_dir(package))
487
488 if 'Scripts' in setup_info:
489 to_scan.extend(setup_info['Scripts'])
490 else:
491 logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.")
492
493 if not to_scan:
494 to_scan = ['.']
495
496 logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan))
497
498 provided_packages = self.parse_pkgdata_for_python_packages()
499 scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan])
500 mapped_deps, unmapped_deps = set(self.base_pkgdeps), set()
501 for dep in scanned_deps:
502 mapped = provided_packages.get(dep)
503 if mapped:
504 mapped_deps.add(mapped)
505 else:
506 unmapped_deps.add(dep)
507 return mapped_deps, unmapped_deps
508
509 def scan_python_dependencies(self, paths):
510 deps = set()
511 try:
512 dep_output = self.run_command(['pythondeps', '-d'] + paths)
513 except (OSError, subprocess.CalledProcessError):
514 pass
515 else:
516 for line in dep_output.splitlines():
517 line = line.rstrip()
518 dep, filename = line.split('\t', 1)
519 if filename.endswith('/setup.py'):
520 continue
521 deps.add(dep)
522
523 try:
524 provides_output = self.run_command(['pythondeps', '-p'] + paths)
525 except (OSError, subprocess.CalledProcessError):
526 pass
527 else:
528 provides_lines = (l.rstrip() for l in provides_output.splitlines())
529 provides = set(l for l in provides_lines if l and l != 'setup')
530 deps -= provides
531
532 return deps
533
534 def parse_pkgdata_for_python_packages(self):
535 suffixes = [t[0] for t in imp.get_suffixes()]
536 pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
537
538 ldata = tinfoil.config_data.createCopy()
539 bb.parse.handle('classes/python-dir.bbclass', ldata, True)
540 python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR', True)
541
542 dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload')
543 python_dirs = [python_sitedir + os.sep,
544 os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep,
545 os.path.dirname(python_sitedir) + os.sep]
546 packages = {}
547 for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)):
548 files_info = None
549 with open(pkgdatafile, 'r') as f:
550 for line in f.readlines():
551 field, value = line.split(': ', 1)
552 if field == 'FILES_INFO':
553 files_info = ast.literal_eval(value)
554 break
555 else:
556 continue
557
558 for fn in files_info.iterkeys():
559 for suffix in suffixes:
560 if fn.endswith(suffix):
561 break
562 else:
563 continue
564
565 if fn.startswith(dynload_dir + os.sep):
566 base = os.path.basename(fn)
567 provided = base.split('.', 1)[0]
568 packages[provided] = os.path.basename(pkgdatafile)
569 continue
570
571 for python_dir in python_dirs:
572 if fn.startswith(python_dir):
573 relpath = fn[len(python_dir):]
574 relstart, _, relremaining = relpath.partition(os.sep)
575 if relstart.endswith('.egg'):
576 relpath = relremaining
577 base, _ = os.path.splitext(relpath)
578
579 if '/.debug/' in base:
580 continue
581 if os.path.basename(base) == '__init__':
582 base = os.path.dirname(base)
583 base = base.replace(os.sep + os.sep, os.sep)
584 provided = base.replace(os.sep, '.')
585 packages[provided] = os.path.basename(pkgdatafile)
586 return packages
587
428 @classmethod 588 @classmethod
429 def run_command(cls, cmd, **popenargs): 589 def run_command(cls, cmd, **popenargs):
430 if 'stderr' not in popenargs: 590 if 'stderr' not in popenargs:
diff --git a/scripts/pythondeps b/scripts/pythondeps
new file mode 100755
index 0000000000..ff92e747ed
--- /dev/null
+++ b/scripts/pythondeps
@@ -0,0 +1,250 @@
1#!/usr/bin/env python
2#
3# Determine dependencies of python scripts or available python modules in a search path.
4#
5# Given the -d argument and a filename/filenames, returns the modules imported by those files.
6# Given the -d argument and a directory/directories, recurses to find all
7# python packages and modules, returns the modules imported by these.
8# Given the -p argument and a path or paths, scans that path for available python modules/packages.
9
10import argparse
11import ast
12import imp
13import logging
14import os.path
15import sys
16
17
18logger = logging.getLogger('pythondeps')
19
20suffixes = []
21for triple in imp.get_suffixes():
22 suffixes.append(triple[0])
23
24
25class PythonDepError(Exception):
26 pass
27
28
29class DependError(PythonDepError):
30 def __init__(self, path, error):
31 self.path = path
32 self.error = error
33 PythonDepError.__init__(self, error)
34
35 def __str__(self):
36 return "Failure determining dependencies of {}: {}".format(self.path, self.error)
37
38
39class ImportVisitor(ast.NodeVisitor):
40 def __init__(self):
41 self.imports = set()
42 self.importsfrom = []
43
44 def visit_Import(self, node):
45 for alias in node.names:
46 self.imports.add(alias.name)
47
48 def visit_ImportFrom(self, node):
49 self.importsfrom.append((node.module, [a.name for a in node.names], node.level))
50
51
52def walk_up(path):
53 while path:
54 yield path
55 path, _, _ = path.rpartition(os.sep)
56
57
58def get_provides(path):
59 path = os.path.realpath(path)
60
61 def get_fn_name(fn):
62 for suffix in suffixes:
63 if fn.endswith(suffix):
64 return fn[:-len(suffix)]
65
66 isdir = os.path.isdir(path)
67 if isdir:
68 pkg_path = path
69 walk_path = path
70 else:
71 pkg_path = get_fn_name(path)
72 if pkg_path is None:
73 return
74 walk_path = os.path.dirname(path)
75
76 for curpath in walk_up(walk_path):
77 if not os.path.exists(os.path.join(curpath, '__init__.py')):
78 libdir = curpath
79 break
80 else:
81 libdir = ''
82
83 package_relpath = pkg_path[len(libdir)+1:]
84 package = '.'.join(package_relpath.split(os.sep))
85 if not isdir:
86 yield package, path
87 else:
88 if os.path.exists(os.path.join(path, '__init__.py')):
89 yield package, path
90
91 for dirpath, dirnames, filenames in os.walk(path):
92 relpath = dirpath[len(path)+1:]
93 if relpath:
94 if '__init__.py' not in filenames:
95 dirnames[:] = []
96 continue
97 else:
98 context = '.'.join(relpath.split(os.sep))
99 if package:
100 context = package + '.' + context
101 yield context, dirpath
102 else:
103 context = package
104
105 for fn in filenames:
106 adjusted_fn = get_fn_name(fn)
107 if not adjusted_fn or adjusted_fn == '__init__':
108 continue
109
110 fullfn = os.path.join(dirpath, fn)
111 if context:
112 yield context + '.' + adjusted_fn, fullfn
113 else:
114 yield adjusted_fn, fullfn
115
116
117def get_code_depends(code_string, path=None, provide=None, ispkg=False):
118 try:
119 code = ast.parse(code_string, path)
120 except TypeError as exc:
121 raise DependError(path, exc)
122 except SyntaxError as exc:
123 raise DependError(path, exc)
124
125 visitor = ImportVisitor()
126 visitor.visit(code)
127 for builtin_module in sys.builtin_module_names:
128 if builtin_module in visitor.imports:
129 visitor.imports.remove(builtin_module)
130
131 if provide:
132 provide_elements = provide.split('.')
133 if ispkg:
134 provide_elements.append("__self__")
135 context = '.'.join(provide_elements[:-1])
136 package_path = os.path.dirname(path)
137 else:
138 context = None
139 package_path = None
140
141 levelzero_importsfrom = (module for module, names, level in visitor.importsfrom
142 if level == 0)
143 for module in visitor.imports | set(levelzero_importsfrom):
144 if context and path:
145 module_basepath = os.path.join(package_path, module.replace('.', '/'))
146 if os.path.exists(module_basepath):
147 # Implicit relative import
148 yield context + '.' + module, path
149 continue
150
151 for suffix in suffixes:
152 if os.path.exists(module_basepath + suffix):
153 # Implicit relative import
154 yield context + '.' + module, path
155 break
156 else:
157 yield module, path
158 else:
159 yield module, path
160
161 for module, names, level in visitor.importsfrom:
162 if level == 0:
163 continue
164 elif not provide:
165 raise DependError("Error: ImportFrom non-zero level outside of a package: {0}".format((module, names, level)), path)
166 elif level > len(provide_elements):
167 raise DependError("Error: ImportFrom level exceeds package depth: {0}".format((module, names, level)), path)
168 else:
169 context = '.'.join(provide_elements[:-level])
170 if module:
171 if context:
172 yield context + '.' + module, path
173 else:
174 yield module, path
175
176
177def get_file_depends(path):
178 try:
179 code_string = open(path, 'r').read()
180 except (OSError, IOError) as exc:
181 raise DependError(path, exc)
182
183 return get_code_depends(code_string, path)
184
185
186def get_depends_recursive(directory):
187 directory = os.path.realpath(directory)
188
189 provides = dict((v, k) for k, v in get_provides(directory))
190 for filename, provide in provides.iteritems():
191 if os.path.isdir(filename):
192 filename = os.path.join(filename, '__init__.py')
193 ispkg = True
194 elif not filename.endswith('.py'):
195 continue
196 else:
197 ispkg = False
198
199 with open(filename, 'r') as f:
200 source = f.read()
201
202 depends = get_code_depends(source, filename, provide, ispkg)
203 for depend, by in depends:
204 yield depend, by
205
206
207def get_depends(path):
208 if os.path.isdir(path):
209 return get_depends_recursive(path)
210 else:
211 return get_file_depends(path)
212
213
214def main():
215 logging.basicConfig()
216
217 parser = argparse.ArgumentParser(description='Determine dependencies and provided packages for python scripts/modules')
218 parser.add_argument('path', nargs='+', help='full path to content to be processed')
219 group = parser.add_mutually_exclusive_group()
220 group.add_argument('-p', '--provides', action='store_true',
221 help='given a path, display the provided python modules')
222 group.add_argument('-d', '--depends', action='store_true',
223 help='given a filename, display the imported python modules')
224
225 args = parser.parse_args()
226 if args.provides:
227 modules = set()
228 for path in args.path:
229 for provide, fn in get_provides(path):
230 modules.add(provide)
231
232 for module in sorted(modules):
233 print(module)
234 elif args.depends:
235 for path in args.path:
236 try:
237 modules = get_depends(path)
238 except PythonDepError as exc:
239 logger.error(str(exc))
240 sys.exit(1)
241
242 for module, imp_by in modules:
243 print("{}\t{}".format(module, imp_by))
244 else:
245 parser.print_help()
246 sys.exit(2)
247
248
249if __name__ == '__main__':
250 main()