diff options
Diffstat (limited to 'scripts')
-rw-r--r-- | scripts/lib/recipetool/create_buildsys_python.py | 160 | ||||
-rwxr-xr-x | scripts/pythondeps | 250 |
2 files changed, 410 insertions, 0 deletions
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py index 9e4e1ebd4c..f4f4212559 100644 --- a/scripts/lib/recipetool/create_buildsys_python.py +++ b/scripts/lib/recipetool/create_buildsys_python.py | |||
@@ -41,6 +41,13 @@ def tinfoil_init(instance): | |||
41 | 41 | ||
42 | 42 | ||
43 | class PythonRecipeHandler(RecipeHandler): | 43 | class PythonRecipeHandler(RecipeHandler): |
44 | base_pkgdeps = ['python-core'] | ||
45 | excluded_pkgdeps = ['python-dbg'] | ||
46 | # os.path is provided by python-core | ||
47 | assume_provided = ['builtins', 'os.path'] | ||
48 | # Assumes that the host python builtin_module_names is sane for target too | ||
49 | assume_provided = assume_provided + list(sys.builtin_module_names) | ||
50 | |||
44 | bbvar_map = { | 51 | bbvar_map = { |
45 | 'Name': 'PN', | 52 | 'Name': 'PN', |
46 | 'Version': 'PV', | 53 | 'Version': 'PV', |
@@ -273,6 +280,8 @@ class PythonRecipeHandler(RecipeHandler): | |||
273 | mdinfo.append('{} = "{}"'.format(k, v)) | 280 | mdinfo.append('{} = "{}"'.format(k, v)) |
274 | lines_before[src_uri_line-1:src_uri_line-1] = mdinfo | 281 | lines_before[src_uri_line-1:src_uri_line-1] = mdinfo |
275 | 282 | ||
283 | mapped_deps, unmapped_deps = self.scan_setup_python_deps(srctree, setup_info, setup_non_literals) | ||
284 | |||
276 | extras_req = set() | 285 | extras_req = set() |
277 | if 'Extras-require' in info: | 286 | if 'Extras-require' in info: |
278 | extras_req = info['Extras-require'] | 287 | extras_req = info['Extras-require'] |
@@ -284,6 +293,8 @@ class PythonRecipeHandler(RecipeHandler): | |||
284 | lines_after.append('# Uncomment this line to enable all the optional features.') | 293 | lines_after.append('# Uncomment this line to enable all the optional features.') |
285 | lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req.iterkeys()))) | 294 | lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req.iterkeys()))) |
286 | for feature, feature_reqs in extras_req.iteritems(): | 295 | for feature, feature_reqs in extras_req.iteritems(): |
296 | unmapped_deps.difference_update(feature_reqs) | ||
297 | |||
287 | feature_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) | 298 | feature_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(feature_reqs)) |
288 | lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) | 299 | lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps))) |
289 | 300 | ||
@@ -293,11 +304,34 @@ class PythonRecipeHandler(RecipeHandler): | |||
293 | lines_after.append('') | 304 | lines_after.append('') |
294 | inst_reqs = info['Install-requires'] | 305 | inst_reqs = info['Install-requires'] |
295 | if inst_reqs: | 306 | if inst_reqs: |
307 | unmapped_deps.difference_update(inst_reqs) | ||
308 | |||
296 | inst_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) | 309 | inst_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(inst_reqs)) |
297 | lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') | 310 | lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These') |
298 | lines_after.append('# upstream names may not correspond exactly to bitbake package names.') | 311 | lines_after.append('# upstream names may not correspond exactly to bitbake package names.') |
299 | lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps))) | 312 | lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps))) |
300 | 313 | ||
314 | if mapped_deps: | ||
315 | name = info.get('Name') | ||
316 | if name and name[0] in mapped_deps: | ||
317 | # Attempt to avoid self-reference | ||
318 | mapped_deps.remove(name[0]) | ||
319 | mapped_deps -= set(self.excluded_pkgdeps) | ||
320 | if inst_reqs or extras_req: | ||
321 | lines_after.append('') | ||
322 | lines_after.append('# WARNING: the following rdepends are determined through basic analysis of the') | ||
323 | lines_after.append('# python sources, and might not be 100% accurate.') | ||
324 | lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(sorted(mapped_deps)))) | ||
325 | |||
326 | unmapped_deps -= set(extensions) | ||
327 | unmapped_deps -= set(self.assume_provided) | ||
328 | if unmapped_deps: | ||
329 | if mapped_deps: | ||
330 | lines_after.append('') | ||
331 | lines_after.append('# WARNING: We were unable to map the following python package/module') | ||
332 | lines_after.append('# dependencies to the bitbake packages which include them:') | ||
333 | lines_after.extend('# {}'.format(d) for d in sorted(unmapped_deps)) | ||
334 | |||
301 | handled.append('buildsystem') | 335 | handled.append('buildsystem') |
302 | 336 | ||
303 | def get_pkginfo(self, pkginfo_fn): | 337 | def get_pkginfo(self, pkginfo_fn): |
@@ -425,6 +459,132 @@ class PythonRecipeHandler(RecipeHandler): | |||
425 | if value != new_list: | 459 | if value != new_list: |
426 | info[variable] = new_list | 460 | info[variable] = new_list |
427 | 461 | ||
462 | def scan_setup_python_deps(self, srctree, setup_info, setup_non_literals): | ||
463 | if 'Package-dir' in setup_info: | ||
464 | package_dir = setup_info['Package-dir'] | ||
465 | else: | ||
466 | package_dir = {} | ||
467 | |||
468 | class PackageDir(distutils.command.build_py.build_py): | ||
469 | def __init__(self, package_dir): | ||
470 | self.package_dir = package_dir | ||
471 | |||
472 | pd = PackageDir(package_dir) | ||
473 | to_scan = [] | ||
474 | if not any(v in setup_non_literals for v in ['Py-modules', 'Scripts', 'Packages']): | ||
475 | if 'Py-modules' in setup_info: | ||
476 | for module in setup_info['Py-modules']: | ||
477 | try: | ||
478 | package, module = module.rsplit('.', 1) | ||
479 | except ValueError: | ||
480 | package, module = '.', module | ||
481 | module_path = os.path.join(pd.get_package_dir(package), module + '.py') | ||
482 | to_scan.append(module_path) | ||
483 | |||
484 | if 'Packages' in setup_info: | ||
485 | for package in setup_info['Packages']: | ||
486 | to_scan.append(pd.get_package_dir(package)) | ||
487 | |||
488 | if 'Scripts' in setup_info: | ||
489 | to_scan.extend(setup_info['Scripts']) | ||
490 | else: | ||
491 | logger.info("Scanning the entire source tree, as one or more of the following setup keywords are non-literal: py_modules, scripts, packages.") | ||
492 | |||
493 | if not to_scan: | ||
494 | to_scan = ['.'] | ||
495 | |||
496 | logger.info("Scanning paths for packages & dependencies: %s", ', '.join(to_scan)) | ||
497 | |||
498 | provided_packages = self.parse_pkgdata_for_python_packages() | ||
499 | scanned_deps = self.scan_python_dependencies([os.path.join(srctree, p) for p in to_scan]) | ||
500 | mapped_deps, unmapped_deps = set(self.base_pkgdeps), set() | ||
501 | for dep in scanned_deps: | ||
502 | mapped = provided_packages.get(dep) | ||
503 | if mapped: | ||
504 | mapped_deps.add(mapped) | ||
505 | else: | ||
506 | unmapped_deps.add(dep) | ||
507 | return mapped_deps, unmapped_deps | ||
508 | |||
509 | def scan_python_dependencies(self, paths): | ||
510 | deps = set() | ||
511 | try: | ||
512 | dep_output = self.run_command(['pythondeps', '-d'] + paths) | ||
513 | except (OSError, subprocess.CalledProcessError): | ||
514 | pass | ||
515 | else: | ||
516 | for line in dep_output.splitlines(): | ||
517 | line = line.rstrip() | ||
518 | dep, filename = line.split('\t', 1) | ||
519 | if filename.endswith('/setup.py'): | ||
520 | continue | ||
521 | deps.add(dep) | ||
522 | |||
523 | try: | ||
524 | provides_output = self.run_command(['pythondeps', '-p'] + paths) | ||
525 | except (OSError, subprocess.CalledProcessError): | ||
526 | pass | ||
527 | else: | ||
528 | provides_lines = (l.rstrip() for l in provides_output.splitlines()) | ||
529 | provides = set(l for l in provides_lines if l and l != 'setup') | ||
530 | deps -= provides | ||
531 | |||
532 | return deps | ||
533 | |||
534 | def parse_pkgdata_for_python_packages(self): | ||
535 | suffixes = [t[0] for t in imp.get_suffixes()] | ||
536 | pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) | ||
537 | |||
538 | ldata = tinfoil.config_data.createCopy() | ||
539 | bb.parse.handle('classes/python-dir.bbclass', ldata, True) | ||
540 | python_sitedir = ldata.getVar('PYTHON_SITEPACKAGES_DIR', True) | ||
541 | |||
542 | dynload_dir = os.path.join(os.path.dirname(python_sitedir), 'lib-dynload') | ||
543 | python_dirs = [python_sitedir + os.sep, | ||
544 | os.path.join(os.path.dirname(python_sitedir), 'dist-packages') + os.sep, | ||
545 | os.path.dirname(python_sitedir) + os.sep] | ||
546 | packages = {} | ||
547 | for pkgdatafile in glob.glob('{}/runtime/*'.format(pkgdata_dir)): | ||
548 | files_info = None | ||
549 | with open(pkgdatafile, 'r') as f: | ||
550 | for line in f.readlines(): | ||
551 | field, value = line.split(': ', 1) | ||
552 | if field == 'FILES_INFO': | ||
553 | files_info = ast.literal_eval(value) | ||
554 | break | ||
555 | else: | ||
556 | continue | ||
557 | |||
558 | for fn in files_info.iterkeys(): | ||
559 | for suffix in suffixes: | ||
560 | if fn.endswith(suffix): | ||
561 | break | ||
562 | else: | ||
563 | continue | ||
564 | |||
565 | if fn.startswith(dynload_dir + os.sep): | ||
566 | base = os.path.basename(fn) | ||
567 | provided = base.split('.', 1)[0] | ||
568 | packages[provided] = os.path.basename(pkgdatafile) | ||
569 | continue | ||
570 | |||
571 | for python_dir in python_dirs: | ||
572 | if fn.startswith(python_dir): | ||
573 | relpath = fn[len(python_dir):] | ||
574 | relstart, _, relremaining = relpath.partition(os.sep) | ||
575 | if relstart.endswith('.egg'): | ||
576 | relpath = relremaining | ||
577 | base, _ = os.path.splitext(relpath) | ||
578 | |||
579 | if '/.debug/' in base: | ||
580 | continue | ||
581 | if os.path.basename(base) == '__init__': | ||
582 | base = os.path.dirname(base) | ||
583 | base = base.replace(os.sep + os.sep, os.sep) | ||
584 | provided = base.replace(os.sep, '.') | ||
585 | packages[provided] = os.path.basename(pkgdatafile) | ||
586 | return packages | ||
587 | |||
428 | @classmethod | 588 | @classmethod |
429 | def run_command(cls, cmd, **popenargs): | 589 | def run_command(cls, cmd, **popenargs): |
430 | if 'stderr' not in popenargs: | 590 | if 'stderr' not in popenargs: |
diff --git a/scripts/pythondeps b/scripts/pythondeps new file mode 100755 index 0000000000..ff92e747ed --- /dev/null +++ b/scripts/pythondeps | |||
@@ -0,0 +1,250 @@ | |||
1 | #!/usr/bin/env python | ||
2 | # | ||
3 | # Determine dependencies of python scripts or available python modules in a search path. | ||
4 | # | ||
5 | # Given the -d argument and a filename/filenames, returns the modules imported by those files. | ||
6 | # Given the -d argument and a directory/directories, recurses to find all | ||
7 | # python packages and modules, returns the modules imported by these. | ||
8 | # Given the -p argument and a path or paths, scans that path for available python modules/packages. | ||
9 | |||
10 | import argparse | ||
11 | import ast | ||
12 | import imp | ||
13 | import logging | ||
14 | import os.path | ||
15 | import sys | ||
16 | |||
17 | |||
18 | logger = logging.getLogger('pythondeps') | ||
19 | |||
20 | suffixes = [] | ||
21 | for triple in imp.get_suffixes(): | ||
22 | suffixes.append(triple[0]) | ||
23 | |||
24 | |||
25 | class PythonDepError(Exception): | ||
26 | pass | ||
27 | |||
28 | |||
29 | class DependError(PythonDepError): | ||
30 | def __init__(self, path, error): | ||
31 | self.path = path | ||
32 | self.error = error | ||
33 | PythonDepError.__init__(self, error) | ||
34 | |||
35 | def __str__(self): | ||
36 | return "Failure determining dependencies of {}: {}".format(self.path, self.error) | ||
37 | |||
38 | |||
39 | class ImportVisitor(ast.NodeVisitor): | ||
40 | def __init__(self): | ||
41 | self.imports = set() | ||
42 | self.importsfrom = [] | ||
43 | |||
44 | def visit_Import(self, node): | ||
45 | for alias in node.names: | ||
46 | self.imports.add(alias.name) | ||
47 | |||
48 | def visit_ImportFrom(self, node): | ||
49 | self.importsfrom.append((node.module, [a.name for a in node.names], node.level)) | ||
50 | |||
51 | |||
52 | def walk_up(path): | ||
53 | while path: | ||
54 | yield path | ||
55 | path, _, _ = path.rpartition(os.sep) | ||
56 | |||
57 | |||
58 | def get_provides(path): | ||
59 | path = os.path.realpath(path) | ||
60 | |||
61 | def get_fn_name(fn): | ||
62 | for suffix in suffixes: | ||
63 | if fn.endswith(suffix): | ||
64 | return fn[:-len(suffix)] | ||
65 | |||
66 | isdir = os.path.isdir(path) | ||
67 | if isdir: | ||
68 | pkg_path = path | ||
69 | walk_path = path | ||
70 | else: | ||
71 | pkg_path = get_fn_name(path) | ||
72 | if pkg_path is None: | ||
73 | return | ||
74 | walk_path = os.path.dirname(path) | ||
75 | |||
76 | for curpath in walk_up(walk_path): | ||
77 | if not os.path.exists(os.path.join(curpath, '__init__.py')): | ||
78 | libdir = curpath | ||
79 | break | ||
80 | else: | ||
81 | libdir = '' | ||
82 | |||
83 | package_relpath = pkg_path[len(libdir)+1:] | ||
84 | package = '.'.join(package_relpath.split(os.sep)) | ||
85 | if not isdir: | ||
86 | yield package, path | ||
87 | else: | ||
88 | if os.path.exists(os.path.join(path, '__init__.py')): | ||
89 | yield package, path | ||
90 | |||
91 | for dirpath, dirnames, filenames in os.walk(path): | ||
92 | relpath = dirpath[len(path)+1:] | ||
93 | if relpath: | ||
94 | if '__init__.py' not in filenames: | ||
95 | dirnames[:] = [] | ||
96 | continue | ||
97 | else: | ||
98 | context = '.'.join(relpath.split(os.sep)) | ||
99 | if package: | ||
100 | context = package + '.' + context | ||
101 | yield context, dirpath | ||
102 | else: | ||
103 | context = package | ||
104 | |||
105 | for fn in filenames: | ||
106 | adjusted_fn = get_fn_name(fn) | ||
107 | if not adjusted_fn or adjusted_fn == '__init__': | ||
108 | continue | ||
109 | |||
110 | fullfn = os.path.join(dirpath, fn) | ||
111 | if context: | ||
112 | yield context + '.' + adjusted_fn, fullfn | ||
113 | else: | ||
114 | yield adjusted_fn, fullfn | ||
115 | |||
116 | |||
117 | def get_code_depends(code_string, path=None, provide=None, ispkg=False): | ||
118 | try: | ||
119 | code = ast.parse(code_string, path) | ||
120 | except TypeError as exc: | ||
121 | raise DependError(path, exc) | ||
122 | except SyntaxError as exc: | ||
123 | raise DependError(path, exc) | ||
124 | |||
125 | visitor = ImportVisitor() | ||
126 | visitor.visit(code) | ||
127 | for builtin_module in sys.builtin_module_names: | ||
128 | if builtin_module in visitor.imports: | ||
129 | visitor.imports.remove(builtin_module) | ||
130 | |||
131 | if provide: | ||
132 | provide_elements = provide.split('.') | ||
133 | if ispkg: | ||
134 | provide_elements.append("__self__") | ||
135 | context = '.'.join(provide_elements[:-1]) | ||
136 | package_path = os.path.dirname(path) | ||
137 | else: | ||
138 | context = None | ||
139 | package_path = None | ||
140 | |||
141 | levelzero_importsfrom = (module for module, names, level in visitor.importsfrom | ||
142 | if level == 0) | ||
143 | for module in visitor.imports | set(levelzero_importsfrom): | ||
144 | if context and path: | ||
145 | module_basepath = os.path.join(package_path, module.replace('.', '/')) | ||
146 | if os.path.exists(module_basepath): | ||
147 | # Implicit relative import | ||
148 | yield context + '.' + module, path | ||
149 | continue | ||
150 | |||
151 | for suffix in suffixes: | ||
152 | if os.path.exists(module_basepath + suffix): | ||
153 | # Implicit relative import | ||
154 | yield context + '.' + module, path | ||
155 | break | ||
156 | else: | ||
157 | yield module, path | ||
158 | else: | ||
159 | yield module, path | ||
160 | |||
161 | for module, names, level in visitor.importsfrom: | ||
162 | if level == 0: | ||
163 | continue | ||
164 | elif not provide: | ||
165 | raise DependError("Error: ImportFrom non-zero level outside of a package: {0}".format((module, names, level)), path) | ||
166 | elif level > len(provide_elements): | ||
167 | raise DependError("Error: ImportFrom level exceeds package depth: {0}".format((module, names, level)), path) | ||
168 | else: | ||
169 | context = '.'.join(provide_elements[:-level]) | ||
170 | if module: | ||
171 | if context: | ||
172 | yield context + '.' + module, path | ||
173 | else: | ||
174 | yield module, path | ||
175 | |||
176 | |||
177 | def get_file_depends(path): | ||
178 | try: | ||
179 | code_string = open(path, 'r').read() | ||
180 | except (OSError, IOError) as exc: | ||
181 | raise DependError(path, exc) | ||
182 | |||
183 | return get_code_depends(code_string, path) | ||
184 | |||
185 | |||
186 | def get_depends_recursive(directory): | ||
187 | directory = os.path.realpath(directory) | ||
188 | |||
189 | provides = dict((v, k) for k, v in get_provides(directory)) | ||
190 | for filename, provide in provides.iteritems(): | ||
191 | if os.path.isdir(filename): | ||
192 | filename = os.path.join(filename, '__init__.py') | ||
193 | ispkg = True | ||
194 | elif not filename.endswith('.py'): | ||
195 | continue | ||
196 | else: | ||
197 | ispkg = False | ||
198 | |||
199 | with open(filename, 'r') as f: | ||
200 | source = f.read() | ||
201 | |||
202 | depends = get_code_depends(source, filename, provide, ispkg) | ||
203 | for depend, by in depends: | ||
204 | yield depend, by | ||
205 | |||
206 | |||
207 | def get_depends(path): | ||
208 | if os.path.isdir(path): | ||
209 | return get_depends_recursive(path) | ||
210 | else: | ||
211 | return get_file_depends(path) | ||
212 | |||
213 | |||
214 | def main(): | ||
215 | logging.basicConfig() | ||
216 | |||
217 | parser = argparse.ArgumentParser(description='Determine dependencies and provided packages for python scripts/modules') | ||
218 | parser.add_argument('path', nargs='+', help='full path to content to be processed') | ||
219 | group = parser.add_mutually_exclusive_group() | ||
220 | group.add_argument('-p', '--provides', action='store_true', | ||
221 | help='given a path, display the provided python modules') | ||
222 | group.add_argument('-d', '--depends', action='store_true', | ||
223 | help='given a filename, display the imported python modules') | ||
224 | |||
225 | args = parser.parse_args() | ||
226 | if args.provides: | ||
227 | modules = set() | ||
228 | for path in args.path: | ||
229 | for provide, fn in get_provides(path): | ||
230 | modules.add(provide) | ||
231 | |||
232 | for module in sorted(modules): | ||
233 | print(module) | ||
234 | elif args.depends: | ||
235 | for path in args.path: | ||
236 | try: | ||
237 | modules = get_depends(path) | ||
238 | except PythonDepError as exc: | ||
239 | logger.error(str(exc)) | ||
240 | sys.exit(1) | ||
241 | |||
242 | for module, imp_by in modules: | ||
243 | print("{}\t{}".format(module, imp_by)) | ||
244 | else: | ||
245 | parser.print_help() | ||
246 | sys.exit(2) | ||
247 | |||
248 | |||
249 | if __name__ == '__main__': | ||
250 | main() | ||