summaryrefslogtreecommitdiffstats
path: root/scripts/lib
diff options
context:
space:
mode:
authorChristopher Larson <kergoth@gmail.com>2015-01-19 11:52:30 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-01-23 11:36:28 +0000
commite490d79fb740319b35227eb66968be852a781036 (patch)
tree6e7c2c721cebb8708bf05e1fad0ebb354f5f7667 /scripts/lib
parent26f7732166c181c0e9b0fba5b63f394986de061b (diff)
downloadpoky-e490d79fb740319b35227eb66968be852a781036.tar.gz
recipetool: add python buildsystem support
- Handles distutils & setuptools. - Supports pulling metadata from PKG-INFO, .egg-info, & setup.py (via two different mechanisms). - Doesn't handle python 3 yet. (From OE-Core rev: 6ff693c71d97b4bcfde198c84cf9fac7185cccfd) Signed-off-by: Christopher Larson <kergoth@gmail.com> Signed-off-by: Ross Burton <ross.burton@intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib')
-rw-r--r--scripts/lib/recipetool/create_buildsys_python.py560
1 files changed, 560 insertions, 0 deletions
diff --git a/scripts/lib/recipetool/create_buildsys_python.py b/scripts/lib/recipetool/create_buildsys_python.py
new file mode 100644
index 0000000000..9e4e1ebd4c
--- /dev/null
+++ b/scripts/lib/recipetool/create_buildsys_python.py
@@ -0,0 +1,560 @@
1# Recipe creation tool - create build system handler for python
2#
3# Copyright (C) 2015 Mentor Graphics Corporation
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU General Public License version 2 as
7# published by the Free Software Foundation.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License along
15# with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
17
18import ast
19import codecs
20import collections
21import distutils.command.build_py
22import email
23import imp
24import glob
25import itertools
26import logging
27import os
28import re
29import sys
30import subprocess
31from recipetool.create import RecipeHandler
32
33logger = logging.getLogger('recipetool')
34
35tinfoil = None
36
37
38def tinfoil_init(instance):
39 global tinfoil
40 tinfoil = instance
41
42
43class PythonRecipeHandler(RecipeHandler):
44 bbvar_map = {
45 'Name': 'PN',
46 'Version': 'PV',
47 'Home-page': 'HOMEPAGE',
48 'Summary': 'SUMMARY',
49 'Description': 'DESCRIPTION',
50 'License': 'LICENSE',
51 'Requires': 'RDEPENDS_${PN}',
52 'Provides': 'RPROVIDES_${PN}',
53 'Obsoletes': 'RREPLACES_${PN}',
54 }
55 # PN/PV are already set by recipetool core & desc can be extremely long
56 excluded_fields = [
57 'Name',
58 'Version',
59 'Description',
60 ]
61 setup_parse_map = {
62 'Url': 'Home-page',
63 'Classifiers': 'Classifier',
64 'Description': 'Summary',
65 }
66 setuparg_map = {
67 'Home-page': 'url',
68 'Classifier': 'classifiers',
69 'Summary': 'description',
70 'Description': 'long-description',
71 }
72 # Values which are lists, used by the setup.py argument based metadata
73 # extraction method, to determine how to process the setup.py output.
74 setuparg_list_fields = [
75 'Classifier',
76 'Requires',
77 'Provides',
78 'Obsoletes',
79 'Platform',
80 'Supported-Platform',
81 ]
82 setuparg_multi_line_values = ['Description']
83 replacements = [
84 ('License', r' ', '-'),
85 ('License', r'-License$', ''),
86 ('License', r'^UNKNOWN$', ''),
87
88 # Remove currently unhandled version numbers from these variables
89 ('Requires', r' *\([^)]*\)', ''),
90 ('Provides', r' *\([^)]*\)', ''),
91 ('Obsoletes', r' *\([^)]*\)', ''),
92 ('Install-requires', r'^([^><= ]+).*', r'\1'),
93 ('Extras-require', r'^([^><= ]+).*', r'\1'),
94 ('Tests-require', r'^([^><= ]+).*', r'\1'),
95
96 # Remove unhandled dependency on particular features (e.g. foo[PDF])
97 ('Install-requires', r'\[[^\]]+\]$', ''),
98 ]
99
100 classifier_license_map = {
101 'License :: OSI Approved :: Academic Free License (AFL)': 'AFL',
102 'License :: OSI Approved :: Apache Software License': 'Apache',
103 'License :: OSI Approved :: Apple Public Source License': 'APSL',
104 'License :: OSI Approved :: Artistic License': 'Artistic',
105 'License :: OSI Approved :: Attribution Assurance License': 'AAL',
106 'License :: OSI Approved :: BSD License': 'BSD',
107 'License :: OSI Approved :: Common Public License': 'CPL',
108 'License :: OSI Approved :: Eiffel Forum License': 'EFL',
109 'License :: OSI Approved :: European Union Public Licence 1.0 (EUPL 1.0)': 'EUPL-1.0',
110 'License :: OSI Approved :: European Union Public Licence 1.1 (EUPL 1.1)': 'EUPL-1.1',
111 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)': 'AGPL-3.0+',
112 'License :: OSI Approved :: GNU Affero General Public License v3': 'AGPL-3.0',
113 'License :: OSI Approved :: GNU Free Documentation License (FDL)': 'GFDL',
114 'License :: OSI Approved :: GNU General Public License (GPL)': 'GPL',
115 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)': 'GPL-2.0',
116 'License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)': 'GPL-2.0+',
117 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)': 'GPL-3.0',
118 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)': 'GPL-3.0+',
119 'License :: OSI Approved :: GNU Lesser General Public License v2 (LGPLv2)': 'LGPL-2.0',
120 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)': 'LGPL-2.0+',
121 'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)': 'LGPL-3.0',
122 'License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)': 'LGPL-3.0+',
123 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)': 'LGPL',
124 'License :: OSI Approved :: IBM Public License': 'IPL',
125 'License :: OSI Approved :: ISC License (ISCL)': 'ISC',
126 'License :: OSI Approved :: Intel Open Source License': 'Intel',
127 'License :: OSI Approved :: Jabber Open Source License': 'Jabber',
128 'License :: OSI Approved :: MIT License': 'MIT',
129 'License :: OSI Approved :: MITRE Collaborative Virtual Workspace License (CVW)': 'CVWL',
130 'License :: OSI Approved :: Motosoto License': 'Motosoto',
131 'License :: OSI Approved :: Mozilla Public License 1.0 (MPL)': 'MPL-1.0',
132 'License :: OSI Approved :: Mozilla Public License 1.1 (MPL 1.1)': 'MPL-1.1',
133 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)': 'MPL-2.0',
134 'License :: OSI Approved :: Nethack General Public License': 'NGPL',
135 'License :: OSI Approved :: Nokia Open Source License': 'Nokia',
136 'License :: OSI Approved :: Open Group Test Suite License': 'OGTSL',
137 'License :: OSI Approved :: Python License (CNRI Python License)': 'CNRI-Python',
138 'License :: OSI Approved :: Python Software Foundation License': 'PSF',
139 'License :: OSI Approved :: Qt Public License (QPL)': 'QPL',
140 'License :: OSI Approved :: Ricoh Source Code Public License': 'RSCPL',
141 'License :: OSI Approved :: Sleepycat License': 'Sleepycat',
142 'License :: OSI Approved :: Sun Industry Standards Source License (SISSL)': '-- Sun Industry Standards Source License (SISSL)',
143 'License :: OSI Approved :: Sun Public License': 'SPL',
144 'License :: OSI Approved :: University of Illinois/NCSA Open Source License': 'NCSA',
145 'License :: OSI Approved :: Vovida Software License 1.0': 'VSL-1.0',
146 'License :: OSI Approved :: W3C License': 'W3C',
147 'License :: OSI Approved :: X.Net License': 'Xnet',
148 'License :: OSI Approved :: Zope Public License': 'ZPL',
149 'License :: OSI Approved :: zlib/libpng License': 'Zlib',
150 }
151
152 def __init__(self):
153 pass
154
155 def process(self, srctree, classes, lines_before, lines_after, handled):
156 if 'buildsystem' in handled:
157 return False
158
159 if not RecipeHandler.checkfiles(srctree, ['setup.py']):
160 return
161
162 # setup.py is always parsed to get at certain required information, such as
163 # distutils vs setuptools
164 #
165 # If egg info is available, we use it for both its PKG-INFO metadata
166 # and for its requires.txt for install_requires.
167 # If PKG-INFO is available but no egg info is, we use that for metadata in preference to
168 # the parsed setup.py, but use the install_requires info from the
169 # parsed setup.py.
170
171 setupscript = os.path.join(srctree, 'setup.py')
172 try:
173 setup_info, uses_setuptools, setup_non_literals, extensions = self.parse_setup_py(setupscript)
174 except Exception:
175 logger.exception("Failed to parse setup.py")
176 setup_info, uses_setuptools, setup_non_literals, extensions = {}, True, [], []
177
178 egginfo = glob.glob(os.path.join(srctree, '*.egg-info'))
179 if egginfo:
180 info = self.get_pkginfo(os.path.join(egginfo[0], 'PKG-INFO'))
181 requires_txt = os.path.join(egginfo[0], 'requires.txt')
182 if os.path.exists(requires_txt):
183 with codecs.open(requires_txt) as f:
184 inst_req = []
185 extras_req = collections.defaultdict(list)
186 current_feature = None
187 for line in f.readlines():
188 line = line.rstrip()
189 if not line:
190 continue
191
192 if line.startswith('['):
193 current_feature = line[1:-1]
194 elif current_feature:
195 extras_req[current_feature].append(line)
196 else:
197 inst_req.append(line)
198 info['Install-requires'] = inst_req
199 info['Extras-require'] = extras_req
200 elif RecipeHandler.checkfiles(srctree, ['PKG-INFO']):
201 info = self.get_pkginfo(os.path.join(srctree, 'PKG-INFO'))
202
203 if setup_info:
204 if 'Install-requires' in setup_info:
205 info['Install-requires'] = setup_info['Install-requires']
206 if 'Extras-require' in setup_info:
207 info['Extras-require'] = setup_info['Extras-require']
208 else:
209 if setup_info:
210 info = setup_info
211 else:
212 info = self.get_setup_args_info(setupscript)
213
214 self.apply_info_replacements(info)
215
216 if uses_setuptools:
217 classes.append('setuptools')
218 else:
219 classes.append('distutils')
220
221 if 'Classifier' in info:
222 licenses = []
223 for classifier in info['Classifier']:
224 if classifier in self.classifier_license_map:
225 license = self.classifier_license_map[classifier]
226 licenses.append(license)
227
228 if licenses:
229 info['License'] = ' & '.join(licenses)
230
231
232 # Map PKG-INFO & setup.py fields to bitbake variables
233 bbinfo = {}
234 for field, values in info.iteritems():
235 if field in self.excluded_fields:
236 continue
237
238 if field not in self.bbvar_map:
239 continue
240
241 if isinstance(values, basestring):
242 value = values
243 else:
244 value = ' '.join(str(v) for v in values if v)
245
246 bbvar = self.bbvar_map[field]
247 if bbvar not in bbinfo and value:
248 bbinfo[bbvar] = value
249
250 comment_lic_line = None
251 for pos, line in enumerate(list(lines_before)):
252 if line.startswith('#') and 'LICENSE' in line:
253 comment_lic_line = pos
254 elif line.startswith('LICENSE =') and 'LICENSE' in bbinfo:
255 if line in ('LICENSE = "Unknown"', 'LICENSE = "CLOSED"'):
256 lines_before[pos] = 'LICENSE = "{}"'.format(bbinfo['LICENSE'])
257 if line == 'LICENSE = "CLOSED"' and comment_lic_line:
258 lines_before[comment_lic_line:pos] = [
259 '# WARNING: the following LICENSE value is a best guess - it is your',
260 '# responsibility to verify that the value is complete and correct.'
261 ]
262 del bbinfo['LICENSE']
263
264 src_uri_line = None
265 for pos, line in enumerate(lines_before):
266 if line.startswith('SRC_URI ='):
267 src_uri_line = pos
268
269 if bbinfo:
270 mdinfo = ['']
271 for k in sorted(bbinfo):
272 v = bbinfo[k]
273 mdinfo.append('{} = "{}"'.format(k, v))
274 lines_before[src_uri_line-1:src_uri_line-1] = mdinfo
275
276 extras_req = set()
277 if 'Extras-require' in info:
278 extras_req = info['Extras-require']
279 if extras_req:
280 lines_after.append('# The following configs & dependencies are from setuptools extras_require.')
281 lines_after.append('# These dependencies are optional, hence can be controlled via PACKAGECONFIG.')
282 lines_after.append('# The upstream names may not correspond exactly to bitbake package names.')
283 lines_after.append('#')
284 lines_after.append('# Uncomment this line to enable all the optional features.')
285 lines_after.append('#PACKAGECONFIG ?= "{}"'.format(' '.join(k.lower() for k in extras_req.iterkeys())))
286 for feature, feature_reqs in extras_req.iteritems():
287 feature_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(feature_reqs))
288 lines_after.append('PACKAGECONFIG[{}] = ",,,{}"'.format(feature.lower(), ' '.join(feature_req_deps)))
289
290 inst_reqs = set()
291 if 'Install-requires' in info:
292 if extras_req:
293 lines_after.append('')
294 inst_reqs = info['Install-requires']
295 if inst_reqs:
296 inst_req_deps = ('python-' + r.replace('.', '-').lower() for r in sorted(inst_reqs))
297 lines_after.append('# WARNING: the following rdepends are from setuptools install_requires. These')
298 lines_after.append('# upstream names may not correspond exactly to bitbake package names.')
299 lines_after.append('RDEPENDS_${{PN}} += "{}"'.format(' '.join(inst_req_deps)))
300
301 handled.append('buildsystem')
302
303 def get_pkginfo(self, pkginfo_fn):
304 msg = email.message_from_file(open(pkginfo_fn, 'r'))
305 msginfo = {}
306 for field in msg.keys():
307 values = msg.get_all(field)
308 if len(values) == 1:
309 msginfo[field] = values[0]
310 else:
311 msginfo[field] = values
312 return msginfo
313
314 def parse_setup_py(self, setupscript='./setup.py'):
315 with codecs.open(setupscript) as f:
316 info, imported_modules, non_literals, extensions = gather_setup_info(f)
317
318 def _map(key):
319 key = key.replace('_', '-')
320 key = key[0].upper() + key[1:]
321 if key in self.setup_parse_map:
322 key = self.setup_parse_map[key]
323 return key
324
325 # Naive mapping of setup() arguments to PKG-INFO field names
326 for d in [info, non_literals]:
327 for key, value in d.items():
328 new_key = _map(key)
329 if new_key != key:
330 del d[key]
331 d[new_key] = value
332
333 return info, 'setuptools' in imported_modules, non_literals, extensions
334
335 def get_setup_args_info(self, setupscript='./setup.py'):
336 cmd = ['python', setupscript]
337 info = {}
338 keys = set(self.bbvar_map.keys())
339 keys |= set(self.setuparg_list_fields)
340 keys |= set(self.setuparg_multi_line_values)
341 grouped_keys = itertools.groupby(keys, lambda k: (k in self.setuparg_list_fields, k in self.setuparg_multi_line_values))
342 for index, keys in grouped_keys:
343 if index == (True, False):
344 # Splitlines output for each arg as a list value
345 for key in keys:
346 arg = self.setuparg_map.get(key, key.lower())
347 try:
348 arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
349 except (OSError, subprocess.CalledProcessError):
350 pass
351 else:
352 info[key] = [l.rstrip() for l in arg_info.splitlines()]
353 elif index == (False, True):
354 # Entire output for each arg
355 for key in keys:
356 arg = self.setuparg_map.get(key, key.lower())
357 try:
358 arg_info = self.run_command(cmd + ['--' + arg], cwd=os.path.dirname(setupscript))
359 except (OSError, subprocess.CalledProcessError):
360 pass
361 else:
362 info[key] = arg_info
363 else:
364 info.update(self.get_setup_byline(list(keys), setupscript))
365 return info
366
367 def get_setup_byline(self, fields, setupscript='./setup.py'):
368 info = {}
369
370 cmd = ['python', setupscript]
371 cmd.extend('--' + self.setuparg_map.get(f, f.lower()) for f in fields)
372 try:
373 info_lines = self.run_command(cmd, cwd=os.path.dirname(setupscript)).splitlines()
374 except (OSError, subprocess.CalledProcessError):
375 pass
376 else:
377 if len(fields) != len(info_lines):
378 logger.error('Mismatch between setup.py output lines and number of fields')
379 sys.exit(1)
380
381 for lineno, line in enumerate(info_lines):
382 line = line.rstrip()
383 info[fields[lineno]] = line
384 return info
385
386 def apply_info_replacements(self, info):
387 for variable, search, replace in self.replacements:
388 if variable not in info:
389 continue
390
391 def replace_value(search, replace, value):
392 if replace is None:
393 if re.search(search, value):
394 return None
395 else:
396 new_value = re.sub(search, replace, value)
397 if value != new_value:
398 return new_value
399 return value
400
401 value = info[variable]
402 if isinstance(value, basestring):
403 new_value = replace_value(search, replace, value)
404 if new_value is None:
405 del info[variable]
406 elif new_value != value:
407 info[variable] = new_value
408 elif hasattr(value, 'iteritems'):
409 for dkey, dvalue in value.iteritems():
410 new_list = []
411 for pos, a_value in enumerate(dvalue):
412 new_value = replace_value(search, replace, a_value)
413 if new_value is not None and new_value != value:
414 new_list.append(new_value)
415
416 if value != new_list:
417 value[dkey] = new_list
418 else:
419 new_list = []
420 for pos, a_value in enumerate(value):
421 new_value = replace_value(search, replace, a_value)
422 if new_value is not None and new_value != value:
423 new_list.append(new_value)
424
425 if value != new_list:
426 info[variable] = new_list
427
428 @classmethod
429 def run_command(cls, cmd, **popenargs):
430 if 'stderr' not in popenargs:
431 popenargs['stderr'] = subprocess.STDOUT
432 try:
433 return subprocess.check_output(cmd, **popenargs)
434 except OSError as exc:
435 logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc)
436 raise
437 except subprocess.CalledProcessError as exc:
438 logger.error('Unable to run `{}`: {}', ' '.join(cmd), exc.output)
439 raise
440
441
442def gather_setup_info(fileobj):
443 parsed = ast.parse(fileobj.read(), fileobj.name)
444 visitor = SetupScriptVisitor()
445 visitor.visit(parsed)
446
447 non_literals, extensions = {}, []
448 for key, value in visitor.keywords.items():
449 if key == 'ext_modules':
450 if isinstance(value, list):
451 for ext in value:
452 if (isinstance(ext, ast.Call) and
453 isinstance(ext.func, ast.Name) and
454 ext.func.id == 'Extension' and
455 not has_non_literals(ext.args)):
456 extensions.append(ext.args[0])
457 elif has_non_literals(value):
458 non_literals[key] = value
459 del visitor.keywords[key]
460
461 return visitor.keywords, visitor.imported_modules, non_literals, extensions
462
463
464class SetupScriptVisitor(ast.NodeVisitor):
465 def __init__(self):
466 ast.NodeVisitor.__init__(self)
467 self.keywords = {}
468 self.non_literals = []
469 self.imported_modules = set()
470
471 def visit_Expr(self, node):
472 if isinstance(node.value, ast.Call) and \
473 isinstance(node.value.func, ast.Name) and \
474 node.value.func.id == 'setup':
475 self.visit_setup(node.value)
476
477 def visit_setup(self, node):
478 call = LiteralAstTransform().visit(node)
479 self.keywords = call.keywords
480 for k, v in self.keywords.iteritems():
481 if has_non_literals(v):
482 self.non_literals.append(k)
483
484 def visit_Import(self, node):
485 for alias in node.names:
486 self.imported_modules.add(alias.name)
487
488 def visit_ImportFrom(self, node):
489 self.imported_modules.add(node.module)
490
491
492class LiteralAstTransform(ast.NodeTransformer):
493 """Simplify the ast through evaluation of literals."""
494 excluded_fields = ['ctx']
495
496 def visit(self, node):
497 if not isinstance(node, ast.AST):
498 return node
499 else:
500 return ast.NodeTransformer.visit(self, node)
501
502 def generic_visit(self, node):
503 try:
504 return ast.literal_eval(node)
505 except ValueError:
506 for field, value in ast.iter_fields(node):
507 if field in self.excluded_fields:
508 delattr(node, field)
509 if value is None:
510 continue
511
512 if isinstance(value, list):
513 if field in ('keywords', 'kwargs'):
514 new_value = dict((kw.arg, self.visit(kw.value)) for kw in value)
515 else:
516 new_value = [self.visit(i) for i in value]
517 else:
518 new_value = self.visit(value)
519 setattr(node, field, new_value)
520 return node
521
522 def visit_Name(self, node):
523 if hasattr('__builtins__', node.id):
524 return getattr(__builtins__, node.id)
525 else:
526 return self.generic_visit(node)
527
528 def visit_Tuple(self, node):
529 return tuple(self.visit(v) for v in node.elts)
530
531 def visit_List(self, node):
532 return [self.visit(v) for v in node.elts]
533
534 def visit_Set(self, node):
535 return set(self.visit(v) for v in node.elts)
536
537 def visit_Dict(self, node):
538 keys = (self.visit(k) for k in node.keys)
539 values = (self.visit(v) for v in node.values)
540 return dict(zip(keys, values))
541
542
543def has_non_literals(value):
544 if isinstance(value, ast.AST):
545 return True
546 elif isinstance(value, basestring):
547 return False
548 elif hasattr(value, 'itervalues'):
549 return any(has_non_literals(v) for v in value.itervalues())
550 elif hasattr(value, '__iter__'):
551 return any(has_non_literals(v) for v in value)
552
553
554def plugin_init(pluginlist):
555 pass
556
557
558def register_recipe_handlers(handlers):
559 # We need to make sure this is ahead of the makefile fallback handler
560 handlers.insert(0, PythonRecipeHandler())