summaryrefslogtreecommitdiffstats
path: root/scripts/lib
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2016-01-19 00:18:31 +1300
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-01-19 17:24:51 +0000
commit726dbda396a49de0fa710e64218457915715a5d2 (patch)
tree25f7673b89afc506f94189c7dc78d63cf90617d4 /scripts/lib
parent788e4bb23ef6a167a95c4f870721ea05bcbe376c (diff)
downloadpoky-726dbda396a49de0fa710e64218457915715a5d2.tar.gz
recipetool: create: move dependency mapping code to RecipeHandler
Some refactoring to allow access to the library/header/pkg-config mappings and the DEPENDS / unmapped dependency output code from other classes than AutotoolsRecipeHandler. (From OE-Core rev: 40c10d998b90dd59c6d36c28f8ba11ec598bfa0f) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib')
-rw-r--r--scripts/lib/recipetool/create.py137
-rw-r--r--scripts/lib/recipetool/create_buildsys.py121
2 files changed, 144 insertions, 114 deletions
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py
index 1218a7d284..dab917faac 100644
--- a/scripts/lib/recipetool/create.py
+++ b/scripts/lib/recipetool/create.py
@@ -21,6 +21,7 @@ import argparse
21import glob 21import glob
22import fnmatch 22import fnmatch
23import re 23import re
24import json
24import logging 25import logging
25import scriptutils 26import scriptutils
26import urlparse 27import urlparse
@@ -39,7 +40,73 @@ def tinfoil_init(instance):
39 global tinfoil 40 global tinfoil
40 tinfoil = instance 41 tinfoil = instance
41 42
42class RecipeHandler(): 43class RecipeHandler(object):
44 recipelibmap = {}
45 recipeheadermap = {}
46
47 @staticmethod
48 def load_libmap(d):
49 '''Load library->recipe mapping'''
50 import oe.package
51
52 if RecipeHandler.recipelibmap:
53 return
54 # First build up library->package mapping
55 shlib_providers = oe.package.read_shlib_providers(d)
56 libdir = d.getVar('libdir', True)
57 base_libdir = d.getVar('base_libdir', True)
58 libpaths = list(set([base_libdir, libdir]))
59 libname_re = re.compile('^lib(.+)\.so.*$')
60 pkglibmap = {}
61 for lib, item in shlib_providers.iteritems():
62 for path, pkg in item.iteritems():
63 if path in libpaths:
64 res = libname_re.match(lib)
65 if res:
66 libname = res.group(1)
67 if not libname in pkglibmap:
68 pkglibmap[libname] = pkg[0]
69 else:
70 logger.debug('unable to extract library name from %s' % lib)
71
72 # Now turn it into a library->recipe mapping
73 pkgdata_dir = d.getVar('PKGDATA_DIR', True)
74 for libname, pkg in pkglibmap.iteritems():
75 try:
76 with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
77 for line in f:
78 if line.startswith('PN:'):
79 RecipeHandler.recipelibmap[libname] = line.split(':', 1)[-1].strip()
80 break
81 except IOError as ioe:
82 if ioe.errno == 2:
83 logger.warn('unable to find a pkgdata file for package %s' % pkg)
84 else:
85 raise
86
87 @staticmethod
88 def load_headermap(d):
89 '''Build up lib headerfile->recipe mapping'''
90 if RecipeHandler.recipeheadermap:
91 return
92 includedir = d.getVar('includedir', True)
93 for pkg in glob.glob(os.path.join(pkgdata_dir, 'runtime', '*-dev')):
94 with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
95 pn = None
96 headers = []
97 for line in f:
98 if line.startswith('PN:'):
99 pn = line.split(':', 1)[-1].strip()
100 elif line.startswith('FILES_INFO:'):
101 val = line.split(':', 1)[1].strip()
102 dictval = json.loads(val)
103 for fullpth in sorted(dictval):
104 if fullpth.startswith(includedir) and fullpth.endswith('.h'):
105 headers.append(os.path.relpath(fullpth, includedir))
106 if pn and headers:
107 for header in headers:
108 RecipeHandler.recipeheadermap[header] = pn
109
43 @staticmethod 110 @staticmethod
44 def checkfiles(path, speclist, recursive=False): 111 def checkfiles(path, speclist, recursive=False):
45 results = [] 112 results = []
@@ -54,6 +121,74 @@ class RecipeHandler():
54 results.extend(glob.glob(os.path.join(path, spec))) 121 results.extend(glob.glob(os.path.join(path, spec)))
55 return results 122 return results
56 123
124 @staticmethod
125 def handle_depends(libdeps, pcdeps, deps, outlines, values, d):
126 if pcdeps:
127 recipemap = read_pkgconfig_provides(d)
128 if libdeps:
129 RecipeHandler.load_libmap(d)
130
131 ignorelibs = ['socket']
132 ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native']
133
134 unmappedpc = []
135 pcdeps = list(set(pcdeps))
136 for pcdep in pcdeps:
137 if isinstance(pcdep, basestring):
138 recipe = recipemap.get(pcdep, None)
139 if recipe:
140 deps.append(recipe)
141 else:
142 if not pcdep.startswith('$'):
143 unmappedpc.append(pcdep)
144 else:
145 for item in pcdep:
146 recipe = recipemap.get(pcdep, None)
147 if recipe:
148 deps.append(recipe)
149 break
150 else:
151 unmappedpc.append('(%s)' % ' or '.join(pcdep))
152
153 unmappedlibs = []
154 for libdep in libdeps:
155 if isinstance(libdep, tuple):
156 lib, header = libdep
157 else:
158 lib = libdep
159 header = None
160
161 if lib in ignorelibs:
162 logger.debug('Ignoring library dependency %s' % lib)
163 continue
164
165 recipe = RecipeHandler.recipelibmap.get(lib, None)
166 if recipe:
167 deps.append(recipe)
168 elif recipe is None:
169 if header:
170 RecipeHandler.load_headermap(d)
171 recipe = RecipeHandler.recipeheadermap.get(header, None)
172 if recipe:
173 deps.append(recipe)
174 elif recipe is None:
175 unmappedlibs.append(lib)
176 else:
177 unmappedlibs.append(lib)
178
179 deps = set(deps).difference(set(ignoredeps))
180
181 if unmappedpc:
182 outlines.append('# NOTE: unable to map the following pkg-config dependencies: %s' % ' '.join(unmappedpc))
183 outlines.append('# (this is based on recipes that have previously been built and packaged)')
184
185 if unmappedlibs:
186 outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmappedlibs))))
187 outlines.append('# (this is based on recipes that have previously been built and packaged)')
188
189 if deps:
190 values['DEPENDS'] = ' '.join(deps)
191
57 def genfunction(self, outlines, funcname, content, python=False, forcespace=False): 192 def genfunction(self, outlines, funcname, content, python=False, forcespace=False):
58 if python: 193 if python:
59 prefix = 'python ' 194 prefix = 'python '
diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py
index 127e13359b..40659d1ea8 100644
--- a/scripts/lib/recipetool/create_buildsys.py
+++ b/scripts/lib/recipetool/create_buildsys.py
@@ -17,7 +17,7 @@
17 17
18import re 18import re
19import logging 19import logging
20from recipetool.create import RecipeHandler, read_pkgconfig_provides, validate_pv 20from recipetool.create import RecipeHandler, validate_pv
21 21
22logger = logging.getLogger('recipetool') 22logger = logging.getLogger('recipetool')
23 23
@@ -143,9 +143,6 @@ class AutotoolsRecipeHandler(RecipeHandler):
143 @staticmethod 143 @staticmethod
144 def extract_autotools_deps(outlines, srctree, extravalues=None, acfile=None): 144 def extract_autotools_deps(outlines, srctree, extravalues=None, acfile=None):
145 import shlex 145 import shlex
146 import oe.package
147 import json
148 import glob
149 146
150 values = {} 147 values = {}
151 inherits = [] 148 inherits = []
@@ -159,9 +156,6 @@ class AutotoolsRecipeHandler(RecipeHandler):
159 progclassmap = {'gconftool-2': 'gconf', 156 progclassmap = {'gconftool-2': 'gconf',
160 'pkg-config': 'pkgconfig'} 157 'pkg-config': 'pkgconfig'}
161 158
162 ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native']
163 ignorelibs = ['socket']
164
165 pkg_re = re.compile('PKG_CHECK_MODULES\(\[?[a-zA-Z0-9_]*\]?, *\[?([^,\]]*)\]?[),].*') 159 pkg_re = re.compile('PKG_CHECK_MODULES\(\[?[a-zA-Z0-9_]*\]?, *\[?([^,\]]*)\]?[),].*')
166 pkgce_re = re.compile('PKG_CHECK_EXISTS\(\[?([^,\]]*)\]?[),].*') 160 pkgce_re = re.compile('PKG_CHECK_EXISTS\(\[?([^,\]]*)\]?[),].*')
167 lib_re = re.compile('AC_CHECK_LIB\(\[?([^,\]]*)\]?,.*') 161 lib_re = re.compile('AC_CHECK_LIB\(\[?([^,\]]*)\]?,.*')
@@ -172,62 +166,6 @@ class AutotoolsRecipeHandler(RecipeHandler):
172 am_init_re = re.compile('AM_INIT_AUTOMAKE\(([^,]+), *([^,]+)[,)].*') 166 am_init_re = re.compile('AM_INIT_AUTOMAKE\(([^,]+), *([^,]+)[,)].*')
173 define_re = re.compile(' *(m4_)?define\(([^,]+), *([^,]+)\)') 167 define_re = re.compile(' *(m4_)?define\(([^,]+), *([^,]+)\)')
174 168
175 # Build up lib library->package mapping
176 shlib_providers = oe.package.read_shlib_providers(tinfoil.config_data)
177 libdir = tinfoil.config_data.getVar('libdir', True)
178 base_libdir = tinfoil.config_data.getVar('base_libdir', True)
179 libpaths = list(set([base_libdir, libdir]))
180 libname_re = re.compile('^lib(.+)\.so.*$')
181 pkglibmap = {}
182 for lib, item in shlib_providers.iteritems():
183 for path, pkg in item.iteritems():
184 if path in libpaths:
185 res = libname_re.match(lib)
186 if res:
187 libname = res.group(1)
188 if not libname in pkglibmap:
189 pkglibmap[libname] = pkg[0]
190 else:
191 logger.debug('unable to extract library name from %s' % lib)
192
193 # Now turn it into a library->recipe mapping
194 recipelibmap = {}
195 recipeheadermap = {}
196 pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True)
197 for libname, pkg in pkglibmap.iteritems():
198 try:
199 with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
200 for line in f:
201 if line.startswith('PN:'):
202 recipelibmap[libname] = line.split(':', 1)[-1].strip()
203 break
204 except IOError as ioe:
205 if ioe.errno == 2:
206 logger.warn('unable to find a pkgdata file for package %s' % pkg)
207 else:
208 raise
209
210 def load_headermap():
211 if recipeheadermap:
212 return
213 includedir = tinfoil.config_data.getVar('includedir', True)
214 for pkg in glob.glob(os.path.join(pkgdata_dir, 'runtime', '*-dev')):
215 with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f:
216 pn = None
217 headers = []
218 for line in f:
219 if line.startswith('PN:'):
220 pn = line.split(':', 1)[-1].strip()
221 elif line.startswith('FILES_INFO:'):
222 val = line.split(':', 1)[1].strip()
223 dictval = json.loads(val)
224 for fullpth in sorted(dictval):
225 if fullpth.startswith(includedir) and fullpth.endswith('.h'):
226 headers.append(os.path.relpath(fullpth, includedir))
227 if pn and headers:
228 for header in headers:
229 recipeheadermap[header] = pn
230
231 defines = {} 169 defines = {}
232 def subst_defines(value): 170 def subst_defines(value):
233 newvalue = value 171 newvalue = value
@@ -263,9 +201,9 @@ class AutotoolsRecipeHandler(RecipeHandler):
263 srcfiles = RecipeHandler.checkfiles(srctree, ['acinclude.m4', 'configure.ac', 'configure.in']) 201 srcfiles = RecipeHandler.checkfiles(srctree, ['acinclude.m4', 'configure.ac', 'configure.in'])
264 202
265 pcdeps = [] 203 pcdeps = []
204 libdeps = []
266 deps = [] 205 deps = []
267 unmapped = [] 206 unmapped = []
268 unmappedlibs = []
269 207
270 def process_macro(keyword, value): 208 def process_macro(keyword, value):
271 if keyword == 'PKG_CHECK_MODULES': 209 if keyword == 'PKG_CHECK_MODULES':
@@ -307,36 +245,15 @@ class AutotoolsRecipeHandler(RecipeHandler):
307 res = lib_re.search(value) 245 res = lib_re.search(value)
308 if res: 246 if res:
309 lib = res.group(1) 247 lib = res.group(1)
310 if lib in ignorelibs: 248 if not lib.startswith('$'):
311 logger.debug('Ignoring library dependency %s' % lib) 249 libdeps.append(lib)
312 else:
313 libdep = recipelibmap.get(lib, None)
314 if libdep:
315 deps.append(libdep)
316 else:
317 if libdep is None:
318 if not lib.startswith('$'):
319 unmappedlibs.append(lib)
320 elif keyword == 'AX_CHECK_LIBRARY': 250 elif keyword == 'AX_CHECK_LIBRARY':
321 res = libx_re.search(value) 251 res = libx_re.search(value)
322 if res: 252 if res:
323 lib = res.group(2) 253 lib = res.group(2)
324 if lib in ignorelibs: 254 if not lib.startswith('$'):
325 logger.debug('Ignoring library dependency %s' % lib) 255 header = res.group(1)
326 else: 256 libdeps.add((lib, header))
327 libdep = recipelibmap.get(lib, None)
328 if libdep:
329 deps.append(libdep)
330 else:
331 if libdep is None:
332 if not lib.startswith('$'):
333 header = res.group(1)
334 load_headermap()
335 libdep = recipeheadermap.get(header, None)
336 if libdep:
337 deps.append(libdep)
338 else:
339 unmappedlibs.append(lib)
340 elif keyword == 'AC_PATH_X': 257 elif keyword == 'AC_PATH_X':
341 deps.append('libx11') 258 deps.append('libx11')
342 elif keyword in ('AX_BOOST', 'BOOST_REQUIRE'): 259 elif keyword in ('AX_BOOST', 'BOOST_REQUIRE'):
@@ -484,29 +401,7 @@ class AutotoolsRecipeHandler(RecipeHandler):
484 if unmapped: 401 if unmapped:
485 outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmapped)))) 402 outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmapped))))
486 403
487 if unmappedlibs: 404 RecipeHandler.handle_depends(libdeps, pcdeps, deps, outlines, values, tinfoil.config_data)
488 outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmappedlibs))))
489 outlines.append('# (this is based on recipes that have previously been built and packaged)')
490
491 recipemap = read_pkgconfig_provides(tinfoil.config_data)
492 unmapped = []
493 pcdeps = list(set(pcdeps))
494 for pcdep in pcdeps:
495 recipe = recipemap.get(pcdep, None)
496 if recipe:
497 deps.append(recipe)
498 else:
499 if not pcdep.startswith('$'):
500 unmapped.append(pcdep)
501
502 deps = set(deps).difference(set(ignoredeps))
503
504 if unmapped:
505 outlines.append('# NOTE: unable to map the following pkg-config dependencies: %s' % ' '.join(unmapped))
506 outlines.append('# (this is based on recipes that have previously been built and packaged)')
507
508 if deps:
509 values['DEPENDS'] = ' '.join(deps)
510 405
511 if inherits: 406 if inherits:
512 values['inherit'] = ' '.join(list(set(inherits))) 407 values['inherit'] = ' '.join(list(set(inherits)))