diff options
-rw-r--r-- | scripts/lib/recipetool/create.py | 137 | ||||
-rw-r--r-- | scripts/lib/recipetool/create_buildsys.py | 121 |
2 files changed, 144 insertions, 114 deletions
diff --git a/scripts/lib/recipetool/create.py b/scripts/lib/recipetool/create.py index 1218a7d284..dab917faac 100644 --- a/scripts/lib/recipetool/create.py +++ b/scripts/lib/recipetool/create.py | |||
@@ -21,6 +21,7 @@ import argparse | |||
21 | import glob | 21 | import glob |
22 | import fnmatch | 22 | import fnmatch |
23 | import re | 23 | import re |
24 | import json | ||
24 | import logging | 25 | import logging |
25 | import scriptutils | 26 | import scriptutils |
26 | import urlparse | 27 | import urlparse |
@@ -39,7 +40,73 @@ def tinfoil_init(instance): | |||
39 | global tinfoil | 40 | global tinfoil |
40 | tinfoil = instance | 41 | tinfoil = instance |
41 | 42 | ||
42 | class RecipeHandler(): | 43 | class RecipeHandler(object): |
44 | recipelibmap = {} | ||
45 | recipeheadermap = {} | ||
46 | |||
47 | @staticmethod | ||
48 | def load_libmap(d): | ||
49 | '''Load library->recipe mapping''' | ||
50 | import oe.package | ||
51 | |||
52 | if RecipeHandler.recipelibmap: | ||
53 | return | ||
54 | # First build up library->package mapping | ||
55 | shlib_providers = oe.package.read_shlib_providers(d) | ||
56 | libdir = d.getVar('libdir', True) | ||
57 | base_libdir = d.getVar('base_libdir', True) | ||
58 | libpaths = list(set([base_libdir, libdir])) | ||
59 | libname_re = re.compile('^lib(.+)\.so.*$') | ||
60 | pkglibmap = {} | ||
61 | for lib, item in shlib_providers.iteritems(): | ||
62 | for path, pkg in item.iteritems(): | ||
63 | if path in libpaths: | ||
64 | res = libname_re.match(lib) | ||
65 | if res: | ||
66 | libname = res.group(1) | ||
67 | if not libname in pkglibmap: | ||
68 | pkglibmap[libname] = pkg[0] | ||
69 | else: | ||
70 | logger.debug('unable to extract library name from %s' % lib) | ||
71 | |||
72 | # Now turn it into a library->recipe mapping | ||
73 | pkgdata_dir = d.getVar('PKGDATA_DIR', True) | ||
74 | for libname, pkg in pkglibmap.iteritems(): | ||
75 | try: | ||
76 | with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: | ||
77 | for line in f: | ||
78 | if line.startswith('PN:'): | ||
79 | RecipeHandler.recipelibmap[libname] = line.split(':', 1)[-1].strip() | ||
80 | break | ||
81 | except IOError as ioe: | ||
82 | if ioe.errno == 2: | ||
83 | logger.warn('unable to find a pkgdata file for package %s' % pkg) | ||
84 | else: | ||
85 | raise | ||
86 | |||
87 | @staticmethod | ||
88 | def load_headermap(d): | ||
89 | '''Build up lib headerfile->recipe mapping''' | ||
90 | if RecipeHandler.recipeheadermap: | ||
91 | return | ||
92 | includedir = d.getVar('includedir', True) | ||
93 | for pkg in glob.glob(os.path.join(pkgdata_dir, 'runtime', '*-dev')): | ||
94 | with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: | ||
95 | pn = None | ||
96 | headers = [] | ||
97 | for line in f: | ||
98 | if line.startswith('PN:'): | ||
99 | pn = line.split(':', 1)[-1].strip() | ||
100 | elif line.startswith('FILES_INFO:'): | ||
101 | val = line.split(':', 1)[1].strip() | ||
102 | dictval = json.loads(val) | ||
103 | for fullpth in sorted(dictval): | ||
104 | if fullpth.startswith(includedir) and fullpth.endswith('.h'): | ||
105 | headers.append(os.path.relpath(fullpth, includedir)) | ||
106 | if pn and headers: | ||
107 | for header in headers: | ||
108 | RecipeHandler.recipeheadermap[header] = pn | ||
109 | |||
43 | @staticmethod | 110 | @staticmethod |
44 | def checkfiles(path, speclist, recursive=False): | 111 | def checkfiles(path, speclist, recursive=False): |
45 | results = [] | 112 | results = [] |
@@ -54,6 +121,74 @@ class RecipeHandler(): | |||
54 | results.extend(glob.glob(os.path.join(path, spec))) | 121 | results.extend(glob.glob(os.path.join(path, spec))) |
55 | return results | 122 | return results |
56 | 123 | ||
124 | @staticmethod | ||
125 | def handle_depends(libdeps, pcdeps, deps, outlines, values, d): | ||
126 | if pcdeps: | ||
127 | recipemap = read_pkgconfig_provides(d) | ||
128 | if libdeps: | ||
129 | RecipeHandler.load_libmap(d) | ||
130 | |||
131 | ignorelibs = ['socket'] | ||
132 | ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native'] | ||
133 | |||
134 | unmappedpc = [] | ||
135 | pcdeps = list(set(pcdeps)) | ||
136 | for pcdep in pcdeps: | ||
137 | if isinstance(pcdep, basestring): | ||
138 | recipe = recipemap.get(pcdep, None) | ||
139 | if recipe: | ||
140 | deps.append(recipe) | ||
141 | else: | ||
142 | if not pcdep.startswith('$'): | ||
143 | unmappedpc.append(pcdep) | ||
144 | else: | ||
145 | for item in pcdep: | ||
146 | recipe = recipemap.get(pcdep, None) | ||
147 | if recipe: | ||
148 | deps.append(recipe) | ||
149 | break | ||
150 | else: | ||
151 | unmappedpc.append('(%s)' % ' or '.join(pcdep)) | ||
152 | |||
153 | unmappedlibs = [] | ||
154 | for libdep in libdeps: | ||
155 | if isinstance(libdep, tuple): | ||
156 | lib, header = libdep | ||
157 | else: | ||
158 | lib = libdep | ||
159 | header = None | ||
160 | |||
161 | if lib in ignorelibs: | ||
162 | logger.debug('Ignoring library dependency %s' % lib) | ||
163 | continue | ||
164 | |||
165 | recipe = RecipeHandler.recipelibmap.get(lib, None) | ||
166 | if recipe: | ||
167 | deps.append(recipe) | ||
168 | elif recipe is None: | ||
169 | if header: | ||
170 | RecipeHandler.load_headermap(d) | ||
171 | recipe = RecipeHandler.recipeheadermap.get(header, None) | ||
172 | if recipe: | ||
173 | deps.append(recipe) | ||
174 | elif recipe is None: | ||
175 | unmappedlibs.append(lib) | ||
176 | else: | ||
177 | unmappedlibs.append(lib) | ||
178 | |||
179 | deps = set(deps).difference(set(ignoredeps)) | ||
180 | |||
181 | if unmappedpc: | ||
182 | outlines.append('# NOTE: unable to map the following pkg-config dependencies: %s' % ' '.join(unmappedpc)) | ||
183 | outlines.append('# (this is based on recipes that have previously been built and packaged)') | ||
184 | |||
185 | if unmappedlibs: | ||
186 | outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmappedlibs)))) | ||
187 | outlines.append('# (this is based on recipes that have previously been built and packaged)') | ||
188 | |||
189 | if deps: | ||
190 | values['DEPENDS'] = ' '.join(deps) | ||
191 | |||
57 | def genfunction(self, outlines, funcname, content, python=False, forcespace=False): | 192 | def genfunction(self, outlines, funcname, content, python=False, forcespace=False): |
58 | if python: | 193 | if python: |
59 | prefix = 'python ' | 194 | prefix = 'python ' |
diff --git a/scripts/lib/recipetool/create_buildsys.py b/scripts/lib/recipetool/create_buildsys.py index 127e13359b..40659d1ea8 100644 --- a/scripts/lib/recipetool/create_buildsys.py +++ b/scripts/lib/recipetool/create_buildsys.py | |||
@@ -17,7 +17,7 @@ | |||
17 | 17 | ||
18 | import re | 18 | import re |
19 | import logging | 19 | import logging |
20 | from recipetool.create import RecipeHandler, read_pkgconfig_provides, validate_pv | 20 | from recipetool.create import RecipeHandler, validate_pv |
21 | 21 | ||
22 | logger = logging.getLogger('recipetool') | 22 | logger = logging.getLogger('recipetool') |
23 | 23 | ||
@@ -143,9 +143,6 @@ class AutotoolsRecipeHandler(RecipeHandler): | |||
143 | @staticmethod | 143 | @staticmethod |
144 | def extract_autotools_deps(outlines, srctree, extravalues=None, acfile=None): | 144 | def extract_autotools_deps(outlines, srctree, extravalues=None, acfile=None): |
145 | import shlex | 145 | import shlex |
146 | import oe.package | ||
147 | import json | ||
148 | import glob | ||
149 | 146 | ||
150 | values = {} | 147 | values = {} |
151 | inherits = [] | 148 | inherits = [] |
@@ -159,9 +156,6 @@ class AutotoolsRecipeHandler(RecipeHandler): | |||
159 | progclassmap = {'gconftool-2': 'gconf', | 156 | progclassmap = {'gconftool-2': 'gconf', |
160 | 'pkg-config': 'pkgconfig'} | 157 | 'pkg-config': 'pkgconfig'} |
161 | 158 | ||
162 | ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'musl', 'tar-native', 'binutils-native'] | ||
163 | ignorelibs = ['socket'] | ||
164 | |||
165 | pkg_re = re.compile('PKG_CHECK_MODULES\(\[?[a-zA-Z0-9_]*\]?, *\[?([^,\]]*)\]?[),].*') | 159 | pkg_re = re.compile('PKG_CHECK_MODULES\(\[?[a-zA-Z0-9_]*\]?, *\[?([^,\]]*)\]?[),].*') |
166 | pkgce_re = re.compile('PKG_CHECK_EXISTS\(\[?([^,\]]*)\]?[),].*') | 160 | pkgce_re = re.compile('PKG_CHECK_EXISTS\(\[?([^,\]]*)\]?[),].*') |
167 | lib_re = re.compile('AC_CHECK_LIB\(\[?([^,\]]*)\]?,.*') | 161 | lib_re = re.compile('AC_CHECK_LIB\(\[?([^,\]]*)\]?,.*') |
@@ -172,62 +166,6 @@ class AutotoolsRecipeHandler(RecipeHandler): | |||
172 | am_init_re = re.compile('AM_INIT_AUTOMAKE\(([^,]+), *([^,]+)[,)].*') | 166 | am_init_re = re.compile('AM_INIT_AUTOMAKE\(([^,]+), *([^,]+)[,)].*') |
173 | define_re = re.compile(' *(m4_)?define\(([^,]+), *([^,]+)\)') | 167 | define_re = re.compile(' *(m4_)?define\(([^,]+), *([^,]+)\)') |
174 | 168 | ||
175 | # Build up lib library->package mapping | ||
176 | shlib_providers = oe.package.read_shlib_providers(tinfoil.config_data) | ||
177 | libdir = tinfoil.config_data.getVar('libdir', True) | ||
178 | base_libdir = tinfoil.config_data.getVar('base_libdir', True) | ||
179 | libpaths = list(set([base_libdir, libdir])) | ||
180 | libname_re = re.compile('^lib(.+)\.so.*$') | ||
181 | pkglibmap = {} | ||
182 | for lib, item in shlib_providers.iteritems(): | ||
183 | for path, pkg in item.iteritems(): | ||
184 | if path in libpaths: | ||
185 | res = libname_re.match(lib) | ||
186 | if res: | ||
187 | libname = res.group(1) | ||
188 | if not libname in pkglibmap: | ||
189 | pkglibmap[libname] = pkg[0] | ||
190 | else: | ||
191 | logger.debug('unable to extract library name from %s' % lib) | ||
192 | |||
193 | # Now turn it into a library->recipe mapping | ||
194 | recipelibmap = {} | ||
195 | recipeheadermap = {} | ||
196 | pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) | ||
197 | for libname, pkg in pkglibmap.iteritems(): | ||
198 | try: | ||
199 | with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: | ||
200 | for line in f: | ||
201 | if line.startswith('PN:'): | ||
202 | recipelibmap[libname] = line.split(':', 1)[-1].strip() | ||
203 | break | ||
204 | except IOError as ioe: | ||
205 | if ioe.errno == 2: | ||
206 | logger.warn('unable to find a pkgdata file for package %s' % pkg) | ||
207 | else: | ||
208 | raise | ||
209 | |||
210 | def load_headermap(): | ||
211 | if recipeheadermap: | ||
212 | return | ||
213 | includedir = tinfoil.config_data.getVar('includedir', True) | ||
214 | for pkg in glob.glob(os.path.join(pkgdata_dir, 'runtime', '*-dev')): | ||
215 | with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: | ||
216 | pn = None | ||
217 | headers = [] | ||
218 | for line in f: | ||
219 | if line.startswith('PN:'): | ||
220 | pn = line.split(':', 1)[-1].strip() | ||
221 | elif line.startswith('FILES_INFO:'): | ||
222 | val = line.split(':', 1)[1].strip() | ||
223 | dictval = json.loads(val) | ||
224 | for fullpth in sorted(dictval): | ||
225 | if fullpth.startswith(includedir) and fullpth.endswith('.h'): | ||
226 | headers.append(os.path.relpath(fullpth, includedir)) | ||
227 | if pn and headers: | ||
228 | for header in headers: | ||
229 | recipeheadermap[header] = pn | ||
230 | |||
231 | defines = {} | 169 | defines = {} |
232 | def subst_defines(value): | 170 | def subst_defines(value): |
233 | newvalue = value | 171 | newvalue = value |
@@ -263,9 +201,9 @@ class AutotoolsRecipeHandler(RecipeHandler): | |||
263 | srcfiles = RecipeHandler.checkfiles(srctree, ['acinclude.m4', 'configure.ac', 'configure.in']) | 201 | srcfiles = RecipeHandler.checkfiles(srctree, ['acinclude.m4', 'configure.ac', 'configure.in']) |
264 | 202 | ||
265 | pcdeps = [] | 203 | pcdeps = [] |
204 | libdeps = [] | ||
266 | deps = [] | 205 | deps = [] |
267 | unmapped = [] | 206 | unmapped = [] |
268 | unmappedlibs = [] | ||
269 | 207 | ||
270 | def process_macro(keyword, value): | 208 | def process_macro(keyword, value): |
271 | if keyword == 'PKG_CHECK_MODULES': | 209 | if keyword == 'PKG_CHECK_MODULES': |
@@ -307,36 +245,15 @@ class AutotoolsRecipeHandler(RecipeHandler): | |||
307 | res = lib_re.search(value) | 245 | res = lib_re.search(value) |
308 | if res: | 246 | if res: |
309 | lib = res.group(1) | 247 | lib = res.group(1) |
310 | if lib in ignorelibs: | 248 | if not lib.startswith('$'): |
311 | logger.debug('Ignoring library dependency %s' % lib) | 249 | libdeps.append(lib) |
312 | else: | ||
313 | libdep = recipelibmap.get(lib, None) | ||
314 | if libdep: | ||
315 | deps.append(libdep) | ||
316 | else: | ||
317 | if libdep is None: | ||
318 | if not lib.startswith('$'): | ||
319 | unmappedlibs.append(lib) | ||
320 | elif keyword == 'AX_CHECK_LIBRARY': | 250 | elif keyword == 'AX_CHECK_LIBRARY': |
321 | res = libx_re.search(value) | 251 | res = libx_re.search(value) |
322 | if res: | 252 | if res: |
323 | lib = res.group(2) | 253 | lib = res.group(2) |
324 | if lib in ignorelibs: | 254 | if not lib.startswith('$'): |
325 | logger.debug('Ignoring library dependency %s' % lib) | 255 | header = res.group(1) |
326 | else: | 256 | libdeps.add((lib, header)) |
327 | libdep = recipelibmap.get(lib, None) | ||
328 | if libdep: | ||
329 | deps.append(libdep) | ||
330 | else: | ||
331 | if libdep is None: | ||
332 | if not lib.startswith('$'): | ||
333 | header = res.group(1) | ||
334 | load_headermap() | ||
335 | libdep = recipeheadermap.get(header, None) | ||
336 | if libdep: | ||
337 | deps.append(libdep) | ||
338 | else: | ||
339 | unmappedlibs.append(lib) | ||
340 | elif keyword == 'AC_PATH_X': | 257 | elif keyword == 'AC_PATH_X': |
341 | deps.append('libx11') | 258 | deps.append('libx11') |
342 | elif keyword in ('AX_BOOST', 'BOOST_REQUIRE'): | 259 | elif keyword in ('AX_BOOST', 'BOOST_REQUIRE'): |
@@ -484,29 +401,7 @@ class AutotoolsRecipeHandler(RecipeHandler): | |||
484 | if unmapped: | 401 | if unmapped: |
485 | outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmapped)))) | 402 | outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmapped)))) |
486 | 403 | ||
487 | if unmappedlibs: | 404 | RecipeHandler.handle_depends(libdeps, pcdeps, deps, outlines, values, tinfoil.config_data) |
488 | outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmappedlibs)))) | ||
489 | outlines.append('# (this is based on recipes that have previously been built and packaged)') | ||
490 | |||
491 | recipemap = read_pkgconfig_provides(tinfoil.config_data) | ||
492 | unmapped = [] | ||
493 | pcdeps = list(set(pcdeps)) | ||
494 | for pcdep in pcdeps: | ||
495 | recipe = recipemap.get(pcdep, None) | ||
496 | if recipe: | ||
497 | deps.append(recipe) | ||
498 | else: | ||
499 | if not pcdep.startswith('$'): | ||
500 | unmapped.append(pcdep) | ||
501 | |||
502 | deps = set(deps).difference(set(ignoredeps)) | ||
503 | |||
504 | if unmapped: | ||
505 | outlines.append('# NOTE: unable to map the following pkg-config dependencies: %s' % ' '.join(unmapped)) | ||
506 | outlines.append('# (this is based on recipes that have previously been built and packaged)') | ||
507 | |||
508 | if deps: | ||
509 | values['DEPENDS'] = ' '.join(deps) | ||
510 | 405 | ||
511 | if inherits: | 406 | if inherits: |
512 | values['inherit'] = ' '.join(list(set(inherits))) | 407 | values['inherit'] = ' '.join(list(set(inherits))) |