summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bblayers/query.py
diff options
context:
space:
mode:
authorChristopher Larson <chris_larson@mentor.com>2016-04-30 12:41:00 -0700
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-05-17 21:16:35 +0100
commit6bbe4fe48c93d1e2b67a36aa82d2ea5e289a219c (patch)
tree802427823189e16ffcd6560acfda961303873adc /bitbake/lib/bblayers/query.py
parent07eebc66898b423ea2e04e96c57d3b2b5eabbb26 (diff)
downloadpoky-6bbe4fe48c93d1e2b67a36aa82d2ea5e289a219c.tar.gz
bitbake: bitbake-layers: convert to plugin-based
This uses bb.utils.load_plugins, based on the plugin handling in recipetool and devtool in oe-core. (Bitbake rev: 5e542df9b966a99b5a5b8aa7cf6100174aff54b2) Signed-off-by: Christopher Larson <chris_larson@mentor.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bblayers/query.py')
-rw-r--r--bitbake/lib/bblayers/query.py500
1 files changed, 500 insertions, 0 deletions
diff --git a/bitbake/lib/bblayers/query.py b/bitbake/lib/bblayers/query.py
new file mode 100644
index 0000000000..b5b98f7639
--- /dev/null
+++ b/bitbake/lib/bblayers/query.py
@@ -0,0 +1,500 @@
1import collections
2import fnmatch
3import logging
4import sys
5import os
6import re
7
8import bb.cache
9import bb.providers
10import bb.utils
11
12from bblayers.common import LayerPlugin
13
14logger = logging.getLogger('bitbake-layers')
15
16
17def plugin_init(plugins):
18 return QueryPlugin()
19
20
21class QueryPlugin(LayerPlugin):
22 def do_show_layers(self, args):
23 """show current configured layers."""
24 logger.plain("%s %s %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
25 logger.plain('=' * 74)
26 for layer, _, regex, pri in self.tinfoil.cooker.recipecache.bbfile_config_priorities:
27 layerdir = self.bbfile_collections.get(layer, None)
28 layername = self.get_layer_name(layerdir)
29 logger.plain("%s %s %d" % (layername.ljust(20), layerdir.ljust(40), pri))
30
31 def version_str(self, pe, pv, pr = None):
32 verstr = "%s" % pv
33 if pr:
34 verstr = "%s-%s" % (verstr, pr)
35 if pe:
36 verstr = "%s:%s" % (pe, verstr)
37 return verstr
38
39 def do_show_overlayed(self, args):
40 """list overlayed recipes (where the same recipe exists in another layer)
41
42Lists the names of overlayed recipes and the available versions in each
43layer, with the preferred version first. Note that skipped recipes that
44are overlayed will also be listed, with a " (skipped)" suffix.
45"""
46
47 items_listed = self.list_recipes('Overlayed recipes', None, True, args.same_version, args.filenames, True, None)
48
49 # Check for overlayed .bbclass files
50 classes = collections.defaultdict(list)
51 for layerdir in self.bblayers:
52 classdir = os.path.join(layerdir, 'classes')
53 if os.path.exists(classdir):
54 for classfile in os.listdir(classdir):
55 if os.path.splitext(classfile)[1] == '.bbclass':
56 classes[classfile].append(classdir)
57
58 # Locating classes and other files is a bit more complicated than recipes -
59 # layer priority is not a factor; instead BitBake uses the first matching
60 # file in BBPATH, which is manipulated directly by each layer's
61 # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a
62 # factor - however, each layer.conf is free to either prepend or append to
63 # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might
64 # not be exactly the order present in bblayers.conf either.
65 bbpath = str(self.tinfoil.config_data.getVar('BBPATH', True))
66 overlayed_class_found = False
67 for (classfile, classdirs) in classes.items():
68 if len(classdirs) > 1:
69 if not overlayed_class_found:
70 logger.plain('=== Overlayed classes ===')
71 overlayed_class_found = True
72
73 mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile))
74 if args.filenames:
75 logger.plain('%s' % mainfile)
76 else:
77 # We effectively have to guess the layer here
78 logger.plain('%s:' % classfile)
79 mainlayername = '?'
80 for layerdir in self.bblayers:
81 classdir = os.path.join(layerdir, 'classes')
82 if mainfile.startswith(classdir):
83 mainlayername = self.get_layer_name(layerdir)
84 logger.plain(' %s' % mainlayername)
85 for classdir in classdirs:
86 fullpath = os.path.join(classdir, classfile)
87 if fullpath != mainfile:
88 if args.filenames:
89 print(' %s' % fullpath)
90 else:
91 print(' %s' % self.get_layer_name(os.path.dirname(classdir)))
92
93 if overlayed_class_found:
94 items_listed = True;
95
96 if not items_listed:
97 logger.plain('No overlayed files found.')
98
99 def do_show_recipes(self, args):
100 """list available recipes, showing the layer they are provided by
101
102Lists the names of recipes and the available versions in each
103layer, with the preferred version first. Optionally you may specify
104pnspec to match a specified recipe name (supports wildcards). Note that
105skipped recipes will also be listed, with a " (skipped)" suffix.
106"""
107
108 inheritlist = args.inherits.split(',') if args.inherits else []
109 if inheritlist or args.pnspec or args.multiple:
110 title = 'Matching recipes:'
111 else:
112 title = 'Available recipes:'
113 self.list_recipes(title, args.pnspec, False, False, args.filenames, args.multiple, inheritlist)
114
115 def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_multi_provider_only, inherits):
116 if inherits:
117 bbpath = str(self.tinfoil.config_data.getVar('BBPATH', True))
118 for classname in inherits:
119 classfile = 'classes/%s.bbclass' % classname
120 if not bb.utils.which(bbpath, classfile, history=False):
121 logger.error('No class named %s found in BBPATH', classfile)
122 sys.exit(1)
123
124 pkg_pn = self.tinfoil.cooker.recipecache.pkg_pn
125 (latest_versions, preferred_versions) = bb.providers.findProviders(self.tinfoil.config_data, self.tinfoil.cooker.recipecache, pkg_pn)
126 allproviders = bb.providers.allProviders(self.tinfoil.cooker.recipecache)
127
128 # Ensure we list skipped recipes
129 # We are largely guessing about PN, PV and the preferred version here,
130 # but we have no choice since skipped recipes are not fully parsed
131 skiplist = self.tinfoil.cooker.skiplist.keys()
132 skiplist.sort( key=lambda fileitem: self.tinfoil.cooker.collection.calc_bbfile_priority(fileitem) )
133 skiplist.reverse()
134 for fn in skiplist:
135 recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
136 p = recipe_parts[0]
137 if len(recipe_parts) > 1:
138 ver = (None, recipe_parts[1], None)
139 else:
140 ver = (None, 'unknown', None)
141 allproviders[p].append((ver, fn))
142 if not p in pkg_pn:
143 pkg_pn[p] = 'dummy'
144 preferred_versions[p] = (ver, fn)
145
146 def print_item(f, pn, ver, layer, ispref):
147 if f in skiplist:
148 skipped = ' (skipped)'
149 else:
150 skipped = ''
151 if show_filenames:
152 if ispref:
153 logger.plain("%s%s", f, skipped)
154 else:
155 logger.plain(" %s%s", f, skipped)
156 else:
157 if ispref:
158 logger.plain("%s:", pn)
159 logger.plain(" %s %s%s", layer.ljust(20), ver, skipped)
160
161 global_inherit = (self.tinfoil.config_data.getVar('INHERIT', True) or "").split()
162 cls_re = re.compile('classes/')
163
164 preffiles = []
165 items_listed = False
166 for p in sorted(pkg_pn):
167 if pnspec:
168 if not fnmatch.fnmatch(p, pnspec):
169 continue
170
171 if len(allproviders[p]) > 1 or not show_multi_provider_only:
172 pref = preferred_versions[p]
173 realfn = bb.cache.Cache.virtualfn2realfn(pref[1])
174 preffile = realfn[0]
175
176 # We only display once per recipe, we should prefer non extended versions of the
177 # recipe if present (so e.g. in OpenEmbedded, openssl rather than nativesdk-openssl
178 # which would otherwise sort first).
179 if realfn[1] and realfn[0] in self.tinfoil.cooker.recipecache.pkg_fn:
180 continue
181
182 if inherits:
183 matchcount = 0
184 recipe_inherits = self.tinfoil.cooker_data.inherits.get(preffile, [])
185 for cls in recipe_inherits:
186 if cls_re.match(cls):
187 continue
188 classname = os.path.splitext(os.path.basename(cls))[0]
189 if classname in global_inherit:
190 continue
191 elif classname in inherits:
192 matchcount += 1
193 if matchcount != len(inherits):
194 # No match - skip this recipe
195 continue
196
197 if preffile not in preffiles:
198 preflayer = self.get_file_layer(preffile)
199 multilayer = False
200 same_ver = True
201 provs = []
202 for prov in allproviders[p]:
203 provfile = bb.cache.Cache.virtualfn2realfn(prov[1])[0]
204 provlayer = self.get_file_layer(provfile)
205 provs.append((provfile, provlayer, prov[0]))
206 if provlayer != preflayer:
207 multilayer = True
208 if prov[0] != pref[0]:
209 same_ver = False
210
211 if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only):
212 if not items_listed:
213 logger.plain('=== %s ===' % title)
214 items_listed = True
215 print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True)
216 for (provfile, provlayer, provver) in provs:
217 if provfile != preffile:
218 print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False)
219 # Ensure we don't show two entries for BBCLASSEXTENDed recipes
220 preffiles.append(preffile)
221
222 return items_listed
223
224 def get_file_layer(self, filename):
225 layerdir = self.get_file_layerdir(filename)
226 if layerdir:
227 return self.get_layer_name(layerdir)
228 else:
229 return '?'
230
231 def get_file_layerdir(self, filename):
232 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data)
233 return self.bbfile_collections.get(layer, None)
234
235 def remove_layer_prefix(self, f):
236 """Remove the layer_dir prefix, e.g., f = /path/to/layer_dir/foo/blah, the
237 return value will be: layer_dir/foo/blah"""
238 f_layerdir = self.get_file_layerdir(f)
239 if not f_layerdir:
240 return f
241 prefix = os.path.join(os.path.dirname(f_layerdir), '')
242 return f[len(prefix):] if f.startswith(prefix) else f
243
244 def do_show_appends(self, args):
245 """list bbappend files and recipe files they apply to
246
247Lists recipes with the bbappends that apply to them as subitems.
248"""
249
250 logger.plain('=== Appended recipes ===')
251
252 pnlist = list(self.tinfoil.cooker_data.pkg_pn.keys())
253 pnlist.sort()
254 appends = False
255 for pn in pnlist:
256 if self.show_appends_for_pn(pn):
257 appends = True
258
259 if self.show_appends_for_skipped():
260 appends = True
261
262 if not appends:
263 logger.plain('No append files found')
264
265 def show_appends_for_pn(self, pn):
266 filenames = self.tinfoil.cooker_data.pkg_pn[pn]
267
268 best = bb.providers.findBestProvider(pn,
269 self.tinfoil.config_data,
270 self.tinfoil.cooker_data,
271 self.tinfoil.cooker_data.pkg_pn)
272 best_filename = os.path.basename(best[3])
273
274 return self.show_appends_output(filenames, best_filename)
275
276 def show_appends_for_skipped(self):
277 filenames = [os.path.basename(f)
278 for f in self.tinfoil.cooker.skiplist.iterkeys()]
279 return self.show_appends_output(filenames, None, " (skipped)")
280
281 def show_appends_output(self, filenames, best_filename, name_suffix = ''):
282 appended, missing = self.get_appends_for_files(filenames)
283 if appended:
284 for basename, appends in appended:
285 logger.plain('%s%s:', basename, name_suffix)
286 for append in appends:
287 logger.plain(' %s', append)
288
289 if best_filename:
290 if best_filename in missing:
291 logger.warning('%s: missing append for preferred version',
292 best_filename)
293 return True
294 else:
295 return False
296
297 def get_appends_for_files(self, filenames):
298 appended, notappended = [], []
299 for filename in filenames:
300 _, cls = bb.cache.Cache.virtualfn2realfn(filename)
301 if cls:
302 continue
303
304 basename = os.path.basename(filename)
305 appends = self.tinfoil.cooker.collection.get_file_appends(basename)
306 if appends:
307 appended.append((basename, list(appends)))
308 else:
309 notappended.append(basename)
310 return appended, notappended
311
312 def do_show_cross_depends(self, args):
313 """Show dependencies between recipes that cross layer boundaries.
314
315Figure out the dependencies between recipes that cross layer boundaries.
316
317NOTE: .bbappend files can impact the dependencies.
318"""
319 ignore_layers = (args.ignore or '').split(',')
320
321 pkg_fn = self.tinfoil.cooker_data.pkg_fn
322 bbpath = str(self.tinfoil.config_data.getVar('BBPATH', True))
323 self.require_re = re.compile(r"require\s+(.+)")
324 self.include_re = re.compile(r"include\s+(.+)")
325 self.inherit_re = re.compile(r"inherit\s+(.+)")
326
327 global_inherit = (self.tinfoil.config_data.getVar('INHERIT', True) or "").split()
328
329 # The bb's DEPENDS and RDEPENDS
330 for f in pkg_fn:
331 f = bb.cache.Cache.virtualfn2realfn(f)[0]
332 # Get the layername that the file is in
333 layername = self.get_file_layer(f)
334
335 # The DEPENDS
336 deps = self.tinfoil.cooker_data.deps[f]
337 for pn in deps:
338 if pn in self.tinfoil.cooker_data.pkg_pn:
339 best = bb.providers.findBestProvider(pn,
340 self.tinfoil.config_data,
341 self.tinfoil.cooker_data,
342 self.tinfoil.cooker_data.pkg_pn)
343 self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers)
344
345 # The RDPENDS
346 all_rdeps = self.tinfoil.cooker_data.rundeps[f].values()
347 # Remove the duplicated or null one.
348 sorted_rdeps = {}
349 # The all_rdeps is the list in list, so we need two for loops
350 for k1 in all_rdeps:
351 for k2 in k1:
352 sorted_rdeps[k2] = 1
353 all_rdeps = sorted_rdeps.keys()
354 for rdep in all_rdeps:
355 all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rdep)
356 if all_p:
357 if f in all_p:
358 # The recipe provides this one itself, ignore
359 continue
360 best = bb.providers.filterProvidersRunTime(all_p, rdep,
361 self.tinfoil.config_data,
362 self.tinfoil.cooker_data)[0][0]
363 self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers)
364
365 # The RRECOMMENDS
366 all_rrecs = self.tinfoil.cooker_data.runrecs[f].values()
367 # Remove the duplicated or null one.
368 sorted_rrecs = {}
369 # The all_rrecs is the list in list, so we need two for loops
370 for k1 in all_rrecs:
371 for k2 in k1:
372 sorted_rrecs[k2] = 1
373 all_rrecs = sorted_rrecs.keys()
374 for rrec in all_rrecs:
375 all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rrec)
376 if all_p:
377 if f in all_p:
378 # The recipe provides this one itself, ignore
379 continue
380 best = bb.providers.filterProvidersRunTime(all_p, rrec,
381 self.tinfoil.config_data,
382 self.tinfoil.cooker_data)[0][0]
383 self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers)
384
385 # The inherit class
386 cls_re = re.compile('classes/')
387 if f in self.tinfoil.cooker_data.inherits:
388 inherits = self.tinfoil.cooker_data.inherits[f]
389 for cls in inherits:
390 # The inherits' format is [classes/cls, /path/to/classes/cls]
391 # ignore the classes/cls.
392 if not cls_re.match(cls):
393 classname = os.path.splitext(os.path.basename(cls))[0]
394 if classname in global_inherit:
395 continue
396 inherit_layername = self.get_file_layer(cls)
397 if inherit_layername != layername and not inherit_layername in ignore_layers:
398 if not args.filenames:
399 f_short = self.remove_layer_prefix(f)
400 cls = self.remove_layer_prefix(cls)
401 else:
402 f_short = f
403 logger.plain("%s inherits %s" % (f_short, cls))
404
405 # The 'require/include xxx' in the bb file
406 pv_re = re.compile(r"\${PV}")
407 with open(f, 'r') as fnfile:
408 line = fnfile.readline()
409 while line:
410 m, keyword = self.match_require_include(line)
411 # Found the 'require/include xxxx'
412 if m:
413 needed_file = m.group(1)
414 # Replace the ${PV} with the real PV
415 if pv_re.search(needed_file) and f in self.tinfoil.cooker_data.pkg_pepvpr:
416 pv = self.tinfoil.cooker_data.pkg_pepvpr[f][1]
417 needed_file = re.sub(r"\${PV}", pv, needed_file)
418 self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers)
419 line = fnfile.readline()
420
421 # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass
422 conf_re = re.compile(".*/conf/machine/[^\/]*\.conf$")
423 inc_re = re.compile(".*\.inc$")
424 # The "inherit xxx" in .bbclass
425 bbclass_re = re.compile(".*\.bbclass$")
426 for layerdir in self.bblayers:
427 layername = self.get_layer_name(layerdir)
428 for dirpath, dirnames, filenames in os.walk(layerdir):
429 for name in filenames:
430 f = os.path.join(dirpath, name)
431 s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f)
432 if s:
433 with open(f, 'r') as ffile:
434 line = ffile.readline()
435 while line:
436 m, keyword = self.match_require_include(line)
437 # Only bbclass has the "inherit xxx" here.
438 bbclass=""
439 if not m and f.endswith(".bbclass"):
440 m, keyword = self.match_inherit(line)
441 bbclass=".bbclass"
442 # Find a 'require/include xxxx'
443 if m:
444 self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers)
445 line = ffile.readline()
446
447 def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames, ignore_layers):
448 """Print the depends that crosses a layer boundary"""
449 needed_file = bb.utils.which(bbpath, needed_filename)
450 if needed_file:
451 # Which layer is this file from
452 needed_layername = self.get_file_layer(needed_file)
453 if needed_layername != layername and not needed_layername in ignore_layers:
454 if not show_filenames:
455 f = self.remove_layer_prefix(f)
456 needed_file = self.remove_layer_prefix(needed_file)
457 logger.plain("%s %s %s" %(f, keyword, needed_file))
458
459 def match_inherit(self, line):
460 """Match the inherit xxx line"""
461 return (self.inherit_re.match(line), "inherits")
462
463 def match_require_include(self, line):
464 """Match the require/include xxx line"""
465 m = self.require_re.match(line)
466 keyword = "requires"
467 if not m:
468 m = self.include_re.match(line)
469 keyword = "includes"
470 return (m, keyword)
471
472 def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames, ignore_layers):
473 """Print the DEPENDS/RDEPENDS file that crosses a layer boundary"""
474 best_realfn = bb.cache.Cache.virtualfn2realfn(needed_file)[0]
475 needed_layername = self.get_file_layer(best_realfn)
476 if needed_layername != layername and not needed_layername in ignore_layers:
477 if not show_filenames:
478 f = self.remove_layer_prefix(f)
479 best_realfn = self.remove_layer_prefix(best_realfn)
480
481 logger.plain("%s %s %s" % (f, keyword, best_realfn))
482
483 def register_commands(self, sp):
484 self.add_command(sp, 'show-layers', self.do_show_layers)
485
486 parser_show_overlayed = self.add_command(sp, 'show-overlayed', self.do_show_overlayed)
487 parser_show_overlayed.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true')
488 parser_show_overlayed.add_argument('-s', '--same-version', help='only list overlayed recipes where the version is the same', action='store_true')
489
490 parser_show_recipes = self.add_command(sp, 'show-recipes', self.do_show_recipes)
491 parser_show_recipes.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true')
492 parser_show_recipes.add_argument('-m', '--multiple', help='only list where multiple recipes (in the same layer or different layers) exist for the same recipe name', action='store_true')
493 parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class', metavar='CLASS', default='')
494 parser_show_recipes.add_argument('pnspec', nargs='?', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)')
495
496 self.add_command(sp, 'show-appends', self.do_show_appends)
497
498 parser_show_cross_depends = self.add_command(sp, 'show-cross-depends', self.do_show_cross_depends)
499 parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true')
500 parser_show_cross_depends.add_argument('-i', '--ignore', help='ignore dependencies on items in the specified layer(s) (split multiple layer names with commas, no spaces)', metavar='LAYERNAME')