summaryrefslogtreecommitdiffstats
path: root/bitbake/bin/bitbake-layers
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/bin/bitbake-layers')
-rwxr-xr-xbitbake/bin/bitbake-layers1065
1 files changed, 60 insertions, 1005 deletions
diff --git a/bitbake/bin/bitbake-layers b/bitbake/bin/bitbake-layers
index 8b17eb0696..d8ffa9592a 100755
--- a/bitbake/bin/bitbake-layers
+++ b/bitbake/bin/bitbake-layers
@@ -23,1048 +23,103 @@
23import logging 23import logging
24import os 24import os
25import sys 25import sys
26import fnmatch
27from collections import defaultdict
28import argparse 26import argparse
29import re
30import httplib, urlparse, json
31import subprocess
32 27
33bindir = os.path.dirname(__file__) 28bindir = os.path.dirname(__file__)
34topdir = os.path.dirname(bindir) 29topdir = os.path.dirname(bindir)
35sys.path[0:0] = [os.path.join(topdir, 'lib')] 30sys.path[0:0] = [os.path.join(topdir, 'lib')]
36 31
37import bb.cache
38import bb.cooker
39import bb.providers
40import bb.utils
41import bb.tinfoil 32import bb.tinfoil
42 33
43 34
35def tinfoil_init(parserecipes):
36 import bb.tinfoil
37 tinfoil = bb.tinfoil.Tinfoil(tracking=True)
38 tinfoil.prepare(not parserecipes)
39 tinfoil.logger.setLevel(logger.getEffectiveLevel())
40 return tinfoil
41
42
44def logger_create(name, output=sys.stderr): 43def logger_create(name, output=sys.stderr):
45 logger = logging.getLogger(name) 44 logger = logging.getLogger(name)
46 console = logging.StreamHandler(output) 45 loggerhandler = logging.StreamHandler(output)
47 format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") 46 loggerhandler.setFormatter(logging.Formatter("%(levelname)s: %(message)s"))
48 if output.isatty(): 47 logger.addHandler(loggerhandler)
49 format.enable_color()
50 console.setFormatter(format)
51 logger.addHandler(console)
52 logger.setLevel(logging.INFO) 48 logger.setLevel(logging.INFO)
53 return logger 49 return logger
54 50
55logger = logger_create('bitbake-layers', sys.stdout)
56
57class UserError(Exception):
58 pass
59
60class Commands():
61 def __init__(self):
62 self.bbhandler = None
63 self.bblayers = []
64
65 def init_bbhandler(self, config_only = False):
66 if not self.bbhandler:
67 self.bbhandler = bb.tinfoil.Tinfoil(tracking=True)
68 self.bblayers = (self.bbhandler.config_data.getVar('BBLAYERS', True) or "").split()
69 self.bbhandler.prepare(config_only)
70 layerconfs = self.bbhandler.config_data.varhistory.get_variable_items_files('BBFILE_COLLECTIONS', self.bbhandler.config_data)
71 self.bbfile_collections = {layer: os.path.dirname(os.path.dirname(path)) for layer, path in layerconfs.iteritems()}
72
73
74 def do_show_layers(self, args):
75 """show current configured layers"""
76 self.init_bbhandler(config_only = True)
77 logger.plain("%s %s %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
78 logger.plain('=' * 74)
79 for layer, _, regex, pri in self.bbhandler.cooker.recipecache.bbfile_config_priorities:
80 layerdir = self.bbfile_collections.get(layer, None)
81 layername = self.get_layer_name(layerdir)
82 logger.plain("%s %s %d" % (layername.ljust(20), layerdir.ljust(40), pri))
83
84
85 def do_add_layer(self, args):
86 """Add a layer to bblayers.conf
87
88Adds the specified layer to bblayers.conf
89"""
90 layerdir = os.path.abspath(args.layerdir)
91 if not os.path.exists(layerdir):
92 sys.stderr.write("Specified layer directory doesn't exist\n")
93 return 1
94
95 layer_conf = os.path.join(layerdir, 'conf', 'layer.conf')
96 if not os.path.exists(layer_conf):
97 sys.stderr.write("Specified layer directory doesn't contain a conf/layer.conf file\n")
98 return 1
99
100 bblayers_conf = os.path.join('conf', 'bblayers.conf')
101 if not os.path.exists(bblayers_conf):
102 sys.stderr.write("Unable to find bblayers.conf\n")
103 return 1
104
105 (notadded, _) = bb.utils.edit_bblayers_conf(bblayers_conf, layerdir, None)
106 if notadded:
107 for item in notadded:
108 sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item)
109
110
111 def do_remove_layer(self, args):
112 """Remove a layer from bblayers.conf
113
114Removes the specified layer from bblayers.conf
115"""
116 bblayers_conf = os.path.join('conf', 'bblayers.conf')
117 if not os.path.exists(bblayers_conf):
118 sys.stderr.write("Unable to find bblayers.conf\n")
119 return 1
120
121 if args.layerdir.startswith('*'):
122 layerdir = args.layerdir
123 elif not '/' in args.layerdir:
124 layerdir = '*/%s' % args.layerdir
125 else:
126 layerdir = os.path.abspath(args.layerdir)
127 (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdir)
128 if notremoved:
129 for item in notremoved:
130 sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
131 return 1
132
133
134 def get_json_data(self, apiurl):
135 proxy_settings = os.environ.get("http_proxy", None)
136 conn = None
137 _parsedurl = urlparse.urlparse(apiurl)
138 path = _parsedurl.path
139 query = _parsedurl.query
140 def parse_url(url):
141 parsedurl = urlparse.urlparse(url)
142 if parsedurl.netloc[0] == '[':
143 host, port = parsedurl.netloc[1:].split(']', 1)
144 if ':' in port:
145 port = port.rsplit(':', 1)[1]
146 else:
147 port = None
148 else:
149 if parsedurl.netloc.count(':') == 1:
150 (host, port) = parsedurl.netloc.split(":")
151 else:
152 host = parsedurl.netloc
153 port = None
154 return (host, 80 if port is None else int(port))
155
156 if proxy_settings is None:
157 host, port = parse_url(apiurl)
158 conn = httplib.HTTPConnection(host, port)
159 conn.request("GET", path + "?" + query)
160 else:
161 host, port = parse_url(proxy_settings)
162 conn = httplib.HTTPConnection(host, port)
163 conn.request("GET", apiurl)
164
165 r = conn.getresponse()
166 if r.status != 200:
167 raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason))
168 return json.loads(r.read())
169
170
171 def get_layer_deps(self, layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=False):
172 def layeritems_info_id(items_name, layeritems):
173 litems_id = None
174 for li in layeritems:
175 if li['name'] == items_name:
176 litems_id = li['id']
177 break
178 return litems_id
179
180 def layerbranches_info(items_id, layerbranches):
181 lbranch = {}
182 for lb in layerbranches:
183 if lb['layer'] == items_id and lb['branch'] == branchnum:
184 lbranch['id'] = lb['id']
185 lbranch['vcs_subdir'] = lb['vcs_subdir']
186 break
187 return lbranch
188
189 def layerdependencies_info(lb_id, layerdependencies):
190 ld_deps = []
191 for ld in layerdependencies:
192 if ld['layerbranch'] == lb_id and not ld['dependency'] in ld_deps:
193 ld_deps.append(ld['dependency'])
194 if not ld_deps:
195 logger.error("The dependency of layerDependencies is not found.")
196 return ld_deps
197
198 def layeritems_info_name_subdir(items_id, layeritems):
199 litems = {}
200 for li in layeritems:
201 if li['id'] == items_id:
202 litems['vcs_url'] = li['vcs_url']
203 litems['name'] = li['name']
204 break
205 return litems
206
207 if selfname:
208 selfid = layeritems_info_id(layername, layeritems)
209 lbinfo = layerbranches_info(selfid, layerbranches)
210 if lbinfo:
211 selfsubdir = lbinfo['vcs_subdir']
212 else:
213 logger.error("%s is not found in the specified branch" % layername)
214 return
215 selfurl = layeritems_info_name_subdir(selfid, layeritems)['vcs_url']
216 if selfurl:
217 return selfurl, selfsubdir
218 else:
219 logger.error("Cannot get layer %s git repo and subdir" % layername)
220 return
221 ldict = {}
222 itemsid = layeritems_info_id(layername, layeritems)
223 if not itemsid:
224 return layername, None
225 lbid = layerbranches_info(itemsid, layerbranches)
226 if lbid:
227 lbid = layerbranches_info(itemsid, layerbranches)['id']
228 else:
229 logger.error("%s is not found in the specified branch" % layername)
230 return None, None
231 for dependency in layerdependencies_info(lbid, layerdependencies):
232 lname = layeritems_info_name_subdir(dependency, layeritems)['name']
233 lurl = layeritems_info_name_subdir(dependency, layeritems)['vcs_url']
234 lsubdir = layerbranches_info(dependency, layerbranches)['vcs_subdir']
235 ldict[lname] = lurl, lsubdir
236 return None, ldict
237
238
239 def get_fetch_layer(self, fetchdir, url, subdir, fetch_layer):
240 layername = self.get_layer_name(url)
241 if os.path.splitext(layername)[1] == '.git':
242 layername = os.path.splitext(layername)[0]
243 repodir = os.path.join(fetchdir, layername)
244 layerdir = os.path.join(repodir, subdir)
245 if not os.path.exists(repodir):
246 if fetch_layer:
247 result = subprocess.call('git clone %s %s' % (url, repodir), shell = True)
248 if result:
249 logger.error("Failed to download %s" % url)
250 return None, None
251 else:
252 return layername, layerdir
253 else:
254 logger.plain("Repository %s needs to be fetched" % url)
255 return layername, layerdir
256 elif os.path.exists(layerdir):
257 return layername, layerdir
258 else:
259 logger.error("%s is not in %s" % (url, subdir))
260 return None, None
261
262
263 def do_layerindex_fetch(self, args):
264 """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
265"""
266 self.init_bbhandler(config_only = True)
267 apiurl = self.bbhandler.config_data.getVar('BBLAYERS_LAYERINDEX_URL', True)
268 if not apiurl:
269 logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
270 return 1
271 else:
272 if apiurl[-1] != '/':
273 apiurl += '/'
274 apiurl += "api/"
275 apilinks = self.get_json_data(apiurl)
276 branches = self.get_json_data(apilinks['branches'])
277
278 branchnum = 0
279 for branch in branches:
280 if branch['name'] == args.branch:
281 branchnum = branch['id']
282 break
283 if branchnum == 0:
284 validbranches = ', '.join([branch['name'] for branch in branches])
285 logger.error('Invalid layer branch name "%s". Valid branches: %s' % (args.branch, validbranches))
286 return 1
287
288 ignore_layers = []
289 for collection in self.bbhandler.config_data.getVar('BBFILE_COLLECTIONS', True).split():
290 lname = self.bbhandler.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection, True)
291 if lname:
292 ignore_layers.append(lname)
293
294 if args.ignore:
295 ignore_layers.extend(args.ignore.split(','))
296
297 layeritems = self.get_json_data(apilinks['layerItems'])
298 layerbranches = self.get_json_data(apilinks['layerBranches'])
299 layerdependencies = self.get_json_data(apilinks['layerDependencies'])
300 invaluenames = []
301 repourls = {}
302 printlayers = []
303 def query_dependencies(layers, layeritems, layerbranches, layerdependencies, branchnum):
304 depslayer = []
305 for layername in layers:
306 invaluename, layerdict = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum)
307 if layerdict:
308 repourls[layername] = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=True)
309 for layer in layerdict:
310 if not layer in ignore_layers:
311 depslayer.append(layer)
312 printlayers.append((layername, layer, layerdict[layer][0], layerdict[layer][1]))
313 if not layer in ignore_layers and not layer in repourls:
314 repourls[layer] = (layerdict[layer][0], layerdict[layer][1])
315 if invaluename and not invaluename in invaluenames:
316 invaluenames.append(invaluename)
317 return depslayer
318
319 depslayers = query_dependencies(args.layername, layeritems, layerbranches, layerdependencies, branchnum)
320 while depslayers:
321 depslayer = query_dependencies(depslayers, layeritems, layerbranches, layerdependencies, branchnum)
322 depslayers = depslayer
323 if invaluenames:
324 for invaluename in invaluenames:
325 logger.error('Layer "%s" not found in layer index' % invaluename)
326 return 1
327 logger.plain("%s %s %s %s" % ("Layer".ljust(19), "Required by".ljust(19), "Git repository".ljust(54), "Subdirectory"))
328 logger.plain('=' * 115)
329 for layername in args.layername:
330 layerurl = repourls[layername]
331 logger.plain("%s %s %s %s" % (layername.ljust(20), '-'.ljust(20), layerurl[0].ljust(55), layerurl[1]))
332 printedlayers = []
333 for layer, dependency, gitrepo, subdirectory in printlayers:
334 if dependency in printedlayers:
335 continue
336 logger.plain("%s %s %s %s" % (dependency.ljust(20), layer.ljust(20), gitrepo.ljust(55), subdirectory))
337 printedlayers.append(dependency)
338
339 if repourls:
340 fetchdir = self.bbhandler.config_data.getVar('BBLAYERS_FETCH_DIR', True)
341 if not fetchdir:
342 logger.error("Cannot get BBLAYERS_FETCH_DIR")
343 return 1
344 if not os.path.exists(fetchdir):
345 os.makedirs(fetchdir)
346 addlayers = []
347 for repourl, subdir in repourls.values():
348 name, layerdir = self.get_fetch_layer(fetchdir, repourl, subdir, not args.show_only)
349 if not name:
350 # Error already shown
351 return 1
352 addlayers.append((subdir, name, layerdir))
353 if not args.show_only:
354 for subdir, name, layerdir in set(addlayers):
355 if os.path.exists(layerdir):
356 if subdir:
357 logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % subdir)
358 else:
359 logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % name)
360 localargs = argparse.Namespace()
361 localargs.layerdir = layerdir
362 self.do_add_layer(localargs)
363 else:
364 break
365
366
367 def do_layerindex_show_depends(self, args):
368 """Find layer dependencies from layer index.
369"""
370 args.show_only = True
371 args.ignore = []
372 self.do_layerindex_fetch(args)
373
374
375 def version_str(self, pe, pv, pr = None):
376 verstr = "%s" % pv
377 if pr:
378 verstr = "%s-%s" % (verstr, pr)
379 if pe:
380 verstr = "%s:%s" % (pe, verstr)
381 return verstr
382
383
384 def do_show_overlayed(self, args):
385 """list overlayed recipes (where the same recipe exists in another layer)
386
387Lists the names of overlayed recipes and the available versions in each
388layer, with the preferred version first. Note that skipped recipes that
389are overlayed will also be listed, with a " (skipped)" suffix.
390"""
391 self.init_bbhandler()
392
393 items_listed = self.list_recipes('Overlayed recipes', None, True, args.same_version, args.filenames, True, None)
394
395 # Check for overlayed .bbclass files
396 classes = defaultdict(list)
397 for layerdir in self.bblayers:
398 classdir = os.path.join(layerdir, 'classes')
399 if os.path.exists(classdir):
400 for classfile in os.listdir(classdir):
401 if os.path.splitext(classfile)[1] == '.bbclass':
402 classes[classfile].append(classdir)
403
404 # Locating classes and other files is a bit more complicated than recipes -
405 # layer priority is not a factor; instead BitBake uses the first matching
406 # file in BBPATH, which is manipulated directly by each layer's
407 # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a
408 # factor - however, each layer.conf is free to either prepend or append to
409 # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might
410 # not be exactly the order present in bblayers.conf either.
411 bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True))
412 overlayed_class_found = False
413 for (classfile, classdirs) in classes.items():
414 if len(classdirs) > 1:
415 if not overlayed_class_found:
416 logger.plain('=== Overlayed classes ===')
417 overlayed_class_found = True
418
419 mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile))
420 if args.filenames:
421 logger.plain('%s' % mainfile)
422 else:
423 # We effectively have to guess the layer here
424 logger.plain('%s:' % classfile)
425 mainlayername = '?'
426 for layerdir in self.bblayers:
427 classdir = os.path.join(layerdir, 'classes')
428 if mainfile.startswith(classdir):
429 mainlayername = self.get_layer_name(layerdir)
430 logger.plain(' %s' % mainlayername)
431 for classdir in classdirs:
432 fullpath = os.path.join(classdir, classfile)
433 if fullpath != mainfile:
434 if args.filenames:
435 print(' %s' % fullpath)
436 else:
437 print(' %s' % self.get_layer_name(os.path.dirname(classdir)))
438
439 if overlayed_class_found:
440 items_listed = True;
441
442 if not items_listed:
443 logger.plain('No overlayed files found.')
444
445
446 def do_show_recipes(self, args):
447 """list available recipes, showing the layer they are provided by
448
449Lists the names of recipes and the available versions in each
450layer, with the preferred version first. Optionally you may specify
451pnspec to match a specified recipe name (supports wildcards). Note that
452skipped recipes will also be listed, with a " (skipped)" suffix.
453"""
454 self.init_bbhandler()
455
456 inheritlist = args.inherits.split(',') if args.inherits else []
457 if inheritlist or args.pnspec or args.multiple:
458 title = 'Matching recipes:'
459 else:
460 title = 'Available recipes:'
461 self.list_recipes(title, args.pnspec, False, False, args.filenames, args.multiple, inheritlist)
462
463
464 def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_multi_provider_only, inherits):
465 if inherits:
466 bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True))
467 for classname in inherits:
468 classfile = 'classes/%s.bbclass' % classname
469 if not bb.utils.which(bbpath, classfile, history=False):
470 raise UserError('No class named %s found in BBPATH' % classfile)
471
472 pkg_pn = self.bbhandler.cooker.recipecache.pkg_pn
473 (latest_versions, preferred_versions) = bb.providers.findProviders(self.bbhandler.config_data, self.bbhandler.cooker.recipecache, pkg_pn)
474 allproviders = bb.providers.allProviders(self.bbhandler.cooker.recipecache)
475
476 # Ensure we list skipped recipes
477 # We are largely guessing about PN, PV and the preferred version here,
478 # but we have no choice since skipped recipes are not fully parsed
479 skiplist = self.bbhandler.cooker.skiplist.keys()
480 skiplist.sort( key=lambda fileitem: self.bbhandler.cooker.collection.calc_bbfile_priority(fileitem) )
481 skiplist.reverse()
482 for fn in skiplist:
483 recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
484 p = recipe_parts[0]
485 if len(recipe_parts) > 1:
486 ver = (None, recipe_parts[1], None)
487 else:
488 ver = (None, 'unknown', None)
489 allproviders[p].append((ver, fn))
490 if not p in pkg_pn:
491 pkg_pn[p] = 'dummy'
492 preferred_versions[p] = (ver, fn)
493
494 def print_item(f, pn, ver, layer, ispref):
495 if f in skiplist:
496 skipped = ' (skipped)'
497 else:
498 skipped = ''
499 if show_filenames:
500 if ispref:
501 logger.plain("%s%s", f, skipped)
502 else:
503 logger.plain(" %s%s", f, skipped)
504 else:
505 if ispref:
506 logger.plain("%s:", pn)
507 logger.plain(" %s %s%s", layer.ljust(20), ver, skipped)
508
509 global_inherit = (self.bbhandler.config_data.getVar('INHERIT', True) or "").split()
510 cls_re = re.compile('classes/')
511
512 preffiles = []
513 items_listed = False
514 for p in sorted(pkg_pn):
515 if pnspec:
516 if not fnmatch.fnmatch(p, pnspec):
517 continue
518
519 if len(allproviders[p]) > 1 or not show_multi_provider_only:
520 pref = preferred_versions[p]
521 realfn = bb.cache.Cache.virtualfn2realfn(pref[1])
522 preffile = realfn[0]
523
524 # We only display once per recipe, we should prefer non extended versions of the
525 # recipe if present (so e.g. in OpenEmbedded, openssl rather than nativesdk-openssl
526 # which would otherwise sort first).
527 if realfn[1] and realfn[0] in self.bbhandler.cooker.recipecache.pkg_fn:
528 continue
529
530 if inherits:
531 matchcount = 0
532 recipe_inherits = self.bbhandler.cooker_data.inherits.get(preffile, [])
533 for cls in recipe_inherits:
534 if cls_re.match(cls):
535 continue
536 classname = os.path.splitext(os.path.basename(cls))[0]
537 if classname in global_inherit:
538 continue
539 elif classname in inherits:
540 matchcount += 1
541 if matchcount != len(inherits):
542 # No match - skip this recipe
543 continue
544
545 if preffile not in preffiles:
546 preflayer = self.get_file_layer(preffile)
547 multilayer = False
548 same_ver = True
549 provs = []
550 for prov in allproviders[p]:
551 provfile = bb.cache.Cache.virtualfn2realfn(prov[1])[0]
552 provlayer = self.get_file_layer(provfile)
553 provs.append((provfile, provlayer, prov[0]))
554 if provlayer != preflayer:
555 multilayer = True
556 if prov[0] != pref[0]:
557 same_ver = False
558
559 if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only):
560 if not items_listed:
561 logger.plain('=== %s ===' % title)
562 items_listed = True
563 print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True)
564 for (provfile, provlayer, provver) in provs:
565 if provfile != preffile:
566 print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False)
567 # Ensure we don't show two entries for BBCLASSEXTENDed recipes
568 preffiles.append(preffile)
569 51
570 return items_listed 52def logger_setup_color(logger, color='auto'):
53 from bb.msg import BBLogFormatter
54 console = logging.StreamHandler(sys.stdout)
55 formatter = BBLogFormatter("%(levelname)s: %(message)s")
56 console.setFormatter(formatter)
57 logger.handlers = [console]
58 if color == 'always' or (color == 'auto' and console.stream.isatty()):
59 formatter.enable_color()
571 60
572 61
573 def do_flatten(self, args): 62logger = logger_create('bitbake-layers', sys.stdout)
574 """flatten layer configuration into a separate output directory.
575
576Takes the specified layers (or all layers in the current layer
577configuration if none are specified) and builds a "flattened" directory
578containing the contents of all layers, with any overlayed recipes removed
579and bbappends appended to the corresponding recipes. Note that some manual
580cleanup may still be necessary afterwards, in particular:
581
582* where non-recipe files (such as patches) are overwritten (the flatten
583 command will show a warning for these)
584* where anything beyond the normal layer setup has been added to
585 layer.conf (only the lowest priority number layer's layer.conf is used)
586* overridden/appended items from bbappends will need to be tidied up
587* when the flattened layers do not have the same directory structure (the
588 flatten command should show a warning when this will cause a problem)
589
590Warning: if you flatten several layers where another layer is intended to
591be used "inbetween" them (in layer priority order) such that recipes /
592bbappends in the layers interact, and then attempt to use the new output
593layer together with that other layer, you may no longer get the same
594build results (as the layer priority order has effectively changed).
595"""
596 if len(args.layer) == 1:
597 logger.error('If you specify layers to flatten you must specify at least two')
598 return 1
599
600 outputdir = args.outputdir
601 if os.path.exists(outputdir) and os.listdir(outputdir):
602 logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir)
603 return 1
604
605 self.init_bbhandler()
606 layers = self.bblayers
607 if len(args.layer) > 2:
608 layernames = args.layer
609 found_layernames = []
610 found_layerdirs = []
611 for layerdir in layers:
612 layername = self.get_layer_name(layerdir)
613 if layername in layernames:
614 found_layerdirs.append(layerdir)
615 found_layernames.append(layername)
616
617 for layername in layernames:
618 if not layername in found_layernames:
619 logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
620 return
621 layers = found_layerdirs
622 else:
623 layernames = []
624
625 # Ensure a specified path matches our list of layers
626 def layer_path_match(path):
627 for layerdir in layers:
628 if path.startswith(os.path.join(layerdir, '')):
629 return layerdir
630 return None
631
632 applied_appends = []
633 for layer in layers:
634 overlayed = []
635 for f in self.bbhandler.cooker.collection.overlayed.iterkeys():
636 for of in self.bbhandler.cooker.collection.overlayed[f]:
637 if of.startswith(layer):
638 overlayed.append(of)
639
640 logger.plain('Copying files from %s...' % layer )
641 for root, dirs, files in os.walk(layer):
642 for f1 in files:
643 f1full = os.sep.join([root, f1])
644 if f1full in overlayed:
645 logger.plain(' Skipping overlayed file %s' % f1full )
646 else:
647 ext = os.path.splitext(f1)[1]
648 if ext != '.bbappend':
649 fdest = f1full[len(layer):]
650 fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
651 bb.utils.mkdirhier(os.path.dirname(fdest))
652 if os.path.exists(fdest):
653 if f1 == 'layer.conf' and root.endswith('/conf'):
654 logger.plain(' Skipping layer config file %s' % f1full )
655 continue
656 else:
657 logger.warning('Overwriting file %s', fdest)
658 bb.utils.copyfile(f1full, fdest)
659 if ext == '.bb':
660 for append in self.bbhandler.cooker.collection.get_file_appends(f1full):
661 if layer_path_match(append):
662 logger.plain(' Applying append %s to %s' % (append, fdest))
663 self.apply_append(append, fdest)
664 applied_appends.append(append)
665
666 # Take care of when some layers are excluded and yet we have included bbappends for those recipes
667 for b in self.bbhandler.cooker.collection.bbappends:
668 (recipename, appendname) = b
669 if appendname not in applied_appends:
670 first_append = None
671 layer = layer_path_match(appendname)
672 if layer:
673 if first_append:
674 self.apply_append(appendname, first_append)
675 else:
676 fdest = appendname[len(layer):]
677 fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
678 bb.utils.mkdirhier(os.path.dirname(fdest))
679 bb.utils.copyfile(appendname, fdest)
680 first_append = fdest
681
682 # Get the regex for the first layer in our list (which is where the conf/layer.conf file will
683 # have come from)
684 first_regex = None
685 layerdir = layers[0]
686 for layername, pattern, regex, _ in self.bbhandler.cooker.recipecache.bbfile_config_priorities:
687 if regex.match(os.path.join(layerdir, 'test')):
688 first_regex = regex
689 break
690
691 if first_regex:
692 # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
693 bbfiles = str(self.bbhandler.config_data.getVar('BBFILES', True)).split()
694 bbfiles_layer = []
695 for item in bbfiles:
696 if first_regex.match(item):
697 newpath = os.path.join(outputdir, item[len(layerdir)+1:])
698 bbfiles_layer.append(newpath)
699
700 if bbfiles_layer:
701 # Check that all important layer files match BBFILES
702 for root, dirs, files in os.walk(outputdir):
703 for f1 in files:
704 ext = os.path.splitext(f1)[1]
705 if ext in ['.bb', '.bbappend']:
706 f1full = os.sep.join([root, f1])
707 entry_found = False
708 for item in bbfiles_layer:
709 if fnmatch.fnmatch(f1full, item):
710 entry_found = True
711 break
712 if not entry_found:
713 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
714
715 def get_file_layer(self, filename):
716 layerdir = self.get_file_layerdir(filename)
717 if layerdir:
718 return self.get_layer_name(layerdir)
719 else:
720 return '?'
721
722 def get_file_layerdir(self, filename):
723 layer = bb.utils.get_file_layer(filename, self.bbhandler.config_data)
724 return self.bbfile_collections.get(layer, None)
725
726 def remove_layer_prefix(self, f):
727 """Remove the layer_dir prefix, e.g., f = /path/to/layer_dir/foo/blah, the
728 return value will be: layer_dir/foo/blah"""
729 f_layerdir = self.get_file_layerdir(f)
730 if not f_layerdir:
731 return f
732 prefix = os.path.join(os.path.dirname(f_layerdir), '')
733 return f[len(prefix):] if f.startswith(prefix) else f
734
735 def get_layer_name(self, layerdir):
736 return os.path.basename(layerdir.rstrip(os.sep))
737
738 def apply_append(self, appendname, recipename):
739 with open(appendname, 'r') as appendfile:
740 with open(recipename, 'a') as recipefile:
741 recipefile.write('\n')
742 recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
743 recipefile.writelines(appendfile.readlines())
744
745 def do_show_appends(self, args):
746 """list bbappend files and recipe files they apply to
747
748Lists recipes with the bbappends that apply to them as subitems.
749"""
750 self.init_bbhandler()
751
752 logger.plain('=== Appended recipes ===')
753
754 pnlist = list(self.bbhandler.cooker_data.pkg_pn.keys())
755 pnlist.sort()
756 appends = False
757 for pn in pnlist:
758 if self.show_appends_for_pn(pn):
759 appends = True
760
761 if self.show_appends_for_skipped():
762 appends = True
763
764 if not appends:
765 logger.plain('No append files found')
766
767 def show_appends_for_pn(self, pn):
768 filenames = self.bbhandler.cooker_data.pkg_pn[pn]
769
770 best = bb.providers.findBestProvider(pn,
771 self.bbhandler.config_data,
772 self.bbhandler.cooker_data,
773 self.bbhandler.cooker_data.pkg_pn)
774 best_filename = os.path.basename(best[3])
775
776 return self.show_appends_output(filenames, best_filename)
777
778 def show_appends_for_skipped(self):
779 filenames = [os.path.basename(f)
780 for f in self.bbhandler.cooker.skiplist.iterkeys()]
781 return self.show_appends_output(filenames, None, " (skipped)")
782
783 def show_appends_output(self, filenames, best_filename, name_suffix = ''):
784 appended, missing = self.get_appends_for_files(filenames)
785 if appended:
786 for basename, appends in appended:
787 logger.plain('%s%s:', basename, name_suffix)
788 for append in appends:
789 logger.plain(' %s', append)
790
791 if best_filename:
792 if best_filename in missing:
793 logger.warning('%s: missing append for preferred version',
794 best_filename)
795 return True
796 else:
797 return False
798
799 def get_appends_for_files(self, filenames):
800 appended, notappended = [], []
801 for filename in filenames:
802 _, cls = bb.cache.Cache.virtualfn2realfn(filename)
803 if cls:
804 continue
805
806 basename = os.path.basename(filename)
807 appends = self.bbhandler.cooker.collection.get_file_appends(basename)
808 if appends:
809 appended.append((basename, list(appends)))
810 else:
811 notappended.append(basename)
812 return appended, notappended
813
814 def do_show_cross_depends(self, args):
815 """Show dependencies between recipes that cross layer boundaries.
816
817Figure out the dependencies between recipes that cross layer boundaries.
818
819NOTE: .bbappend files can impact the dependencies.
820"""
821 ignore_layers = (args.ignore or '').split(',')
822
823 self.init_bbhandler()
824
825 pkg_fn = self.bbhandler.cooker_data.pkg_fn
826 bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True))
827 self.require_re = re.compile(r"require\s+(.+)")
828 self.include_re = re.compile(r"include\s+(.+)")
829 self.inherit_re = re.compile(r"inherit\s+(.+)")
830
831 global_inherit = (self.bbhandler.config_data.getVar('INHERIT', True) or "").split()
832
833 # The bb's DEPENDS and RDEPENDS
834 for f in pkg_fn:
835 f = bb.cache.Cache.virtualfn2realfn(f)[0]
836 # Get the layername that the file is in
837 layername = self.get_file_layer(f)
838
839 # The DEPENDS
840 deps = self.bbhandler.cooker_data.deps[f]
841 for pn in deps:
842 if pn in self.bbhandler.cooker_data.pkg_pn:
843 best = bb.providers.findBestProvider(pn,
844 self.bbhandler.config_data,
845 self.bbhandler.cooker_data,
846 self.bbhandler.cooker_data.pkg_pn)
847 self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers)
848
849 # The RDPENDS
850 all_rdeps = self.bbhandler.cooker_data.rundeps[f].values()
851 # Remove the duplicated or null one.
852 sorted_rdeps = {}
853 # The all_rdeps is the list in list, so we need two for loops
854 for k1 in all_rdeps:
855 for k2 in k1:
856 sorted_rdeps[k2] = 1
857 all_rdeps = sorted_rdeps.keys()
858 for rdep in all_rdeps:
859 all_p = bb.providers.getRuntimeProviders(self.bbhandler.cooker_data, rdep)
860 if all_p:
861 if f in all_p:
862 # The recipe provides this one itself, ignore
863 continue
864 best = bb.providers.filterProvidersRunTime(all_p, rdep,
865 self.bbhandler.config_data,
866 self.bbhandler.cooker_data)[0][0]
867 self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers)
868
869 # The RRECOMMENDS
870 all_rrecs = self.bbhandler.cooker_data.runrecs[f].values()
871 # Remove the duplicated or null one.
872 sorted_rrecs = {}
873 # The all_rrecs is the list in list, so we need two for loops
874 for k1 in all_rrecs:
875 for k2 in k1:
876 sorted_rrecs[k2] = 1
877 all_rrecs = sorted_rrecs.keys()
878 for rrec in all_rrecs:
879 all_p = bb.providers.getRuntimeProviders(self.bbhandler.cooker_data, rrec)
880 if all_p:
881 if f in all_p:
882 # The recipe provides this one itself, ignore
883 continue
884 best = bb.providers.filterProvidersRunTime(all_p, rrec,
885 self.bbhandler.config_data,
886 self.bbhandler.cooker_data)[0][0]
887 self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers)
888
889 # The inherit class
890 cls_re = re.compile('classes/')
891 if f in self.bbhandler.cooker_data.inherits:
892 inherits = self.bbhandler.cooker_data.inherits[f]
893 for cls in inherits:
894 # The inherits' format is [classes/cls, /path/to/classes/cls]
895 # ignore the classes/cls.
896 if not cls_re.match(cls):
897 classname = os.path.splitext(os.path.basename(cls))[0]
898 if classname in global_inherit:
899 continue
900 inherit_layername = self.get_file_layer(cls)
901 if inherit_layername != layername and not inherit_layername in ignore_layers:
902 if not args.filenames:
903 f_short = self.remove_layer_prefix(f)
904 cls = self.remove_layer_prefix(cls)
905 else:
906 f_short = f
907 logger.plain("%s inherits %s" % (f_short, cls))
908
909 # The 'require/include xxx' in the bb file
910 pv_re = re.compile(r"\${PV}")
911 with open(f, 'r') as fnfile:
912 line = fnfile.readline()
913 while line:
914 m, keyword = self.match_require_include(line)
915 # Found the 'require/include xxxx'
916 if m:
917 needed_file = m.group(1)
918 # Replace the ${PV} with the real PV
919 if pv_re.search(needed_file) and f in self.bbhandler.cooker_data.pkg_pepvpr:
920 pv = self.bbhandler.cooker_data.pkg_pepvpr[f][1]
921 needed_file = re.sub(r"\${PV}", pv, needed_file)
922 self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers)
923 line = fnfile.readline()
924
925 # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass
926 conf_re = re.compile(".*/conf/machine/[^\/]*\.conf$")
927 inc_re = re.compile(".*\.inc$")
928 # The "inherit xxx" in .bbclass
929 bbclass_re = re.compile(".*\.bbclass$")
930 for layerdir in self.bblayers:
931 layername = self.get_layer_name(layerdir)
932 for dirpath, dirnames, filenames in os.walk(layerdir):
933 for name in filenames:
934 f = os.path.join(dirpath, name)
935 s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f)
936 if s:
937 with open(f, 'r') as ffile:
938 line = ffile.readline()
939 while line:
940 m, keyword = self.match_require_include(line)
941 # Only bbclass has the "inherit xxx" here.
942 bbclass=""
943 if not m and f.endswith(".bbclass"):
944 m, keyword = self.match_inherit(line)
945 bbclass=".bbclass"
946 # Find a 'require/include xxxx'
947 if m:
948 self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers)
949 line = ffile.readline()
950
951 def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames, ignore_layers):
952 """Print the depends that crosses a layer boundary"""
953 needed_file = bb.utils.which(bbpath, needed_filename)
954 if needed_file:
955 # Which layer is this file from
956 needed_layername = self.get_file_layer(needed_file)
957 if needed_layername != layername and not needed_layername in ignore_layers:
958 if not show_filenames:
959 f = self.remove_layer_prefix(f)
960 needed_file = self.remove_layer_prefix(needed_file)
961 logger.plain("%s %s %s" %(f, keyword, needed_file))
962
963 def match_inherit(self, line):
964 """Match the inherit xxx line"""
965 return (self.inherit_re.match(line), "inherits")
966
967 def match_require_include(self, line):
968 """Match the require/include xxx line"""
969 m = self.require_re.match(line)
970 keyword = "requires"
971 if not m:
972 m = self.include_re.match(line)
973 keyword = "includes"
974 return (m, keyword)
975
976 def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames, ignore_layers):
977 """Print the DEPENDS/RDEPENDS file that crosses a layer boundary"""
978 best_realfn = bb.cache.Cache.virtualfn2realfn(needed_file)[0]
979 needed_layername = self.get_file_layer(best_realfn)
980 if needed_layername != layername and not needed_layername in ignore_layers:
981 if not show_filenames:
982 f = self.remove_layer_prefix(f)
983 best_realfn = self.remove_layer_prefix(best_realfn)
984
985 logger.plain("%s %s %s" % (f, keyword, best_realfn))
986 63
987 64
988def main(): 65def main():
989 66 parser = argparse.ArgumentParser(
990 cmds = Commands() 67 description="BitBake layers utility",
991 68 epilog="Use %(prog)s <subcommand> --help to get help on a specific command",
992 def add_command(cmdname, function, *args, **kwargs): 69 add_help=False)
993 # Convert docstring for function to help (one-liner shown in main --help) and description (shown in subcommand --help)
994 docsplit = function.__doc__.splitlines()
995 help = docsplit[0]
996 if len(docsplit) > 1:
997 desc = '\n'.join(docsplit[1:])
998 else:
999 desc = help
1000 subparser = subparsers.add_parser(cmdname, *args, help=help, description=desc, formatter_class=argparse.RawTextHelpFormatter, **kwargs)
1001 subparser.set_defaults(func=function)
1002 return subparser
1003
1004 parser = argparse.ArgumentParser(description="BitBake layers utility",
1005 epilog="Use %(prog)s <subcommand> --help to get help on a specific command")
1006 parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true') 70 parser.add_argument('-d', '--debug', help='Enable debug output', action='store_true')
1007 parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true') 71 parser.add_argument('-q', '--quiet', help='Print only errors', action='store_true')
1008 subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>') 72 parser.add_argument('--color', choices=['auto', 'always', 'never'], default='auto', help='Colorize output (where %(metavar)s is %(choices)s)', metavar='COLOR')
1009
1010 parser_show_layers = add_command('show-layers', cmds.do_show_layers)
1011
1012 parser_add_layer = add_command('add-layer', cmds.do_add_layer)
1013 parser_add_layer.add_argument('layerdir', help='Layer directory to add')
1014
1015 parser_remove_layer = add_command('remove-layer', cmds.do_remove_layer)
1016 parser_remove_layer.add_argument('layerdir', help='Layer directory to remove (wildcards allowed, enclose in quotes to avoid shell expansion)')
1017 parser_remove_layer.set_defaults(func=cmds.do_remove_layer)
1018
1019 parser_show_overlayed = add_command('show-overlayed', cmds.do_show_overlayed)
1020 parser_show_overlayed.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true')
1021 parser_show_overlayed.add_argument('-s', '--same-version', help='only list overlayed recipes where the version is the same', action='store_true')
1022 73
1023 parser_show_recipes = add_command('show-recipes', cmds.do_show_recipes) 74 global_args, unparsed_args = parser.parse_known_args()
1024 parser_show_recipes.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true')
1025 parser_show_recipes.add_argument('-m', '--multiple', help='only list where multiple recipes (in the same layer or different layers) exist for the same recipe name', action='store_true')
1026 parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class', metavar='CLASS', default='')
1027 parser_show_recipes.add_argument('pnspec', nargs='?', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)')
1028 75
1029 parser_show_appends = add_command('show-appends', cmds.do_show_appends) 76 # Help is added here rather than via add_help=True, as we don't want it to
77 # be handled by parse_known_args()
78 parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS,
79 help='show this help message and exit')
80 subparsers = parser.add_subparsers(title='subcommands', metavar='<subcommand>')
1030 81
1031 parser_flatten = add_command('flatten', cmds.do_flatten) 82 if global_args.debug:
1032 parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)') 83 logger.setLevel(logging.DEBUG)
1033 parser_flatten.add_argument('outputdir', help='Output directory') 84 elif global_args.quiet:
85 logger.setLevel(logging.ERROR)
1034 86
1035 parser_show_cross_depends = add_command('show-cross-depends', cmds.do_show_cross_depends) 87 logger_setup_color(logger, global_args.color)
1036 parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true')
1037 parser_show_cross_depends.add_argument('-i', '--ignore', help='ignore dependencies on items in the specified layer(s) (split multiple layer names with commas, no spaces)', metavar='LAYERNAME')
1038 88
1039 parser_layerindex_fetch = add_command('layerindex-fetch', cmds.do_layerindex_fetch) 89 plugins = []
1040 parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true') 90 tinfoil = tinfoil_init(False)
1041 parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master') 91 for path in ([topdir] +
1042 parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER') 92 tinfoil.config_data.getVar('BBPATH', True).split(':')):
1043 parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch') 93 pluginpath = os.path.join(path, 'lib', 'bblayers')
94 bb.utils.load_plugins(logger, plugins, pluginpath)
1044 95
1045 parser_layerindex_show_depends = add_command('layerindex-show-depends', cmds.do_layerindex_show_depends) 96 registered = False
1046 parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master') 97 for plugin in plugins:
1047 parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query') 98 if hasattr(plugin, 'register_commands'):
99 registered = True
100 plugin.register_commands(subparsers)
101 if hasattr(plugin, 'tinfoil_init'):
102 plugin.tinfoil_init(tinfoil)
1048 103
1049 args = parser.parse_args() 104 if not registered:
105 logger.error("No commands registered - missing plugins?")
106 sys.exit(1)
1050 107
1051 if args.debug: 108 args = parser.parse_args(unparsed_args, namespace=global_args)
1052 logger.setLevel(logging.DEBUG)
1053 elif args.quiet:
1054 logger.setLevel(logging.ERROR)
1055 109
1056 try: 110 if getattr(args, 'parserecipes', False):
1057 ret = args.func(args) 111 tinfoil.config_data.disableTracking()
1058 except UserError as err: 112 tinfoil.parseRecipes()
1059 logger.error(str(err)) 113 tinfoil.config_data.enableTracking()
1060 ret = 1
1061 114
1062 return ret 115 return args.func(args)
1063 116
1064 117
1065if __name__ == "__main__": 118if __name__ == "__main__":
1066 try: 119 try:
1067 ret = main() 120 ret = main()
121 except bb.BBHandledException:
122 ret = 1
1068 except Exception: 123 except Exception:
1069 ret = 1 124 ret = 1
1070 import traceback 125 import traceback
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2025-01-16 23:45:42 +0000