summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bblayers
diff options
context:
space:
mode:
authorChristopher Larson <chris_larson@mentor.com>2016-04-30 19:41:00 (GMT)
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-05-17 20:16:35 (GMT)
commit6bbe4fe48c93d1e2b67a36aa82d2ea5e289a219c (patch)
tree802427823189e16ffcd6560acfda961303873adc /bitbake/lib/bblayers
parent07eebc66898b423ea2e04e96c57d3b2b5eabbb26 (diff)
downloadpoky-6bbe4fe48c93d1e2b67a36aa82d2ea5e289a219c.tar.gz
bitbake: bitbake-layers: convert to plugin-based
This uses bb.utils.load_plugins, based on the plugin handling in recipetool and devtool in oe-core. (Bitbake rev: 5e542df9b966a99b5a5b8aa7cf6100174aff54b2) Signed-off-by: Christopher Larson <chris_larson@mentor.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bblayers')
-rw-r--r--bitbake/lib/bblayers/__init__.py2
-rw-r--r--bitbake/lib/bblayers/action.py233
-rw-r--r--bitbake/lib/bblayers/common.py33
-rw-r--r--bitbake/lib/bblayers/layerindex.py270
-rw-r--r--bitbake/lib/bblayers/query.py500
5 files changed, 1038 insertions, 0 deletions
diff --git a/bitbake/lib/bblayers/__init__.py b/bitbake/lib/bblayers/__init__.py
new file mode 100644
index 0000000..3ad9513
--- /dev/null
+++ b/bitbake/lib/bblayers/__init__.py
@@ -0,0 +1,2 @@
1from pkgutil import extend_path
2__path__ = extend_path(__path__, __name__)
diff --git a/bitbake/lib/bblayers/action.py b/bitbake/lib/bblayers/action.py
new file mode 100644
index 0000000..5b95e2e
--- /dev/null
+++ b/bitbake/lib/bblayers/action.py
@@ -0,0 +1,233 @@
1import fnmatch
2import logging
3import os
4import sys
5
6import bb.utils
7
8from bblayers.common import LayerPlugin
9
10logger = logging.getLogger('bitbake-layers')
11
12
13def plugin_init(plugins):
14 return ActionPlugin()
15
16
17class ActionPlugin(LayerPlugin):
18 def do_add_layer(self, args):
19 """Add a layer to bblayers.conf."""
20 layerdir = os.path.abspath(args.layerdir)
21 if not os.path.exists(layerdir):
22 sys.stderr.write("Specified layer directory doesn't exist\n")
23 return 1
24
25 layer_conf = os.path.join(layerdir, 'conf', 'layer.conf')
26 if not os.path.exists(layer_conf):
27 sys.stderr.write("Specified layer directory doesn't contain a conf/layer.conf file\n")
28 return 1
29
30 bblayers_conf = os.path.join('conf', 'bblayers.conf')
31 if not os.path.exists(bblayers_conf):
32 sys.stderr.write("Unable to find bblayers.conf\n")
33 return 1
34
35 notadded, _ = bb.utils.edit_bblayers_conf(bblayers_conf, layerdir, None)
36 if notadded:
37 for item in notadded:
38 sys.stderr.write("Specified layer %s is already in BBLAYERS\n" % item)
39
40 def do_remove_layer(self, args):
41 """Remove a layer from bblayers.conf."""
42 bblayers_conf = os.path.join('conf', 'bblayers.conf')
43 if not os.path.exists(bblayers_conf):
44 sys.stderr.write("Unable to find bblayers.conf\n")
45 return 1
46
47 if args.layerdir.startswith('*'):
48 layerdir = args.layerdir
49 elif not '/' in args.layerdir:
50 layerdir = '*/%s' % args.layerdir
51 else:
52 layerdir = os.path.abspath(args.layerdir)
53 (_, notremoved) = bb.utils.edit_bblayers_conf(bblayers_conf, None, layerdir)
54 if notremoved:
55 for item in notremoved:
56 sys.stderr.write("No layers matching %s found in BBLAYERS\n" % item)
57 return 1
58
59 def do_flatten(self, args):
60 """flatten layer configuration into a separate output directory.
61
62Takes the specified layers (or all layers in the current layer
63configuration if none are specified) and builds a "flattened" directory
64containing the contents of all layers, with any overlayed recipes removed
65and bbappends appended to the corresponding recipes. Note that some manual
66cleanup may still be necessary afterwards, in particular:
67
68* where non-recipe files (such as patches) are overwritten (the flatten
69 command will show a warning for these)
70* where anything beyond the normal layer setup has been added to
71 layer.conf (only the lowest priority number layer's layer.conf is used)
72* overridden/appended items from bbappends will need to be tidied up
73* when the flattened layers do not have the same directory structure (the
74 flatten command should show a warning when this will cause a problem)
75
76Warning: if you flatten several layers where another layer is intended to
77be used "inbetween" them (in layer priority order) such that recipes /
78bbappends in the layers interact, and then attempt to use the new output
79layer together with that other layer, you may no longer get the same
80build results (as the layer priority order has effectively changed).
81"""
82 if len(args.layer) == 1:
83 logger.error('If you specify layers to flatten you must specify at least two')
84 return 1
85
86 outputdir = args.outputdir
87 if os.path.exists(outputdir) and os.listdir(outputdir):
88 logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir)
89 return 1
90
91 layers = self.bblayers
92 if len(args.layer) > 2:
93 layernames = args.layer
94 found_layernames = []
95 found_layerdirs = []
96 for layerdir in layers:
97 layername = self.get_layer_name(layerdir)
98 if layername in layernames:
99 found_layerdirs.append(layerdir)
100 found_layernames.append(layername)
101
102 for layername in layernames:
103 if not layername in found_layernames:
104 logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0])))
105 return
106 layers = found_layerdirs
107 else:
108 layernames = []
109
110 # Ensure a specified path matches our list of layers
111 def layer_path_match(path):
112 for layerdir in layers:
113 if path.startswith(os.path.join(layerdir, '')):
114 return layerdir
115 return None
116
117 applied_appends = []
118 for layer in layers:
119 overlayed = []
120 for f in self.tinfoil.cooker.collection.overlayed.iterkeys():
121 for of in self.tinfoil.cooker.collection.overlayed[f]:
122 if of.startswith(layer):
123 overlayed.append(of)
124
125 logger.plain('Copying files from %s...' % layer )
126 for root, dirs, files in os.walk(layer):
127 if '.git' in dirs:
128 dirs.remove('.git')
129 if '.hg' in dirs:
130 dirs.remove('.hg')
131
132 for f1 in files:
133 f1full = os.sep.join([root, f1])
134 if f1full in overlayed:
135 logger.plain(' Skipping overlayed file %s' % f1full )
136 else:
137 ext = os.path.splitext(f1)[1]
138 if ext != '.bbappend':
139 fdest = f1full[len(layer):]
140 fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
141 bb.utils.mkdirhier(os.path.dirname(fdest))
142 if os.path.exists(fdest):
143 if f1 == 'layer.conf' and root.endswith('/conf'):
144 logger.plain(' Skipping layer config file %s' % f1full )
145 continue
146 else:
147 logger.warning('Overwriting file %s', fdest)
148 bb.utils.copyfile(f1full, fdest)
149 if ext == '.bb':
150 for append in self.tinfoil.cooker.collection.get_file_appends(f1full):
151 if layer_path_match(append):
152 logger.plain(' Applying append %s to %s' % (append, fdest))
153 self.apply_append(append, fdest)
154 applied_appends.append(append)
155
156 # Take care of when some layers are excluded and yet we have included bbappends for those recipes
157 for b in self.tinfoil.cooker.collection.bbappends:
158 (recipename, appendname) = b
159 if appendname not in applied_appends:
160 first_append = None
161 layer = layer_path_match(appendname)
162 if layer:
163 if first_append:
164 self.apply_append(appendname, first_append)
165 else:
166 fdest = appendname[len(layer):]
167 fdest = os.path.normpath(os.sep.join([outputdir,fdest]))
168 bb.utils.mkdirhier(os.path.dirname(fdest))
169 bb.utils.copyfile(appendname, fdest)
170 first_append = fdest
171
172 # Get the regex for the first layer in our list (which is where the conf/layer.conf file will
173 # have come from)
174 first_regex = None
175 layerdir = layers[0]
176 for layername, pattern, regex, _ in self.tinfoil.cooker.recipecache.bbfile_config_priorities:
177 if regex.match(os.path.join(layerdir, 'test')):
178 first_regex = regex
179 break
180
181 if first_regex:
182 # Find the BBFILES entries that match (which will have come from this conf/layer.conf file)
183 bbfiles = str(self.tinfoil.config_data.getVar('BBFILES', True)).split()
184 bbfiles_layer = []
185 for item in bbfiles:
186 if first_regex.match(item):
187 newpath = os.path.join(outputdir, item[len(layerdir)+1:])
188 bbfiles_layer.append(newpath)
189
190 if bbfiles_layer:
191 # Check that all important layer files match BBFILES
192 for root, dirs, files in os.walk(outputdir):
193 for f1 in files:
194 ext = os.path.splitext(f1)[1]
195 if ext in ['.bb', '.bbappend']:
196 f1full = os.sep.join([root, f1])
197 entry_found = False
198 for item in bbfiles_layer:
199 if fnmatch.fnmatch(f1full, item):
200 entry_found = True
201 break
202 if not entry_found:
203 logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full)
204
205 def get_file_layer(self, filename):
206 layerdir = self.get_file_layerdir(filename)
207 if layerdir:
208 return self.get_layer_name(layerdir)
209 else:
210 return '?'
211
212 def get_file_layerdir(self, filename):
213 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data)
214 return self.bbfile_collections.get(layer, None)
215
216 def apply_append(self, appendname, recipename):
217 with open(appendname, 'r') as appendfile:
218 with open(recipename, 'a') as recipefile:
219 recipefile.write('\n')
220 recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname))
221 recipefile.writelines(appendfile.readlines())
222
223 def register_commands(self, sp):
224 parser_add_layer = self.add_command(sp, 'add-layer', self.do_add_layer, parserecipes=False)
225 parser_add_layer.add_argument('layerdir', help='Layer directory to add')
226
227 parser_remove_layer = self.add_command(sp, 'remove-layer', self.do_remove_layer, parserecipes=False)
228 parser_remove_layer.add_argument('layerdir', help='Layer directory to remove (wildcards allowed, enclose in quotes to avoid shell expansion)')
229 parser_remove_layer.set_defaults(func=self.do_remove_layer)
230
231 parser_flatten = self.add_command(sp, 'flatten', self.do_flatten)
232 parser_flatten.add_argument('layer', nargs='*', help='Optional layer(s) to flatten (otherwise all are flattened)')
233 parser_flatten.add_argument('outputdir', help='Output directory')
diff --git a/bitbake/lib/bblayers/common.py b/bitbake/lib/bblayers/common.py
new file mode 100644
index 0000000..360b9d7
--- /dev/null
+++ b/bitbake/lib/bblayers/common.py
@@ -0,0 +1,33 @@
1import argparse
2import logging
3import os
4
5logger = logging.getLogger('bitbake-layers')
6
7
8class LayerPlugin():
9 def __init__(self):
10 self.tinfoil = None
11 self.bblayers = []
12
13 def tinfoil_init(self, tinfoil):
14 self.tinfoil = tinfoil
15 self.bblayers = (self.tinfoil.config_data.getVar('BBLAYERS', True) or "").split()
16 layerconfs = self.tinfoil.config_data.varhistory.get_variable_items_files('BBFILE_COLLECTIONS', self.tinfoil.config_data)
17 self.bbfile_collections = {layer: os.path.dirname(os.path.dirname(path)) for layer, path in layerconfs.iteritems()}
18
19 @staticmethod
20 def add_command(subparsers, cmdname, function, parserecipes=True, *args, **kwargs):
21 """Convert docstring for function to help."""
22 docsplit = function.__doc__.splitlines()
23 help = docsplit[0]
24 if len(docsplit) > 1:
25 desc = '\n'.join(docsplit[1:])
26 else:
27 desc = help
28 subparser = subparsers.add_parser(cmdname, *args, help=help, description=desc, formatter_class=argparse.RawTextHelpFormatter, **kwargs)
29 subparser.set_defaults(func=function, parserecipes=parserecipes)
30 return subparser
31
32 def get_layer_name(self, layerdir):
33 return os.path.basename(layerdir.rstrip(os.sep))
diff --git a/bitbake/lib/bblayers/layerindex.py b/bitbake/lib/bblayers/layerindex.py
new file mode 100644
index 0000000..3c39d8a
--- /dev/null
+++ b/bitbake/lib/bblayers/layerindex.py
@@ -0,0 +1,270 @@
1import argparse
2import httplib
3import json
4import logging
5import os
6import subprocess
7import urlparse
8
9from bblayers.action import ActionPlugin
10
11logger = logging.getLogger('bitbake-layers')
12
13
14def plugin_init(plugins):
15 return LayerIndexPlugin()
16
17
18class LayerIndexPlugin(ActionPlugin):
19 """Subcommands for interacting with the layer index.
20
21 This class inherits ActionPlugin to get do_add_layer.
22 """
23
24 def get_json_data(self, apiurl):
25 proxy_settings = os.environ.get("http_proxy", None)
26 conn = None
27 _parsedurl = urlparse.urlparse(apiurl)
28 path = _parsedurl.path
29 query = _parsedurl.query
30
31 def parse_url(url):
32 parsedurl = urlparse.urlparse(url)
33 if parsedurl.netloc[0] == '[':
34 host, port = parsedurl.netloc[1:].split(']', 1)
35 if ':' in port:
36 port = port.rsplit(':', 1)[1]
37 else:
38 port = None
39 else:
40 if parsedurl.netloc.count(':') == 1:
41 (host, port) = parsedurl.netloc.split(":")
42 else:
43 host = parsedurl.netloc
44 port = None
45 return (host, 80 if port is None else int(port))
46
47 if proxy_settings is None:
48 host, port = parse_url(apiurl)
49 conn = httplib.HTTPConnection(host, port)
50 conn.request("GET", path + "?" + query)
51 else:
52 host, port = parse_url(proxy_settings)
53 conn = httplib.HTTPConnection(host, port)
54 conn.request("GET", apiurl)
55
56 r = conn.getresponse()
57 if r.status != 200:
58 raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason))
59 return json.loads(r.read())
60
61 def get_layer_deps(self, layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=False):
62 def layeritems_info_id(items_name, layeritems):
63 litems_id = None
64 for li in layeritems:
65 if li['name'] == items_name:
66 litems_id = li['id']
67 break
68 return litems_id
69
70 def layerbranches_info(items_id, layerbranches):
71 lbranch = {}
72 for lb in layerbranches:
73 if lb['layer'] == items_id and lb['branch'] == branchnum:
74 lbranch['id'] = lb['id']
75 lbranch['vcs_subdir'] = lb['vcs_subdir']
76 break
77 return lbranch
78
79 def layerdependencies_info(lb_id, layerdependencies):
80 ld_deps = []
81 for ld in layerdependencies:
82 if ld['layerbranch'] == lb_id and not ld['dependency'] in ld_deps:
83 ld_deps.append(ld['dependency'])
84 if not ld_deps:
85 logger.error("The dependency of layerDependencies is not found.")
86 return ld_deps
87
88 def layeritems_info_name_subdir(items_id, layeritems):
89 litems = {}
90 for li in layeritems:
91 if li['id'] == items_id:
92 litems['vcs_url'] = li['vcs_url']
93 litems['name'] = li['name']
94 break
95 return litems
96
97 if selfname:
98 selfid = layeritems_info_id(layername, layeritems)
99 lbinfo = layerbranches_info(selfid, layerbranches)
100 if lbinfo:
101 selfsubdir = lbinfo['vcs_subdir']
102 else:
103 logger.error("%s is not found in the specified branch" % layername)
104 return
105 selfurl = layeritems_info_name_subdir(selfid, layeritems)['vcs_url']
106 if selfurl:
107 return selfurl, selfsubdir
108 else:
109 logger.error("Cannot get layer %s git repo and subdir" % layername)
110 return
111 ldict = {}
112 itemsid = layeritems_info_id(layername, layeritems)
113 if not itemsid:
114 return layername, None
115 lbid = layerbranches_info(itemsid, layerbranches)
116 if lbid:
117 lbid = layerbranches_info(itemsid, layerbranches)['id']
118 else:
119 logger.error("%s is not found in the specified branch" % layername)
120 return None, None
121 for dependency in layerdependencies_info(lbid, layerdependencies):
122 lname = layeritems_info_name_subdir(dependency, layeritems)['name']
123 lurl = layeritems_info_name_subdir(dependency, layeritems)['vcs_url']
124 lsubdir = layerbranches_info(dependency, layerbranches)['vcs_subdir']
125 ldict[lname] = lurl, lsubdir
126 return None, ldict
127
128 def get_fetch_layer(self, fetchdir, url, subdir, fetch_layer):
129 layername = self.get_layer_name(url)
130 if os.path.splitext(layername)[1] == '.git':
131 layername = os.path.splitext(layername)[0]
132 repodir = os.path.join(fetchdir, layername)
133 layerdir = os.path.join(repodir, subdir)
134 if not os.path.exists(repodir):
135 if fetch_layer:
136 result = subprocess.call('git clone %s %s' % (url, repodir), shell = True)
137 if result:
138 logger.error("Failed to download %s" % url)
139 return None, None
140 else:
141 return layername, layerdir
142 else:
143 logger.plain("Repository %s needs to be fetched" % url)
144 return layername, layerdir
145 elif os.path.exists(layerdir):
146 return layername, layerdir
147 else:
148 logger.error("%s is not in %s" % (url, subdir))
149 return None, None
150
151 def do_layerindex_fetch(self, args):
152 """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
153"""
154 apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL', True)
155 if not apiurl:
156 logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
157 return 1
158 else:
159 if apiurl[-1] != '/':
160 apiurl += '/'
161 apiurl += "api/"
162 apilinks = self.get_json_data(apiurl)
163 branches = self.get_json_data(apilinks['branches'])
164
165 branchnum = 0
166 for branch in branches:
167 if branch['name'] == args.branch:
168 branchnum = branch['id']
169 break
170 if branchnum == 0:
171 validbranches = ', '.join([branch['name'] for branch in branches])
172 logger.error('Invalid layer branch name "%s". Valid branches: %s' % (args.branch, validbranches))
173 return 1
174
175 ignore_layers = []
176 for collection in self.tinfoil.config_data.getVar('BBFILE_COLLECTIONS', True).split():
177 lname = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection, True)
178 if lname:
179 ignore_layers.append(lname)
180
181 if args.ignore:
182 ignore_layers.extend(args.ignore.split(','))
183
184 layeritems = self.get_json_data(apilinks['layerItems'])
185 layerbranches = self.get_json_data(apilinks['layerBranches'])
186 layerdependencies = self.get_json_data(apilinks['layerDependencies'])
187 invaluenames = []
188 repourls = {}
189 printlayers = []
190
191 def query_dependencies(layers, layeritems, layerbranches, layerdependencies, branchnum):
192 depslayer = []
193 for layername in layers:
194 invaluename, layerdict = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum)
195 if layerdict:
196 repourls[layername] = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=True)
197 for layer in layerdict:
198 if not layer in ignore_layers:
199 depslayer.append(layer)
200 printlayers.append((layername, layer, layerdict[layer][0], layerdict[layer][1]))
201 if not layer in ignore_layers and not layer in repourls:
202 repourls[layer] = (layerdict[layer][0], layerdict[layer][1])
203 if invaluename and not invaluename in invaluenames:
204 invaluenames.append(invaluename)
205 return depslayer
206
207 depslayers = query_dependencies(args.layername, layeritems, layerbranches, layerdependencies, branchnum)
208 while depslayers:
209 depslayer = query_dependencies(depslayers, layeritems, layerbranches, layerdependencies, branchnum)
210 depslayers = depslayer
211 if invaluenames:
212 for invaluename in invaluenames:
213 logger.error('Layer "%s" not found in layer index' % invaluename)
214 return 1
215 logger.plain("%s %s %s %s" % ("Layer".ljust(19), "Required by".ljust(19), "Git repository".ljust(54), "Subdirectory"))
216 logger.plain('=' * 115)
217 for layername in args.layername:
218 layerurl = repourls[layername]
219 logger.plain("%s %s %s %s" % (layername.ljust(20), '-'.ljust(20), layerurl[0].ljust(55), layerurl[1]))
220 printedlayers = []
221 for layer, dependency, gitrepo, subdirectory in printlayers:
222 if dependency in printedlayers:
223 continue
224 logger.plain("%s %s %s %s" % (dependency.ljust(20), layer.ljust(20), gitrepo.ljust(55), subdirectory))
225 printedlayers.append(dependency)
226
227 if repourls:
228 fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR', True)
229 if not fetchdir:
230 logger.error("Cannot get BBLAYERS_FETCH_DIR")
231 return 1
232 if not os.path.exists(fetchdir):
233 os.makedirs(fetchdir)
234 addlayers = []
235 for repourl, subdir in repourls.values():
236 name, layerdir = self.get_fetch_layer(fetchdir, repourl, subdir, not args.show_only)
237 if not name:
238 # Error already shown
239 return 1
240 addlayers.append((subdir, name, layerdir))
241 if not args.show_only:
242 for subdir, name, layerdir in set(addlayers):
243 if os.path.exists(layerdir):
244 if subdir:
245 logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % subdir)
246 else:
247 logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % name)
248 localargs = argparse.Namespace()
249 localargs.layerdir = layerdir
250 self.do_add_layer(localargs)
251 else:
252 break
253
254 def do_layerindex_show_depends(self, args):
255 """Find layer dependencies from layer index.
256"""
257 args.show_only = True
258 args.ignore = []
259 self.do_layerindex_fetch(args)
260
261 def register_commands(self, sp):
262 parser_layerindex_fetch = self.add_command(sp, 'layerindex-fetch', self.do_layerindex_fetch)
263 parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true')
264 parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master')
265 parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER')
266 parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch')
267
268 parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends)
269 parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master')
270 parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query')
diff --git a/bitbake/lib/bblayers/query.py b/bitbake/lib/bblayers/query.py
new file mode 100644
index 0000000..b5b98f7
--- /dev/null
+++ b/bitbake/lib/bblayers/query.py
@@ -0,0 +1,500 @@
1import collections
2import fnmatch
3import logging
4import sys
5import os
6import re
7
8import bb.cache
9import bb.providers
10import bb.utils
11
12from bblayers.common import LayerPlugin
13
14logger = logging.getLogger('bitbake-layers')
15
16
17def plugin_init(plugins):
18 return QueryPlugin()
19
20
21class QueryPlugin(LayerPlugin):
22 def do_show_layers(self, args):
23 """show current configured layers."""
24 logger.plain("%s %s %s" % ("layer".ljust(20), "path".ljust(40), "priority"))
25 logger.plain('=' * 74)
26 for layer, _, regex, pri in self.tinfoil.cooker.recipecache.bbfile_config_priorities:
27 layerdir = self.bbfile_collections.get(layer, None)
28 layername = self.get_layer_name(layerdir)
29 logger.plain("%s %s %d" % (layername.ljust(20), layerdir.ljust(40), pri))
30
31 def version_str(self, pe, pv, pr = None):
32 verstr = "%s" % pv
33 if pr:
34 verstr = "%s-%s" % (verstr, pr)
35 if pe:
36 verstr = "%s:%s" % (pe, verstr)
37 return verstr
38
39 def do_show_overlayed(self, args):
40 """list overlayed recipes (where the same recipe exists in another layer)
41
42Lists the names of overlayed recipes and the available versions in each
43layer, with the preferred version first. Note that skipped recipes that
44are overlayed will also be listed, with a " (skipped)" suffix.
45"""
46
47 items_listed = self.list_recipes('Overlayed recipes', None, True, args.same_version, args.filenames, True, None)
48
49 # Check for overlayed .bbclass files
50 classes = collections.defaultdict(list)
51 for layerdir in self.bblayers:
52 classdir = os.path.join(layerdir, 'classes')
53 if os.path.exists(classdir):
54 for classfile in os.listdir(classdir):
55 if os.path.splitext(classfile)[1] == '.bbclass':
56 classes[classfile].append(classdir)
57
58 # Locating classes and other files is a bit more complicated than recipes -
59 # layer priority is not a factor; instead BitBake uses the first matching
60 # file in BBPATH, which is manipulated directly by each layer's
61 # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a
62 # factor - however, each layer.conf is free to either prepend or append to
63 # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might
64 # not be exactly the order present in bblayers.conf either.
65 bbpath = str(self.tinfoil.config_data.getVar('BBPATH', True))
66 overlayed_class_found = False
67 for (classfile, classdirs) in classes.items():
68 if len(classdirs) > 1:
69 if not overlayed_class_found:
70 logger.plain('=== Overlayed classes ===')
71 overlayed_class_found = True
72
73 mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile))
74 if args.filenames:
75 logger.plain('%s' % mainfile)
76 else:
77 # We effectively have to guess the layer here
78 logger.plain('%s:' % classfile)
79 mainlayername = '?'
80 for layerdir in self.bblayers:
81 classdir = os.path.join(layerdir, 'classes')
82 if mainfile.startswith(classdir):
83 mainlayername = self.get_layer_name(layerdir)
84 logger.plain(' %s' % mainlayername)
85 for classdir in classdirs:
86 fullpath = os.path.join(classdir, classfile)
87 if fullpath != mainfile:
88 if args.filenames:
89 print(' %s' % fullpath)
90 else:
91 print(' %s' % self.get_layer_name(os.path.dirname(classdir)))
92
93 if overlayed_class_found:
94 items_listed = True;
95
96 if not items_listed:
97 logger.plain('No overlayed files found.')
98
99 def do_show_recipes(self, args):
100 """list available recipes, showing the layer they are provided by
101
102Lists the names of recipes and the available versions in each
103layer, with the preferred version first. Optionally you may specify
104pnspec to match a specified recipe name (supports wildcards). Note that
105skipped recipes will also be listed, with a " (skipped)" suffix.
106"""
107
108 inheritlist = args.inherits.split(',') if args.inherits else []
109 if inheritlist or args.pnspec or args.multiple:
110 title = 'Matching recipes:'
111 else:
112 title = 'Available recipes:'
113 self.list_recipes(title, args.pnspec, False, False, args.filenames, args.multiple, inheritlist)
114
115 def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_multi_provider_only, inherits):
116 if inherits:
117 bbpath = str(self.tinfoil.config_data.getVar('BBPATH', True))
118 for classname in inherits:
119 classfile = 'classes/%s.bbclass' % classname
120 if not bb.utils.which(bbpath, classfile, history=False):
121 logger.error('No class named %s found in BBPATH', classfile)
122 sys.exit(1)
123
124 pkg_pn = self.tinfoil.cooker.recipecache.pkg_pn
125 (latest_versions, preferred_versions) = bb.providers.findProviders(self.tinfoil.config_data, self.tinfoil.cooker.recipecache, pkg_pn)
126 allproviders = bb.providers.allProviders(self.tinfoil.cooker.recipecache)
127
128 # Ensure we list skipped recipes
129 # We are largely guessing about PN, PV and the preferred version here,
130 # but we have no choice since skipped recipes are not fully parsed
131 skiplist = self.tinfoil.cooker.skiplist.keys()
132 skiplist.sort( key=lambda fileitem: self.tinfoil.cooker.collection.calc_bbfile_priority(fileitem) )
133 skiplist.reverse()
134 for fn in skiplist:
135 recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
136 p = recipe_parts[0]
137 if len(recipe_parts) > 1:
138 ver = (None, recipe_parts[1], None)
139 else:
140 ver = (None, 'unknown', None)
141 allproviders[p].append((ver, fn))
142 if not p in pkg_pn:
143 pkg_pn[p] = 'dummy'
144 preferred_versions[p] = (ver, fn)
145
146 def print_item(f, pn, ver, layer, ispref):
147 if f in skiplist:
148 skipped = ' (skipped)'
149 else:
150 skipped = ''
151 if show_filenames:
152 if ispref:
153 logger.plain("%s%s", f, skipped)
154 else:
155 logger.plain(" %s%s", f, skipped)
156 else:
157 if ispref:
158 logger.plain("%s:", pn)
159 logger.plain(" %s %s%s", layer.ljust(20), ver, skipped)
160
161 global_inherit = (self.tinfoil.config_data.getVar('INHERIT', True) or "").split()
162 cls_re = re.compile('classes/')
163
164 preffiles = []
165 items_listed = False
166 for p in sorted(pkg_pn):
167 if pnspec:
168 if not fnmatch.fnmatch(p, pnspec):
169 continue
170
171 if len(allproviders[p]) > 1 or not show_multi_provider_only:
172 pref = preferred_versions[p]
173 realfn = bb.cache.Cache.virtualfn2realfn(pref[1])
174 preffile = realfn[0]
175
176 # We only display once per recipe, we should prefer non extended versions of the
177 # recipe if present (so e.g. in OpenEmbedded, openssl rather than nativesdk-openssl
178 # which would otherwise sort first).
179 if realfn[1] and realfn[0] in self.tinfoil.cooker.recipecache.pkg_fn:
180 continue
181
182 if inherits:
183 matchcount = 0
184 recipe_inherits = self.tinfoil.cooker_data.inherits.get(preffile, [])
185 for cls in recipe_inherits:
186 if cls_re.match(cls):
187 continue
188 classname = os.path.splitext(os.path.basename(cls))[0]
189 if classname in global_inherit:
190 continue
191 elif classname in inherits:
192 matchcount += 1
193 if matchcount != len(inherits):
194 # No match - skip this recipe
195 continue
196
197 if preffile not in preffiles:
198 preflayer = self.get_file_layer(preffile)
199 multilayer = False
200 same_ver = True
201 provs = []
202 for prov in allproviders[p]:
203 provfile = bb.cache.Cache.virtualfn2realfn(prov[1])[0]
204 provlayer = self.get_file_layer(provfile)
205 provs.append((provfile, provlayer, prov[0]))
206 if provlayer != preflayer:
207 multilayer = True
208 if prov[0] != pref[0]:
209 same_ver = False
210
211 if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only):
212 if not items_listed:
213 logger.plain('=== %s ===' % title)
214 items_listed = True
215 print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True)
216 for (provfile, provlayer, provver) in provs:
217 if provfile != preffile:
218 print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False)
219 # Ensure we don't show two entries for BBCLASSEXTENDed recipes
220 preffiles.append(preffile)
221
222 return items_listed
223
224 def get_file_layer(self, filename):
225 layerdir = self.get_file_layerdir(filename)
226 if layerdir:
227 return self.get_layer_name(layerdir)
228 else:
229 return '?'
230
231 def get_file_layerdir(self, filename):
232 layer = bb.utils.get_file_layer(filename, self.tinfoil.config_data)
233 return self.bbfile_collections.get(layer, None)
234
235 def remove_layer_prefix(self, f):
236 """Remove the layer_dir prefix, e.g., f = /path/to/layer_dir/foo/blah, the
237 return value will be: layer_dir/foo/blah"""
238 f_layerdir = self.get_file_layerdir(f)
239 if not f_layerdir:
240 return f
241 prefix = os.path.join(os.path.dirname(f_layerdir), '')
242 return f[len(prefix):] if f.startswith(prefix) else f
243
244 def do_show_appends(self, args):
245 """list bbappend files and recipe files they apply to
246
247Lists recipes with the bbappends that apply to them as subitems.
248"""
249
250 logger.plain('=== Appended recipes ===')
251
252 pnlist = list(self.tinfoil.cooker_data.pkg_pn.keys())
253 pnlist.sort()
254 appends = False
255 for pn in pnlist:
256 if self.show_appends_for_pn(pn):
257 appends = True
258
259 if self.show_appends_for_skipped():
260 appends = True
261
262 if not appends:
263 logger.plain('No append files found')
264
265 def show_appends_for_pn(self, pn):
266 filenames = self.tinfoil.cooker_data.pkg_pn[pn]
267
268 best = bb.providers.findBestProvider(pn,
269 self.tinfoil.config_data,
270 self.tinfoil.cooker_data,
271 self.tinfoil.cooker_data.pkg_pn)
272 best_filename = os.path.basename(best[3])
273
274 return self.show_appends_output(filenames, best_filename)
275
276 def show_appends_for_skipped(self):
277 filenames = [os.path.basename(f)
278 for f in self.tinfoil.cooker.skiplist.iterkeys()]
279 return self.show_appends_output(filenames, None, " (skipped)")
280
281 def show_appends_output(self, filenames, best_filename, name_suffix = ''):
282 appended, missing = self.get_appends_for_files(filenames)
283 if appended:
284 for basename, appends in appended:
285 logger.plain('%s%s:', basename, name_suffix)
286 for append in appends:
287 logger.plain(' %s', append)
288
289 if best_filename:
290 if best_filename in missing:
291 logger.warning('%s: missing append for preferred version',
292 best_filename)
293 return True
294 else:
295 return False
296
297 def get_appends_for_files(self, filenames):
298 appended, notappended = [], []
299 for filename in filenames:
300 _, cls = bb.cache.Cache.virtualfn2realfn(filename)
301 if cls:
302 continue
303
304 basename = os.path.basename(filename)
305 appends = self.tinfoil.cooker.collection.get_file_appends(basename)
306 if appends:
307 appended.append((basename, list(appends)))
308 else:
309 notappended.append(basename)
310 return appended, notappended
311
312 def do_show_cross_depends(self, args):
313 """Show dependencies between recipes that cross layer boundaries.
314
315Figure out the dependencies between recipes that cross layer boundaries.
316
317NOTE: .bbappend files can impact the dependencies.
318"""
319 ignore_layers = (args.ignore or '').split(',')
320
321 pkg_fn = self.tinfoil.cooker_data.pkg_fn
322 bbpath = str(self.tinfoil.config_data.getVar('BBPATH', True))
323 self.require_re = re.compile(r"require\s+(.+)")
324 self.include_re = re.compile(r"include\s+(.+)")
325 self.inherit_re = re.compile(r"inherit\s+(.+)")
326
327 global_inherit = (self.tinfoil.config_data.getVar('INHERIT', True) or "").split()
328
329 # The bb's DEPENDS and RDEPENDS
330 for f in pkg_fn:
331 f = bb.cache.Cache.virtualfn2realfn(f)[0]
332 # Get the layername that the file is in
333 layername = self.get_file_layer(f)
334
335 # The DEPENDS
336 deps = self.tinfoil.cooker_data.deps[f]
337 for pn in deps:
338 if pn in self.tinfoil.cooker_data.pkg_pn:
339 best = bb.providers.findBestProvider(pn,
340 self.tinfoil.config_data,
341 self.tinfoil.cooker_data,
342 self.tinfoil.cooker_data.pkg_pn)
343 self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers)
344
345 # The RDPENDS
346 all_rdeps = self.tinfoil.cooker_data.rundeps[f].values()
347 # Remove the duplicated or null one.
348 sorted_rdeps = {}
349 # The all_rdeps is the list in list, so we need two for loops
350 for k1 in all_rdeps:
351 for k2 in k1:
352 sorted_rdeps[k2] = 1
353 all_rdeps = sorted_rdeps.keys()
354 for rdep in all_rdeps:
355 all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rdep)
356 if all_p:
357 if f in all_p:
358 # The recipe provides this one itself, ignore
359 continue
360 best = bb.providers.filterProvidersRunTime(all_p, rdep,
361 self.tinfoil.config_data,
362 self.tinfoil.cooker_data)[0][0]
363 self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers)
364
365 # The RRECOMMENDS
366 all_rrecs = self.tinfoil.cooker_data.runrecs[f].values()
367 # Remove the duplicated or null one.
368 sorted_rrecs = {}
369 # The all_rrecs is the list in list, so we need two for loops
370 for k1 in all_rrecs:
371 for k2 in k1:
372 sorted_rrecs[k2] = 1
373 all_rrecs = sorted_rrecs.keys()
374 for rrec in all_rrecs:
375 all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rrec)
376 if all_p:
377 if f in all_p:
378 # The recipe provides this one itself, ignore
379 continue
380 best = bb.providers.filterProvidersRunTime(all_p, rrec,
381 self.tinfoil.config_data,
382 self.tinfoil.cooker_data)[0][0]
383 self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers)
384
385 # The inherit class
386 cls_re = re.compile('classes/')
387 if f in self.tinfoil.cooker_data.inherits:
388 inherits = self.tinfoil.cooker_data.inherits[f]
389 for cls in inherits:
390 # The inherits' format is [classes/cls, /path/to/classes/cls]
391 # ignore the classes/cls.
392 if not cls_re.match(cls):
393 classname = os.path.splitext(os.path.basename(cls))[0]
394 if classname in global_inherit:
395 continue
396 inherit_layername = self.get_file_layer(cls)
397 if inherit_layername != layername and not inherit_layername in ignore_layers:
398 if not args.filenames:
399 f_short = self.remove_layer_prefix(f)
400 cls = self.remove_layer_prefix(cls)
401 else:
402 f_short = f
403 logger.plain("%s inherits %s" % (f_short, cls))
404
405 # The 'require/include xxx' in the bb file
406 pv_re = re.compile(r"\${PV}")
407 with open(f, 'r') as fnfile:
408 line = fnfile.readline()
409 while line:
410 m, keyword = self.match_require_include(line)
411 # Found the 'require/include xxxx'
412 if m:
413 needed_file = m.group(1)
414 # Replace the ${PV} with the real PV
415 if pv_re.search(needed_file) and f in self.tinfoil.cooker_data.pkg_pepvpr:
416 pv = self.tinfoil.cooker_data.pkg_pepvpr[f][1]
417 needed_file = re.sub(r"\${PV}", pv, needed_file)
418 self.print_cross_files(bbpath, keyword, layername, f, needed_file, args.filenames, ignore_layers)
419 line = fnfile.readline()
420
421 # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass
422 conf_re = re.compile(".*/conf/machine/[^\/]*\.conf$")
423 inc_re = re.compile(".*\.inc$")
424 # The "inherit xxx" in .bbclass
425 bbclass_re = re.compile(".*\.bbclass$")
426 for layerdir in self.bblayers:
427 layername = self.get_layer_name(layerdir)
428 for dirpath, dirnames, filenames in os.walk(layerdir):
429 for name in filenames:
430 f = os.path.join(dirpath, name)
431 s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f)
432 if s:
433 with open(f, 'r') as ffile:
434 line = ffile.readline()
435 while line:
436 m, keyword = self.match_require_include(line)
437 # Only bbclass has the "inherit xxx" here.
438 bbclass=""
439 if not m and f.endswith(".bbclass"):
440 m, keyword = self.match_inherit(line)
441 bbclass=".bbclass"
442 # Find a 'require/include xxxx'
443 if m:
444 self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, args.filenames, ignore_layers)
445 line = ffile.readline()
446
447 def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames, ignore_layers):
448 """Print the depends that crosses a layer boundary"""
449 needed_file = bb.utils.which(bbpath, needed_filename)
450 if needed_file:
451 # Which layer is this file from
452 needed_layername = self.get_file_layer(needed_file)
453 if needed_layername != layername and not needed_layername in ignore_layers:
454 if not show_filenames:
455 f = self.remove_layer_prefix(f)
456 needed_file = self.remove_layer_prefix(needed_file)
457 logger.plain("%s %s %s" %(f, keyword, needed_file))
458
459 def match_inherit(self, line):
460 """Match the inherit xxx line"""
461 return (self.inherit_re.match(line), "inherits")
462
463 def match_require_include(self, line):
464 """Match the require/include xxx line"""
465 m = self.require_re.match(line)
466 keyword = "requires"
467 if not m:
468 m = self.include_re.match(line)
469 keyword = "includes"
470 return (m, keyword)
471
472 def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames, ignore_layers):
473 """Print the DEPENDS/RDEPENDS file that crosses a layer boundary"""
474 best_realfn = bb.cache.Cache.virtualfn2realfn(needed_file)[0]
475 needed_layername = self.get_file_layer(best_realfn)
476 if needed_layername != layername and not needed_layername in ignore_layers:
477 if not show_filenames:
478 f = self.remove_layer_prefix(f)
479 best_realfn = self.remove_layer_prefix(best_realfn)
480
481 logger.plain("%s %s %s" % (f, keyword, best_realfn))
482
483 def register_commands(self, sp):
484 self.add_command(sp, 'show-layers', self.do_show_layers)
485
486 parser_show_overlayed = self.add_command(sp, 'show-overlayed', self.do_show_overlayed)
487 parser_show_overlayed.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true')
488 parser_show_overlayed.add_argument('-s', '--same-version', help='only list overlayed recipes where the version is the same', action='store_true')
489
490 parser_show_recipes = self.add_command(sp, 'show-recipes', self.do_show_recipes)
491 parser_show_recipes.add_argument('-f', '--filenames', help='instead of the default formatting, list filenames of higher priority recipes with the ones they overlay indented underneath', action='store_true')
492 parser_show_recipes.add_argument('-m', '--multiple', help='only list where multiple recipes (in the same layer or different layers) exist for the same recipe name', action='store_true')
493 parser_show_recipes.add_argument('-i', '--inherits', help='only list recipes that inherit the named class', metavar='CLASS', default='')
494 parser_show_recipes.add_argument('pnspec', nargs='?', help='optional recipe name specification (wildcards allowed, enclose in quotes to avoid shell expansion)')
495
496 self.add_command(sp, 'show-appends', self.do_show_appends)
497
498 parser_show_cross_depends = self.add_command(sp, 'show-cross-depends', self.do_show_cross_depends)
499 parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true')
500 parser_show_cross_depends.add_argument('-i', '--ignore', help='ignore dependencies on items in the specified layer(s) (split multiple layer names with commas, no spaces)', metavar='LAYERNAME')