summaryrefslogtreecommitdiffstats
path: root/bitbake/bin
diff options
context:
space:
mode:
authorChong Lu <Chong.Lu@windriver.com>2015-02-20 17:52:43 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-02-21 22:05:37 +0000
commitdfa28485fc07ef9c194823358f53e6a3ace20a9d (patch)
treee06c332076ff1b82dae25e7898696e32164856dc /bitbake/bin
parent90ffdc01ba2d75b171720baa4232f6cc1ae8b0b1 (diff)
downloadpoky-dfa28485fc07ef9c194823358f53e6a3ace20a9d.tar.gz
bitbake: bitbake-layers: add ability to fetch layers and their dependencies from layer index
Add a command to query layer dependencies from a layer index such as the OpenEmbedded Layer Index at http://layers.openembedded.org. Fetches the layer and its dependencies and adds them into conf/bblayers.conf. [YOCTO #5348] (Bitbake rev: 4b8fcf9a5bc802793bf332334217faace55f14f6) Signed-off-by: Chong Lu <Chong.Lu@windriver.com> Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/bin')
-rwxr-xr-xbitbake/bin/bitbake-layers252
1 files changed, 252 insertions, 0 deletions
diff --git a/bitbake/bin/bitbake-layers b/bitbake/bin/bitbake-layers
index fc62386b64..a86ad1c0f1 100755
--- a/bitbake/bin/bitbake-layers
+++ b/bitbake/bin/bitbake-layers
@@ -27,6 +27,8 @@ import fnmatch
27from collections import defaultdict 27from collections import defaultdict
28import argparse 28import argparse
29import re 29import re
30import httplib, urlparse, json
31import subprocess
30 32
31bindir = os.path.dirname(__file__) 33bindir = os.path.dirname(__file__)
32topdir = os.path.dirname(bindir) 34topdir = os.path.dirname(bindir)
@@ -127,6 +129,246 @@ Removes the specified layer from bblayers.conf
127 return 1 129 return 1
128 130
129 131
132 def get_json_data(self, apiurl):
133 proxy_settings = os.environ.get("http_proxy", None)
134 conn = None
135 _parsedurl = urlparse.urlparse(apiurl)
136 path = _parsedurl.path
137 query = _parsedurl.query
138 def parse_url(url):
139 parsedurl = urlparse.urlparse(url)
140 if parsedurl.netloc[0] == '[':
141 host, port = parsedurl.netloc[1:].split(']', 1)
142 if ':' in port:
143 port = port.rsplit(':', 1)[1]
144 else:
145 port = None
146 else:
147 if parsedurl.netloc.count(':') == 1:
148 (host, port) = parsedurl.netloc.split(":")
149 else:
150 host = parsedurl.netloc
151 port = None
152 return (host, 80 if port is None else int(port))
153
154 if proxy_settings is None:
155 host, port = parse_url(apiurl)
156 conn = httplib.HTTPConnection(host, port)
157 conn.request("GET", path + "?" + query)
158 else:
159 host, port = parse_url(proxy_settings)
160 conn = httplib.HTTPConnection(host, port)
161 conn.request("GET", apiurl)
162
163 r = conn.getresponse()
164 if r.status != 200:
165 raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason))
166 return json.loads(r.read())
167
168
169 def get_layer_deps(self, layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=False):
170 def layeritems_info_id(items_name, layeritems):
171 litems_id = None
172 for li in layeritems:
173 if li['name'] == items_name:
174 litems_id = li['id']
175 break
176 return litems_id
177
178 def layerbranches_info(items_id, layerbranches):
179 lbranch = {}
180 for lb in layerbranches:
181 if lb['layer'] == items_id and lb['branch'] == branchnum:
182 lbranch['id'] = lb['id']
183 lbranch['vcs_subdir'] = lb['vcs_subdir']
184 break
185 return lbranch
186
187 def layerdependencies_info(lb_id, layerdependencies):
188 ld_deps = []
189 for ld in layerdependencies:
190 if ld['layerbranch'] == lb_id and not ld['dependency'] in ld_deps:
191 ld_deps.append(ld['dependency'])
192 if not ld_deps:
193 logger.error("The dependency of layerDependencies is not found.")
194 return ld_deps
195
196 def layeritems_info_name_subdir(items_id, layeritems):
197 litems = {}
198 for li in layeritems:
199 if li['id'] == items_id:
200 litems['vcs_url'] = li['vcs_url']
201 litems['name'] = li['name']
202 break
203 return litems
204
205 if selfname:
206 selfid = layeritems_info_id(layername, layeritems)
207 lbinfo = layerbranches_info(selfid, layerbranches)
208 if lbinfo:
209 selfsubdir = lbinfo['vcs_subdir']
210 else:
211 logger.error("%s is not found in the specified branch" % layername)
212 return
213 selfurl = layeritems_info_name_subdir(selfid, layeritems)['vcs_url']
214 if selfurl:
215 return selfurl, selfsubdir
216 else:
217 logger.error("Cannot get layer %s git repo and subdir" % layername)
218 return
219 ldict = {}
220 itemsid = layeritems_info_id(layername, layeritems)
221 if not itemsid:
222 return layername, None
223 lbid = layerbranches_info(itemsid, layerbranches)
224 if lbid:
225 lbid = layerbranches_info(itemsid, layerbranches)['id']
226 else:
227 logger.error("%s is not found in the specified branch" % layername)
228 return None, None
229 for dependency in layerdependencies_info(lbid, layerdependencies):
230 lname = layeritems_info_name_subdir(dependency, layeritems)['name']
231 lurl = layeritems_info_name_subdir(dependency, layeritems)['vcs_url']
232 lsubdir = layerbranches_info(dependency, layerbranches)['vcs_subdir']
233 ldict[lname] = lurl, lsubdir
234 return None, ldict
235
236
237 def get_fetch_layer(self, fetchdir, url, subdir, fetch_layer):
238 layername = self.get_layer_name(url)
239 if os.path.splitext(layername)[1] == '.git':
240 layername = os.path.splitext(layername)[0]
241 repodir = os.path.join(fetchdir, layername)
242 layerdir = os.path.join(repodir, subdir)
243 if not os.path.exists(repodir):
244 if fetch_layer:
245 result = subprocess.call('git clone %s %s' % (url, repodir), shell = True)
246 if result:
247 logger.error("Failed to download %s" % url)
248 return None, None
249 else:
250 return layername, layerdir
251 else:
252 logger.plain("Repository %s needs to be fetched" % url)
253 return layername, layerdir
254 elif os.path.exists(layerdir):
255 return layername, layerdir
256 else:
257 logger.error("%s is not in %s" % (url, subdir))
258 return None, None
259
260
261 def do_layerindex_fetch(self, args):
262 """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
263"""
264 self.init_bbhandler(config_only = True)
265 apiurl = self.bbhandler.config_data.getVar('BBLAYERS_LAYERINDEX_URL', True)
266 if not apiurl:
267 logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
268 else:
269 if apiurl[-1] != '/':
270 apiurl += '/'
271 apiurl += "api/"
272 apilinks = self.get_json_data(apiurl)
273 branches = self.get_json_data(apilinks['branches'])
274
275 branchnum = 0
276 for branch in branches:
277 if branch['name'] == args.branch:
278 branchnum = branch['id']
279 break
280 if branchnum == 0:
281 validbranches = ', '.join([branch['name'] for branch in branches])
282 logger.error('Invalid layer branch name "%s". Valid branches: %s' % (args.branch, validbranches))
283 return 1
284
285 ignore_layers = []
286 for collection in self.bbhandler.config_data.getVar('BBFILE_COLLECTIONS', True).split():
287 lname = self.bbhandler.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection, True)
288 if lname:
289 ignore_layers.append(lname)
290
291 if args.ignore:
292 ignore_layers.extend(args.ignore.split(','))
293
294 layeritems = self.get_json_data(apilinks['layerItems'])
295 layerbranches = self.get_json_data(apilinks['layerBranches'])
296 layerdependencies = self.get_json_data(apilinks['layerDependencies'])
297 invaluenames = []
298 repourls = {}
299 printlayers = []
300 def query_dependencies(layers, layeritems, layerbranches, layerdependencies, branchnum):
301 depslayer = []
302 for layername in layers:
303 invaluename, layerdict = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum)
304 if layerdict:
305 repourls[layername] = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=True)
306 for layer in layerdict:
307 if not layer in ignore_layers:
308 depslayer.append(layer)
309 printlayers.append((layername, layer, layerdict[layer][0], layerdict[layer][1]))
310 if not layer in ignore_layers and not layer in repourls:
311 repourls[layer] = (layerdict[layer][0], layerdict[layer][1])
312 if invaluename and not invaluename in invaluenames:
313 invaluenames.append(invaluename)
314 return depslayer
315
316 depslayers = query_dependencies(args.layername, layeritems, layerbranches, layerdependencies, branchnum)
317 while depslayers:
318 depslayer = query_dependencies(depslayers, layeritems, layerbranches, layerdependencies, branchnum)
319 depslayers = depslayer
320 if invaluenames:
321 for invaluename in invaluenames:
322 logger.error('Layer "%s" not found in layer index' % invaluename)
323 return 1
324 logger.plain("%s %s %s %s" % ("Layer".ljust(19), "Required by".ljust(19), "Git repository".ljust(54), "Subdirectory"))
325 logger.plain('=' * 115)
326 for layername in args.layername:
327 layerurl = repourls[layername]
328 logger.plain("%s %s %s %s" % (layername.ljust(20), '-'.ljust(20), layerurl[0].ljust(55), layerurl[1]))
329 printedlayers = []
330 for layer, dependency, gitrepo, subdirectory in printlayers:
331 if dependency in printedlayers:
332 continue
333 logger.plain("%s %s %s %s" % (dependency.ljust(20), layer.ljust(20), gitrepo.ljust(55), subdirectory))
334 printedlayers.append(dependency)
335
336 if repourls:
337 fetchdir = self.bbhandler.config_data.getVar('BBLAYERS_FETCH_DIR', True)
338 if not fetchdir:
339 logger.error("Cannot get BBLAYERS_FETCH_DIR")
340 return 1
341 if not os.path.exists(fetchdir):
342 os.makedirs(fetchdir)
343 addlayers = []
344 for repourl, subdir in repourls.values():
345 name, layerdir = self.get_fetch_layer(fetchdir, repourl, subdir, not args.show_only)
346 if not name:
347 # Error already shown
348 return 1
349 addlayers.append((subdir, name, layerdir))
350 if not args.show_only:
351 for subdir, name, layerdir in set(addlayers):
352 if os.path.exists(layerdir):
353 if subdir:
354 logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % subdir)
355 else:
356 logger.plain("Adding layer \"%s\" to conf/bblayers.conf" % name)
357 localargs = argparse.Namespace()
358 localargs.layerdir = layerdir
359 self.do_add_layer(localargs)
360 else:
361 break
362
363
364 def do_layerindex_show_depends(self, args):
365 """Find layer dependencies from layer index.
366"""
367 args.show_only = True
368 args.ignore = []
369 self.do_layerindex_fetch(args)
370
371
130 def version_str(self, pe, pv, pr = None): 372 def version_str(self, pe, pv, pr = None):
131 verstr = "%s" % pv 373 verstr = "%s" % pv
132 if pr: 374 if pr:
@@ -759,6 +1001,16 @@ def main():
759 parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true') 1001 parser_show_cross_depends.add_argument('-f', '--filenames', help='show full file path', action='store_true')
760 parser_show_cross_depends.add_argument('-i', '--ignore', help='ignore dependencies on items in the specified layer(s) (split multiple layer names with commas, no spaces)', metavar='LAYERNAME') 1002 parser_show_cross_depends.add_argument('-i', '--ignore', help='ignore dependencies on items in the specified layer(s) (split multiple layer names with commas, no spaces)', metavar='LAYERNAME')
761 1003
1004 parser_layerindex_fetch = add_command('layerindex-fetch', cmds.do_layerindex_fetch)
1005 parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true')
1006 parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master')
1007 parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER')
1008 parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch')
1009
1010 parser_layerindex_show_depends = add_command('layerindex-show-depends', cmds.do_layerindex_show_depends)
1011 parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master')
1012 parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query')
1013
762 args = parser.parse_args() 1014 args = parser.parse_args()
763 1015
764 if args.debug: 1016 if args.debug: