import argparse import http.client import json import logging import os import subprocess import urllib.parse from bblayers.action import ActionPlugin logger = logging.getLogger('bitbake-layers') def plugin_init(plugins): return LayerIndexPlugin() class LayerIndexPlugin(ActionPlugin): """Subcommands for interacting with the layer index. This class inherits ActionPlugin to get do_add_layer. """ def get_json_data(self, apiurl): proxy_settings = os.environ.get("http_proxy", None) conn = None _parsedurl = urllib.parse.urlparse(apiurl) path = _parsedurl.path query = _parsedurl.query def parse_url(url): parsedurl = urllib.parse.urlparse(url) if parsedurl.netloc[0] == '[': host, port = parsedurl.netloc[1:].split(']', 1) if ':' in port: port = port.rsplit(':', 1)[1] else: port = None else: if parsedurl.netloc.count(':') == 1: (host, port) = parsedurl.netloc.split(":") else: host = parsedurl.netloc port = None return (host, 80 if port is None else int(port)) if proxy_settings is None: host, port = parse_url(apiurl) conn = http.client.HTTPConnection(host, port) conn.request("GET", path + "?" + query) else: host, port = parse_url(proxy_settings) conn = http.client.HTTPConnection(host, port) conn.request("GET", apiurl) r = conn.getresponse() if r.status != 200: raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason)) return json.loads(r.read().decode()) def get_layer_deps(self, layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=False): def layeritems_info_id(items_name, layeritems): litems_id = None for li in layeritems: if li['name'] == items_name: litems_id = li['id'] break return litems_id def layerbranches_info(items_id, layerbranches): lbranch = {} for lb in layerbranches: if lb['layer'] == items_id and lb['branch'] == branchnum: lbranch['id'] = lb['id'] lbranch['vcs_subdir'] = lb['vcs_subdir'] break return lbranch def layerdependencies_info(lb_id, layerdependencies): ld_deps = [] for ld in layerdependencies: if ld['layerbranch'] == lb_id and not ld['dependency'] in ld_deps: ld_deps.append(ld['dependency']) if not ld_deps: logger.error("The dependency of layerDependencies is not found.") return ld_deps def layeritems_info_name_subdir(items_id, layeritems): litems = {} for li in layeritems: if li['id'] == items_id: litems['vcs_url'] = li['vcs_url'] litems['name'] = li['name'] break return litems if selfname: selfid = layeritems_info_id(layername, layeritems) lbinfo = layerbranches_info(selfid, layerbranches) if lbinfo: selfsubdir = lbinfo['vcs_subdir'] else: logger.error("%s is not found in the specified branch" % layername) return selfurl = layeritems_info_name_subdir(selfid, layeritems)['vcs_url'] if selfurl: return selfurl, selfsubdir else: logger.error("Cannot get layer %s git repo and subdir" % layername) return ldict = {} itemsid = layeritems_info_id(layername, layeritems) if not itemsid: return layername, None lbid = layerbranches_info(itemsid, layerbranches) if lbid: lbid = layerbranches_info(itemsid, layerbranches)['id'] else: logger.error("%s is not found in the specified branch" % layername) return None, None for dependency in layerdependencies_info(lbid, layerdependencies): lname = layeritems_info_name_subdir(dependency, layeritems)['name'] lurl = layeritems_info_name_subdir(dependency, layeritems)['vcs_url'] lsubdir = layerbranches_info(dependency, layerbranches)['vcs_subdir'] ldict[lname] = lurl, lsubdir return None, ldict def get_fetch_layer(self, fetchdir, url, subdir, fetch_layer): layername = self.get_layer_name(url) if os.path.splitext(layername)[1] == '.git': layername = os.path.splitext(layername)[0] repodir = os.path.join(fetchdir, layername) layerdir = os.path.join(repodir, subdir) if not os.path.exists(repodir): if fetch_layer: result = subprocess.call('git clone %s %s' % (url, repodir), shell = True) if result: logger.error("Failed to download %s" % url) return None, None else: return layername, layerdir else: logger.plain("Repository %s needs to be fetched" % url) return layername, layerdir elif os.path.exists(layerdir): return layername, layerdir else: logger.error("%s is not in %s" % (url, subdir)) return None, None def do_layerindex_fetch(self, args): """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf. """ apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL') if not apiurl: logger.error("Cannot get BBLAYERS_LAYERINDEX_URL") return 1 else: if apiurl[-1] != '/': apiurl += '/' apiurl += "api/" apilinks = self.get_json_data(apiurl) branches = self.get_json_data(apilinks['branches']) branchnum = 0 for branch in branches: if branch['name'] == args.branch: branchnum = branch['id'] break if branchnum == 0: validbranches = ', '.join([branch['name'] for branch in branches]) logger.error('Invalid layer branch name "%s". Valid branches: %s' % (args.branch, validbranches)) return 1 ignore_layers = [] for collection in self.tinfoil.config_data.getVar('BBFILE_COLLECTIONS').split(): lname = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection) if lname: ignore_layers.append(lname) if args.ignore: ignore_layers.extend(args.ignore.split(',')) layeritems = self.get_json_data(apilinks['layerItems']) layerbranches = self.get_json_data(apilinks['layerBranches']) layerdependencies = self.get_json_data(apilinks['layerDependencies']) invaluenames = [] repourls = {} printlayers = [] def query_dependencies(layers, layeritems, layerbranches, layerdependencies, branchnum): depslayer = [] for layername in layers: invaluename, layerdict = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum) if layerdict: repourls[layername] = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=True) for layer in layerdict: if not layer in ignore_layers: depslayer.append(layer) printlayers.append((layername, layer, layerdict[layer][0], layerdict[layer][1])) if not layer in ignore_layers and not layer in repourls: repourls[layer] = (layerdict[layer][0], layerdict[layer][1]) if invaluename and not invaluename in invaluenames: invaluenames.append(invaluename) return depslayer depslayers = query_dependencies(args.layername, layeritems, layerbranches, layerdependencies, branchnum) while depslayers: depslayer = query_dependencies(depslayers, layeritems, layerbranches, layerdependencies, branchnum) depslayers = depslayer if invaluenames: for invaluename in invaluenames: logger.error('Layer "%s" not found in layer index' % invaluename) return 1 logger.plain("%s %s %s %s" % ("Layer".ljust(19), "Required by".ljust(19), "Git repository".ljust(54), "Subdirectory")) logger.plain('=' * 115) for layername in args.layername: layerurl = repourls[layername] logger.plain("%s %s %s %s" % (layername.ljust(20), '-'.ljust(20), layerurl[0].ljust(55), layerurl[1])) printedlayers = [] for layer, dependency, gitrepo, subdirectory in printlayers: if dependency in printedlayers: continue logger.plain("%s %s %s %s" % (dependency.ljust(20), layer.ljust(20), gitrepo.ljust(55), subdirectory)) printedlayers.append(dependency) if repourls: fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR') if not fetchdir: logger.error("Cannot get BBLAYERS_FETCH_DIR") return 1 if not os.path.exists(fetchdir): os.makedirs(fetchdir) addlayers = [] for repourl, subdir in repourls.values(): name, layerdir = self.get_fetch_layer(fetchdir, repourl, subdir, not args.show_only) if not name: # Error already shown return 1 addlayers.append((subdir, name, layerdir)) if not args.show_only: localargs = argparse.Namespace() localargs.layerdir = [] localargs.force = args.force for subdir, name, layerdir in set(addlayers): if os.path.exists(layerdir): if subdir: logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (subdir, layerdir)) else: logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (name, layerdir)) localargs.layerdir.append(layerdir) else: break if localargs.layerdir: self.do_add_layer(localargs) def do_layerindex_show_depends(self, args): """Find layer dependencies from layer index. """ args.show_only = True args.ignore = [] self.do_layerindex_fetch(args) def register_commands(self, sp): parser_layerindex_fetch = self.add_command(sp, 'layerindex-fetch', self.do_layerindex_fetch) parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true') parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master') parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER') parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch') parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends) parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master') parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query')