summaryrefslogtreecommitdiffstats
path: root/bitbake/lib
diff options
context:
space:
mode:
authorMark Hatle <mark.hatle@windriver.com>2018-07-23 22:29:12 -0400
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-08-02 10:18:27 +0100
commitddfcda7ec3e03c5d9dae1396dfa7d0c2920a3619 (patch)
tree8f59840758771f4d8b10ec6a5558eca91a9e46ca /bitbake/lib
parent1ac19d1bf111a4836625f5cbb28a751d5c427395 (diff)
downloadpoky-ddfcda7ec3e03c5d9dae1396dfa7d0c2920a3619.tar.gz
bitbake: bblayers/layerindex.py: Switch to use the new layerindexlib class
Display changes: The output will now include references to the layers that the user already has on their system. It does this by querying the cooker derived index. The code that enables this behavior is labeled as 'TODO' currently. As part of the work we need to make a final determination if this is the desired output. Also changed the default branch to no longer define itself as 'master'. When the user does NOT set a branch, the default is now the 'LAYERSERIES_CORENAMES', and if that doesn't exist 'master'. This is subtly different in behavior, but more consistent with user expectations. (Bitbake rev: 478c4b5489943f756ae03d6b6d3b5d665cedbce0) Signed-off-by: Mark Hatle <mark.hatle@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib')
-rw-r--r--bitbake/lib/bblayers/layerindex.py304
1 files changed, 119 insertions, 185 deletions
diff --git a/bitbake/lib/bblayers/layerindex.py b/bitbake/lib/bblayers/layerindex.py
index 53c858db16..121c7d515a 100644
--- a/bitbake/lib/bblayers/layerindex.py
+++ b/bitbake/lib/bblayers/layerindex.py
@@ -1,10 +1,9 @@
1import layerindexlib
2
1import argparse 3import argparse
2import http.client
3import json
4import logging 4import logging
5import os 5import os
6import subprocess 6import subprocess
7import urllib.parse
8 7
9from bblayers.action import ActionPlugin 8from bblayers.action import ActionPlugin
10 9
@@ -21,110 +20,6 @@ class LayerIndexPlugin(ActionPlugin):
21 This class inherits ActionPlugin to get do_add_layer. 20 This class inherits ActionPlugin to get do_add_layer.
22 """ 21 """
23 22
24 def get_json_data(self, apiurl):
25 proxy_settings = os.environ.get("http_proxy", None)
26 conn = None
27 _parsedurl = urllib.parse.urlparse(apiurl)
28 path = _parsedurl.path
29 query = _parsedurl.query
30
31 def parse_url(url):
32 parsedurl = urllib.parse.urlparse(url)
33 if parsedurl.netloc[0] == '[':
34 host, port = parsedurl.netloc[1:].split(']', 1)
35 if ':' in port:
36 port = port.rsplit(':', 1)[1]
37 else:
38 port = None
39 else:
40 if parsedurl.netloc.count(':') == 1:
41 (host, port) = parsedurl.netloc.split(":")
42 else:
43 host = parsedurl.netloc
44 port = None
45 return (host, 80 if port is None else int(port))
46
47 if proxy_settings is None:
48 host, port = parse_url(apiurl)
49 conn = http.client.HTTPConnection(host, port)
50 conn.request("GET", path + "?" + query)
51 else:
52 host, port = parse_url(proxy_settings)
53 conn = http.client.HTTPConnection(host, port)
54 conn.request("GET", apiurl)
55
56 r = conn.getresponse()
57 if r.status != 200:
58 raise Exception("Failed to read " + path + ": %d %s" % (r.status, r.reason))
59 return json.loads(r.read().decode())
60
61 def get_layer_deps(self, layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=False):
62 def layeritems_info_id(items_name, layeritems):
63 litems_id = None
64 for li in layeritems:
65 if li['name'] == items_name:
66 litems_id = li['id']
67 break
68 return litems_id
69
70 def layerbranches_info(items_id, layerbranches):
71 lbranch = {}
72 for lb in layerbranches:
73 if lb['layer'] == items_id and lb['branch'] == branchnum:
74 lbranch['id'] = lb['id']
75 lbranch['vcs_subdir'] = lb['vcs_subdir']
76 break
77 return lbranch
78
79 def layerdependencies_info(lb_id, layerdependencies):
80 ld_deps = []
81 for ld in layerdependencies:
82 if ld['layerbranch'] == lb_id and not ld['dependency'] in ld_deps:
83 ld_deps.append(ld['dependency'])
84 if not ld_deps:
85 logger.error("The dependency of layerDependencies is not found.")
86 return ld_deps
87
88 def layeritems_info_name_subdir(items_id, layeritems):
89 litems = {}
90 for li in layeritems:
91 if li['id'] == items_id:
92 litems['vcs_url'] = li['vcs_url']
93 litems['name'] = li['name']
94 break
95 return litems
96
97 if selfname:
98 selfid = layeritems_info_id(layername, layeritems)
99 lbinfo = layerbranches_info(selfid, layerbranches)
100 if lbinfo:
101 selfsubdir = lbinfo['vcs_subdir']
102 else:
103 logger.error("%s is not found in the specified branch" % layername)
104 return
105 selfurl = layeritems_info_name_subdir(selfid, layeritems)['vcs_url']
106 if selfurl:
107 return selfurl, selfsubdir
108 else:
109 logger.error("Cannot get layer %s git repo and subdir" % layername)
110 return
111 ldict = {}
112 itemsid = layeritems_info_id(layername, layeritems)
113 if not itemsid:
114 return layername, None
115 lbid = layerbranches_info(itemsid, layerbranches)
116 if lbid:
117 lbid = layerbranches_info(itemsid, layerbranches)['id']
118 else:
119 logger.error("%s is not found in the specified branch" % layername)
120 return None, None
121 for dependency in layerdependencies_info(lbid, layerdependencies):
122 lname = layeritems_info_name_subdir(dependency, layeritems)['name']
123 lurl = layeritems_info_name_subdir(dependency, layeritems)['vcs_url']
124 lsubdir = layerbranches_info(dependency, layerbranches)['vcs_subdir']
125 ldict[lname] = lurl, lsubdir
126 return None, ldict
127
128 def get_fetch_layer(self, fetchdir, url, subdir, fetch_layer): 23 def get_fetch_layer(self, fetchdir, url, subdir, fetch_layer):
129 layername = self.get_layer_name(url) 24 layername = self.get_layer_name(url)
130 if os.path.splitext(layername)[1] == '.git': 25 if os.path.splitext(layername)[1] == '.git':
@@ -136,95 +31,124 @@ class LayerIndexPlugin(ActionPlugin):
136 result = subprocess.call('git clone %s %s' % (url, repodir), shell = True) 31 result = subprocess.call('git clone %s %s' % (url, repodir), shell = True)
137 if result: 32 if result:
138 logger.error("Failed to download %s" % url) 33 logger.error("Failed to download %s" % url)
139 return None, None 34 return None, None, None
140 else: 35 else:
141 return layername, layerdir 36 return subdir, layername, layerdir
142 else: 37 else:
143 logger.plain("Repository %s needs to be fetched" % url) 38 logger.plain("Repository %s needs to be fetched" % url)
144 return layername, layerdir 39 return subdir, layername, layerdir
145 elif os.path.exists(layerdir): 40 elif os.path.exists(layerdir):
146 return layername, layerdir 41 return subdir, layername, layerdir
147 else: 42 else:
148 logger.error("%s is not in %s" % (url, subdir)) 43 logger.error("%s is not in %s" % (url, subdir))
149 return None, None 44 return None, None, None
150 45
151 def do_layerindex_fetch(self, args): 46 def do_layerindex_fetch(self, args):
152 """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf. 47 """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
153""" 48"""
154 apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL') 49
155 if not apiurl: 50 def _construct_url(baseurls, branches):
156 logger.error("Cannot get BBLAYERS_LAYERINDEX_URL") 51 urls = []
157 return 1 52 for baseurl in baseurls:
53 if baseurl[-1] != '/':
54 baseurl += '/'
55
56 if not baseurl.startswith('cooker'):
57 baseurl += "api/"
58
59 if branches:
60 baseurl += ";branch=%s" % ','.join(branches)
61
62 urls.append(baseurl)
63
64 return urls
65
66
67 # Set the default...
68 if args.branch:
69 branches = [args.branch]
158 else: 70 else:
159 if apiurl[-1] != '/': 71 branches = (self.tinfoil.config_data.getVar('LAYERSERIES_CORENAMES') or 'master').split()
160 apiurl += '/' 72 logger.debug(1, 'Trying branches: %s' % branches)
161 apiurl += "api/"
162 apilinks = self.get_json_data(apiurl)
163 branches = self.get_json_data(apilinks['branches'])
164
165 branchnum = 0
166 for branch in branches:
167 if branch['name'] == args.branch:
168 branchnum = branch['id']
169 break
170 if branchnum == 0:
171 validbranches = ', '.join([branch['name'] for branch in branches])
172 logger.error('Invalid layer branch name "%s". Valid branches: %s' % (args.branch, validbranches))
173 return 1
174 73
175 ignore_layers = [] 74 ignore_layers = []
176 for collection in self.tinfoil.config_data.getVar('BBFILE_COLLECTIONS').split():
177 lname = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % collection)
178 if lname:
179 ignore_layers.append(lname)
180
181 if args.ignore: 75 if args.ignore:
182 ignore_layers.extend(args.ignore.split(',')) 76 ignore_layers.extend(args.ignore.split(','))
183 77
184 layeritems = self.get_json_data(apilinks['layerItems']) 78 # Load the cooker DB
185 layerbranches = self.get_json_data(apilinks['layerBranches']) 79 cookerIndex = layerindexlib.LayerIndex(self.tinfoil.config_data)
186 layerdependencies = self.get_json_data(apilinks['layerDependencies']) 80 cookerIndex.load_layerindex('cooker://', load='layerDependencies')
187 invaluenames = [] 81
188 repourls = {} 82 # Fast path, check if we already have what has been requested!
189 printlayers = [] 83 (dependencies, invalidnames) = cookerIndex.find_dependencies(names=args.layername, ignores=ignore_layers)
190 84 if not args.show_only and not invalidnames:
191 def query_dependencies(layers, layeritems, layerbranches, layerdependencies, branchnum): 85 logger.plain("You already have the requested layer(s): %s" % args.layername)
192 depslayer = [] 86 return 0
193 for layername in layers: 87
194 invaluename, layerdict = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum) 88 # The information to show is already in the cookerIndex
195 if layerdict: 89 if invalidnames:
196 repourls[layername] = self.get_layer_deps(layername, layeritems, layerbranches, layerdependencies, branchnum, selfname=True) 90 # General URL to use to access the layer index
197 for layer in layerdict: 91 # While there is ONE right now, we're expect users could enter several
198 if not layer in ignore_layers: 92 apiurl = self.tinfoil.config_data.getVar('BBLAYERS_LAYERINDEX_URL').split()
199 depslayer.append(layer) 93 if not apiurl:
200 printlayers.append((layername, layer, layerdict[layer][0], layerdict[layer][1])) 94 logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
201 if not layer in ignore_layers and not layer in repourls: 95 return 1
202 repourls[layer] = (layerdict[layer][0], layerdict[layer][1]) 96
203 if invaluename and not invaluename in invaluenames: 97 remoteIndex = layerindexlib.LayerIndex(self.tinfoil.config_data)
204 invaluenames.append(invaluename) 98
205 return depslayer 99 for remoteurl in _construct_url(apiurl, branches):
206 100 logger.plain("Loading %s..." % remoteurl)
207 depslayers = query_dependencies(args.layername, layeritems, layerbranches, layerdependencies, branchnum) 101 remoteIndex.load_layerindex(remoteurl)
208 while depslayers: 102
209 depslayer = query_dependencies(depslayers, layeritems, layerbranches, layerdependencies, branchnum) 103 if remoteIndex.is_empty():
210 depslayers = depslayer 104 logger.error("Remote layer index %s is empty for branches %s" % (apiurl, branches))
211 if invaluenames: 105 return 1
212 for invaluename in invaluenames: 106
213 logger.error('Layer "%s" not found in layer index' % invaluename) 107 lIndex = cookerIndex + remoteIndex
214 return 1 108
215 logger.plain("%s %s %s %s" % ("Layer".ljust(19), "Required by".ljust(19), "Git repository".ljust(54), "Subdirectory")) 109 (dependencies, invalidnames) = lIndex.find_dependencies(names=args.layername, ignores=ignore_layers)
216 logger.plain('=' * 115) 110
217 for layername in args.layername: 111 if invalidnames:
218 layerurl = repourls[layername] 112 for invaluename in invalidnames:
219 logger.plain("%s %s %s %s" % (layername.ljust(20), '-'.ljust(20), layerurl[0].ljust(55), layerurl[1])) 113 logger.error('Layer "%s" not found in layer index' % invaluename)
220 printedlayers = [] 114 return 1
221 for layer, dependency, gitrepo, subdirectory in printlayers: 115
222 if dependency in printedlayers: 116 logger.plain("%s %s %s" % ("Layer".ljust(49), "Git repository (branch)".ljust(54), "Subdirectory"))
223 continue 117 logger.plain('=' * 125)
224 logger.plain("%s %s %s %s" % (dependency.ljust(20), layer.ljust(20), gitrepo.ljust(55), subdirectory)) 118
225 printedlayers.append(dependency) 119 for deplayerbranch in dependencies:
226 120 layerBranch = dependencies[deplayerbranch][0]
227 if repourls: 121
122 # TODO: Determine display behavior
123 # This is the local content, uncomment to hide local
124 # layers from the display.
125 #if layerBranch.index.config['TYPE'] == 'cooker':
126 # continue
127
128 layerDeps = dependencies[deplayerbranch][1:]
129
130 requiredby = []
131 recommendedby = []
132 for dep in layerDeps:
133 if dep.required:
134 requiredby.append(dep.layer.name)
135 else:
136 recommendedby.append(dep.layer.name)
137
138 logger.plain('%s %s %s' % (("%s:%s:%s" %
139 (layerBranch.index.config['DESCRIPTION'],
140 layerBranch.branch.name,
141 layerBranch.layer.name)).ljust(50),
142 ("%s (%s)" % (layerBranch.layer.vcs_url,
143 layerBranch.actual_branch)).ljust(55),
144 layerBranch.vcs_subdir
145 ))
146 if requiredby:
147 logger.plain(' required by: %s' % ' '.join(requiredby))
148 if recommendedby:
149 logger.plain(' recommended by: %s' % ' '.join(recommendedby))
150
151 if dependencies:
228 fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR') 152 fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
229 if not fetchdir: 153 if not fetchdir:
230 logger.error("Cannot get BBLAYERS_FETCH_DIR") 154 logger.error("Cannot get BBLAYERS_FETCH_DIR")
@@ -232,8 +156,18 @@ class LayerIndexPlugin(ActionPlugin):
232 if not os.path.exists(fetchdir): 156 if not os.path.exists(fetchdir):
233 os.makedirs(fetchdir) 157 os.makedirs(fetchdir)
234 addlayers = [] 158 addlayers = []
235 for repourl, subdir in repourls.values(): 159
236 name, layerdir = self.get_fetch_layer(fetchdir, repourl, subdir, not args.show_only) 160 for deplayerbranch in dependencies:
161 layerBranch = dependencies[deplayerbranch][0]
162
163 if layerBranch.index.config['TYPE'] == 'cooker':
164 # Anything loaded via cooker is already local, skip it
165 continue
166
167 subdir, name, layerdir = self.get_fetch_layer(fetchdir,
168 layerBranch.layer.vcs_url,
169 layerBranch.vcs_subdir,
170 not args.show_only)
237 if not name: 171 if not name:
238 # Error already shown 172 # Error already shown
239 return 1 173 return 1
@@ -242,7 +176,7 @@ class LayerIndexPlugin(ActionPlugin):
242 localargs = argparse.Namespace() 176 localargs = argparse.Namespace()
243 localargs.layerdir = [] 177 localargs.layerdir = []
244 localargs.force = args.force 178 localargs.force = args.force
245 for subdir, name, layerdir in set(addlayers): 179 for subdir, name, layerdir in addlayers:
246 if os.path.exists(layerdir): 180 if os.path.exists(layerdir):
247 if subdir: 181 if subdir:
248 logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (subdir, layerdir)) 182 logger.plain("Adding layer \"%s\" (%s) to conf/bblayers.conf" % (subdir, layerdir))
@@ -265,10 +199,10 @@ class LayerIndexPlugin(ActionPlugin):
265 def register_commands(self, sp): 199 def register_commands(self, sp):
266 parser_layerindex_fetch = self.add_command(sp, 'layerindex-fetch', self.do_layerindex_fetch) 200 parser_layerindex_fetch = self.add_command(sp, 'layerindex-fetch', self.do_layerindex_fetch)
267 parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true') 201 parser_layerindex_fetch.add_argument('-n', '--show-only', help='show dependencies and do nothing else', action='store_true')
268 parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master') 202 parser_layerindex_fetch.add_argument('-b', '--branch', help='branch name to fetch')
269 parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER') 203 parser_layerindex_fetch.add_argument('-i', '--ignore', help='assume the specified layers do not need to be fetched/added (separate multiple layers with commas, no spaces)', metavar='LAYER')
270 parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch') 204 parser_layerindex_fetch.add_argument('layername', nargs='+', help='layer to fetch')
271 205
272 parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends) 206 parser_layerindex_show_depends = self.add_command(sp, 'layerindex-show-depends', self.do_layerindex_show_depends)
273 parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch (default %(default)s)', default='master') 207 parser_layerindex_show_depends.add_argument('-b', '--branch', help='branch name to fetch')
274 parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query') 208 parser_layerindex_show_depends.add_argument('layername', nargs='+', help='layer to query')