diff options
Diffstat (limited to 'bitbake/lib/layerindexlib/restapi.py')
-rw-r--r-- | bitbake/lib/layerindexlib/restapi.py | 398 |
1 files changed, 398 insertions, 0 deletions
diff --git a/bitbake/lib/layerindexlib/restapi.py b/bitbake/lib/layerindexlib/restapi.py new file mode 100644 index 0000000000..d08eb20555 --- /dev/null +++ b/bitbake/lib/layerindexlib/restapi.py | |||
@@ -0,0 +1,398 @@ | |||
1 | # Copyright (C) 2016-2018 Wind River Systems, Inc. | ||
2 | # | ||
3 | # This program is free software; you can redistribute it and/or modify | ||
4 | # it under the terms of the GNU General Public License version 2 as | ||
5 | # published by the Free Software Foundation. | ||
6 | # | ||
7 | # This program is distributed in the hope that it will be useful, | ||
8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. | ||
10 | # See the GNU General Public License for more details. | ||
11 | # | ||
12 | # You should have received a copy of the GNU General Public License | ||
13 | # along with this program; if not, write to the Free Software | ||
14 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | ||
15 | |||
16 | import logging | ||
17 | import json | ||
18 | from urllib.parse import unquote | ||
19 | from urllib.parse import urlparse | ||
20 | |||
21 | import layerindexlib | ||
22 | import layerindexlib.plugin | ||
23 | |||
24 | logger = logging.getLogger('BitBake.layerindexlib.restapi') | ||
25 | |||
26 | def plugin_init(plugins): | ||
27 | return RestApiPlugin() | ||
28 | |||
29 | class RestApiPlugin(layerindexlib.plugin.IndexPlugin): | ||
30 | def __init__(self): | ||
31 | self.type = "restapi" | ||
32 | |||
33 | def load_index(self, url, load): | ||
34 | """ | ||
35 | Fetches layer information from a local or remote layer index. | ||
36 | |||
37 | The return value is a LayerIndexObj. | ||
38 | |||
39 | url is the url to the rest api of the layer index, such as: | ||
40 | http://layers.openembedded.org/layerindex/api/ | ||
41 | |||
42 | Or a local file... | ||
43 | """ | ||
44 | |||
45 | up = urlparse(url) | ||
46 | |||
47 | if up.scheme == 'file': | ||
48 | return self.load_index_file(up, url, load) | ||
49 | |||
50 | if up.scheme == 'http' or up.scheme == 'https': | ||
51 | return self.load_index_web(up, url, load) | ||
52 | |||
53 | raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url) | ||
54 | |||
55 | |||
56 | def load_index_file(self, up, url, load): | ||
57 | """ | ||
58 | Fetches layer information from a local file or directory. | ||
59 | |||
60 | The return value is a LayerIndexObj. | ||
61 | |||
62 | ud is the parsed url to the local file or directory. | ||
63 | """ | ||
64 | if not os.path.exists(up.path): | ||
65 | raise FileNotFoundError(up.path) | ||
66 | |||
67 | index = layerindexlib.LayerIndexObj() | ||
68 | |||
69 | index.config = {} | ||
70 | index.config['TYPE'] = self.type | ||
71 | index.config['URL'] = url | ||
72 | |||
73 | params = self.layerindex._parse_params(up.params) | ||
74 | |||
75 | if 'desc' in params: | ||
76 | index.config['DESCRIPTION'] = unquote(params['desc']) | ||
77 | else: | ||
78 | index.config['DESCRIPTION'] = up.path | ||
79 | |||
80 | if 'cache' in params: | ||
81 | index.config['CACHE'] = params['cache'] | ||
82 | |||
83 | if 'branch' in params: | ||
84 | branches = params['branch'].split(',') | ||
85 | index.config['BRANCH'] = branches | ||
86 | else: | ||
87 | branches = ['*'] | ||
88 | |||
89 | |||
90 | def load_cache(path, index, branches=[]): | ||
91 | logger.debug(1, 'Loading json file %s' % path) | ||
92 | with open(path, 'rt', encoding='utf-8') as f: | ||
93 | pindex = json.load(f) | ||
94 | |||
95 | # Filter the branches on loaded files... | ||
96 | newpBranch = [] | ||
97 | for branch in branches: | ||
98 | if branch != '*': | ||
99 | if 'branches' in pindex: | ||
100 | for br in pindex['branches']: | ||
101 | if br['name'] == branch: | ||
102 | newpBranch.append(br) | ||
103 | else: | ||
104 | if 'branches' in pindex: | ||
105 | for br in pindex['branches']: | ||
106 | newpBranch.append(br) | ||
107 | |||
108 | if newpBranch: | ||
109 | index.add_raw_element('branches', layerindexlib.Branch, newpBranch) | ||
110 | else: | ||
111 | logger.debug(1, 'No matching branches (%s) in index file(s)' % branches) | ||
112 | # No matching branches.. return nothing... | ||
113 | return | ||
114 | |||
115 | for (lName, lType) in [("layerItems", layerindexlib.LayerItem), | ||
116 | ("layerBranches", layerindexlib.LayerBranch), | ||
117 | ("layerDependencies", layerindexlib.LayerDependency), | ||
118 | ("recipes", layerindexlib.Recipe), | ||
119 | ("machines", layerindexlib.Machine), | ||
120 | ("distros", layerindexlib.Distro)]: | ||
121 | if lName in pindex: | ||
122 | index.add_raw_element(lName, lType, pindex[lName]) | ||
123 | |||
124 | |||
125 | if not os.path.isdir(up.path): | ||
126 | load_cache(up.path, index, branches) | ||
127 | return index | ||
128 | |||
129 | logger.debug(1, 'Loading from dir %s...' % (up.path)) | ||
130 | for (dirpath, _, filenames) in os.walk(up.path): | ||
131 | for filename in filenames: | ||
132 | if not filename.endswith('.json'): | ||
133 | continue | ||
134 | fpath = os.path.join(dirpath, filename) | ||
135 | load_cache(fpath, index, branches) | ||
136 | |||
137 | return index | ||
138 | |||
139 | |||
140 | def load_index_web(self, up, url, load): | ||
141 | """ | ||
142 | Fetches layer information from a remote layer index. | ||
143 | |||
144 | The return value is a LayerIndexObj. | ||
145 | |||
146 | ud is the parsed url to the rest api of the layer index, such as: | ||
147 | http://layers.openembedded.org/layerindex/api/ | ||
148 | """ | ||
149 | |||
150 | def _get_json_response(apiurl=None, username=None, password=None, retry=True): | ||
151 | assert apiurl is not None | ||
152 | |||
153 | logger.debug(1, "fetching %s" % apiurl) | ||
154 | |||
155 | up = urlparse(apiurl) | ||
156 | |||
157 | username=up.username | ||
158 | password=up.password | ||
159 | |||
160 | # Strip username/password and params | ||
161 | if up.port: | ||
162 | up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port)) | ||
163 | else: | ||
164 | up_stripped = up._replace(params="", netloc=up.hostname) | ||
165 | |||
166 | res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password) | ||
167 | |||
168 | try: | ||
169 | parsed = json.loads(res.read().decode('utf-8')) | ||
170 | except ConnectionResetError: | ||
171 | if retry: | ||
172 | logger.debug(1, "%s: Connection reset by peer. Retrying..." % url) | ||
173 | parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False) | ||
174 | logger.debug(1, "%s: retry successful.") | ||
175 | else: | ||
176 | raise LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl) | ||
177 | |||
178 | return parsed | ||
179 | |||
180 | index = layerindexlib.LayerIndexObj() | ||
181 | |||
182 | index.config = {} | ||
183 | index.config['TYPE'] = self.type | ||
184 | index.config['URL'] = url | ||
185 | |||
186 | params = self.layerindex._parse_params(up.params) | ||
187 | |||
188 | if 'desc' in params: | ||
189 | index.config['DESCRIPTION'] = unquote(params['desc']) | ||
190 | else: | ||
191 | index.config['DESCRIPTION'] = up.hostname | ||
192 | |||
193 | if 'cache' in params: | ||
194 | index.config['CACHE'] = params['cache'] | ||
195 | |||
196 | if 'branch' in params: | ||
197 | branches = params['branch'].split(',') | ||
198 | index.config['BRANCH'] = branches | ||
199 | else: | ||
200 | branches = ['*'] | ||
201 | |||
202 | try: | ||
203 | index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password) | ||
204 | except Exception as e: | ||
205 | raise layerindexlib.LayerIndexFetchError(url, e) | ||
206 | |||
207 | # Local raw index set... | ||
208 | pindex = {} | ||
209 | |||
210 | # Load all the requested branches at the same time time, | ||
211 | # a special branch of '*' means load all branches | ||
212 | filter = "" | ||
213 | if "*" not in branches: | ||
214 | filter = "?filter=name:%s" % "OR".join(branches) | ||
215 | |||
216 | logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches'])) | ||
217 | |||
218 | # The link won't include username/password, so pull it from the original url | ||
219 | pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter, | ||
220 | username=up.username, password=up.password) | ||
221 | if not pindex['branches']: | ||
222 | logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url)) | ||
223 | return index | ||
224 | index.add_raw_element("branches", layerindexlib.Branch, pindex['branches']) | ||
225 | |||
226 | # Load all of the layerItems (these can not be easily filtered) | ||
227 | logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems'])) | ||
228 | |||
229 | |||
230 | # The link won't include username/password, so pull it from the original url | ||
231 | pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'], | ||
232 | username=up.username, password=up.password) | ||
233 | if not pindex['layerItems']: | ||
234 | logger.debug(1, "No layers were found at url %s." % (url)) | ||
235 | return index | ||
236 | index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems']) | ||
237 | |||
238 | |||
239 | # From this point on load the contents for each branch. Otherwise we | ||
240 | # could run into a timeout. | ||
241 | for branch in index.branches: | ||
242 | filter = "?filter=branch__name:%s" % index.branches[branch].name | ||
243 | |||
244 | logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches'])) | ||
245 | |||
246 | # The link won't include username/password, so pull it from the original url | ||
247 | pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter, | ||
248 | username=up.username, password=up.password) | ||
249 | if not pindex['layerBranches']: | ||
250 | logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url)) | ||
251 | return index | ||
252 | index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches']) | ||
253 | |||
254 | |||
255 | # Load the rest, they all have a similar format | ||
256 | # Note: the layer index has a few more items, we can add them if necessary | ||
257 | # in the future. | ||
258 | filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name | ||
259 | for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency), | ||
260 | ("recipes", layerindexlib.Recipe), | ||
261 | ("machines", layerindexlib.Machine), | ||
262 | ("distros", layerindexlib.Distro)]: | ||
263 | if lName not in load: | ||
264 | continue | ||
265 | logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName])) | ||
266 | |||
267 | # The link won't include username/password, so pull it from the original url | ||
268 | pindex[lName] = _get_json_response(index.apilinks[lName] + filter, | ||
269 | username=up.username, password=up.password) | ||
270 | index.add_raw_element(lName, lType, pindex[lName]) | ||
271 | |||
272 | return index | ||
273 | |||
274 | def store_index(self, url, index): | ||
275 | """ | ||
276 | Store layer information into a local file/dir. | ||
277 | |||
278 | The return value is a dictionary containing API, | ||
279 | layer, branch, dependency, recipe, machine, distro, information. | ||
280 | |||
281 | ud is a parsed url to a directory or file. If the path is a | ||
282 | directory, we will split the files into one file per layer. | ||
283 | If the path is to a file (exists or not) the entire DB will be | ||
284 | dumped into that one file. | ||
285 | """ | ||
286 | |||
287 | up = urlparse(url) | ||
288 | |||
289 | if up.scheme != 'file': | ||
290 | raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url) | ||
291 | |||
292 | logger.debug(1, "Storing to %s..." % up.path) | ||
293 | |||
294 | try: | ||
295 | layerbranches = index.layerBranches | ||
296 | except KeyError: | ||
297 | logger.error('No layerBranches to write.') | ||
298 | return | ||
299 | |||
300 | |||
301 | def filter_item(layerbranchid, objects): | ||
302 | filtered = [] | ||
303 | for obj in getattr(index, objects, None): | ||
304 | try: | ||
305 | if getattr(index, objects)[obj].layerbranch_id == layerbranchid: | ||
306 | filtered.append(getattr(index, objects)[obj]._data) | ||
307 | except AttributeError: | ||
308 | logger.debug(1, 'No obj.layerbranch_id: %s' % objects) | ||
309 | # No simple filter method, just include it... | ||
310 | try: | ||
311 | filtered.append(getattr(index, objects)[obj]._data) | ||
312 | except AttributeError: | ||
313 | logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj))) | ||
314 | filtered.append(obj) | ||
315 | return filtered | ||
316 | |||
317 | |||
318 | # Write out to a single file. | ||
319 | # Filter out unnecessary items, then sort as we write for determinism | ||
320 | if not os.path.isdir(up.path): | ||
321 | pindex = {} | ||
322 | |||
323 | pindex['branches'] = [] | ||
324 | pindex['layerItems'] = [] | ||
325 | pindex['layerBranches'] = [] | ||
326 | |||
327 | for layerbranchid in layerbranches: | ||
328 | if layerbranches[layerbranchid].branch._data not in pindex['branches']: | ||
329 | pindex['branches'].append(layerbranches[layerbranchid].branch._data) | ||
330 | |||
331 | if layerbranches[layerbranchid].layer._data not in pindex['layerItems']: | ||
332 | pindex['layerItems'].append(layerbranches[layerbranchid].layer._data) | ||
333 | |||
334 | if layerbranches[layerbranchid]._data not in pindex['layerBranches']: | ||
335 | pindex['layerBranches'].append(layerbranches[layerbranchid]._data) | ||
336 | |||
337 | for entry in index._index: | ||
338 | # Skip local items, apilinks and items already processed | ||
339 | if entry in index.config['local'] or \ | ||
340 | entry == 'apilinks' or \ | ||
341 | entry == 'branches' or \ | ||
342 | entry == 'layerBranches' or \ | ||
343 | entry == 'layerItems': | ||
344 | continue | ||
345 | if entry not in pindex: | ||
346 | pindex[entry] = [] | ||
347 | pindex[entry].extend(filter_item(layerbranchid, entry)) | ||
348 | |||
349 | bb.debug(1, 'Writing index to %s' % up.path) | ||
350 | with open(up.path, 'wt') as f: | ||
351 | json.dump(layerindexlib.sort_entry(pindex), f, indent=4) | ||
352 | return | ||
353 | |||
354 | |||
355 | # Write out to a directory one file per layerBranch | ||
356 | # Prepare all layer related items, to create a minimal file. | ||
357 | # We have to sort the entries as we write so they are deterministic | ||
358 | for layerbranchid in layerbranches: | ||
359 | pindex = {} | ||
360 | |||
361 | for entry in index._index: | ||
362 | # Skip local items, apilinks and items already processed | ||
363 | if entry in index.config['local'] or \ | ||
364 | entry == 'apilinks' or \ | ||
365 | entry == 'branches' or \ | ||
366 | entry == 'layerBranches' or \ | ||
367 | entry == 'layerItems': | ||
368 | continue | ||
369 | pindex[entry] = filter_item(layerbranchid, entry) | ||
370 | |||
371 | # Add the layer we're processing as the first one... | ||
372 | pindex['branches'] = [layerbranches[layerbranchid].branch._data] | ||
373 | pindex['layerItems'] = [layerbranches[layerbranchid].layer._data] | ||
374 | pindex['layerBranches'] = [layerbranches[layerbranchid]._data] | ||
375 | |||
376 | # We also need to include the layerbranch for any dependencies... | ||
377 | for layerdep in pindex['layerDependencies']: | ||
378 | layerdependency = layerindexlib.LayerDependency(index, layerdep) | ||
379 | |||
380 | layeritem = layerdependency.dependency | ||
381 | layerbranch = layerdependency.dependency_layerBranch | ||
382 | |||
383 | # We need to avoid duplicates... | ||
384 | if layeritem._data not in pindex['layerItems']: | ||
385 | pindex['layerItems'].append(layeritem._data) | ||
386 | |||
387 | if layerbranch._data not in pindex['layerBranches']: | ||
388 | pindex['layerBranches'].append(layerbranch._data) | ||
389 | |||
390 | # apply mirroring adjustments here.... | ||
391 | |||
392 | fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name'] | ||
393 | fname = fname.translate(str.maketrans('/ ', '__')) | ||
394 | fpath = os.path.join(up.path, fname) | ||
395 | |||
396 | bb.debug(1, 'Writing index to %s' % fpath + '.json') | ||
397 | with open(fpath + '.json', 'wt') as f: | ||
398 | json.dump(layerindexlib.sort_entry(pindex), f, indent=4) | ||