summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMark Hatle <mark.hatle@windriver.com>2018-07-23 22:29:11 -0400
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-08-02 10:18:27 +0100
commit1ac19d1bf111a4836625f5cbb28a751d5c427395 (patch)
treeae9fa1448b425522952cbc8af90cf79912c746b1
parent0dea95093115acc08f6ad19dc931d532a601cbec (diff)
downloadpoky-1ac19d1bf111a4836625f5cbb28a751d5c427395.tar.gz
bitbake: layerindexlib: Initial layer index processing module implementation
The layer index module is expected to be used by various parts of the system in order to access a layerindex-web (such as layers.openembedded.org) and perform basic processing on the information, such as dependency scanning. Along with the layerindex implementation are associated tests. The tests properly honor BB_SKIP_NETTESTS='yes' to prevent test failures. Tests Implemented: - Branch, LayerItem, LayerBranch, LayerDependency, Recipe, Machine and Distro objects - LayerIndex setup using the layers.openembedded.org restapi - LayerIndex storing and retrieving from a file - LayerIndex verify dependency resolution ordering - LayerIndex setup using simulated cooker data (Bitbake rev: fd0ee6c10dbb5592731e56f4c592fe687682a3e6) Signed-off-by: Mark Hatle <mark.hatle@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rwxr-xr-xbitbake/bin/bitbake-selftest6
-rw-r--r--bitbake/lib/layerindexlib/README28
-rw-r--r--bitbake/lib/layerindexlib/__init__.py1364
-rw-r--r--bitbake/lib/layerindexlib/cooker.py341
-rw-r--r--bitbake/lib/layerindexlib/plugin.py60
-rw-r--r--bitbake/lib/layerindexlib/restapi.py398
-rw-r--r--bitbake/lib/layerindexlib/tests/__init__.py0
-rw-r--r--bitbake/lib/layerindexlib/tests/common.py43
-rw-r--r--bitbake/lib/layerindexlib/tests/cooker.py123
-rw-r--r--bitbake/lib/layerindexlib/tests/layerindexobj.py226
-rw-r--r--bitbake/lib/layerindexlib/tests/restapi.py174
-rw-r--r--bitbake/lib/layerindexlib/tests/testdata/README11
-rw-r--r--bitbake/lib/layerindexlib/tests/testdata/build/conf/bblayers.conf15
-rw-r--r--bitbake/lib/layerindexlib/tests/testdata/layer1/conf/layer.conf17
-rw-r--r--bitbake/lib/layerindexlib/tests/testdata/layer2/conf/layer.conf20
-rw-r--r--bitbake/lib/layerindexlib/tests/testdata/layer3/conf/layer.conf19
-rw-r--r--bitbake/lib/layerindexlib/tests/testdata/layer4/conf/layer.conf22
17 files changed, 2866 insertions, 1 deletions
diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest
index afe1603d04..7564de304c 100755
--- a/bitbake/bin/bitbake-selftest
+++ b/bitbake/bin/bitbake-selftest
@@ -22,6 +22,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib
22import unittest 22import unittest
23try: 23try:
24 import bb 24 import bb
25 import layerindexlib
25except RuntimeError as exc: 26except RuntimeError as exc:
26 sys.exit(str(exc)) 27 sys.exit(str(exc))
27 28
@@ -31,7 +32,10 @@ tests = ["bb.tests.codeparser",
31 "bb.tests.event", 32 "bb.tests.event",
32 "bb.tests.fetch", 33 "bb.tests.fetch",
33 "bb.tests.parse", 34 "bb.tests.parse",
34 "bb.tests.utils"] 35 "bb.tests.utils",
36 "layerindexlib.tests.layerindexobj",
37 "layerindexlib.tests.restapi",
38 "layerindexlib.tests.cooker"]
35 39
36for t in tests: 40for t in tests:
37 t = '.'.join(t.split('.')[:3]) 41 t = '.'.join(t.split('.')[:3])
diff --git a/bitbake/lib/layerindexlib/README b/bitbake/lib/layerindexlib/README
new file mode 100644
index 0000000000..5d927afdf7
--- /dev/null
+++ b/bitbake/lib/layerindexlib/README
@@ -0,0 +1,28 @@
1The layerindexlib module is designed to permit programs to work directly
2with layer index information. (See layers.openembedded.org...)
3
4The layerindexlib module includes a plugin interface that is used to extend
5the basic functionality. There are two primary plugins available: restapi
6and cooker.
7
8The restapi plugin works with a web based REST Api compatible with the
9layerindex-web project, as well as the ability to store and retried a
10the information for one or more files on the disk.
11
12The cooker plugin works by reading the information from the current build
13project and processing it as if it were a layer index.
14
15
16TODO:
17
18__init__.py:
19Implement local on-disk caching (using the rest api store/load)
20Implement layer index style query operations on a combined index
21
22common.py:
23Stop network access if BB_NO_NETWORK or allowed hosts is restricted
24
25cooker.py:
26Cooker - Implement recipe parsing
27
28
diff --git a/bitbake/lib/layerindexlib/__init__.py b/bitbake/lib/layerindexlib/__init__.py
new file mode 100644
index 0000000000..74f3e2e93e
--- /dev/null
+++ b/bitbake/lib/layerindexlib/__init__.py
@@ -0,0 +1,1364 @@
1# Copyright (C) 2016-2018 Wind River Systems, Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License version 2 as
5# published by the Free Software Foundation.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
10# See the GNU General Public License for more details.
11#
12# You should have received a copy of the GNU General Public License
13# along with this program; if not, write to the Free Software
14# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
16import datetime
17
18import logging
19import imp
20
21from collections import OrderedDict
22from layerindexlib.plugin import LayerIndexPluginUrlError
23
24logger = logging.getLogger('BitBake.layerindexlib')
25
26# Exceptions
27
28class LayerIndexException(Exception):
29 '''LayerIndex Generic Exception'''
30 def __init__(self, message):
31 self.msg = message
32 Exception.__init__(self, message)
33
34 def __str__(self):
35 return self.msg
36
37class LayerIndexUrlError(LayerIndexException):
38 '''Exception raised when unable to access a URL for some reason'''
39 def __init__(self, url, message=""):
40 if message:
41 msg = "Unable to access layerindex url %s: %s" % (url, message)
42 else:
43 msg = "Unable to access layerindex url %s" % url
44 self.url = url
45 LayerIndexException.__init__(self, msg)
46
47class LayerIndexFetchError(LayerIndexException):
48 '''General layerindex fetcher exception when something fails'''
49 def __init__(self, url, message=""):
50 if message:
51 msg = "Unable to fetch layerindex url %s: %s" % (url, message)
52 else:
53 msg = "Unable to fetch layerindex url %s" % url
54 self.url = url
55 LayerIndexException.__init__(self, msg)
56
57
58# Interface to the overall layerindex system
59# the layer may contain one or more individual indexes
60class LayerIndex():
61 def __init__(self, d):
62 if not d:
63 raise LayerIndexException("Must be initialized with bb.data.")
64
65 self.data = d
66
67 # List of LayerIndexObj
68 self.indexes = []
69
70 self.plugins = []
71
72 import bb.utils
73 bb.utils.load_plugins(logger, self.plugins, os.path.dirname(__file__))
74 for plugin in self.plugins:
75 if hasattr(plugin, 'init'):
76 plugin.init(self)
77
78 def __add__(self, other):
79 newIndex = LayerIndex(self.data)
80
81 if self.__class__ != newIndex.__class__ or \
82 other.__class__ != newIndex.__class__:
83 raise TypeException("Can not add different types.")
84
85 for indexEnt in self.indexes:
86 newIndex.indexes.append(indexEnt)
87
88 for indexEnt in other.indexes:
89 newIndex.indexes.append(indexEnt)
90
91 return newIndex
92
93 def _parse_params(self, params):
94 '''Take a parameter list, return a dictionary of parameters.
95
96 Expected to be called from the data of urllib.parse.urlparse(url).params
97
98 If there are two conflicting parameters, last in wins...
99 '''
100
101 param_dict = {}
102 for param in params.split(';'):
103 if not param:
104 continue
105 item = param.split('=', 1)
106 logger.debug(1, item)
107 param_dict[item[0]] = item[1]
108
109 return param_dict
110
111 def _fetch_url(self, url, username=None, password=None, debuglevel=0):
112 '''Fetch data from a specific URL.
113
114 Fetch something from a specific URL. This is specifically designed to
115 fetch data from a layerindex-web instance, but may be useful for other
116 raw fetch actions.
117
118 It is not designed to be used to fetch recipe sources or similar. the
119 regular fetcher class should used for that.
120
121 It is the responsibility of the caller to check BB_NO_NETWORK and related
122 BB_ALLOWED_NETWORKS.
123 '''
124
125 if not url:
126 raise LayerIndexUrlError(url, "empty url")
127
128 import urllib
129 from urllib.request import urlopen, Request
130 from urllib.parse import urlparse
131
132 up = urlparse(url)
133
134 if username:
135 logger.debug(1, "Configuring authentication for %s..." % url)
136 password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
137 password_mgr.add_password(None, "%s://%s" % (up.scheme, up.netloc), username, password)
138 handler = urllib.request.HTTPBasicAuthHandler(password_mgr)
139 opener = urllib.request.build_opener(handler, urllib.request.HTTPSHandler(debuglevel=debuglevel))
140 else:
141 opener = urllib.request.build_opener(urllib.request.HTTPSHandler(debuglevel=debuglevel))
142
143 urllib.request.install_opener(opener)
144
145 logger.debug(1, "Fetching %s (%s)..." % (url, ["without authentication", "with authentication"][bool(username)]))
146
147 try:
148 res = urlopen(Request(url, headers={'User-Agent': 'Mozilla/5.0 (bitbake/lib/layerindex)'}, unverifiable=True))
149 except urllib.error.HTTPError as e:
150 logger.debug(1, "HTTP Error: %s: %s" % (e.code, e.reason))
151 logger.debug(1, " Requested: %s" % (url))
152 logger.debug(1, " Actual: %s" % (e.geturl()))
153
154 if e.code == 404:
155 logger.debug(1, "Request not found.")
156 raise LayerIndexFetchError(url, e)
157 else:
158 logger.debug(1, "Headers:\n%s" % (e.headers))
159 raise LayerIndexFetchError(url, e)
160 except OSError as e:
161 error = 0
162 reason = ""
163
164 # Process base OSError first...
165 if hasattr(e, 'errno'):
166 error = e.errno
167 reason = e.strerror
168
169 # Process gaierror (socket error) subclass if available.
170 if hasattr(e, 'reason') and hasattr(e.reason, 'errno') and hasattr(e.reason, 'strerror'):
171 error = e.reason.errno
172 reason = e.reason.strerror
173 if error == -2:
174 raise LayerIndexFetchError(url, "%s: %s" % (e, reason))
175
176 if error and error != 0:
177 raise LayerIndexFetchError(url, "Unexpected exception: [Error %s] %s" % (error, reason))
178 else:
179 raise LayerIndexFetchError(url, "Unable to fetch OSError exception: %s" % e)
180
181 finally:
182 logger.debug(1, "...fetching %s (%s), done." % (url, ["without authentication", "with authentication"][bool(username)]))
183
184 return res
185
186
187 def load_layerindex(self, indexURI, load=['layerDependencies', 'recipes', 'machines', 'distros'], reload=False):
188 '''Load the layerindex.
189
190 indexURI - An index to load. (Use multiple calls to load multiple indexes)
191
192 reload - If reload is True, then any previously loaded indexes will be forgotten.
193
194 load - List of elements to load. Default loads all items.
195 Note: plugs may ignore this.
196
197The format of the indexURI:
198
199 <url>;branch=<branch>;cache=<cache>;desc=<description>
200
201 Note: the 'branch' parameter if set can select multiple branches by using
202 comma, such as 'branch=master,morty,pyro'. However, many operations only look
203 at the -first- branch specified!
204
205 The cache value may be undefined, in this case a network failure will
206 result in an error, otherwise the system will look for a file of the cache
207 name and load that instead.
208
209 For example:
210
211 http://layers.openembedded.org/layerindex/api/;branch=master;desc=OpenEmbedded%20Layer%20Index
212 cooker://
213'''
214 if reload:
215 self.indexes = []
216
217 logger.debug(1, 'Loading: %s' % indexURI)
218
219 if not self.plugins:
220 raise LayerIndexException("No LayerIndex Plugins available")
221
222 for plugin in self.plugins:
223 # Check if the plugin was initialized
224 logger.debug(1, 'Trying %s' % plugin.__class__)
225 if not hasattr(plugin, 'type') or not plugin.type:
226 continue
227 try:
228 # TODO: Implement 'cache', for when the network is not available
229 indexEnt = plugin.load_index(indexURI, load)
230 break
231 except LayerIndexPluginUrlError as e:
232 logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url))
233 except NotImplementedError:
234 pass
235 else:
236 logger.debug(1, "No plugins support %s" % indexURI)
237 raise LayerIndexException("No plugins support %s" % indexURI)
238
239 # Mark CONFIG data as something we've added...
240 indexEnt.config['local'] = []
241 indexEnt.config['local'].append('config')
242
243 # No longer permit changes..
244 indexEnt.lockData()
245
246 self.indexes.append(indexEnt)
247
248 def store_layerindex(self, indexURI, index=None):
249 '''Store one layerindex
250
251Typically this will be used to create a local cache file of a remote index.
252
253 file://<path>;branch=<branch>
254
255We can write out in either the restapi or django formats. The split option
256will write out the individual elements split by layer and related components.
257'''
258 if not index:
259 logger.warning('No index to write, nothing to do.')
260 return
261
262 if not self.plugins:
263 raise LayerIndexException("No LayerIndex Plugins available")
264
265 for plugin in self.plugins:
266 # Check if the plugin was initialized
267 logger.debug(1, 'Trying %s' % plugin.__class__)
268 if not hasattr(plugin, 'type') or not plugin.type:
269 continue
270 try:
271 plugin.store_index(indexURI, index)
272 break
273 except LayerIndexPluginUrlError as e:
274 logger.debug(1, "%s doesn't support %s" % (plugin.type, e.url))
275 except NotImplementedError:
276 logger.debug(1, "Store not implemented in %s" % plugin.type)
277 pass
278 else:
279 logger.debug(1, "No plugins support %s" % url)
280 raise LayerIndexException("No plugins support %s" % url)
281
282
283 def is_empty(self):
284 '''Return True or False if the index has any usable data.
285
286We check the indexes entries to see if they have a branch set, as well as
287layerBranches set. If not, they are effectively blank.'''
288
289 found = False
290 for index in self.indexes:
291 if index.__bool__():
292 found = True
293 break
294 return not found
295
296
297 def find_vcs_url(self, vcs_url, branch=None):
298 '''Return the first layerBranch with the given vcs_url
299
300 If a branch has not been specified, we will iterate over the branches in
301 the default configuration until the first vcs_url/branch match.'''
302
303 for index in self.indexes:
304 logger.debug(1, ' searching %s' % index.config['DESCRIPTION'])
305 layerBranch = index.find_vcs_url(vcs_url, [branch])
306 if layerBranch:
307 return layerBranch
308 return None
309
310 def find_collection(self, collection, version=None, branch=None):
311 '''Return the first layerBranch with the given collection name
312
313 If a branch has not been specified, we will iterate over the branches in
314 the default configuration until the first collection/branch match.'''
315
316 logger.debug(1, 'find_collection: %s (%s) %s' % (collection, version, branch))
317
318 if branch:
319 branches = [branch]
320 else:
321 branches = None
322
323 for index in self.indexes:
324 logger.debug(1, ' searching %s' % index.config['DESCRIPTION'])
325 layerBranch = index.find_collection(collection, version, branches)
326 if layerBranch:
327 return layerBranch
328 else:
329 logger.debug(1, 'Collection %s (%s) not found for branch (%s)' % (collection, version, branch))
330 return None
331
332 def find_layerbranch(self, name, branch=None):
333 '''Return the layerBranch item for a given name and branch
334
335 If a branch has not been specified, we will iterate over the branches in
336 the default configuration until the first name/branch match.'''
337
338 if branch:
339 branches = [branch]
340 else:
341 branches = None
342
343 for index in self.indexes:
344 layerBranch = index.find_layerbranch(name, branches)
345 if layerBranch:
346 return layerBranch
347 return None
348
349 def find_dependencies(self, names=None, layerbranches=None, ignores=None):
350 '''Return a tuple of all dependencies and valid items for the list of (layer) names
351
352 The dependency scanning happens depth-first. The returned
353 dependencies should be in the best order to define bblayers.
354
355 names - list of layer names (searching layerItems)
356 branches - when specified (with names) only this list of branches are evaluated
357
358 layerbranches - list of layerbranches to resolve dependencies
359
360 ignores - list of layer names to ignore
361
362 return: (dependencies, invalid)
363
364 dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ]
365 invalid = [ LayerItem.name1, LayerItem.name2, ... ]
366 '''
367
368 invalid = []
369
370 # Convert name/branch to layerbranches
371 if layerbranches is None:
372 layerbranches = []
373
374 for name in names:
375 if ignores and name in ignores:
376 continue
377
378 for index in self.indexes:
379 layerbranch = index.find_layerbranch(name)
380 if not layerbranch:
381 # Not in this index, hopefully it's in another...
382 continue
383 layerbranches.append(layerbranch)
384 break
385 else:
386 invalid.append(name)
387
388
389 def _resolve_dependencies(layerbranches, ignores, dependencies, invalid):
390 for layerbranch in layerbranches:
391 if ignores and layerbranch.layer.name in ignores:
392 continue
393
394 # Get a list of dependencies and then recursively process them
395 for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerbranch.id]:
396 deplayerbranch = layerdependency.dependency_layerBranch
397
398 if ignores and deplayerbranch.layer.name in ignores:
399 continue
400
401 # This little block is why we can't re-use the LayerIndexObj version,
402 # we must be able to satisfy each dependencies across layer indexes and
403 # use the layer index order for priority. (r stands for replacement below)
404
405 # If this is the primary index, we can fast path and skip this
406 if deplayerbranch.index != self.indexes[0]:
407 # Is there an entry in a prior index for this collection/version?
408 rdeplayerbranch = self.find_collection(
409 collection=deplayerbranch.collection,
410 version=deplayerbranch.version
411 )
412 if rdeplayerbranch != deplayerbranch:
413 logger.debug(1, 'Replaced %s:%s:%s with %s:%s:%s' % \
414 (deplayerbranch.index.config['DESCRIPTION'],
415 deplayerbranch.branch.name,
416 deplayerbranch.layer.name,
417 rdeplayerbranch.index.config['DESCRIPTION'],
418 rdeplayerbranch.branch.name,
419 rdeplayerbranch.layer.name))
420 deplayerbranch = rdeplayerbranch
421
422 # New dependency, we need to resolve it now... depth-first
423 if deplayerbranch.layer.name not in dependencies:
424 (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid)
425
426 if deplayerbranch.layer.name not in dependencies:
427 dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency]
428 else:
429 if layerdependency not in dependencies[deplayerbranch.layer.name]:
430 dependencies[deplayerbranch.layer.name].append(layerdependency)
431
432 return (dependencies, invalid)
433
434 # OK, resolve this one...
435 dependencies = OrderedDict()
436 (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid)
437
438 for layerbranch in layerbranches:
439 if layerbranch.layer.name not in dependencies:
440 dependencies[layerbranch.layer.name] = [layerbranch]
441
442 return (dependencies, invalid)
443
444
445 def list_obj(self, object):
446 '''Print via the plain logger object information
447
448This function is used to implement debugging and provide the user info.
449'''
450 for lix in self.indexes:
451 if object not in lix:
452 continue
453
454 logger.plain ('')
455 logger.plain ('Index: %s' % lix.config['DESCRIPTION'])
456
457 output = []
458
459 if object == 'branches':
460 logger.plain ('%s %s %s' % ('{:26}'.format('branch'), '{:34}'.format('description'), '{:22}'.format('bitbake branch')))
461 logger.plain ('{:-^80}'.format(""))
462 for branchid in lix.branches:
463 output.append('%s %s %s' % (
464 '{:26}'.format(lix.branches[branchid].name),
465 '{:34}'.format(lix.branches[branchid].short_description),
466 '{:22}'.format(lix.branches[branchid].bitbake_branch)
467 ))
468 for line in sorted(output):
469 logger.plain (line)
470
471 continue
472
473 if object == 'layerItems':
474 logger.plain ('%s %s' % ('{:26}'.format('layer'), '{:34}'.format('description')))
475 logger.plain ('{:-^80}'.format(""))
476 for layerid in lix.layerItems:
477 output.append('%s %s' % (
478 '{:26}'.format(lix.layerItems[layerid].name),
479 '{:34}'.format(lix.layerItems[layerid].summary)
480 ))
481 for line in sorted(output):
482 logger.plain (line)
483
484 continue
485
486 if object == 'layerBranches':
487 logger.plain ('%s %s %s' % ('{:26}'.format('layer'), '{:34}'.format('description'), '{:19}'.format('collection:version')))
488 logger.plain ('{:-^80}'.format(""))
489 for layerbranchid in lix.layerBranches:
490 output.append('%s %s %s' % (
491 '{:26}'.format(lix.layerBranches[layerbranchid].layer.name),
492 '{:34}'.format(lix.layerBranches[layerbranchid].layer.summary),
493 '{:19}'.format("%s:%s" %
494 (lix.layerBranches[layerbranchid].collection,
495 lix.layerBranches[layerbranchid].version)
496 )
497 ))
498 for line in sorted(output):
499 logger.plain (line)
500
501 continue
502
503 if object == 'layerDependencies':
504 logger.plain ('%s %s %s %s' % ('{:19}'.format('branch'), '{:26}'.format('layer'), '{:11}'.format('dependency'), '{:26}'.format('layer')))
505 logger.plain ('{:-^80}'.format(""))
506 for layerDependency in lix.layerDependencies:
507 if not lix.layerDependencies[layerDependency].dependency_layerBranch:
508 continue
509
510 output.append('%s %s %s %s' % (
511 '{:19}'.format(lix.layerDependencies[layerDependency].layerbranch.branch.name),
512 '{:26}'.format(lix.layerDependencies[layerDependency].layerbranch.layer.name),
513 '{:11}'.format('requires' if lix.layerDependencies[layerDependency].required else 'recommends'),
514 '{:26}'.format(lix.layerDependencies[layerDependency].dependency_layerBranch.layer.name)
515 ))
516 for line in sorted(output):
517 logger.plain (line)
518
519 continue
520
521 if object == 'recipes':
522 logger.plain ('%s %s %s' % ('{:20}'.format('recipe'), '{:10}'.format('version'), 'layer'))
523 logger.plain ('{:-^80}'.format(""))
524 output = []
525 for recipe in lix.recipes:
526 output.append('%s %s %s' % (
527 '{:30}'.format(lix.recipes[recipe].pn),
528 '{:30}'.format(lix.recipes[recipe].pv),
529 lix.recipes[recipe].layer.name
530 ))
531 for line in sorted(output):
532 logger.plain (line)
533
534 continue
535
536 if object == 'machines':
537 logger.plain ('%s %s %s' % ('{:24}'.format('machine'), '{:34}'.format('description'), '{:19}'.format('layer')))
538 logger.plain ('{:-^80}'.format(""))
539 for machine in lix.machines:
540 output.append('%s %s %s' % (
541 '{:24}'.format(lix.machines[machine].name),
542 '{:34}'.format(lix.machines[machine].description)[:34],
543 '{:19}'.format(lix.machines[machine].layerbranch.layer.name)
544 ))
545 for line in sorted(output):
546 logger.plain (line)
547
548 continue
549
550 if object == 'distros':
551 logger.plain ('%s %s %s' % ('{:24}'.format('distro'), '{:34}'.format('description'), '{:19}'.format('layer')))
552 logger.plain ('{:-^80}'.format(""))
553 for distro in lix.distros:
554 output.append('%s %s %s' % (
555 '{:24}'.format(lix.distros[distro].name),
556 '{:34}'.format(lix.distros[distro].description)[:34],
557 '{:19}'.format(lix.distros[distro].layerbranch.layer.name)
558 ))
559 for line in sorted(output):
560 logger.plain (line)
561
562 continue
563
564 logger.plain ('')
565
566
567# This class holds a single layer index instance
568# The LayerIndexObj is made up of dictionary of elements, such as:
569# index['config'] - configuration data for this index
570# index['branches'] - dictionary of Branch objects, by id number
571# index['layerItems'] - dictionary of layerItem objects, by id number
572# ...etc... (See: http://layers.openembedded.org/layerindex/api/)
573#
574# The class needs to manage the 'index' entries and allow easily adding
575# of new items, as well as simply loading of the items.
576class LayerIndexObj():
577 def __init__(self):
578 super().__setattr__('_index', {})
579 super().__setattr__('_lock', False)
580
581 def __bool__(self):
582 '''False if the index is effectively empty
583
584 We check the index to see if it has a branch set, as well as
585 layerbranches set. If not, it is effectively blank.'''
586
587 if not bool(self._index):
588 return False
589
590 try:
591 if self.branches and self.layerBranches:
592 return True
593 except AttributeError:
594 pass
595
596 return False
597
598 def __getattr__(self, name):
599 if name.startswith('_'):
600 return super().__getattribute__(name)
601
602 if name not in self._index:
603 raise AttributeError('%s not in index datastore' % name)
604
605 return self._index[name]
606
607 def __setattr__(self, name, value):
608 if self.isLocked():
609 raise TypeError("Can not set attribute '%s': index is locked" % name)
610
611 if name.startswith('_'):
612 super().__setattr__(name, value)
613 return
614
615 self._index[name] = value
616
617 def __delattr__(self, name):
618 if self.isLocked():
619 raise TypeError("Can not delete attribute '%s': index is locked" % name)
620
621 if name.startswith('_'):
622 super().__delattr__(name)
623
624 self._index.pop(name)
625
626 def lockData(self):
627 '''Lock data object (make it readonly)'''
628 super().__setattr__("_lock", True)
629
630 def unlockData(self):
631 '''unlock data object (make it readonly)'''
632 super().__setattr__("_lock", False)
633
634 # When the data is unlocked, we have to clear the caches, as
635 # modification is allowed!
636 del(self._layerBranches_layerId_branchId)
637 del(self._layerDependencies_layerBranchId)
638 del(self._layerBranches_vcsUrl)
639
640 def isLocked(self):
641 '''Is this object locked (readonly)?'''
642 return self._lock
643
644 def add_element(self, indexname, objs):
645 '''Add a layer index object to index.<indexname>'''
646 if indexname not in self._index:
647 self._index[indexname] = {}
648
649 for obj in objs:
650 if obj.id in self._index[indexname]:
651 if self._index[indexname][obj.id] == obj:
652 continue
653 raise LayerIndexError('Conflict adding object %s(%s) to index' % (indexname, obj.id))
654 self._index[indexname][obj.id] = obj
655
656 def add_raw_element(self, indexname, objtype, rawobjs):
657 '''Convert a raw layer index data item to a layer index item object and add to the index'''
658 objs = []
659 for entry in rawobjs:
660 objs.append(objtype(self, entry))
661 self.add_element(indexname, objs)
662
663 # Quick lookup table for searching layerId and branchID combos
664 @property
665 def layerBranches_layerId_branchId(self):
666 def createCache(self):
667 cache = {}
668 for layerbranchid in self.layerBranches:
669 layerbranch = self.layerBranches[layerbranchid]
670 cache["%s:%s" % (layerbranch.layer_id, layerbranch.branch_id)] = layerbranch
671 return cache
672
673 if self.isLocked():
674 cache = getattr(self, '_layerBranches_layerId_branchId', None)
675 else:
676 cache = None
677
678 if not cache:
679 cache = createCache(self)
680
681 if self.isLocked():
682 super().__setattr__('_layerBranches_layerId_branchId', cache)
683
684 return cache
685
686 # Quick lookup table for finding all dependencies of a layerBranch
687 @property
688 def layerDependencies_layerBranchId(self):
689 def createCache(self):
690 cache = {}
691 # This ensures empty lists for all branchids
692 for layerbranchid in self.layerBranches:
693 cache[layerbranchid] = []
694
695 for layerdependencyid in self.layerDependencies:
696 layerdependency = self.layerDependencies[layerdependencyid]
697 cache[layerdependency.layerbranch_id].append(layerdependency)
698 return cache
699
700 if self.isLocked():
701 cache = getattr(self, '_layerDependencies_layerBranchId', None)
702 else:
703 cache = None
704
705 if not cache:
706 cache = createCache(self)
707
708 if self.isLocked():
709 super().__setattr__('_layerDependencies_layerBranchId', cache)
710
711 return cache
712
713 # Quick lookup table for finding all instances of a vcs_url
714 @property
715 def layerBranches_vcsUrl(self):
716 def createCache(self):
717 cache = {}
718 for layerbranchid in self.layerBranches:
719 layerbranch = self.layerBranches[layerbranchid]
720 if layerbranch.layer.vcs_url not in cache:
721 cache[layerbranch.layer.vcs_url] = [layerbranch]
722 else:
723 cache[layerbranch.layer.vcs_url].append(layerbranch)
724 return cache
725
726 if self.isLocked():
727 cache = getattr(self, '_layerBranches_vcsUrl', None)
728 else:
729 cache = None
730
731 if not cache:
732 cache = createCache(self)
733
734 if self.isLocked():
735 super().__setattr__('_layerBranches_vcsUrl', cache)
736
737 return cache
738
739
740 def find_vcs_url(self, vcs_url, branches=None):
741 ''''Return the first layerBranch with the given vcs_url
742
743 If a list of branches has not been specified, we will iterate on
744 all branches until the first vcs_url is found.'''
745
746 if not self.__bool__():
747 return None
748
749 for layerbranch in self.layerBranches_vcsUrl:
750 if branches and layerbranch.branch.name not in branches:
751 continue
752
753 return layerbranch
754
755 return None
756
757
758 def find_collection(self, collection, version=None, branches=None):
759 '''Return the first layerBranch with the given collection name
760
761 If a list of branches has not been specified, we will iterate on
762 all branches until the first collection is found.'''
763
764 if not self.__bool__():
765 return None
766
767 for layerbranchid in self.layerBranches:
768 layerbranch = self.layerBranches[layerbranchid]
769 if branches and layerbranch.branch.name not in branches:
770 continue
771
772 if layerbranch.collection == collection and \
773 (version is None or version == layerbranch.version):
774 return layerbranch
775
776 return None
777
778
779 def find_layerbranch(self, name, branches=None):
780 '''Return the first layerbranch whose layer name matches
781
782 If a list of branches has not been specified, we will iterate on
783 all branches until the first layer with that name is found.'''
784
785 if not self.__bool__():
786 return None
787
788 for layerbranchid in self.layerBranches:
789 layerbranch = self.layerBranches[layerbranchid]
790 if branches and layerbranch.branch.name not in branches:
791 continue
792
793 if layerbranch.layer.name == name:
794 return layerbranch
795
796 return None
797
798 def find_dependencies(self, names=None, branches=None, layerBranches=None, ignores=None):
799 '''Return a tuple of all dependencies and valid items for the list of (layer) names
800
801 The dependency scanning happens depth-first. The returned
802 dependencies should be in the best order to define bblayers.
803
804 names - list of layer names (searching layerItems)
805 branches - when specified (with names) only this list of branches are evaluated
806
807 layerBranches - list of layerBranches to resolve dependencies
808
809 ignores - list of layer names to ignore
810
811 return: (dependencies, invalid)
812
813 dependencies[LayerItem.name] = [ LayerBranch, LayerDependency1, LayerDependency2, ... ]
814 invalid = [ LayerItem.name1, LayerItem.name2, ... ]'''
815
816 invalid = []
817
818 # Convert name/branch to layerBranches
819 if layerbranches is None:
820 layerbranches = []
821
822 for name in names:
823 if ignores and name in ignores:
824 continue
825
826 layerbranch = self.find_layerbranch(name, branches)
827 if not layerbranch:
828 invalid.append(name)
829 else:
830 layerbranches.append(layerbranch)
831
832 for layerbranch in layerbranches:
833 if layerbranch.index != self:
834 raise LayerIndexException("Can not resolve dependencies across indexes with this class function!")
835
836 def _resolve_dependencies(layerbranches, ignores, dependencies, invalid):
837 for layerbranch in layerbranches:
838 if ignores and layerBranch.layer.name in ignores:
839 continue
840
841 for layerdependency in layerbranch.index.layerDependencies_layerBranchId[layerBranch.id]:
842 deplayerbranch = layerDependency.dependency_layerBranch
843
844 if ignores and deplayerbranch.layer.name in ignores:
845 continue
846
847 # New dependency, we need to resolve it now... depth-first
848 if deplayerbranch.layer.name not in dependencies:
849 (dependencies, invalid) = _resolve_dependencies([deplayerbranch], ignores, dependencies, invalid)
850
851 if deplayerbranch.layer.name not in dependencies:
852 dependencies[deplayerbranch.layer.name] = [deplayerbranch, layerdependency]
853 else:
854 if layerdependency not in dependencies[deplayerbranch.layer.name]:
855 dependencies[deplayerbranch.layer.name].append(layerdependency)
856
857 return (dependencies, invalid)
858
859 # OK, resolve this one...
860 dependencies = OrderedDict()
861 (dependencies, invalid) = _resolve_dependencies(layerbranches, ignores, dependencies, invalid)
862
863 # Is this item already in the list, if not add it
864 for layerbranch in layerbranches:
865 if layerbranch.layer.name not in dependencies:
866 dependencies[layerbranch.layer.name] = [layerbranch]
867
868 return (dependencies, invalid)
869
870
871# Define a basic LayerIndexItemObj. This object forms the basis for all other
872# objects. The raw Layer Index data is stored in the _data element, but we
873# do not want users to access data directly. So wrap this and protect it
874# from direct manipulation.
875#
876# It is up to the insantiators of the objects to fill them out, and once done
877# lock the objects to prevent further accidently manipulation.
878#
879# Using the getattr, setattr and properties we can access and manipulate
880# the data within the data element.
881class LayerIndexItemObj():
882 def __init__(self, index, data=None, lock=False):
883 if data is None:
884 data = {}
885
886 if type(data) != type(dict()):
887 raise TypeError('data (%s) is not a dict' % type(data))
888
889 super().__setattr__('_lock', lock)
890 super().__setattr__('index', index)
891 super().__setattr__('_data', data)
892
893 def __eq__(self, other):
894 if self.__class__ != other.__class__:
895 return False
896 res=(self._data == other._data)
897 return res
898
899 def __bool__(self):
900 return bool(self._data)
901
902 def __getattr__(self, name):
903 # These are internal to THIS class, and not part of data
904 if name == "index" or name.startswith('_'):
905 return super().__getattribute__(name)
906
907 if name not in self._data:
908 raise AttributeError('%s not in datastore' % name)
909
910 return self._data[name]
911
912 def _setattr(self, name, value, prop=True):
913 '''__setattr__ like function, but with control over property object behavior'''
914 if self.isLocked():
915 raise TypeError("Can not set attribute '%s': Object data is locked" % name)
916
917 if name.startswith('_'):
918 super().__setattr__(name, value)
919 return
920
921 # Since __setattr__ runs before properties, we need to check if
922 # there is a setter property and then execute it
923 # ... or return self._data[name]
924 propertyobj = getattr(self.__class__, name, None)
925 if prop and isinstance(propertyobj, property):
926 if propertyobj.fset:
927 propertyobj.fset(self, value)
928 else:
929 raise AttributeError('Attribute %s is readonly, and may not be set' % name)
930 else:
931 self._data[name] = value
932
933 def __setattr__(self, name, value):
934 self._setattr(name, value, prop=True)
935
936 def _delattr(self, name, prop=True):
937 # Since __delattr__ runs before properties, we need to check if
938 # there is a deleter property and then execute it
939 # ... or we pop it ourselves..
940 propertyobj = getattr(self.__class__, name, None)
941 if prop and isinstance(propertyobj, property):
942 if propertyobj.fdel:
943 propertyobj.fdel(self)
944 else:
945 raise AttributeError('Attribute %s is readonly, and may not be deleted' % name)
946 else:
947 self._data.pop(name)
948
949 def __delattr__(self, name):
950 self._delattr(name, prop=True)
951
952 def lockData(self):
953 '''Lock data object (make it readonly)'''
954 super().__setattr__("_lock", True)
955
956 def unlockData(self):
957 '''unlock data object (make it readonly)'''
958 super().__setattr__("_lock", False)
959
960 def isLocked(self):
961 '''Is this object locked (readonly)?'''
962 return self._lock
963
964# Branch object
965class Branch(LayerIndexItemObj):
966 def define_data(self, id, name, bitbake_branch,
967 short_description=None, sort_priority=1,
968 updates_enabled=True, updated=None,
969 update_environment=None):
970 self.id = id
971 self.name = name
972 self.bitbake_branch = bitbake_branch
973 self.short_description = short_description or name
974 self.sort_priority = sort_priority
975 self.updates_enabled = updates_enabled
976 self.updated = updated or datetime.datetime.today().isoformat()
977 self.update_environment = update_environment
978
979 @property
980 def name(self):
981 return self.__getattr__('name')
982
983 @name.setter
984 def name(self, value):
985 self._data['name'] = value
986
987 if self.bitbake_branch == value:
988 self.bitbake_branch = ""
989
990 @name.deleter
991 def name(self):
992 self._delattr('name', prop=False)
993
994 @property
995 def bitbake_branch(self):
996 try:
997 return self.__getattr__('bitbake_branch')
998 except AttributeError:
999 return self.name
1000
1001 @bitbake_branch.setter
1002 def bitbake_branch(self, value):
1003 if self.name == value:
1004 self._data['bitbake_branch'] = ""
1005 else:
1006 self._data['bitbake_branch'] = value
1007
1008 @bitbake_branch.deleter
1009 def bitbake_branch(self):
1010 self._delattr('bitbake_branch', prop=False)
1011
1012
1013class LayerItem(LayerIndexItemObj):
1014 def define_data(self, id, name, status='P',
1015 layer_type='A', summary=None,
1016 description=None,
1017 vcs_url=None, vcs_web_url=None,
1018 vcs_web_tree_base_url=None,
1019 vcs_web_file_base_url=None,
1020 usage_url=None,
1021 mailing_list_url=None,
1022 index_preference=1,
1023 classic=False,
1024 updated=None):
1025 self.id = id
1026 self.name = name
1027 self.status = status
1028 self.layer_type = layer_type
1029 self.summary = summary or name
1030 self.description = description or summary or name
1031 self.vcs_url = vcs_url
1032 self.vcs_web_url = vcs_web_url
1033 self.vcs_web_tree_base_url = vcs_web_tree_base_url
1034 self.vcs_web_file_base_url = vcs_web_file_base_url
1035 self.index_preference = index_preference
1036 self.classic = classic
1037 self.updated = updated or datetime.datetime.today().isoformat()
1038
1039
1040class LayerBranch(LayerIndexItemObj):
1041 def define_data(self, id, collection, version, layer, branch,
1042 vcs_subdir="", vcs_last_fetch=None,
1043 vcs_last_rev=None, vcs_last_commit=None,
1044 actual_branch="",
1045 updated=None):
1046 self.id = id
1047 self.collection = collection
1048 self.version = version
1049 if type(layer) != type(LayerItem):
1050 self.layer_id = layer
1051 else:
1052 self.layer = layer
1053
1054 if type(branch) != type(Branch):
1055 self.branch_id = branch
1056 else:
1057 self.branch = branch
1058
1059 self.vcs_subdir = vcs_subdir
1060 self.vcs_last_fetch = vcs_last_fetch
1061 self.vcs_last_rev = vcs_last_rev
1062 self.vcs_last_commit = vcs_last_commit
1063 self.actual_branch = actual_branch
1064 self.updated = updated or datetime.datetime.today().isoformat()
1065
1066 # This is a little odd, the _data attribute is 'layer', but it's really
1067 # referring to the layer id.. so lets adjust this to make it useful
1068 @property
1069 def layer_id(self):
1070 return self.__getattr__('layer')
1071
1072 @layer_id.setter
1073 def layer_id(self, value):
1074 self._setattr('layer', value, prop=False)
1075
1076 @layer_id.deleter
1077 def layer_id(self):
1078 self._delattr('layer', prop=False)
1079
1080 @property
1081 def layer(self):
1082 try:
1083 return self.index.layerItems[self.layer_id]
1084 except KeyError:
1085 raise AttributeError('Unable to find layerItems in index to map layer_id %s' % self.layer_id)
1086 except IndexError:
1087 raise AttributeError('Unable to find layer_id %s in index layerItems' % self.layer_id)
1088
1089 @layer.setter
1090 def layer(self, value):
1091 if type(value) != type(LayerItem):
1092 raise TypeError('value is not a LayerItem')
1093 if self.index != value.index:
1094 raise AttributeError('Object and value do not share the same index and thus key set.')
1095 self.layer_id = value.id
1096
1097 @layer.deleter
1098 def layer(self):
1099 del self.layer_id
1100
1101 @property
1102 def branch_id(self):
1103 return self.__getattr__('branch')
1104
1105 @branch_id.setter
1106 def branch_id(self, value):
1107 self._setattr('branch', value, prop=False)
1108
1109 @branch_id.deleter
1110 def branch_id(self):
1111 self._delattr('branch', prop=False)
1112
1113 @property
1114 def branch(self):
1115 try:
1116 logger.debug(1, "Get branch object from branches[%s]" % (self.branch_id))
1117 return self.index.branches[self.branch_id]
1118 except KeyError:
1119 raise AttributeError('Unable to find branches in index to map branch_id %s' % self.branch_id)
1120 except IndexError:
1121 raise AttributeError('Unable to find branch_id %s in index branches' % self.branch_id)
1122
1123 @branch.setter
1124 def branch(self, value):
1125 if type(value) != type(LayerItem):
1126 raise TypeError('value is not a LayerItem')
1127 if self.index != value.index:
1128 raise AttributeError('Object and value do not share the same index and thus key set.')
1129 self.branch_id = value.id
1130
1131 @branch.deleter
1132 def branch(self):
1133 del self.branch_id
1134
1135 @property
1136 def actual_branch(self):
1137 if self.__getattr__('actual_branch'):
1138 return self.__getattr__('actual_branch')
1139 else:
1140 return self.branch.name
1141
1142 @actual_branch.setter
1143 def actual_branch(self, value):
1144 logger.debug(1, "Set actual_branch to %s .. name is %s" % (value, self.branch.name))
1145 if value != self.branch.name:
1146 self._setattr('actual_branch', value, prop=False)
1147 else:
1148 self._setattr('actual_branch', '', prop=False)
1149
1150 @actual_branch.deleter
1151 def actual_branch(self):
1152 self._delattr('actual_branch', prop=False)
1153
1154# Extend LayerIndexItemObj with common LayerBranch manipulations
1155# All of the remaining LayerIndex objects refer to layerbranch, and it is
1156# up to the user to follow that back through the LayerBranch object into
1157# the layer object to get various attributes. So add an intermediate set
1158# of attributes that can easily get us the layerbranch as well as layer.
1159
1160class LayerIndexItemObj_LayerBranch(LayerIndexItemObj):
1161 @property
1162 def layerbranch_id(self):
1163 return self.__getattr__('layerbranch')
1164
1165 @layerbranch_id.setter
1166 def layerbranch_id(self, value):
1167 self._setattr('layerbranch', value, prop=False)
1168
1169 @layerbranch_id.deleter
1170 def layerbranch_id(self):
1171 self._delattr('layerbranch', prop=False)
1172
1173 @property
1174 def layerbranch(self):
1175 try:
1176 return self.index.layerBranches[self.layerbranch_id]
1177 except KeyError:
1178 raise AttributeError('Unable to find layerBranches in index to map layerbranch_id %s' % self.layerbranch_id)
1179 except IndexError:
1180 raise AttributeError('Unable to find layerbranch_id %s in index branches' % self.layerbranch_id)
1181
1182 @layerbranch.setter
1183 def layerbranch(self, value):
1184 if type(value) != type(LayerBranch):
1185 raise TypeError('value (%s) is not a layerBranch' % type(value))
1186 if self.index != value.index:
1187 raise AttributeError('Object and value do not share the same index and thus key set.')
1188 self.layerbranch_id = value.id
1189
1190 @layerbranch.deleter
1191 def layerbranch(self):
1192 del self.layerbranch_id
1193
1194 @property
1195 def layer_id(self):
1196 return self.layerbranch.layer_id
1197
1198 # Doesn't make sense to set or delete layer_id
1199
1200 @property
1201 def layer(self):
1202 return self.layerbranch.layer
1203
1204 # Doesn't make sense to set or delete layer
1205
1206
1207class LayerDependency(LayerIndexItemObj_LayerBranch):
1208 def define_data(self, id, layerbranch, dependency, required=True):
1209 self.id = id
1210 if type(layerbranch) != type(LayerBranch):
1211 self.layerbranch_id = layerbranch
1212 else:
1213 self.layerbranch = layerbranch
1214 if type(dependency) != type(LayerDependency):
1215 self.dependency_id = dependency
1216 else:
1217 self.dependency = dependency
1218 self.required = required
1219
1220 @property
1221 def dependency_id(self):
1222 return self.__getattr__('dependency')
1223
1224 @dependency_id.setter
1225 def dependency_id(self, value):
1226 self._setattr('dependency', value, prop=False)
1227
1228 @dependency_id.deleter
1229 def dependency_id(self):
1230 self._delattr('dependency', prop=False)
1231
1232 @property
1233 def dependency(self):
1234 try:
1235 return self.index.layerItems[self.dependency_id]
1236 except KeyError:
1237 raise AttributeError('Unable to find layerItems in index to map layerbranch_id %s' % self.dependency_id)
1238 except IndexError:
1239 raise AttributeError('Unable to find dependency_id %s in index layerItems' % self.dependency_id)
1240
1241 @dependency.setter
1242 def dependency(self, value):
1243 if type(value) != type(LayerDependency):
1244 raise TypeError('value (%s) is not a dependency' % type(value))
1245 if self.index != value.index:
1246 raise AttributeError('Object and value do not share the same index and thus key set.')
1247 self.dependency_id = value.id
1248
1249 @dependency.deleter
1250 def dependency(self):
1251 self._delattr('dependency', prop=False)
1252
1253 @property
1254 def dependency_layerBranch(self):
1255 layerid = self.dependency_id
1256 branchid = self.layerbranch.branch_id
1257
1258 try:
1259 return self.index.layerBranches_layerId_branchId["%s:%s" % (layerid, branchid)]
1260 except IndexError:
1261 # layerBranches_layerId_branchId -- but not layerId:branchId
1262 raise AttributeError('Unable to find layerId:branchId %s:%s in index layerBranches_layerId_branchId' % (layerid, branchid))
1263 except KeyError:
1264 raise AttributeError('Unable to find layerId:branchId %s:%s in layerItems and layerBranches' % (layerid, branchid))
1265
1266 # dependency_layerBranch doesn't make sense to set or del
1267
1268
1269class Recipe(LayerIndexItemObj_LayerBranch):
1270 def define_data(self, id,
1271 filename, filepath, pn, pv, layerbranch,
1272 summary="", description="", section="", license="",
1273 homepage="", bugtracker="", provides="", bbclassextend="",
1274 inherits="", blacklisted="", updated=None):
1275 self.id = id
1276 self.filename = filename
1277 self.filepath = filepath
1278 self.pn = pn
1279 self.pv = pv
1280 self.summary = summary
1281 self.description = description
1282 self.section = section
1283 self.license = license
1284 self.homepage = homepage
1285 self.bugtracker = bugtracker
1286 self.provides = provides
1287 self.bbclassextend = bbclassextend
1288 self.inherits = inherits
1289 self.updated = updated or datetime.datetime.today().isoformat()
1290 self.blacklisted = blacklisted
1291 if type(layerbranch) != type(LayerBranch):
1292 self.layerbranch_id = layerbranch
1293 else:
1294 self.layerbranch = layerbranch
1295
1296 @property
1297 def fullpath(self):
1298 return os.path.join(self.filepath, self.filename)
1299
1300 # Set would need to understand how to split it
1301 # del would we del both parts?
1302
1303 @property
1304 def inherits(self):
1305 if 'inherits' not in self._data:
1306 # Older indexes may not have this, so emulate it
1307 if '-image-' in self.pn:
1308 return 'image'
1309 return self.__getattr__('inherits')
1310
1311 @inherits.setter
1312 def inherits(self, value):
1313 return self._setattr('inherits', value, prop=False)
1314
1315 @inherits.deleter
1316 def inherits(self):
1317 return self._delattr('inherits', prop=False)
1318
1319
1320class Machine(LayerIndexItemObj_LayerBranch):
1321 def define_data(self, id,
1322 name, description, layerbranch,
1323 updated=None):
1324 self.id = id
1325 self.name = name
1326 self.description = description
1327 if type(layerbranch) != type(LayerBranch):
1328 self.layerbranch_id = layerbranch
1329 else:
1330 self.layerbranch = layerbranch
1331 self.updated = updated or datetime.datetime.today().isoformat()
1332
1333class Distro(LayerIndexItemObj_LayerBranch):
1334 def define_data(self, id,
1335 name, description, layerbranch,
1336 updated=None):
1337 self.id = id
1338 self.name = name
1339 self.description = description
1340 if type(layerbranch) != type(LayerBranch):
1341 self.layerbranch_id = layerbranch
1342 else:
1343 self.layerbranch = layerbranch
1344 self.updated = updated or datetime.datetime.today().isoformat()
1345
1346
1347# When performing certain actions, we may need to sort the data.
1348# This will allow us to keep it consistent from run to run.
1349def sort_entry(item):
1350 newitem = item
1351 try:
1352 if type(newitem) == type(dict()):
1353 newitem = OrderedDict(sorted(newitem.items(), key=lambda t: t[0]))
1354 for index in newitem:
1355 newitem[index] = sort_entry(newitem[index])
1356 elif type(newitem) == type(list()):
1357 newitem.sort(key=lambda obj: obj['id'])
1358 for index, _ in enumerate(newitem):
1359 newitem[index] = sort_entry(newitem[index])
1360 except:
1361 logger.error('Sort failed for item %s' % type(item))
1362 pass
1363
1364 return newitem
diff --git a/bitbake/lib/layerindexlib/cooker.py b/bitbake/lib/layerindexlib/cooker.py
new file mode 100644
index 0000000000..248a597754
--- /dev/null
+++ b/bitbake/lib/layerindexlib/cooker.py
@@ -0,0 +1,341 @@
1# Copyright (C) 2016-2018 Wind River Systems, Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License version 2 as
5# published by the Free Software Foundation.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
10# See the GNU General Public License for more details.
11#
12# You should have received a copy of the GNU General Public License
13# along with this program; if not, write to the Free Software
14# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
16import logging
17import json
18
19from collections import OrderedDict, defaultdict
20
21from urllib.parse import unquote, urlparse
22
23import layerindexlib
24
25import layerindexlib.plugin
26
27logger = logging.getLogger('BitBake.layerindexlib.cooker')
28
29import bb.utils
30
31def plugin_init(plugins):
32 return CookerPlugin()
33
34class CookerPlugin(layerindexlib.plugin.IndexPlugin):
35 def __init__(self):
36 self.type = "cooker"
37
38 self.server_connection = None
39 self.ui_module = None
40 self.server = None
41
42 def _run_command(self, command, path, default=None):
43 try:
44 result, _ = bb.process.run(command, cwd=path)
45 result = result.strip()
46 except bb.process.ExecutionError:
47 result = default
48 return result
49
50 def _handle_git_remote(self, remote):
51 if "://" not in remote:
52 if ':' in remote:
53 # This is assumed to be ssh
54 remote = "ssh://" + remote
55 else:
56 # This is assumed to be a file path
57 remote = "file://" + remote
58 return remote
59
60 def _get_bitbake_info(self):
61 """Return a tuple of bitbake information"""
62
63 # Our path SHOULD be .../bitbake/lib/layerindex/cooker.py
64 bb_path = os.path.dirname(__file__) # .../bitbake/lib/layerindex/cooker.py
65 bb_path = os.path.dirname(bb_path) # .../bitbake/lib/layerindex
66 bb_path = os.path.dirname(bb_path) # .../bitbake/lib
67 bb_path = os.path.dirname(bb_path) # .../bitbake
68 bb_path = self._run_command('git rev-parse --show-toplevel', os.path.dirname(__file__), default=bb_path)
69 bb_branch = self._run_command('git rev-parse --abbrev-ref HEAD', bb_path, default="<unknown>")
70 bb_rev = self._run_command('git rev-parse HEAD', bb_path, default="<unknown>")
71 for remotes in self._run_command('git remote -v', bb_path, default="").split("\n"):
72 remote = remotes.split("\t")[1].split(" ")[0]
73 if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
74 bb_remote = self._handle_git_remote(remote)
75 break
76 else:
77 bb_remote = self._handle_git_remote(bb_path)
78
79 return (bb_remote, bb_branch, bb_rev, bb_path)
80
81 def _load_bblayers(self, branches=None):
82 """Load the BBLAYERS and related collection information"""
83
84 d = self.layerindex.data
85
86 if not branches:
87 raise LayerIndexFetchError("No branches specified for _load_bblayers!")
88
89 index = layerindexlib.LayerIndexObj()
90
91 branchId = 0
92 index.branches = {}
93
94 layerItemId = 0
95 index.layerItems = {}
96
97 layerBranchId = 0
98 index.layerBranches = {}
99
100 bblayers = d.getVar('BBLAYERS').split()
101
102 if not bblayers:
103 # It's blank! Nothing to process...
104 return index
105
106 collections = d.getVar('BBFILE_COLLECTIONS')
107 layerconfs = d.varhistory.get_variable_items_files('BBFILE_COLLECTIONS', d)
108 bbfile_collections = {layer: os.path.dirname(os.path.dirname(path)) for layer, path in layerconfs.items()}
109
110 (_, bb_branch, _, _) = self._get_bitbake_info()
111
112 for branch in branches:
113 branchId += 1
114 index.branches[branchId] = layerindexlib.Branch(index, None)
115 index.branches[branchId].define_data(branchId, branch, bb_branch)
116
117 for entry in collections.split():
118 layerpath = entry
119 if entry in bbfile_collections:
120 layerpath = bbfile_collections[entry]
121
122 layername = d.getVar('BBLAYERS_LAYERINDEX_NAME_%s' % entry) or os.path.basename(layerpath)
123 layerversion = d.getVar('LAYERVERSION_%s' % entry) or ""
124 layerurl = self._handle_git_remote(layerpath)
125
126 layersubdir = ""
127 layerrev = "<unknown>"
128 layerbranch = "<unknown>"
129
130 if os.path.isdir(layerpath):
131 layerbasepath = self._run_command('git rev-parse --show-toplevel', layerpath, default=layerpath)
132 if os.path.abspath(layerpath) != os.path.abspath(layerbasepath):
133 layersubdir = os.path.abspath(layerpath)[len(layerbasepath) + 1:]
134
135 layerbranch = self._run_command('git rev-parse --abbrev-ref HEAD', layerpath, default="<unknown>")
136 layerrev = self._run_command('git rev-parse HEAD', layerpath, default="<unknown>")
137
138 for remotes in self._run_command('git remote -v', layerpath, default="").split("\n"):
139 remote = remotes.split("\t")[1].split(" ")[0]
140 if "(fetch)" == remotes.split("\t")[1].split(" ")[1]:
141 layerurl = self._handle_git_remote(remote)
142 break
143
144 layerItemId += 1
145 index.layerItems[layerItemId] = layerindexlib.LayerItem(index, None)
146 index.layerItems[layerItemId].define_data(layerItemId, layername, description=layerpath, vcs_url=layerurl)
147
148 for branchId in index.branches:
149 layerBranchId += 1
150 index.layerBranches[layerBranchId] = layerindexlib.LayerBranch(index, None)
151 index.layerBranches[layerBranchId].define_data(layerBranchId, entry, layerversion, layerItemId, branchId,
152 vcs_subdir=layersubdir, vcs_last_rev=layerrev, actual_branch=layerbranch)
153
154 return index
155
156
157 def load_index(self, url, load):
158 """
159 Fetches layer information from a build configuration.
160
161 The return value is a dictionary containing API,
162 layer, branch, dependency, recipe, machine, distro, information.
163
164 url type should be 'cooker'.
165 url path is ignored
166 """
167
168 up = urlparse(url)
169
170 if up.scheme != 'cooker':
171 raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
172
173 d = self.layerindex.data
174
175 params = self.layerindex._parse_params(up.params)
176
177 # Only reason to pass a branch is to emulate them...
178 if 'branch' in params:
179 branches = params['branch'].split(',')
180 else:
181 branches = ['HEAD']
182
183 logger.debug(1, "Loading cooker data branches %s" % branches)
184
185 index = self._load_bblayers(branches=branches)
186
187 index.config = {}
188 index.config['TYPE'] = self.type
189 index.config['URL'] = url
190
191 if 'desc' in params:
192 index.config['DESCRIPTION'] = unquote(params['desc'])
193 else:
194 index.config['DESCRIPTION'] = 'local'
195
196 if 'cache' in params:
197 index.config['CACHE'] = params['cache']
198
199 index.config['BRANCH'] = branches
200
201 # ("layerDependencies", layerindexlib.LayerDependency)
202 layerDependencyId = 0
203 if "layerDependencies" in load:
204 index.layerDependencies = {}
205 for layerBranchId in index.layerBranches:
206 branchName = index.layerBranches[layerBranchId].branch.name
207 collection = index.layerBranches[layerBranchId].collection
208
209 def add_dependency(layerDependencyId, index, deps, required):
210 try:
211 depDict = bb.utils.explode_dep_versions2(deps)
212 except bb.utils.VersionStringException as vse:
213 bb.fatal('Error parsing LAYERDEPENDS_%s: %s' % (c, str(vse)))
214
215 for dep, oplist in list(depDict.items()):
216 # We need to search ourselves, so use the _ version...
217 depLayerBranch = index.find_collection(dep, branches=[branchName])
218 if not depLayerBranch:
219 # Missing dependency?!
220 logger.error('Missing dependency %s (%s)' % (dep, branchName))
221 continue
222
223 # We assume that the oplist matches...
224 layerDependencyId += 1
225 layerDependency = layerindexlib.LayerDependency(index, None)
226 layerDependency.define_data(id=layerDependencyId,
227 required=required, layerbranch=layerBranchId,
228 dependency=depLayerBranch.layer_id)
229
230 logger.debug(1, '%s requires %s' % (layerDependency.layer.name, layerDependency.dependency.name))
231 index.add_element("layerDependencies", [layerDependency])
232
233 return layerDependencyId
234
235 deps = d.getVar("LAYERDEPENDS_%s" % collection)
236 if deps:
237 layerDependencyId = add_dependency(layerDependencyId, index, deps, True)
238
239 deps = d.getVar("LAYERRECOMMENDS_%s" % collection)
240 if deps:
241 layerDependencyId = add_dependency(layerDependencyId, index, deps, False)
242
243 # Need to load recipes here (requires cooker access)
244 recipeId = 0
245 ## TODO: NOT IMPLEMENTED
246 # The code following this is an example of what needs to be
247 # implemented. However, it does not work as-is.
248 if False and 'recipes' in load:
249 index.recipes = {}
250
251 ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
252
253 all_versions = self._run_command('allProviders')
254
255 all_versions_list = defaultdict(list, all_versions)
256 for pn in all_versions_list:
257 for ((pe, pv, pr), fpath) in all_versions_list[pn]:
258 realfn = bb.cache.virtualfn2realfn(fpath)
259
260 filepath = os.path.dirname(realfn[0])
261 filename = os.path.basename(realfn[0])
262
263 # This is all HORRIBLY slow, and likely unnecessary
264 #dscon = self._run_command('parseRecipeFile', fpath, False, [])
265 #connector = myDataStoreConnector(self, dscon.dsindex)
266 #recipe_data = bb.data.init()
267 #recipe_data.setVar('_remote_data', connector)
268
269 #summary = recipe_data.getVar('SUMMARY')
270 #description = recipe_data.getVar('DESCRIPTION')
271 #section = recipe_data.getVar('SECTION')
272 #license = recipe_data.getVar('LICENSE')
273 #homepage = recipe_data.getVar('HOMEPAGE')
274 #bugtracker = recipe_data.getVar('BUGTRACKER')
275 #provides = recipe_data.getVar('PROVIDES')
276
277 layer = bb.utils.get_file_layer(realfn[0], self.config_data)
278
279 depBranchId = collection_layerbranch[layer]
280
281 recipeId += 1
282 recipe = layerindexlib.Recipe(index, None)
283 recipe.define_data(id=recipeId,
284 filename=filename, filepath=filepath,
285 pn=pn, pv=pv,
286 summary=pn, description=pn, section='?',
287 license='?', homepage='?', bugtracker='?',
288 provides='?', bbclassextend='?', inherits='?',
289 blacklisted='?', layerbranch=depBranchId)
290
291 index = addElement("recipes", [recipe], index)
292
293 # ("machines", layerindexlib.Machine)
294 machineId = 0
295 if 'machines' in load:
296 index.machines = {}
297
298 for layerBranchId in index.layerBranches:
299 # load_bblayers uses the description to cache the actual path...
300 machine_path = index.layerBranches[layerBranchId].getDescription()
301 machine_path = os.path.join(machine_path, 'conf/machine')
302 if os.path.isdir(machine_path):
303 for (dirpath, _, filenames) in os.walk(machine_path):
304 # Ignore subdirs...
305 if not dirpath.endswith('conf/machine'):
306 continue
307 for fname in filenames:
308 if fname.endswith('.conf'):
309 machineId += 1
310 machine = layerindexlib.Machine(index, None)
311 machine.define_data(id=machineId, name=fname[:-5],
312 description=fname[:-5],
313 layerbranch=collection_layerbranch[entry])
314
315 index.add_element("machines", [machine])
316
317 # ("distros", layerindexlib.Distro)
318 distroId = 0
319 if 'distros' in load:
320 index.distros = {}
321
322 for layerBranchId in index.layerBranches:
323 # load_bblayers uses the description to cache the actual path...
324 distro_path = index.layerBranches[layerBranchId].getDescription()
325 distro_path = os.path.join(distro_path, 'conf/distro')
326 if os.path.isdir(distro_path):
327 for (dirpath, _, filenames) in os.walk(distro_path):
328 # Ignore subdirs...
329 if not dirpath.endswith('conf/distro'):
330 continue
331 for fname in filenames:
332 if fname.endswith('.conf'):
333 distroId += 1
334 distro = layerindexlib.Distro(index, None)
335 distro.define_data(id=distroId, name=fname[:-5],
336 description=fname[:-5],
337 layerbranch=collection_layerbranch[entry])
338
339 index.add_element("distros", [distro])
340
341 return index
diff --git a/bitbake/lib/layerindexlib/plugin.py b/bitbake/lib/layerindexlib/plugin.py
new file mode 100644
index 0000000000..92a2e978ba
--- /dev/null
+++ b/bitbake/lib/layerindexlib/plugin.py
@@ -0,0 +1,60 @@
1# Copyright (C) 2016-2018 Wind River Systems, Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License version 2 as
5# published by the Free Software Foundation.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
10# See the GNU General Public License for more details.
11#
12# You should have received a copy of the GNU General Public License
13# along with this program; if not, write to the Free Software
14# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
16# The file contains:
17# LayerIndex exceptions
18# Plugin base class
19# Utility Functions for working on layerindex data
20
21import argparse
22import logging
23import os
24import bb.msg
25
26logger = logging.getLogger('BitBake.layerindexlib.plugin')
27
28class LayerIndexPluginException(Exception):
29 """LayerIndex Generic Exception"""
30 def __init__(self, message):
31 self.msg = message
32 Exception.__init__(self, message)
33
34 def __str__(self):
35 return self.msg
36
37class LayerIndexPluginUrlError(LayerIndexPluginException):
38 """Exception raised when a plugin does not support a given URL type"""
39 def __init__(self, plugin, url):
40 msg = "%s does not support %s:" % (plugin, url)
41 self.plugin = plugin
42 self.url = url
43 LayerIndexPluginException.__init__(self, msg)
44
45class IndexPlugin():
46 def __init__(self):
47 self.type = None
48
49 def init(self, layerindex):
50 self.layerindex = layerindex
51
52 def plugin_type(self):
53 return self.type
54
55 def load_index(self, uri):
56 raise NotImplementedError('load_index is not implemented')
57
58 def store_index(self, uri, index):
59 raise NotImplementedError('store_index is not implemented')
60
diff --git a/bitbake/lib/layerindexlib/restapi.py b/bitbake/lib/layerindexlib/restapi.py
new file mode 100644
index 0000000000..d08eb20555
--- /dev/null
+++ b/bitbake/lib/layerindexlib/restapi.py
@@ -0,0 +1,398 @@
1# Copyright (C) 2016-2018 Wind River Systems, Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License version 2 as
5# published by the Free Software Foundation.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
10# See the GNU General Public License for more details.
11#
12# You should have received a copy of the GNU General Public License
13# along with this program; if not, write to the Free Software
14# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
16import logging
17import json
18from urllib.parse import unquote
19from urllib.parse import urlparse
20
21import layerindexlib
22import layerindexlib.plugin
23
24logger = logging.getLogger('BitBake.layerindexlib.restapi')
25
26def plugin_init(plugins):
27 return RestApiPlugin()
28
29class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
30 def __init__(self):
31 self.type = "restapi"
32
33 def load_index(self, url, load):
34 """
35 Fetches layer information from a local or remote layer index.
36
37 The return value is a LayerIndexObj.
38
39 url is the url to the rest api of the layer index, such as:
40 http://layers.openembedded.org/layerindex/api/
41
42 Or a local file...
43 """
44
45 up = urlparse(url)
46
47 if up.scheme == 'file':
48 return self.load_index_file(up, url, load)
49
50 if up.scheme == 'http' or up.scheme == 'https':
51 return self.load_index_web(up, url, load)
52
53 raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
54
55
56 def load_index_file(self, up, url, load):
57 """
58 Fetches layer information from a local file or directory.
59
60 The return value is a LayerIndexObj.
61
62 ud is the parsed url to the local file or directory.
63 """
64 if not os.path.exists(up.path):
65 raise FileNotFoundError(up.path)
66
67 index = layerindexlib.LayerIndexObj()
68
69 index.config = {}
70 index.config['TYPE'] = self.type
71 index.config['URL'] = url
72
73 params = self.layerindex._parse_params(up.params)
74
75 if 'desc' in params:
76 index.config['DESCRIPTION'] = unquote(params['desc'])
77 else:
78 index.config['DESCRIPTION'] = up.path
79
80 if 'cache' in params:
81 index.config['CACHE'] = params['cache']
82
83 if 'branch' in params:
84 branches = params['branch'].split(',')
85 index.config['BRANCH'] = branches
86 else:
87 branches = ['*']
88
89
90 def load_cache(path, index, branches=[]):
91 logger.debug(1, 'Loading json file %s' % path)
92 with open(path, 'rt', encoding='utf-8') as f:
93 pindex = json.load(f)
94
95 # Filter the branches on loaded files...
96 newpBranch = []
97 for branch in branches:
98 if branch != '*':
99 if 'branches' in pindex:
100 for br in pindex['branches']:
101 if br['name'] == branch:
102 newpBranch.append(br)
103 else:
104 if 'branches' in pindex:
105 for br in pindex['branches']:
106 newpBranch.append(br)
107
108 if newpBranch:
109 index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
110 else:
111 logger.debug(1, 'No matching branches (%s) in index file(s)' % branches)
112 # No matching branches.. return nothing...
113 return
114
115 for (lName, lType) in [("layerItems", layerindexlib.LayerItem),
116 ("layerBranches", layerindexlib.LayerBranch),
117 ("layerDependencies", layerindexlib.LayerDependency),
118 ("recipes", layerindexlib.Recipe),
119 ("machines", layerindexlib.Machine),
120 ("distros", layerindexlib.Distro)]:
121 if lName in pindex:
122 index.add_raw_element(lName, lType, pindex[lName])
123
124
125 if not os.path.isdir(up.path):
126 load_cache(up.path, index, branches)
127 return index
128
129 logger.debug(1, 'Loading from dir %s...' % (up.path))
130 for (dirpath, _, filenames) in os.walk(up.path):
131 for filename in filenames:
132 if not filename.endswith('.json'):
133 continue
134 fpath = os.path.join(dirpath, filename)
135 load_cache(fpath, index, branches)
136
137 return index
138
139
140 def load_index_web(self, up, url, load):
141 """
142 Fetches layer information from a remote layer index.
143
144 The return value is a LayerIndexObj.
145
146 ud is the parsed url to the rest api of the layer index, such as:
147 http://layers.openembedded.org/layerindex/api/
148 """
149
150 def _get_json_response(apiurl=None, username=None, password=None, retry=True):
151 assert apiurl is not None
152
153 logger.debug(1, "fetching %s" % apiurl)
154
155 up = urlparse(apiurl)
156
157 username=up.username
158 password=up.password
159
160 # Strip username/password and params
161 if up.port:
162 up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port))
163 else:
164 up_stripped = up._replace(params="", netloc=up.hostname)
165
166 res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password)
167
168 try:
169 parsed = json.loads(res.read().decode('utf-8'))
170 except ConnectionResetError:
171 if retry:
172 logger.debug(1, "%s: Connection reset by peer. Retrying..." % url)
173 parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
174 logger.debug(1, "%s: retry successful.")
175 else:
176 raise LayerIndexFetchError('%s: Connection reset by peer. Is there a firewall blocking your connection?' % apiurl)
177
178 return parsed
179
180 index = layerindexlib.LayerIndexObj()
181
182 index.config = {}
183 index.config['TYPE'] = self.type
184 index.config['URL'] = url
185
186 params = self.layerindex._parse_params(up.params)
187
188 if 'desc' in params:
189 index.config['DESCRIPTION'] = unquote(params['desc'])
190 else:
191 index.config['DESCRIPTION'] = up.hostname
192
193 if 'cache' in params:
194 index.config['CACHE'] = params['cache']
195
196 if 'branch' in params:
197 branches = params['branch'].split(',')
198 index.config['BRANCH'] = branches
199 else:
200 branches = ['*']
201
202 try:
203 index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password)
204 except Exception as e:
205 raise layerindexlib.LayerIndexFetchError(url, e)
206
207 # Local raw index set...
208 pindex = {}
209
210 # Load all the requested branches at the same time time,
211 # a special branch of '*' means load all branches
212 filter = ""
213 if "*" not in branches:
214 filter = "?filter=name:%s" % "OR".join(branches)
215
216 logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches']))
217
218 # The link won't include username/password, so pull it from the original url
219 pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
220 username=up.username, password=up.password)
221 if not pindex['branches']:
222 logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url))
223 return index
224 index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
225
226 # Load all of the layerItems (these can not be easily filtered)
227 logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
228
229
230 # The link won't include username/password, so pull it from the original url
231 pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
232 username=up.username, password=up.password)
233 if not pindex['layerItems']:
234 logger.debug(1, "No layers were found at url %s." % (url))
235 return index
236 index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
237
238
239 # From this point on load the contents for each branch. Otherwise we
240 # could run into a timeout.
241 for branch in index.branches:
242 filter = "?filter=branch__name:%s" % index.branches[branch].name
243
244 logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
245
246 # The link won't include username/password, so pull it from the original url
247 pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
248 username=up.username, password=up.password)
249 if not pindex['layerBranches']:
250 logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url))
251 return index
252 index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
253
254
255 # Load the rest, they all have a similar format
256 # Note: the layer index has a few more items, we can add them if necessary
257 # in the future.
258 filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name
259 for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency),
260 ("recipes", layerindexlib.Recipe),
261 ("machines", layerindexlib.Machine),
262 ("distros", layerindexlib.Distro)]:
263 if lName not in load:
264 continue
265 logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName]))
266
267 # The link won't include username/password, so pull it from the original url
268 pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
269 username=up.username, password=up.password)
270 index.add_raw_element(lName, lType, pindex[lName])
271
272 return index
273
274 def store_index(self, url, index):
275 """
276 Store layer information into a local file/dir.
277
278 The return value is a dictionary containing API,
279 layer, branch, dependency, recipe, machine, distro, information.
280
281 ud is a parsed url to a directory or file. If the path is a
282 directory, we will split the files into one file per layer.
283 If the path is to a file (exists or not) the entire DB will be
284 dumped into that one file.
285 """
286
287 up = urlparse(url)
288
289 if up.scheme != 'file':
290 raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
291
292 logger.debug(1, "Storing to %s..." % up.path)
293
294 try:
295 layerbranches = index.layerBranches
296 except KeyError:
297 logger.error('No layerBranches to write.')
298 return
299
300
301 def filter_item(layerbranchid, objects):
302 filtered = []
303 for obj in getattr(index, objects, None):
304 try:
305 if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
306 filtered.append(getattr(index, objects)[obj]._data)
307 except AttributeError:
308 logger.debug(1, 'No obj.layerbranch_id: %s' % objects)
309 # No simple filter method, just include it...
310 try:
311 filtered.append(getattr(index, objects)[obj]._data)
312 except AttributeError:
313 logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj)))
314 filtered.append(obj)
315 return filtered
316
317
318 # Write out to a single file.
319 # Filter out unnecessary items, then sort as we write for determinism
320 if not os.path.isdir(up.path):
321 pindex = {}
322
323 pindex['branches'] = []
324 pindex['layerItems'] = []
325 pindex['layerBranches'] = []
326
327 for layerbranchid in layerbranches:
328 if layerbranches[layerbranchid].branch._data not in pindex['branches']:
329 pindex['branches'].append(layerbranches[layerbranchid].branch._data)
330
331 if layerbranches[layerbranchid].layer._data not in pindex['layerItems']:
332 pindex['layerItems'].append(layerbranches[layerbranchid].layer._data)
333
334 if layerbranches[layerbranchid]._data not in pindex['layerBranches']:
335 pindex['layerBranches'].append(layerbranches[layerbranchid]._data)
336
337 for entry in index._index:
338 # Skip local items, apilinks and items already processed
339 if entry in index.config['local'] or \
340 entry == 'apilinks' or \
341 entry == 'branches' or \
342 entry == 'layerBranches' or \
343 entry == 'layerItems':
344 continue
345 if entry not in pindex:
346 pindex[entry] = []
347 pindex[entry].extend(filter_item(layerbranchid, entry))
348
349 bb.debug(1, 'Writing index to %s' % up.path)
350 with open(up.path, 'wt') as f:
351 json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
352 return
353
354
355 # Write out to a directory one file per layerBranch
356 # Prepare all layer related items, to create a minimal file.
357 # We have to sort the entries as we write so they are deterministic
358 for layerbranchid in layerbranches:
359 pindex = {}
360
361 for entry in index._index:
362 # Skip local items, apilinks and items already processed
363 if entry in index.config['local'] or \
364 entry == 'apilinks' or \
365 entry == 'branches' or \
366 entry == 'layerBranches' or \
367 entry == 'layerItems':
368 continue
369 pindex[entry] = filter_item(layerbranchid, entry)
370
371 # Add the layer we're processing as the first one...
372 pindex['branches'] = [layerbranches[layerbranchid].branch._data]
373 pindex['layerItems'] = [layerbranches[layerbranchid].layer._data]
374 pindex['layerBranches'] = [layerbranches[layerbranchid]._data]
375
376 # We also need to include the layerbranch for any dependencies...
377 for layerdep in pindex['layerDependencies']:
378 layerdependency = layerindexlib.LayerDependency(index, layerdep)
379
380 layeritem = layerdependency.dependency
381 layerbranch = layerdependency.dependency_layerBranch
382
383 # We need to avoid duplicates...
384 if layeritem._data not in pindex['layerItems']:
385 pindex['layerItems'].append(layeritem._data)
386
387 if layerbranch._data not in pindex['layerBranches']:
388 pindex['layerBranches'].append(layerbranch._data)
389
390 # apply mirroring adjustments here....
391
392 fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name']
393 fname = fname.translate(str.maketrans('/ ', '__'))
394 fpath = os.path.join(up.path, fname)
395
396 bb.debug(1, 'Writing index to %s' % fpath + '.json')
397 with open(fpath + '.json', 'wt') as f:
398 json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
diff --git a/bitbake/lib/layerindexlib/tests/__init__.py b/bitbake/lib/layerindexlib/tests/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/bitbake/lib/layerindexlib/tests/__init__.py
diff --git a/bitbake/lib/layerindexlib/tests/common.py b/bitbake/lib/layerindexlib/tests/common.py
new file mode 100644
index 0000000000..22a54585c8
--- /dev/null
+++ b/bitbake/lib/layerindexlib/tests/common.py
@@ -0,0 +1,43 @@
1# Copyright (C) 2017-2018 Wind River Systems, Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License version 2 as
5# published by the Free Software Foundation.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
10# See the GNU General Public License for more details.
11#
12# You should have received a copy of the GNU General Public License
13# along with this program; if not, write to the Free Software
14# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
16import unittest
17import tempfile
18import os
19import bb
20
21import logging
22
23class LayersTest(unittest.TestCase):
24
25 def setUp(self):
26 self.origdir = os.getcwd()
27 self.d = bb.data.init()
28 # At least one variable needs to be set
29 self.d.setVar('DL_DIR', os.getcwd())
30
31 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
32 self.d.setVar('BB_NO_NETWORK', '1')
33
34 self.tempdir = tempfile.mkdtemp()
35 self.logger = logging.getLogger("BitBake")
36
37 def tearDown(self):
38 os.chdir(self.origdir)
39 if os.environ.get("BB_TMPDIR_NOCLEAN") == "yes":
40 print("Not cleaning up %s. Please remove manually." % self.tempdir)
41 else:
42 bb.utils.prunedir(self.tempdir)
43
diff --git a/bitbake/lib/layerindexlib/tests/cooker.py b/bitbake/lib/layerindexlib/tests/cooker.py
new file mode 100644
index 0000000000..9ce6e8c3ae
--- /dev/null
+++ b/bitbake/lib/layerindexlib/tests/cooker.py
@@ -0,0 +1,123 @@
1# Copyright (C) 2018 Wind River Systems, Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License version 2 as
5# published by the Free Software Foundation.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
10# See the GNU General Public License for more details.
11#
12# You should have received a copy of the GNU General Public License
13# along with this program; if not, write to the Free Software
14# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
16import unittest
17import tempfile
18import os
19import bb
20
21import layerindexlib
22from layerindexlib.tests.common import LayersTest
23
24import logging
25
26class LayerIndexCookerTest(LayersTest):
27
28 def setUp(self):
29 LayersTest.setUp(self)
30
31 # Note this is NOT a comprehensive test of cooker, as we can't easily
32 # configure the test data. But we can emulate the basics of the layer.conf
33 # files, so that is what we will do.
34
35 new_topdir = os.path.join(os.path.dirname(__file__), "testdata")
36 new_bbpath = os.path.join(new_topdir, "build")
37
38 self.d.setVar('TOPDIR', new_topdir)
39 self.d.setVar('BBPATH', new_bbpath)
40
41 self.d = bb.parse.handle("%s/conf/bblayers.conf" % new_bbpath, self.d, True)
42 for layer in self.d.getVar('BBLAYERS').split():
43 self.d = bb.parse.handle("%s/conf/layer.conf" % layer, self.d, True)
44
45 self.layerindex = layerindexlib.LayerIndex(self.d)
46 self.layerindex.load_layerindex('cooker://', load=['layerDependencies'])
47
48 def test_layerindex_is_empty(self):
49 self.assertFalse(self.layerindex.is_empty(), msg="Layerindex is not empty!")
50
51 def test_dependency_resolution(self):
52 # Verify depth first searching...
53 (dependencies, invalidnames) = self.layerindex.find_dependencies(names=['meta-python'])
54
55 first = True
56 for deplayerbranch in dependencies:
57 layerBranch = dependencies[deplayerbranch][0]
58 layerDeps = dependencies[deplayerbranch][1:]
59
60 if not first:
61 continue
62
63 first = False
64
65 # Top of the deps should be openembedded-core, since everything depends on it.
66 self.assertEqual(layerBranch.layer.name, "openembedded-core", msg='Top dependency not openembedded-core')
67
68 # meta-python should cause an openembedded-core dependency, if not assert!
69 for dep in layerDeps:
70 if dep.layer.name == 'meta-python':
71 break
72 else:
73 self.assertTrue(False, msg='meta-python was not found')
74
75 # Only check the first element...
76 break
77 else:
78 if first:
79 # Empty list, this is bad.
80 self.assertTrue(False, msg='Empty list of dependencies')
81
82 # Last dep should be the requested item
83 layerBranch = dependencies[deplayerbranch][0]
84 self.assertEqual(layerBranch.layer.name, "meta-python", msg='Last dependency not meta-python')
85
86 def test_find_collection(self):
87 def _check(collection, expected):
88 self.logger.debug(1, "Looking for collection %s..." % collection)
89 result = self.layerindex.find_collection(collection)
90 if expected:
91 self.assertIsNotNone(result, msg="Did not find %s when it shouldn't be there" % collection)
92 else:
93 self.assertIsNone(result, msg="Found %s when it should be there" % collection)
94
95 tests = [ ('core', True),
96 ('openembedded-core', False),
97 ('networking-layer', True),
98 ('meta-python', True),
99 ('openembedded-layer', True),
100 ('notpresent', False) ]
101
102 for collection,result in tests:
103 _check(collection, result)
104
105 def test_find_layerbranch(self):
106 def _check(name, expected):
107 self.logger.debug(1, "Looking for layerbranch %s..." % name)
108 result = self.layerindex.find_layerbranch(name)
109 if expected:
110 self.assertIsNotNone(result, msg="Did not find %s when it shouldn't be there" % collection)
111 else:
112 self.assertIsNone(result, msg="Found %s when it should be there" % collection)
113
114 tests = [ ('openembedded-core', True),
115 ('core', False),
116 ('networking-layer', True),
117 ('meta-python', True),
118 ('openembedded-layer', True),
119 ('notpresent', False) ]
120
121 for collection,result in tests:
122 _check(collection, result)
123
diff --git a/bitbake/lib/layerindexlib/tests/layerindexobj.py b/bitbake/lib/layerindexlib/tests/layerindexobj.py
new file mode 100644
index 0000000000..e2fbb950b0
--- /dev/null
+++ b/bitbake/lib/layerindexlib/tests/layerindexobj.py
@@ -0,0 +1,226 @@
1# Copyright (C) 2017-2018 Wind River Systems, Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License version 2 as
5# published by the Free Software Foundation.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
10# See the GNU General Public License for more details.
11#
12# You should have received a copy of the GNU General Public License
13# along with this program; if not, write to the Free Software
14# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
16import unittest
17import tempfile
18import os
19import bb
20
21from layerindexlib.tests.common import LayersTest
22
23import logging
24
25class LayerIndexObjectsTest(LayersTest):
26 def setUp(self):
27 from layerindexlib import LayerIndexObj, Branch, LayerItem, LayerBranch, LayerDependency, Recipe, Machine, Distro
28
29 LayersTest.setUp(self)
30
31 self.index = LayerIndexObj()
32
33 branchId = 0
34 layerItemId = 0
35 layerBranchId = 0
36 layerDependencyId = 0
37 recipeId = 0
38 machineId = 0
39 distroId = 0
40
41 self.index.branches = {}
42 self.index.layerItems = {}
43 self.index.layerBranches = {}
44 self.index.layerDependencies = {}
45 self.index.recipes = {}
46 self.index.machines = {}
47 self.index.distros = {}
48
49 branchId += 1
50 self.index.branches[branchId] = Branch(self.index)
51 self.index.branches[branchId].define_data(branchId,
52 'test_branch', 'bb_test_branch')
53 self.index.branches[branchId].lockData()
54
55 layerItemId +=1
56 self.index.layerItems[layerItemId] = LayerItem(self.index)
57 self.index.layerItems[layerItemId].define_data(layerItemId,
58 'test_layerItem', vcs_url='git://git_test_url/test_layerItem')
59 self.index.layerItems[layerItemId].lockData()
60
61 layerBranchId +=1
62 self.index.layerBranches[layerBranchId] = LayerBranch(self.index)
63 self.index.layerBranches[layerBranchId].define_data(layerBranchId,
64 'test_collection', '99', layerItemId,
65 branchId)
66
67 recipeId += 1
68 self.index.recipes[recipeId] = Recipe(self.index)
69 self.index.recipes[recipeId].define_data(recipeId, 'test_git.bb',
70 'recipes-test', 'test', 'git',
71 layerBranchId)
72
73 machineId += 1
74 self.index.machines[machineId] = Machine(self.index)
75 self.index.machines[machineId].define_data(machineId,
76 'test_machine', 'test_machine',
77 layerBranchId)
78
79 distroId += 1
80 self.index.distros[distroId] = Distro(self.index)
81 self.index.distros[distroId].define_data(distroId,
82 'test_distro', 'test_distro',
83 layerBranchId)
84
85 layerItemId +=1
86 self.index.layerItems[layerItemId] = LayerItem(self.index)
87 self.index.layerItems[layerItemId].define_data(layerItemId, 'test_layerItem 2',
88 vcs_url='git://git_test_url/test_layerItem')
89
90 layerBranchId +=1
91 self.index.layerBranches[layerBranchId] = LayerBranch(self.index)
92 self.index.layerBranches[layerBranchId].define_data(layerBranchId,
93 'test_collection_2', '72', layerItemId,
94 branchId, actual_branch='some_other_branch')
95
96 layerDependencyId += 1
97 self.index.layerDependencies[layerDependencyId] = LayerDependency(self.index)
98 self.index.layerDependencies[layerDependencyId].define_data(layerDependencyId,
99 layerBranchId, 1)
100
101 layerDependencyId += 1
102 self.index.layerDependencies[layerDependencyId] = LayerDependency(self.index)
103 self.index.layerDependencies[layerDependencyId].define_data(layerDependencyId,
104 layerBranchId, 1, required=False)
105
106 def test_branch(self):
107 branch = self.index.branches[1]
108 self.assertEqual(branch.id, 1)
109 self.assertEqual(branch.name, 'test_branch')
110 self.assertEqual(branch.short_description, 'test_branch')
111 self.assertEqual(branch.bitbake_branch, 'bb_test_branch')
112
113 def test_layerItem(self):
114 layerItem = self.index.layerItems[1]
115 self.assertEqual(layerItem.id, 1)
116 self.assertEqual(layerItem.name, 'test_layerItem')
117 self.assertEqual(layerItem.summary, 'test_layerItem')
118 self.assertEqual(layerItem.description, 'test_layerItem')
119 self.assertEqual(layerItem.vcs_url, 'git://git_test_url/test_layerItem')
120 self.assertEqual(layerItem.vcs_web_url, None)
121 self.assertIsNone(layerItem.vcs_web_tree_base_url)
122 self.assertIsNone(layerItem.vcs_web_file_base_url)
123 self.assertIsNotNone(layerItem.updated)
124
125 layerItem = self.index.layerItems[2]
126 self.assertEqual(layerItem.id, 2)
127 self.assertEqual(layerItem.name, 'test_layerItem 2')
128 self.assertEqual(layerItem.summary, 'test_layerItem 2')
129 self.assertEqual(layerItem.description, 'test_layerItem 2')
130 self.assertEqual(layerItem.vcs_url, 'git://git_test_url/test_layerItem')
131 self.assertIsNone(layerItem.vcs_web_url)
132 self.assertIsNone(layerItem.vcs_web_tree_base_url)
133 self.assertIsNone(layerItem.vcs_web_file_base_url)
134 self.assertIsNotNone(layerItem.updated)
135
136 def test_layerBranch(self):
137 layerBranch = self.index.layerBranches[1]
138 self.assertEqual(layerBranch.id, 1)
139 self.assertEqual(layerBranch.collection, 'test_collection')
140 self.assertEqual(layerBranch.version, '99')
141 self.assertEqual(layerBranch.vcs_subdir, '')
142 self.assertEqual(layerBranch.actual_branch, 'test_branch')
143 self.assertIsNotNone(layerBranch.updated)
144 self.assertEqual(layerBranch.layer_id, 1)
145 self.assertEqual(layerBranch.branch_id, 1)
146 self.assertEqual(layerBranch.layer, self.index.layerItems[1])
147 self.assertEqual(layerBranch.branch, self.index.branches[1])
148
149 layerBranch = self.index.layerBranches[2]
150 self.assertEqual(layerBranch.id, 2)
151 self.assertEqual(layerBranch.collection, 'test_collection_2')
152 self.assertEqual(layerBranch.version, '72')
153 self.assertEqual(layerBranch.vcs_subdir, '')
154 self.assertEqual(layerBranch.actual_branch, 'some_other_branch')
155 self.assertIsNotNone(layerBranch.updated)
156 self.assertEqual(layerBranch.layer_id, 2)
157 self.assertEqual(layerBranch.branch_id, 1)
158 self.assertEqual(layerBranch.layer, self.index.layerItems[2])
159 self.assertEqual(layerBranch.branch, self.index.branches[1])
160
161 def test_layerDependency(self):
162 layerDependency = self.index.layerDependencies[1]
163 self.assertEqual(layerDependency.id, 1)
164 self.assertEqual(layerDependency.layerbranch_id, 2)
165 self.assertEqual(layerDependency.layerbranch, self.index.layerBranches[2])
166 self.assertEqual(layerDependency.layer_id, 2)
167 self.assertEqual(layerDependency.layer, self.index.layerItems[2])
168 self.assertTrue(layerDependency.required)
169 self.assertEqual(layerDependency.dependency_id, 1)
170 self.assertEqual(layerDependency.dependency, self.index.layerItems[1])
171 self.assertEqual(layerDependency.dependency_layerBranch, self.index.layerBranches[1])
172
173 layerDependency = self.index.layerDependencies[2]
174 self.assertEqual(layerDependency.id, 2)
175 self.assertEqual(layerDependency.layerbranch_id, 2)
176 self.assertEqual(layerDependency.layerbranch, self.index.layerBranches[2])
177 self.assertEqual(layerDependency.layer_id, 2)
178 self.assertEqual(layerDependency.layer, self.index.layerItems[2])
179 self.assertFalse(layerDependency.required)
180 self.assertEqual(layerDependency.dependency_id, 1)
181 self.assertEqual(layerDependency.dependency, self.index.layerItems[1])
182 self.assertEqual(layerDependency.dependency_layerBranch, self.index.layerBranches[1])
183
184 def test_recipe(self):
185 recipe = self.index.recipes[1]
186 self.assertEqual(recipe.id, 1)
187 self.assertEqual(recipe.layerbranch_id, 1)
188 self.assertEqual(recipe.layerbranch, self.index.layerBranches[1])
189 self.assertEqual(recipe.layer_id, 1)
190 self.assertEqual(recipe.layer, self.index.layerItems[1])
191 self.assertEqual(recipe.filename, 'test_git.bb')
192 self.assertEqual(recipe.filepath, 'recipes-test')
193 self.assertEqual(recipe.fullpath, 'recipes-test/test_git.bb')
194 self.assertEqual(recipe.summary, "")
195 self.assertEqual(recipe.description, "")
196 self.assertEqual(recipe.section, "")
197 self.assertEqual(recipe.pn, 'test')
198 self.assertEqual(recipe.pv, 'git')
199 self.assertEqual(recipe.license, "")
200 self.assertEqual(recipe.homepage, "")
201 self.assertEqual(recipe.bugtracker, "")
202 self.assertEqual(recipe.provides, "")
203 self.assertIsNotNone(recipe.updated)
204 self.assertEqual(recipe.inherits, "")
205
206 def test_machine(self):
207 machine = self.index.machines[1]
208 self.assertEqual(machine.id, 1)
209 self.assertEqual(machine.layerbranch_id, 1)
210 self.assertEqual(machine.layerbranch, self.index.layerBranches[1])
211 self.assertEqual(machine.layer_id, 1)
212 self.assertEqual(machine.layer, self.index.layerItems[1])
213 self.assertEqual(machine.name, 'test_machine')
214 self.assertEqual(machine.description, 'test_machine')
215 self.assertIsNotNone(machine.updated)
216
217 def test_distro(self):
218 distro = self.index.distros[1]
219 self.assertEqual(distro.id, 1)
220 self.assertEqual(distro.layerbranch_id, 1)
221 self.assertEqual(distro.layerbranch, self.index.layerBranches[1])
222 self.assertEqual(distro.layer_id, 1)
223 self.assertEqual(distro.layer, self.index.layerItems[1])
224 self.assertEqual(distro.name, 'test_distro')
225 self.assertEqual(distro.description, 'test_distro')
226 self.assertIsNotNone(distro.updated)
diff --git a/bitbake/lib/layerindexlib/tests/restapi.py b/bitbake/lib/layerindexlib/tests/restapi.py
new file mode 100644
index 0000000000..bfaac43db4
--- /dev/null
+++ b/bitbake/lib/layerindexlib/tests/restapi.py
@@ -0,0 +1,174 @@
1# Copyright (C) 2017-2018 Wind River Systems, Inc.
2#
3# This program is free software; you can redistribute it and/or modify
4# it under the terms of the GNU General Public License version 2 as
5# published by the Free Software Foundation.
6#
7# This program is distributed in the hope that it will be useful,
8# but WITHOUT ANY WARRANTY; without even the implied warranty of
9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
10# See the GNU General Public License for more details.
11#
12# You should have received a copy of the GNU General Public License
13# along with this program; if not, write to the Free Software
14# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
15
16import unittest
17import tempfile
18import os
19import bb
20
21import layerindexlib
22from layerindexlib.tests.common import LayersTest
23
24import logging
25
26class LayerIndexWebRestApiTest(LayersTest):
27
28 if os.environ.get("BB_SKIP_NETTESTS") == "yes":
29 print("Unset BB_SKIP_NETTESTS to run network tests")
30 else:
31 def setUp(self):
32 LayersTest.setUp(self)
33 self.layerindex = layerindexlib.LayerIndex(self.d)
34 self.layerindex.load_layerindex('http://layers.openembedded.org/layerindex/api/;branch=sumo', load=['layerDependencies'])
35
36 def test_layerindex_is_empty(self):
37 self.assertFalse(self.layerindex.is_empty(), msg="Layerindex is empty")
38
39 def test_layerindex_store_file(self):
40 self.layerindex.store_layerindex('file://%s/file.json' % self.tempdir, self.layerindex.indexes[0])
41
42 self.assertTrue(os.path.isfile('%s/file.json' % self.tempdir), msg="Temporary file was not created by store_layerindex")
43
44 reload = layerindexlib.LayerIndex(self.d)
45 reload.load_layerindex('file://%s/file.json' % self.tempdir)
46
47 self.assertFalse(reload.is_empty(), msg="Layerindex is empty")
48
49 # Calculate layerItems in original index that should NOT be in reload
50 layerItemNames = []
51 for itemId in self.layerindex.indexes[0].layerItems:
52 layerItemNames.append(self.layerindex.indexes[0].layerItems[itemId].name)
53
54 for layerBranchId in self.layerindex.indexes[0].layerBranches:
55 layerItemNames.remove(self.layerindex.indexes[0].layerBranches[layerBranchId].layer.name)
56
57 for itemId in reload.indexes[0].layerItems:
58 self.assertFalse(reload.indexes[0].layerItems[itemId].name in layerItemNames, msg="Item reloaded when it shouldn't have been")
59
60 # Compare the original to what we wrote...
61 for type in self.layerindex.indexes[0]._index:
62 if type == 'apilinks' or \
63 type == 'layerItems' or \
64 type in self.layerindex.indexes[0].config['local']:
65 continue
66 for id in getattr(self.layerindex.indexes[0], type):
67 self.logger.debug(1, "type %s" % (type))
68
69 self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number not in reloaded index")
70
71 self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id]))
72
73 self.assertEqual(getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id], msg="Reloaded contents different")
74
75 def test_layerindex_store_split(self):
76 self.layerindex.store_layerindex('file://%s' % self.tempdir, self.layerindex.indexes[0])
77
78 reload = layerindexlib.LayerIndex(self.d)
79 reload.load_layerindex('file://%s' % self.tempdir)
80
81 self.assertFalse(reload.is_empty(), msg="Layer index is empty")
82
83 for type in self.layerindex.indexes[0]._index:
84 if type == 'apilinks' or \
85 type == 'layerItems' or \
86 type in self.layerindex.indexes[0].config['local']:
87 continue
88 for id in getattr(self.layerindex.indexes[0] ,type):
89 self.logger.debug(1, "type %s" % (type))
90
91 self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number missing from reloaded data")
92
93 self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id]))
94
95 self.assertEqual(getattr(self.layerindex.indexes[0] ,type)[id], getattr(reload.indexes[0], type)[id], msg="reloaded data does not match original")
96
97 def test_dependency_resolution(self):
98 # Verify depth first searching...
99 (dependencies, invalidnames) = self.layerindex.find_dependencies(names=['meta-python'])
100
101 first = True
102 for deplayerbranch in dependencies:
103 layerBranch = dependencies[deplayerbranch][0]
104 layerDeps = dependencies[deplayerbranch][1:]
105
106 if not first:
107 continue
108
109 first = False
110
111 # Top of the deps should be openembedded-core, since everything depends on it.
112 self.assertEqual(layerBranch.layer.name, "openembedded-core", msg='OpenEmbedded-Core is no the first dependency')
113
114 # meta-python should cause an openembedded-core dependency, if not assert!
115 for dep in layerDeps:
116 if dep.layer.name == 'meta-python':
117 break
118 else:
119 self.logger.debug(1, "meta-python was not found")
120 self.assetTrue(False)
121
122 # Only check the first element...
123 break
124 else:
125 # Empty list, this is bad.
126 self.logger.debug(1, "Empty list of dependencies")
127 self.assertIsNotNone(first, msg="Empty list of dependencies")
128
129 # Last dep should be the requested item
130 layerBranch = dependencies[deplayerbranch][0]
131 self.assertEqual(layerBranch.layer.name, "meta-python", msg="Last dependency not meta-python")
132
133 def test_find_collection(self):
134 def _check(collection, expected):
135 self.logger.debug(1, "Looking for collection %s..." % collection)
136 result = self.layerindex.find_collection(collection)
137 if expected:
138 self.assertIsNotNone(result, msg="Did not find %s when it should be there" % collection)
139 else:
140 self.assertIsNone(result, msg="Found %s when it shouldn't be there" % collection)
141
142 tests = [ ('core', True),
143 ('openembedded-core', False),
144 ('networking-layer', True),
145 ('meta-python', True),
146 ('openembedded-layer', True),
147 ('notpresent', False) ]
148
149 for collection,result in tests:
150 _check(collection, result)
151
152 def test_find_layerbranch(self):
153 def _check(name, expected):
154 self.logger.debug(1, "Looking for layerbranch %s..." % name)
155
156 for index in self.layerindex.indexes:
157 for layerbranchid in index.layerBranches:
158 self.logger.debug(1, "Present: %s" % index.layerBranches[layerbranchid].layer.name)
159 result = self.layerindex.find_layerbranch(name)
160 if expected:
161 self.assertIsNotNone(result, msg="Did not find %s when it should be there" % collection)
162 else:
163 self.assertIsNone(result, msg="Found %s when it shouldn't be there" % collection)
164
165 tests = [ ('openembedded-core', True),
166 ('core', False),
167 ('meta-networking', True),
168 ('meta-python', True),
169 ('meta-oe', True),
170 ('notpresent', False) ]
171
172 for collection,result in tests:
173 _check(collection, result)
174
diff --git a/bitbake/lib/layerindexlib/tests/testdata/README b/bitbake/lib/layerindexlib/tests/testdata/README
new file mode 100644
index 0000000000..36ab40bebe
--- /dev/null
+++ b/bitbake/lib/layerindexlib/tests/testdata/README
@@ -0,0 +1,11 @@
1This test data is used to verify the 'cooker' module of the layerindex.
2
3The module consists of a faux project bblayers.conf with four layers defined.
4
5layer1 - openembedded-core
6layer2 - networking-layer
7layer3 - meta-python
8layer4 - openembedded-layer (meta-oe)
9
10Since we do not have a fully populated cooker, we use this to test the
11basic index generation, and not any deep recipe based contents.
diff --git a/bitbake/lib/layerindexlib/tests/testdata/build/conf/bblayers.conf b/bitbake/lib/layerindexlib/tests/testdata/build/conf/bblayers.conf
new file mode 100644
index 0000000000..40429b2f66
--- /dev/null
+++ b/bitbake/lib/layerindexlib/tests/testdata/build/conf/bblayers.conf
@@ -0,0 +1,15 @@
1LAYERSERIES_CORENAMES = "sumo"
2
3# LAYER_CONF_VERSION is increased each time build/conf/bblayers.conf
4# changes incompatibly
5LCONF_VERSION = "7"
6
7BBPATH = "${TOPDIR}"
8BBFILES ?= ""
9
10BBLAYERS ?= " \
11 ${TOPDIR}/layer1 \
12 ${TOPDIR}/layer2 \
13 ${TOPDIR}/layer3 \
14 ${TOPDIR}/layer4 \
15 "
diff --git a/bitbake/lib/layerindexlib/tests/testdata/layer1/conf/layer.conf b/bitbake/lib/layerindexlib/tests/testdata/layer1/conf/layer.conf
new file mode 100644
index 0000000000..966d531959
--- /dev/null
+++ b/bitbake/lib/layerindexlib/tests/testdata/layer1/conf/layer.conf
@@ -0,0 +1,17 @@
1# We have a conf and classes directory, add to BBPATH
2BBPATH .= ":${LAYERDIR}"
3# We have recipes-* directories, add to BBFILES
4BBFILES += "${LAYERDIR}/recipes-*/*/*.bb"
5
6BBFILE_COLLECTIONS += "core"
7BBFILE_PATTERN_core = "^${LAYERDIR}/"
8BBFILE_PRIORITY_core = "5"
9
10LAYERSERIES_CORENAMES = "sumo"
11
12# This should only be incremented on significant changes that will
13# cause compatibility issues with other layers
14LAYERVERSION_core = "11"
15LAYERSERIES_COMPAT_core = "sumo"
16
17BBLAYERS_LAYERINDEX_NAME_core = "openembedded-core"
diff --git a/bitbake/lib/layerindexlib/tests/testdata/layer2/conf/layer.conf b/bitbake/lib/layerindexlib/tests/testdata/layer2/conf/layer.conf
new file mode 100644
index 0000000000..7569d1c217
--- /dev/null
+++ b/bitbake/lib/layerindexlib/tests/testdata/layer2/conf/layer.conf
@@ -0,0 +1,20 @@
1# We have a conf and classes directory, add to BBPATH
2BBPATH .= ":${LAYERDIR}"
3
4# We have a packages directory, add to BBFILES
5BBFILES += "${LAYERDIR}/recipes-*/*/*.bb \
6 ${LAYERDIR}/recipes-*/*/*.bbappend"
7
8BBFILE_COLLECTIONS += "networking-layer"
9BBFILE_PATTERN_networking-layer := "^${LAYERDIR}/"
10BBFILE_PRIORITY_networking-layer = "5"
11
12# This should only be incremented on significant changes that will
13# cause compatibility issues with other layers
14LAYERVERSION_networking-layer = "1"
15
16LAYERDEPENDS_networking-layer = "core"
17LAYERDEPENDS_networking-layer += "openembedded-layer"
18LAYERDEPENDS_networking-layer += "meta-python"
19
20LAYERSERIES_COMPAT_networking-layer = "sumo"
diff --git a/bitbake/lib/layerindexlib/tests/testdata/layer3/conf/layer.conf b/bitbake/lib/layerindexlib/tests/testdata/layer3/conf/layer.conf
new file mode 100644
index 0000000000..7089071faf
--- /dev/null
+++ b/bitbake/lib/layerindexlib/tests/testdata/layer3/conf/layer.conf
@@ -0,0 +1,19 @@
1# We might have a conf and classes directory, append to BBPATH
2BBPATH .= ":${LAYERDIR}"
3
4# We have recipes directories, add to BBFILES
5BBFILES += "${LAYERDIR}/recipes*/*/*.bb ${LAYERDIR}/recipes*/*/*.bbappend"
6
7BBFILE_COLLECTIONS += "meta-python"
8BBFILE_PATTERN_meta-python := "^${LAYERDIR}/"
9BBFILE_PRIORITY_meta-python = "7"
10
11# This should only be incremented on significant changes that will
12# cause compatibility issues with other layers
13LAYERVERSION_meta-python = "1"
14
15LAYERDEPENDS_meta-python = "core openembedded-layer"
16
17LAYERSERIES_COMPAT_meta-python = "sumo"
18
19LICENSE_PATH += "${LAYERDIR}/licenses"
diff --git a/bitbake/lib/layerindexlib/tests/testdata/layer4/conf/layer.conf b/bitbake/lib/layerindexlib/tests/testdata/layer4/conf/layer.conf
new file mode 100644
index 0000000000..6649ee0208
--- /dev/null
+++ b/bitbake/lib/layerindexlib/tests/testdata/layer4/conf/layer.conf
@@ -0,0 +1,22 @@
1# We have a conf and classes directory, append to BBPATH
2BBPATH .= ":${LAYERDIR}"
3
4# We have a recipes directory, add to BBFILES
5BBFILES += "${LAYERDIR}/recipes-*/*/*.bb ${LAYERDIR}/recipes-*/*/*.bbappend"
6
7BBFILE_COLLECTIONS += "openembedded-layer"
8BBFILE_PATTERN_openembedded-layer := "^${LAYERDIR}/"
9
10# Define the priority for recipes (.bb files) from this layer,
11# choosing carefully how this layer interacts with all of the
12# other layers.
13
14BBFILE_PRIORITY_openembedded-layer = "6"
15
16# This should only be incremented on significant changes that will
17# cause compatibility issues with other layers
18LAYERVERSION_openembedded-layer = "1"
19
20LAYERDEPENDS_openembedded-layer = "core"
21
22LAYERSERIES_COMPAT_openembedded-layer = "sumo"