summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/command.py195
-rw-r--r--bitbake/lib/bb/cooker.py19
-rwxr-xr-xbitbake/lib/bb/main.py83
-rw-r--r--bitbake/lib/bb/remotedata.py74
-rw-r--r--bitbake/lib/bb/tinfoil.py385
-rw-r--r--bitbake/lib/bblayers/query.py28
6 files changed, 665 insertions, 119 deletions
diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py
index 012b35faf6..d5be86dab8 100644
--- a/bitbake/lib/bb/command.py
+++ b/bitbake/lib/bb/command.py
@@ -28,8 +28,15 @@ and must not trigger events, directly or indirectly.
28Commands are queued in a CommandQueue 28Commands are queued in a CommandQueue
29""" 29"""
30 30
31from collections import OrderedDict, defaultdict
32
31import bb.event 33import bb.event
32import bb.cooker 34import bb.cooker
35import bb.remotedata
36
37class DataStoreConnectionHandle(object):
38 def __init__(self, dsindex=0):
39 self.dsindex = dsindex
33 40
34class CommandCompleted(bb.event.Event): 41class CommandCompleted(bb.event.Event):
35 pass 42 pass
@@ -55,6 +62,7 @@ class Command:
55 self.cooker = cooker 62 self.cooker = cooker
56 self.cmds_sync = CommandsSync() 63 self.cmds_sync = CommandsSync()
57 self.cmds_async = CommandsAsync() 64 self.cmds_async = CommandsAsync()
65 self.remotedatastores = bb.remotedata.RemoteDatastores(cooker)
58 66
59 # FIXME Add lock for this 67 # FIXME Add lock for this
60 self.currentAsyncCommand = None 68 self.currentAsyncCommand = None
@@ -298,6 +306,193 @@ class CommandsSync:
298 command.cooker.updateConfigOpts(options, environment) 306 command.cooker.updateConfigOpts(options, environment)
299 updateConfig.needconfig = False 307 updateConfig.needconfig = False
300 308
309 def parseConfiguration(self, command, params):
310 """Instruct bitbake to parse its configuration
311 NOTE: it is only necessary to call this if you aren't calling any normal action
312 (otherwise parsing is taken care of automatically)
313 """
314 command.cooker.parseConfiguration()
315 parseConfiguration.needconfig = False
316
317 def getLayerPriorities(self, command, params):
318 ret = []
319 # regex objects cannot be marshalled by xmlrpc
320 for collection, pattern, regex, pri in command.cooker.bbfile_config_priorities:
321 ret.append((collection, pattern, regex.pattern, pri))
322 return ret
323 getLayerPriorities.readonly = True
324
325 def getRecipes(self, command, params):
326 try:
327 mc = params[0]
328 except IndexError:
329 mc = ''
330 return list(command.cooker.recipecaches[mc].pkg_pn.items())
331 getRecipes.readonly = True
332
333 def getRecipeDepends(self, command, params):
334 try:
335 mc = params[0]
336 except IndexError:
337 mc = ''
338 return list(command.cooker.recipecaches[mc].deps.items())
339 getRecipeDepends.readonly = True
340
341 def getRecipeVersions(self, command, params):
342 try:
343 mc = params[0]
344 except IndexError:
345 mc = ''
346 return command.cooker.recipecaches[mc].pkg_pepvpr
347 getRecipeVersions.readonly = True
348
349 def getRuntimeDepends(self, command, params):
350 ret = []
351 try:
352 mc = params[0]
353 except IndexError:
354 mc = ''
355 rundeps = command.cooker.recipecaches[mc].rundeps
356 for key, value in rundeps.items():
357 if isinstance(value, defaultdict):
358 value = dict(value)
359 ret.append((key, value))
360 return ret
361 getRuntimeDepends.readonly = True
362
363 def getRuntimeRecommends(self, command, params):
364 ret = []
365 try:
366 mc = params[0]
367 except IndexError:
368 mc = ''
369 runrecs = command.cooker.recipecaches[mc].runrecs
370 for key, value in runrecs.items():
371 if isinstance(value, defaultdict):
372 value = dict(value)
373 ret.append((key, value))
374 return ret
375 getRuntimeRecommends.readonly = True
376
377 def getRecipeInherits(self, command, params):
378 try:
379 mc = params[0]
380 except IndexError:
381 mc = ''
382 return command.cooker.recipecaches[mc].inherits
383 getRecipeInherits.readonly = True
384
385 def getBbFilePriority(self, command, params):
386 try:
387 mc = params[0]
388 except IndexError:
389 mc = ''
390 return command.cooker.recipecaches[mc].bbfile_priority
391 getBbFilePriority.readonly = True
392
393 def getDefaultPreference(self, command, params):
394 try:
395 mc = params[0]
396 except IndexError:
397 mc = ''
398 return command.cooker.recipecaches[mc].pkg_dp
399 getDefaultPreference.readonly = True
400
401 def getSkippedRecipes(self, command, params):
402 # Return list sorted by reverse priority order
403 import bb.cache
404 skipdict = OrderedDict(sorted(command.cooker.skiplist.items(),
405 key=lambda x: (-command.cooker.collection.calc_bbfile_priority(bb.cache.virtualfn2realfn(x[0])[0]), x[0])))
406 return list(skipdict.items())
407 getSkippedRecipes.readonly = True
408
409 def getOverlayedRecipes(self, command, params):
410 return list(command.cooker.collection.overlayed.items())
411 getOverlayedRecipes.readonly = True
412
413 def getFileAppends(self, command, params):
414 fn = params[0]
415 return command.cooker.collection.get_file_appends(fn)
416 getFileAppends.readonly = True
417
418 def getAllAppends(self, command, params):
419 return command.cooker.collection.bbappends
420 getAllAppends.readonly = True
421
422 def findProviders(self, command, params):
423 return command.cooker.findProviders()
424 findProviders.readonly = True
425
426 def findBestProvider(self, command, params):
427 pn = params[0]
428 return command.cooker.findBestProvider(pn)
429 findBestProvider.readonly = True
430
431 def allProviders(self, command, params):
432 try:
433 mc = params[0]
434 except IndexError:
435 mc = ''
436 return list(bb.providers.allProviders(command.cooker.recipecaches[mc]).items())
437 allProviders.readonly = True
438
439 def getRuntimeProviders(self, command, params):
440 rprovide = params[0]
441 try:
442 mc = params[1]
443 except IndexError:
444 mc = ''
445 all_p = bb.providers.getRuntimeProviders(command.cooker.recipecaches[mc], rprovide)
446 if all_p:
447 best = bb.providers.filterProvidersRunTime(all_p, rprovide,
448 command.cooker.data,
449 command.cooker.recipecaches[mc])[0][0]
450 else:
451 best = None
452 return all_p, best
453 getRuntimeProviders.readonly = True
454
455 def dataStoreConnectorFindVar(self, command, params):
456 dsindex = params[0]
457 name = params[1]
458 datastore = command.remotedatastores[dsindex]
459 value = datastore._findVar(name)
460
461 if value:
462 content = value.get('_content', None)
463 if isinstance(content, bb.data_smart.DataSmart):
464 # Value is a datastore (e.g. BB_ORIGENV) - need to handle this carefully
465 idx = command.remotedatastores.check_store(content, True)
466 return {'_content': DataStoreConnectionHandle(idx), '_connector_origtype': 'DataStoreConnectionHandle'}
467 elif isinstance(content, set):
468 return {'_content': list(content), '_connector_origtype': 'set'}
469 return value
470 dataStoreConnectorFindVar.readonly = True
471
472 def dataStoreConnectorGetKeys(self, command, params):
473 dsindex = params[0]
474 datastore = command.remotedatastores[dsindex]
475 return list(datastore.keys())
476 dataStoreConnectorGetKeys.readonly = True
477
478 def dataStoreConnectorGetVarHistory(self, command, params):
479 dsindex = params[0]
480 name = params[1]
481 datastore = command.remotedatastores[dsindex]
482 return datastore.varhistory.variable(name)
483 dataStoreConnectorGetVarHistory.readonly = True
484
485 def dataStoreConnectorExpandPythonRef(self, command, params):
486 dsindex = params[0]
487 varname = params[1]
488 expr = params[2]
489 if dsindex:
490 datastore = self.dataStores[dsindex]
491 else:
492 datastore = command.cooker.data
493 varparse = bb.data_smart.VariableParse(varname, datastore)
494 return varparse.python_sub(expr)
495
301class CommandsAsync: 496class CommandsAsync:
302 """ 497 """
303 A class of asynchronous commands 498 A class of asynchronous commands
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 2614c4485a..48904a52d6 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -583,13 +583,12 @@ class BBCooker:
583 583
584 def showVersions(self): 584 def showVersions(self):
585 585
586 pkg_pn = self.recipecaches[''].pkg_pn 586 (latest_versions, preferred_versions) = self.findProviders()
587 (latest_versions, preferred_versions) = bb.providers.findProviders(self.data, self.recipecaches[''], pkg_pn)
588 587
589 logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version") 588 logger.plain("%-35s %25s %25s", "Recipe Name", "Latest Version", "Preferred Version")
590 logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================") 589 logger.plain("%-35s %25s %25s\n", "===========", "==============", "=================")
591 590
592 for p in sorted(pkg_pn): 591 for p in sorted(self.recipecaches[''].pkg_pn):
593 pref = preferred_versions[p] 592 pref = preferred_versions[p]
594 latest = latest_versions[p] 593 latest = latest_versions[p]
595 594
@@ -1084,6 +1083,20 @@ class BBCooker:
1084 if matches: 1083 if matches:
1085 bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data) 1084 bb.event.fire(bb.event.FilesMatchingFound(filepattern, matches), self.data)
1086 1085
1086 def findProviders(self, mc=''):
1087 return bb.providers.findProviders(self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
1088
1089 def findBestProvider(self, pn, mc=''):
1090 if pn in self.recipecaches[mc].providers:
1091 filenames = self.recipecaches[mc].providers[pn]
1092 eligible, foundUnique = bb.providers.filterProviders(filenames, pn, self.expanded_data, self.recipecaches[mc])
1093 filename = eligible[0]
1094 return None, None, None, filename
1095 elif pn in self.recipecaches[mc].pkg_pn:
1096 return bb.providers.findBestProvider(pn, self.data, self.recipecaches[mc], self.recipecaches[mc].pkg_pn)
1097 else:
1098 return None, None, None, None
1099
1087 def findConfigFiles(self, varname): 1100 def findConfigFiles(self, varname):
1088 """ 1101 """
1089 Find config files which are appropriate values for varname. 1102 Find config files which are appropriate values for varname.
diff --git a/bitbake/lib/bb/main.py b/bitbake/lib/bb/main.py
index a544c0aecb..443f5ec2fd 100755
--- a/bitbake/lib/bb/main.py
+++ b/bitbake/lib/bb/main.py
@@ -389,12 +389,8 @@ def bitbake_main(configParams, configuration):
389 except: 389 except:
390 pass 390 pass
391 391
392
393 configuration.setConfigParameters(configParams) 392 configuration.setConfigParameters(configParams)
394 393
395 ui_module = import_extension_module(bb.ui, configParams.ui, 'main')
396 servermodule = import_extension_module(bb.server, configParams.servertype, 'BitBakeServer')
397
398 if configParams.server_only: 394 if configParams.server_only:
399 if configParams.servertype != "xmlrpc": 395 if configParams.servertype != "xmlrpc":
400 raise BBMainException("FATAL: If '--server-only' is defined, we must set the " 396 raise BBMainException("FATAL: If '--server-only' is defined, we must set the "
@@ -442,6 +438,31 @@ def bitbake_main(configParams, configuration):
442 bb.msg.init_msgconfig(configParams.verbose, configuration.debug, 438 bb.msg.init_msgconfig(configParams.verbose, configuration.debug,
443 configuration.debug_domains) 439 configuration.debug_domains)
444 440
441 server, server_connection, ui_module = setup_bitbake(configParams, configuration)
442 if server_connection is None and configParams.kill_server:
443 return 0
444
445 if not configParams.server_only:
446 if configParams.status_only:
447 server_connection.terminate()
448 return 0
449
450 try:
451 return ui_module.main(server_connection.connection, server_connection.events,
452 configParams)
453 finally:
454 bb.event.ui_queue = []
455 server_connection.terminate()
456 else:
457 print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host,
458 server.serverImpl.port))
459 if configParams.foreground:
460 server.serverImpl.serve_forever()
461 return 0
462
463 return 1
464
465def setup_bitbake(configParams, configuration, extrafeatures=None):
445 # Ensure logging messages get sent to the UI as events 466 # Ensure logging messages get sent to the UI as events
446 handler = bb.event.LogHandler() 467 handler = bb.event.LogHandler()
447 if not configParams.status_only: 468 if not configParams.status_only:
@@ -451,8 +472,11 @@ def bitbake_main(configParams, configuration):
451 # Clear away any spurious environment variables while we stoke up the cooker 472 # Clear away any spurious environment variables while we stoke up the cooker
452 cleanedvars = bb.utils.clean_environment() 473 cleanedvars = bb.utils.clean_environment()
453 474
454 featureset = [] 475 if configParams.server_only:
455 if not configParams.server_only: 476 featureset = []
477 ui_module = None
478 else:
479 ui_module = import_extension_module(bb.ui, configParams.ui, 'main')
456 # Collect the feature set for the UI 480 # Collect the feature set for the UI
457 featureset = getattr(ui_module, "featureSet", []) 481 featureset = getattr(ui_module, "featureSet", [])
458 482
@@ -463,11 +487,15 @@ def bitbake_main(configParams, configuration):
463 setattr(configuration, "%s_server" % param, value) 487 setattr(configuration, "%s_server" % param, value)
464 param = "%s_server" % param 488 param = "%s_server" % param
465 489
466 if not configParams.remote_server: 490 if extrafeatures:
467 # we start a server with a given configuration 491 for feature in extrafeatures:
468 server = start_server(servermodule, configParams, configuration, featureset) 492 if not feature in featureset:
469 bb.event.ui_queue = [] 493 featureset.append(feature)
470 else: 494
495 servermodule = import_extension_module(bb.server,
496 configParams.servertype,
497 'BitBakeServer')
498 if configParams.remote_server:
471 if os.getenv('BBSERVER') == 'autostart': 499 if os.getenv('BBSERVER') == 'autostart':
472 if configParams.remote_server == 'autostart' or \ 500 if configParams.remote_server == 'autostart' or \
473 not servermodule.check_connection(configParams.remote_server, timeout=2): 501 not servermodule.check_connection(configParams.remote_server, timeout=2):
@@ -475,14 +503,19 @@ def bitbake_main(configParams, configuration):
475 srv = start_server(servermodule, configParams, configuration, featureset) 503 srv = start_server(servermodule, configParams, configuration, featureset)
476 configParams.remote_server = '%s:%d' % tuple(configuration.interface) 504 configParams.remote_server = '%s:%d' % tuple(configuration.interface)
477 bb.event.ui_queue = [] 505 bb.event.ui_queue = []
478
479 # we start a stub server that is actually a XMLRPClient that connects to a real server 506 # we start a stub server that is actually a XMLRPClient that connects to a real server
507 from bb.server.xmlrpc import BitBakeXMLRPCClient
480 server = servermodule.BitBakeXMLRPCClient(configParams.observe_only, 508 server = servermodule.BitBakeXMLRPCClient(configParams.observe_only,
481 configParams.xmlrpctoken) 509 configParams.xmlrpctoken)
482 server.saveConnectionDetails(configParams.remote_server) 510 server.saveConnectionDetails(configParams.remote_server)
511 else:
512 # we start a server with a given configuration
513 server = start_server(servermodule, configParams, configuration, featureset)
514 bb.event.ui_queue = []
483 515
484 516 if configParams.server_only:
485 if not configParams.server_only: 517 server_connection = None
518 else:
486 try: 519 try:
487 server_connection = server.establishConnection(featureset) 520 server_connection = server.establishConnection(featureset)
488 except Exception as e: 521 except Exception as e:
@@ -491,7 +524,7 @@ def bitbake_main(configParams, configuration):
491 if configParams.kill_server: 524 if configParams.kill_server:
492 server_connection.connection.terminateServer() 525 server_connection.connection.terminateServer()
493 bb.event.ui_queue = [] 526 bb.event.ui_queue = []
494 return 0 527 return None, None, None
495 528
496 server_connection.setupEventQueue() 529 server_connection.setupEventQueue()
497 530
@@ -501,22 +534,4 @@ def bitbake_main(configParams, configuration):
501 534
502 logger.removeHandler(handler) 535 logger.removeHandler(handler)
503 536
504 537 return server, server_connection, ui_module
505 if configParams.status_only:
506 server_connection.terminate()
507 return 0
508
509 try:
510 return ui_module.main(server_connection.connection, server_connection.events,
511 configParams)
512 finally:
513 bb.event.ui_queue = []
514 server_connection.terminate()
515 else:
516 print("Bitbake server address: %s, server port: %s" % (server.serverImpl.host,
517 server.serverImpl.port))
518 if configParams.foreground:
519 server.serverImpl.serve_forever()
520 return 0
521
522 return 1
diff --git a/bitbake/lib/bb/remotedata.py b/bitbake/lib/bb/remotedata.py
new file mode 100644
index 0000000000..932ee430ea
--- /dev/null
+++ b/bitbake/lib/bb/remotedata.py
@@ -0,0 +1,74 @@
1"""
2BitBake 'remotedata' module
3
4Provides support for using a datastore from the bitbake client
5"""
6
7# Copyright (C) 2016 Intel Corporation
8#
9# This program is free software; you can redistribute it and/or modify
10# it under the terms of the GNU General Public License version 2 as
11# published by the Free Software Foundation.
12#
13# This program is distributed in the hope that it will be useful,
14# but WITHOUT ANY WARRANTY; without even the implied warranty of
15# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16# GNU General Public License for more details.
17#
18# You should have received a copy of the GNU General Public License along
19# with this program; if not, write to the Free Software Foundation, Inc.,
20# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
21
22import bb.data
23
24class RemoteDatastores:
25 """Used on the server side to manage references to server-side datastores"""
26 def __init__(self, cooker):
27 self.cooker = cooker
28 self.datastores = {}
29 self.locked = []
30 self.nextindex = 1
31
32 def __len__(self):
33 return len(self.datastores)
34
35 def __getitem__(self, key):
36 if key is None:
37 return self.cooker.data
38 else:
39 return self.datastores[key]
40
41 def items(self):
42 return self.datastores.items()
43
44 def store(self, d, locked=False):
45 """
46 Put a datastore into the collection. If locked=True then the datastore
47 is understood to be managed externally and cannot be released by calling
48 release().
49 """
50 idx = self.nextindex
51 self.datastores[idx] = d
52 if locked:
53 self.locked.append(idx)
54 self.nextindex += 1
55 return idx
56
57 def check_store(self, d, locked=False):
58 """
59 Put a datastore into the collection if it's not already in there;
60 in either case return the index
61 """
62 for key, val in self.datastores.items():
63 if val is d:
64 idx = key
65 break
66 else:
67 idx = self.store(d, locked)
68 return idx
69
70 def release(self, idx):
71 """Discard a datastore in the collection"""
72 if idx in self.locked:
73 raise Exception('Tried to release locked datastore %d' % idx)
74 del self.datastores[idx]
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py
index 8899e861c3..459f6c1286 100644
--- a/bitbake/lib/bb/tinfoil.py
+++ b/bitbake/lib/bb/tinfoil.py
@@ -1,6 +1,6 @@
1# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities 1# tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities
2# 2#
3# Copyright (C) 2012 Intel Corporation 3# Copyright (C) 2012-2016 Intel Corporation
4# Copyright (C) 2011 Mentor Graphics Corporation 4# Copyright (C) 2011 Mentor Graphics Corporation
5# 5#
6# This program is free software; you can redistribute it and/or modify 6# This program is free software; you can redistribute it and/or modify
@@ -17,47 +17,172 @@
17# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 17# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18 18
19import logging 19import logging
20import warnings
21import os 20import os
22import sys 21import sys
22import atexit
23import re
24from collections import OrderedDict, defaultdict
23 25
24import bb.cache 26import bb.cache
25import bb.cooker 27import bb.cooker
26import bb.providers 28import bb.providers
27import bb.utils 29import bb.utils
28from bb.cooker import state, BBCooker, CookerFeatures 30import bb.command
29from bb.cookerdata import CookerConfiguration, ConfigParameters 31from bb.cookerdata import CookerConfiguration, ConfigParameters
32from bb.main import setup_bitbake, BitBakeConfigParameters, BBMainException
30import bb.fetch2 33import bb.fetch2
31 34
35
36# We need this in order to shut down the connection to the bitbake server,
37# otherwise the process will never properly exit
38_server_connections = []
39def _terminate_connections():
40 for connection in _server_connections:
41 connection.terminate()
42atexit.register(_terminate_connections)
43
44class TinfoilUIException(Exception):
45 """Exception raised when the UI returns non-zero from its main function"""
46 def __init__(self, returncode):
47 self.returncode = returncode
48 def __repr__(self):
49 return 'UI module main returned %d' % self.returncode
50
51class TinfoilCommandFailed(Exception):
52 """Exception raised when run_command fails"""
53
54class TinfoilDataStoreConnector:
55
56 def __init__(self, tinfoil, dsindex):
57 self.tinfoil = tinfoil
58 self.dsindex = dsindex
59 def getVar(self, name):
60 value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name)
61 if isinstance(value, dict):
62 if '_connector_origtype' in value:
63 value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype'])
64 del value['_connector_origtype']
65
66 return value
67 def getKeys(self):
68 return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex))
69 def getVarHistory(self, name):
70 return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name)
71 def expandPythonRef(self, varname, expr):
72 ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', self.dsindex, varname, expr)
73 return ret
74 def setVar(self, varname, value):
75 if self.dsindex is None:
76 self.tinfoil.run_command('setVariable', varname, value)
77 else:
78 # Not currently implemented - indicate that setting should
79 # be redirected to local side
80 return True
81
82class TinfoilCookerAdapter:
83 """
84 Provide an adapter for existing code that expects to access a cooker object via Tinfoil,
85 since now Tinfoil is on the client side it no longer has direct access.
86 """
87
88 class TinfoilCookerCollectionAdapter:
89 """ cooker.collection adapter """
90 def __init__(self, tinfoil):
91 self.tinfoil = tinfoil
92 def get_file_appends(self, fn):
93 return self.tinfoil.run_command('getFileAppends', fn)
94 def __getattr__(self, name):
95 if name == 'overlayed':
96 return self.tinfoil.get_overlayed_recipes()
97 elif name == 'bbappends':
98 return self.tinfoil.run_command('getAllAppends')
99 else:
100 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
101
102 class TinfoilRecipeCacheAdapter:
103 """ cooker.recipecache adapter """
104 def __init__(self, tinfoil):
105 self.tinfoil = tinfoil
106 self._cache = {}
107
108 def get_pkg_pn_fn(self):
109 pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes') or [])
110 pkg_fn = {}
111 for pn, fnlist in pkg_pn.items():
112 for fn in fnlist:
113 pkg_fn[fn] = pn
114 self._cache['pkg_pn'] = pkg_pn
115 self._cache['pkg_fn'] = pkg_fn
116
117 def __getattr__(self, name):
118 # Grab these only when they are requested since they aren't always used
119 if name in self._cache:
120 return self._cache[name]
121 elif name == 'pkg_pn':
122 self.get_pkg_pn_fn()
123 return self._cache[name]
124 elif name == 'pkg_fn':
125 self.get_pkg_pn_fn()
126 return self._cache[name]
127 elif name == 'deps':
128 attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends') or [])
129 elif name == 'rundeps':
130 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends') or [])
131 elif name == 'runrecs':
132 attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends') or [])
133 elif name == 'pkg_pepvpr':
134 attrvalue = self.tinfoil.run_command('getRecipeVersions') or {}
135 elif name == 'inherits':
136 attrvalue = self.tinfoil.run_command('getRecipeInherits') or {}
137 elif name == 'bbfile_priority':
138 attrvalue = self.tinfoil.run_command('getBbFilePriority') or {}
139 elif name == 'pkg_dp':
140 attrvalue = self.tinfoil.run_command('getDefaultPreference') or {}
141 else:
142 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
143
144 self._cache[name] = attrvalue
145 return attrvalue
146
147 def __init__(self, tinfoil):
148 self.tinfoil = tinfoil
149 self.collection = self.TinfoilCookerCollectionAdapter(tinfoil)
150 self.recipecaches = {}
151 # FIXME all machines
152 self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil)
153 self._cache = {}
154 def __getattr__(self, name):
155 # Grab these only when they are requested since they aren't always used
156 if name in self._cache:
157 return self._cache[name]
158 elif name == 'skiplist':
159 attrvalue = self.tinfoil.get_skipped_recipes()
160 elif name == 'bbfile_config_priorities':
161 ret = self.tinfoil.run_command('getLayerPriorities')
162 bbfile_config_priorities = []
163 for collection, pattern, regex, pri in ret:
164 bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri))
165
166 attrvalue = bbfile_config_priorities
167 else:
168 raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name))
169
170 self._cache[name] = attrvalue
171 return attrvalue
172
173 def findBestProvider(self, pn):
174 return self.tinfoil.find_best_provider(pn)
175
176
32class Tinfoil: 177class Tinfoil:
33 def __init__(self, output=sys.stdout, tracking=False):
34 # Needed to avoid deprecation warnings with python 2.6
35 warnings.filterwarnings("ignore", category=DeprecationWarning)
36 178
37 # Set up logging 179 def __init__(self, output=sys.stdout, tracking=False):
38 self.logger = logging.getLogger('BitBake') 180 self.logger = logging.getLogger('BitBake')
39 self._log_hdlr = logging.StreamHandler(output) 181 self.config_data = None
40 bb.msg.addDefaultlogFilter(self._log_hdlr) 182 self.cooker = None
41 format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") 183 self.tracking = tracking
42 if output.isatty(): 184 self.ui_module = None
43 format.enable_color() 185 self.server_connection = None
44 self._log_hdlr.setFormatter(format)
45 self.logger.addHandler(self._log_hdlr)
46
47 self.config = CookerConfiguration()
48 configparams = TinfoilConfigParameters(parse_only=True)
49 self.config.setConfigParameters(configparams)
50 self.config.setServerRegIdleCallback(self.register_idle_function)
51 features = []
52 if tracking:
53 features.append(CookerFeatures.BASEDATASTORE_TRACKING)
54 self.cooker = BBCooker(self.config, features)
55 self.config_data = self.cooker.data
56 bb.providers.logger.setLevel(logging.ERROR)
57 self.cooker_data = None
58
59 def register_idle_function(self, function, data):
60 pass
61 186
62 def __enter__(self): 187 def __enter__(self):
63 return self 188 return self
@@ -65,30 +190,120 @@ class Tinfoil:
65 def __exit__(self, type, value, traceback): 190 def __exit__(self, type, value, traceback):
66 self.shutdown() 191 self.shutdown()
67 192
68 def parseRecipes(self): 193 def prepare(self, config_only=False, config_params=None, quiet=0):
69 sys.stderr.write("Parsing recipes..") 194 if self.tracking:
70 self.logger.setLevel(logging.WARNING) 195 extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING]
196 else:
197 extrafeatures = []
71 198
72 try: 199 if not config_params:
73 while self.cooker.state in (state.initial, state.parsing): 200 config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet)
74 self.cooker.updateCache()
75 except KeyboardInterrupt:
76 self.cooker.shutdown()
77 self.cooker.updateCache()
78 sys.exit(2)
79 201
80 self.logger.setLevel(logging.INFO) 202 cookerconfig = CookerConfiguration()
81 sys.stderr.write("done.\n") 203 cookerconfig.setConfigParameters(config_params)
82 204
83 self.cooker_data = self.cooker.recipecaches[''] 205 server, self.server_connection, ui_module = setup_bitbake(config_params,
206 cookerconfig,
207 extrafeatures)
84 208
85 def prepare(self, config_only = False): 209 self.ui_module = ui_module
86 if not self.cooker_data: 210
211 if self.server_connection:
212 _server_connections.append(self.server_connection)
87 if config_only: 213 if config_only:
88 self.cooker.parseConfiguration() 214 config_params.updateToServer(self.server_connection.connection, os.environ.copy())
89 self.cooker_data = self.cooker.recipecaches[''] 215 self.run_command('parseConfiguration')
90 else: 216 else:
91 self.parseRecipes() 217 self.run_actions(config_params)
218
219 self.config_data = bb.data.init()
220 connector = TinfoilDataStoreConnector(self, None)
221 self.config_data.setVar('_remote_data', connector)
222 self.cooker = TinfoilCookerAdapter(self)
223 self.cooker_data = self.cooker.recipecaches['']
224 else:
225 raise Exception('Failed to start bitbake server')
226
227 def run_actions(self, config_params):
228 """
229 Run the actions specified in config_params through the UI.
230 """
231 ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params)
232 if ret:
233 raise TinfoilUIException(ret)
234
235 def parseRecipes(self):
236 """
237 Force a parse of all recipes. Normally you should specify
238 config_only=False when calling prepare() instead of using this
239 function; this function is designed for situations where you need
240 to initialise Tinfoil and use it with config_only=True first and
241 then conditionally call this function to parse recipes later.
242 """
243 config_params = TinfoilConfigParameters(config_only=False)
244 self.run_actions(config_params)
245
246 def run_command(self, command, *params):
247 """
248 Run a command on the server (as implemented in bb.command).
249 Note that there are two types of command - synchronous and
250 asynchronous; in order to receive the results of asynchronous
251 commands you will need to set an appropriate event mask
252 using set_event_mask() and listen for the result using
253 wait_event() - with the correct event mask you'll at least get
254 bb.command.CommandCompleted and possibly other events before
255 that depending on the command.
256 """
257 if not self.server_connection:
258 raise Exception('Not connected to server (did you call .prepare()?)')
259
260 commandline = [command]
261 if params:
262 commandline.extend(params)
263 result = self.server_connection.connection.runCommand(commandline)
264 if result[1]:
265 raise TinfoilCommandFailed(result[1])
266 return result[0]
267
268 def set_event_mask(self, eventlist):
269 """Set the event mask which will be applied within wait_event()"""
270 if not self.server_connection:
271 raise Exception('Not connected to server (did you call .prepare()?)')
272 llevel, debug_domains = bb.msg.constructLogOptions()
273 ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist)
274 if not ret:
275 raise Exception('setEventMask failed')
276
277 def wait_event(self, timeout=0):
278 """
279 Wait for an event from the server for the specified time.
280 A timeout of 0 means don't wait if there are no events in the queue.
281 Returns the next event in the queue or None if the timeout was
282 reached. Note that in order to recieve any events you will
283 first need to set the internal event mask using set_event_mask()
284 (otherwise whatever event mask the UI set up will be in effect).
285 """
286 if not self.server_connection:
287 raise Exception('Not connected to server (did you call .prepare()?)')
288 return self.server_connection.events.waitEvent(timeout)
289
290 def get_overlayed_recipes(self):
291 return defaultdict(list, self.run_command('getOverlayedRecipes'))
292
293 def get_skipped_recipes(self):
294 return OrderedDict(self.run_command('getSkippedRecipes'))
295
296 def get_all_providers(self):
297 return defaultdict(list, self.run_command('allProviders'))
298
299 def find_providers(self):
300 return self.run_command('findProviders')
301
302 def find_best_provider(self, pn):
303 return self.run_command('findBestProvider', pn)
304
305 def get_runtime_providers(self, rdep):
306 return self.run_command('getRuntimeProviders', rdep)
92 307
93 def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None): 308 def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None):
94 """ 309 """
@@ -126,22 +341,72 @@ class Tinfoil:
126 envdata = parser.loadDataFull(fn, appendfiles) 341 envdata = parser.loadDataFull(fn, appendfiles)
127 return envdata 342 return envdata
128 343
344 def build_file(self, buildfile, task):
345 """
346 Runs the specified task for just a single recipe (i.e. no dependencies).
347 This is equivalent to bitbake -b.
348 """
349 return self.run_command('buildFile', buildfile, task)
350
129 def shutdown(self): 351 def shutdown(self):
130 self.cooker.shutdown(force=True) 352 if self.server_connection:
131 self.cooker.post_serve() 353 self.run_command('clientComplete')
132 self.cooker.unlockBitbake() 354 _server_connections.remove(self.server_connection)
133 self.logger.removeHandler(self._log_hdlr) 355 bb.event.ui_queue = []
356 self.server_connection.terminate()
357 self.server_connection = None
134 358
135class TinfoilConfigParameters(ConfigParameters): 359 def _reconvert_type(self, obj, origtypename):
360 """
361 Convert an object back to the right type, in the case
362 that marshalling has changed it (especially with xmlrpc)
363 """
364 supported_types = {
365 'set': set,
366 'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle,
367 }
136 368
137 def __init__(self, **options): 369 origtype = supported_types.get(origtypename, None)
370 if origtype is None:
371 raise Exception('Unsupported type "%s"' % origtypename)
372 if type(obj) == origtype:
373 newobj = obj
374 elif isinstance(obj, dict):
375 # New style class
376 newobj = origtype()
377 for k,v in obj.items():
378 setattr(newobj, k, v)
379 else:
380 # Assume we can coerce the type
381 newobj = origtype(obj)
382
383 if isinstance(newobj, bb.command.DataStoreConnectionHandle):
384 connector = TinfoilDataStoreConnector(self, newobj.dsindex)
385 newobj = bb.data.init()
386 newobj.setVar('_remote_data', connector)
387
388 return newobj
389
390
391class TinfoilConfigParameters(BitBakeConfigParameters):
392
393 def __init__(self, config_only, **options):
138 self.initial_options = options 394 self.initial_options = options
139 super(TinfoilConfigParameters, self).__init__() 395 # Apply some sane defaults
396 if not 'parse_only' in options:
397 self.initial_options['parse_only'] = not config_only
398 #if not 'status_only' in options:
399 # self.initial_options['status_only'] = config_only
400 if not 'ui' in options:
401 self.initial_options['ui'] = 'knotty'
402 if not 'argv' in options:
403 self.initial_options['argv'] = []
140 404
141 def parseCommandLine(self, argv=sys.argv): 405 super(TinfoilConfigParameters, self).__init__()
142 class DummyOptions:
143 def __init__(self, initial_options):
144 for key, val in initial_options.items():
145 setattr(self, key, val)
146 406
147 return DummyOptions(self.initial_options), None 407 def parseCommandLine(self, argv=None):
408 # We don't want any parameters parsed from the command line
409 opts = super(TinfoilConfigParameters, self).parseCommandLine([])
410 for key, val in self.initial_options.items():
411 setattr(opts[0], key, val)
412 return opts
diff --git a/bitbake/lib/bblayers/query.py b/bitbake/lib/bblayers/query.py
index 29491163c2..5def7179ce 100644
--- a/bitbake/lib/bblayers/query.py
+++ b/bitbake/lib/bblayers/query.py
@@ -5,8 +5,6 @@ import sys
5import os 5import os
6import re 6import re
7 7
8import bb.cache
9import bb.providers
10import bb.utils 8import bb.utils
11 9
12from bblayers.common import LayerPlugin 10from bblayers.common import LayerPlugin
@@ -122,15 +120,13 @@ skipped recipes will also be listed, with a " (skipped)" suffix.
122 sys.exit(1) 120 sys.exit(1)
123 121
124 pkg_pn = self.tinfoil.cooker.recipecaches[''].pkg_pn 122 pkg_pn = self.tinfoil.cooker.recipecaches[''].pkg_pn
125 (latest_versions, preferred_versions) = bb.providers.findProviders(self.tinfoil.config_data, self.tinfoil.cooker.recipecaches[''], pkg_pn) 123 (latest_versions, preferred_versions) = self.tinfoil.find_providers()
126 allproviders = bb.providers.allProviders(self.tinfoil.cooker.recipecaches['']) 124 allproviders = self.tinfoil.get_all_providers()
127 125
128 # Ensure we list skipped recipes 126 # Ensure we list skipped recipes
129 # We are largely guessing about PN, PV and the preferred version here, 127 # We are largely guessing about PN, PV and the preferred version here,
130 # but we have no choice since skipped recipes are not fully parsed 128 # but we have no choice since skipped recipes are not fully parsed
131 skiplist = list(self.tinfoil.cooker.skiplist.keys()) 129 skiplist = list(self.tinfoil.cooker.skiplist.keys())
132 skiplist.sort( key=lambda fileitem: self.tinfoil.cooker.collection.calc_bbfile_priority(fileitem) )
133 skiplist.reverse()
134 for fn in skiplist: 130 for fn in skiplist:
135 recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_') 131 recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_')
136 p = recipe_parts[0] 132 p = recipe_parts[0]
@@ -265,10 +261,7 @@ Lists recipes with the bbappends that apply to them as subitems.
265 def show_appends_for_pn(self, pn): 261 def show_appends_for_pn(self, pn):
266 filenames = self.tinfoil.cooker_data.pkg_pn[pn] 262 filenames = self.tinfoil.cooker_data.pkg_pn[pn]
267 263
268 best = bb.providers.findBestProvider(pn, 264 best = self.tinfoil.find_best_provider(pn)
269 self.tinfoil.config_data,
270 self.tinfoil.cooker_data,
271 self.tinfoil.cooker_data.pkg_pn)
272 best_filename = os.path.basename(best[3]) 265 best_filename = os.path.basename(best[3])
273 266
274 return self.show_appends_output(filenames, best_filename) 267 return self.show_appends_output(filenames, best_filename)
@@ -336,10 +329,7 @@ NOTE: .bbappend files can impact the dependencies.
336 deps = self.tinfoil.cooker_data.deps[f] 329 deps = self.tinfoil.cooker_data.deps[f]
337 for pn in deps: 330 for pn in deps:
338 if pn in self.tinfoil.cooker_data.pkg_pn: 331 if pn in self.tinfoil.cooker_data.pkg_pn:
339 best = bb.providers.findBestProvider(pn, 332 best = self.tinfoil.find_best_provider(pn)
340 self.tinfoil.config_data,
341 self.tinfoil.cooker_data,
342 self.tinfoil.cooker_data.pkg_pn)
343 self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers) 333 self.check_cross_depends("DEPENDS", layername, f, best[3], args.filenames, ignore_layers)
344 334
345 # The RDPENDS 335 # The RDPENDS
@@ -352,14 +342,11 @@ NOTE: .bbappend files can impact the dependencies.
352 sorted_rdeps[k2] = 1 342 sorted_rdeps[k2] = 1
353 all_rdeps = sorted_rdeps.keys() 343 all_rdeps = sorted_rdeps.keys()
354 for rdep in all_rdeps: 344 for rdep in all_rdeps:
355 all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rdep) 345 all_p, best = self.tinfoil.get_runtime_providers(rdep)
356 if all_p: 346 if all_p:
357 if f in all_p: 347 if f in all_p:
358 # The recipe provides this one itself, ignore 348 # The recipe provides this one itself, ignore
359 continue 349 continue
360 best = bb.providers.filterProvidersRunTime(all_p, rdep,
361 self.tinfoil.config_data,
362 self.tinfoil.cooker_data)[0][0]
363 self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers) 350 self.check_cross_depends("RDEPENDS", layername, f, best, args.filenames, ignore_layers)
364 351
365 # The RRECOMMENDS 352 # The RRECOMMENDS
@@ -372,14 +359,11 @@ NOTE: .bbappend files can impact the dependencies.
372 sorted_rrecs[k2] = 1 359 sorted_rrecs[k2] = 1
373 all_rrecs = sorted_rrecs.keys() 360 all_rrecs = sorted_rrecs.keys()
374 for rrec in all_rrecs: 361 for rrec in all_rrecs:
375 all_p = bb.providers.getRuntimeProviders(self.tinfoil.cooker_data, rrec) 362 all_p, best = self.tinfoil.get_runtime_providers(rrec)
376 if all_p: 363 if all_p:
377 if f in all_p: 364 if f in all_p:
378 # The recipe provides this one itself, ignore 365 # The recipe provides this one itself, ignore
379 continue 366 continue
380 best = bb.providers.filterProvidersRunTime(all_p, rrec,
381 self.tinfoil.config_data,
382 self.tinfoil.cooker_data)[0][0]
383 self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers) 367 self.check_cross_depends("RRECOMMENDS", layername, f, best, args.filenames, ignore_layers)
384 368
385 # The inherit class 369 # The inherit class