diff options
Diffstat (limited to 'bitbake/lib/bb/tinfoil.py')
-rw-r--r-- | bitbake/lib/bb/tinfoil.py | 385 |
1 files changed, 325 insertions, 60 deletions
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py index 8899e861c3..459f6c1286 100644 --- a/bitbake/lib/bb/tinfoil.py +++ b/bitbake/lib/bb/tinfoil.py | |||
@@ -1,6 +1,6 @@ | |||
1 | # tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities | 1 | # tinfoil: a simple wrapper around cooker for bitbake-based command-line utilities |
2 | # | 2 | # |
3 | # Copyright (C) 2012 Intel Corporation | 3 | # Copyright (C) 2012-2016 Intel Corporation |
4 | # Copyright (C) 2011 Mentor Graphics Corporation | 4 | # Copyright (C) 2011 Mentor Graphics Corporation |
5 | # | 5 | # |
6 | # This program is free software; you can redistribute it and/or modify | 6 | # This program is free software; you can redistribute it and/or modify |
@@ -17,47 +17,172 @@ | |||
17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | 17 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
18 | 18 | ||
19 | import logging | 19 | import logging |
20 | import warnings | ||
21 | import os | 20 | import os |
22 | import sys | 21 | import sys |
22 | import atexit | ||
23 | import re | ||
24 | from collections import OrderedDict, defaultdict | ||
23 | 25 | ||
24 | import bb.cache | 26 | import bb.cache |
25 | import bb.cooker | 27 | import bb.cooker |
26 | import bb.providers | 28 | import bb.providers |
27 | import bb.utils | 29 | import bb.utils |
28 | from bb.cooker import state, BBCooker, CookerFeatures | 30 | import bb.command |
29 | from bb.cookerdata import CookerConfiguration, ConfigParameters | 31 | from bb.cookerdata import CookerConfiguration, ConfigParameters |
32 | from bb.main import setup_bitbake, BitBakeConfigParameters, BBMainException | ||
30 | import bb.fetch2 | 33 | import bb.fetch2 |
31 | 34 | ||
35 | |||
36 | # We need this in order to shut down the connection to the bitbake server, | ||
37 | # otherwise the process will never properly exit | ||
38 | _server_connections = [] | ||
39 | def _terminate_connections(): | ||
40 | for connection in _server_connections: | ||
41 | connection.terminate() | ||
42 | atexit.register(_terminate_connections) | ||
43 | |||
44 | class TinfoilUIException(Exception): | ||
45 | """Exception raised when the UI returns non-zero from its main function""" | ||
46 | def __init__(self, returncode): | ||
47 | self.returncode = returncode | ||
48 | def __repr__(self): | ||
49 | return 'UI module main returned %d' % self.returncode | ||
50 | |||
51 | class TinfoilCommandFailed(Exception): | ||
52 | """Exception raised when run_command fails""" | ||
53 | |||
54 | class TinfoilDataStoreConnector: | ||
55 | |||
56 | def __init__(self, tinfoil, dsindex): | ||
57 | self.tinfoil = tinfoil | ||
58 | self.dsindex = dsindex | ||
59 | def getVar(self, name): | ||
60 | value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name) | ||
61 | if isinstance(value, dict): | ||
62 | if '_connector_origtype' in value: | ||
63 | value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype']) | ||
64 | del value['_connector_origtype'] | ||
65 | |||
66 | return value | ||
67 | def getKeys(self): | ||
68 | return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex)) | ||
69 | def getVarHistory(self, name): | ||
70 | return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name) | ||
71 | def expandPythonRef(self, varname, expr): | ||
72 | ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', self.dsindex, varname, expr) | ||
73 | return ret | ||
74 | def setVar(self, varname, value): | ||
75 | if self.dsindex is None: | ||
76 | self.tinfoil.run_command('setVariable', varname, value) | ||
77 | else: | ||
78 | # Not currently implemented - indicate that setting should | ||
79 | # be redirected to local side | ||
80 | return True | ||
81 | |||
82 | class TinfoilCookerAdapter: | ||
83 | """ | ||
84 | Provide an adapter for existing code that expects to access a cooker object via Tinfoil, | ||
85 | since now Tinfoil is on the client side it no longer has direct access. | ||
86 | """ | ||
87 | |||
88 | class TinfoilCookerCollectionAdapter: | ||
89 | """ cooker.collection adapter """ | ||
90 | def __init__(self, tinfoil): | ||
91 | self.tinfoil = tinfoil | ||
92 | def get_file_appends(self, fn): | ||
93 | return self.tinfoil.run_command('getFileAppends', fn) | ||
94 | def __getattr__(self, name): | ||
95 | if name == 'overlayed': | ||
96 | return self.tinfoil.get_overlayed_recipes() | ||
97 | elif name == 'bbappends': | ||
98 | return self.tinfoil.run_command('getAllAppends') | ||
99 | else: | ||
100 | raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) | ||
101 | |||
102 | class TinfoilRecipeCacheAdapter: | ||
103 | """ cooker.recipecache adapter """ | ||
104 | def __init__(self, tinfoil): | ||
105 | self.tinfoil = tinfoil | ||
106 | self._cache = {} | ||
107 | |||
108 | def get_pkg_pn_fn(self): | ||
109 | pkg_pn = defaultdict(list, self.tinfoil.run_command('getRecipes') or []) | ||
110 | pkg_fn = {} | ||
111 | for pn, fnlist in pkg_pn.items(): | ||
112 | for fn in fnlist: | ||
113 | pkg_fn[fn] = pn | ||
114 | self._cache['pkg_pn'] = pkg_pn | ||
115 | self._cache['pkg_fn'] = pkg_fn | ||
116 | |||
117 | def __getattr__(self, name): | ||
118 | # Grab these only when they are requested since they aren't always used | ||
119 | if name in self._cache: | ||
120 | return self._cache[name] | ||
121 | elif name == 'pkg_pn': | ||
122 | self.get_pkg_pn_fn() | ||
123 | return self._cache[name] | ||
124 | elif name == 'pkg_fn': | ||
125 | self.get_pkg_pn_fn() | ||
126 | return self._cache[name] | ||
127 | elif name == 'deps': | ||
128 | attrvalue = defaultdict(list, self.tinfoil.run_command('getRecipeDepends') or []) | ||
129 | elif name == 'rundeps': | ||
130 | attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeDepends') or []) | ||
131 | elif name == 'runrecs': | ||
132 | attrvalue = defaultdict(lambda: defaultdict(list), self.tinfoil.run_command('getRuntimeRecommends') or []) | ||
133 | elif name == 'pkg_pepvpr': | ||
134 | attrvalue = self.tinfoil.run_command('getRecipeVersions') or {} | ||
135 | elif name == 'inherits': | ||
136 | attrvalue = self.tinfoil.run_command('getRecipeInherits') or {} | ||
137 | elif name == 'bbfile_priority': | ||
138 | attrvalue = self.tinfoil.run_command('getBbFilePriority') or {} | ||
139 | elif name == 'pkg_dp': | ||
140 | attrvalue = self.tinfoil.run_command('getDefaultPreference') or {} | ||
141 | else: | ||
142 | raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) | ||
143 | |||
144 | self._cache[name] = attrvalue | ||
145 | return attrvalue | ||
146 | |||
147 | def __init__(self, tinfoil): | ||
148 | self.tinfoil = tinfoil | ||
149 | self.collection = self.TinfoilCookerCollectionAdapter(tinfoil) | ||
150 | self.recipecaches = {} | ||
151 | # FIXME all machines | ||
152 | self.recipecaches[''] = self.TinfoilRecipeCacheAdapter(tinfoil) | ||
153 | self._cache = {} | ||
154 | def __getattr__(self, name): | ||
155 | # Grab these only when they are requested since they aren't always used | ||
156 | if name in self._cache: | ||
157 | return self._cache[name] | ||
158 | elif name == 'skiplist': | ||
159 | attrvalue = self.tinfoil.get_skipped_recipes() | ||
160 | elif name == 'bbfile_config_priorities': | ||
161 | ret = self.tinfoil.run_command('getLayerPriorities') | ||
162 | bbfile_config_priorities = [] | ||
163 | for collection, pattern, regex, pri in ret: | ||
164 | bbfile_config_priorities.append((collection, pattern, re.compile(regex), pri)) | ||
165 | |||
166 | attrvalue = bbfile_config_priorities | ||
167 | else: | ||
168 | raise AttributeError("%s instance has no attribute '%s'" % (self.__class__.__name__, name)) | ||
169 | |||
170 | self._cache[name] = attrvalue | ||
171 | return attrvalue | ||
172 | |||
173 | def findBestProvider(self, pn): | ||
174 | return self.tinfoil.find_best_provider(pn) | ||
175 | |||
176 | |||
32 | class Tinfoil: | 177 | class Tinfoil: |
33 | def __init__(self, output=sys.stdout, tracking=False): | ||
34 | # Needed to avoid deprecation warnings with python 2.6 | ||
35 | warnings.filterwarnings("ignore", category=DeprecationWarning) | ||
36 | 178 | ||
37 | # Set up logging | 179 | def __init__(self, output=sys.stdout, tracking=False): |
38 | self.logger = logging.getLogger('BitBake') | 180 | self.logger = logging.getLogger('BitBake') |
39 | self._log_hdlr = logging.StreamHandler(output) | 181 | self.config_data = None |
40 | bb.msg.addDefaultlogFilter(self._log_hdlr) | 182 | self.cooker = None |
41 | format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") | 183 | self.tracking = tracking |
42 | if output.isatty(): | 184 | self.ui_module = None |
43 | format.enable_color() | 185 | self.server_connection = None |
44 | self._log_hdlr.setFormatter(format) | ||
45 | self.logger.addHandler(self._log_hdlr) | ||
46 | |||
47 | self.config = CookerConfiguration() | ||
48 | configparams = TinfoilConfigParameters(parse_only=True) | ||
49 | self.config.setConfigParameters(configparams) | ||
50 | self.config.setServerRegIdleCallback(self.register_idle_function) | ||
51 | features = [] | ||
52 | if tracking: | ||
53 | features.append(CookerFeatures.BASEDATASTORE_TRACKING) | ||
54 | self.cooker = BBCooker(self.config, features) | ||
55 | self.config_data = self.cooker.data | ||
56 | bb.providers.logger.setLevel(logging.ERROR) | ||
57 | self.cooker_data = None | ||
58 | |||
59 | def register_idle_function(self, function, data): | ||
60 | pass | ||
61 | 186 | ||
62 | def __enter__(self): | 187 | def __enter__(self): |
63 | return self | 188 | return self |
@@ -65,30 +190,120 @@ class Tinfoil: | |||
65 | def __exit__(self, type, value, traceback): | 190 | def __exit__(self, type, value, traceback): |
66 | self.shutdown() | 191 | self.shutdown() |
67 | 192 | ||
68 | def parseRecipes(self): | 193 | def prepare(self, config_only=False, config_params=None, quiet=0): |
69 | sys.stderr.write("Parsing recipes..") | 194 | if self.tracking: |
70 | self.logger.setLevel(logging.WARNING) | 195 | extrafeatures = [bb.cooker.CookerFeatures.BASEDATASTORE_TRACKING] |
196 | else: | ||
197 | extrafeatures = [] | ||
71 | 198 | ||
72 | try: | 199 | if not config_params: |
73 | while self.cooker.state in (state.initial, state.parsing): | 200 | config_params = TinfoilConfigParameters(config_only=config_only, quiet=quiet) |
74 | self.cooker.updateCache() | ||
75 | except KeyboardInterrupt: | ||
76 | self.cooker.shutdown() | ||
77 | self.cooker.updateCache() | ||
78 | sys.exit(2) | ||
79 | 201 | ||
80 | self.logger.setLevel(logging.INFO) | 202 | cookerconfig = CookerConfiguration() |
81 | sys.stderr.write("done.\n") | 203 | cookerconfig.setConfigParameters(config_params) |
82 | 204 | ||
83 | self.cooker_data = self.cooker.recipecaches[''] | 205 | server, self.server_connection, ui_module = setup_bitbake(config_params, |
206 | cookerconfig, | ||
207 | extrafeatures) | ||
84 | 208 | ||
85 | def prepare(self, config_only = False): | 209 | self.ui_module = ui_module |
86 | if not self.cooker_data: | 210 | |
211 | if self.server_connection: | ||
212 | _server_connections.append(self.server_connection) | ||
87 | if config_only: | 213 | if config_only: |
88 | self.cooker.parseConfiguration() | 214 | config_params.updateToServer(self.server_connection.connection, os.environ.copy()) |
89 | self.cooker_data = self.cooker.recipecaches[''] | 215 | self.run_command('parseConfiguration') |
90 | else: | 216 | else: |
91 | self.parseRecipes() | 217 | self.run_actions(config_params) |
218 | |||
219 | self.config_data = bb.data.init() | ||
220 | connector = TinfoilDataStoreConnector(self, None) | ||
221 | self.config_data.setVar('_remote_data', connector) | ||
222 | self.cooker = TinfoilCookerAdapter(self) | ||
223 | self.cooker_data = self.cooker.recipecaches[''] | ||
224 | else: | ||
225 | raise Exception('Failed to start bitbake server') | ||
226 | |||
227 | def run_actions(self, config_params): | ||
228 | """ | ||
229 | Run the actions specified in config_params through the UI. | ||
230 | """ | ||
231 | ret = self.ui_module.main(self.server_connection.connection, self.server_connection.events, config_params) | ||
232 | if ret: | ||
233 | raise TinfoilUIException(ret) | ||
234 | |||
235 | def parseRecipes(self): | ||
236 | """ | ||
237 | Force a parse of all recipes. Normally you should specify | ||
238 | config_only=False when calling prepare() instead of using this | ||
239 | function; this function is designed for situations where you need | ||
240 | to initialise Tinfoil and use it with config_only=True first and | ||
241 | then conditionally call this function to parse recipes later. | ||
242 | """ | ||
243 | config_params = TinfoilConfigParameters(config_only=False) | ||
244 | self.run_actions(config_params) | ||
245 | |||
246 | def run_command(self, command, *params): | ||
247 | """ | ||
248 | Run a command on the server (as implemented in bb.command). | ||
249 | Note that there are two types of command - synchronous and | ||
250 | asynchronous; in order to receive the results of asynchronous | ||
251 | commands you will need to set an appropriate event mask | ||
252 | using set_event_mask() and listen for the result using | ||
253 | wait_event() - with the correct event mask you'll at least get | ||
254 | bb.command.CommandCompleted and possibly other events before | ||
255 | that depending on the command. | ||
256 | """ | ||
257 | if not self.server_connection: | ||
258 | raise Exception('Not connected to server (did you call .prepare()?)') | ||
259 | |||
260 | commandline = [command] | ||
261 | if params: | ||
262 | commandline.extend(params) | ||
263 | result = self.server_connection.connection.runCommand(commandline) | ||
264 | if result[1]: | ||
265 | raise TinfoilCommandFailed(result[1]) | ||
266 | return result[0] | ||
267 | |||
268 | def set_event_mask(self, eventlist): | ||
269 | """Set the event mask which will be applied within wait_event()""" | ||
270 | if not self.server_connection: | ||
271 | raise Exception('Not connected to server (did you call .prepare()?)') | ||
272 | llevel, debug_domains = bb.msg.constructLogOptions() | ||
273 | ret = self.run_command('setEventMask', self.server_connection.connection.getEventHandle(), llevel, debug_domains, eventlist) | ||
274 | if not ret: | ||
275 | raise Exception('setEventMask failed') | ||
276 | |||
277 | def wait_event(self, timeout=0): | ||
278 | """ | ||
279 | Wait for an event from the server for the specified time. | ||
280 | A timeout of 0 means don't wait if there are no events in the queue. | ||
281 | Returns the next event in the queue or None if the timeout was | ||
282 | reached. Note that in order to recieve any events you will | ||
283 | first need to set the internal event mask using set_event_mask() | ||
284 | (otherwise whatever event mask the UI set up will be in effect). | ||
285 | """ | ||
286 | if not self.server_connection: | ||
287 | raise Exception('Not connected to server (did you call .prepare()?)') | ||
288 | return self.server_connection.events.waitEvent(timeout) | ||
289 | |||
290 | def get_overlayed_recipes(self): | ||
291 | return defaultdict(list, self.run_command('getOverlayedRecipes')) | ||
292 | |||
293 | def get_skipped_recipes(self): | ||
294 | return OrderedDict(self.run_command('getSkippedRecipes')) | ||
295 | |||
296 | def get_all_providers(self): | ||
297 | return defaultdict(list, self.run_command('allProviders')) | ||
298 | |||
299 | def find_providers(self): | ||
300 | return self.run_command('findProviders') | ||
301 | |||
302 | def find_best_provider(self, pn): | ||
303 | return self.run_command('findBestProvider', pn) | ||
304 | |||
305 | def get_runtime_providers(self, rdep): | ||
306 | return self.run_command('getRuntimeProviders', rdep) | ||
92 | 307 | ||
93 | def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None): | 308 | def parse_recipe_file(self, fn, appends=True, appendlist=None, config_data=None): |
94 | """ | 309 | """ |
@@ -126,22 +341,72 @@ class Tinfoil: | |||
126 | envdata = parser.loadDataFull(fn, appendfiles) | 341 | envdata = parser.loadDataFull(fn, appendfiles) |
127 | return envdata | 342 | return envdata |
128 | 343 | ||
344 | def build_file(self, buildfile, task): | ||
345 | """ | ||
346 | Runs the specified task for just a single recipe (i.e. no dependencies). | ||
347 | This is equivalent to bitbake -b. | ||
348 | """ | ||
349 | return self.run_command('buildFile', buildfile, task) | ||
350 | |||
129 | def shutdown(self): | 351 | def shutdown(self): |
130 | self.cooker.shutdown(force=True) | 352 | if self.server_connection: |
131 | self.cooker.post_serve() | 353 | self.run_command('clientComplete') |
132 | self.cooker.unlockBitbake() | 354 | _server_connections.remove(self.server_connection) |
133 | self.logger.removeHandler(self._log_hdlr) | 355 | bb.event.ui_queue = [] |
356 | self.server_connection.terminate() | ||
357 | self.server_connection = None | ||
134 | 358 | ||
135 | class TinfoilConfigParameters(ConfigParameters): | 359 | def _reconvert_type(self, obj, origtypename): |
360 | """ | ||
361 | Convert an object back to the right type, in the case | ||
362 | that marshalling has changed it (especially with xmlrpc) | ||
363 | """ | ||
364 | supported_types = { | ||
365 | 'set': set, | ||
366 | 'DataStoreConnectionHandle': bb.command.DataStoreConnectionHandle, | ||
367 | } | ||
136 | 368 | ||
137 | def __init__(self, **options): | 369 | origtype = supported_types.get(origtypename, None) |
370 | if origtype is None: | ||
371 | raise Exception('Unsupported type "%s"' % origtypename) | ||
372 | if type(obj) == origtype: | ||
373 | newobj = obj | ||
374 | elif isinstance(obj, dict): | ||
375 | # New style class | ||
376 | newobj = origtype() | ||
377 | for k,v in obj.items(): | ||
378 | setattr(newobj, k, v) | ||
379 | else: | ||
380 | # Assume we can coerce the type | ||
381 | newobj = origtype(obj) | ||
382 | |||
383 | if isinstance(newobj, bb.command.DataStoreConnectionHandle): | ||
384 | connector = TinfoilDataStoreConnector(self, newobj.dsindex) | ||
385 | newobj = bb.data.init() | ||
386 | newobj.setVar('_remote_data', connector) | ||
387 | |||
388 | return newobj | ||
389 | |||
390 | |||
391 | class TinfoilConfigParameters(BitBakeConfigParameters): | ||
392 | |||
393 | def __init__(self, config_only, **options): | ||
138 | self.initial_options = options | 394 | self.initial_options = options |
139 | super(TinfoilConfigParameters, self).__init__() | 395 | # Apply some sane defaults |
396 | if not 'parse_only' in options: | ||
397 | self.initial_options['parse_only'] = not config_only | ||
398 | #if not 'status_only' in options: | ||
399 | # self.initial_options['status_only'] = config_only | ||
400 | if not 'ui' in options: | ||
401 | self.initial_options['ui'] = 'knotty' | ||
402 | if not 'argv' in options: | ||
403 | self.initial_options['argv'] = [] | ||
140 | 404 | ||
141 | def parseCommandLine(self, argv=sys.argv): | 405 | super(TinfoilConfigParameters, self).__init__() |
142 | class DummyOptions: | ||
143 | def __init__(self, initial_options): | ||
144 | for key, val in initial_options.items(): | ||
145 | setattr(self, key, val) | ||
146 | 406 | ||
147 | return DummyOptions(self.initial_options), None | 407 | def parseCommandLine(self, argv=None): |
408 | # We don't want any parameters parsed from the command line | ||
409 | opts = super(TinfoilConfigParameters, self).parseCommandLine([]) | ||
410 | for key, val in self.initial_options.items(): | ||
411 | setattr(opts[0], key, val) | ||
412 | return opts | ||