diff options
author | Adrian Dudau <adrian.dudau@enea.com> | 2013-12-12 13:38:32 +0100 |
---|---|---|
committer | Adrian Dudau <adrian.dudau@enea.com> | 2013-12-12 13:50:20 +0100 |
commit | e2e6f6fe07049f33cb6348780fa975162752e421 (patch) | |
tree | b1813295411235d1297a0ed642b1346b24fdfb12 /bitbake/bin | |
download | poky-e2e6f6fe07049f33cb6348780fa975162752e421.tar.gz |
initial commit of Enea Linux 3.1
Migrated from the internal git server on the dora-enea branch
Signed-off-by: Adrian Dudau <adrian.dudau@enea.com>
Diffstat (limited to 'bitbake/bin')
-rwxr-xr-x | bitbake/bin/bitbake | 334 | ||||
-rwxr-xr-x | bitbake/bin/bitbake-diffsigs | 122 | ||||
-rwxr-xr-x | bitbake/bin/bitbake-dumpsig | 65 | ||||
-rwxr-xr-x | bitbake/bin/bitbake-layers | 720 | ||||
-rwxr-xr-x | bitbake/bin/bitbake-prserv | 55 | ||||
-rwxr-xr-x | bitbake/bin/bitbake-selftest | 38 | ||||
-rwxr-xr-x | bitbake/bin/bitbake-worker | 363 | ||||
-rwxr-xr-x | bitbake/bin/bitdoc | 531 | ||||
-rwxr-xr-x | bitbake/bin/image-writer | 122 |
9 files changed, 2350 insertions, 0 deletions
diff --git a/bitbake/bin/bitbake b/bitbake/bin/bitbake new file mode 100755 index 0000000000..88893f8de6 --- /dev/null +++ b/bitbake/bin/bitbake | |||
@@ -0,0 +1,334 @@ | |||
1 | #!/usr/bin/env python | ||
2 | # ex:ts=4:sw=4:sts=4:et | ||
3 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | ||
4 | # | ||
5 | # Copyright (C) 2003, 2004 Chris Larson | ||
6 | # Copyright (C) 2003, 2004 Phil Blundell | ||
7 | # Copyright (C) 2003 - 2005 Michael 'Mickey' Lauer | ||
8 | # Copyright (C) 2005 Holger Hans Peter Freyther | ||
9 | # Copyright (C) 2005 ROAD GmbH | ||
10 | # Copyright (C) 2006 Richard Purdie | ||
11 | # | ||
12 | # This program is free software; you can redistribute it and/or modify | ||
13 | # it under the terms of the GNU General Public License version 2 as | ||
14 | # published by the Free Software Foundation. | ||
15 | # | ||
16 | # This program is distributed in the hope that it will be useful, | ||
17 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
18 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
19 | # GNU General Public License for more details. | ||
20 | # | ||
21 | # You should have received a copy of the GNU General Public License along | ||
22 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
23 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
24 | |||
25 | import os | ||
26 | import sys, logging | ||
27 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), | ||
28 | 'lib')) | ||
29 | |||
30 | import optparse | ||
31 | import warnings | ||
32 | from traceback import format_exception | ||
33 | try: | ||
34 | import bb | ||
35 | except RuntimeError as exc: | ||
36 | sys.exit(str(exc)) | ||
37 | from bb import event | ||
38 | import bb.msg | ||
39 | from bb import cooker | ||
40 | from bb import ui | ||
41 | from bb import server | ||
42 | from bb import cookerdata | ||
43 | |||
44 | __version__ = "1.20.0" | ||
45 | logger = logging.getLogger("BitBake") | ||
46 | |||
47 | # Python multiprocessing requires /dev/shm | ||
48 | if not os.access('/dev/shm', os.W_OK | os.X_OK): | ||
49 | sys.exit("FATAL: /dev/shm does not exist or is not writable") | ||
50 | |||
51 | # Unbuffer stdout to avoid log truncation in the event | ||
52 | # of an unorderly exit as well as to provide timely | ||
53 | # updates to log files for use with tail | ||
54 | try: | ||
55 | if sys.stdout.name == '<stdout>': | ||
56 | sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) | ||
57 | except: | ||
58 | pass | ||
59 | |||
60 | |||
61 | def get_ui(config): | ||
62 | if not config.ui: | ||
63 | # modify 'ui' attribute because it is also read by cooker | ||
64 | config.ui = os.environ.get('BITBAKE_UI', 'knotty') | ||
65 | |||
66 | interface = config.ui | ||
67 | |||
68 | try: | ||
69 | # Dynamically load the UI based on the ui name. Although we | ||
70 | # suggest a fixed set this allows you to have flexibility in which | ||
71 | # ones are available. | ||
72 | module = __import__("bb.ui", fromlist = [interface]) | ||
73 | return getattr(module, interface) | ||
74 | except AttributeError: | ||
75 | sys.exit("FATAL: Invalid user interface '%s' specified.\n" | ||
76 | "Valid interfaces: depexp, goggle, ncurses, hob, knotty [default]." % interface) | ||
77 | |||
78 | |||
79 | # Display bitbake/OE warnings via the BitBake.Warnings logger, ignoring others""" | ||
80 | warnlog = logging.getLogger("BitBake.Warnings") | ||
81 | _warnings_showwarning = warnings.showwarning | ||
82 | def _showwarning(message, category, filename, lineno, file=None, line=None): | ||
83 | if file is not None: | ||
84 | if _warnings_showwarning is not None: | ||
85 | _warnings_showwarning(message, category, filename, lineno, file, line) | ||
86 | else: | ||
87 | s = warnings.formatwarning(message, category, filename, lineno) | ||
88 | warnlog.warn(s) | ||
89 | |||
90 | warnings.showwarning = _showwarning | ||
91 | warnings.filterwarnings("ignore") | ||
92 | warnings.filterwarnings("default", module="(<string>$|(oe|bb)\.)") | ||
93 | warnings.filterwarnings("ignore", category=PendingDeprecationWarning) | ||
94 | warnings.filterwarnings("ignore", category=ImportWarning) | ||
95 | warnings.filterwarnings("ignore", category=DeprecationWarning, module="<string>$") | ||
96 | warnings.filterwarnings("ignore", message="With-statements now directly support multiple context managers") | ||
97 | |||
98 | class BitBakeConfigParameters(cookerdata.ConfigParameters): | ||
99 | |||
100 | def parseCommandLine(self): | ||
101 | parser = optparse.OptionParser( | ||
102 | version = "BitBake Build Tool Core version %s, %%prog version %s" % (bb.__version__, __version__), | ||
103 | usage = """%prog [options] [recipename/target ...] | ||
104 | |||
105 | Executes the specified task (default is 'build') for a given set of target recipes (.bb files). | ||
106 | It is assumed there is a conf/bblayers.conf available in cwd or in BBPATH which | ||
107 | will provide the layer, BBFILES and other configuration information.""") | ||
108 | |||
109 | parser.add_option("-b", "--buildfile", help = "Execute tasks from a specific .bb recipe directly. WARNING: Does not handle any dependencies from other recipes.", | ||
110 | action = "store", dest = "buildfile", default = None) | ||
111 | |||
112 | parser.add_option("-k", "--continue", help = "Continue as much as possible after an error. While the target that failed and anything depending on it cannot be built, as much as possible will be built before stopping.", | ||
113 | action = "store_false", dest = "abort", default = True) | ||
114 | |||
115 | parser.add_option("-a", "--tryaltconfigs", help = "Continue with builds by trying to use alternative providers where possible.", | ||
116 | action = "store_true", dest = "tryaltconfigs", default = False) | ||
117 | |||
118 | parser.add_option("-f", "--force", help = "Force the specified targets/task to run (invalidating any existing stamp file).", | ||
119 | action = "store_true", dest = "force", default = False) | ||
120 | |||
121 | parser.add_option("-c", "--cmd", help = "Specify the task to execute. The exact options available depend on the metadata. Some examples might be 'compile' or 'populate_sysroot' or 'listtasks' may give a list of the tasks available.", | ||
122 | action = "store", dest = "cmd") | ||
123 | |||
124 | parser.add_option("-C", "--clear-stamp", help = "Invalidate the stamp for the specified task such as 'compile' and then run the default task for the specified target(s).", | ||
125 | action = "store", dest = "invalidate_stamp") | ||
126 | |||
127 | parser.add_option("-r", "--read", help = "Read the specified file before bitbake.conf.", | ||
128 | action = "append", dest = "prefile", default = []) | ||
129 | |||
130 | parser.add_option("-R", "--postread", help = "Read the specified file after bitbake.conf.", | ||
131 | action = "append", dest = "postfile", default = []) | ||
132 | |||
133 | parser.add_option("-v", "--verbose", help = "Output more log message data to the terminal.", | ||
134 | action = "store_true", dest = "verbose", default = False) | ||
135 | |||
136 | parser.add_option("-D", "--debug", help = "Increase the debug level. You can specify this more than once.", | ||
137 | action = "count", dest="debug", default = 0) | ||
138 | |||
139 | parser.add_option("-n", "--dry-run", help = "Don't execute, just go through the motions.", | ||
140 | action = "store_true", dest = "dry_run", default = False) | ||
141 | |||
142 | parser.add_option("-S", "--dump-signatures", help = "Don't execute, just dump out the signature construction information.", | ||
143 | action = "store_true", dest = "dump_signatures", default = False) | ||
144 | |||
145 | parser.add_option("-p", "--parse-only", help = "Quit after parsing the BB recipes.", | ||
146 | action = "store_true", dest = "parse_only", default = False) | ||
147 | |||
148 | parser.add_option("-s", "--show-versions", help = "Show current and preferred versions of all recipes.", | ||
149 | action = "store_true", dest = "show_versions", default = False) | ||
150 | |||
151 | parser.add_option("-e", "--environment", help = "Show the global or per-package environment complete with information about where variables were set/changed.", | ||
152 | action = "store_true", dest = "show_environment", default = False) | ||
153 | |||
154 | parser.add_option("-g", "--graphviz", help = "Save dependency tree information for the specified targets in the dot syntax.", | ||
155 | action = "store_true", dest = "dot_graph", default = False) | ||
156 | |||
157 | parser.add_option("-I", "--ignore-deps", help = """Assume these dependencies don't exist and are already provided (equivalent to ASSUME_PROVIDED). Useful to make dependency graphs more appealing""", | ||
158 | action = "append", dest = "extra_assume_provided", default = []) | ||
159 | |||
160 | parser.add_option("-l", "--log-domains", help = """Show debug logging for the specified logging domains""", | ||
161 | action = "append", dest = "debug_domains", default = []) | ||
162 | |||
163 | parser.add_option("-P", "--profile", help = "Profile the command and save reports.", | ||
164 | action = "store_true", dest = "profile", default = False) | ||
165 | |||
166 | parser.add_option("-u", "--ui", help = "The user interface to use (e.g. knotty, hob, depexp).", | ||
167 | action = "store", dest = "ui") | ||
168 | |||
169 | parser.add_option("-t", "--servertype", help = "Choose which server to use, process or xmlrpc.", | ||
170 | action = "store", dest = "servertype") | ||
171 | |||
172 | parser.add_option("", "--revisions-changed", help = "Set the exit code depending on whether upstream floating revisions have changed or not.", | ||
173 | action = "store_true", dest = "revisions_changed", default = False) | ||
174 | |||
175 | parser.add_option("", "--server-only", help = "Run bitbake without a UI, only starting a server (cooker) process.", | ||
176 | action = "store_true", dest = "server_only", default = False) | ||
177 | |||
178 | parser.add_option("-B", "--bind", help = "The name/address for the bitbake server to bind to.", | ||
179 | action = "store", dest = "bind", default = False) | ||
180 | |||
181 | parser.add_option("", "--no-setscene", help = "Do not run any setscene tasks. sstate will be ignored and everything needed, built.", | ||
182 | action = "store_true", dest = "nosetscene", default = False) | ||
183 | |||
184 | parser.add_option("", "--remote-server", help = "Connect to the specified server.", | ||
185 | action = "store", dest = "remote_server", default = False) | ||
186 | |||
187 | parser.add_option("-m", "--kill-server", help = "Terminate the remote server.", | ||
188 | action = "store_true", dest = "kill_server", default = False) | ||
189 | |||
190 | parser.add_option("", "--observe-only", help = "Connect to a server as an observing-only client.", | ||
191 | action = "store_true", dest = "observe_only", default = False) | ||
192 | |||
193 | options, targets = parser.parse_args(sys.argv) | ||
194 | |||
195 | # some environmental variables set also configuration options | ||
196 | if "BBSERVER" in os.environ: | ||
197 | options.servertype = "xmlrpc" | ||
198 | options.remote_server = os.environ["BBSERVER"] | ||
199 | |||
200 | return options, targets[1:] | ||
201 | |||
202 | |||
203 | def start_server(servermodule, configParams, configuration): | ||
204 | server = servermodule.BitBakeServer() | ||
205 | if configParams.bind: | ||
206 | (host, port) = configParams.bind.split(':') | ||
207 | server.initServer((host, int(port))) | ||
208 | else: | ||
209 | server.initServer() | ||
210 | |||
211 | try: | ||
212 | configuration.setServerRegIdleCallback(server.getServerIdleCB()) | ||
213 | |||
214 | cooker = bb.cooker.BBCooker(configuration) | ||
215 | |||
216 | server.addcooker(cooker) | ||
217 | server.saveConnectionDetails() | ||
218 | except Exception as e: | ||
219 | exc_info = sys.exc_info() | ||
220 | while True: | ||
221 | try: | ||
222 | import queue | ||
223 | except ImportError: | ||
224 | import Queue as queue | ||
225 | try: | ||
226 | event = server.event_queue.get(block=False) | ||
227 | except (queue.Empty, IOError): | ||
228 | break | ||
229 | if isinstance(event, logging.LogRecord): | ||
230 | logger.handle(event) | ||
231 | raise exc_info[1], None, exc_info[2] | ||
232 | server.detach() | ||
233 | return server | ||
234 | |||
235 | |||
236 | |||
237 | def main(): | ||
238 | |||
239 | configParams = BitBakeConfigParameters() | ||
240 | configuration = cookerdata.CookerConfiguration() | ||
241 | configuration.setConfigParameters(configParams) | ||
242 | |||
243 | ui_module = get_ui(configParams) | ||
244 | |||
245 | # Server type can be xmlrpc or process currently, if nothing is specified, | ||
246 | # the default server is process | ||
247 | if configParams.servertype: | ||
248 | server_type = configParams.servertype | ||
249 | else: | ||
250 | server_type = 'process' | ||
251 | |||
252 | try: | ||
253 | module = __import__("bb.server", fromlist = [server_type]) | ||
254 | servermodule = getattr(module, server_type) | ||
255 | except AttributeError: | ||
256 | sys.exit("FATAL: Invalid server type '%s' specified.\n" | ||
257 | "Valid interfaces: xmlrpc, process [default]." % servertype) | ||
258 | |||
259 | if configParams.server_only: | ||
260 | if configParams.servertype != "xmlrpc": | ||
261 | sys.exit("FATAL: If '--server-only' is defined, we must set the servertype as 'xmlrpc'.\n") | ||
262 | if not configParams.bind: | ||
263 | sys.exit("FATAL: The '--server-only' option requires a name/address to bind to with the -B option.\n") | ||
264 | if configParams.remote_server: | ||
265 | sys.exit("FATAL: The '--server-only' option conflicts with %s.\n" % | ||
266 | ("the BBSERVER environment variable" if "BBSERVER" in os.environ else "the '--remote-server' option" )) | ||
267 | |||
268 | if configParams.bind and configParams.servertype != "xmlrpc": | ||
269 | sys.exit("FATAL: If '-B' or '--bind' is defined, we must set the servertype as 'xmlrpc'.\n") | ||
270 | |||
271 | if configParams.remote_server and configParams.servertype != "xmlrpc": | ||
272 | sys.exit("FATAL: If '--remote-server' is defined, we must set the servertype as 'xmlrpc'.\n") | ||
273 | |||
274 | if configParams.observe_only and (not configParams.remote_server or configParams.bind): | ||
275 | sys.exit("FATAL: '--observe-only' can only be used by UI clients connecting to a server.\n") | ||
276 | |||
277 | if "BBDEBUG" in os.environ: | ||
278 | level = int(os.environ["BBDEBUG"]) | ||
279 | if level > configuration.debug: | ||
280 | configuration.debug = level | ||
281 | |||
282 | bb.msg.init_msgconfig(configParams.verbose, configuration.debug, | ||
283 | configuration.debug_domains) | ||
284 | |||
285 | # Ensure logging messages get sent to the UI as events | ||
286 | handler = bb.event.LogHandler() | ||
287 | logger.addHandler(handler) | ||
288 | |||
289 | # Clear away any spurious environment variables while we stoke up the cooker | ||
290 | cleanedvars = bb.utils.clean_environment() | ||
291 | |||
292 | if not configParams.remote_server: | ||
293 | # we start a server with a given configuration | ||
294 | server = start_server(servermodule, configParams, configuration) | ||
295 | bb.event.ui_queue = [] | ||
296 | else: | ||
297 | # we start a stub server that is actually a XMLRPClient that connects to a real server | ||
298 | server = servermodule.BitBakeXMLRPCClient(configParams.observe_only) | ||
299 | server.saveConnectionDetails(configParams.remote_server) | ||
300 | |||
301 | if not configParams.server_only: | ||
302 | # Collect the feature set for the UI | ||
303 | featureset = getattr(ui_module, "featureSet", []) | ||
304 | |||
305 | # Setup a connection to the server (cooker) | ||
306 | server_connection = server.establishConnection(featureset) | ||
307 | |||
308 | # Restore the environment in case the UI needs it | ||
309 | for k in cleanedvars: | ||
310 | os.environ[k] = cleanedvars[k] | ||
311 | |||
312 | logger.removeHandler(handler) | ||
313 | |||
314 | try: | ||
315 | return ui_module.main(server_connection.connection, server_connection.events, configParams) | ||
316 | finally: | ||
317 | bb.event.ui_queue = [] | ||
318 | server_connection.terminate() | ||
319 | else: | ||
320 | print("server address: %s, server port: %s" % (server.serverImpl.host, server.serverImpl.port)) | ||
321 | |||
322 | return 1 | ||
323 | |||
324 | if __name__ == "__main__": | ||
325 | try: | ||
326 | ret = main() | ||
327 | except bb.BBHandledException: | ||
328 | ret = 1 | ||
329 | except Exception: | ||
330 | ret = 1 | ||
331 | import traceback | ||
332 | traceback.print_exc() | ||
333 | sys.exit(ret) | ||
334 | |||
diff --git a/bitbake/bin/bitbake-diffsigs b/bitbake/bin/bitbake-diffsigs new file mode 100755 index 0000000000..78757b0aae --- /dev/null +++ b/bitbake/bin/bitbake-diffsigs | |||
@@ -0,0 +1,122 @@ | |||
1 | #!/usr/bin/env python | ||
2 | |||
3 | # bitbake-diffsigs | ||
4 | # BitBake task signature data comparison utility | ||
5 | # | ||
6 | # Copyright (C) 2012-2013 Intel Corporation | ||
7 | # | ||
8 | # This program is free software; you can redistribute it and/or modify | ||
9 | # it under the terms of the GNU General Public License version 2 as | ||
10 | # published by the Free Software Foundation. | ||
11 | # | ||
12 | # This program is distributed in the hope that it will be useful, | ||
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
15 | # GNU General Public License for more details. | ||
16 | # | ||
17 | # You should have received a copy of the GNU General Public License along | ||
18 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
20 | |||
21 | import os | ||
22 | import sys | ||
23 | import warnings | ||
24 | import fnmatch | ||
25 | import optparse | ||
26 | import logging | ||
27 | |||
28 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) | ||
29 | |||
30 | import bb.tinfoil | ||
31 | import bb.siggen | ||
32 | |||
33 | def logger_create(name, output=sys.stderr): | ||
34 | logger = logging.getLogger(name) | ||
35 | console = logging.StreamHandler(output) | ||
36 | format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") | ||
37 | if output.isatty(): | ||
38 | format.enable_color() | ||
39 | console.setFormatter(format) | ||
40 | logger.addHandler(console) | ||
41 | logger.setLevel(logging.INFO) | ||
42 | return logger | ||
43 | |||
44 | logger = logger_create('bitbake-diffsigs') | ||
45 | |||
46 | def find_compare_task(bbhandler, pn, taskname): | ||
47 | """ Find the most recent signature files for the specified PN/task and compare them """ | ||
48 | |||
49 | if not hasattr(bb.siggen, 'find_siginfo'): | ||
50 | logger.error('Metadata does not support finding signature data files') | ||
51 | sys.exit(1) | ||
52 | |||
53 | if not taskname.startswith('do_'): | ||
54 | taskname = 'do_%s' % taskname | ||
55 | |||
56 | filedates = bb.siggen.find_siginfo(pn, taskname, None, bbhandler.config_data) | ||
57 | latestfiles = sorted(filedates.keys(), key=lambda f: filedates[f])[-2:] | ||
58 | if not latestfiles: | ||
59 | logger.error('No sigdata files found matching %s %s' % (pn, taskname)) | ||
60 | sys.exit(1) | ||
61 | elif len(latestfiles) < 2: | ||
62 | logger.error('Only one matching sigdata file found for the specified task (%s %s)' % (pn, taskname)) | ||
63 | sys.exit(1) | ||
64 | else: | ||
65 | # Define recursion callback | ||
66 | def recursecb(key, hash1, hash2): | ||
67 | hashes = [hash1, hash2] | ||
68 | hashfiles = bb.siggen.find_siginfo(key, None, hashes, bbhandler.config_data) | ||
69 | |||
70 | recout = [] | ||
71 | if len(hashfiles) == 2: | ||
72 | out2 = bb.siggen.compare_sigfiles(hashfiles[hash1], hashfiles[hash2], recursecb) | ||
73 | recout.extend(list(' ' + l for l in out2)) | ||
74 | else: | ||
75 | recout.append("Unable to find matching sigdata for %s with hashes %s or %s" % (key, hash1, hash2)) | ||
76 | |||
77 | return recout | ||
78 | |||
79 | # Recurse into signature comparison | ||
80 | output = bb.siggen.compare_sigfiles(latestfiles[0], latestfiles[1], recursecb) | ||
81 | if output: | ||
82 | print '\n'.join(output) | ||
83 | sys.exit(0) | ||
84 | |||
85 | |||
86 | |||
87 | parser = optparse.OptionParser( | ||
88 | description = "Compares siginfo/sigdata files written out by BitBake", | ||
89 | usage = """ | ||
90 | %prog -t recipename taskname | ||
91 | %prog sigdatafile1 sigdatafile2 | ||
92 | %prog sigdatafile1""") | ||
93 | |||
94 | parser.add_option("-t", "--task", | ||
95 | help = "find the signature data files for last two runs of the specified task and compare them", | ||
96 | action="store", dest="taskargs", nargs=2, metavar='recipename taskname') | ||
97 | |||
98 | options, args = parser.parse_args(sys.argv) | ||
99 | |||
100 | if options.taskargs: | ||
101 | tinfoil = bb.tinfoil.Tinfoil() | ||
102 | tinfoil.prepare(config_only = True) | ||
103 | find_compare_task(tinfoil, options.taskargs[0], options.taskargs[1]) | ||
104 | else: | ||
105 | if len(args) == 1: | ||
106 | parser.print_help() | ||
107 | else: | ||
108 | import cPickle | ||
109 | try: | ||
110 | if len(args) == 2: | ||
111 | output = bb.siggen.dump_sigfile(sys.argv[1]) | ||
112 | else: | ||
113 | output = bb.siggen.compare_sigfiles(sys.argv[1], sys.argv[2]) | ||
114 | except IOError as e: | ||
115 | logger.error(str(e)) | ||
116 | sys.exit(1) | ||
117 | except cPickle.UnpicklingError, EOFError: | ||
118 | logger.error('Invalid signature data - ensure you are specifying sigdata/siginfo files') | ||
119 | sys.exit(1) | ||
120 | |||
121 | if output: | ||
122 | print '\n'.join(output) | ||
diff --git a/bitbake/bin/bitbake-dumpsig b/bitbake/bin/bitbake-dumpsig new file mode 100755 index 0000000000..656d93a5ac --- /dev/null +++ b/bitbake/bin/bitbake-dumpsig | |||
@@ -0,0 +1,65 @@ | |||
1 | #!/usr/bin/env python | ||
2 | |||
3 | # bitbake-dumpsig | ||
4 | # BitBake task signature dump utility | ||
5 | # | ||
6 | # Copyright (C) 2013 Intel Corporation | ||
7 | # | ||
8 | # This program is free software; you can redistribute it and/or modify | ||
9 | # it under the terms of the GNU General Public License version 2 as | ||
10 | # published by the Free Software Foundation. | ||
11 | # | ||
12 | # This program is distributed in the hope that it will be useful, | ||
13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
15 | # GNU General Public License for more details. | ||
16 | # | ||
17 | # You should have received a copy of the GNU General Public License along | ||
18 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
19 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
20 | |||
21 | import os | ||
22 | import sys | ||
23 | import warnings | ||
24 | import optparse | ||
25 | import logging | ||
26 | |||
27 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) | ||
28 | |||
29 | import bb.siggen | ||
30 | |||
31 | def logger_create(name, output=sys.stderr): | ||
32 | logger = logging.getLogger(name) | ||
33 | console = logging.StreamHandler(output) | ||
34 | format = bb.msg.BBLogFormatter("%(levelname)s: %(message)s") | ||
35 | if output.isatty(): | ||
36 | format.enable_color() | ||
37 | console.setFormatter(format) | ||
38 | logger.addHandler(console) | ||
39 | logger.setLevel(logging.INFO) | ||
40 | return logger | ||
41 | |||
42 | logger = logger_create('bitbake-dumpsig') | ||
43 | |||
44 | parser = optparse.OptionParser( | ||
45 | description = "Dumps siginfo/sigdata files written out by BitBake", | ||
46 | usage = """ | ||
47 | %prog sigdatafile""") | ||
48 | |||
49 | options, args = parser.parse_args(sys.argv) | ||
50 | |||
51 | if len(args) == 1: | ||
52 | parser.print_help() | ||
53 | else: | ||
54 | import cPickle | ||
55 | try: | ||
56 | output = bb.siggen.dump_sigfile(args[1]) | ||
57 | except IOError as e: | ||
58 | logger.error(str(e)) | ||
59 | sys.exit(1) | ||
60 | except cPickle.UnpicklingError, EOFError: | ||
61 | logger.error('Invalid signature data - ensure you are specifying a sigdata/siginfo file') | ||
62 | sys.exit(1) | ||
63 | |||
64 | if output: | ||
65 | print '\n'.join(output) | ||
diff --git a/bitbake/bin/bitbake-layers b/bitbake/bin/bitbake-layers new file mode 100755 index 0000000000..047583c497 --- /dev/null +++ b/bitbake/bin/bitbake-layers | |||
@@ -0,0 +1,720 @@ | |||
1 | #!/usr/bin/env python | ||
2 | |||
3 | # This script has subcommands which operate against your bitbake layers, either | ||
4 | # displaying useful information, or acting against them. | ||
5 | # See the help output for details on available commands. | ||
6 | |||
7 | # Copyright (C) 2011 Mentor Graphics Corporation | ||
8 | # Copyright (C) 2012 Intel Corporation | ||
9 | # | ||
10 | # This program is free software; you can redistribute it and/or modify | ||
11 | # it under the terms of the GNU General Public License version 2 as | ||
12 | # published by the Free Software Foundation. | ||
13 | # | ||
14 | # This program is distributed in the hope that it will be useful, | ||
15 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
16 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
17 | # GNU General Public License for more details. | ||
18 | # | ||
19 | # You should have received a copy of the GNU General Public License along | ||
20 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
21 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
22 | |||
23 | import cmd | ||
24 | import logging | ||
25 | import os | ||
26 | import sys | ||
27 | import fnmatch | ||
28 | from collections import defaultdict | ||
29 | import re | ||
30 | |||
31 | bindir = os.path.dirname(__file__) | ||
32 | topdir = os.path.dirname(bindir) | ||
33 | sys.path[0:0] = [os.path.join(topdir, 'lib')] | ||
34 | |||
35 | import bb.cache | ||
36 | import bb.cooker | ||
37 | import bb.providers | ||
38 | import bb.utils | ||
39 | import bb.tinfoil | ||
40 | |||
41 | |||
42 | logger = logging.getLogger('BitBake') | ||
43 | |||
44 | |||
45 | def main(args): | ||
46 | cmds = Commands() | ||
47 | if args: | ||
48 | # Allow user to specify e.g. show-layers instead of show_layers | ||
49 | args = [args[0].replace('-', '_')] + args[1:] | ||
50 | cmds.onecmd(' '.join(args)) | ||
51 | else: | ||
52 | cmds.do_help('') | ||
53 | return cmds.returncode | ||
54 | |||
55 | |||
56 | class Commands(cmd.Cmd): | ||
57 | def __init__(self): | ||
58 | cmd.Cmd.__init__(self) | ||
59 | self.bbhandler = bb.tinfoil.Tinfoil() | ||
60 | self.returncode = 0 | ||
61 | self.bblayers = (self.bbhandler.config_data.getVar('BBLAYERS', True) or "").split() | ||
62 | |||
63 | def default(self, line): | ||
64 | """Handle unrecognised commands""" | ||
65 | sys.stderr.write("Unrecognised command or option\n") | ||
66 | self.do_help('') | ||
67 | |||
68 | def do_help(self, topic): | ||
69 | """display general help or help on a specified command""" | ||
70 | if topic: | ||
71 | sys.stdout.write('%s: ' % topic) | ||
72 | cmd.Cmd.do_help(self, topic.replace('-', '_')) | ||
73 | else: | ||
74 | sys.stdout.write("usage: bitbake-layers <command> [arguments]\n\n") | ||
75 | sys.stdout.write("Available commands:\n") | ||
76 | procnames = list(set(self.get_names())) | ||
77 | for procname in procnames: | ||
78 | if procname[:3] == 'do_': | ||
79 | sys.stdout.write(" %s\n" % procname[3:].replace('_', '-')) | ||
80 | doc = getattr(self, procname).__doc__ | ||
81 | if doc: | ||
82 | sys.stdout.write(" %s\n" % doc.splitlines()[0]) | ||
83 | |||
84 | def do_show_layers(self, args): | ||
85 | """show current configured layers""" | ||
86 | self.bbhandler.prepare(config_only = True) | ||
87 | logger.plain("%s %s %s" % ("layer".ljust(20), "path".ljust(40), "priority")) | ||
88 | logger.plain('=' * 74) | ||
89 | for layerdir in self.bblayers: | ||
90 | layername = self.get_layer_name(layerdir) | ||
91 | layerpri = 0 | ||
92 | for layer, _, regex, pri in self.bbhandler.cooker.recipecache.bbfile_config_priorities: | ||
93 | if regex.match(os.path.join(layerdir, 'test')): | ||
94 | layerpri = pri | ||
95 | break | ||
96 | |||
97 | logger.plain("%s %s %d" % (layername.ljust(20), layerdir.ljust(40), layerpri)) | ||
98 | |||
99 | |||
100 | def version_str(self, pe, pv, pr = None): | ||
101 | verstr = "%s" % pv | ||
102 | if pr: | ||
103 | verstr = "%s-%s" % (verstr, pr) | ||
104 | if pe: | ||
105 | verstr = "%s:%s" % (pe, verstr) | ||
106 | return verstr | ||
107 | |||
108 | |||
109 | def do_show_overlayed(self, args): | ||
110 | """list overlayed recipes (where the same recipe exists in another layer) | ||
111 | |||
112 | usage: show-overlayed [-f] [-s] | ||
113 | |||
114 | Lists the names of overlayed recipes and the available versions in each | ||
115 | layer, with the preferred version first. Note that skipped recipes that | ||
116 | are overlayed will also be listed, with a " (skipped)" suffix. | ||
117 | |||
118 | Options: | ||
119 | -f instead of the default formatting, list filenames of higher priority | ||
120 | recipes with the ones they overlay indented underneath | ||
121 | -s only list overlayed recipes where the version is the same | ||
122 | """ | ||
123 | self.bbhandler.prepare() | ||
124 | |||
125 | show_filenames = False | ||
126 | show_same_ver_only = False | ||
127 | for arg in args.split(): | ||
128 | if arg == '-f': | ||
129 | show_filenames = True | ||
130 | elif arg == '-s': | ||
131 | show_same_ver_only = True | ||
132 | else: | ||
133 | sys.stderr.write("show-overlayed: invalid option %s\n" % arg) | ||
134 | self.do_help('') | ||
135 | return | ||
136 | |||
137 | items_listed = self.list_recipes('Overlayed recipes', None, True, show_same_ver_only, show_filenames, True) | ||
138 | |||
139 | # Check for overlayed .bbclass files | ||
140 | classes = defaultdict(list) | ||
141 | for layerdir in self.bblayers: | ||
142 | classdir = os.path.join(layerdir, 'classes') | ||
143 | if os.path.exists(classdir): | ||
144 | for classfile in os.listdir(classdir): | ||
145 | if os.path.splitext(classfile)[1] == '.bbclass': | ||
146 | classes[classfile].append(classdir) | ||
147 | |||
148 | # Locating classes and other files is a bit more complicated than recipes - | ||
149 | # layer priority is not a factor; instead BitBake uses the first matching | ||
150 | # file in BBPATH, which is manipulated directly by each layer's | ||
151 | # conf/layer.conf in turn, thus the order of layers in bblayers.conf is a | ||
152 | # factor - however, each layer.conf is free to either prepend or append to | ||
153 | # BBPATH (or indeed do crazy stuff with it). Thus the order in BBPATH might | ||
154 | # not be exactly the order present in bblayers.conf either. | ||
155 | bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True)) | ||
156 | overlayed_class_found = False | ||
157 | for (classfile, classdirs) in classes.items(): | ||
158 | if len(classdirs) > 1: | ||
159 | if not overlayed_class_found: | ||
160 | logger.plain('=== Overlayed classes ===') | ||
161 | overlayed_class_found = True | ||
162 | |||
163 | mainfile = bb.utils.which(bbpath, os.path.join('classes', classfile)) | ||
164 | if show_filenames: | ||
165 | logger.plain('%s' % mainfile) | ||
166 | else: | ||
167 | # We effectively have to guess the layer here | ||
168 | logger.plain('%s:' % classfile) | ||
169 | mainlayername = '?' | ||
170 | for layerdir in self.bblayers: | ||
171 | classdir = os.path.join(layerdir, 'classes') | ||
172 | if mainfile.startswith(classdir): | ||
173 | mainlayername = self.get_layer_name(layerdir) | ||
174 | logger.plain(' %s' % mainlayername) | ||
175 | for classdir in classdirs: | ||
176 | fullpath = os.path.join(classdir, classfile) | ||
177 | if fullpath != mainfile: | ||
178 | if show_filenames: | ||
179 | print(' %s' % fullpath) | ||
180 | else: | ||
181 | print(' %s' % self.get_layer_name(os.path.dirname(classdir))) | ||
182 | |||
183 | if overlayed_class_found: | ||
184 | items_listed = True; | ||
185 | |||
186 | if not items_listed: | ||
187 | logger.plain('No overlayed files found.') | ||
188 | |||
189 | |||
190 | def do_show_recipes(self, args): | ||
191 | """list available recipes, showing the layer they are provided by | ||
192 | |||
193 | usage: show-recipes [-f] [-m] [pnspec] | ||
194 | |||
195 | Lists the names of overlayed recipes and the available versions in each | ||
196 | layer, with the preferred version first. Optionally you may specify | ||
197 | pnspec to match a specified recipe name (supports wildcards). Note that | ||
198 | skipped recipes will also be listed, with a " (skipped)" suffix. | ||
199 | |||
200 | Options: | ||
201 | -f instead of the default formatting, list filenames of higher priority | ||
202 | recipes with other available recipes indented underneath | ||
203 | -m only list where multiple recipes (in the same layer or different | ||
204 | layers) exist for the same recipe name | ||
205 | """ | ||
206 | self.bbhandler.prepare() | ||
207 | |||
208 | show_filenames = False | ||
209 | show_multi_provider_only = False | ||
210 | pnspec = None | ||
211 | title = 'Available recipes:' | ||
212 | for arg in args.split(): | ||
213 | if arg == '-f': | ||
214 | show_filenames = True | ||
215 | elif arg == '-m': | ||
216 | show_multi_provider_only = True | ||
217 | elif not arg.startswith('-'): | ||
218 | pnspec = arg | ||
219 | title = 'Available recipes matching %s:' % pnspec | ||
220 | else: | ||
221 | sys.stderr.write("show-recipes: invalid option %s\n" % arg) | ||
222 | self.do_help('') | ||
223 | return | ||
224 | self.list_recipes(title, pnspec, False, False, show_filenames, show_multi_provider_only) | ||
225 | |||
226 | |||
227 | def list_recipes(self, title, pnspec, show_overlayed_only, show_same_ver_only, show_filenames, show_multi_provider_only): | ||
228 | pkg_pn = self.bbhandler.cooker.recipecache.pkg_pn | ||
229 | (latest_versions, preferred_versions) = bb.providers.findProviders(self.bbhandler.config_data, self.bbhandler.cooker.recipecache, pkg_pn) | ||
230 | allproviders = bb.providers.allProviders(self.bbhandler.cooker.recipecache) | ||
231 | |||
232 | # Ensure we list skipped recipes | ||
233 | # We are largely guessing about PN, PV and the preferred version here, | ||
234 | # but we have no choice since skipped recipes are not fully parsed | ||
235 | skiplist = self.bbhandler.cooker.skiplist.keys() | ||
236 | skiplist.sort( key=lambda fileitem: self.bbhandler.cooker.collection.calc_bbfile_priority(fileitem) ) | ||
237 | skiplist.reverse() | ||
238 | for fn in skiplist: | ||
239 | recipe_parts = os.path.splitext(os.path.basename(fn))[0].split('_') | ||
240 | p = recipe_parts[0] | ||
241 | if len(recipe_parts) > 1: | ||
242 | ver = (None, recipe_parts[1], None) | ||
243 | else: | ||
244 | ver = (None, 'unknown', None) | ||
245 | allproviders[p].append((ver, fn)) | ||
246 | if not p in pkg_pn: | ||
247 | pkg_pn[p] = 'dummy' | ||
248 | preferred_versions[p] = (ver, fn) | ||
249 | |||
250 | def print_item(f, pn, ver, layer, ispref): | ||
251 | if f in skiplist: | ||
252 | skipped = ' (skipped)' | ||
253 | else: | ||
254 | skipped = '' | ||
255 | if show_filenames: | ||
256 | if ispref: | ||
257 | logger.plain("%s%s", f, skipped) | ||
258 | else: | ||
259 | logger.plain(" %s%s", f, skipped) | ||
260 | else: | ||
261 | if ispref: | ||
262 | logger.plain("%s:", pn) | ||
263 | logger.plain(" %s %s%s", layer.ljust(20), ver, skipped) | ||
264 | |||
265 | preffiles = [] | ||
266 | items_listed = False | ||
267 | for p in sorted(pkg_pn): | ||
268 | if pnspec: | ||
269 | if not fnmatch.fnmatch(p, pnspec): | ||
270 | continue | ||
271 | |||
272 | if len(allproviders[p]) > 1 or not show_multi_provider_only: | ||
273 | pref = preferred_versions[p] | ||
274 | preffile = bb.cache.Cache.virtualfn2realfn(pref[1])[0] | ||
275 | if preffile not in preffiles: | ||
276 | preflayer = self.get_file_layer(preffile) | ||
277 | multilayer = False | ||
278 | same_ver = True | ||
279 | provs = [] | ||
280 | for prov in allproviders[p]: | ||
281 | provfile = bb.cache.Cache.virtualfn2realfn(prov[1])[0] | ||
282 | provlayer = self.get_file_layer(provfile) | ||
283 | provs.append((provfile, provlayer, prov[0])) | ||
284 | if provlayer != preflayer: | ||
285 | multilayer = True | ||
286 | if prov[0] != pref[0]: | ||
287 | same_ver = False | ||
288 | |||
289 | if (multilayer or not show_overlayed_only) and (same_ver or not show_same_ver_only): | ||
290 | if not items_listed: | ||
291 | logger.plain('=== %s ===' % title) | ||
292 | items_listed = True | ||
293 | print_item(preffile, p, self.version_str(pref[0][0], pref[0][1]), preflayer, True) | ||
294 | for (provfile, provlayer, provver) in provs: | ||
295 | if provfile != preffile: | ||
296 | print_item(provfile, p, self.version_str(provver[0], provver[1]), provlayer, False) | ||
297 | # Ensure we don't show two entries for BBCLASSEXTENDed recipes | ||
298 | preffiles.append(preffile) | ||
299 | |||
300 | return items_listed | ||
301 | |||
302 | |||
303 | def do_flatten(self, args): | ||
304 | """flattens layer configuration into a separate output directory. | ||
305 | |||
306 | usage: flatten [layer1 layer2 [layer3]...] <outputdir> | ||
307 | |||
308 | Takes the specified layers (or all layers in the current layer | ||
309 | configuration if none are specified) and builds a "flattened" directory | ||
310 | containing the contents of all layers, with any overlayed recipes removed | ||
311 | and bbappends appended to the corresponding recipes. Note that some manual | ||
312 | cleanup may still be necessary afterwards, in particular: | ||
313 | |||
314 | * where non-recipe files (such as patches) are overwritten (the flatten | ||
315 | command will show a warning for these) | ||
316 | * where anything beyond the normal layer setup has been added to | ||
317 | layer.conf (only the lowest priority number layer's layer.conf is used) | ||
318 | * overridden/appended items from bbappends will need to be tidied up | ||
319 | * when the flattened layers do not have the same directory structure (the | ||
320 | flatten command should show a warning when this will cause a problem) | ||
321 | |||
322 | Warning: if you flatten several layers where another layer is intended to | ||
323 | be used "inbetween" them (in layer priority order) such that recipes / | ||
324 | bbappends in the layers interact, and then attempt to use the new output | ||
325 | layer together with that other layer, you may no longer get the same | ||
326 | build results (as the layer priority order has effectively changed). | ||
327 | """ | ||
328 | arglist = args.split() | ||
329 | if len(arglist) < 1: | ||
330 | logger.error('Please specify an output directory') | ||
331 | self.do_help('flatten') | ||
332 | return | ||
333 | |||
334 | if len(arglist) == 2: | ||
335 | logger.error('If you specify layers to flatten you must specify at least two') | ||
336 | self.do_help('flatten') | ||
337 | return | ||
338 | |||
339 | outputdir = arglist[-1] | ||
340 | if os.path.exists(outputdir) and os.listdir(outputdir): | ||
341 | logger.error('Directory %s exists and is non-empty, please clear it out first' % outputdir) | ||
342 | return | ||
343 | |||
344 | self.bbhandler.prepare() | ||
345 | layers = self.bblayers | ||
346 | if len(arglist) > 2: | ||
347 | layernames = arglist[:-1] | ||
348 | found_layernames = [] | ||
349 | found_layerdirs = [] | ||
350 | for layerdir in layers: | ||
351 | layername = self.get_layer_name(layerdir) | ||
352 | if layername in layernames: | ||
353 | found_layerdirs.append(layerdir) | ||
354 | found_layernames.append(layername) | ||
355 | |||
356 | for layername in layernames: | ||
357 | if not layername in found_layernames: | ||
358 | logger.error('Unable to find layer %s in current configuration, please run "%s show-layers" to list configured layers' % (layername, os.path.basename(sys.argv[0]))) | ||
359 | return | ||
360 | layers = found_layerdirs | ||
361 | else: | ||
362 | layernames = [] | ||
363 | |||
364 | # Ensure a specified path matches our list of layers | ||
365 | def layer_path_match(path): | ||
366 | for layerdir in layers: | ||
367 | if path.startswith(os.path.join(layerdir, '')): | ||
368 | return layerdir | ||
369 | return None | ||
370 | |||
371 | appended_recipes = [] | ||
372 | for layer in layers: | ||
373 | overlayed = [] | ||
374 | for f in self.bbhandler.cooker.collection.overlayed.iterkeys(): | ||
375 | for of in self.bbhandler.cooker.collection.overlayed[f]: | ||
376 | if of.startswith(layer): | ||
377 | overlayed.append(of) | ||
378 | |||
379 | logger.plain('Copying files from %s...' % layer ) | ||
380 | for root, dirs, files in os.walk(layer): | ||
381 | for f1 in files: | ||
382 | f1full = os.sep.join([root, f1]) | ||
383 | if f1full in overlayed: | ||
384 | logger.plain(' Skipping overlayed file %s' % f1full ) | ||
385 | else: | ||
386 | ext = os.path.splitext(f1)[1] | ||
387 | if ext != '.bbappend': | ||
388 | fdest = f1full[len(layer):] | ||
389 | fdest = os.path.normpath(os.sep.join([outputdir,fdest])) | ||
390 | bb.utils.mkdirhier(os.path.dirname(fdest)) | ||
391 | if os.path.exists(fdest): | ||
392 | if f1 == 'layer.conf' and root.endswith('/conf'): | ||
393 | logger.plain(' Skipping layer config file %s' % f1full ) | ||
394 | continue | ||
395 | else: | ||
396 | logger.warn('Overwriting file %s', fdest) | ||
397 | bb.utils.copyfile(f1full, fdest) | ||
398 | if ext == '.bb': | ||
399 | if f1 in self.bbhandler.cooker.collection.appendlist: | ||
400 | appends = self.bbhandler.cooker.collection.appendlist[f1] | ||
401 | if appends: | ||
402 | logger.plain(' Applying appends to %s' % fdest ) | ||
403 | for appendname in appends: | ||
404 | if layer_path_match(appendname): | ||
405 | self.apply_append(appendname, fdest) | ||
406 | appended_recipes.append(f1) | ||
407 | |||
408 | # Take care of when some layers are excluded and yet we have included bbappends for those recipes | ||
409 | for recipename in self.bbhandler.cooker.collection.appendlist.iterkeys(): | ||
410 | if recipename not in appended_recipes: | ||
411 | appends = self.bbhandler.cooker.collection.appendlist[recipename] | ||
412 | first_append = None | ||
413 | for appendname in appends: | ||
414 | layer = layer_path_match(appendname) | ||
415 | if layer: | ||
416 | if first_append: | ||
417 | self.apply_append(appendname, first_append) | ||
418 | else: | ||
419 | fdest = appendname[len(layer):] | ||
420 | fdest = os.path.normpath(os.sep.join([outputdir,fdest])) | ||
421 | bb.utils.mkdirhier(os.path.dirname(fdest)) | ||
422 | bb.utils.copyfile(appendname, fdest) | ||
423 | first_append = fdest | ||
424 | |||
425 | # Get the regex for the first layer in our list (which is where the conf/layer.conf file will | ||
426 | # have come from) | ||
427 | first_regex = None | ||
428 | layerdir = layers[0] | ||
429 | for layername, pattern, regex, _ in self.bbhandler.cooker.recipecache.bbfile_config_priorities: | ||
430 | if regex.match(os.path.join(layerdir, 'test')): | ||
431 | first_regex = regex | ||
432 | break | ||
433 | |||
434 | if first_regex: | ||
435 | # Find the BBFILES entries that match (which will have come from this conf/layer.conf file) | ||
436 | bbfiles = str(self.bbhandler.config_data.getVar('BBFILES', True)).split() | ||
437 | bbfiles_layer = [] | ||
438 | for item in bbfiles: | ||
439 | if first_regex.match(item): | ||
440 | newpath = os.path.join(outputdir, item[len(layerdir)+1:]) | ||
441 | bbfiles_layer.append(newpath) | ||
442 | |||
443 | if bbfiles_layer: | ||
444 | # Check that all important layer files match BBFILES | ||
445 | for root, dirs, files in os.walk(outputdir): | ||
446 | for f1 in files: | ||
447 | ext = os.path.splitext(f1)[1] | ||
448 | if ext in ['.bb', '.bbappend']: | ||
449 | f1full = os.sep.join([root, f1]) | ||
450 | entry_found = False | ||
451 | for item in bbfiles_layer: | ||
452 | if fnmatch.fnmatch(f1full, item): | ||
453 | entry_found = True | ||
454 | break | ||
455 | if not entry_found: | ||
456 | logger.warning("File %s does not match the flattened layer's BBFILES setting, you may need to edit conf/layer.conf or move the file elsewhere" % f1full) | ||
457 | |||
458 | def get_file_layer(self, filename): | ||
459 | for layer, _, regex, _ in self.bbhandler.cooker.recipecache.bbfile_config_priorities: | ||
460 | if regex.match(filename): | ||
461 | for layerdir in self.bblayers: | ||
462 | if regex.match(os.path.join(layerdir, 'test')) and re.match(layerdir, filename): | ||
463 | return self.get_layer_name(layerdir) | ||
464 | return "?" | ||
465 | |||
466 | def get_file_layerdir(self, filename): | ||
467 | for layer, _, regex, _ in self.bbhandler.cooker.recipecache.bbfile_config_priorities: | ||
468 | if regex.match(filename): | ||
469 | for layerdir in self.bblayers: | ||
470 | if regex.match(os.path.join(layerdir, 'test')) and re.match(layerdir, filename): | ||
471 | return layerdir | ||
472 | return "?" | ||
473 | |||
474 | def remove_layer_prefix(self, f): | ||
475 | """Remove the layer_dir prefix, e.g., f = /path/to/layer_dir/foo/blah, the | ||
476 | return value will be: layer_dir/foo/blah""" | ||
477 | f_layerdir = self.get_file_layerdir(f) | ||
478 | prefix = os.path.join(os.path.dirname(f_layerdir), '') | ||
479 | return f[len(prefix):] if f.startswith(prefix) else f | ||
480 | |||
481 | def get_layer_name(self, layerdir): | ||
482 | return os.path.basename(layerdir.rstrip(os.sep)) | ||
483 | |||
484 | def apply_append(self, appendname, recipename): | ||
485 | appendfile = open(appendname, 'r') | ||
486 | recipefile = open(recipename, 'a') | ||
487 | recipefile.write('\n') | ||
488 | recipefile.write('##### bbappended from %s #####\n' % self.get_file_layer(appendname)) | ||
489 | recipefile.writelines(appendfile.readlines()) | ||
490 | recipefile.close() | ||
491 | appendfile.close() | ||
492 | |||
493 | def do_show_appends(self, args): | ||
494 | """list bbappend files and recipe files they apply to | ||
495 | |||
496 | usage: show-appends | ||
497 | |||
498 | Recipes are listed with the bbappends that apply to them as subitems. | ||
499 | """ | ||
500 | self.bbhandler.prepare() | ||
501 | if not self.bbhandler.cooker.collection.appendlist: | ||
502 | logger.plain('No append files found') | ||
503 | return | ||
504 | |||
505 | logger.plain('=== Appended recipes ===') | ||
506 | |||
507 | pnlist = list(self.bbhandler.cooker_data.pkg_pn.keys()) | ||
508 | pnlist.sort() | ||
509 | for pn in pnlist: | ||
510 | self.show_appends_for_pn(pn) | ||
511 | |||
512 | self.show_appends_for_skipped() | ||
513 | |||
514 | def show_appends_for_pn(self, pn): | ||
515 | filenames = self.bbhandler.cooker_data.pkg_pn[pn] | ||
516 | |||
517 | best = bb.providers.findBestProvider(pn, | ||
518 | self.bbhandler.config_data, | ||
519 | self.bbhandler.cooker_data, | ||
520 | self.bbhandler.cooker_data.pkg_pn) | ||
521 | best_filename = os.path.basename(best[3]) | ||
522 | |||
523 | self.show_appends_output(filenames, best_filename) | ||
524 | |||
525 | def show_appends_for_skipped(self): | ||
526 | filenames = [os.path.basename(f) | ||
527 | for f in self.bbhandler.cooker.skiplist.iterkeys()] | ||
528 | self.show_appends_output(filenames, None, " (skipped)") | ||
529 | |||
530 | def show_appends_output(self, filenames, best_filename, name_suffix = ''): | ||
531 | appended, missing = self.get_appends_for_files(filenames) | ||
532 | if appended: | ||
533 | for basename, appends in appended: | ||
534 | logger.plain('%s%s:', basename, name_suffix) | ||
535 | for append in appends: | ||
536 | logger.plain(' %s', append) | ||
537 | |||
538 | if best_filename: | ||
539 | if best_filename in missing: | ||
540 | logger.warn('%s: missing append for preferred version', | ||
541 | best_filename) | ||
542 | self.returncode |= 1 | ||
543 | |||
544 | |||
545 | def get_appends_for_files(self, filenames): | ||
546 | appended, notappended = [], [] | ||
547 | for filename in filenames: | ||
548 | _, cls = bb.cache.Cache.virtualfn2realfn(filename) | ||
549 | if cls: | ||
550 | continue | ||
551 | |||
552 | basename = os.path.basename(filename) | ||
553 | appends = self.bbhandler.cooker.collection.appendlist.get(basename) | ||
554 | if appends: | ||
555 | appended.append((basename, list(appends))) | ||
556 | else: | ||
557 | notappended.append(basename) | ||
558 | return appended, notappended | ||
559 | |||
560 | def do_show_cross_depends(self, args): | ||
561 | """figure out the dependency between recipes that crosses a layer boundary. | ||
562 | |||
563 | usage: show-cross-depends [-f] | ||
564 | |||
565 | Figure out the dependency between recipes that crosses a layer boundary. | ||
566 | |||
567 | Options: | ||
568 | -f show full file path | ||
569 | |||
570 | NOTE: | ||
571 | The .bbappend file can impact the dependency. | ||
572 | """ | ||
573 | self.bbhandler.prepare() | ||
574 | |||
575 | show_filenames = False | ||
576 | for arg in args.split(): | ||
577 | if arg == '-f': | ||
578 | show_filenames = True | ||
579 | else: | ||
580 | sys.stderr.write("show-cross-depends: invalid option %s\n" % arg) | ||
581 | self.do_help('') | ||
582 | return | ||
583 | |||
584 | pkg_fn = self.bbhandler.cooker_data.pkg_fn | ||
585 | bbpath = str(self.bbhandler.config_data.getVar('BBPATH', True)) | ||
586 | self.require_re = re.compile(r"require\s+(.+)") | ||
587 | self.include_re = re.compile(r"include\s+(.+)") | ||
588 | self.inherit_re = re.compile(r"inherit\s+(.+)") | ||
589 | |||
590 | # The bb's DEPENDS and RDEPENDS | ||
591 | for f in pkg_fn: | ||
592 | f = bb.cache.Cache.virtualfn2realfn(f)[0] | ||
593 | # Get the layername that the file is in | ||
594 | layername = self.get_file_layer(f) | ||
595 | |||
596 | # The DEPENDS | ||
597 | deps = self.bbhandler.cooker_data.deps[f] | ||
598 | for pn in deps: | ||
599 | if pn in self.bbhandler.cooker_data.pkg_pn: | ||
600 | best = bb.providers.findBestProvider(pn, | ||
601 | self.bbhandler.config_data, | ||
602 | self.bbhandler.cooker_data, | ||
603 | self.bbhandler.cooker_data.pkg_pn) | ||
604 | self.check_cross_depends("DEPENDS", layername, f, best[3], show_filenames) | ||
605 | |||
606 | # The RDPENDS | ||
607 | all_rdeps = self.bbhandler.cooker_data.rundeps[f].values() | ||
608 | # Remove the duplicated or null one. | ||
609 | sorted_rdeps = {} | ||
610 | # The all_rdeps is the list in list, so we need two for loops | ||
611 | for k1 in all_rdeps: | ||
612 | for k2 in k1: | ||
613 | sorted_rdeps[k2] = 1 | ||
614 | all_rdeps = sorted_rdeps.keys() | ||
615 | for rdep in all_rdeps: | ||
616 | all_p = bb.providers.getRuntimeProviders(self.bbhandler.cooker_data, rdep) | ||
617 | if all_p: | ||
618 | best = bb.providers.filterProvidersRunTime(all_p, rdep, | ||
619 | self.bbhandler.config_data, | ||
620 | self.bbhandler.cooker_data)[0][0] | ||
621 | self.check_cross_depends("RDEPENDS", layername, f, best, show_filenames) | ||
622 | |||
623 | # The inherit class | ||
624 | cls_re = re.compile('classes/') | ||
625 | if f in self.bbhandler.cooker_data.inherits: | ||
626 | inherits = self.bbhandler.cooker_data.inherits[f] | ||
627 | for cls in inherits: | ||
628 | # The inherits' format is [classes/cls, /path/to/classes/cls] | ||
629 | # ignore the classes/cls. | ||
630 | if not cls_re.match(cls): | ||
631 | inherit_layername = self.get_file_layer(cls) | ||
632 | if inherit_layername != layername: | ||
633 | if not show_filenames: | ||
634 | f_short = self.remove_layer_prefix(f) | ||
635 | cls = self.remove_layer_prefix(cls) | ||
636 | else: | ||
637 | f_short = f | ||
638 | logger.plain("%s inherits %s" % (f_short, cls)) | ||
639 | |||
640 | # The 'require/include xxx' in the bb file | ||
641 | pv_re = re.compile(r"\${PV}") | ||
642 | fnfile = open(f, 'r') | ||
643 | line = fnfile.readline() | ||
644 | while line: | ||
645 | m, keyword = self.match_require_include(line) | ||
646 | # Found the 'require/include xxxx' | ||
647 | if m: | ||
648 | needed_file = m.group(1) | ||
649 | # Replace the ${PV} with the real PV | ||
650 | if pv_re.search(needed_file) and f in self.bbhandler.cooker_data.pkg_pepvpr: | ||
651 | pv = self.bbhandler.cooker_data.pkg_pepvpr[f][1] | ||
652 | needed_file = re.sub(r"\${PV}", pv, needed_file) | ||
653 | self.print_cross_files(bbpath, keyword, layername, f, needed_file, show_filenames) | ||
654 | line = fnfile.readline() | ||
655 | fnfile.close() | ||
656 | |||
657 | # The "require/include xxx" in conf/machine/*.conf, .inc and .bbclass | ||
658 | conf_re = re.compile(".*/conf/machine/[^\/]*\.conf$") | ||
659 | inc_re = re.compile(".*\.inc$") | ||
660 | # The "inherit xxx" in .bbclass | ||
661 | bbclass_re = re.compile(".*\.bbclass$") | ||
662 | for layerdir in self.bblayers: | ||
663 | layername = self.get_layer_name(layerdir) | ||
664 | for dirpath, dirnames, filenames in os.walk(layerdir): | ||
665 | for name in filenames: | ||
666 | f = os.path.join(dirpath, name) | ||
667 | s = conf_re.match(f) or inc_re.match(f) or bbclass_re.match(f) | ||
668 | if s: | ||
669 | ffile = open(f, 'r') | ||
670 | line = ffile.readline() | ||
671 | while line: | ||
672 | m, keyword = self.match_require_include(line) | ||
673 | # Only bbclass has the "inherit xxx" here. | ||
674 | bbclass="" | ||
675 | if not m and f.endswith(".bbclass"): | ||
676 | m, keyword = self.match_inherit(line) | ||
677 | bbclass=".bbclass" | ||
678 | # Find a 'require/include xxxx' | ||
679 | if m: | ||
680 | self.print_cross_files(bbpath, keyword, layername, f, m.group(1) + bbclass, show_filenames) | ||
681 | line = ffile.readline() | ||
682 | ffile.close() | ||
683 | |||
684 | def print_cross_files(self, bbpath, keyword, layername, f, needed_filename, show_filenames): | ||
685 | """Print the depends that crosses a layer boundary""" | ||
686 | needed_file = bb.utils.which(bbpath, needed_filename) | ||
687 | if needed_file: | ||
688 | # Which layer is this file from | ||
689 | needed_layername = self.get_file_layer(needed_file) | ||
690 | if needed_layername != layername: | ||
691 | if not show_filenames: | ||
692 | f = self.remove_layer_prefix(f) | ||
693 | needed_file = self.remove_layer_prefix(needed_file) | ||
694 | logger.plain("%s %s %s" %(f, keyword, needed_file)) | ||
695 | def match_inherit(self, line): | ||
696 | """Match the inherit xxx line""" | ||
697 | return (self.inherit_re.match(line), "inherits") | ||
698 | |||
699 | def match_require_include(self, line): | ||
700 | """Match the require/include xxx line""" | ||
701 | m = self.require_re.match(line) | ||
702 | keyword = "requires" | ||
703 | if not m: | ||
704 | m = self.include_re.match(line) | ||
705 | keyword = "includes" | ||
706 | return (m, keyword) | ||
707 | |||
708 | def check_cross_depends(self, keyword, layername, f, needed_file, show_filenames): | ||
709 | """Print the DEPENDS/RDEPENDS file that crosses a layer boundary""" | ||
710 | best_realfn = bb.cache.Cache.virtualfn2realfn(needed_file)[0] | ||
711 | needed_layername = self.get_file_layer(best_realfn) | ||
712 | if needed_layername != layername: | ||
713 | if not show_filenames: | ||
714 | f = self.remove_layer_prefix(f) | ||
715 | best_realfn = self.remove_layer_prefix(best_realfn) | ||
716 | |||
717 | logger.plain("%s %s %s" % (f, keyword, best_realfn)) | ||
718 | |||
719 | if __name__ == '__main__': | ||
720 | sys.exit(main(sys.argv[1:]) or 0) | ||
diff --git a/bitbake/bin/bitbake-prserv b/bitbake/bin/bitbake-prserv new file mode 100755 index 0000000000..a8d7acb4c2 --- /dev/null +++ b/bitbake/bin/bitbake-prserv | |||
@@ -0,0 +1,55 @@ | |||
1 | #!/usr/bin/env python | ||
2 | import os | ||
3 | import sys,logging | ||
4 | import optparse | ||
5 | |||
6 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)),'lib')) | ||
7 | |||
8 | import prserv | ||
9 | import prserv.serv | ||
10 | |||
11 | __version__="1.0.0" | ||
12 | |||
13 | PRHOST_DEFAULT='0.0.0.0' | ||
14 | PRPORT_DEFAULT=8585 | ||
15 | |||
16 | def main(): | ||
17 | parser = optparse.OptionParser( | ||
18 | version="Bitbake PR Service Core version %s, %%prog version %s" % (prserv.__version__, __version__), | ||
19 | usage = "%prog < --start | --stop > [options]") | ||
20 | |||
21 | parser.add_option("-f", "--file", help="database filename(default: prserv.sqlite3)", action="store", | ||
22 | dest="dbfile", type="string", default="prserv.sqlite3") | ||
23 | parser.add_option("-l", "--log", help="log filename(default: prserv.log)", action="store", | ||
24 | dest="logfile", type="string", default="prserv.log") | ||
25 | parser.add_option("--loglevel", help="logging level, i.e. CRITICAL, ERROR, WARNING, INFO, DEBUG", | ||
26 | action = "store", type="string", dest="loglevel", default = "INFO") | ||
27 | parser.add_option("--start", help="start daemon", | ||
28 | action="store_true", dest="start") | ||
29 | parser.add_option("--stop", help="stop daemon", | ||
30 | action="store_true", dest="stop") | ||
31 | parser.add_option("--host", help="ip address to bind", action="store", | ||
32 | dest="host", type="string", default=PRHOST_DEFAULT) | ||
33 | parser.add_option("--port", help="port number(default: 8585)", action="store", | ||
34 | dest="port", type="int", default=PRPORT_DEFAULT) | ||
35 | |||
36 | options, args = parser.parse_args(sys.argv) | ||
37 | prserv.init_logger(os.path.abspath(options.logfile),options.loglevel) | ||
38 | |||
39 | if options.start: | ||
40 | ret=prserv.serv.start_daemon(options.dbfile, options.host, options.port,os.path.abspath(options.logfile)) | ||
41 | elif options.stop: | ||
42 | ret=prserv.serv.stop_daemon(options.host, options.port) | ||
43 | else: | ||
44 | ret=parser.print_help() | ||
45 | return ret | ||
46 | |||
47 | if __name__ == "__main__": | ||
48 | try: | ||
49 | ret = main() | ||
50 | except Exception: | ||
51 | ret = 1 | ||
52 | import traceback | ||
53 | traceback.print_exc(5) | ||
54 | sys.exit(ret) | ||
55 | |||
diff --git a/bitbake/bin/bitbake-selftest b/bitbake/bin/bitbake-selftest new file mode 100755 index 0000000000..48a58fef67 --- /dev/null +++ b/bitbake/bin/bitbake-selftest | |||
@@ -0,0 +1,38 @@ | |||
1 | #!/usr/bin/env python | ||
2 | # | ||
3 | # Copyright (C) 2012 Richard Purdie | ||
4 | # | ||
5 | # This program is free software; you can redistribute it and/or modify | ||
6 | # it under the terms of the GNU General Public License version 2 as | ||
7 | # published by the Free Software Foundation. | ||
8 | # | ||
9 | # This program is distributed in the hope that it will be useful, | ||
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
12 | # GNU General Public License for more details. | ||
13 | # | ||
14 | # You should have received a copy of the GNU General Public License along | ||
15 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
16 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
17 | |||
18 | import os | ||
19 | import sys, logging | ||
20 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), 'lib')) | ||
21 | |||
22 | import unittest | ||
23 | try: | ||
24 | import bb | ||
25 | except RuntimeError as exc: | ||
26 | sys.exit(str(exc)) | ||
27 | |||
28 | tests = ["bb.tests.codeparser", | ||
29 | "bb.tests.cow", | ||
30 | "bb.tests.data", | ||
31 | "bb.tests.fetch", | ||
32 | "bb.tests.utils"] | ||
33 | |||
34 | for t in tests: | ||
35 | __import__(t) | ||
36 | |||
37 | unittest.main(argv=["bitbake-selftest"] + tests) | ||
38 | |||
diff --git a/bitbake/bin/bitbake-worker b/bitbake/bin/bitbake-worker new file mode 100755 index 0000000000..66b6aabfdb --- /dev/null +++ b/bitbake/bin/bitbake-worker | |||
@@ -0,0 +1,363 @@ | |||
1 | #!/usr/bin/env python | ||
2 | |||
3 | import os | ||
4 | import sys | ||
5 | import warnings | ||
6 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(sys.argv[0])), 'lib')) | ||
7 | from bb import fetch2 | ||
8 | import logging | ||
9 | import bb | ||
10 | import select | ||
11 | import errno | ||
12 | import signal | ||
13 | |||
14 | # Users shouldn't be running this code directly | ||
15 | if len(sys.argv) != 2 or sys.argv[1] != "decafbad": | ||
16 | print("bitbake-worker is meant for internal execution by bitbake itself, please don't use it standalone.") | ||
17 | sys.exit(1) | ||
18 | |||
19 | logger = logging.getLogger("BitBake") | ||
20 | |||
21 | try: | ||
22 | import cPickle as pickle | ||
23 | except ImportError: | ||
24 | import pickle | ||
25 | bb.msg.note(1, bb.msg.domain.Cache, "Importing cPickle failed. Falling back to a very slow implementation.") | ||
26 | |||
27 | |||
28 | worker_pipe = sys.stdout.fileno() | ||
29 | bb.utils.nonblockingfd(worker_pipe) | ||
30 | |||
31 | handler = bb.event.LogHandler() | ||
32 | logger.addHandler(handler) | ||
33 | |||
34 | if 0: | ||
35 | # Code to write out a log file of all events passing through the worker | ||
36 | logfilename = "/tmp/workerlogfile" | ||
37 | format_str = "%(levelname)s: %(message)s" | ||
38 | conlogformat = bb.msg.BBLogFormatter(format_str) | ||
39 | consolelog = logging.FileHandler(logfilename) | ||
40 | bb.msg.addDefaultlogFilter(consolelog) | ||
41 | consolelog.setFormatter(conlogformat) | ||
42 | logger.addHandler(consolelog) | ||
43 | |||
44 | worker_queue = "" | ||
45 | |||
46 | def worker_fire(event, d): | ||
47 | data = "<event>" + pickle.dumps(event) + "</event>" | ||
48 | worker_fire_prepickled(data) | ||
49 | |||
50 | def worker_fire_prepickled(event): | ||
51 | global worker_queue | ||
52 | |||
53 | worker_queue = worker_queue + event | ||
54 | worker_flush() | ||
55 | |||
56 | def worker_flush(): | ||
57 | global worker_queue, worker_pipe | ||
58 | |||
59 | if not worker_queue: | ||
60 | return | ||
61 | |||
62 | try: | ||
63 | written = os.write(worker_pipe, worker_queue) | ||
64 | worker_queue = worker_queue[written:] | ||
65 | except (IOError, OSError) as e: | ||
66 | if e.errno != errno.EAGAIN: | ||
67 | raise | ||
68 | |||
69 | def worker_child_fire(event, d): | ||
70 | global worker_pipe | ||
71 | |||
72 | data = "<event>" + pickle.dumps(event) + "</event>" | ||
73 | worker_pipe.write(data) | ||
74 | |||
75 | bb.event.worker_fire = worker_fire | ||
76 | |||
77 | lf = None | ||
78 | #lf = open("/tmp/workercommandlog", "w+") | ||
79 | def workerlog_write(msg): | ||
80 | if lf: | ||
81 | lf.write(msg) | ||
82 | lf.flush() | ||
83 | |||
84 | def fork_off_task(cfg, data, workerdata, fn, task, taskname, appends, quieterrors=False): | ||
85 | # We need to setup the environment BEFORE the fork, since | ||
86 | # a fork() or exec*() activates PSEUDO... | ||
87 | |||
88 | envbackup = {} | ||
89 | fakeenv = {} | ||
90 | umask = None | ||
91 | |||
92 | taskdep = workerdata["taskdeps"][fn] | ||
93 | if 'umask' in taskdep and taskname in taskdep['umask']: | ||
94 | # umask might come in as a number or text string.. | ||
95 | try: | ||
96 | umask = int(taskdep['umask'][taskname],8) | ||
97 | except TypeError: | ||
98 | umask = taskdep['umask'][taskname] | ||
99 | |||
100 | if 'fakeroot' in taskdep and taskname in taskdep['fakeroot']: | ||
101 | envvars = (workerdata["fakerootenv"][fn] or "").split() | ||
102 | for key, value in (var.split('=') for var in envvars): | ||
103 | envbackup[key] = os.environ.get(key) | ||
104 | os.environ[key] = value | ||
105 | fakeenv[key] = value | ||
106 | |||
107 | fakedirs = (workerdata["fakerootdirs"][fn] or "").split() | ||
108 | for p in fakedirs: | ||
109 | bb.utils.mkdirhier(p) | ||
110 | logger.debug(2, 'Running %s:%s under fakeroot, fakedirs: %s' % | ||
111 | (fn, taskname, ', '.join(fakedirs))) | ||
112 | else: | ||
113 | envvars = (workerdata["fakerootnoenv"][fn] or "").split() | ||
114 | for key, value in (var.split('=') for var in envvars): | ||
115 | envbackup[key] = os.environ.get(key) | ||
116 | os.environ[key] = value | ||
117 | fakeenv[key] = value | ||
118 | |||
119 | sys.stdout.flush() | ||
120 | sys.stderr.flush() | ||
121 | |||
122 | try: | ||
123 | pipein, pipeout = os.pipe() | ||
124 | pipein = os.fdopen(pipein, 'rb', 4096) | ||
125 | pipeout = os.fdopen(pipeout, 'wb', 0) | ||
126 | pid = os.fork() | ||
127 | except OSError as e: | ||
128 | bb.msg.fatal("RunQueue", "fork failed: %d (%s)" % (e.errno, e.strerror)) | ||
129 | |||
130 | if pid == 0: | ||
131 | global worker_pipe | ||
132 | pipein.close() | ||
133 | |||
134 | # Save out the PID so that the event can include it the | ||
135 | # events | ||
136 | bb.event.worker_pid = os.getpid() | ||
137 | bb.event.worker_fire = worker_child_fire | ||
138 | worker_pipe = pipeout | ||
139 | |||
140 | # Make the child the process group leader | ||
141 | os.setpgid(0, 0) | ||
142 | # No stdin | ||
143 | newsi = os.open(os.devnull, os.O_RDWR) | ||
144 | os.dup2(newsi, sys.stdin.fileno()) | ||
145 | |||
146 | if umask: | ||
147 | os.umask(umask) | ||
148 | |||
149 | data.setVar("BB_WORKERCONTEXT", "1") | ||
150 | data.setVar("BUILDNAME", workerdata["buildname"]) | ||
151 | data.setVar("DATE", workerdata["date"]) | ||
152 | data.setVar("TIME", workerdata["time"]) | ||
153 | bb.parse.siggen.set_taskdata(workerdata["hashes"], workerdata["hash_deps"], workerdata["sigchecksums"]) | ||
154 | ret = 0 | ||
155 | try: | ||
156 | the_data = bb.cache.Cache.loadDataFull(fn, appends, data) | ||
157 | the_data.setVar('BB_TASKHASH', workerdata["runq_hash"][task]) | ||
158 | for h in workerdata["hashes"]: | ||
159 | the_data.setVar("BBHASH_%s" % h, workerdata["hashes"][h]) | ||
160 | for h in workerdata["hash_deps"]: | ||
161 | the_data.setVar("BBHASHDEPS_%s" % h, workerdata["hash_deps"][h]) | ||
162 | |||
163 | # exported_vars() returns a generator which *cannot* be passed to os.environ.update() | ||
164 | # successfully. We also need to unset anything from the environment which shouldn't be there | ||
165 | exports = bb.data.exported_vars(the_data) | ||
166 | bb.utils.empty_environment() | ||
167 | for e, v in exports: | ||
168 | os.environ[e] = v | ||
169 | for e in fakeenv: | ||
170 | os.environ[e] = fakeenv[e] | ||
171 | the_data.setVar(e, fakeenv[e]) | ||
172 | the_data.setVarFlag(e, 'export', "1") | ||
173 | |||
174 | if quieterrors: | ||
175 | the_data.setVarFlag(taskname, "quieterrors", "1") | ||
176 | |||
177 | except Exception as exc: | ||
178 | if not quieterrors: | ||
179 | logger.critical(str(exc)) | ||
180 | os._exit(1) | ||
181 | try: | ||
182 | if not cfg.dry_run: | ||
183 | ret = bb.build.exec_task(fn, taskname, the_data, cfg.profile) | ||
184 | os._exit(ret) | ||
185 | except: | ||
186 | os._exit(1) | ||
187 | else: | ||
188 | for key, value in envbackup.iteritems(): | ||
189 | if value is None: | ||
190 | del os.environ[key] | ||
191 | else: | ||
192 | os.environ[key] = value | ||
193 | |||
194 | return pid, pipein, pipeout | ||
195 | |||
196 | class runQueueWorkerPipe(): | ||
197 | """ | ||
198 | Abstraction for a pipe between a worker thread and the worker server | ||
199 | """ | ||
200 | def __init__(self, pipein, pipeout): | ||
201 | self.input = pipein | ||
202 | if pipeout: | ||
203 | pipeout.close() | ||
204 | bb.utils.nonblockingfd(self.input) | ||
205 | self.queue = "" | ||
206 | |||
207 | def read(self): | ||
208 | start = len(self.queue) | ||
209 | try: | ||
210 | self.queue = self.queue + self.input.read(102400) | ||
211 | except (OSError, IOError) as e: | ||
212 | if e.errno != errno.EAGAIN: | ||
213 | raise | ||
214 | |||
215 | end = len(self.queue) | ||
216 | index = self.queue.find("</event>") | ||
217 | while index != -1: | ||
218 | worker_fire_prepickled(self.queue[:index+8]) | ||
219 | self.queue = self.queue[index+8:] | ||
220 | index = self.queue.find("</event>") | ||
221 | return (end > start) | ||
222 | |||
223 | def close(self): | ||
224 | while self.read(): | ||
225 | continue | ||
226 | if len(self.queue) > 0: | ||
227 | print("Warning, worker child left partial message: %s" % self.queue) | ||
228 | self.input.close() | ||
229 | |||
230 | normalexit = False | ||
231 | |||
232 | class BitbakeWorker(object): | ||
233 | def __init__(self, din): | ||
234 | self.input = din | ||
235 | bb.utils.nonblockingfd(self.input) | ||
236 | self.queue = "" | ||
237 | self.cookercfg = None | ||
238 | self.databuilder = None | ||
239 | self.data = None | ||
240 | self.build_pids = {} | ||
241 | self.build_pipes = {} | ||
242 | |||
243 | def serve(self): | ||
244 | while True: | ||
245 | (ready, _, _) = select.select([self.input] + [i.input for i in self.build_pipes.values()], [] , [], 1) | ||
246 | if self.input in ready or len(self.queue): | ||
247 | start = len(self.queue) | ||
248 | try: | ||
249 | self.queue = self.queue + self.input.read() | ||
250 | except (OSError, IOError): | ||
251 | pass | ||
252 | end = len(self.queue) | ||
253 | self.handle_item("cookerconfig", self.handle_cookercfg) | ||
254 | self.handle_item("workerdata", self.handle_workerdata) | ||
255 | self.handle_item("runtask", self.handle_runtask) | ||
256 | self.handle_item("finishnow", self.handle_finishnow) | ||
257 | self.handle_item("ping", self.handle_ping) | ||
258 | self.handle_item("quit", self.handle_quit) | ||
259 | |||
260 | for pipe in self.build_pipes: | ||
261 | self.build_pipes[pipe].read() | ||
262 | if len(self.build_pids): | ||
263 | self.process_waitpid() | ||
264 | worker_flush() | ||
265 | |||
266 | |||
267 | def handle_item(self, item, func): | ||
268 | if self.queue.startswith("<" + item + ">"): | ||
269 | index = self.queue.find("</" + item + ">") | ||
270 | while index != -1: | ||
271 | func(self.queue[(len(item) + 2):index]) | ||
272 | self.queue = self.queue[(index + len(item) + 3):] | ||
273 | index = self.queue.find("</" + item + ">") | ||
274 | |||
275 | def handle_cookercfg(self, data): | ||
276 | self.cookercfg = pickle.loads(data) | ||
277 | self.databuilder = bb.cookerdata.CookerDataBuilder(self.cookercfg, worker=True) | ||
278 | self.databuilder.parseBaseConfiguration() | ||
279 | self.data = self.databuilder.data | ||
280 | |||
281 | def handle_workerdata(self, data): | ||
282 | self.workerdata = pickle.loads(data) | ||
283 | bb.msg.loggerDefaultDebugLevel = self.workerdata["logdefaultdebug"] | ||
284 | bb.msg.loggerDefaultVerbose = self.workerdata["logdefaultverbose"] | ||
285 | bb.msg.loggerVerboseLogs = self.workerdata["logdefaultverboselogs"] | ||
286 | bb.msg.loggerDefaultDomains = self.workerdata["logdefaultdomain"] | ||
287 | self.data.setVar("PRSERV_HOST", self.workerdata["prhost"]) | ||
288 | |||
289 | def handle_ping(self, _): | ||
290 | workerlog_write("Handling ping\n") | ||
291 | |||
292 | logger.warn("Pong from bitbake-worker!") | ||
293 | |||
294 | def handle_quit(self, data): | ||
295 | workerlog_write("Handling quit\n") | ||
296 | |||
297 | global normalexit | ||
298 | normalexit = True | ||
299 | sys.exit(0) | ||
300 | |||
301 | def handle_runtask(self, data): | ||
302 | fn, task, taskname, quieterrors, appends = pickle.loads(data) | ||
303 | workerlog_write("Handling runtask %s %s %s\n" % (task, fn, taskname)) | ||
304 | |||
305 | pid, pipein, pipeout = fork_off_task(self.cookercfg, self.data, self.workerdata, fn, task, taskname, appends, quieterrors) | ||
306 | |||
307 | self.build_pids[pid] = task | ||
308 | self.build_pipes[pid] = runQueueWorkerPipe(pipein, pipeout) | ||
309 | |||
310 | def process_waitpid(self): | ||
311 | """ | ||
312 | Return none is there are no processes awaiting result collection, otherwise | ||
313 | collect the process exit codes and close the information pipe. | ||
314 | """ | ||
315 | try: | ||
316 | pid, status = os.waitpid(-1, os.WNOHANG) | ||
317 | if pid == 0 or os.WIFSTOPPED(status): | ||
318 | return None | ||
319 | except OSError: | ||
320 | return None | ||
321 | |||
322 | workerlog_write("Exit code of %s for pid %s\n" % (status, pid)) | ||
323 | |||
324 | if os.WIFEXITED(status): | ||
325 | status = os.WEXITSTATUS(status) | ||
326 | elif os.WIFSIGNALED(status): | ||
327 | # Per shell conventions for $?, when a process exits due to | ||
328 | # a signal, we return an exit code of 128 + SIGNUM | ||
329 | status = 128 + os.WTERMSIG(status) | ||
330 | |||
331 | task = self.build_pids[pid] | ||
332 | del self.build_pids[pid] | ||
333 | |||
334 | self.build_pipes[pid].close() | ||
335 | del self.build_pipes[pid] | ||
336 | |||
337 | worker_fire_prepickled("<exitcode>" + pickle.dumps((task, status)) + "</exitcode>") | ||
338 | |||
339 | def handle_finishnow(self, _): | ||
340 | if self.build_pids: | ||
341 | logger.info("Sending SIGTERM to remaining %s tasks", len(self.build_pids)) | ||
342 | for k, v in self.build_pids.iteritems(): | ||
343 | try: | ||
344 | os.kill(-k, signal.SIGTERM) | ||
345 | os.waitpid(-1, 0) | ||
346 | except: | ||
347 | pass | ||
348 | for pipe in self.build_pipes: | ||
349 | self.build_pipes[pipe].read() | ||
350 | |||
351 | try: | ||
352 | worker = BitbakeWorker(sys.stdin) | ||
353 | worker.serve() | ||
354 | except BaseException as e: | ||
355 | if not normalexit: | ||
356 | import traceback | ||
357 | sys.stderr.write(traceback.format_exc()) | ||
358 | sys.stderr.write(str(e)) | ||
359 | while len(worker_queue): | ||
360 | worker_flush() | ||
361 | workerlog_write("exitting") | ||
362 | sys.exit(0) | ||
363 | |||
diff --git a/bitbake/bin/bitdoc b/bitbake/bin/bitdoc new file mode 100755 index 0000000000..576d88b574 --- /dev/null +++ b/bitbake/bin/bitdoc | |||
@@ -0,0 +1,531 @@ | |||
1 | #!/usr/bin/env python | ||
2 | # ex:ts=4:sw=4:sts=4:et | ||
3 | # -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- | ||
4 | # | ||
5 | # Copyright (C) 2005 Holger Hans Peter Freyther | ||
6 | # | ||
7 | # This program is free software; you can redistribute it and/or modify | ||
8 | # it under the terms of the GNU General Public License version 2 as | ||
9 | # published by the Free Software Foundation. | ||
10 | # | ||
11 | # This program is distributed in the hope that it will be useful, | ||
12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
14 | # GNU General Public License for more details. | ||
15 | # | ||
16 | # You should have received a copy of the GNU General Public License along | ||
17 | # with this program; if not, write to the Free Software Foundation, Inc., | ||
18 | # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. | ||
19 | |||
20 | import optparse, os, sys | ||
21 | |||
22 | # bitbake | ||
23 | sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__), 'lib')) | ||
24 | import bb | ||
25 | import bb.parse | ||
26 | from string import split, join | ||
27 | |||
28 | __version__ = "0.0.2" | ||
29 | |||
30 | class HTMLFormatter: | ||
31 | """ | ||
32 | Simple class to help to generate some sort of HTML files. It is | ||
33 | quite inferior solution compared to docbook, gtkdoc, doxygen but it | ||
34 | should work for now. | ||
35 | We've a global introduction site (index.html) and then one site for | ||
36 | the list of keys (alphabetical sorted) and one for the list of groups, | ||
37 | one site for each key with links to the relations and groups. | ||
38 | |||
39 | index.html | ||
40 | all_keys.html | ||
41 | all_groups.html | ||
42 | groupNAME.html | ||
43 | keyNAME.html | ||
44 | """ | ||
45 | |||
46 | def replace(self, text, *pairs): | ||
47 | """ | ||
48 | From pydoc... almost identical at least | ||
49 | """ | ||
50 | while pairs: | ||
51 | (a, b) = pairs[0] | ||
52 | text = join(split(text, a), b) | ||
53 | pairs = pairs[1:] | ||
54 | return text | ||
55 | def escape(self, text): | ||
56 | """ | ||
57 | Escape string to be conform HTML | ||
58 | """ | ||
59 | return self.replace(text, | ||
60 | ('&', '&'), | ||
61 | ('<', '<' ), | ||
62 | ('>', '>' ) ) | ||
63 | def createNavigator(self): | ||
64 | """ | ||
65 | Create the navgiator | ||
66 | """ | ||
67 | return """<table class="navigation" width="100%" summary="Navigation header" cellpadding="2" cellspacing="2"> | ||
68 | <tr valign="middle"> | ||
69 | <td><a accesskey="g" href="index.html">Home</a></td> | ||
70 | <td><a accesskey="n" href="all_groups.html">Groups</a></td> | ||
71 | <td><a accesskey="u" href="all_keys.html">Keys</a></td> | ||
72 | </tr></table> | ||
73 | """ | ||
74 | |||
75 | def relatedKeys(self, item): | ||
76 | """ | ||
77 | Create HTML to link to foreign keys | ||
78 | """ | ||
79 | |||
80 | if len(item.related()) == 0: | ||
81 | return "" | ||
82 | |||
83 | txt = "<p><b>See also:</b><br>" | ||
84 | txts = [] | ||
85 | for it in item.related(): | ||
86 | txts.append("""<a href="key%(it)s.html">%(it)s</a>""" % vars() ) | ||
87 | |||
88 | return txt + ",".join(txts) | ||
89 | |||
90 | def groups(self, item): | ||
91 | """ | ||
92 | Create HTML to link to related groups | ||
93 | """ | ||
94 | |||
95 | if len(item.groups()) == 0: | ||
96 | return "" | ||
97 | |||
98 | |||
99 | txt = "<p><b>See also:</b><br>" | ||
100 | txts = [] | ||
101 | for group in item.groups(): | ||
102 | txts.append( """<a href="group%s.html">%s</a> """ % (group, group) ) | ||
103 | |||
104 | return txt + ",".join(txts) | ||
105 | |||
106 | |||
107 | def createKeySite(self, item): | ||
108 | """ | ||
109 | Create a site for a key. It contains the header/navigator, a heading, | ||
110 | the description, links to related keys and to the groups. | ||
111 | """ | ||
112 | |||
113 | return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> | ||
114 | <html><head><title>Key %s</title></head> | ||
115 | <link rel="stylesheet" href="style.css" type="text/css"> | ||
116 | <body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"> | ||
117 | %s | ||
118 | <h2><span class="refentrytitle">%s</span></h2> | ||
119 | |||
120 | <div class="refsynopsisdiv"> | ||
121 | <h2>Synopsis</h2> | ||
122 | <p> | ||
123 | %s | ||
124 | </p> | ||
125 | </div> | ||
126 | |||
127 | <div class="refsynopsisdiv"> | ||
128 | <h2>Related Keys</h2> | ||
129 | <p> | ||
130 | %s | ||
131 | </p> | ||
132 | </div> | ||
133 | |||
134 | <div class="refsynopsisdiv"> | ||
135 | <h2>Groups</h2> | ||
136 | <p> | ||
137 | %s | ||
138 | </p> | ||
139 | </div> | ||
140 | |||
141 | |||
142 | </body> | ||
143 | """ % (item.name(), self.createNavigator(), item.name(), | ||
144 | self.escape(item.description()), self.relatedKeys(item), self.groups(item)) | ||
145 | |||
146 | def createGroupsSite(self, doc): | ||
147 | """ | ||
148 | Create the Group Overview site | ||
149 | """ | ||
150 | |||
151 | groups = "" | ||
152 | sorted_groups = sorted(doc.groups()) | ||
153 | for group in sorted_groups: | ||
154 | groups += """<a href="group%s.html">%s</a><br>""" % (group, group) | ||
155 | |||
156 | return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> | ||
157 | <html><head><title>Group overview</title></head> | ||
158 | <link rel="stylesheet" href="style.css" type="text/css"> | ||
159 | <body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"> | ||
160 | %s | ||
161 | <h2>Available Groups</h2> | ||
162 | %s | ||
163 | </body> | ||
164 | """ % (self.createNavigator(), groups) | ||
165 | |||
166 | def createIndex(self): | ||
167 | """ | ||
168 | Create the index file | ||
169 | """ | ||
170 | |||
171 | return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> | ||
172 | <html><head><title>Bitbake Documentation</title></head> | ||
173 | <link rel="stylesheet" href="style.css" type="text/css"> | ||
174 | <body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"> | ||
175 | %s | ||
176 | <h2>Documentation Entrance</h2> | ||
177 | <a href="all_groups.html">All available groups</a><br> | ||
178 | <a href="all_keys.html">All available keys</a><br> | ||
179 | </body> | ||
180 | """ % self.createNavigator() | ||
181 | |||
182 | def createKeysSite(self, doc): | ||
183 | """ | ||
184 | Create Overview of all avilable keys | ||
185 | """ | ||
186 | keys = "" | ||
187 | sorted_keys = sorted(doc.doc_keys()) | ||
188 | for key in sorted_keys: | ||
189 | keys += """<a href="key%s.html">%s</a><br>""" % (key, key) | ||
190 | |||
191 | return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> | ||
192 | <html><head><title>Key overview</title></head> | ||
193 | <link rel="stylesheet" href="style.css" type="text/css"> | ||
194 | <body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"> | ||
195 | %s | ||
196 | <h2>Available Keys</h2> | ||
197 | %s | ||
198 | </body> | ||
199 | """ % (self.createNavigator(), keys) | ||
200 | |||
201 | def createGroupSite(self, gr, items, _description = None): | ||
202 | """ | ||
203 | Create a site for a group: | ||
204 | Group the name of the group, items contain the name of the keys | ||
205 | inside this group | ||
206 | """ | ||
207 | groups = "" | ||
208 | description = "" | ||
209 | |||
210 | # create a section with the group descriptions | ||
211 | if _description: | ||
212 | description += "<h2 Description of Grozp %s</h2>" % gr | ||
213 | description += _description | ||
214 | |||
215 | items.sort(lambda x, y:cmp(x.name(), y.name())) | ||
216 | for group in items: | ||
217 | groups += """<a href="key%s.html">%s</a><br>""" % (group.name(), group.name()) | ||
218 | |||
219 | return """<!doctype html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> | ||
220 | <html><head><title>Group %s</title></head> | ||
221 | <link rel="stylesheet" href="style.css" type="text/css"> | ||
222 | <body bgcolor="white" text="black" link="#0000FF" vlink="#840084" alink="#0000FF"> | ||
223 | %s | ||
224 | %s | ||
225 | <div class="refsynopsisdiv"> | ||
226 | <h2>Keys in Group %s</h2> | ||
227 | <pre class="synopsis"> | ||
228 | %s | ||
229 | </pre> | ||
230 | </div> | ||
231 | </body> | ||
232 | """ % (gr, self.createNavigator(), description, gr, groups) | ||
233 | |||
234 | |||
235 | |||
236 | def createCSS(self): | ||
237 | """ | ||
238 | Create the CSS file | ||
239 | """ | ||
240 | return """.synopsis, .classsynopsis | ||
241 | { | ||
242 | background: #eeeeee; | ||
243 | border: solid 1px #aaaaaa; | ||
244 | padding: 0.5em; | ||
245 | } | ||
246 | .programlisting | ||
247 | { | ||
248 | background: #eeeeff; | ||
249 | border: solid 1px #aaaaff; | ||
250 | padding: 0.5em; | ||
251 | } | ||
252 | .variablelist | ||
253 | { | ||
254 | padding: 4px; | ||
255 | margin-left: 3em; | ||
256 | } | ||
257 | .variablelist td:first-child | ||
258 | { | ||
259 | vertical-align: top; | ||
260 | } | ||
261 | table.navigation | ||
262 | { | ||
263 | background: #ffeeee; | ||
264 | border: solid 1px #ffaaaa; | ||
265 | margin-top: 0.5em; | ||
266 | margin-bottom: 0.5em; | ||
267 | } | ||
268 | .navigation a | ||
269 | { | ||
270 | color: #770000; | ||
271 | } | ||
272 | .navigation a:visited | ||
273 | { | ||
274 | color: #550000; | ||
275 | } | ||
276 | .navigation .title | ||
277 | { | ||
278 | font-size: 200%; | ||
279 | } | ||
280 | div.refnamediv | ||
281 | { | ||
282 | margin-top: 2em; | ||
283 | } | ||
284 | div.gallery-float | ||
285 | { | ||
286 | float: left; | ||
287 | padding: 10px; | ||
288 | } | ||
289 | div.gallery-float img | ||
290 | { | ||
291 | border-style: none; | ||
292 | } | ||
293 | div.gallery-spacer | ||
294 | { | ||
295 | clear: both; | ||
296 | } | ||
297 | a | ||
298 | { | ||
299 | text-decoration: none; | ||
300 | } | ||
301 | a:hover | ||
302 | { | ||
303 | text-decoration: underline; | ||
304 | color: #FF0000; | ||
305 | } | ||
306 | """ | ||
307 | |||
308 | |||
309 | |||
310 | class DocumentationItem: | ||
311 | """ | ||
312 | A class to hold information about a configuration | ||
313 | item. It contains the key name, description, a list of related names, | ||
314 | and the group this item is contained in. | ||
315 | """ | ||
316 | |||
317 | def __init__(self): | ||
318 | self._groups = [] | ||
319 | self._related = [] | ||
320 | self._name = "" | ||
321 | self._desc = "" | ||
322 | |||
323 | def groups(self): | ||
324 | return self._groups | ||
325 | |||
326 | def name(self): | ||
327 | return self._name | ||
328 | |||
329 | def description(self): | ||
330 | return self._desc | ||
331 | |||
332 | def related(self): | ||
333 | return self._related | ||
334 | |||
335 | def setName(self, name): | ||
336 | self._name = name | ||
337 | |||
338 | def setDescription(self, desc): | ||
339 | self._desc = desc | ||
340 | |||
341 | def addGroup(self, group): | ||
342 | self._groups.append(group) | ||
343 | |||
344 | def addRelation(self, relation): | ||
345 | self._related.append(relation) | ||
346 | |||
347 | def sort(self): | ||
348 | self._related.sort() | ||
349 | self._groups.sort() | ||
350 | |||
351 | |||
352 | class Documentation: | ||
353 | """ | ||
354 | Holds the documentation... with mappings from key to items... | ||
355 | """ | ||
356 | |||
357 | def __init__(self): | ||
358 | self.__keys = {} | ||
359 | self.__groups = {} | ||
360 | |||
361 | def insert_doc_item(self, item): | ||
362 | """ | ||
363 | Insert the Doc Item into the internal list | ||
364 | of representation | ||
365 | """ | ||
366 | item.sort() | ||
367 | self.__keys[item.name()] = item | ||
368 | |||
369 | for group in item.groups(): | ||
370 | if not group in self.__groups: | ||
371 | self.__groups[group] = [] | ||
372 | self.__groups[group].append(item) | ||
373 | self.__groups[group].sort() | ||
374 | |||
375 | |||
376 | def doc_item(self, key): | ||
377 | """ | ||
378 | Return the DocumentationInstance describing the key | ||
379 | """ | ||
380 | try: | ||
381 | return self.__keys[key] | ||
382 | except KeyError: | ||
383 | return None | ||
384 | |||
385 | def doc_keys(self): | ||
386 | """ | ||
387 | Return the documented KEYS (names) | ||
388 | """ | ||
389 | return self.__keys.keys() | ||
390 | |||
391 | def groups(self): | ||
392 | """ | ||
393 | Return the names of available groups | ||
394 | """ | ||
395 | return self.__groups.keys() | ||
396 | |||
397 | def group_content(self, group_name): | ||
398 | """ | ||
399 | Return a list of keys/names that are in a specefic | ||
400 | group or the empty list | ||
401 | """ | ||
402 | try: | ||
403 | return self.__groups[group_name] | ||
404 | except KeyError: | ||
405 | return [] | ||
406 | |||
407 | |||
408 | def parse_cmdline(args): | ||
409 | """ | ||
410 | Parse the CMD line and return the result as a n-tuple | ||
411 | """ | ||
412 | |||
413 | parser = optparse.OptionParser( version = "Bitbake Documentation Tool Core version %s, %%prog version %s" % (bb.__version__, __version__)) | ||
414 | usage = """%prog [options] | ||
415 | |||
416 | Create a set of html pages (documentation) for a bitbake.conf.... | ||
417 | """ | ||
418 | |||
419 | # Add the needed options | ||
420 | parser.add_option( "-c", "--config", help = "Use the specified configuration file as source", | ||
421 | action = "store", dest = "config", default = os.path.join("conf", "documentation.conf") ) | ||
422 | |||
423 | parser.add_option( "-o", "--output", help = "Output directory for html files", | ||
424 | action = "store", dest = "output", default = "html/" ) | ||
425 | |||
426 | parser.add_option( "-D", "--debug", help = "Increase the debug level", | ||
427 | action = "count", dest = "debug", default = 0 ) | ||
428 | |||
429 | parser.add_option( "-v", "--verbose", help = "output more chit-char to the terminal", | ||
430 | action = "store_true", dest = "verbose", default = False ) | ||
431 | |||
432 | options, args = parser.parse_args( sys.argv ) | ||
433 | |||
434 | bb.msg.init_msgconfig(options.verbose, options.debug) | ||
435 | |||
436 | return options.config, options.output | ||
437 | |||
438 | def main(): | ||
439 | """ | ||
440 | The main Method | ||
441 | """ | ||
442 | |||
443 | (config_file, output_dir) = parse_cmdline( sys.argv ) | ||
444 | |||
445 | # right to let us load the file now | ||
446 | try: | ||
447 | documentation = bb.parse.handle( config_file, bb.data.init() ) | ||
448 | except IOError: | ||
449 | bb.fatal( "Unable to open %s" % config_file ) | ||
450 | except bb.parse.ParseError: | ||
451 | bb.fatal( "Unable to parse %s" % config_file ) | ||
452 | |||
453 | if isinstance(documentation, dict): | ||
454 | documentation = documentation[""] | ||
455 | |||
456 | # Assuming we've the file loaded now, we will initialize the 'tree' | ||
457 | doc = Documentation() | ||
458 | |||
459 | # defined states | ||
460 | state_begin = 0 | ||
461 | state_see = 1 | ||
462 | state_group = 2 | ||
463 | |||
464 | for key in bb.data.keys(documentation): | ||
465 | data = documentation.getVarFlag(key, "doc") | ||
466 | if not data: | ||
467 | continue | ||
468 | |||
469 | # The Documentation now starts | ||
470 | doc_ins = DocumentationItem() | ||
471 | doc_ins.setName(key) | ||
472 | |||
473 | |||
474 | tokens = data.split(' ') | ||
475 | state = state_begin | ||
476 | string= "" | ||
477 | for token in tokens: | ||
478 | token = token.strip(',') | ||
479 | |||
480 | if not state == state_see and token == "@see": | ||
481 | state = state_see | ||
482 | continue | ||
483 | elif not state == state_group and token == "@group": | ||
484 | state = state_group | ||
485 | continue | ||
486 | |||
487 | if state == state_begin: | ||
488 | string += " %s" % token | ||
489 | elif state == state_see: | ||
490 | doc_ins.addRelation(token) | ||
491 | elif state == state_group: | ||
492 | doc_ins.addGroup(token) | ||
493 | |||
494 | # set the description | ||
495 | doc_ins.setDescription(string) | ||
496 | doc.insert_doc_item(doc_ins) | ||
497 | |||
498 | # let us create the HTML now | ||
499 | bb.utils.mkdirhier(output_dir) | ||
500 | os.chdir(output_dir) | ||
501 | |||
502 | # Let us create the sites now. We do it in the following order | ||
503 | # Start with the index.html. It will point to sites explaining all | ||
504 | # keys and groups | ||
505 | html_slave = HTMLFormatter() | ||
506 | |||
507 | f = file('style.css', 'w') | ||
508 | print >> f, html_slave.createCSS() | ||
509 | |||
510 | f = file('index.html', 'w') | ||
511 | print >> f, html_slave.createIndex() | ||
512 | |||
513 | f = file('all_groups.html', 'w') | ||
514 | print >> f, html_slave.createGroupsSite(doc) | ||
515 | |||
516 | f = file('all_keys.html', 'w') | ||
517 | print >> f, html_slave.createKeysSite(doc) | ||
518 | |||
519 | # now for each group create the site | ||
520 | for group in doc.groups(): | ||
521 | f = file('group%s.html' % group, 'w') | ||
522 | print >> f, html_slave.createGroupSite(group, doc.group_content(group)) | ||
523 | |||
524 | # now for the keys | ||
525 | for key in doc.doc_keys(): | ||
526 | f = file('key%s.html' % doc.doc_item(key).name(), 'w') | ||
527 | print >> f, html_slave.createKeySite(doc.doc_item(key)) | ||
528 | |||
529 | |||
530 | if __name__ == "__main__": | ||
531 | main() | ||
diff --git a/bitbake/bin/image-writer b/bitbake/bin/image-writer new file mode 100755 index 0000000000..86c38b5769 --- /dev/null +++ b/bitbake/bin/image-writer | |||
@@ -0,0 +1,122 @@ | |||
1 | #!/usr/bin/env python | ||
2 | |||
3 | # Copyright (c) 2012 Wind River Systems, Inc. | ||
4 | # | ||
5 | # This program is free software; you can redistribute it and/or modify | ||
6 | # it under the terms of the GNU General Public License version 2 as | ||
7 | # published by the Free Software Foundation. | ||
8 | # | ||
9 | # This program is distributed in the hope that it will be useful, | ||
10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. | ||
12 | # See the GNU General Public License for more details. | ||
13 | # | ||
14 | # You should have received a copy of the GNU General Public License | ||
15 | # along with this program; if not, write to the Free Software | ||
16 | # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | ||
17 | |||
18 | import os | ||
19 | import sys | ||
20 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname( \ | ||
21 | os.path.abspath(__file__))), 'lib')) | ||
22 | try: | ||
23 | import bb | ||
24 | except RuntimeError as exc: | ||
25 | sys.exit(str(exc)) | ||
26 | |||
27 | import gtk | ||
28 | import optparse | ||
29 | import pygtk | ||
30 | |||
31 | from bb.ui.crumbs.hobwidget import HobAltButton, HobButton | ||
32 | from bb.ui.crumbs.hig.crumbsmessagedialog import CrumbsMessageDialog | ||
33 | from bb.ui.crumbs.hig.deployimagedialog import DeployImageDialog | ||
34 | from bb.ui.crumbs.hig.imageselectiondialog import ImageSelectionDialog | ||
35 | |||
36 | # I put all the fs bitbake supported here. Need more test. | ||
37 | DEPLOYABLE_IMAGE_TYPES = ["jffs2", "cramfs", "ext2", "ext3", "btrfs", "squashfs", "ubi", "vmdk"] | ||
38 | Title = "USB Image Writer" | ||
39 | |||
40 | class DeployWindow(gtk.Window): | ||
41 | def __init__(self, image_path=''): | ||
42 | super(DeployWindow, self).__init__() | ||
43 | |||
44 | if len(image_path) > 0: | ||
45 | valid = True | ||
46 | if not os.path.exists(image_path): | ||
47 | valid = False | ||
48 | lbl = "<b>Invalid image file path: %s.</b>\nPress <b>Select Image</b> to select an image." % image_path | ||
49 | else: | ||
50 | image_path = os.path.abspath(image_path) | ||
51 | extend_name = os.path.splitext(image_path)[1][1:] | ||
52 | if extend_name not in DEPLOYABLE_IMAGE_TYPES: | ||
53 | valid = False | ||
54 | lbl = "<b>Undeployable imge type: %s</b>\nPress <b>Select Image</b> to select an image." % extend_name | ||
55 | |||
56 | if not valid: | ||
57 | image_path = '' | ||
58 | crumbs_dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO) | ||
59 | button = crumbs_dialog.add_button("Close", gtk.RESPONSE_OK) | ||
60 | HobButton.style_button(button) | ||
61 | crumbs_dialog.run() | ||
62 | crumbs_dialog.destroy() | ||
63 | |||
64 | self.deploy_dialog = DeployImageDialog(Title, image_path, self, | ||
65 | gtk.DIALOG_MODAL | gtk.DIALOG_DESTROY_WITH_PARENT | ||
66 | | gtk.DIALOG_NO_SEPARATOR, None, standalone=True) | ||
67 | close_button = self.deploy_dialog.add_button("Close", gtk.RESPONSE_NO) | ||
68 | HobAltButton.style_button(close_button) | ||
69 | close_button.connect('clicked', gtk.main_quit) | ||
70 | |||
71 | write_button = self.deploy_dialog.add_button("Write USB image", gtk.RESPONSE_YES) | ||
72 | HobAltButton.style_button(write_button) | ||
73 | |||
74 | self.deploy_dialog.connect('select_image_clicked', self.select_image_clicked_cb) | ||
75 | self.deploy_dialog.connect('destroy', gtk.main_quit) | ||
76 | response = self.deploy_dialog.show() | ||
77 | |||
78 | def select_image_clicked_cb(self, dialog): | ||
79 | cwd = os.getcwd() | ||
80 | dialog = ImageSelectionDialog(cwd, DEPLOYABLE_IMAGE_TYPES, Title, self, gtk.FILE_CHOOSER_ACTION_SAVE ) | ||
81 | button = dialog.add_button("Cancel", gtk.RESPONSE_NO) | ||
82 | HobAltButton.style_button(button) | ||
83 | button = dialog.add_button("Open", gtk.RESPONSE_YES) | ||
84 | HobAltButton.style_button(button) | ||
85 | response = dialog.run() | ||
86 | |||
87 | if response == gtk.RESPONSE_YES: | ||
88 | if not dialog.image_names: | ||
89 | lbl = "<b>No selections made</b>\nClicked the radio button to select a image." | ||
90 | crumbs_dialog = CrumbsMessageDialog(self, lbl, gtk.STOCK_DIALOG_INFO) | ||
91 | button = crumbs_dialog.add_button("Close", gtk.RESPONSE_OK) | ||
92 | HobButton.style_button(button) | ||
93 | crumbs_dialog.run() | ||
94 | crumbs_dialog.destroy() | ||
95 | dialog.destroy() | ||
96 | return | ||
97 | |||
98 | # get the full path of image | ||
99 | image_path = os.path.join(dialog.image_folder, dialog.image_names[0]) | ||
100 | self.deploy_dialog.set_image_text_buffer(image_path) | ||
101 | self.deploy_dialog.set_image_path(image_path) | ||
102 | |||
103 | dialog.destroy() | ||
104 | |||
105 | def main(): | ||
106 | parser = optparse.OptionParser( | ||
107 | usage = """%prog [-h] [image_file] | ||
108 | |||
109 | %prog writes bootable images to USB devices. You can | ||
110 | provide the image file on the command line or select it using the GUI.""") | ||
111 | |||
112 | options, args = parser.parse_args(sys.argv) | ||
113 | image_file = args[1] if len(args) > 1 else '' | ||
114 | dw = DeployWindow(image_file) | ||
115 | |||
116 | if __name__ == '__main__': | ||
117 | try: | ||
118 | main() | ||
119 | gtk.main() | ||
120 | except Exception: | ||
121 | import traceback | ||
122 | traceback.print_exc(3) | ||