summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/build.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/build.py')
-rw-r--r--bitbake/lib/bb/build.py709
1 files changed, 709 insertions, 0 deletions
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
new file mode 100644
index 0000000..5cb4c06
--- /dev/null
+++ b/bitbake/lib/bb/build.py
@@ -0,0 +1,709 @@
1# ex:ts=4:sw=4:sts=4:et
2# -*- tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*-
3#
4# BitBake 'Build' implementation
5#
6# Core code for function execution and task handling in the
7# BitBake build tools.
8#
9# Copyright (C) 2003, 2004 Chris Larson
10#
11# Based on Gentoo's portage.py.
12#
13# This program is free software; you can redistribute it and/or modify
14# it under the terms of the GNU General Public License version 2 as
15# published by the Free Software Foundation.
16#
17# This program is distributed in the hope that it will be useful,
18# but WITHOUT ANY WARRANTY; without even the implied warranty of
19# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20# GNU General Public License for more details.
21#
22# You should have received a copy of the GNU General Public License along
23# with this program; if not, write to the Free Software Foundation, Inc.,
24# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
25#
26#Based on functions from the base bb module, Copyright 2003 Holger Schurig
27
28import os
29import sys
30import logging
31import shlex
32import glob
33import time
34import bb
35import bb.msg
36import bb.process
37from contextlib import nested
38from bb import event, utils
39
40bblogger = logging.getLogger('BitBake')
41logger = logging.getLogger('BitBake.Build')
42
43NULL = open(os.devnull, 'r+')
44
45
46# When we execute a python function we'd like certain things
47# in all namespaces, hence we add them to __builtins__
48# If we do not do this and use the exec globals, they will
49# not be available to subfunctions.
50__builtins__['bb'] = bb
51__builtins__['os'] = os
52
53class FuncFailed(Exception):
54 def __init__(self, name = None, logfile = None):
55 self.logfile = logfile
56 self.name = name
57 if name:
58 self.msg = 'Function failed: %s' % name
59 else:
60 self.msg = "Function failed"
61
62 def __str__(self):
63 if self.logfile and os.path.exists(self.logfile):
64 msg = ("%s (log file is located at %s)" %
65 (self.msg, self.logfile))
66 else:
67 msg = self.msg
68 return msg
69
70class TaskBase(event.Event):
71 """Base class for task events"""
72
73 def __init__(self, t, logfile, d):
74 self._task = t
75 self._package = d.getVar("PF", True)
76 self.taskfile = d.getVar("FILE", True)
77 self.taskname = self._task
78 self.logfile = logfile
79 self.time = time.time()
80 event.Event.__init__(self)
81 self._message = "recipe %s: task %s: %s" % (d.getVar("PF", True), t, self.getDisplayName())
82
83 def getTask(self):
84 return self._task
85
86 def setTask(self, task):
87 self._task = task
88
89 def getDisplayName(self):
90 return bb.event.getName(self)[4:]
91
92 task = property(getTask, setTask, None, "task property")
93
94class TaskStarted(TaskBase):
95 """Task execution started"""
96 def __init__(self, t, logfile, taskflags, d):
97 super(TaskStarted, self).__init__(t, logfile, d)
98 self.taskflags = taskflags
99
100class TaskSucceeded(TaskBase):
101 """Task execution completed"""
102
103class TaskFailed(TaskBase):
104 """Task execution failed"""
105
106 def __init__(self, task, logfile, metadata, errprinted = False):
107 self.errprinted = errprinted
108 super(TaskFailed, self).__init__(task, logfile, metadata)
109
110class TaskFailedSilent(TaskBase):
111 """Task execution failed (silently)"""
112 def getDisplayName(self):
113 # Don't need to tell the user it was silent
114 return "Failed"
115
116class TaskInvalid(TaskBase):
117
118 def __init__(self, task, metadata):
119 super(TaskInvalid, self).__init__(task, None, metadata)
120 self._message = "No such task '%s'" % task
121
122
123class LogTee(object):
124 def __init__(self, logger, outfile):
125 self.outfile = outfile
126 self.logger = logger
127 self.name = self.outfile.name
128
129 def write(self, string):
130 self.logger.plain(string)
131 self.outfile.write(string)
132
133 def __enter__(self):
134 self.outfile.__enter__()
135 return self
136
137 def __exit__(self, *excinfo):
138 self.outfile.__exit__(*excinfo)
139
140 def __repr__(self):
141 return '<LogTee {0}>'.format(self.name)
142 def flush(self):
143 self.outfile.flush()
144
145def exec_func(func, d, dirs = None):
146 """Execute an BB 'function'"""
147
148 body = d.getVar(func)
149 if not body:
150 if body is None:
151 logger.warn("Function %s doesn't exist", func)
152 return
153
154 flags = d.getVarFlags(func)
155 cleandirs = flags.get('cleandirs')
156 if cleandirs:
157 for cdir in d.expand(cleandirs).split():
158 bb.utils.remove(cdir, True)
159 bb.utils.mkdirhier(cdir)
160
161 if dirs is None:
162 dirs = flags.get('dirs')
163 if dirs:
164 dirs = d.expand(dirs).split()
165
166 if dirs:
167 for adir in dirs:
168 bb.utils.mkdirhier(adir)
169 adir = dirs[-1]
170 else:
171 adir = d.getVar('B', True)
172 bb.utils.mkdirhier(adir)
173
174 ispython = flags.get('python')
175
176 lockflag = flags.get('lockfiles')
177 if lockflag:
178 lockfiles = [f for f in d.expand(lockflag).split()]
179 else:
180 lockfiles = None
181
182 tempdir = d.getVar('T', True)
183
184 # or func allows items to be executed outside of the normal
185 # task set, such as buildhistory
186 task = d.getVar('BB_RUNTASK', True) or func
187 if task == func:
188 taskfunc = task
189 else:
190 taskfunc = "%s.%s" % (task, func)
191
192 runfmt = d.getVar('BB_RUNFMT', True) or "run.{func}.{pid}"
193 runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid())
194 runfile = os.path.join(tempdir, runfn)
195 bb.utils.mkdirhier(os.path.dirname(runfile))
196
197 # Setup the courtesy link to the runfn, only for tasks
198 # we create the link 'just' before the run script is created
199 # if we create it after, and if the run script fails, then the
200 # link won't be created as an exception would be fired.
201 if task == func:
202 runlink = os.path.join(tempdir, 'run.{0}'.format(task))
203 if runlink:
204 bb.utils.remove(runlink)
205
206 try:
207 os.symlink(runfn, runlink)
208 except OSError:
209 pass
210
211 with bb.utils.fileslocked(lockfiles):
212 if ispython:
213 exec_func_python(func, d, runfile, cwd=adir)
214 else:
215 exec_func_shell(func, d, runfile, cwd=adir)
216
217_functionfmt = """
218def {function}(d):
219{body}
220
221{function}(d)
222"""
223logformatter = bb.msg.BBLogFormatter("%(levelname)s: %(message)s")
224def exec_func_python(func, d, runfile, cwd=None):
225 """Execute a python BB 'function'"""
226
227 bbfile = d.getVar('FILE', True)
228 code = _functionfmt.format(function=func, body=d.getVar(func, True))
229 bb.utils.mkdirhier(os.path.dirname(runfile))
230 with open(runfile, 'w') as script:
231 script.write(code)
232
233 if cwd:
234 try:
235 olddir = os.getcwd()
236 except OSError:
237 olddir = None
238 os.chdir(cwd)
239
240 bb.debug(2, "Executing python function %s" % func)
241
242 try:
243 comp = utils.better_compile(code, func, bbfile)
244 utils.better_exec(comp, {"d": d}, code, bbfile)
245 except:
246 if sys.exc_info()[0] in (bb.parse.SkipPackage, bb.build.FuncFailed):
247 raise
248
249 raise FuncFailed(func, None)
250 finally:
251 bb.debug(2, "Python function %s finished" % func)
252
253 if cwd and olddir:
254 try:
255 os.chdir(olddir)
256 except OSError:
257 pass
258
259def shell_trap_code():
260 return '''#!/bin/sh\n
261# Emit a useful diagnostic if something fails:
262bb_exit_handler() {
263 ret=$?
264 case $ret in
265 0) ;;
266 *) case $BASH_VERSION in
267 "") echo "WARNING: exit code $ret from a shell command.";;
268 *) echo "WARNING: ${BASH_SOURCE[0]}:${BASH_LINENO[0]} exit $ret from
269 \"$BASH_COMMAND\"";;
270 esac
271 exit $ret
272 esac
273}
274trap 'bb_exit_handler' 0
275set -e
276'''
277
278def exec_func_shell(func, d, runfile, cwd=None):
279 """Execute a shell function from the metadata
280
281 Note on directory behavior. The 'dirs' varflag should contain a list
282 of the directories you need created prior to execution. The last
283 item in the list is where we will chdir/cd to.
284 """
285
286 # Don't let the emitted shell script override PWD
287 d.delVarFlag('PWD', 'export')
288
289 with open(runfile, 'w') as script:
290 script.write(shell_trap_code())
291
292 bb.data.emit_func(func, script, d)
293
294 if bb.msg.loggerVerboseLogs:
295 script.write("set -x\n")
296 if cwd:
297 script.write("cd '%s'\n" % cwd)
298 script.write("%s\n" % func)
299 script.write('''
300# cleanup
301ret=$?
302trap '' 0
303exit $?
304''')
305
306 os.chmod(runfile, 0775)
307
308 cmd = runfile
309 if d.getVarFlag(func, 'fakeroot'):
310 fakerootcmd = d.getVar('FAKEROOT', True)
311 if fakerootcmd:
312 cmd = [fakerootcmd, runfile]
313
314 if bb.msg.loggerDefaultVerbose:
315 logfile = LogTee(logger, sys.stdout)
316 else:
317 logfile = sys.stdout
318
319 bb.debug(2, "Executing shell function %s" % func)
320
321 try:
322 with open(os.devnull, 'r+') as stdin:
323 bb.process.run(cmd, shell=False, stdin=stdin, log=logfile)
324 except bb.process.CmdError:
325 logfn = d.getVar('BB_LOGFILE', True)
326 raise FuncFailed(func, logfn)
327
328 bb.debug(2, "Shell function %s finished" % func)
329
330def _task_data(fn, task, d):
331 localdata = bb.data.createCopy(d)
332 localdata.setVar('BB_FILENAME', fn)
333 localdata.setVar('BB_CURRENTTASK', task[3:])
334 localdata.setVar('OVERRIDES', 'task-%s:%s' %
335 (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False)))
336 localdata.finalize()
337 bb.data.expandKeys(localdata)
338 return localdata
339
340def _exec_task(fn, task, d, quieterr):
341 """Execute a BB 'task'
342
343 Execution of a task involves a bit more setup than executing a function,
344 running it with its own local metadata, and with some useful variables set.
345 """
346 if not d.getVarFlag(task, 'task'):
347 event.fire(TaskInvalid(task, d), d)
348 logger.error("No such task: %s" % task)
349 return 1
350
351 logger.debug(1, "Executing task %s", task)
352
353 localdata = _task_data(fn, task, d)
354 tempdir = localdata.getVar('T', True)
355 if not tempdir:
356 bb.fatal("T variable not set, unable to build")
357
358 # Change nice level if we're asked to
359 nice = localdata.getVar("BB_TASK_NICE_LEVEL", True)
360 if nice:
361 curnice = os.nice(0)
362 nice = int(nice) - curnice
363 newnice = os.nice(nice)
364 logger.debug(1, "Renice to %s " % newnice)
365
366 bb.utils.mkdirhier(tempdir)
367
368 # Determine the logfile to generate
369 logfmt = localdata.getVar('BB_LOGFMT', True) or 'log.{task}.{pid}'
370 logbase = logfmt.format(task=task, pid=os.getpid())
371
372 # Document the order of the tasks...
373 logorder = os.path.join(tempdir, 'log.task_order')
374 try:
375 with open(logorder, 'a') as logorderfile:
376 logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase))
377 except OSError:
378 logger.exception("Opening log file '%s'", logorder)
379 pass
380
381 # Setup the courtesy link to the logfn
382 loglink = os.path.join(tempdir, 'log.{0}'.format(task))
383 logfn = os.path.join(tempdir, logbase)
384 if loglink:
385 bb.utils.remove(loglink)
386
387 try:
388 os.symlink(logbase, loglink)
389 except OSError:
390 pass
391
392 prefuncs = localdata.getVarFlag(task, 'prefuncs', expand=True)
393 postfuncs = localdata.getVarFlag(task, 'postfuncs', expand=True)
394
395 class ErrorCheckHandler(logging.Handler):
396 def __init__(self):
397 self.triggered = False
398 logging.Handler.__init__(self, logging.ERROR)
399 def emit(self, record):
400 self.triggered = True
401
402 # Handle logfiles
403 si = open('/dev/null', 'r')
404 try:
405 bb.utils.mkdirhier(os.path.dirname(logfn))
406 logfile = open(logfn, 'w')
407 except OSError:
408 logger.exception("Opening log file '%s'", logfn)
409 pass
410
411 # Dup the existing fds so we dont lose them
412 osi = [os.dup(sys.stdin.fileno()), sys.stdin.fileno()]
413 oso = [os.dup(sys.stdout.fileno()), sys.stdout.fileno()]
414 ose = [os.dup(sys.stderr.fileno()), sys.stderr.fileno()]
415
416 # Replace those fds with our own
417 os.dup2(si.fileno(), osi[1])
418 os.dup2(logfile.fileno(), oso[1])
419 os.dup2(logfile.fileno(), ose[1])
420
421 # Ensure python logging goes to the logfile
422 handler = logging.StreamHandler(logfile)
423 handler.setFormatter(logformatter)
424 # Always enable full debug output into task logfiles
425 handler.setLevel(logging.DEBUG - 2)
426 bblogger.addHandler(handler)
427
428 errchk = ErrorCheckHandler()
429 bblogger.addHandler(errchk)
430
431 localdata.setVar('BB_LOGFILE', logfn)
432 localdata.setVar('BB_RUNTASK', task)
433
434 flags = localdata.getVarFlags(task)
435
436 event.fire(TaskStarted(task, logfn, flags, localdata), localdata)
437 try:
438 for func in (prefuncs or '').split():
439 exec_func(func, localdata)
440 exec_func(task, localdata)
441 for func in (postfuncs or '').split():
442 exec_func(func, localdata)
443 except FuncFailed as exc:
444 if quieterr:
445 event.fire(TaskFailedSilent(task, logfn, localdata), localdata)
446 else:
447 errprinted = errchk.triggered
448 logger.error(str(exc))
449 event.fire(TaskFailed(task, logfn, localdata, errprinted), localdata)
450 return 1
451 finally:
452 sys.stdout.flush()
453 sys.stderr.flush()
454
455 bblogger.removeHandler(handler)
456
457 # Restore the backup fds
458 os.dup2(osi[0], osi[1])
459 os.dup2(oso[0], oso[1])
460 os.dup2(ose[0], ose[1])
461
462 # Close the backup fds
463 os.close(osi[0])
464 os.close(oso[0])
465 os.close(ose[0])
466 si.close()
467
468 logfile.close()
469 if os.path.exists(logfn) and os.path.getsize(logfn) == 0:
470 logger.debug(2, "Zero size logfn %s, removing", logfn)
471 bb.utils.remove(logfn)
472 bb.utils.remove(loglink)
473 event.fire(TaskSucceeded(task, logfn, localdata), localdata)
474
475 if not localdata.getVarFlag(task, 'nostamp') and not localdata.getVarFlag(task, 'selfstamp'):
476 make_stamp(task, localdata)
477
478 return 0
479
480def exec_task(fn, task, d, profile = False):
481 try:
482 quieterr = False
483 if d.getVarFlag(task, "quieterrors") is not None:
484 quieterr = True
485
486 if profile:
487 profname = "profile-%s.log" % (d.getVar("PN", True) + "-" + task)
488 try:
489 import cProfile as profile
490 except:
491 import profile
492 prof = profile.Profile()
493 ret = profile.Profile.runcall(prof, _exec_task, fn, task, d, quieterr)
494 prof.dump_stats(profname)
495 bb.utils.process_profilelog(profname)
496
497 return ret
498 else:
499 return _exec_task(fn, task, d, quieterr)
500
501 except Exception:
502 from traceback import format_exc
503 if not quieterr:
504 logger.error("Build of %s failed" % (task))
505 logger.error(format_exc())
506 failedevent = TaskFailed(task, None, d, True)
507 event.fire(failedevent, d)
508 return 1
509
510def stamp_internal(taskname, d, file_name):
511 """
512 Internal stamp helper function
513 Makes sure the stamp directory exists
514 Returns the stamp path+filename
515
516 In the bitbake core, d can be a CacheData and file_name will be set.
517 When called in task context, d will be a data store, file_name will not be set
518 """
519 taskflagname = taskname
520 if taskname.endswith("_setscene") and taskname != "do_setscene":
521 taskflagname = taskname.replace("_setscene", "")
522
523 if file_name:
524 stamp = d.stamp_base[file_name].get(taskflagname) or d.stamp[file_name]
525 extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
526 else:
527 stamp = d.getVarFlag(taskflagname, 'stamp-base', True) or d.getVar('STAMP', True)
528 file_name = d.getVar('BB_FILENAME', True)
529 extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or ""
530
531 if not stamp:
532 return
533
534 stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo)
535
536 stampdir = os.path.dirname(stamp)
537 if bb.parse.cached_mtime_noerror(stampdir) == 0:
538 bb.utils.mkdirhier(stampdir)
539
540 return stamp
541
542def stamp_cleanmask_internal(taskname, d, file_name):
543 """
544 Internal stamp helper function to generate stamp cleaning mask
545 Returns the stamp path+filename
546
547 In the bitbake core, d can be a CacheData and file_name will be set.
548 When called in task context, d will be a data store, file_name will not be set
549 """
550 taskflagname = taskname
551 if taskname.endswith("_setscene") and taskname != "do_setscene":
552 taskflagname = taskname.replace("_setscene", "")
553
554 if file_name:
555 stamp = d.stamp_base_clean[file_name].get(taskflagname) or d.stampclean[file_name]
556 extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or ""
557 else:
558 stamp = d.getVarFlag(taskflagname, 'stamp-base-clean', True) or d.getVar('STAMPCLEAN', True)
559 file_name = d.getVar('BB_FILENAME', True)
560 extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info', True) or ""
561
562 if not stamp:
563 return []
564
565 cleanmask = bb.parse.siggen.stampcleanmask(stamp, file_name, taskname, extrainfo)
566
567 return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")]
568
569def make_stamp(task, d, file_name = None):
570 """
571 Creates/updates a stamp for a given task
572 (d can be a data dict or dataCache)
573 """
574 cleanmask = stamp_cleanmask_internal(task, d, file_name)
575 for mask in cleanmask:
576 for name in glob.glob(mask):
577 # Preserve sigdata files in the stamps directory
578 if "sigdata" in name:
579 continue
580 # Preserve taint files in the stamps directory
581 if name.endswith('.taint'):
582 continue
583 os.unlink(name)
584
585 stamp = stamp_internal(task, d, file_name)
586 # Remove the file and recreate to force timestamp
587 # change on broken NFS filesystems
588 if stamp:
589 bb.utils.remove(stamp)
590 open(stamp, "w").close()
591
592 # If we're in task context, write out a signature file for each task
593 # as it completes
594 if not task.endswith("_setscene") and task != "do_setscene" and not file_name:
595 file_name = d.getVar('BB_FILENAME', True)
596 bb.parse.siggen.dump_sigtask(file_name, task, d.getVar('STAMP', True), True)
597
598def del_stamp(task, d, file_name = None):
599 """
600 Removes a stamp for a given task
601 (d can be a data dict or dataCache)
602 """
603 stamp = stamp_internal(task, d, file_name)
604 bb.utils.remove(stamp)
605
606def write_taint(task, d, file_name = None):
607 """
608 Creates a "taint" file which will force the specified task and its
609 dependents to be re-run the next time by influencing the value of its
610 taskhash.
611 (d can be a data dict or dataCache)
612 """
613 import uuid
614 if file_name:
615 taintfn = d.stamp[file_name] + '.' + task + '.taint'
616 else:
617 taintfn = d.getVar('STAMP', True) + '.' + task + '.taint'
618 bb.utils.mkdirhier(os.path.dirname(taintfn))
619 # The specific content of the taint file is not really important,
620 # we just need it to be random, so a random UUID is used
621 with open(taintfn, 'w') as taintf:
622 taintf.write(str(uuid.uuid4()))
623
624def stampfile(taskname, d, file_name = None):
625 """
626 Return the stamp for a given task
627 (d can be a data dict or dataCache)
628 """
629 return stamp_internal(taskname, d, file_name)
630
631def add_tasks(tasklist, deltasklist, d):
632 task_deps = d.getVar('_task_deps')
633 if not task_deps:
634 task_deps = {}
635 if not 'tasks' in task_deps:
636 task_deps['tasks'] = []
637 if not 'parents' in task_deps:
638 task_deps['parents'] = {}
639
640 for task in tasklist:
641 task = d.expand(task)
642
643 if task in deltasklist:
644 continue
645
646 d.setVarFlag(task, 'task', 1)
647
648 if not task in task_deps['tasks']:
649 task_deps['tasks'].append(task)
650
651 flags = d.getVarFlags(task)
652 def getTask(name):
653 if not name in task_deps:
654 task_deps[name] = {}
655 if name in flags:
656 deptask = d.expand(flags[name])
657 task_deps[name][task] = deptask
658 getTask('depends')
659 getTask('rdepends')
660 getTask('deptask')
661 getTask('rdeptask')
662 getTask('recrdeptask')
663 getTask('recideptask')
664 getTask('nostamp')
665 getTask('fakeroot')
666 getTask('noexec')
667 getTask('umask')
668 task_deps['parents'][task] = []
669 if 'deps' in flags:
670 for dep in flags['deps']:
671 dep = d.expand(dep)
672 task_deps['parents'][task].append(dep)
673
674 # don't assume holding a reference
675 d.setVar('_task_deps', task_deps)
676
677def addtask(task, before, after, d):
678 if task[:3] != "do_":
679 task = "do_" + task
680
681 d.setVarFlag(task, "task", 1)
682 bbtasks = d.getVar('__BBTASKS') or []
683 if not task in bbtasks:
684 bbtasks.append(task)
685 d.setVar('__BBTASKS', bbtasks)
686
687 existing = d.getVarFlag(task, "deps") or []
688 if after is not None:
689 # set up deps for function
690 for entry in after.split():
691 if entry not in existing:
692 existing.append(entry)
693 d.setVarFlag(task, "deps", existing)
694 if before is not None:
695 # set up things that depend on this func
696 for entry in before.split():
697 existing = d.getVarFlag(entry, "deps") or []
698 if task not in existing:
699 d.setVarFlag(entry, "deps", [task] + existing)
700
701def deltask(task, d):
702 if task[:3] != "do_":
703 task = "do_" + task
704
705 bbtasks = d.getVar('__BBDELTASKS') or []
706 if not task in bbtasks:
707 bbtasks.append(task)
708 d.setVar('__BBDELTASKS', bbtasks)
709