diff options
Diffstat (limited to 'bitbake/lib/bb/build.py')
-rw-r--r-- | bitbake/lib/bb/build.py | 270 |
1 files changed, 134 insertions, 136 deletions
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py index f4f897e41a..44d08f5c55 100644 --- a/bitbake/lib/bb/build.py +++ b/bitbake/lib/bb/build.py | |||
@@ -20,10 +20,12 @@ import itertools | |||
20 | import time | 20 | import time |
21 | import re | 21 | import re |
22 | import stat | 22 | import stat |
23 | import datetime | ||
23 | import bb | 24 | import bb |
24 | import bb.msg | 25 | import bb.msg |
25 | import bb.process | 26 | import bb.process |
26 | import bb.progress | 27 | import bb.progress |
28 | from io import StringIO | ||
27 | from bb import data, event, utils | 29 | from bb import data, event, utils |
28 | 30 | ||
29 | bblogger = logging.getLogger('BitBake') | 31 | bblogger = logging.getLogger('BitBake') |
@@ -176,7 +178,9 @@ class StdoutNoopContextManager: | |||
176 | 178 | ||
177 | @property | 179 | @property |
178 | def name(self): | 180 | def name(self): |
179 | return sys.stdout.name | 181 | if "name" in dir(sys.stdout): |
182 | return sys.stdout.name | ||
183 | return "<mem>" | ||
180 | 184 | ||
181 | 185 | ||
182 | def exec_func(func, d, dirs = None): | 186 | def exec_func(func, d, dirs = None): |
@@ -295,9 +299,25 @@ def exec_func_python(func, d, runfile, cwd=None): | |||
295 | lineno = int(d.getVarFlag(func, "lineno", False)) | 299 | lineno = int(d.getVarFlag(func, "lineno", False)) |
296 | bb.methodpool.insert_method(func, text, fn, lineno - 1) | 300 | bb.methodpool.insert_method(func, text, fn, lineno - 1) |
297 | 301 | ||
298 | comp = utils.better_compile(code, func, "exec_python_func() autogenerated") | 302 | if verboseStdoutLogging: |
299 | utils.better_exec(comp, {"d": d}, code, "exec_python_func() autogenerated") | 303 | sys.stdout.flush() |
304 | sys.stderr.flush() | ||
305 | currout = sys.stdout | ||
306 | currerr = sys.stderr | ||
307 | sys.stderr = sys.stdout = execio = StringIO() | ||
308 | comp = utils.better_compile(code, func, "exec_func_python() autogenerated") | ||
309 | utils.better_exec(comp, {"d": d}, code, "exec_func_python() autogenerated") | ||
300 | finally: | 310 | finally: |
311 | if verboseStdoutLogging: | ||
312 | execio.flush() | ||
313 | logger.plain("%s" % execio.getvalue()) | ||
314 | sys.stdout = currout | ||
315 | sys.stderr = currerr | ||
316 | execio.close() | ||
317 | # We want any stdout/stderr to be printed before any other log messages to make debugging | ||
318 | # more accurate. In some cases we seem to lose stdout/stderr entirely in logging tests without this. | ||
319 | sys.stdout.flush() | ||
320 | sys.stderr.flush() | ||
301 | bb.debug(2, "Python function %s finished" % func) | 321 | bb.debug(2, "Python function %s finished" % func) |
302 | 322 | ||
303 | if cwd and olddir: | 323 | if cwd and olddir: |
@@ -436,7 +456,11 @@ exit $ret | |||
436 | if fakerootcmd: | 456 | if fakerootcmd: |
437 | cmd = [fakerootcmd, runfile] | 457 | cmd = [fakerootcmd, runfile] |
438 | 458 | ||
439 | if verboseStdoutLogging: | 459 | # We only want to output to logger via LogTee if stdout is sys.__stdout__ (which will either |
460 | # be real stdout or subprocess PIPE or similar). In other cases we are being run "recursively", | ||
461 | # ie. inside another function, in which case stdout is already being captured so we don't | ||
462 | # want to Tee here as output would be printed twice, and out of order. | ||
463 | if verboseStdoutLogging and sys.stdout == sys.__stdout__: | ||
440 | logfile = LogTee(logger, StdoutNoopContextManager()) | 464 | logfile = LogTee(logger, StdoutNoopContextManager()) |
441 | else: | 465 | else: |
442 | logfile = StdoutNoopContextManager() | 466 | logfile = StdoutNoopContextManager() |
@@ -565,10 +589,8 @@ exit $ret | |||
565 | def _task_data(fn, task, d): | 589 | def _task_data(fn, task, d): |
566 | localdata = bb.data.createCopy(d) | 590 | localdata = bb.data.createCopy(d) |
567 | localdata.setVar('BB_FILENAME', fn) | 591 | localdata.setVar('BB_FILENAME', fn) |
568 | localdata.setVar('BB_CURRENTTASK', task[3:]) | ||
569 | localdata.setVar('OVERRIDES', 'task-%s:%s' % | 592 | localdata.setVar('OVERRIDES', 'task-%s:%s' % |
570 | (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False))) | 593 | (task[3:].replace('_', '-'), d.getVar('OVERRIDES', False))) |
571 | localdata.finalize() | ||
572 | bb.data.expandKeys(localdata) | 594 | bb.data.expandKeys(localdata) |
573 | return localdata | 595 | return localdata |
574 | 596 | ||
@@ -579,7 +601,7 @@ def _exec_task(fn, task, d, quieterr): | |||
579 | running it with its own local metadata, and with some useful variables set. | 601 | running it with its own local metadata, and with some useful variables set. |
580 | """ | 602 | """ |
581 | if not d.getVarFlag(task, 'task', False): | 603 | if not d.getVarFlag(task, 'task', False): |
582 | event.fire(TaskInvalid(task, d), d) | 604 | event.fire(TaskInvalid(task, fn, d), d) |
583 | logger.error("No such task: %s" % task) | 605 | logger.error("No such task: %s" % task) |
584 | return 1 | 606 | return 1 |
585 | 607 | ||
@@ -615,7 +637,8 @@ def _exec_task(fn, task, d, quieterr): | |||
615 | logorder = os.path.join(tempdir, 'log.task_order') | 637 | logorder = os.path.join(tempdir, 'log.task_order') |
616 | try: | 638 | try: |
617 | with open(logorder, 'a') as logorderfile: | 639 | with open(logorder, 'a') as logorderfile: |
618 | logorderfile.write('{0} ({1}): {2}\n'.format(task, os.getpid(), logbase)) | 640 | timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S.%f") |
641 | logorderfile.write('{0} {1} ({2}): {3}\n'.format(timestamp, task, os.getpid(), logbase)) | ||
619 | except OSError: | 642 | except OSError: |
620 | logger.exception("Opening log file '%s'", logorder) | 643 | logger.exception("Opening log file '%s'", logorder) |
621 | pass | 644 | pass |
@@ -682,47 +705,55 @@ def _exec_task(fn, task, d, quieterr): | |||
682 | try: | 705 | try: |
683 | try: | 706 | try: |
684 | event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata) | 707 | event.fire(TaskStarted(task, fn, logfn, flags, localdata), localdata) |
685 | except (bb.BBHandledException, SystemExit): | ||
686 | return 1 | ||
687 | 708 | ||
688 | try: | ||
689 | for func in (prefuncs or '').split(): | 709 | for func in (prefuncs or '').split(): |
690 | exec_func(func, localdata) | 710 | exec_func(func, localdata) |
691 | exec_func(task, localdata) | 711 | exec_func(task, localdata) |
692 | for func in (postfuncs or '').split(): | 712 | for func in (postfuncs or '').split(): |
693 | exec_func(func, localdata) | 713 | exec_func(func, localdata) |
694 | except bb.BBHandledException: | 714 | finally: |
695 | event.fire(TaskFailed(task, fn, logfn, localdata, True), localdata) | 715 | # Need to flush and close the logs before sending events where the |
696 | return 1 | 716 | # UI may try to look at the logs. |
697 | except Exception as exc: | 717 | sys.stdout.flush() |
698 | if quieterr: | 718 | sys.stderr.flush() |
699 | event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata) | 719 | |
700 | else: | 720 | bblogger.removeHandler(handler) |
701 | errprinted = errchk.triggered | 721 | |
702 | logger.error(str(exc)) | 722 | # Restore the backup fds |
703 | event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata) | 723 | os.dup2(osi[0], osi[1]) |
704 | return 1 | 724 | os.dup2(oso[0], oso[1]) |
705 | finally: | 725 | os.dup2(ose[0], ose[1]) |
706 | sys.stdout.flush() | 726 | |
707 | sys.stderr.flush() | 727 | # Close the backup fds |
708 | 728 | os.close(osi[0]) | |
709 | bblogger.removeHandler(handler) | 729 | os.close(oso[0]) |
710 | 730 | os.close(ose[0]) | |
711 | # Restore the backup fds | 731 | |
712 | os.dup2(osi[0], osi[1]) | 732 | logfile.close() |
713 | os.dup2(oso[0], oso[1]) | 733 | if os.path.exists(logfn) and os.path.getsize(logfn) == 0: |
714 | os.dup2(ose[0], ose[1]) | 734 | logger.debug2("Zero size logfn %s, removing", logfn) |
715 | 735 | bb.utils.remove(logfn) | |
716 | # Close the backup fds | 736 | bb.utils.remove(loglink) |
717 | os.close(osi[0]) | 737 | except (Exception, SystemExit) as exc: |
718 | os.close(oso[0]) | 738 | handled = False |
719 | os.close(ose[0]) | 739 | if isinstance(exc, bb.BBHandledException): |
740 | handled = True | ||
741 | |||
742 | if quieterr: | ||
743 | if not handled: | ||
744 | logger.warning(repr(exc)) | ||
745 | event.fire(TaskFailedSilent(task, fn, logfn, localdata), localdata) | ||
746 | else: | ||
747 | errprinted = errchk.triggered | ||
748 | # If the output is already on stdout, we've printed the information in the | ||
749 | # logs once already so don't duplicate | ||
750 | if verboseStdoutLogging or handled: | ||
751 | errprinted = True | ||
752 | if not handled: | ||
753 | logger.error(repr(exc)) | ||
754 | event.fire(TaskFailed(task, fn, logfn, localdata, errprinted), localdata) | ||
755 | return 1 | ||
720 | 756 | ||
721 | logfile.close() | ||
722 | if os.path.exists(logfn) and os.path.getsize(logfn) == 0: | ||
723 | logger.debug2("Zero size logfn %s, removing", logfn) | ||
724 | bb.utils.remove(logfn) | ||
725 | bb.utils.remove(loglink) | ||
726 | event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata) | 757 | event.fire(TaskSucceeded(task, fn, logfn, localdata), localdata) |
727 | 758 | ||
728 | if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False): | 759 | if not localdata.getVarFlag(task, 'nostamp', False) and not localdata.getVarFlag(task, 'selfstamp', False): |
@@ -760,132 +791,92 @@ def exec_task(fn, task, d, profile = False): | |||
760 | event.fire(failedevent, d) | 791 | event.fire(failedevent, d) |
761 | return 1 | 792 | return 1 |
762 | 793 | ||
763 | def stamp_internal(taskname, d, file_name, baseonly=False, noextra=False): | 794 | def _get_cleanmask(taskname, mcfn): |
764 | """ | 795 | """ |
765 | Internal stamp helper function | 796 | Internal stamp helper function to generate stamp cleaning mask |
766 | Makes sure the stamp directory exists | ||
767 | Returns the stamp path+filename | 797 | Returns the stamp path+filename |
768 | 798 | ||
769 | In the bitbake core, d can be a CacheData and file_name will be set. | 799 | In the bitbake core, d can be a CacheData and file_name will be set. |
770 | When called in task context, d will be a data store, file_name will not be set | 800 | When called in task context, d will be a data store, file_name will not be set |
771 | """ | 801 | """ |
772 | taskflagname = taskname | 802 | cleanmask = bb.parse.siggen.stampcleanmask_mcfn(taskname, mcfn) |
773 | if taskname.endswith("_setscene") and taskname != "do_setscene": | 803 | taskflagname = taskname.replace("_setscene", "") |
774 | taskflagname = taskname.replace("_setscene", "") | 804 | if cleanmask: |
775 | 805 | return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")] | |
776 | if file_name: | 806 | return [] |
777 | stamp = d.stamp[file_name] | 807 | |
778 | extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" | 808 | def clean_stamp_mcfn(task, mcfn): |
779 | else: | 809 | cleanmask = _get_cleanmask(task, mcfn) |
780 | stamp = d.getVar('STAMP') | 810 | for mask in cleanmask: |
781 | file_name = d.getVar('BB_FILENAME') | 811 | for name in glob.glob(mask): |
782 | extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or "" | 812 | # Preserve sigdata files in the stamps directory |
813 | if "sigdata" in name or "sigbasedata" in name: | ||
814 | continue | ||
815 | # Preserve taint files in the stamps directory | ||
816 | if name.endswith('.taint'): | ||
817 | continue | ||
818 | os.unlink(name) | ||
783 | 819 | ||
784 | if baseonly: | 820 | def clean_stamp(task, d): |
785 | return stamp | 821 | mcfn = d.getVar('BB_FILENAME') |
786 | if noextra: | 822 | clean_stamp_mcfn(task, mcfn) |
787 | extrainfo = "" | ||
788 | 823 | ||
789 | if not stamp: | 824 | def make_stamp_mcfn(task, mcfn): |
790 | return | ||
791 | 825 | ||
792 | stamp = bb.parse.siggen.stampfile(stamp, file_name, taskname, extrainfo) | 826 | basestamp = bb.parse.siggen.stampfile_mcfn(task, mcfn) |
793 | 827 | ||
794 | stampdir = os.path.dirname(stamp) | 828 | stampdir = os.path.dirname(basestamp) |
795 | if cached_mtime_noerror(stampdir) == 0: | 829 | if cached_mtime_noerror(stampdir) == 0: |
796 | bb.utils.mkdirhier(stampdir) | 830 | bb.utils.mkdirhier(stampdir) |
797 | 831 | ||
798 | return stamp | 832 | clean_stamp_mcfn(task, mcfn) |
799 | 833 | ||
800 | def stamp_cleanmask_internal(taskname, d, file_name): | 834 | # Remove the file and recreate to force timestamp |
801 | """ | 835 | # change on broken NFS filesystems |
802 | Internal stamp helper function to generate stamp cleaning mask | 836 | if basestamp: |
803 | Returns the stamp path+filename | 837 | bb.utils.remove(basestamp) |
838 | open(basestamp, "w").close() | ||
804 | 839 | ||
805 | In the bitbake core, d can be a CacheData and file_name will be set. | 840 | def make_stamp(task, d): |
806 | When called in task context, d will be a data store, file_name will not be set | ||
807 | """ | 841 | """ |
808 | taskflagname = taskname | 842 | Creates/updates a stamp for a given task |
809 | if taskname.endswith("_setscene") and taskname != "do_setscene": | 843 | """ |
810 | taskflagname = taskname.replace("_setscene", "") | 844 | mcfn = d.getVar('BB_FILENAME') |
811 | |||
812 | if file_name: | ||
813 | stamp = d.stampclean[file_name] | ||
814 | extrainfo = d.stamp_extrainfo[file_name].get(taskflagname) or "" | ||
815 | else: | ||
816 | stamp = d.getVar('STAMPCLEAN') | ||
817 | file_name = d.getVar('BB_FILENAME') | ||
818 | extrainfo = d.getVarFlag(taskflagname, 'stamp-extra-info') or "" | ||
819 | 845 | ||
820 | if not stamp: | 846 | make_stamp_mcfn(task, mcfn) |
821 | return [] | ||
822 | 847 | ||
823 | cleanmask = bb.parse.siggen.stampcleanmask(stamp, file_name, taskname, extrainfo) | 848 | # If we're in task context, write out a signature file for each task |
849 | # as it completes | ||
850 | if not task.endswith("_setscene"): | ||
851 | stampbase = bb.parse.siggen.stampfile_base(mcfn) | ||
852 | bb.parse.siggen.dump_sigtask(mcfn, task, stampbase, True) | ||
824 | 853 | ||
825 | return [cleanmask, cleanmask.replace(taskflagname, taskflagname + "_setscene")] | ||
826 | 854 | ||
827 | def make_stamp(task, d, file_name = None): | 855 | def find_stale_stamps(task, mcfn): |
828 | """ | 856 | current = bb.parse.siggen.stampfile_mcfn(task, mcfn) |
829 | Creates/updates a stamp for a given task | 857 | current2 = bb.parse.siggen.stampfile_mcfn(task + "_setscene", mcfn) |
830 | (d can be a data dict or dataCache) | 858 | cleanmask = _get_cleanmask(task, mcfn) |
831 | """ | 859 | found = [] |
832 | cleanmask = stamp_cleanmask_internal(task, d, file_name) | ||
833 | for mask in cleanmask: | 860 | for mask in cleanmask: |
834 | for name in glob.glob(mask): | 861 | for name in glob.glob(mask): |
835 | # Preserve sigdata files in the stamps directory | ||
836 | if "sigdata" in name or "sigbasedata" in name: | 862 | if "sigdata" in name or "sigbasedata" in name: |
837 | continue | 863 | continue |
838 | # Preserve taint files in the stamps directory | ||
839 | if name.endswith('.taint'): | 864 | if name.endswith('.taint'): |
840 | continue | 865 | continue |
841 | os.unlink(name) | 866 | if name == current or name == current2: |
842 | 867 | continue | |
843 | stamp = stamp_internal(task, d, file_name) | 868 | logger.debug2("Stampfile %s does not match %s or %s" % (name, current, current2)) |
844 | # Remove the file and recreate to force timestamp | 869 | found.append(name) |
845 | # change on broken NFS filesystems | 870 | return found |
846 | if stamp: | ||
847 | bb.utils.remove(stamp) | ||
848 | open(stamp, "w").close() | ||
849 | |||
850 | # If we're in task context, write out a signature file for each task | ||
851 | # as it completes | ||
852 | if not task.endswith("_setscene") and task != "do_setscene" and not file_name: | ||
853 | stampbase = stamp_internal(task, d, None, True) | ||
854 | file_name = d.getVar('BB_FILENAME') | ||
855 | bb.parse.siggen.dump_sigtask(file_name, task, stampbase, True) | ||
856 | |||
857 | def del_stamp(task, d, file_name = None): | ||
858 | """ | ||
859 | Removes a stamp for a given task | ||
860 | (d can be a data dict or dataCache) | ||
861 | """ | ||
862 | stamp = stamp_internal(task, d, file_name) | ||
863 | bb.utils.remove(stamp) | ||
864 | 871 | ||
865 | def write_taint(task, d, file_name = None): | 872 | def write_taint(task, d): |
866 | """ | 873 | """ |
867 | Creates a "taint" file which will force the specified task and its | 874 | Creates a "taint" file which will force the specified task and its |
868 | dependents to be re-run the next time by influencing the value of its | 875 | dependents to be re-run the next time by influencing the value of its |
869 | taskhash. | 876 | taskhash. |
870 | (d can be a data dict or dataCache) | ||
871 | """ | 877 | """ |
872 | import uuid | 878 | mcfn = d.getVar('BB_FILENAME') |
873 | if file_name: | 879 | bb.parse.siggen.invalidate_task(task, mcfn) |
874 | taintfn = d.stamp[file_name] + '.' + task + '.taint' | ||
875 | else: | ||
876 | taintfn = d.getVar('STAMP') + '.' + task + '.taint' | ||
877 | bb.utils.mkdirhier(os.path.dirname(taintfn)) | ||
878 | # The specific content of the taint file is not really important, | ||
879 | # we just need it to be random, so a random UUID is used | ||
880 | with open(taintfn, 'w') as taintf: | ||
881 | taintf.write(str(uuid.uuid4())) | ||
882 | |||
883 | def stampfile(taskname, d, file_name = None, noextra=False): | ||
884 | """ | ||
885 | Return the stamp for a given task | ||
886 | (d can be a data dict or dataCache) | ||
887 | """ | ||
888 | return stamp_internal(taskname, d, file_name, noextra=noextra) | ||
889 | 880 | ||
890 | def add_tasks(tasklist, d): | 881 | def add_tasks(tasklist, d): |
891 | task_deps = d.getVar('_task_deps', False) | 882 | task_deps = d.getVar('_task_deps', False) |
@@ -910,6 +901,11 @@ def add_tasks(tasklist, d): | |||
910 | task_deps[name] = {} | 901 | task_deps[name] = {} |
911 | if name in flags: | 902 | if name in flags: |
912 | deptask = d.expand(flags[name]) | 903 | deptask = d.expand(flags[name]) |
904 | if name in ['noexec', 'fakeroot', 'nostamp']: | ||
905 | if deptask != '1': | ||
906 | bb.warn("In a future version of BitBake, setting the '{}' flag to something other than '1' " | ||
907 | "will result in the flag not being set. See YP bug #13808.".format(name)) | ||
908 | |||
913 | task_deps[name][task] = deptask | 909 | task_deps[name][task] = deptask |
914 | getTask('mcdepends') | 910 | getTask('mcdepends') |
915 | getTask('depends') | 911 | getTask('depends') |
@@ -1008,6 +1004,8 @@ def tasksbetween(task_start, task_end, d): | |||
1008 | def follow_chain(task, endtask, chain=None): | 1004 | def follow_chain(task, endtask, chain=None): |
1009 | if not chain: | 1005 | if not chain: |
1010 | chain = [] | 1006 | chain = [] |
1007 | if task in chain: | ||
1008 | bb.fatal("Circular task dependencies as %s depends on itself via the chain %s" % (task, " -> ".join(chain))) | ||
1011 | chain.append(task) | 1009 | chain.append(task) |
1012 | for othertask in tasks: | 1010 | for othertask in tasks: |
1013 | if othertask == task: | 1011 | if othertask == task: |