summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/progress.py42
-rw-r--r--bitbake/lib/bb/runqueue.py70
2 files changed, 109 insertions, 3 deletions
diff --git a/bitbake/lib/bb/progress.py b/bitbake/lib/bb/progress.py
index 93e42dfcdb..49417761bb 100644
--- a/bitbake/lib/bb/progress.py
+++ b/bitbake/lib/bb/progress.py
@@ -195,3 +195,45 @@ class MultiStageProgressReporter(object):
195 else: 195 else:
196 out.append('Up to finish: %d' % stage_weight) 196 out.append('Up to finish: %d' % stage_weight)
197 bb.warn('Stage times:\n %s' % '\n '.join(out)) 197 bb.warn('Stage times:\n %s' % '\n '.join(out))
198
199class MultiStageProcessProgressReporter(MultiStageProgressReporter):
200 """
201 Version of MultiStageProgressReporter intended for use with
202 standalone processes (such as preparing the runqueue)
203 """
204 def __init__(self, d, processname, stage_weights, debug=False):
205 self._processname = processname
206 MultiStageProgressReporter.__init__(self, d, stage_weights, debug)
207
208 def start(self):
209 bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
210
211 def _fire_progress(self, taskprogress):
212 bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data)
213
214 def finish(self):
215 MultiStageProgressReporter.finish(self)
216 bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)
217
218class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
219 """
220 MultiStageProcessProgressReporter that takes the calls and does nothing
221 with them (to avoid a bunch of "if progress_reporter:" checks)
222 """
223 def __init__(self):
224 MultiStageProcessProgressReporter.__init__(self, "", None, [])
225
226 def _fire_progress(self, taskprogress, rate=None):
227 pass
228
229 def start(self):
230 pass
231
232 def next_stage(self, stage_total=None):
233 pass
234
235 def update(self, stage_progress):
236 pass
237
238 def finish(self):
239 pass
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index b62a28a2be..57be15a62b 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -241,6 +241,7 @@ class RunQueueData:
241 self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or "" 241 self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or ""
242 self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split() 242 self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split()
243 self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData) 243 self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData)
244 self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter()
244 245
245 self.reset() 246 self.reset()
246 247
@@ -432,7 +433,8 @@ class RunQueueData:
432 # Nothing to do 433 # Nothing to do
433 return 0 434 return 0
434 435
435 logger.info("Preparing RunQueue") 436 self.init_progress_reporter.start()
437 self.init_progress_reporter.next_stage()
436 438
437 # Step A - Work out a list of tasks to run 439 # Step A - Work out a list of tasks to run
438 # 440 #
@@ -562,8 +564,9 @@ class RunQueueData:
562 # e.g. do_sometask[recrdeptask] = "do_someothertask" 564 # e.g. do_sometask[recrdeptask] = "do_someothertask"
563 # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) 565 # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively)
564 # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed 566 # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed
567 self.init_progress_reporter.next_stage(len(recursivetasks))
565 extradeps = {} 568 extradeps = {}
566 for tid in recursivetasks: 569 for taskcounter, tid in enumerate(recursivetasks):
567 extradeps[tid] = set(self.runtaskentries[tid].depends) 570 extradeps[tid] = set(self.runtaskentries[tid].depends)
568 571
569 tasknames = recursivetasks[tid] 572 tasknames = recursivetasks[tid]
@@ -585,6 +588,7 @@ class RunQueueData:
585 if tid in recursiveitasks: 588 if tid in recursiveitasks:
586 for dep in recursiveitasks[tid]: 589 for dep in recursiveitasks[tid]:
587 generate_recdeps(dep) 590 generate_recdeps(dep)
591 self.init_progress_reporter.update(taskcounter)
588 592
589 # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work 593 # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work
590 for tid in recursivetasks: 594 for tid in recursivetasks:
@@ -600,6 +604,8 @@ class RunQueueData:
600 logger.debug(2, "Task %s contains self reference!", tid) 604 logger.debug(2, "Task %s contains self reference!", tid)
601 self.runtaskentries[tid].depends.remove(tid) 605 self.runtaskentries[tid].depends.remove(tid)
602 606
607 self.init_progress_reporter.next_stage()
608
603 # Step B - Mark all active tasks 609 # Step B - Mark all active tasks
604 # 610 #
605 # Start with the tasks we were asked to run and mark all dependencies 611 # Start with the tasks we were asked to run and mark all dependencies
@@ -664,6 +670,8 @@ class RunQueueData:
664 else: 670 else:
665 mark_active(tid, 1) 671 mark_active(tid, 1)
666 672
673 self.init_progress_reporter.next_stage()
674
667 # Step C - Prune all inactive tasks 675 # Step C - Prune all inactive tasks
668 # 676 #
669 # Once all active tasks are marked, prune the ones we don't need. 677 # Once all active tasks are marked, prune the ones we don't need.
@@ -674,6 +682,8 @@ class RunQueueData:
674 del self.runtaskentries[tid] 682 del self.runtaskentries[tid]
675 delcount += 1 683 delcount += 1
676 684
685 self.init_progress_reporter.next_stage()
686
677 # 687 #
678 # Step D - Sanity checks and computation 688 # Step D - Sanity checks and computation
679 # 689 #
@@ -689,11 +699,15 @@ class RunQueueData:
689 699
690 logger.verbose("Assign Weightings") 700 logger.verbose("Assign Weightings")
691 701
702 self.init_progress_reporter.next_stage()
703
692 # Generate a list of reverse dependencies to ease future calculations 704 # Generate a list of reverse dependencies to ease future calculations
693 for tid in self.runtaskentries: 705 for tid in self.runtaskentries:
694 for dep in self.runtaskentries[tid].depends: 706 for dep in self.runtaskentries[tid].depends:
695 self.runtaskentries[dep].revdeps.add(tid) 707 self.runtaskentries[dep].revdeps.add(tid)
696 708
709 self.init_progress_reporter.next_stage()
710
697 # Identify tasks at the end of dependency chains 711 # Identify tasks at the end of dependency chains
698 # Error on circular dependency loops (length two) 712 # Error on circular dependency loops (length two)
699 endpoints = [] 713 endpoints = []
@@ -709,10 +723,14 @@ class RunQueueData:
709 723
710 logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints)) 724 logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints))
711 725
726 self.init_progress_reporter.next_stage()
727
712 # Calculate task weights 728 # Calculate task weights
713 # Check of higher length circular dependencies 729 # Check of higher length circular dependencies
714 self.runq_weight = self.calculate_task_weights(endpoints) 730 self.runq_weight = self.calculate_task_weights(endpoints)
715 731
732 self.init_progress_reporter.next_stage()
733
716 # Sanity Check - Check for multiple tasks building the same provider 734 # Sanity Check - Check for multiple tasks building the same provider
717 prov_list = {} 735 prov_list = {}
718 seen_fn = [] 736 seen_fn = []
@@ -804,6 +822,8 @@ class RunQueueData:
804 else: 822 else:
805 logger.error(msg) 823 logger.error(msg)
806 824
825 self.init_progress_reporter.next_stage()
826
807 # Create a whitelist usable by the stamp checks 827 # Create a whitelist usable by the stamp checks
808 stampfnwhitelist = [] 828 stampfnwhitelist = []
809 for entry in self.stampwhitelist.split(): 829 for entry in self.stampwhitelist.split():
@@ -813,6 +833,8 @@ class RunQueueData:
813 stampfnwhitelist.append(fn) 833 stampfnwhitelist.append(fn)
814 self.stampfnwhitelist = stampfnwhitelist 834 self.stampfnwhitelist = stampfnwhitelist
815 835
836 self.init_progress_reporter.next_stage()
837
816 # Iterate over the task list looking for tasks with a 'setscene' function 838 # Iterate over the task list looking for tasks with a 'setscene' function
817 self.runq_setscene_tids = [] 839 self.runq_setscene_tids = []
818 if not self.cooker.configuration.nosetscene: 840 if not self.cooker.configuration.nosetscene:
@@ -837,6 +859,8 @@ class RunQueueData:
837 logger.verbose("Invalidate task %s, %s", taskname, fn) 859 logger.verbose("Invalidate task %s, %s", taskname, fn)
838 bb.parse.siggen.invalidate_task(taskname, self.dataCache, fn) 860 bb.parse.siggen.invalidate_task(taskname, self.dataCache, fn)
839 861
862 self.init_progress_reporter.next_stage()
863
840 # Invalidate task if force mode active 864 # Invalidate task if force mode active
841 if self.cooker.configuration.force: 865 if self.cooker.configuration.force:
842 for (fn, target) in self.target_pairs: 866 for (fn, target) in self.target_pairs:
@@ -850,6 +874,8 @@ class RunQueueData:
850 st = "do_%s" % st 874 st = "do_%s" % st
851 invalidate_task(fn, st, True) 875 invalidate_task(fn, st, True)
852 876
877 self.init_progress_reporter.next_stage()
878
853 # Create and print to the logs a virtual/xxxx -> PN (fn) table 879 # Create and print to the logs a virtual/xxxx -> PN (fn) table
854 virtmap = taskData.get_providermap(prefix="virtual/") 880 virtmap = taskData.get_providermap(prefix="virtual/")
855 virtpnmap = {} 881 virtpnmap = {}
@@ -859,6 +885,8 @@ class RunQueueData:
859 if hasattr(bb.parse.siggen, "tasks_resolved"): 885 if hasattr(bb.parse.siggen, "tasks_resolved"):
860 bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCache) 886 bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCache)
861 887
888 self.init_progress_reporter.next_stage()
889
862 # Iterate over the task list and call into the siggen code 890 # Iterate over the task list and call into the siggen code
863 dealtwith = set() 891 dealtwith = set()
864 todeal = set(self.runtaskentries) 892 todeal = set(self.runtaskentries)
@@ -1096,14 +1124,25 @@ class RunQueue:
1096 1124
1097 if self.state is runQueuePrepare: 1125 if self.state is runQueuePrepare:
1098 self.rqexe = RunQueueExecuteDummy(self) 1126 self.rqexe = RunQueueExecuteDummy(self)
1127 # NOTE: if you add, remove or significantly refactor the stages of this
1128 # process then you should recalculate the weightings here. This is quite
1129 # easy to do - just change the next line temporarily to pass debug=True as
1130 # the last parameter and you'll get a printout of the weightings as well
1131 # as a map to the lines where next_stage() was called. Of course this isn't
1132 # critical, but it helps to keep the progress reporting accurate.
1133 self.rqdata.init_progress_reporter = bb.progress.MultiStageProcessProgressReporter(self.cooker.data,
1134 "Initialising tasks",
1135 [43, 967, 4, 3, 1, 5, 3, 7, 13, 1, 2, 1, 1, 246, 35, 1, 38, 1, 35, 2, 338, 204, 142, 3, 3, 37, 244])
1099 if self.rqdata.prepare() == 0: 1136 if self.rqdata.prepare() == 0:
1100 self.state = runQueueComplete 1137 self.state = runQueueComplete
1101 else: 1138 else:
1102 self.state = runQueueSceneInit 1139 self.state = runQueueSceneInit
1140 self.rqdata.init_progress_reporter.next_stage()
1103 1141
1104 # we are ready to run, emit dependency info to any UI or class which 1142 # we are ready to run, emit dependency info to any UI or class which
1105 # needs it 1143 # needs it
1106 depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData) 1144 depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData)
1145 self.rqdata.init_progress_reporter.next_stage()
1107 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data) 1146 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data)
1108 1147
1109 if self.state is runQueueSceneInit: 1148 if self.state is runQueueSceneInit:
@@ -1116,7 +1155,9 @@ class RunQueue:
1116 self.write_diffscenetasks(invalidtasks) 1155 self.write_diffscenetasks(invalidtasks)
1117 self.state = runQueueComplete 1156 self.state = runQueueComplete
1118 else: 1157 else:
1158 self.rqdata.init_progress_reporter.next_stage()
1119 self.start_worker() 1159 self.start_worker()
1160 self.rqdata.init_progress_reporter.next_stage()
1120 self.rqexe = RunQueueExecuteScenequeue(self) 1161 self.rqexe = RunQueueExecuteScenequeue(self)
1121 1162
1122 if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp]: 1163 if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp]:
@@ -1129,6 +1170,8 @@ class RunQueue:
1129 if self.cooker.configuration.setsceneonly: 1170 if self.cooker.configuration.setsceneonly:
1130 self.state = runQueueComplete 1171 self.state = runQueueComplete
1131 else: 1172 else:
1173 # Just in case we didn't setscene
1174 self.rqdata.init_progress_reporter.finish()
1132 logger.info("Executing RunQueue Tasks") 1175 logger.info("Executing RunQueue Tasks")
1133 self.rqexe = RunQueueExecuteTasks(self) 1176 self.rqexe = RunQueueExecuteTasks(self)
1134 self.state = runQueueRunning 1177 self.state = runQueueRunning
@@ -1769,6 +1812,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1769 # therefore aims to collapse the huge runqueue dependency tree into a smaller one 1812 # therefore aims to collapse the huge runqueue dependency tree into a smaller one
1770 # only containing the setscene functions. 1813 # only containing the setscene functions.
1771 1814
1815 self.rqdata.init_progress_reporter.next_stage()
1816
1772 # First process the chains up to the first setscene task. 1817 # First process the chains up to the first setscene task.
1773 endpoints = {} 1818 endpoints = {}
1774 for tid in self.rqdata.runtaskentries: 1819 for tid in self.rqdata.runtaskentries:
@@ -1778,6 +1823,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1778 #bb.warn("Added endpoint %s" % (tid)) 1823 #bb.warn("Added endpoint %s" % (tid))
1779 endpoints[tid] = set() 1824 endpoints[tid] = set()
1780 1825
1826 self.rqdata.init_progress_reporter.next_stage()
1827
1781 # Secondly process the chains between setscene tasks. 1828 # Secondly process the chains between setscene tasks.
1782 for tid in self.rqdata.runq_setscene_tids: 1829 for tid in self.rqdata.runq_setscene_tids:
1783 #bb.warn("Added endpoint 2 %s" % (tid)) 1830 #bb.warn("Added endpoint 2 %s" % (tid))
@@ -1787,6 +1834,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1787 #bb.warn(" Added endpoint 3 %s" % (dep)) 1834 #bb.warn(" Added endpoint 3 %s" % (dep))
1788 endpoints[dep].add(tid) 1835 endpoints[dep].add(tid)
1789 1836
1837 self.rqdata.init_progress_reporter.next_stage()
1838
1790 def process_endpoints(endpoints): 1839 def process_endpoints(endpoints):
1791 newendpoints = {} 1840 newendpoints = {}
1792 for point, task in endpoints.items(): 1841 for point, task in endpoints.items():
@@ -1811,6 +1860,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1811 1860
1812 process_endpoints(endpoints) 1861 process_endpoints(endpoints)
1813 1862
1863 self.rqdata.init_progress_reporter.next_stage()
1864
1814 # Build a list of setscene tasks which are "unskippable" 1865 # Build a list of setscene tasks which are "unskippable"
1815 # These are direct endpoints referenced by the build 1866 # These are direct endpoints referenced by the build
1816 endpoints2 = {} 1867 endpoints2 = {}
@@ -1847,7 +1898,9 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1847 if sq_revdeps_new2[tid]: 1898 if sq_revdeps_new2[tid]:
1848 self.unskippable.append(tid) 1899 self.unskippable.append(tid)
1849 1900
1850 for tid in self.rqdata.runtaskentries: 1901 self.rqdata.init_progress_reporter.next_stage(len(self.rqdata.runtaskentries))
1902
1903 for taskcounter, tid in enumerate(self.rqdata.runtaskentries):
1851 if tid in self.rqdata.runq_setscene_tids: 1904 if tid in self.rqdata.runq_setscene_tids:
1852 deps = set() 1905 deps = set()
1853 for dep in sq_revdeps_new[tid]: 1906 for dep in sq_revdeps_new[tid]:
@@ -1855,6 +1908,9 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1855 sq_revdeps_squash[tid] = deps 1908 sq_revdeps_squash[tid] = deps
1856 elif len(sq_revdeps_new[tid]) != 0: 1909 elif len(sq_revdeps_new[tid]) != 0:
1857 bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.") 1910 bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.")
1911 self.rqdata.init_progress_reporter.update(taskcounter)
1912
1913 self.rqdata.init_progress_reporter.next_stage()
1858 1914
1859 # Resolve setscene inter-task dependencies 1915 # Resolve setscene inter-task dependencies
1860 # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene" 1916 # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene"
@@ -1882,10 +1938,14 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1882 # Have to zero this to avoid circular dependencies 1938 # Have to zero this to avoid circular dependencies
1883 sq_revdeps_squash[deptid] = set() 1939 sq_revdeps_squash[deptid] = set()
1884 1940
1941 self.rqdata.init_progress_reporter.next_stage()
1942
1885 for task in self.sq_harddeps: 1943 for task in self.sq_harddeps:
1886 for dep in self.sq_harddeps[task]: 1944 for dep in self.sq_harddeps[task]:
1887 sq_revdeps_squash[dep].add(task) 1945 sq_revdeps_squash[dep].add(task)
1888 1946
1947 self.rqdata.init_progress_reporter.next_stage()
1948
1889 #for tid in sq_revdeps_squash: 1949 #for tid in sq_revdeps_squash:
1890 # for dep in sq_revdeps_squash[tid]: 1950 # for dep in sq_revdeps_squash[tid]:
1891 # data = data + "\n %s" % dep 1951 # data = data + "\n %s" % dep
@@ -1901,6 +1961,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1901 for dep in self.sq_revdeps[tid]: 1961 for dep in self.sq_revdeps[tid]:
1902 self.sq_deps[dep].add(tid) 1962 self.sq_deps[dep].add(tid)
1903 1963
1964 self.rqdata.init_progress_reporter.next_stage()
1965
1904 for tid in self.sq_revdeps: 1966 for tid in self.sq_revdeps:
1905 if len(self.sq_revdeps[tid]) == 0: 1967 if len(self.sq_revdeps[tid]) == 0:
1906 self.runq_buildable.add(tid) 1968 self.runq_buildable.add(tid)
@@ -1956,6 +2018,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1956 logger.debug(2, 'No package found, so skipping setscene task %s', tid) 2018 logger.debug(2, 'No package found, so skipping setscene task %s', tid)
1957 self.outrightfail.append(tid) 2019 self.outrightfail.append(tid)
1958 2020
2021 self.rqdata.init_progress_reporter.finish()
2022
1959 logger.info('Executing SetScene Tasks') 2023 logger.info('Executing SetScene Tasks')
1960 2024
1961 self.rq.state = runQueueSceneRun 2025 self.rq.state = runQueueSceneRun