summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2016-06-23 22:59:12 +1200
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-07-08 09:57:27 +0100
commit1b930b41a5e677eea0adae3d247f43b77d1945f6 (patch)
treee3d3c0099efb6820c3bdc403d8842db4b4d73200
parentf3b62c1c2e47968084342d1fe2a9af596a3ba93c (diff)
downloadpoky-1b930b41a5e677eea0adae3d247f43b77d1945f6.tar.gz
bitbake: runqueue: report progress for "Preparing RunQueue" step
When "Preparing RunQueue" shows up you can expect to wait up to 30 seconds while it works - which is a bit long to leave the user waiting without any kind of output. Since the work being carried out during this time is divided into stages such that it's practical to determine internally how it's progressing, replace the message with a progress bar. Actually what happens during this time is two major steps rather than just one - the runqueue preparation itself, followed by the initialisation prior to running setscene tasks. I elected to have the progress bar cover both as one (there doesn't appear to be much point in doing otherwise from a user perspective). I did however describe it as "initialising tasks". (Bitbake rev: 591e9741e108487ff437e77cb439ef2dbca42e03) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/progress.py42
-rw-r--r--bitbake/lib/bb/runqueue.py70
2 files changed, 109 insertions, 3 deletions
diff --git a/bitbake/lib/bb/progress.py b/bitbake/lib/bb/progress.py
index 93e42dfcdb..49417761bb 100644
--- a/bitbake/lib/bb/progress.py
+++ b/bitbake/lib/bb/progress.py
@@ -195,3 +195,45 @@ class MultiStageProgressReporter(object):
195 else: 195 else:
196 out.append('Up to finish: %d' % stage_weight) 196 out.append('Up to finish: %d' % stage_weight)
197 bb.warn('Stage times:\n %s' % '\n '.join(out)) 197 bb.warn('Stage times:\n %s' % '\n '.join(out))
198
199class MultiStageProcessProgressReporter(MultiStageProgressReporter):
200 """
201 Version of MultiStageProgressReporter intended for use with
202 standalone processes (such as preparing the runqueue)
203 """
204 def __init__(self, d, processname, stage_weights, debug=False):
205 self._processname = processname
206 MultiStageProgressReporter.__init__(self, d, stage_weights, debug)
207
208 def start(self):
209 bb.event.fire(bb.event.ProcessStarted(self._processname, 100), self._data)
210
211 def _fire_progress(self, taskprogress):
212 bb.event.fire(bb.event.ProcessProgress(self._processname, taskprogress), self._data)
213
214 def finish(self):
215 MultiStageProgressReporter.finish(self)
216 bb.event.fire(bb.event.ProcessFinished(self._processname), self._data)
217
218class DummyMultiStageProcessProgressReporter(MultiStageProgressReporter):
219 """
220 MultiStageProcessProgressReporter that takes the calls and does nothing
221 with them (to avoid a bunch of "if progress_reporter:" checks)
222 """
223 def __init__(self):
224 MultiStageProcessProgressReporter.__init__(self, "", None, [])
225
226 def _fire_progress(self, taskprogress, rate=None):
227 pass
228
229 def start(self):
230 pass
231
232 def next_stage(self, stage_total=None):
233 pass
234
235 def update(self, stage_progress):
236 pass
237
238 def finish(self):
239 pass
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index b62a28a2be..57be15a62b 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -241,6 +241,7 @@ class RunQueueData:
241 self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or "" 241 self.stampwhitelist = cfgData.getVar("BB_STAMP_WHITELIST", True) or ""
242 self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split() 242 self.multi_provider_whitelist = (cfgData.getVar("MULTI_PROVIDER_WHITELIST", True) or "").split()
243 self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData) 243 self.setscenewhitelist = get_setscene_enforce_whitelist(cfgData)
244 self.init_progress_reporter = bb.progress.DummyMultiStageProcessProgressReporter()
244 245
245 self.reset() 246 self.reset()
246 247
@@ -432,7 +433,8 @@ class RunQueueData:
432 # Nothing to do 433 # Nothing to do
433 return 0 434 return 0
434 435
435 logger.info("Preparing RunQueue") 436 self.init_progress_reporter.start()
437 self.init_progress_reporter.next_stage()
436 438
437 # Step A - Work out a list of tasks to run 439 # Step A - Work out a list of tasks to run
438 # 440 #
@@ -562,8 +564,9 @@ class RunQueueData:
562 # e.g. do_sometask[recrdeptask] = "do_someothertask" 564 # e.g. do_sometask[recrdeptask] = "do_someothertask"
563 # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively) 565 # (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies, recursively)
564 # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed 566 # We need to do this separately since we need all of runtaskentries[*].depends to be complete before this is processed
567 self.init_progress_reporter.next_stage(len(recursivetasks))
565 extradeps = {} 568 extradeps = {}
566 for tid in recursivetasks: 569 for taskcounter, tid in enumerate(recursivetasks):
567 extradeps[tid] = set(self.runtaskentries[tid].depends) 570 extradeps[tid] = set(self.runtaskentries[tid].depends)
568 571
569 tasknames = recursivetasks[tid] 572 tasknames = recursivetasks[tid]
@@ -585,6 +588,7 @@ class RunQueueData:
585 if tid in recursiveitasks: 588 if tid in recursiveitasks:
586 for dep in recursiveitasks[tid]: 589 for dep in recursiveitasks[tid]:
587 generate_recdeps(dep) 590 generate_recdeps(dep)
591 self.init_progress_reporter.update(taskcounter)
588 592
589 # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work 593 # Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work
590 for tid in recursivetasks: 594 for tid in recursivetasks:
@@ -600,6 +604,8 @@ class RunQueueData:
600 logger.debug(2, "Task %s contains self reference!", tid) 604 logger.debug(2, "Task %s contains self reference!", tid)
601 self.runtaskentries[tid].depends.remove(tid) 605 self.runtaskentries[tid].depends.remove(tid)
602 606
607 self.init_progress_reporter.next_stage()
608
603 # Step B - Mark all active tasks 609 # Step B - Mark all active tasks
604 # 610 #
605 # Start with the tasks we were asked to run and mark all dependencies 611 # Start with the tasks we were asked to run and mark all dependencies
@@ -664,6 +670,8 @@ class RunQueueData:
664 else: 670 else:
665 mark_active(tid, 1) 671 mark_active(tid, 1)
666 672
673 self.init_progress_reporter.next_stage()
674
667 # Step C - Prune all inactive tasks 675 # Step C - Prune all inactive tasks
668 # 676 #
669 # Once all active tasks are marked, prune the ones we don't need. 677 # Once all active tasks are marked, prune the ones we don't need.
@@ -674,6 +682,8 @@ class RunQueueData:
674 del self.runtaskentries[tid] 682 del self.runtaskentries[tid]
675 delcount += 1 683 delcount += 1
676 684
685 self.init_progress_reporter.next_stage()
686
677 # 687 #
678 # Step D - Sanity checks and computation 688 # Step D - Sanity checks and computation
679 # 689 #
@@ -689,11 +699,15 @@ class RunQueueData:
689 699
690 logger.verbose("Assign Weightings") 700 logger.verbose("Assign Weightings")
691 701
702 self.init_progress_reporter.next_stage()
703
692 # Generate a list of reverse dependencies to ease future calculations 704 # Generate a list of reverse dependencies to ease future calculations
693 for tid in self.runtaskentries: 705 for tid in self.runtaskentries:
694 for dep in self.runtaskentries[tid].depends: 706 for dep in self.runtaskentries[tid].depends:
695 self.runtaskentries[dep].revdeps.add(tid) 707 self.runtaskentries[dep].revdeps.add(tid)
696 708
709 self.init_progress_reporter.next_stage()
710
697 # Identify tasks at the end of dependency chains 711 # Identify tasks at the end of dependency chains
698 # Error on circular dependency loops (length two) 712 # Error on circular dependency loops (length two)
699 endpoints = [] 713 endpoints = []
@@ -709,10 +723,14 @@ class RunQueueData:
709 723
710 logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints)) 724 logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints))
711 725
726 self.init_progress_reporter.next_stage()
727
712 # Calculate task weights 728 # Calculate task weights
713 # Check of higher length circular dependencies 729 # Check of higher length circular dependencies
714 self.runq_weight = self.calculate_task_weights(endpoints) 730 self.runq_weight = self.calculate_task_weights(endpoints)
715 731
732 self.init_progress_reporter.next_stage()
733
716 # Sanity Check - Check for multiple tasks building the same provider 734 # Sanity Check - Check for multiple tasks building the same provider
717 prov_list = {} 735 prov_list = {}
718 seen_fn = [] 736 seen_fn = []
@@ -804,6 +822,8 @@ class RunQueueData:
804 else: 822 else:
805 logger.error(msg) 823 logger.error(msg)
806 824
825 self.init_progress_reporter.next_stage()
826
807 # Create a whitelist usable by the stamp checks 827 # Create a whitelist usable by the stamp checks
808 stampfnwhitelist = [] 828 stampfnwhitelist = []
809 for entry in self.stampwhitelist.split(): 829 for entry in self.stampwhitelist.split():
@@ -813,6 +833,8 @@ class RunQueueData:
813 stampfnwhitelist.append(fn) 833 stampfnwhitelist.append(fn)
814 self.stampfnwhitelist = stampfnwhitelist 834 self.stampfnwhitelist = stampfnwhitelist
815 835
836 self.init_progress_reporter.next_stage()
837
816 # Iterate over the task list looking for tasks with a 'setscene' function 838 # Iterate over the task list looking for tasks with a 'setscene' function
817 self.runq_setscene_tids = [] 839 self.runq_setscene_tids = []
818 if not self.cooker.configuration.nosetscene: 840 if not self.cooker.configuration.nosetscene:
@@ -837,6 +859,8 @@ class RunQueueData:
837 logger.verbose("Invalidate task %s, %s", taskname, fn) 859 logger.verbose("Invalidate task %s, %s", taskname, fn)
838 bb.parse.siggen.invalidate_task(taskname, self.dataCache, fn) 860 bb.parse.siggen.invalidate_task(taskname, self.dataCache, fn)
839 861
862 self.init_progress_reporter.next_stage()
863
840 # Invalidate task if force mode active 864 # Invalidate task if force mode active
841 if self.cooker.configuration.force: 865 if self.cooker.configuration.force:
842 for (fn, target) in self.target_pairs: 866 for (fn, target) in self.target_pairs:
@@ -850,6 +874,8 @@ class RunQueueData:
850 st = "do_%s" % st 874 st = "do_%s" % st
851 invalidate_task(fn, st, True) 875 invalidate_task(fn, st, True)
852 876
877 self.init_progress_reporter.next_stage()
878
853 # Create and print to the logs a virtual/xxxx -> PN (fn) table 879 # Create and print to the logs a virtual/xxxx -> PN (fn) table
854 virtmap = taskData.get_providermap(prefix="virtual/") 880 virtmap = taskData.get_providermap(prefix="virtual/")
855 virtpnmap = {} 881 virtpnmap = {}
@@ -859,6 +885,8 @@ class RunQueueData:
859 if hasattr(bb.parse.siggen, "tasks_resolved"): 885 if hasattr(bb.parse.siggen, "tasks_resolved"):
860 bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCache) 886 bb.parse.siggen.tasks_resolved(virtmap, virtpnmap, self.dataCache)
861 887
888 self.init_progress_reporter.next_stage()
889
862 # Iterate over the task list and call into the siggen code 890 # Iterate over the task list and call into the siggen code
863 dealtwith = set() 891 dealtwith = set()
864 todeal = set(self.runtaskentries) 892 todeal = set(self.runtaskentries)
@@ -1096,14 +1124,25 @@ class RunQueue:
1096 1124
1097 if self.state is runQueuePrepare: 1125 if self.state is runQueuePrepare:
1098 self.rqexe = RunQueueExecuteDummy(self) 1126 self.rqexe = RunQueueExecuteDummy(self)
1127 # NOTE: if you add, remove or significantly refactor the stages of this
1128 # process then you should recalculate the weightings here. This is quite
1129 # easy to do - just change the next line temporarily to pass debug=True as
1130 # the last parameter and you'll get a printout of the weightings as well
1131 # as a map to the lines where next_stage() was called. Of course this isn't
1132 # critical, but it helps to keep the progress reporting accurate.
1133 self.rqdata.init_progress_reporter = bb.progress.MultiStageProcessProgressReporter(self.cooker.data,
1134 "Initialising tasks",
1135 [43, 967, 4, 3, 1, 5, 3, 7, 13, 1, 2, 1, 1, 246, 35, 1, 38, 1, 35, 2, 338, 204, 142, 3, 3, 37, 244])
1099 if self.rqdata.prepare() == 0: 1136 if self.rqdata.prepare() == 0:
1100 self.state = runQueueComplete 1137 self.state = runQueueComplete
1101 else: 1138 else:
1102 self.state = runQueueSceneInit 1139 self.state = runQueueSceneInit
1140 self.rqdata.init_progress_reporter.next_stage()
1103 1141
1104 # we are ready to run, emit dependency info to any UI or class which 1142 # we are ready to run, emit dependency info to any UI or class which
1105 # needs it 1143 # needs it
1106 depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData) 1144 depgraph = self.cooker.buildDependTree(self, self.rqdata.taskData)
1145 self.rqdata.init_progress_reporter.next_stage()
1107 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data) 1146 bb.event.fire(bb.event.DepTreeGenerated(depgraph), self.cooker.data)
1108 1147
1109 if self.state is runQueueSceneInit: 1148 if self.state is runQueueSceneInit:
@@ -1116,7 +1155,9 @@ class RunQueue:
1116 self.write_diffscenetasks(invalidtasks) 1155 self.write_diffscenetasks(invalidtasks)
1117 self.state = runQueueComplete 1156 self.state = runQueueComplete
1118 else: 1157 else:
1158 self.rqdata.init_progress_reporter.next_stage()
1119 self.start_worker() 1159 self.start_worker()
1160 self.rqdata.init_progress_reporter.next_stage()
1120 self.rqexe = RunQueueExecuteScenequeue(self) 1161 self.rqexe = RunQueueExecuteScenequeue(self)
1121 1162
1122 if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp]: 1163 if self.state in [runQueueSceneRun, runQueueRunning, runQueueCleanUp]:
@@ -1129,6 +1170,8 @@ class RunQueue:
1129 if self.cooker.configuration.setsceneonly: 1170 if self.cooker.configuration.setsceneonly:
1130 self.state = runQueueComplete 1171 self.state = runQueueComplete
1131 else: 1172 else:
1173 # Just in case we didn't setscene
1174 self.rqdata.init_progress_reporter.finish()
1132 logger.info("Executing RunQueue Tasks") 1175 logger.info("Executing RunQueue Tasks")
1133 self.rqexe = RunQueueExecuteTasks(self) 1176 self.rqexe = RunQueueExecuteTasks(self)
1134 self.state = runQueueRunning 1177 self.state = runQueueRunning
@@ -1769,6 +1812,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1769 # therefore aims to collapse the huge runqueue dependency tree into a smaller one 1812 # therefore aims to collapse the huge runqueue dependency tree into a smaller one
1770 # only containing the setscene functions. 1813 # only containing the setscene functions.
1771 1814
1815 self.rqdata.init_progress_reporter.next_stage()
1816
1772 # First process the chains up to the first setscene task. 1817 # First process the chains up to the first setscene task.
1773 endpoints = {} 1818 endpoints = {}
1774 for tid in self.rqdata.runtaskentries: 1819 for tid in self.rqdata.runtaskentries:
@@ -1778,6 +1823,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1778 #bb.warn("Added endpoint %s" % (tid)) 1823 #bb.warn("Added endpoint %s" % (tid))
1779 endpoints[tid] = set() 1824 endpoints[tid] = set()
1780 1825
1826 self.rqdata.init_progress_reporter.next_stage()
1827
1781 # Secondly process the chains between setscene tasks. 1828 # Secondly process the chains between setscene tasks.
1782 for tid in self.rqdata.runq_setscene_tids: 1829 for tid in self.rqdata.runq_setscene_tids:
1783 #bb.warn("Added endpoint 2 %s" % (tid)) 1830 #bb.warn("Added endpoint 2 %s" % (tid))
@@ -1787,6 +1834,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1787 #bb.warn(" Added endpoint 3 %s" % (dep)) 1834 #bb.warn(" Added endpoint 3 %s" % (dep))
1788 endpoints[dep].add(tid) 1835 endpoints[dep].add(tid)
1789 1836
1837 self.rqdata.init_progress_reporter.next_stage()
1838
1790 def process_endpoints(endpoints): 1839 def process_endpoints(endpoints):
1791 newendpoints = {} 1840 newendpoints = {}
1792 for point, task in endpoints.items(): 1841 for point, task in endpoints.items():
@@ -1811,6 +1860,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1811 1860
1812 process_endpoints(endpoints) 1861 process_endpoints(endpoints)
1813 1862
1863 self.rqdata.init_progress_reporter.next_stage()
1864
1814 # Build a list of setscene tasks which are "unskippable" 1865 # Build a list of setscene tasks which are "unskippable"
1815 # These are direct endpoints referenced by the build 1866 # These are direct endpoints referenced by the build
1816 endpoints2 = {} 1867 endpoints2 = {}
@@ -1847,7 +1898,9 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1847 if sq_revdeps_new2[tid]: 1898 if sq_revdeps_new2[tid]:
1848 self.unskippable.append(tid) 1899 self.unskippable.append(tid)
1849 1900
1850 for tid in self.rqdata.runtaskentries: 1901 self.rqdata.init_progress_reporter.next_stage(len(self.rqdata.runtaskentries))
1902
1903 for taskcounter, tid in enumerate(self.rqdata.runtaskentries):
1851 if tid in self.rqdata.runq_setscene_tids: 1904 if tid in self.rqdata.runq_setscene_tids:
1852 deps = set() 1905 deps = set()
1853 for dep in sq_revdeps_new[tid]: 1906 for dep in sq_revdeps_new[tid]:
@@ -1855,6 +1908,9 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1855 sq_revdeps_squash[tid] = deps 1908 sq_revdeps_squash[tid] = deps
1856 elif len(sq_revdeps_new[tid]) != 0: 1909 elif len(sq_revdeps_new[tid]) != 0:
1857 bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.") 1910 bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting. Please report this problem.")
1911 self.rqdata.init_progress_reporter.update(taskcounter)
1912
1913 self.rqdata.init_progress_reporter.next_stage()
1858 1914
1859 # Resolve setscene inter-task dependencies 1915 # Resolve setscene inter-task dependencies
1860 # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene" 1916 # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene"
@@ -1882,10 +1938,14 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1882 # Have to zero this to avoid circular dependencies 1938 # Have to zero this to avoid circular dependencies
1883 sq_revdeps_squash[deptid] = set() 1939 sq_revdeps_squash[deptid] = set()
1884 1940
1941 self.rqdata.init_progress_reporter.next_stage()
1942
1885 for task in self.sq_harddeps: 1943 for task in self.sq_harddeps:
1886 for dep in self.sq_harddeps[task]: 1944 for dep in self.sq_harddeps[task]:
1887 sq_revdeps_squash[dep].add(task) 1945 sq_revdeps_squash[dep].add(task)
1888 1946
1947 self.rqdata.init_progress_reporter.next_stage()
1948
1889 #for tid in sq_revdeps_squash: 1949 #for tid in sq_revdeps_squash:
1890 # for dep in sq_revdeps_squash[tid]: 1950 # for dep in sq_revdeps_squash[tid]:
1891 # data = data + "\n %s" % dep 1951 # data = data + "\n %s" % dep
@@ -1901,6 +1961,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1901 for dep in self.sq_revdeps[tid]: 1961 for dep in self.sq_revdeps[tid]:
1902 self.sq_deps[dep].add(tid) 1962 self.sq_deps[dep].add(tid)
1903 1963
1964 self.rqdata.init_progress_reporter.next_stage()
1965
1904 for tid in self.sq_revdeps: 1966 for tid in self.sq_revdeps:
1905 if len(self.sq_revdeps[tid]) == 0: 1967 if len(self.sq_revdeps[tid]) == 0:
1906 self.runq_buildable.add(tid) 1968 self.runq_buildable.add(tid)
@@ -1956,6 +2018,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1956 logger.debug(2, 'No package found, so skipping setscene task %s', tid) 2018 logger.debug(2, 'No package found, so skipping setscene task %s', tid)
1957 self.outrightfail.append(tid) 2019 self.outrightfail.append(tid)
1958 2020
2021 self.rqdata.init_progress_reporter.finish()
2022
1959 logger.info('Executing SetScene Tasks') 2023 logger.info('Executing SetScene Tasks')
1960 2024
1961 self.rq.state = runQueueSceneRun 2025 self.rq.state = runQueueSceneRun