summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2016-06-12 23:55:08 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-06-15 08:35:07 +0100
commit29b4fb17966bfaec74b41cce533725c04ec1078f (patch)
tree4a10c618d264af930d0d8dd9b42e434d8e0d16c8 /bitbake/lib/bb
parent7e54da0581faaafa9bfb13680ad6618c4c8f9831 (diff)
downloadpoky-29b4fb17966bfaec74b41cce533725c04ec1078f.tar.gz
bitbake: runqueue: Change buildable/running lists to sets
Using positions in lists for flags is an odd choice and makes the code hard to maintain. Maintaining a list is slow since list searches are slow (watch bitbake -n slow massively with it) but we can use a set() instead. This patch uses python sets to maintain the lists of tasks in each state and this prepares for changing the task IDs from being integers. (Bitbake rev: 8c1ed57f6ea475b714eca6673b48e8e5f5f0f9c3) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/runqueue.py63
1 files changed, 27 insertions, 36 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index 376c9f51e8..de618ff1d8 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -108,7 +108,7 @@ class RunQueueScheduler(object):
108 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]] 108 fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]]
109 taskname = self.rqdata.runq_task[taskid] 109 taskname = self.rqdata.runq_task[taskid]
110 self.stamps[taskid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) 110 self.stamps[taskid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
111 if self.rq.runq_buildable[taskid] == 1: 111 if taskid in self.rq.runq_buildable:
112 self.buildable.append(taskid) 112 self.buildable.append(taskid)
113 113
114 self.rev_prio_map = None 114 self.rev_prio_map = None
@@ -117,7 +117,7 @@ class RunQueueScheduler(object):
117 """ 117 """
118 Return the id of the first task we find that is buildable 118 Return the id of the first task we find that is buildable
119 """ 119 """
120 self.buildable = [x for x in self.buildable if not self.rq.runq_running[x] == 1] 120 self.buildable = [x for x in self.buildable if x not in self.rq.runq_running]
121 if not self.buildable: 121 if not self.buildable:
122 return None 122 return None
123 if len(self.buildable) == 1: 123 if len(self.buildable) == 1:
@@ -1361,9 +1361,9 @@ class RunQueueExecute:
1361 self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS", True) or 1) 1361 self.number_tasks = int(self.cfgData.getVar("BB_NUMBER_THREADS", True) or 1)
1362 self.scheduler = self.cfgData.getVar("BB_SCHEDULER", True) or "speed" 1362 self.scheduler = self.cfgData.getVar("BB_SCHEDULER", True) or "speed"
1363 1363
1364 self.runq_buildable = [] 1364 self.runq_buildable = set()
1365 self.runq_running = [] 1365 self.runq_running = set()
1366 self.runq_complete = [] 1366 self.runq_complete = set()
1367 1367
1368 self.build_stamps = {} 1368 self.build_stamps = {}
1369 self.build_stamps2 = [] 1369 self.build_stamps2 = []
@@ -1464,12 +1464,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
1464 1464
1465 # Mark initial buildable tasks 1465 # Mark initial buildable tasks
1466 for task in range(self.stats.total): 1466 for task in range(self.stats.total):
1467 self.runq_running.append(0)
1468 self.runq_complete.append(0)
1469 if len(self.rqdata.runq_depends[task]) == 0: 1467 if len(self.rqdata.runq_depends[task]) == 0:
1470 self.runq_buildable.append(1) 1468 self.runq_buildable.add(task)
1471 else:
1472 self.runq_buildable.append(0)
1473 if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered): 1469 if len(self.rqdata.runq_revdeps[task]) > 0 and self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered):
1474 self.rq.scenequeue_covered.add(task) 1470 self.rq.scenequeue_covered.add(task)
1475 1471
@@ -1574,7 +1570,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1574 return schedulers 1570 return schedulers
1575 1571
1576 def setbuildable(self, task): 1572 def setbuildable(self, task):
1577 self.runq_buildable[task] = 1 1573 self.runq_buildable.add(task)
1578 self.sched.newbuilable(task) 1574 self.sched.newbuilable(task)
1579 1575
1580 def task_completeoutright(self, task): 1576 def task_completeoutright(self, task):
@@ -1583,15 +1579,15 @@ class RunQueueExecuteTasks(RunQueueExecute):
1583 Look at the reverse dependencies and mark any task with 1579 Look at the reverse dependencies and mark any task with
1584 completed dependencies as buildable 1580 completed dependencies as buildable
1585 """ 1581 """
1586 self.runq_complete[task] = 1 1582 self.runq_complete.add(task)
1587 for revdep in self.rqdata.runq_revdeps[task]: 1583 for revdep in self.rqdata.runq_revdeps[task]:
1588 if self.runq_running[revdep] == 1: 1584 if revdep in self.runq_running:
1589 continue 1585 continue
1590 if self.runq_buildable[revdep] == 1: 1586 if revdep in self.runq_buildable:
1591 continue 1587 continue
1592 alldeps = 1 1588 alldeps = 1
1593 for dep in self.rqdata.runq_depends[revdep]: 1589 for dep in self.rqdata.runq_depends[revdep]:
1594 if self.runq_complete[dep] != 1: 1590 if dep not in self.runq_complete:
1595 alldeps = 0 1591 alldeps = 0
1596 if alldeps == 1: 1592 if alldeps == 1:
1597 self.setbuildable(revdep) 1593 self.setbuildable(revdep)
@@ -1617,7 +1613,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1617 self.rq.state = runQueueCleanUp 1613 self.rq.state = runQueueCleanUp
1618 1614
1619 def task_skip(self, task, reason): 1615 def task_skip(self, task, reason):
1620 self.runq_running[task] = 1 1616 self.runq_running.add(task)
1621 self.setbuildable(task) 1617 self.setbuildable(task)
1622 bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, reason), self.cfgData) 1618 bb.event.fire(runQueueTaskSkipped(task, self.stats, self.rq, reason), self.cfgData)
1623 self.task_completeoutright(task) 1619 self.task_completeoutright(task)
@@ -1658,7 +1654,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1658 startevent = runQueueTaskStarted(task, self.stats, self.rq, 1654 startevent = runQueueTaskStarted(task, self.stats, self.rq,
1659 noexec=True) 1655 noexec=True)
1660 bb.event.fire(startevent, self.cfgData) 1656 bb.event.fire(startevent, self.cfgData)
1661 self.runq_running[task] = 1 1657 self.runq_running.add(task)
1662 self.stats.taskActive() 1658 self.stats.taskActive()
1663 if not self.cooker.configuration.dry_run: 1659 if not self.cooker.configuration.dry_run:
1664 bb.build.make_stamp(taskname, self.rqdata.dataCache, fn) 1660 bb.build.make_stamp(taskname, self.rqdata.dataCache, fn)
@@ -1687,7 +1683,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1687 1683
1688 self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn) 1684 self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
1689 self.build_stamps2.append(self.build_stamps[task]) 1685 self.build_stamps2.append(self.build_stamps[task])
1690 self.runq_running[task] = 1 1686 self.runq_running.add(task)
1691 self.stats.taskActive() 1687 self.stats.taskActive()
1692 if self.stats.active < self.number_tasks: 1688 if self.stats.active < self.number_tasks:
1693 return True 1689 return True
@@ -1702,11 +1698,11 @@ class RunQueueExecuteTasks(RunQueueExecute):
1702 1698
1703 # Sanity Checks 1699 # Sanity Checks
1704 for task in range(self.stats.total): 1700 for task in range(self.stats.total):
1705 if self.runq_buildable[task] == 0: 1701 if task not in self.runq_buildable:
1706 logger.error("Task %s never buildable!", task) 1702 logger.error("Task %s never buildable!", task)
1707 if self.runq_running[task] == 0: 1703 if task not in self.runq_running:
1708 logger.error("Task %s never ran!", task) 1704 logger.error("Task %s never ran!", task)
1709 if self.runq_complete[task] == 0: 1705 if task not in self.runq_complete:
1710 logger.error("Task %s never completed!", task) 1706 logger.error("Task %s never completed!", task)
1711 self.rq.state = runQueueComplete 1707 self.rq.state = runQueueComplete
1712 1708
@@ -1759,11 +1755,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1759 # therefore aims to collapse the huge runqueue dependency tree into a smaller one 1755 # therefore aims to collapse the huge runqueue dependency tree into a smaller one
1760 # only containing the setscene functions. 1756 # only containing the setscene functions.
1761 1757
1762 for task in range(self.stats.total):
1763 self.runq_running.append(0)
1764 self.runq_complete.append(0)
1765 self.runq_buildable.append(0)
1766
1767 # First process the chains up to the first setscene task. 1758 # First process the chains up to the first setscene task.
1768 endpoints = {} 1759 endpoints = {}
1769 for task in range(len(self.rqdata.runq_fnid)): 1760 for task in range(len(self.rqdata.runq_fnid)):
@@ -1894,7 +1885,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1894 1885
1895 for task in range(len(self.sq_revdeps)): 1886 for task in range(len(self.sq_revdeps)):
1896 if len(self.sq_revdeps[task]) == 0: 1887 if len(self.sq_revdeps[task]) == 0:
1897 self.runq_buildable[task] = 1 1888 self.runq_buildable.add(task)
1898 1889
1899 self.outrightfail = [] 1890 self.outrightfail = []
1900 if self.rq.hashvalidate: 1891 if self.rq.hashvalidate:
@@ -1966,7 +1957,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1966 continue 1957 continue
1967 self.sq_revdeps2[dep].remove(task) 1958 self.sq_revdeps2[dep].remove(task)
1968 if len(self.sq_revdeps2[dep]) == 0: 1959 if len(self.sq_revdeps2[dep]) == 0:
1969 self.runq_buildable[dep] = 1 1960 self.runq_buildable.add(dep)
1970 1961
1971 def task_completeoutright(self, task): 1962 def task_completeoutright(self, task):
1972 """ 1963 """
@@ -1994,8 +1985,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1994 self.scenequeue_updatecounters(task, True) 1985 self.scenequeue_updatecounters(task, True)
1995 1986
1996 def task_failoutright(self, task): 1987 def task_failoutright(self, task):
1997 self.runq_running[task] = 1 1988 self.runq_running.add(task)
1998 self.runq_buildable[task] = 1 1989 self.runq_buildable.add(task)
1999 self.stats.taskCompleted() 1990 self.stats.taskCompleted()
2000 self.stats.taskSkipped() 1991 self.stats.taskSkipped()
2001 index = self.rqdata.runq_setscene[task] 1992 index = self.rqdata.runq_setscene[task]
@@ -2003,8 +1994,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
2003 self.scenequeue_updatecounters(task, True) 1994 self.scenequeue_updatecounters(task, True)
2004 1995
2005 def task_skip(self, task): 1996 def task_skip(self, task):
2006 self.runq_running[task] = 1 1997 self.runq_running.add(task)
2007 self.runq_buildable[task] = 1 1998 self.runq_buildable.add(task)
2008 self.task_completeoutright(task) 1999 self.task_completeoutright(task)
2009 self.stats.taskCompleted() 2000 self.stats.taskCompleted()
2010 self.stats.taskSkipped() 2001 self.stats.taskSkipped()
@@ -2020,7 +2011,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
2020 if self.stats.active < self.number_tasks: 2011 if self.stats.active < self.number_tasks:
2021 # Find the next setscene to run 2012 # Find the next setscene to run
2022 for nexttask in range(self.stats.total): 2013 for nexttask in range(self.stats.total):
2023 if self.runq_buildable[nexttask] == 1 and self.runq_running[nexttask] != 1: 2014 if nexttask in self.runq_buildable and nexttask not in self.runq_running:
2024 if nexttask in self.unskippable: 2015 if nexttask in self.unskippable:
2025 logger.debug(2, "Setscene task %s is unskippable" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask])) 2016 logger.debug(2, "Setscene task %s is unskippable" % self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask]))
2026 if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True): 2017 if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask, self.sq_revdeps[nexttask], True):
@@ -2077,7 +2068,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
2077 self.rq.worker.stdin.write(b"<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>") 2068 self.rq.worker.stdin.write(b"<runtask>" + pickle.dumps((fn, realtask, taskname, True, self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>")
2078 self.rq.worker.stdin.flush() 2069 self.rq.worker.stdin.flush()
2079 2070
2080 self.runq_running[task] = 1 2071 self.runq_running.add(task)
2081 self.stats.taskActive() 2072 self.stats.taskActive()
2082 if self.stats.active < self.number_tasks: 2073 if self.stats.active < self.number_tasks:
2083 return True 2074 return True
@@ -2087,8 +2078,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
2087 return self.rq.active_fds() 2078 return self.rq.active_fds()
2088 2079
2089 #for task in range(self.stats.total): 2080 #for task in range(self.stats.total):
2090 # if self.runq_running[task] != 1: 2081 # if task not in self.runq_running:
2091 # buildable = self.runq_buildable[task] 2082 # buildable = task in self.runq_buildable
2092 # revdeps = self.sq_revdeps[task] 2083 # revdeps = self.sq_revdeps[task]
2093 # bb.warn("Found we didn't run %s %s %s %s" % (task, buildable, str(revdeps), self.rqdata.get_user_idstring(self.rqdata.runq_setscene[task]))) 2084 # bb.warn("Found we didn't run %s %s %s %s" % (task, buildable, str(revdeps), self.rqdata.get_user_idstring(self.rqdata.runq_setscene[task])))
2094 2085