summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/runqueue.py
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2016-10-07 08:26:48 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2016-10-09 12:33:26 +0100
commit77e56194dc8a81509b1977e1f329f91ce128a9c0 (patch)
treec38550c7106a48ff73c3df6c45422a657492d5d3 /bitbake/lib/bb/runqueue.py
parent146eb22b0e7b77bf3b3dcd750b4124b6cb27f090 (diff)
downloadpoky-77e56194dc8a81509b1977e1f329f91ce128a9c0.tar.gz
bitbake: runqueue: Optimise task id string manipulations
Some task id manipulations were suboptimal: * taskfn_fromtid and fn_from_tid were effectively the same function * many calls to split_tid(), then taskfn_fromtid() * taskfn_fromtid() called split_tid() internally This patch adds split_tid_mcfn() to replace split_tid() and returns the "taskfn" variant being used in many places. We update all core calls to the new function and ignore the return values we don't need since the function call overhead of the split_tid wrapper is higher than ignoring a return value. The one remaining standalone use of taskfn_fromtid is replaced with fn_from_tid. I couldn't see any external usage so it was dropped. There is external usage of split_tid so a wrapper remains for it. Combined together these changes should improve some of the runqueue task manipulation performance. (Bitbake rev: 1bf2ef874fbe47f1320007efa0bdeef8d630b8a1) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r--bitbake/lib/bb/runqueue.py79
1 files changed, 33 insertions, 46 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index df7c50fe96..13062630c0 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -49,30 +49,30 @@ def taskname_from_tid(tid):
49 return tid.rsplit(":", 1)[1] 49 return tid.rsplit(":", 1)[1]
50 50
51def split_tid(tid): 51def split_tid(tid):
52 (mc, fn, taskname, _) = split_tid_mcfn(tid)
53 return (mc, fn, taskname)
54
55def split_tid_mcfn(tid):
52 if tid.startswith('multiconfig:'): 56 if tid.startswith('multiconfig:'):
53 elems = tid.split(':') 57 elems = tid.split(':')
54 mc = elems[1] 58 mc = elems[1]
55 fn = ":".join(elems[2:-1]) 59 fn = ":".join(elems[2:-1])
56 taskname = elems[-1] 60 taskname = elems[-1]
61 mcfn = "multiconfig:" + mc + ":" + fn
57 else: 62 else:
58 tid = tid.rsplit(":", 1) 63 tid = tid.rsplit(":", 1)
59 mc = "" 64 mc = ""
60 fn = tid[0] 65 fn = tid[0]
61 taskname = tid[1] 66 taskname = tid[1]
67 mcfn = fn
62 68
63 return (mc, fn, taskname) 69 return (mc, fn, taskname, mcfn)
64 70
65def build_tid(mc, fn, taskname): 71def build_tid(mc, fn, taskname):
66 if mc: 72 if mc:
67 return "multiconfig:" + mc + ":" + fn + ":" + taskname 73 return "multiconfig:" + mc + ":" + fn + ":" + taskname
68 return fn + ":" + taskname 74 return fn + ":" + taskname
69 75
70def taskfn_fromtid(tid):
71 (mc, fn, taskname) = split_tid(tid)
72 if mc:
73 return "multiconfig:" + mc + ":" + fn
74 return fn
75
76class RunQueueStats: 76class RunQueueStats:
77 """ 77 """
78 Holds statistics on the tasks handled by the associated runQueue 78 Holds statistics on the tasks handled by the associated runQueue
@@ -135,8 +135,7 @@ class RunQueueScheduler(object):
135 self.buildable = [] 135 self.buildable = []
136 self.stamps = {} 136 self.stamps = {}
137 for tid in self.rqdata.runtaskentries: 137 for tid in self.rqdata.runtaskentries:
138 (mc, fn, taskname) = split_tid(tid) 138 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
139 taskfn = taskfn_fromtid(tid)
140 self.stamps[tid] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True) 139 self.stamps[tid] = bb.build.stampfile(taskname, self.rqdata.dataCaches[mc], taskfn, noextra=True)
141 if tid in self.rq.runq_buildable: 140 if tid in self.rq.runq_buildable:
142 self.buildable.append(tid) 141 self.buildable.append(tid)
@@ -289,7 +288,7 @@ class RunQueueData:
289 return tid + task_name_suffix 288 return tid + task_name_suffix
290 289
291 def get_short_user_idstring(self, task, task_name_suffix = ""): 290 def get_short_user_idstring(self, task, task_name_suffix = ""):
292 (mc, fn, taskname) = split_tid(task) 291 (mc, fn, taskname, _) = split_tid_mcfn(task)
293 pn = self.dataCaches[mc].pkg_fn[fn] 292 pn = self.dataCaches[mc].pkg_fn[fn]
294 taskname = taskname_from_tid(task) + task_name_suffix 293 taskname = taskname_from_tid(task) + task_name_suffix
295 return "%s:%s" % (pn, taskname) 294 return "%s:%s" % (pn, taskname)
@@ -511,9 +510,8 @@ class RunQueueData:
511 for mc in taskData: 510 for mc in taskData:
512 for tid in taskData[mc].taskentries: 511 for tid in taskData[mc].taskentries:
513 512
514 (mc, fn, taskname) = split_tid(tid) 513 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
515 #runtid = build_tid(mc, fn, taskname) 514 #runtid = build_tid(mc, fn, taskname)
516 taskfn = taskfn_fromtid(tid)
517 515
518 #logger.debug(2, "Processing %s,%s:%s", mc, fn, taskname) 516 #logger.debug(2, "Processing %s,%s:%s", mc, fn, taskname)
519 517
@@ -529,7 +527,7 @@ class RunQueueData:
529 # 527 #
530 # e.g. addtask before X after Y 528 # e.g. addtask before X after Y
531 for t in taskData[mc].taskentries[tid].tdepends: 529 for t in taskData[mc].taskentries[tid].tdepends:
532 (_, depfn, deptaskname) = split_tid(t) 530 (_, depfn, deptaskname, _) = split_tid_mcfn(t)
533 depends.add(build_tid(mc, depfn, deptaskname)) 531 depends.add(build_tid(mc, depfn, deptaskname))
534 532
535 # Resolve 'deptask' dependencies 533 # Resolve 'deptask' dependencies
@@ -611,7 +609,7 @@ class RunQueueData:
611 609
612 def generate_recdeps(t): 610 def generate_recdeps(t):
613 newdeps = set() 611 newdeps = set()
614 (mc, fn, taskname) = split_tid(t) 612 (mc, fn, taskname, _) = split_tid_mcfn(t)
615 add_resolved_dependencies(mc, fn, tasknames, newdeps) 613 add_resolved_dependencies(mc, fn, tasknames, newdeps)
616 extradeps[tid].update(newdeps) 614 extradeps[tid].update(newdeps)
617 seendeps.add(t) 615 seendeps.add(t)
@@ -774,8 +772,7 @@ class RunQueueData:
774 prov_list = {} 772 prov_list = {}
775 seen_fn = [] 773 seen_fn = []
776 for tid in self.runtaskentries: 774 for tid in self.runtaskentries:
777 (tidmc, fn, taskname) = split_tid(tid) 775 (tidmc, fn, taskname, taskfn) = split_tid_mcfn(tid)
778 taskfn = taskfn_fromtid(tid)
779 if taskfn in seen_fn: 776 if taskfn in seen_fn:
780 continue 777 continue
781 if mc != tidmc: 778 if mc != tidmc:
@@ -885,14 +882,14 @@ class RunQueueData:
885 self.runq_setscene_tids = [] 882 self.runq_setscene_tids = []
886 if not self.cooker.configuration.nosetscene: 883 if not self.cooker.configuration.nosetscene:
887 for tid in self.runtaskentries: 884 for tid in self.runtaskentries:
888 (mc, fn, taskname) = split_tid(tid) 885 (mc, fn, taskname, _) = split_tid_mcfn(tid)
889 setscenetid = fn + ":" + taskname + "_setscene" 886 setscenetid = fn + ":" + taskname + "_setscene"
890 if setscenetid not in taskData[mc].taskentries: 887 if setscenetid not in taskData[mc].taskentries:
891 continue 888 continue
892 self.runq_setscene_tids.append(tid) 889 self.runq_setscene_tids.append(tid)
893 890
894 def invalidate_task(tid, error_nostamp): 891 def invalidate_task(tid, error_nostamp):
895 (mc, fn, taskname) = split_tid(tid) 892 (mc, fn, taskname, _) = split_tid_mcfn(tid)
896 taskdep = self.dataCaches[mc].task_deps[fn] 893 taskdep = self.dataCaches[mc].task_deps[fn]
897 if fn + ":" + taskname not in taskData[mc].taskentries: 894 if fn + ":" + taskname not in taskData[mc].taskentries:
898 logger.warning("Task %s does not exist, invalidating this task will have no effect" % taskname) 895 logger.warning("Task %s does not exist, invalidating this task will have no effect" % taskname)
@@ -946,8 +943,7 @@ class RunQueueData:
946 procdep = [] 943 procdep = []
947 for dep in self.runtaskentries[tid].depends: 944 for dep in self.runtaskentries[tid].depends:
948 procdep.append(fn_from_tid(dep) + "." + taskname_from_tid(dep)) 945 procdep.append(fn_from_tid(dep) + "." + taskname_from_tid(dep))
949 (mc, fn, taskname) = split_tid(tid) 946 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
950 taskfn = taskfn_fromtid(tid)
951 self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(taskfn, taskname, procdep, self.dataCaches[mc]) 947 self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(taskfn, taskname, procdep, self.dataCaches[mc])
952 task = self.runtaskentries[tid].task 948 task = self.runtaskentries[tid].task
953 949
@@ -1099,8 +1095,7 @@ class RunQueue:
1099 except: 1095 except:
1100 return None 1096 return None
1101 1097
1102 (mc, fn, tn) = split_tid(tid) 1098 (mc, fn, tn, taskfn) = split_tid_mcfn(tid)
1103 taskfn = taskfn_fromtid(tid)
1104 if taskname is None: 1099 if taskname is None:
1105 taskname = tn 1100 taskname = tn
1106 1101
@@ -1134,8 +1129,7 @@ class RunQueue:
1134 t1 = get_timestamp(stampfile) 1129 t1 = get_timestamp(stampfile)
1135 for dep in self.rqdata.runtaskentries[tid].depends: 1130 for dep in self.rqdata.runtaskentries[tid].depends:
1136 if iscurrent: 1131 if iscurrent:
1137 (mc2, fn2, taskname2) = split_tid(dep) 1132 (mc2, fn2, taskname2, taskfn2) = split_tid_mcfn(dep)
1138 taskfn2 = taskfn_fromtid(dep)
1139 stampfile2 = bb.build.stampfile(taskname2, self.rqdata.dataCaches[mc2], taskfn2) 1133 stampfile2 = bb.build.stampfile(taskname2, self.rqdata.dataCaches[mc2], taskfn2)
1140 stampfile3 = bb.build.stampfile(taskname2 + "_setscene", self.rqdata.dataCaches[mc2], taskfn2) 1134 stampfile3 = bb.build.stampfile(taskname2 + "_setscene", self.rqdata.dataCaches[mc2], taskfn2)
1141 t2 = get_timestamp(stampfile2) 1135 t2 = get_timestamp(stampfile2)
@@ -1247,7 +1241,7 @@ class RunQueue:
1247 if not self.rqdata.taskData[''].tryaltconfigs: 1241 if not self.rqdata.taskData[''].tryaltconfigs:
1248 raise bb.runqueue.TaskFailure(self.rqexe.failed_tids) 1242 raise bb.runqueue.TaskFailure(self.rqexe.failed_tids)
1249 for tid in self.rqexe.failed_tids: 1243 for tid in self.rqexe.failed_tids:
1250 (mc, fn, tn) = split_tid(tid) 1244 (mc, fn, tn, _) = split_tid_mcfn(tid)
1251 self.rqdata.taskData[mc].fail_fn(fn) 1245 self.rqdata.taskData[mc].fail_fn(fn)
1252 self.rqdata.reset() 1246 self.rqdata.reset()
1253 1247
@@ -1297,7 +1291,7 @@ class RunQueue:
1297 bb.note("Reparsing files to collect dependency data") 1291 bb.note("Reparsing files to collect dependency data")
1298 bb_cache = bb.cache.NoCache(self.cooker.databuilder) 1292 bb_cache = bb.cache.NoCache(self.cooker.databuilder)
1299 for tid in self.rqdata.runtaskentries: 1293 for tid in self.rqdata.runtaskentries:
1300 fn = taskfn_fromtid(tid) 1294 fn = fn_from_tid(tid)
1301 if fn not in done: 1295 if fn not in done:
1302 the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn)) 1296 the_data = bb_cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn))
1303 done.add(fn) 1297 done.add(fn)
@@ -1319,8 +1313,7 @@ class RunQueue:
1319 valid_new = set() 1313 valid_new = set()
1320 1314
1321 for tid in self.rqdata.runtaskentries: 1315 for tid in self.rqdata.runtaskentries:
1322 (mc, fn, taskname) = split_tid(tid) 1316 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
1323 taskfn = taskfn_fromtid(tid)
1324 taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] 1317 taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
1325 1318
1326 if 'noexec' in taskdep and taskname in taskdep['noexec']: 1319 if 'noexec' in taskdep and taskname in taskdep['noexec']:
@@ -1408,7 +1401,7 @@ class RunQueue:
1408 1401
1409 1402
1410 for tid in invalidtasks: 1403 for tid in invalidtasks:
1411 (mc, fn, taskname) = split_tid(tid) 1404 (mc, fn, taskname, _) = split_tid_mcfn(tid)
1412 pn = self.rqdata.dataCaches[mc].pkg_fn[fn] 1405 pn = self.rqdata.dataCaches[mc].pkg_fn[fn]
1413 h = self.rqdata.runtaskentries[tid].hash 1406 h = self.rqdata.runtaskentries[tid].hash
1414 matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData) 1407 matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData)
@@ -1512,7 +1505,7 @@ class RunQueueExecute:
1512 taskdata = {} 1505 taskdata = {}
1513 taskdeps.add(task) 1506 taskdeps.add(task)
1514 for dep in taskdeps: 1507 for dep in taskdeps:
1515 (mc, fn, taskname) = split_tid(dep) 1508 (mc, fn, taskname, _) = split_tid_mcfn(dep)
1516 pn = self.rqdata.dataCaches[mc].pkg_fn[fn] 1509 pn = self.rqdata.dataCaches[mc].pkg_fn[fn]
1517 taskdata[dep] = [pn, taskname, fn] 1510 taskdata[dep] = [pn, taskname, fn]
1518 call = self.rq.depvalidate + "(task, taskdata, notneeded, d)" 1511 call = self.rq.depvalidate + "(task, taskdata, notneeded, d)"
@@ -1569,8 +1562,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1569 tasknames = {} 1562 tasknames = {}
1570 fns = {} 1563 fns = {}
1571 for tid in self.rqdata.runtaskentries: 1564 for tid in self.rqdata.runtaskentries:
1572 (mc, fn, taskname) = split_tid(tid) 1565 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
1573 taskfn = taskfn_fromtid(tid)
1574 taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn] 1566 taskdep = self.rqdata.dataCaches[mc].task_deps[taskfn]
1575 fns[tid] = taskfn 1567 fns[tid] = taskfn
1576 tasknames[tid] = taskname 1568 tasknames[tid] = taskname
@@ -1589,9 +1581,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
1589 covered_remove = bb.utils.better_eval(call, locs) 1581 covered_remove = bb.utils.better_eval(call, locs)
1590 1582
1591 def removecoveredtask(tid): 1583 def removecoveredtask(tid):
1592 (mc, fn, taskname) = split_tid(tid) 1584 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
1593 taskname = taskname + '_setscene' 1585 taskname = taskname + '_setscene'
1594 taskfn = taskfn_fromtid(tid)
1595 bb.build.del_stamp(taskname, self.rqdata.dataCaches[mc], taskfn) 1586 bb.build.del_stamp(taskname, self.rqdata.dataCaches[mc], taskfn)
1596 self.rq.scenequeue_covered.remove(tid) 1587 self.rq.scenequeue_covered.remove(tid)
1597 1588
@@ -1617,7 +1608,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1617 for mc in self.rqdata.dataCaches: 1608 for mc in self.rqdata.dataCaches:
1618 target_pairs = [] 1609 target_pairs = []
1619 for tid in self.rqdata.target_tids: 1610 for tid in self.rqdata.target_tids:
1620 (tidmc, fn, taskname) = split_tid(tid) 1611 (tidmc, fn, taskname, _) = split_tid_mcfn(tid)
1621 if tidmc == mc: 1612 if tidmc == mc:
1622 target_pairs.append((fn, taskname)) 1613 target_pairs.append((fn, taskname))
1623 1614
@@ -1713,7 +1704,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1713 if self.rqdata.setscenewhitelist: 1704 if self.rqdata.setscenewhitelist:
1714 # Check tasks that are going to run against the whitelist 1705 # Check tasks that are going to run against the whitelist
1715 def check_norun_task(tid, showerror=False): 1706 def check_norun_task(tid, showerror=False):
1716 (mc, fn, taskname) = split_tid(tid) 1707 (mc, fn, taskname, _) = split_tid_mcfn(tid)
1717 # Ignore covered tasks 1708 # Ignore covered tasks
1718 if tid in self.rq.scenequeue_covered: 1709 if tid in self.rq.scenequeue_covered:
1719 return False 1710 return False
@@ -1761,8 +1752,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1761 1752
1762 task = self.sched.next() 1753 task = self.sched.next()
1763 if task is not None: 1754 if task is not None:
1764 (mc, fn, taskname) = split_tid(task) 1755 (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
1765 taskfn = taskfn_fromtid(task)
1766 1756
1767 if task in self.rq.scenequeue_covered: 1757 if task in self.rq.scenequeue_covered:
1768 logger.debug(2, "Setscene covered task %s", task) 1758 logger.debug(2, "Setscene covered task %s", task)
@@ -1842,8 +1832,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
1842 while next: 1832 while next:
1843 additional = [] 1833 additional = []
1844 for revdep in next: 1834 for revdep in next:
1845 (mc, fn, taskname) = split_tid(revdep) 1835 (mc, fn, taskname, taskfn) = split_tid_mcfn(revdep)
1846 taskfn = taskfn_fromtid(revdep)
1847 pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn] 1836 pn = self.rqdata.dataCaches[mc].pkg_fn[taskfn]
1848 deps = self.rqdata.runtaskentries[revdep].depends 1837 deps = self.rqdata.runtaskentries[revdep].depends
1849 provides = self.rqdata.dataCaches[mc].fn_provides[taskfn] 1838 provides = self.rqdata.dataCaches[mc].fn_provides[taskfn]
@@ -1986,7 +1975,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
1986 # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene" 1975 # e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene"
1987 # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies 1976 # Note that anything explicitly depended upon will have its reverse dependencies removed to avoid circular dependencies
1988 for tid in self.rqdata.runq_setscene_tids: 1977 for tid in self.rqdata.runq_setscene_tids:
1989 (mc, fn, taskname) = split_tid(tid) 1978 (mc, fn, taskname, _) = split_tid_mcfn(tid)
1990 realtid = fn + ":" + taskname + "_setscene" 1979 realtid = fn + ":" + taskname + "_setscene"
1991 idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends 1980 idepends = self.rqdata.taskData[mc].taskentries[realtid].idepends
1992 for (depname, idependtask) in idepends: 1981 for (depname, idependtask) in idepends:
@@ -2050,8 +2039,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
2050 noexec = [] 2039 noexec = []
2051 stamppresent = [] 2040 stamppresent = []
2052 for tid in self.sq_revdeps: 2041 for tid in self.sq_revdeps:
2053 (mc, fn, taskname) = split_tid(tid) 2042 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
2054 taskfn = taskfn_fromtid(tid)
2055 2043
2056 taskdep = self.rqdata.dataCaches[mc].task_deps[fn] 2044 taskdep = self.rqdata.dataCaches[mc].task_deps[fn]
2057 2045
@@ -2122,7 +2110,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
2122 def check_taskfail(self, task): 2110 def check_taskfail(self, task):
2123 if self.rqdata.setscenewhitelist: 2111 if self.rqdata.setscenewhitelist:
2124 realtask = task.split('_setscene')[0] 2112 realtask = task.split('_setscene')[0]
2125 (mc, fn, taskname) = split_tid(realtask) 2113 (mc, fn, taskname, _) = split_tid_mcfn(realtask)
2126 pn = self.rqdata.dataCaches[mc].pkg_fn[fn] 2114 pn = self.rqdata.dataCaches[mc].pkg_fn[fn]
2127 if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist): 2115 if not check_setscene_enforce_whitelist(pn, taskname, self.rqdata.setscenewhitelist):
2128 logger.error('Task %s.%s failed' % (pn, taskname + "_setscene")) 2116 logger.error('Task %s.%s failed' % (pn, taskname + "_setscene"))
@@ -2186,8 +2174,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
2186 task = nexttask 2174 task = nexttask
2187 break 2175 break
2188 if task is not None: 2176 if task is not None:
2189 (mc, fn, taskname) = split_tid(task) 2177 (mc, fn, taskname, taskfn) = split_tid_mcfn(task)
2190 taskfn = taskfn_fromtid(task)
2191 taskname = taskname + "_setscene" 2178 taskname = taskname + "_setscene"
2192 if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache): 2179 if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True, cache=self.stampcache):
2193 logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task) 2180 logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant', task)