summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/runqueue.py
diff options
context:
space:
mode:
authorMark Hatle <mark.hatle@windriver.com>2018-05-14 10:21:39 -0400
committerRichard Purdie <richard.purdie@linuxfoundation.org>2018-06-15 17:57:30 +0100
commita0cb1021c04c37b972bbf3d3626c7eef8a9876d1 (patch)
treed3e2a4177c6402a9db46f81e879d2abde3362721 /bitbake/lib/bb/runqueue.py
parent27101e647f6323a783a17391b57fe4145fa5dec5 (diff)
downloadpoky-a0cb1021c04c37b972bbf3d3626c7eef8a9876d1.tar.gz
bitbake: runqueue.py: Initial implementation of per task process limits
On high core machines, in do_fetch, it is possible to DDoS your own machine. A method to limit any arbitrary task type to a certain number of simultaneous threads is needed. (Similar to how BB_NUMBER_THREADS works in the general case.) The format of this new limitation is: do_fetch[number_threads] = "2" This should be set globally. If it is set in individual recipes it could result in unpredictable behavior. Note: a value for number_threads > BB_NUMBER_THREADS will have no effect. (Bitbake rev: 055865047c63b9c3b213b47a1884924ce0adeda0) Signed-off-by: Mark Hatle <mark.hatle@windriver.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb/runqueue.py')
-rw-r--r--bitbake/lib/bb/runqueue.py21
1 files changed, 21 insertions, 0 deletions
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index a937a0ba76..2d9e18d888 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -134,6 +134,7 @@ class RunQueueScheduler(object):
134 self.prio_map = [self.rqdata.runtaskentries.keys()] 134 self.prio_map = [self.rqdata.runtaskentries.keys()]
135 135
136 self.buildable = [] 136 self.buildable = []
137 self.skip_maxthread = {}
137 self.stamps = {} 138 self.stamps = {}
138 for tid in self.rqdata.runtaskentries: 139 for tid in self.rqdata.runtaskentries:
139 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid) 140 (mc, fn, taskname, taskfn) = split_tid_mcfn(tid)
@@ -150,8 +151,25 @@ class RunQueueScheduler(object):
150 self.buildable = [x for x in self.buildable if x not in self.rq.runq_running] 151 self.buildable = [x for x in self.buildable if x not in self.rq.runq_running]
151 if not self.buildable: 152 if not self.buildable:
152 return None 153 return None
154
155 # Filter out tasks that have a max number of threads that have been exceeded
156 skip_buildable = {}
157 for running in self.rq.runq_running.difference(self.rq.runq_complete):
158 rtaskname = taskname_from_tid(running)
159 if rtaskname not in self.skip_maxthread:
160 self.skip_maxthread[rtaskname] = self.rq.cfgData.getVarFlag(rtaskname, "number_threads")
161 if not self.skip_maxthread[rtaskname]:
162 continue
163 if rtaskname in skip_buildable:
164 skip_buildable[rtaskname] += 1
165 else:
166 skip_buildable[rtaskname] = 1
167
153 if len(self.buildable) == 1: 168 if len(self.buildable) == 1:
154 tid = self.buildable[0] 169 tid = self.buildable[0]
170 taskname = taskname_from_tid(tid)
171 if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
172 return None
155 stamp = self.stamps[tid] 173 stamp = self.stamps[tid]
156 if stamp not in self.rq.build_stamps.values(): 174 if stamp not in self.rq.build_stamps.values():
157 return tid 175 return tid
@@ -164,6 +182,9 @@ class RunQueueScheduler(object):
164 best = None 182 best = None
165 bestprio = None 183 bestprio = None
166 for tid in self.buildable: 184 for tid in self.buildable:
185 taskname = taskname_from_tid(tid)
186 if taskname in skip_buildable and skip_buildable[taskname] >= int(self.skip_maxthread[taskname]):
187 continue
167 prio = self.rev_prio_map[tid] 188 prio = self.rev_prio_map[tid]
168 if bestprio is None or bestprio > prio: 189 if bestprio is None or bestprio > prio:
169 stamp = self.stamps[tid] 190 stamp = self.stamps[tid]