summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul Eggleton <paul.eggleton@linux.intel.com>2012-10-11 18:43:51 (GMT)
committerRichard Purdie <richard.purdie@linuxfoundation.org>2012-10-12 07:58:34 (GMT)
commit732007cbb627f648ce536554c28471f9acd58e9e (patch)
tree1442a4dfc16f3ba9ba4d5acef199ba24853ee277
parent9945923c8bc2479a72a913ee718ca823c69360b3 (diff)
downloadpoky-732007cbb627f648ce536554c28471f9acd58e9e.tar.gz
bitbake: bitbake: ensure -f causes dependent tasks to be re-run
If -f is specified, force dependent tasks to be re-run next time. This works by changing the force behaviour so that instead of deleting the task's stamp, we write a "taint" file into the stamps directory, which will alter the taskhash randomly and thus trigger the task to re-run next time we evaluate whether or not that should be done as well as influencing the taskhashes of any dependent tasks so that they are similarly re-triggered. As a bonus because we write this file as <stamp file name>.taskname.taint, the existing code which deletes the stamp files in OE's do_clean will already handle removing it. This means you can now do the following: bitbake somepackage [ change the source code in the package's WORKDIR ] bitbake -c compile -f somepackage bitbake somepackage and the result will be that all of the tasks that depend on do_compile (do_install, do_package, etc.) will be re-run in the last step. Note that to operate in the manner described above you need full hashing enabled (i.e. BB_SIGNATURE_HANDLER must be set to a signature handler that inherits from BasicHash). If this is not the case, -f will just delete the stamp for the specified task as it did before. This fix is required for [YOCTO #2615] and [YOCTO #2256]. (Bitbake rev: f7b55a94226f9acd985f87946e26d01bd86a35bb) Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--bitbake/lib/bb/build.py18
-rw-r--r--bitbake/lib/bb/cooker.py6
-rw-r--r--bitbake/lib/bb/runqueue.py12
-rw-r--r--bitbake/lib/bb/siggen.py34
4 files changed, 61 insertions, 9 deletions
diff --git a/bitbake/lib/bb/build.py b/bitbake/lib/bb/build.py
index 95f1dcf..f912a6b 100644
--- a/bitbake/lib/bb/build.py
+++ b/bitbake/lib/bb/build.py
@@ -458,6 +458,24 @@ def del_stamp(task, d, file_name = None):
458 stamp = stamp_internal(task, d, file_name) 458 stamp = stamp_internal(task, d, file_name)
459 bb.utils.remove(stamp) 459 bb.utils.remove(stamp)
460 460
461def write_taint(task, d, file_name = None):
462 """
463 Creates a "taint" file which will force the specified task and its
464 dependents to be re-run the next time by influencing the value of its
465 taskhash.
466 (d can be a data dict or dataCache)
467 """
468 import uuid
469 if file_name:
470 taintfn = d.stamp[file_name] + '.' + task + '.taint'
471 else:
472 taintfn = d.getVar('STAMP', True) + '.' + task + '.taint'
473 bb.utils.mkdirhier(os.path.dirname(taintfn))
474 # The specific content of the taint file is not really important,
475 # we just need it to be random, so a random UUID is used
476 with open(taintfn, 'w') as taintf:
477 taintf.write(str(uuid.uuid4()))
478
461def stampfile(taskname, d, file_name = None): 479def stampfile(taskname, d, file_name = None):
462 """ 480 """
463 Return the stamp for a given task 481 Return the stamp for a given task
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index 4016f3b..745f591 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -1066,10 +1066,10 @@ class BBCooker:
1066 self.status.rundeps[fn] = [] 1066 self.status.rundeps[fn] = []
1067 self.status.runrecs[fn] = [] 1067 self.status.runrecs[fn] = []
1068 1068
1069 # Remove stamp for target if force mode active 1069 # Invalidate task for target if force mode active
1070 if self.configuration.force: 1070 if self.configuration.force:
1071 logger.verbose("Remove stamp %s, %s", task, fn) 1071 logger.verbose("Invalidate task %s, %s", task, fn)
1072 bb.build.del_stamp('do_%s' % task, self.status, fn) 1072 bb.parse.siggen.invalidate_task('do_%s' % task, self.status, fn)
1073 1073
1074 # Setup taskdata structure 1074 # Setup taskdata structure
1075 taskdata = bb.taskdata.TaskData(self.configuration.abort) 1075 taskdata = bb.taskdata.TaskData(self.configuration.abort)
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index b870caf..c1e9e8c 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -705,6 +705,12 @@ class RunQueueData:
705 continue 705 continue
706 self.runq_setscene.append(task) 706 self.runq_setscene.append(task)
707 707
708 # Invalidate task if force mode active
709 if self.cooker.configuration.force:
710 for (fn, target) in self.target_pairs:
711 logger.verbose("Invalidate task %s, %s", target, fn)
712 bb.parse.siggen.invalidate_task(target, self.dataCache, fn)
713
708 # Interate over the task list and call into the siggen code 714 # Interate over the task list and call into the siggen code
709 dealtwith = set() 715 dealtwith = set()
710 todeal = set(range(len(self.runq_fnid))) 716 todeal = set(range(len(self.runq_fnid)))
@@ -731,12 +737,6 @@ class RunQueueData:
731 deps.append(depidentifier) 737 deps.append(depidentifier)
732 self.hash_deps[identifier] = deps 738 self.hash_deps[identifier] = deps
733 739
734 # Remove stamps for targets if force mode active
735 if self.cooker.configuration.force:
736 for (fn, target) in self.target_pairs:
737 logger.verbose("Remove stamp %s, %s", target, fn)
738 bb.build.del_stamp(target, self.dataCache, fn)
739
740 return len(self.runq_fnid) 740 return len(self.runq_fnid)
741 741
742 def dump_data(self, taskQueue): 742 def dump_data(self, taskQueue):
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 8c79b17..4a0af94 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -50,6 +50,10 @@ class SignatureGenerator(object):
50 def dump_sigtask(self, fn, task, stampbase, runtime): 50 def dump_sigtask(self, fn, task, stampbase, runtime):
51 return 51 return
52 52
53 def invalidate_task(self, task, d, fn):
54 bb.build.del_stamp(task, d, fn)
55
56
53class SignatureGeneratorBasic(SignatureGenerator): 57class SignatureGeneratorBasic(SignatureGenerator):
54 """ 58 """
55 """ 59 """
@@ -148,6 +152,15 @@ class SignatureGeneratorBasic(SignatureGenerator):
148 return False 152 return False
149 return True 153 return True
150 154
155 def read_taint(self, fn, task, stampbase):
156 taint = None
157 try:
158 with open(stampbase + '.' + task + '.taint', 'r') as taintf:
159 taint = taintf.read()
160 except IOError:
161 pass
162 return taint
163
151 def get_taskhash(self, fn, task, deps, dataCache): 164 def get_taskhash(self, fn, task, deps, dataCache):
152 k = fn + "." + task 165 k = fn + "." + task
153 data = dataCache.basetaskhash[k] 166 data = dataCache.basetaskhash[k]
@@ -161,6 +174,11 @@ class SignatureGeneratorBasic(SignatureGenerator):
161 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep) 174 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep)
162 data = data + self.taskhash[dep] 175 data = data + self.taskhash[dep]
163 self.runtaskdeps[k].append(dep) 176 self.runtaskdeps[k].append(dep)
177
178 taint = self.read_taint(fn, task, dataCache.stamp[fn])
179 if taint:
180 data = data + taint
181
164 h = hashlib.md5(data).hexdigest() 182 h = hashlib.md5(data).hexdigest()
165 self.taskhash[k] = h 183 self.taskhash[k] = h
166 #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) 184 #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
@@ -201,9 +219,14 @@ class SignatureGeneratorBasic(SignatureGenerator):
201 for dep in data['runtaskdeps']: 219 for dep in data['runtaskdeps']:
202 data['runtaskhashes'][dep] = self.taskhash[dep] 220 data['runtaskhashes'][dep] = self.taskhash[dep]
203 221
222 taint = self.read_taint(fn, task, stampbase)
223 if taint:
224 data['taint'] = taint
225
204 p = pickle.Pickler(file(sigfile, "wb"), -1) 226 p = pickle.Pickler(file(sigfile, "wb"), -1)
205 p.dump(data) 227 p.dump(data)
206 228
229
207 def dump_sigs(self, dataCache): 230 def dump_sigs(self, dataCache):
208 for fn in self.taskdeps: 231 for fn in self.taskdeps:
209 for task in self.taskdeps[fn]: 232 for task in self.taskdeps[fn]:
@@ -230,6 +253,9 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
230 h = self.basehash[k] 253 h = self.basehash[k]
231 return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.') 254 return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.')
232 255
256 def invalidate_task(self, task, d, fn):
257 bb.build.write_taint(task, d, fn)
258
233def dump_this_task(outfile, d): 259def dump_this_task(outfile, d):
234 import bb.parse 260 import bb.parse
235 fn = d.getVar("BB_FILENAME", True) 261 fn = d.getVar("BB_FILENAME", True)
@@ -330,6 +356,11 @@ def compare_sigfiles(a, b):
330 for dep in changed: 356 for dep in changed:
331 print "Hash for dependent task %s changed from %s to %s" % (dep, a[dep], b[dep]) 357 print "Hash for dependent task %s changed from %s to %s" % (dep, a[dep], b[dep])
332 358
359 a_taint = a_data.get('taint', None)
360 b_taint = b_data.get('taint', None)
361 if a_taint != b_taint:
362 print "Taint (by forced/invalidated task) changed from %s to %s" % (a_taint, b_taint)
363
333def dump_sigfile(a): 364def dump_sigfile(a):
334 p1 = pickle.Unpickler(file(a, "rb")) 365 p1 = pickle.Unpickler(file(a, "rb"))
335 a_data = p1.load() 366 a_data = p1.load()
@@ -354,3 +385,6 @@ def dump_sigfile(a):
354 if 'runtaskhashes' in a_data: 385 if 'runtaskhashes' in a_data:
355 for dep in a_data['runtaskhashes']: 386 for dep in a_data['runtaskhashes']:
356 print "Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]) 387 print "Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep])
388
389 if 'taint' in a_data:
390 print "Tainted (by forced/invalidated task): %s" % a_data['taint']