From 8b8be74ed21b878b2fe30d5b76ff0648e6e48c18 Mon Sep 17 00:00:00 2001 From: Paul Eggleton Date: Wed, 23 May 2012 00:23:32 +0100 Subject: bitbake: implement checksums for local files in SRC_URI Gathers a list of paths to have checksums calculated at parse time, and processes these when calculating task hashes. Checksums are cached with the file's current mtime. Thus, changing any local file in SRC_URI will now cause the do_fetch taskhash to change, thus forcing a rebuild. This change adds very roughly about an 8% increase in parse time (a few seconds) and maybe a few seconds during runqueue generation, so a fairly moderate performance hit. Note that since paths are resolved at parse time, this will not force a rebuild when files are introduced which would cause that resolved path to be different - for example, where a machine-specific version of a file was added without otherwise changing the recipe. This will need to be handled in a future update. Code to hook this into the signature generator was courtesy of Richard Purdie . Implements [YOCTO #2044]. (Bitbake rev: c993b7c457f8b7776e8a5dff253bfa0724bc2cae) Signed-off-by: Paul Eggleton Signed-off-by: Richard Purdie --- bitbake/lib/bb/siggen.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'bitbake/lib/bb/siggen.py') diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py index 5a0b80e8a9..daf56770f9 100644 --- a/bitbake/lib/bb/siggen.py +++ b/bitbake/lib/bb/siggen.py @@ -60,6 +60,7 @@ class SignatureGeneratorBasic(SignatureGenerator): self.taskhash = {} self.taskdeps = {} self.runtaskdeps = {} + self.file_checksum_values = {} self.gendeps = {} self.lookupcache = {} self.pkgnameextract = re.compile("(?P.*)\..*") @@ -152,6 +153,7 @@ class SignatureGeneratorBasic(SignatureGenerator): k = fn + "." + task data = dataCache.basetaskhash[k] self.runtaskdeps[k] = [] + self.file_checksum_values[k] = {} recipename = dataCache.pkg_fn[fn] for dep in sorted(deps, key=clean_basepath): depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')] @@ -161,6 +163,12 @@ class SignatureGeneratorBasic(SignatureGenerator): bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep) data = data + self.taskhash[dep] self.runtaskdeps[k].append(dep) + + if task in dataCache.file_checksums[fn]: + checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) + for (f,cs) in checksums: + self.file_checksum_values[k][f] = cs + data = data + cs h = hashlib.md5(data).hexdigest() self.taskhash[k] = h #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) @@ -197,6 +205,7 @@ class SignatureGeneratorBasic(SignatureGenerator): if runtime and k in self.taskhash: data['runtaskdeps'] = self.runtaskdeps[k] + data['file_checksum_values'] = self.file_checksum_values[k] data['runtaskhashes'] = {} for dep in data['runtaskdeps']: data['runtaskhashes'][dep] = self.taskhash[dep] @@ -304,6 +313,18 @@ def compare_sigfiles(a, b): for dep in changed: print "Variable %s value changed from %s to %s" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]) + changed, added, removed = dict_diff(a_data['file_checksum_values'], b_data['file_checksum_values']) + if changed: + for f in changed: + print "Checksum for file %s changed from %s to %s" % (f, a_data['file_checksum_values'][f], b_data['file_checksum_values'][f]) + if added: + for f in added: + print "Dependency on checksum of file %s was added" % (f) + if removed: + for f in removed: + print "Dependency on checksum of file %s was removed" % (f) + + if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data: a = clean_basepaths(a_data['runtaskhashes']) b = clean_basepaths(b_data['runtaskhashes']) @@ -353,6 +374,9 @@ def dump_sigfile(a): if 'runtaskdeps' in a_data: print "Tasks this task depends on: %s" % (a_data['runtaskdeps']) + if 'file_checksum_values' in a_data: + print "This task depends on the checksums of files: %s" % (a_data['file_checksum_values']) + if 'runtaskhashes' in a_data: for dep in a_data['runtaskhashes']: print "Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]) -- cgit v1.2.3-54-g00ecf