bitbake: implement checksums for local files in SRC_URI

Gathers a list of paths to have checksums calculated at parse time, and
processes these when calculating task hashes. Checksums are cached with
the file's current mtime. Thus, changing any local file in SRC_URI will
now cause the do_fetch taskhash to change, thus forcing a rebuild.

This change adds very roughly about an 8% increase in parse time (a few
seconds) and maybe a few seconds during runqueue generation, so a fairly
moderate performance hit.

Note that since paths are resolved at parse time, this will not force
a rebuild when files are introduced which would cause that resolved path
to be different - for example, where a machine-specific version of a file
was added without otherwise changing the recipe. This will need to be
handled in a future update.

Code to hook this into the signature generator was courtesy of
Richard Purdie <richard.purdie@linuxfoundation.org>.

Implements [YOCTO #2044].

(Bitbake rev: c993b7c457f8b7776e8a5dff253bfa0724bc2cae)

Signed-off-by: Paul Eggleton <paul.eggleton@linux.intel.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 24 insertions, 0 deletions

diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 5a0b80e8a9..daf56770f9 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -60,6 +60,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
         self.taskhash = {}
         self.taskdeps = {}
         self.runtaskdeps = {}
+        self.file_checksum_values = {}
         self.gendeps = {}
         self.lookupcache = {}
         self.pkgnameextract = re.compile("(?P<fn>.*)\..*")
@@ -152,6 +153,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
         k = fn + "." + task
         data = dataCache.basetaskhash[k]
         self.runtaskdeps[k] = []
+        self.file_checksum_values[k] = {}
         recipename = dataCache.pkg_fn[fn]
         for dep in sorted(deps, key=clean_basepath):
             depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')]
@@ -161,6 +163,12 @@ class SignatureGeneratorBasic(SignatureGenerator):
                 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep)
             data = data + self.taskhash[dep]
             self.runtaskdeps[k].append(dep)
+
+        if task in dataCache.file_checksums[fn]:
+            checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
+            for (f,cs) in checksums:
+               self.file_checksum_values[k][f] = cs
+               data = data + cs
         h = hashlib.md5(data).hexdigest()
         self.taskhash[k] = h
         #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
@@ -197,6 +205,7 @@ class SignatureGeneratorBasic(SignatureGenerator):
 
         if runtime and k in self.taskhash:
             data['runtaskdeps'] = self.runtaskdeps[k]
+            data['file_checksum_values'] = self.file_checksum_values[k]
             data['runtaskhashes'] = {}
             for dep in data['runtaskdeps']:
                 data['runtaskhashes'][dep] = self.taskhash[dep]
@@ -304,6 +313,18 @@ def compare_sigfiles(a, b):
         for dep in changed:
             print "Variable %s value changed from %s to %s" % (dep, a_data['varvals'][dep], b_data['varvals'][dep])
 
+    changed, added, removed = dict_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
+    if changed:
+        for f in changed:
+            print "Checksum for file %s changed from %s to %s" % (f, a_data['file_checksum_values'][f], b_data['file_checksum_values'][f])
+    if added:
+        for f in added:
+            print "Dependency on checksum of file %s was added" % (f)
+    if removed:
+        for f in removed:
+            print "Dependency on checksum of file %s was removed" % (f)
+
+
     if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data:
         a = clean_basepaths(a_data['runtaskhashes'])
         b = clean_basepaths(b_data['runtaskhashes'])
@@ -353,6 +374,9 @@ def dump_sigfile(a):
     if 'runtaskdeps' in a_data:
         print "Tasks this task depends on: %s" % (a_data['runtaskdeps'])
 
+    if 'file_checksum_values' in a_data:
+        print "This task depends on the checksums of files: %s" % (a_data['file_checksum_values'])
+
     if 'runtaskhashes' in a_data:
         for dep in a_data['runtaskhashes']:
             print "Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep])