summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2014-12-08 21:25:23 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-01-08 09:52:52 +0000
commit97d44bf526080c31188dfb5b0e0ffcd05a15455a (patch)
tree992c4630920fe14f7616d7c73c31b36921adfc75 /bitbake/lib/bb
parent26cd67e63a9451455def3be8ef03114dcac81ea5 (diff)
downloadpoky-97d44bf526080c31188dfb5b0e0ffcd05a15455a.tar.gz
bitbake: cache/fetch2/siggen: Ensure we track include history for file checksums
Currently, if you reference a file url, its checksum is included in the task hash, however if you change to a different file at a different location, perhaps taking advantage of the FILESPATH functionality, the system will not reparse the file in question and change its checksum to match the new file. To correctly handle this, the system not only needs to know if the existing file still exists or not, but also check the existance of every file it would have looked at when computing the original file. We already do this in the bitbake parsing code for class inclusion. This change uses the same technique to log the file list we looked at and if files in these locations exist when they previously did not, to invalidate and reparse the file. Since data stored in the cache is flattened text, we have to use a string form of the data and split on the ":" character which is ugly, but is an internal detail we can improve later if a better method is found. The cache version changes to trigger a reparse since the previous cache data is now incompatible. [YOCTO #7019] (Bitbake rev: 6c0706a28d72c591f1b75b6e3f3b645859387c7e) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r--bitbake/lib/bb/cache.py9
-rw-r--r--bitbake/lib/bb/fetch2/__init__.py35
-rw-r--r--bitbake/lib/bb/siggen.py3
3 files changed, 26 insertions, 21 deletions
diff --git a/bitbake/lib/bb/cache.py b/bitbake/lib/bb/cache.py
index ac0c27f922..715da07e8d 100644
--- a/bitbake/lib/bb/cache.py
+++ b/bitbake/lib/bb/cache.py
@@ -43,7 +43,7 @@ except ImportError:
43 logger.info("Importing cPickle failed. " 43 logger.info("Importing cPickle failed. "
44 "Falling back to a very slow implementation.") 44 "Falling back to a very slow implementation.")
45 45
46__cache_version__ = "147" 46__cache_version__ = "148"
47 47
48def getCacheFile(path, filename, data_hash): 48def getCacheFile(path, filename, data_hash):
49 return os.path.join(path, filename + "." + data_hash) 49 return os.path.join(path, filename + "." + data_hash)
@@ -529,8 +529,11 @@ class Cache(object):
529 if hasattr(info_array[0], 'file_checksums'): 529 if hasattr(info_array[0], 'file_checksums'):
530 for _, fl in info_array[0].file_checksums.items(): 530 for _, fl in info_array[0].file_checksums.items():
531 for f in fl.split(): 531 for f in fl.split():
532 if not ('*' in f or os.path.exists(f)): 532 if "*" in f:
533 logger.debug(2, "Cache: %s's file checksum list file %s was removed", 533 continue
534 f, exist = f.split(":")
535 if (exist == "True" and not os.path.exists(f)) or (exist == "False" and os.path.exists(f)):
536 logger.debug(2, "Cache: %s's file checksum list file %s changed",
534 fn, f) 537 fn, f)
535 self.remove(fn) 538 self.remove(fn)
536 return False 539 return False
diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py
index 5b26524f45..190de9ed15 100644
--- a/bitbake/lib/bb/fetch2/__init__.py
+++ b/bitbake/lib/bb/fetch2/__init__.py
@@ -936,22 +936,21 @@ def get_checksum_file_list(d):
936 ud = fetch.ud[u] 936 ud = fetch.ud[u]
937 937
938 if ud and isinstance(ud.method, local.Local): 938 if ud and isinstance(ud.method, local.Local):
939 ud.setup_localpath(d) 939 paths = ud.method.localpaths(ud, d)
940 f = ud.localpath 940 for f in paths:
941 pth = ud.decodedurl 941 pth = ud.decodedurl
942 if '*' in pth: 942 if '*' in pth:
943 f = os.path.join(os.path.abspath(f), pth) 943 f = os.path.join(os.path.abspath(f), pth)
944 if f.startswith(dl_dir): 944 if f.startswith(dl_dir):
945 # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else 945 # The local fetcher's behaviour is to return a path under DL_DIR if it couldn't find the file anywhere else
946 if os.path.exists(f): 946 if os.path.exists(f):
947 bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN', True), os.path.basename(f))) 947 bb.warn("Getting checksum for %s SRC_URI entry %s: file not found except in DL_DIR" % (d.getVar('PN', True), os.path.basename(f)))
948 else: 948 else:
949 bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN', True), os.path.basename(f))) 949 bb.warn("Unable to get checksum for %s SRC_URI entry %s: file could not be found" % (d.getVar('PN', True), os.path.basename(f)))
950 filelist.append(f) 950 filelist.append(f + ":" + str(os.path.exists(f)))
951 951
952 return " ".join(filelist) 952 return " ".join(filelist)
953 953
954
955def get_file_checksums(filelist, pn): 954def get_file_checksums(filelist, pn):
956 """Get a list of the checksums for a list of local files 955 """Get a list of the checksums for a list of local files
957 956
@@ -981,6 +980,10 @@ def get_file_checksums(filelist, pn):
981 980
982 checksums = [] 981 checksums = []
983 for pth in filelist.split(): 982 for pth in filelist.split():
983 exist = pth.split(":")[1]
984 if exist == "False":
985 continue
986 pth = pth.split(":")[0]
984 if '*' in pth: 987 if '*' in pth:
985 # Handle globs 988 # Handle globs
986 for f in glob.glob(pth): 989 for f in glob.glob(pth):
@@ -988,14 +991,12 @@ def get_file_checksums(filelist, pn):
988 checksums.extend(checksum_dir(f)) 991 checksums.extend(checksum_dir(f))
989 else: 992 else:
990 checksum = checksum_file(f) 993 checksum = checksum_file(f)
991 if checksum: 994 checksums.append((f, checksum))
992 checksums.append((f, checksum))
993 elif os.path.isdir(pth): 995 elif os.path.isdir(pth):
994 checksums.extend(checksum_dir(pth)) 996 checksums.extend(checksum_dir(pth))
995 else: 997 else:
996 checksum = checksum_file(pth) 998 checksum = checksum_file(pth)
997 if checksum: 999 checksums.append((pth, checksum))
998 checksums.append((pth, checksum))
999 1000
1000 checksums.sort(key=operator.itemgetter(1)) 1001 checksums.sort(key=operator.itemgetter(1))
1001 return checksums 1002 return checksums
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 0c77d72112..a0e70a8ee4 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -185,7 +185,8 @@ class SignatureGeneratorBasic(SignatureGenerator):
185 checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) 185 checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
186 for (f,cs) in checksums: 186 for (f,cs) in checksums:
187 self.file_checksum_values[k][f] = cs 187 self.file_checksum_values[k][f] = cs
188 data = data + cs 188 if cs:
189 data = data + cs
189 190
190 taskdep = dataCache.task_deps[fn] 191 taskdep = dataCache.task_deps[fn]
191 if 'nostamp' in taskdep and task in taskdep['nostamp']: 192 if 'nostamp' in taskdep and task in taskdep['nostamp']: