summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2015-06-08 23:28:12 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2015-06-10 12:00:13 +0100
commitccb2d1b481c0fba69c06a0c0f2553e2a45ee937a (patch)
treef6f363baa27dcd0b25b1d31222dab4f30ac19ba0
parent926de56da9469e3d4ea8349d8e7c9777ae6f93c1 (diff)
downloadpoky-ccb2d1b481c0fba69c06a0c0f2553e2a45ee937a.tar.gz
sstate: Add eventhandler which cleans up stale recipe data
"Incremental builds do not work well when renaming recipes or changing architecture" is a long standing issue which causes people considerable pain. We've struggled for a long time to come up with a way to generically address the problem. There are additional issues where removal of a layer caused data to continue to exist and additionally, changing DISTRO_FEATURES also caused problems in an existing TMPDIR. This patch attempts to address this by adding a mapping between stamp files and manifests. After parsing we can easily tell which stamp files are still reachable, if any manifest has a stamp that can no longer be reached, we can remove it. Since this code ties this to the sstate architecture list, it will not remove data from other than the current MACHINE (and its active architectures). It does not clean the sstate cache so if another build activates something which was cleaned, it should reinstall from sstate. We can also go one step further, depending on the setting of SSTATE_PRUNE_OBSOLETEWORKDIR, workdirs which are no longer active can also be removed. This avoids the buildup of many old copies of data in WORKDIR for example when versions are upgraded. The one thing which may surprise people with this change is if you remove a layer, data added by that layer will be "uninstalled" before the next build continues. I believe this is a feature and a good thing to do though. This code is safe with existing builds. If something isn't in the new index it simply isn't removed. Since changes to the sstate code trigger a rebuild, after this merges, we can assume the code will start to detect changes from that point onwards. [YOCTO #4102] (From OE-Core rev: 4ea39427eedeadd51439a62fa015c86be30c3445) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/classes/sstate.bbclass65
1 files changed, 65 insertions, 0 deletions
diff --git a/meta/classes/sstate.bbclass b/meta/classes/sstate.bbclass
index 949ba4a288..89df28af5d 100644
--- a/meta/classes/sstate.bbclass
+++ b/meta/classes/sstate.bbclass
@@ -33,6 +33,16 @@ SSTATE_SCAN_CMD ?= 'find ${SSTATE_BUILDDIR} \( -name "${@"\" -o -name \"".join(d
33 33
34BB_HASHFILENAME = "${SSTATE_EXTRAPATH} ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}" 34BB_HASHFILENAME = "${SSTATE_EXTRAPATH} ${SSTATE_PKGSPEC} ${SSTATE_SWSPEC}"
35 35
36SSTATE_ARCHS = " \
37 ${BUILD_ARCH} \
38 ${BUILD_ARCH}_${SDK_ARCH}_${SDK_OS} \
39 ${BUILD_ARCH}_${TARGET_ARCH} \
40 ${SDK_ARCH}_${SDK_OS} \
41 ${SDK_ARCH}_${PACKAGE_ARCH} \
42 allarch \
43 ${PACKAGE_ARCH} \
44 ${MACHINE}"
45
36SSTATE_MANMACH ?= "${SSTATE_PKGARCH}" 46SSTATE_MANMACH ?= "${SSTATE_PKGARCH}"
37 47
38SSTATECREATEFUNCS = "sstate_hardcode_path" 48SSTATECREATEFUNCS = "sstate_hardcode_path"
@@ -233,6 +243,20 @@ def sstate_install(ss, d):
233 f.write(di + "\n") 243 f.write(di + "\n")
234 f.close() 244 f.close()
235 245
246 # Append to the list of manifests for this PACKAGE_ARCH
247
248 i = d2.expand("${SSTATE_MANIFESTS}/index-${SSTATE_MANMACH}")
249 l = bb.utils.lockfile(i + ".lock")
250 filedata = d.getVar("STAMP", True) + " " + d2.getVar("SSTATE_MANFILEPREFIX", True) + " " + d.getVar("WORKDIR", True) + "\n"
251 manifests = []
252 if os.path.exists(i):
253 with open(i, "r") as f:
254 manifests = f.readlines()
255 if filedata not in manifests:
256 with open(i, "a+") as f:
257 f.write(filedata)
258 bb.utils.unlockfile(l)
259
236 # Run the actual file install 260 # Run the actual file install
237 for state in ss['dirs']: 261 for state in ss['dirs']:
238 if os.path.exists(state[1]): 262 if os.path.exists(state[1]):
@@ -858,3 +882,44 @@ python sstate_eventhandler() {
858 bb.siggen.dump_this_task(sstatepkg + '_' + taskname + ".tgz" ".siginfo", d) 882 bb.siggen.dump_this_task(sstatepkg + '_' + taskname + ".tgz" ".siginfo", d)
859} 883}
860 884
885SSTATE_PRUNE_OBSOLETEWORKDIR = "1"
886
887# Event handler which removes manifests and stamps file for
888# recipes which are no longer reachable in a build where they
889# once were.
890# Also optionally removes the workdir of those tasks/recipes
891#
892addhandler sstate_eventhandler2
893sstate_eventhandler2[eventmask] = "bb.event.ReachableStamps"
894python sstate_eventhandler2() {
895 import glob
896 d = e.data
897 stamps = e.stamps.values()
898 removeworkdir = (d.getVar("SSTATE_PRUNE_OBSOLETEWORKDIR") == "1")
899 seen = []
900 for a in d.getVar("SSTATE_ARCHS", True).split():
901 toremove = []
902 i = d.expand("${SSTATE_MANIFESTS}/index-" + a)
903 if not os.path.exists(i):
904 continue
905 with open(i, "r") as f:
906 lines = f.readlines()
907 for l in lines:
908 (stamp, manifest, workdir) = l.split()
909 if stamp not in stamps:
910 toremove.append(l)
911 if stamp not in seen:
912 bb.note("Stamp %s is not reachable, removing related manifests" % stamp)
913 seen.append(stamp)
914 for r in toremove:
915 (stamp, manifest, workdir) = r.split()
916 for m in glob.glob(manifest + ".*"):
917 sstate_clean_manifest(m, d)
918 bb.utils.remove(stamp + "*")
919 if removeworkdir:
920 bb.utils.remove(workdir, recurse = True)
921 lines.remove(r)
922 with open(i, "w") as f:
923 for l in lines:
924 f.write(l)
925}