summaryrefslogtreecommitdiffstats
path: root/meta/lib
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2014-09-05 10:40:02 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2014-09-17 22:00:25 +0100
commitc5cc4993f0555d3fc7a7aa5a471ec2b8e940dec6 (patch)
tree5d590aa0bb70f013793dc43b2fbe5683bc17dae7 /meta/lib
parent7d80f8e9468253496a7097685aac8f468940a9c5 (diff)
downloadpoky-c5cc4993f0555d3fc7a7aa5a471ec2b8e940dec6.tar.gz
sstatesig/sstate: Add support for locked down sstate cache usage
I've been giving things some thought, specifically why sstate doesn't get used more and why we have people requesting external toolchains. I'm guessing the issue is that people don't like how often sstate can change and the lack of an easy way to lock it down. Locking it down is actually quite easy so patch implements some basics of how you can do this (for example to a specific toolchain). With an addition like this to local.conf (or wherever): SIGGEN_LOCKEDSIGS = "\ gcc-cross:do_populate_sysroot:a8d91b35b98e1494957a2ddaf4598956 \ eglibc:do_populate_sysroot:13e8c68553dc61f9d67564f13b9b2d67 \ eglibc:do_packagedata:bfca0db1782c719d373f8636282596ee \ gcc-cross:do_packagedata:4b601ff4f67601395ee49c46701122f6 \ " the code at the end of the email will force the hashes to those values for the recipes mentioned. The system would then find and use those specific objects from the sstate cache instead of trying to build anything. Obviously this is a little simplistic, you might need to put an override against this to only apply those revisions for a specific architecture for example. You'd also probably want to put code in the sstate hash validation code to ensure it really did install these from sstate since if it didn't you'd want to abort the build. This patch also implements support to add to bitbake -S which dumps the locked sstate checksums for each task into a ready prepared include file locked-sigs.inc (currently placed into cwd). There is a function, bb.parse.siggen.dump_lockedsigs() which can be called to trigger the same functionality from task space. A warning is added to sstate.bbclass through a call back into the siggen class to warn if objects are not used from the locked cache. The SIGGEN_ENFORCE_LOCKEDSIGS variable controls whether this is just a warning or a fatal error. A script is provided to generate sstate directory from a locked-sigs file. (From OE-Core rev: 7e14784f2493a19c6bfe3ec3f05a5cf9797a2f22) (From OE-Core rev: 884d4fa3e77cf32836f14a113c11489076f4a84d) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'meta/lib')
-rw-r--r--meta/lib/oe/sstatesig.py74
1 files changed, 74 insertions, 0 deletions
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index 4188873c20..7b860c5b0d 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -61,6 +61,16 @@ def sstate_rundepfilter(siggen, fn, recipename, task, dep, depname, dataCache):
61 # Default to keep dependencies 61 # Default to keep dependencies
62 return True 62 return True
63 63
64def sstate_lockedsigs(d):
65 sigs = {}
66 lockedsigs = (d.getVar("SIGGEN_LOCKEDSIGS", True) or "").split()
67 for ls in lockedsigs:
68 pn, task, h = ls.split(":", 2)
69 if pn not in sigs:
70 sigs[pn] = {}
71 sigs[pn][task] = h
72 return sigs
73
64class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic): 74class SignatureGeneratorOEBasic(bb.siggen.SignatureGeneratorBasic):
65 name = "OEBasic" 75 name = "OEBasic"
66 def init_rundepcheck(self, data): 76 def init_rundepcheck(self, data):
@@ -75,10 +85,74 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
75 def init_rundepcheck(self, data): 85 def init_rundepcheck(self, data):
76 self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE", True) or "").split() 86 self.abisaferecipes = (data.getVar("SIGGEN_EXCLUDERECIPES_ABISAFE", True) or "").split()
77 self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS", True) or "").split() 87 self.saferecipedeps = (data.getVar("SIGGEN_EXCLUDE_SAFE_RECIPE_DEPS", True) or "").split()
88 self.lockedsigs = sstate_lockedsigs(data)
89 self.lockedhashes = {}
90 self.lockedpnmap = {}
78 pass 91 pass
79 def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None): 92 def rundep_check(self, fn, recipename, task, dep, depname, dataCache = None):
80 return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache) 93 return sstate_rundepfilter(self, fn, recipename, task, dep, depname, dataCache)
81 94
95 def get_taskdata(self):
96 data = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskdata()
97 return (data, self.lockedpnmap)
98
99 def set_taskdata(self, data):
100 coredata, self.lockedpnmap = data
101 super(bb.siggen.SignatureGeneratorBasicHash, self).set_taskdata(coredata)
102
103 def dump_sigs(self, dataCache, options):
104 self.dump_lockedsigs()
105 return super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigs(dataCache, options)
106
107 def get_taskhash(self, fn, task, deps, dataCache):
108 recipename = dataCache.pkg_fn[fn]
109 self.lockedpnmap[fn] = recipename
110 if recipename in self.lockedsigs:
111 if task in self.lockedsigs[recipename]:
112 k = fn + "." + task
113 h = self.lockedsigs[recipename][task]
114 self.lockedhashes[k] = h
115 self.taskhash[k] = h
116 #bb.warn("Using %s %s %s" % (recipename, task, h))
117 return h
118 h = super(bb.siggen.SignatureGeneratorBasicHash, self).get_taskhash(fn, task, deps, dataCache)
119 #bb.warn("%s %s %s" % (recipename, task, h))
120 return h
121
122 def dump_sigtask(self, fn, task, stampbase, runtime):
123 k = fn + "." + task
124 if k in self.lockedhashes:
125 return
126 super(bb.siggen.SignatureGeneratorBasicHash, self).dump_sigtask(fn, task, stampbase, runtime)
127
128 def dump_lockedsigs(self):
129 bb.plain("Writing locked sigs to " + os.getcwd() + "/locked-sigs.inc")
130 with open("locked-sigs.inc", "w") as f:
131 f.write('SIGGEN_LOCKEDSIGS = "\\\n')
132 #for fn in self.taskdeps:
133 for k in self.runtaskdeps:
134 #k = fn + "." + task
135 fn = k.rsplit(".",1)[0]
136 task = k.rsplit(".",1)[1]
137 if k not in self.taskhash:
138 continue
139 f.write(" " + self.lockedpnmap[fn] + ":" + task + ":" + self.taskhash[k] + " \\\n")
140 f.write(' "\n')
141
142 def checkhashes(self, missed, ret, sq_fn, sq_task, sq_hash, sq_hashfn, d):
143 enforce = (d.getVar("SIGGEN_ENFORCE_LOCKEDSIGS", True) or "1") == "1"
144 msgs = []
145 for task in range(len(sq_fn)):
146 if task not in ret:
147 for pn in self.lockedsigs:
148 if sq_hash[task] in self.lockedsigs[pn].itervalues():
149 msgs.append("Locked sig is set for %s:%s (%s) yet not in sstate cache?" % (pn, sq_task[task], sq_hash[task]))
150 if msgs and enforce:
151 bb.fatal("\n".join(msgs))
152 elif msgs:
153 bb.warn("\n".join(msgs))
154
155
82# Insert these classes into siggen's namespace so it can see and select them 156# Insert these classes into siggen's namespace so it can see and select them
83bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic 157bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic
84bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash 158bb.siggen.SignatureGeneratorOEBasicHash = SignatureGeneratorOEBasicHash