summaryrefslogtreecommitdiffstats
path: root/bitbake/lib/bb/siggen.py
diff options
context:
space:
mode:
Diffstat (limited to 'bitbake/lib/bb/siggen.py')
-rw-r--r--bitbake/lib/bb/siggen.py483
1 files changed, 483 insertions, 0 deletions
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
new file mode 100644
index 0000000000..a6d28597ed
--- /dev/null
+++ b/bitbake/lib/bb/siggen.py
@@ -0,0 +1,483 @@
1import hashlib
2import logging
3import os
4import re
5import tempfile
6import bb.data
7
8logger = logging.getLogger('BitBake.SigGen')
9
10try:
11 import cPickle as pickle
12except ImportError:
13 import pickle
14 logger.info('Importing cPickle failed. Falling back to a very slow implementation.')
15
16def init(d):
17 siggens = [obj for obj in globals().itervalues()
18 if type(obj) is type and issubclass(obj, SignatureGenerator)]
19
20 desired = d.getVar("BB_SIGNATURE_HANDLER", True) or "noop"
21 for sg in siggens:
22 if desired == sg.name:
23 return sg(d)
24 break
25 else:
26 logger.error("Invalid signature generator '%s', using default 'noop'\n"
27 "Available generators: %s", desired,
28 ', '.join(obj.name for obj in siggens))
29 return SignatureGenerator(d)
30
31class SignatureGenerator(object):
32 """
33 """
34 name = "noop"
35
36 def __init__(self, data):
37 self.taskhash = {}
38 self.runtaskdeps = {}
39 self.file_checksum_values = {}
40
41 def finalise(self, fn, d, varient):
42 return
43
44 def get_taskhash(self, fn, task, deps, dataCache):
45 return "0"
46
47 def set_taskdata(self, hashes, deps, checksum):
48 return
49
50 def stampfile(self, stampbase, file_name, taskname, extrainfo):
51 return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
52
53 def stampcleanmask(self, stampbase, file_name, taskname, extrainfo):
54 return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.')
55
56 def dump_sigtask(self, fn, task, stampbase, runtime):
57 return
58
59 def invalidate_task(self, task, d, fn):
60 bb.build.del_stamp(task, d, fn)
61
62 def dump_sigs(self, dataCache, options):
63 return
64
65class SignatureGeneratorBasic(SignatureGenerator):
66 """
67 """
68 name = "basic"
69
70 def __init__(self, data):
71 self.basehash = {}
72 self.taskhash = {}
73 self.taskdeps = {}
74 self.runtaskdeps = {}
75 self.file_checksum_values = {}
76 self.gendeps = {}
77 self.lookupcache = {}
78 self.pkgnameextract = re.compile("(?P<fn>.*)\..*")
79 self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split())
80 self.taskwhitelist = None
81 self.init_rundepcheck(data)
82
83 def init_rundepcheck(self, data):
84 self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None
85 if self.taskwhitelist:
86 self.twl = re.compile(self.taskwhitelist)
87 else:
88 self.twl = None
89
90 def _build_data(self, fn, d):
91
92 tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d)
93
94 taskdeps = {}
95 basehash = {}
96
97 for task in tasklist:
98 data = lookupcache[task]
99
100 if data is None:
101 bb.error("Task %s from %s seems to be empty?!" % (task, fn))
102 data = ''
103
104 gendeps[task] -= self.basewhitelist
105 newdeps = gendeps[task]
106 seen = set()
107 while newdeps:
108 nextdeps = newdeps
109 seen |= nextdeps
110 newdeps = set()
111 for dep in nextdeps:
112 if dep in self.basewhitelist:
113 continue
114 gendeps[dep] -= self.basewhitelist
115 newdeps |= gendeps[dep]
116 newdeps -= seen
117
118 alldeps = sorted(seen)
119 for dep in alldeps:
120 data = data + dep
121 var = lookupcache[dep]
122 if var is not None:
123 data = data + str(var)
124 self.basehash[fn + "." + task] = hashlib.md5(data).hexdigest()
125 taskdeps[task] = alldeps
126
127 self.taskdeps[fn] = taskdeps
128 self.gendeps[fn] = gendeps
129 self.lookupcache[fn] = lookupcache
130
131 return taskdeps
132
133 def finalise(self, fn, d, variant):
134
135 if variant:
136 fn = "virtual:" + variant + ":" + fn
137
138 try:
139 taskdeps = self._build_data(fn, d)
140 except:
141 bb.note("Error during finalise of %s" % fn)
142 raise
143
144 #Slow but can be useful for debugging mismatched basehashes
145 #for task in self.taskdeps[fn]:
146 # self.dump_sigtask(fn, task, d.getVar("STAMP", True), False)
147
148 for task in taskdeps:
149 d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task])
150
151 def rundep_check(self, fn, recipename, task, dep, depname, dataCache):
152 # Return True if we should keep the dependency, False to drop it
153 # We only manipulate the dependencies for packages not in the whitelist
154 if self.twl and not self.twl.search(recipename):
155 # then process the actual dependencies
156 if self.twl.search(depname):
157 return False
158 return True
159
160 def read_taint(self, fn, task, stampbase):
161 taint = None
162 try:
163 with open(stampbase + '.' + task + '.taint', 'r') as taintf:
164 taint = taintf.read()
165 except IOError:
166 pass
167 return taint
168
169 def get_taskhash(self, fn, task, deps, dataCache):
170 k = fn + "." + task
171 data = dataCache.basetaskhash[k]
172 self.runtaskdeps[k] = []
173 self.file_checksum_values[k] = {}
174 recipename = dataCache.pkg_fn[fn]
175 for dep in sorted(deps, key=clean_basepath):
176 depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')]
177 if not self.rundep_check(fn, recipename, task, dep, depname, dataCache):
178 continue
179 if dep not in self.taskhash:
180 bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep)
181 data = data + self.taskhash[dep]
182 self.runtaskdeps[k].append(dep)
183
184 if task in dataCache.file_checksums[fn]:
185 checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename)
186 for (f,cs) in checksums:
187 self.file_checksum_values[k][f] = cs
188 data = data + cs
189
190 taint = self.read_taint(fn, task, dataCache.stamp[fn])
191 if taint:
192 data = data + taint
193
194 h = hashlib.md5(data).hexdigest()
195 self.taskhash[k] = h
196 #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task])
197 return h
198
199 def set_taskdata(self, hashes, deps, checksums):
200 self.runtaskdeps = deps
201 self.taskhash = hashes
202 self.file_checksum_values = checksums
203
204 def dump_sigtask(self, fn, task, stampbase, runtime):
205 k = fn + "." + task
206 if runtime == "customfile":
207 sigfile = stampbase
208 elif runtime and k in self.taskhash:
209 sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k]
210 else:
211 sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k]
212
213 bb.utils.mkdirhier(os.path.dirname(sigfile))
214
215 data = {}
216 data['basewhitelist'] = self.basewhitelist
217 data['taskwhitelist'] = self.taskwhitelist
218 data['taskdeps'] = self.taskdeps[fn][task]
219 data['basehash'] = self.basehash[k]
220 data['gendeps'] = {}
221 data['varvals'] = {}
222 data['varvals'][task] = self.lookupcache[fn][task]
223 for dep in self.taskdeps[fn][task]:
224 if dep in self.basewhitelist:
225 continue
226 data['gendeps'][dep] = self.gendeps[fn][dep]
227 data['varvals'][dep] = self.lookupcache[fn][dep]
228
229 if runtime and k in self.taskhash:
230 data['runtaskdeps'] = self.runtaskdeps[k]
231 data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k].items()]
232 data['runtaskhashes'] = {}
233 for dep in data['runtaskdeps']:
234 data['runtaskhashes'][dep] = self.taskhash[dep]
235
236 taint = self.read_taint(fn, task, stampbase)
237 if taint:
238 data['taint'] = taint
239
240 fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.")
241 try:
242 with os.fdopen(fd, "wb") as stream:
243 p = pickle.dump(data, stream, -1)
244 stream.flush()
245 os.chmod(tmpfile, 0664)
246 os.rename(tmpfile, sigfile)
247 except (OSError, IOError) as err:
248 try:
249 os.unlink(tmpfile)
250 except OSError:
251 pass
252 raise err
253
254 def dump_sigs(self, dataCache, options):
255 for fn in self.taskdeps:
256 for task in self.taskdeps[fn]:
257 k = fn + "." + task
258 if k not in self.taskhash:
259 continue
260 if dataCache.basetaskhash[k] != self.basehash[k]:
261 bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k)
262 bb.error("The mismatched hashes were %s and %s" % (dataCache.basetaskhash[k], self.basehash[k]))
263 self.dump_sigtask(fn, task, dataCache.stamp[fn], True)
264
265class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
266 name = "basichash"
267
268 def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False):
269 if taskname != "do_setscene" and taskname.endswith("_setscene"):
270 k = fn + "." + taskname[:-9]
271 else:
272 k = fn + "." + taskname
273 if clean:
274 h = "*"
275 elif k in self.taskhash:
276 h = self.taskhash[k]
277 else:
278 # If k is not in basehash, then error
279 h = self.basehash[k]
280 return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.')
281
282 def stampcleanmask(self, stampbase, fn, taskname, extrainfo):
283 return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True)
284
285 def invalidate_task(self, task, d, fn):
286 bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
287 bb.build.write_taint(task, d, fn)
288
289def dump_this_task(outfile, d):
290 import bb.parse
291 fn = d.getVar("BB_FILENAME", True)
292 task = "do_" + d.getVar("BB_CURRENTTASK", True)
293 bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile")
294
295def clean_basepath(a):
296 if a.startswith("virtual:"):
297 b = a.rsplit(":", 1)[0] + ":" + a.rsplit("/", 1)[1]
298 else:
299 b = a.rsplit("/", 1)[1]
300 return b
301
302def clean_basepaths(a):
303 b = {}
304 for x in a:
305 b[clean_basepath(x)] = a[x]
306 return b
307
308def compare_sigfiles(a, b, recursecb = None):
309 output = []
310
311 p1 = pickle.Unpickler(open(a, "rb"))
312 a_data = p1.load()
313 p2 = pickle.Unpickler(open(b, "rb"))
314 b_data = p2.load()
315
316 def dict_diff(a, b, whitelist=set()):
317 sa = set(a.keys())
318 sb = set(b.keys())
319 common = sa & sb
320 changed = set()
321 for i in common:
322 if a[i] != b[i] and i not in whitelist:
323 changed.add(i)
324 added = sb - sa
325 removed = sa - sb
326 return changed, added, removed
327
328 def file_checksums_diff(a, b):
329 from collections import Counter
330 # Handle old siginfo format
331 if isinstance(a, dict):
332 a = [(os.path.basename(f), cs) for f, cs in a.items()]
333 if isinstance(b, dict):
334 b = [(os.path.basename(f), cs) for f, cs in b.items()]
335 # Compare lists, ensuring we can handle duplicate filenames if they exist
336 removedcount = Counter(a)
337 removedcount.subtract(b)
338 addedcount = Counter(b)
339 addedcount.subtract(a)
340 added = []
341 for x in b:
342 if addedcount[x] > 0:
343 addedcount[x] -= 1
344 added.append(x)
345 removed = []
346 changed = []
347 for x in a:
348 if removedcount[x] > 0:
349 removedcount[x] -= 1
350 for y in added:
351 if y[0] == x[0]:
352 changed.append((x[0], x[1], y[1]))
353 added.remove(y)
354 break
355 else:
356 removed.append(x)
357 added = [x[0] for x in added]
358 removed = [x[0] for x in removed]
359 return changed, added, removed
360
361 if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']:
362 output.append("basewhitelist changed from '%s' to '%s'" % (a_data['basewhitelist'], b_data['basewhitelist']))
363 if a_data['basewhitelist'] and b_data['basewhitelist']:
364 output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist']))
365
366 if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']:
367 output.append("taskwhitelist changed from '%s' to '%s'" % (a_data['taskwhitelist'], b_data['taskwhitelist']))
368 if a_data['taskwhitelist'] and b_data['taskwhitelist']:
369 output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist']))
370
371 if a_data['taskdeps'] != b_data['taskdeps']:
372 output.append("Task dependencies changed from:\n%s\nto:\n%s" % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps'])))
373
374 if a_data['basehash'] != b_data['basehash']:
375 output.append("basehash changed from %s to %s" % (a_data['basehash'], b_data['basehash']))
376
377 changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist'])
378 if changed:
379 for dep in changed:
380 output.append("List of dependencies for variable %s changed from '%s' to '%s'" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep]))
381 if a_data['gendeps'][dep] and b_data['gendeps'][dep]:
382 output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep]))
383 if added:
384 for dep in added:
385 output.append("Dependency on variable %s was added" % (dep))
386 if removed:
387 for dep in removed:
388 output.append("Dependency on Variable %s was removed" % (dep))
389
390
391 changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals'])
392 if changed:
393 for dep in changed:
394 output.append("Variable %s value changed from '%s' to '%s'" % (dep, a_data['varvals'][dep], b_data['varvals'][dep]))
395
396 changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values'])
397 if changed:
398 for f, old, new in changed:
399 output.append("Checksum for file %s changed from %s to %s" % (f, old, new))
400 if added:
401 for f in added:
402 output.append("Dependency on checksum of file %s was added" % (f))
403 if removed:
404 for f in removed:
405 output.append("Dependency on checksum of file %s was removed" % (f))
406
407
408 if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data:
409 a = a_data['runtaskhashes']
410 b = b_data['runtaskhashes']
411 changed, added, removed = dict_diff(a, b)
412 if added:
413 for dep in added:
414 bdep_found = False
415 if removed:
416 for bdep in removed:
417 if b[dep] == a[bdep]:
418 #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep))
419 bdep_found = True
420 if not bdep_found:
421 output.append("Dependency on task %s was added with hash %s" % (clean_basepath(dep), b[dep]))
422 if removed:
423 for dep in removed:
424 adep_found = False
425 if added:
426 for adep in added:
427 if b[adep] == a[dep]:
428 #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep))
429 adep_found = True
430 if not adep_found:
431 output.append("Dependency on task %s was removed with hash %s" % (clean_basepath(dep), a[dep]))
432 if changed:
433 for dep in changed:
434 output.append("Hash for dependent task %s changed from %s to %s" % (clean_basepath(dep), a[dep], b[dep]))
435 if callable(recursecb):
436 # If a dependent hash changed, might as well print the line above and then defer to the changes in
437 # that hash since in all likelyhood, they're the same changes this task also saw.
438 recout = recursecb(dep, a[dep], b[dep])
439 if recout:
440 output = [output[-1]] + recout
441
442 a_taint = a_data.get('taint', None)
443 b_taint = b_data.get('taint', None)
444 if a_taint != b_taint:
445 output.append("Taint (by forced/invalidated task) changed from %s to %s" % (a_taint, b_taint))
446
447 return output
448
449
450def dump_sigfile(a):
451 output = []
452
453 p1 = pickle.Unpickler(open(a, "rb"))
454 a_data = p1.load()
455
456 output.append("basewhitelist: %s" % (a_data['basewhitelist']))
457
458 output.append("taskwhitelist: %s" % (a_data['taskwhitelist']))
459
460 output.append("Task dependencies: %s" % (sorted(a_data['taskdeps'])))
461
462 output.append("basehash: %s" % (a_data['basehash']))
463
464 for dep in a_data['gendeps']:
465 output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep]))
466
467 for dep in a_data['varvals']:
468 output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep]))
469
470 if 'runtaskdeps' in a_data:
471 output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps']))
472
473 if 'file_checksum_values' in a_data:
474 output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values']))
475
476 if 'runtaskhashes' in a_data:
477 for dep in a_data['runtaskhashes']:
478 output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep]))
479
480 if 'taint' in a_data:
481 output.append("Tainted (by forced/invalidated task): %s" % a_data['taint'])
482
483 return output