diff options
Diffstat (limited to 'bitbake/lib/bb/siggen.py')
-rw-r--r-- | bitbake/lib/bb/siggen.py | 483 |
1 files changed, 483 insertions, 0 deletions
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py new file mode 100644 index 0000000000..a6d28597ed --- /dev/null +++ b/bitbake/lib/bb/siggen.py | |||
@@ -0,0 +1,483 @@ | |||
1 | import hashlib | ||
2 | import logging | ||
3 | import os | ||
4 | import re | ||
5 | import tempfile | ||
6 | import bb.data | ||
7 | |||
8 | logger = logging.getLogger('BitBake.SigGen') | ||
9 | |||
10 | try: | ||
11 | import cPickle as pickle | ||
12 | except ImportError: | ||
13 | import pickle | ||
14 | logger.info('Importing cPickle failed. Falling back to a very slow implementation.') | ||
15 | |||
16 | def init(d): | ||
17 | siggens = [obj for obj in globals().itervalues() | ||
18 | if type(obj) is type and issubclass(obj, SignatureGenerator)] | ||
19 | |||
20 | desired = d.getVar("BB_SIGNATURE_HANDLER", True) or "noop" | ||
21 | for sg in siggens: | ||
22 | if desired == sg.name: | ||
23 | return sg(d) | ||
24 | break | ||
25 | else: | ||
26 | logger.error("Invalid signature generator '%s', using default 'noop'\n" | ||
27 | "Available generators: %s", desired, | ||
28 | ', '.join(obj.name for obj in siggens)) | ||
29 | return SignatureGenerator(d) | ||
30 | |||
31 | class SignatureGenerator(object): | ||
32 | """ | ||
33 | """ | ||
34 | name = "noop" | ||
35 | |||
36 | def __init__(self, data): | ||
37 | self.taskhash = {} | ||
38 | self.runtaskdeps = {} | ||
39 | self.file_checksum_values = {} | ||
40 | |||
41 | def finalise(self, fn, d, varient): | ||
42 | return | ||
43 | |||
44 | def get_taskhash(self, fn, task, deps, dataCache): | ||
45 | return "0" | ||
46 | |||
47 | def set_taskdata(self, hashes, deps, checksum): | ||
48 | return | ||
49 | |||
50 | def stampfile(self, stampbase, file_name, taskname, extrainfo): | ||
51 | return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.') | ||
52 | |||
53 | def stampcleanmask(self, stampbase, file_name, taskname, extrainfo): | ||
54 | return ("%s.%s.%s" % (stampbase, taskname, extrainfo)).rstrip('.') | ||
55 | |||
56 | def dump_sigtask(self, fn, task, stampbase, runtime): | ||
57 | return | ||
58 | |||
59 | def invalidate_task(self, task, d, fn): | ||
60 | bb.build.del_stamp(task, d, fn) | ||
61 | |||
62 | def dump_sigs(self, dataCache, options): | ||
63 | return | ||
64 | |||
65 | class SignatureGeneratorBasic(SignatureGenerator): | ||
66 | """ | ||
67 | """ | ||
68 | name = "basic" | ||
69 | |||
70 | def __init__(self, data): | ||
71 | self.basehash = {} | ||
72 | self.taskhash = {} | ||
73 | self.taskdeps = {} | ||
74 | self.runtaskdeps = {} | ||
75 | self.file_checksum_values = {} | ||
76 | self.gendeps = {} | ||
77 | self.lookupcache = {} | ||
78 | self.pkgnameextract = re.compile("(?P<fn>.*)\..*") | ||
79 | self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split()) | ||
80 | self.taskwhitelist = None | ||
81 | self.init_rundepcheck(data) | ||
82 | |||
83 | def init_rundepcheck(self, data): | ||
84 | self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None | ||
85 | if self.taskwhitelist: | ||
86 | self.twl = re.compile(self.taskwhitelist) | ||
87 | else: | ||
88 | self.twl = None | ||
89 | |||
90 | def _build_data(self, fn, d): | ||
91 | |||
92 | tasklist, gendeps, lookupcache = bb.data.generate_dependencies(d) | ||
93 | |||
94 | taskdeps = {} | ||
95 | basehash = {} | ||
96 | |||
97 | for task in tasklist: | ||
98 | data = lookupcache[task] | ||
99 | |||
100 | if data is None: | ||
101 | bb.error("Task %s from %s seems to be empty?!" % (task, fn)) | ||
102 | data = '' | ||
103 | |||
104 | gendeps[task] -= self.basewhitelist | ||
105 | newdeps = gendeps[task] | ||
106 | seen = set() | ||
107 | while newdeps: | ||
108 | nextdeps = newdeps | ||
109 | seen |= nextdeps | ||
110 | newdeps = set() | ||
111 | for dep in nextdeps: | ||
112 | if dep in self.basewhitelist: | ||
113 | continue | ||
114 | gendeps[dep] -= self.basewhitelist | ||
115 | newdeps |= gendeps[dep] | ||
116 | newdeps -= seen | ||
117 | |||
118 | alldeps = sorted(seen) | ||
119 | for dep in alldeps: | ||
120 | data = data + dep | ||
121 | var = lookupcache[dep] | ||
122 | if var is not None: | ||
123 | data = data + str(var) | ||
124 | self.basehash[fn + "." + task] = hashlib.md5(data).hexdigest() | ||
125 | taskdeps[task] = alldeps | ||
126 | |||
127 | self.taskdeps[fn] = taskdeps | ||
128 | self.gendeps[fn] = gendeps | ||
129 | self.lookupcache[fn] = lookupcache | ||
130 | |||
131 | return taskdeps | ||
132 | |||
133 | def finalise(self, fn, d, variant): | ||
134 | |||
135 | if variant: | ||
136 | fn = "virtual:" + variant + ":" + fn | ||
137 | |||
138 | try: | ||
139 | taskdeps = self._build_data(fn, d) | ||
140 | except: | ||
141 | bb.note("Error during finalise of %s" % fn) | ||
142 | raise | ||
143 | |||
144 | #Slow but can be useful for debugging mismatched basehashes | ||
145 | #for task in self.taskdeps[fn]: | ||
146 | # self.dump_sigtask(fn, task, d.getVar("STAMP", True), False) | ||
147 | |||
148 | for task in taskdeps: | ||
149 | d.setVar("BB_BASEHASH_task-%s" % task, self.basehash[fn + "." + task]) | ||
150 | |||
151 | def rundep_check(self, fn, recipename, task, dep, depname, dataCache): | ||
152 | # Return True if we should keep the dependency, False to drop it | ||
153 | # We only manipulate the dependencies for packages not in the whitelist | ||
154 | if self.twl and not self.twl.search(recipename): | ||
155 | # then process the actual dependencies | ||
156 | if self.twl.search(depname): | ||
157 | return False | ||
158 | return True | ||
159 | |||
160 | def read_taint(self, fn, task, stampbase): | ||
161 | taint = None | ||
162 | try: | ||
163 | with open(stampbase + '.' + task + '.taint', 'r') as taintf: | ||
164 | taint = taintf.read() | ||
165 | except IOError: | ||
166 | pass | ||
167 | return taint | ||
168 | |||
169 | def get_taskhash(self, fn, task, deps, dataCache): | ||
170 | k = fn + "." + task | ||
171 | data = dataCache.basetaskhash[k] | ||
172 | self.runtaskdeps[k] = [] | ||
173 | self.file_checksum_values[k] = {} | ||
174 | recipename = dataCache.pkg_fn[fn] | ||
175 | for dep in sorted(deps, key=clean_basepath): | ||
176 | depname = dataCache.pkg_fn[self.pkgnameextract.search(dep).group('fn')] | ||
177 | if not self.rundep_check(fn, recipename, task, dep, depname, dataCache): | ||
178 | continue | ||
179 | if dep not in self.taskhash: | ||
180 | bb.fatal("%s is not in taskhash, caller isn't calling in dependency order?", dep) | ||
181 | data = data + self.taskhash[dep] | ||
182 | self.runtaskdeps[k].append(dep) | ||
183 | |||
184 | if task in dataCache.file_checksums[fn]: | ||
185 | checksums = bb.fetch2.get_file_checksums(dataCache.file_checksums[fn][task], recipename) | ||
186 | for (f,cs) in checksums: | ||
187 | self.file_checksum_values[k][f] = cs | ||
188 | data = data + cs | ||
189 | |||
190 | taint = self.read_taint(fn, task, dataCache.stamp[fn]) | ||
191 | if taint: | ||
192 | data = data + taint | ||
193 | |||
194 | h = hashlib.md5(data).hexdigest() | ||
195 | self.taskhash[k] = h | ||
196 | #d.setVar("BB_TASKHASH_task-%s" % task, taskhash[task]) | ||
197 | return h | ||
198 | |||
199 | def set_taskdata(self, hashes, deps, checksums): | ||
200 | self.runtaskdeps = deps | ||
201 | self.taskhash = hashes | ||
202 | self.file_checksum_values = checksums | ||
203 | |||
204 | def dump_sigtask(self, fn, task, stampbase, runtime): | ||
205 | k = fn + "." + task | ||
206 | if runtime == "customfile": | ||
207 | sigfile = stampbase | ||
208 | elif runtime and k in self.taskhash: | ||
209 | sigfile = stampbase + "." + task + ".sigdata" + "." + self.taskhash[k] | ||
210 | else: | ||
211 | sigfile = stampbase + "." + task + ".sigbasedata" + "." + self.basehash[k] | ||
212 | |||
213 | bb.utils.mkdirhier(os.path.dirname(sigfile)) | ||
214 | |||
215 | data = {} | ||
216 | data['basewhitelist'] = self.basewhitelist | ||
217 | data['taskwhitelist'] = self.taskwhitelist | ||
218 | data['taskdeps'] = self.taskdeps[fn][task] | ||
219 | data['basehash'] = self.basehash[k] | ||
220 | data['gendeps'] = {} | ||
221 | data['varvals'] = {} | ||
222 | data['varvals'][task] = self.lookupcache[fn][task] | ||
223 | for dep in self.taskdeps[fn][task]: | ||
224 | if dep in self.basewhitelist: | ||
225 | continue | ||
226 | data['gendeps'][dep] = self.gendeps[fn][dep] | ||
227 | data['varvals'][dep] = self.lookupcache[fn][dep] | ||
228 | |||
229 | if runtime and k in self.taskhash: | ||
230 | data['runtaskdeps'] = self.runtaskdeps[k] | ||
231 | data['file_checksum_values'] = [(os.path.basename(f), cs) for f,cs in self.file_checksum_values[k].items()] | ||
232 | data['runtaskhashes'] = {} | ||
233 | for dep in data['runtaskdeps']: | ||
234 | data['runtaskhashes'][dep] = self.taskhash[dep] | ||
235 | |||
236 | taint = self.read_taint(fn, task, stampbase) | ||
237 | if taint: | ||
238 | data['taint'] = taint | ||
239 | |||
240 | fd, tmpfile = tempfile.mkstemp(dir=os.path.dirname(sigfile), prefix="sigtask.") | ||
241 | try: | ||
242 | with os.fdopen(fd, "wb") as stream: | ||
243 | p = pickle.dump(data, stream, -1) | ||
244 | stream.flush() | ||
245 | os.chmod(tmpfile, 0664) | ||
246 | os.rename(tmpfile, sigfile) | ||
247 | except (OSError, IOError) as err: | ||
248 | try: | ||
249 | os.unlink(tmpfile) | ||
250 | except OSError: | ||
251 | pass | ||
252 | raise err | ||
253 | |||
254 | def dump_sigs(self, dataCache, options): | ||
255 | for fn in self.taskdeps: | ||
256 | for task in self.taskdeps[fn]: | ||
257 | k = fn + "." + task | ||
258 | if k not in self.taskhash: | ||
259 | continue | ||
260 | if dataCache.basetaskhash[k] != self.basehash[k]: | ||
261 | bb.error("Bitbake's cached basehash does not match the one we just generated (%s)!" % k) | ||
262 | bb.error("The mismatched hashes were %s and %s" % (dataCache.basetaskhash[k], self.basehash[k])) | ||
263 | self.dump_sigtask(fn, task, dataCache.stamp[fn], True) | ||
264 | |||
265 | class SignatureGeneratorBasicHash(SignatureGeneratorBasic): | ||
266 | name = "basichash" | ||
267 | |||
268 | def stampfile(self, stampbase, fn, taskname, extrainfo, clean=False): | ||
269 | if taskname != "do_setscene" and taskname.endswith("_setscene"): | ||
270 | k = fn + "." + taskname[:-9] | ||
271 | else: | ||
272 | k = fn + "." + taskname | ||
273 | if clean: | ||
274 | h = "*" | ||
275 | elif k in self.taskhash: | ||
276 | h = self.taskhash[k] | ||
277 | else: | ||
278 | # If k is not in basehash, then error | ||
279 | h = self.basehash[k] | ||
280 | return ("%s.%s.%s.%s" % (stampbase, taskname, h, extrainfo)).rstrip('.') | ||
281 | |||
282 | def stampcleanmask(self, stampbase, fn, taskname, extrainfo): | ||
283 | return self.stampfile(stampbase, fn, taskname, extrainfo, clean=True) | ||
284 | |||
285 | def invalidate_task(self, task, d, fn): | ||
286 | bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task)) | ||
287 | bb.build.write_taint(task, d, fn) | ||
288 | |||
289 | def dump_this_task(outfile, d): | ||
290 | import bb.parse | ||
291 | fn = d.getVar("BB_FILENAME", True) | ||
292 | task = "do_" + d.getVar("BB_CURRENTTASK", True) | ||
293 | bb.parse.siggen.dump_sigtask(fn, task, outfile, "customfile") | ||
294 | |||
295 | def clean_basepath(a): | ||
296 | if a.startswith("virtual:"): | ||
297 | b = a.rsplit(":", 1)[0] + ":" + a.rsplit("/", 1)[1] | ||
298 | else: | ||
299 | b = a.rsplit("/", 1)[1] | ||
300 | return b | ||
301 | |||
302 | def clean_basepaths(a): | ||
303 | b = {} | ||
304 | for x in a: | ||
305 | b[clean_basepath(x)] = a[x] | ||
306 | return b | ||
307 | |||
308 | def compare_sigfiles(a, b, recursecb = None): | ||
309 | output = [] | ||
310 | |||
311 | p1 = pickle.Unpickler(open(a, "rb")) | ||
312 | a_data = p1.load() | ||
313 | p2 = pickle.Unpickler(open(b, "rb")) | ||
314 | b_data = p2.load() | ||
315 | |||
316 | def dict_diff(a, b, whitelist=set()): | ||
317 | sa = set(a.keys()) | ||
318 | sb = set(b.keys()) | ||
319 | common = sa & sb | ||
320 | changed = set() | ||
321 | for i in common: | ||
322 | if a[i] != b[i] and i not in whitelist: | ||
323 | changed.add(i) | ||
324 | added = sb - sa | ||
325 | removed = sa - sb | ||
326 | return changed, added, removed | ||
327 | |||
328 | def file_checksums_diff(a, b): | ||
329 | from collections import Counter | ||
330 | # Handle old siginfo format | ||
331 | if isinstance(a, dict): | ||
332 | a = [(os.path.basename(f), cs) for f, cs in a.items()] | ||
333 | if isinstance(b, dict): | ||
334 | b = [(os.path.basename(f), cs) for f, cs in b.items()] | ||
335 | # Compare lists, ensuring we can handle duplicate filenames if they exist | ||
336 | removedcount = Counter(a) | ||
337 | removedcount.subtract(b) | ||
338 | addedcount = Counter(b) | ||
339 | addedcount.subtract(a) | ||
340 | added = [] | ||
341 | for x in b: | ||
342 | if addedcount[x] > 0: | ||
343 | addedcount[x] -= 1 | ||
344 | added.append(x) | ||
345 | removed = [] | ||
346 | changed = [] | ||
347 | for x in a: | ||
348 | if removedcount[x] > 0: | ||
349 | removedcount[x] -= 1 | ||
350 | for y in added: | ||
351 | if y[0] == x[0]: | ||
352 | changed.append((x[0], x[1], y[1])) | ||
353 | added.remove(y) | ||
354 | break | ||
355 | else: | ||
356 | removed.append(x) | ||
357 | added = [x[0] for x in added] | ||
358 | removed = [x[0] for x in removed] | ||
359 | return changed, added, removed | ||
360 | |||
361 | if 'basewhitelist' in a_data and a_data['basewhitelist'] != b_data['basewhitelist']: | ||
362 | output.append("basewhitelist changed from '%s' to '%s'" % (a_data['basewhitelist'], b_data['basewhitelist'])) | ||
363 | if a_data['basewhitelist'] and b_data['basewhitelist']: | ||
364 | output.append("changed items: %s" % a_data['basewhitelist'].symmetric_difference(b_data['basewhitelist'])) | ||
365 | |||
366 | if 'taskwhitelist' in a_data and a_data['taskwhitelist'] != b_data['taskwhitelist']: | ||
367 | output.append("taskwhitelist changed from '%s' to '%s'" % (a_data['taskwhitelist'], b_data['taskwhitelist'])) | ||
368 | if a_data['taskwhitelist'] and b_data['taskwhitelist']: | ||
369 | output.append("changed items: %s" % a_data['taskwhitelist'].symmetric_difference(b_data['taskwhitelist'])) | ||
370 | |||
371 | if a_data['taskdeps'] != b_data['taskdeps']: | ||
372 | output.append("Task dependencies changed from:\n%s\nto:\n%s" % (sorted(a_data['taskdeps']), sorted(b_data['taskdeps']))) | ||
373 | |||
374 | if a_data['basehash'] != b_data['basehash']: | ||
375 | output.append("basehash changed from %s to %s" % (a_data['basehash'], b_data['basehash'])) | ||
376 | |||
377 | changed, added, removed = dict_diff(a_data['gendeps'], b_data['gendeps'], a_data['basewhitelist'] & b_data['basewhitelist']) | ||
378 | if changed: | ||
379 | for dep in changed: | ||
380 | output.append("List of dependencies for variable %s changed from '%s' to '%s'" % (dep, a_data['gendeps'][dep], b_data['gendeps'][dep])) | ||
381 | if a_data['gendeps'][dep] and b_data['gendeps'][dep]: | ||
382 | output.append("changed items: %s" % a_data['gendeps'][dep].symmetric_difference(b_data['gendeps'][dep])) | ||
383 | if added: | ||
384 | for dep in added: | ||
385 | output.append("Dependency on variable %s was added" % (dep)) | ||
386 | if removed: | ||
387 | for dep in removed: | ||
388 | output.append("Dependency on Variable %s was removed" % (dep)) | ||
389 | |||
390 | |||
391 | changed, added, removed = dict_diff(a_data['varvals'], b_data['varvals']) | ||
392 | if changed: | ||
393 | for dep in changed: | ||
394 | output.append("Variable %s value changed from '%s' to '%s'" % (dep, a_data['varvals'][dep], b_data['varvals'][dep])) | ||
395 | |||
396 | changed, added, removed = file_checksums_diff(a_data['file_checksum_values'], b_data['file_checksum_values']) | ||
397 | if changed: | ||
398 | for f, old, new in changed: | ||
399 | output.append("Checksum for file %s changed from %s to %s" % (f, old, new)) | ||
400 | if added: | ||
401 | for f in added: | ||
402 | output.append("Dependency on checksum of file %s was added" % (f)) | ||
403 | if removed: | ||
404 | for f in removed: | ||
405 | output.append("Dependency on checksum of file %s was removed" % (f)) | ||
406 | |||
407 | |||
408 | if 'runtaskhashes' in a_data and 'runtaskhashes' in b_data: | ||
409 | a = a_data['runtaskhashes'] | ||
410 | b = b_data['runtaskhashes'] | ||
411 | changed, added, removed = dict_diff(a, b) | ||
412 | if added: | ||
413 | for dep in added: | ||
414 | bdep_found = False | ||
415 | if removed: | ||
416 | for bdep in removed: | ||
417 | if b[dep] == a[bdep]: | ||
418 | #output.append("Dependency on task %s was replaced by %s with same hash" % (dep, bdep)) | ||
419 | bdep_found = True | ||
420 | if not bdep_found: | ||
421 | output.append("Dependency on task %s was added with hash %s" % (clean_basepath(dep), b[dep])) | ||
422 | if removed: | ||
423 | for dep in removed: | ||
424 | adep_found = False | ||
425 | if added: | ||
426 | for adep in added: | ||
427 | if b[adep] == a[dep]: | ||
428 | #output.append("Dependency on task %s was replaced by %s with same hash" % (adep, dep)) | ||
429 | adep_found = True | ||
430 | if not adep_found: | ||
431 | output.append("Dependency on task %s was removed with hash %s" % (clean_basepath(dep), a[dep])) | ||
432 | if changed: | ||
433 | for dep in changed: | ||
434 | output.append("Hash for dependent task %s changed from %s to %s" % (clean_basepath(dep), a[dep], b[dep])) | ||
435 | if callable(recursecb): | ||
436 | # If a dependent hash changed, might as well print the line above and then defer to the changes in | ||
437 | # that hash since in all likelyhood, they're the same changes this task also saw. | ||
438 | recout = recursecb(dep, a[dep], b[dep]) | ||
439 | if recout: | ||
440 | output = [output[-1]] + recout | ||
441 | |||
442 | a_taint = a_data.get('taint', None) | ||
443 | b_taint = b_data.get('taint', None) | ||
444 | if a_taint != b_taint: | ||
445 | output.append("Taint (by forced/invalidated task) changed from %s to %s" % (a_taint, b_taint)) | ||
446 | |||
447 | return output | ||
448 | |||
449 | |||
450 | def dump_sigfile(a): | ||
451 | output = [] | ||
452 | |||
453 | p1 = pickle.Unpickler(open(a, "rb")) | ||
454 | a_data = p1.load() | ||
455 | |||
456 | output.append("basewhitelist: %s" % (a_data['basewhitelist'])) | ||
457 | |||
458 | output.append("taskwhitelist: %s" % (a_data['taskwhitelist'])) | ||
459 | |||
460 | output.append("Task dependencies: %s" % (sorted(a_data['taskdeps']))) | ||
461 | |||
462 | output.append("basehash: %s" % (a_data['basehash'])) | ||
463 | |||
464 | for dep in a_data['gendeps']: | ||
465 | output.append("List of dependencies for variable %s is %s" % (dep, a_data['gendeps'][dep])) | ||
466 | |||
467 | for dep in a_data['varvals']: | ||
468 | output.append("Variable %s value is %s" % (dep, a_data['varvals'][dep])) | ||
469 | |||
470 | if 'runtaskdeps' in a_data: | ||
471 | output.append("Tasks this task depends on: %s" % (a_data['runtaskdeps'])) | ||
472 | |||
473 | if 'file_checksum_values' in a_data: | ||
474 | output.append("This task depends on the checksums of files: %s" % (a_data['file_checksum_values'])) | ||
475 | |||
476 | if 'runtaskhashes' in a_data: | ||
477 | for dep in a_data['runtaskhashes']: | ||
478 | output.append("Hash for dependent task %s is %s" % (dep, a_data['runtaskhashes'][dep])) | ||
479 | |||
480 | if 'taint' in a_data: | ||
481 | output.append("Tainted (by forced/invalidated task): %s" % a_data['taint']) | ||
482 | |||
483 | return output | ||