summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--bitbake/lib/bb/siggen.py163
1 files changed, 163 insertions, 0 deletions
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 6a729f3b1e..0105fa7251 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -354,6 +354,169 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
354 bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task)) 354 bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
355 bb.build.write_taint(task, d, fn) 355 bb.build.write_taint(task, d, fn)
356 356
357class SignatureGeneratorUniHashMixIn(object):
358 def get_taskdata(self):
359 return (self.server, self.method) + super().get_taskdata()
360
361 def set_taskdata(self, data):
362 self.server, self.method = data[:2]
363 super().set_taskdata(data[2:])
364
365 def __get_task_unihash_key(self, task):
366 # TODO: The key only *needs* to be the taskhash, the task is just
367 # convenient
368 return '%s:%s' % (task, self.taskhash[task])
369
370 def get_stampfile_hash(self, task):
371 if task in self.taskhash:
372 # If a unique hash is reported, use it as the stampfile hash. This
373 # ensures that if a task won't be re-run if the taskhash changes,
374 # but it would result in the same output hash
375 unihash = self.unihashes.get(self.__get_task_unihash_key(task))
376 if unihash is not None:
377 return unihash
378
379 return super().get_stampfile_hash(task)
380
381 def get_unihash(self, task):
382 import urllib
383 import json
384
385 taskhash = self.taskhash[task]
386
387 key = self.__get_task_unihash_key(task)
388
389 # TODO: This cache can grow unbounded. It probably only needs to keep
390 # for each task
391 unihash = self.unihashes.get(key)
392 if unihash is not None:
393 return unihash
394
395 # In the absence of being able to discover a unique hash from the
396 # server, make it be equivalent to the taskhash. The unique "hash" only
397 # really needs to be a unique string (not even necessarily a hash), but
398 # making it match the taskhash has a few advantages:
399 #
400 # 1) All of the sstate code that assumes hashes can be the same
401 # 2) It provides maximal compatibility with builders that don't use
402 # an equivalency server
403 # 3) The value is easy for multiple independent builders to derive the
404 # same unique hash from the same input. This means that if the
405 # independent builders find the same taskhash, but it isn't reported
406 # to the server, there is a better chance that they will agree on
407 # the unique hash.
408 unihash = taskhash
409
410 try:
411 url = '%s/v1/equivalent?%s' % (self.server,
412 urllib.parse.urlencode({'method': self.method, 'taskhash': self.taskhash[task]}))
413
414 request = urllib.request.Request(url)
415 response = urllib.request.urlopen(request)
416 data = response.read().decode('utf-8')
417
418 json_data = json.loads(data)
419
420 if json_data:
421 unihash = json_data['unihash']
422 # A unique hash equal to the taskhash is not very interesting,
423 # so it is reported it at debug level 2. If they differ, that
424 # is much more interesting, so it is reported at debug level 1
425 bb.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, task, self.server))
426 else:
427 bb.debug(2, 'No reported unihash for %s:%s from %s' % (task, taskhash, self.server))
428 except urllib.error.URLError as e:
429 bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
430 except (KeyError, json.JSONDecodeError) as e:
431 bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e)))
432
433 self.unihashes[key] = unihash
434 return unihash
435
436 def report_unihash(self, path, task, d):
437 import urllib
438 import json
439 import tempfile
440 import base64
441 import importlib
442
443 taskhash = d.getVar('BB_TASKHASH')
444 unihash = d.getVar('BB_UNIHASH')
445 report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
446 tempdir = d.getVar('T')
447 fn = d.getVar('BB_FILENAME')
448 key = fn + '.do_' + task + ':' + taskhash
449
450 # Sanity checks
451 cache_unihash = self.unihashes.get(key)
452 if cache_unihash is None:
453 bb.fatal('%s not in unihash cache. Please report this error' % key)
454
455 if cache_unihash != unihash:
456 bb.fatal("Cache unihash %s doesn't match BB_UNIHASH %s" % (cache_unihash, unihash))
457
458 sigfile = None
459 sigfile_name = "depsig.do_%s.%d" % (task, os.getpid())
460 sigfile_link = "depsig.do_%s" % task
461
462 try:
463 sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b')
464
465 locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d}
466
467 (module, method) = self.method.rsplit('.', 1)
468 locs['method'] = getattr(importlib.import_module(module), method)
469
470 outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs)
471
472 try:
473 url = '%s/v1/equivalent' % self.server
474 task_data = {
475 'taskhash': taskhash,
476 'method': self.method,
477 'outhash': outhash,
478 'unihash': unihash,
479 'owner': d.getVar('SSTATE_HASHEQUIV_OWNER')
480 }
481
482 if report_taskdata:
483 sigfile.seek(0)
484
485 task_data['PN'] = d.getVar('PN')
486 task_data['PV'] = d.getVar('PV')
487 task_data['PR'] = d.getVar('PR')
488 task_data['task'] = task
489 task_data['outhash_siginfo'] = sigfile.read().decode('utf-8')
490
491 headers = {'content-type': 'application/json'}
492
493 request = urllib.request.Request(url, json.dumps(task_data).encode('utf-8'), headers)
494 response = urllib.request.urlopen(request)
495 data = response.read().decode('utf-8')
496
497 json_data = json.loads(data)
498 new_unihash = json_data['unihash']
499
500 if new_unihash != unihash:
501 bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
502 else:
503 bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
504 except urllib.error.URLError as e:
505 bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
506 except (KeyError, json.JSONDecodeError) as e:
507 bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e)))
508 finally:
509 if sigfile:
510 sigfile.close()
511
512 sigfile_link_path = os.path.join(tempdir, sigfile_link)
513 bb.utils.remove(sigfile_link_path)
514
515 try:
516 os.symlink(sigfile_name, sigfile_link_path)
517 except OSError:
518 pass
519
357def dump_this_task(outfile, d): 520def dump_this_task(outfile, d):
358 import bb.parse 521 import bb.parse
359 fn = d.getVar("BB_FILENAME") 522 fn = d.getVar("BB_FILENAME")