summaryrefslogtreecommitdiffstats
path: root/bitbake
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-07-29 14:22:23 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-08-06 11:21:31 +0100
commitdb31374efa9b0b5f545c39259e5e0aa255e70086 (patch)
tree8dc376a6a93eb02720d4f59ffc0a1f1d9dc39988 /bitbake
parent9802b2e6509bfc67f979f742e93b35340af62af8 (diff)
downloadpoky-db31374efa9b0b5f545c39259e5e0aa255e70086.tar.gz
bitbake: siggen: Import unihash code from OE-Core
This code is closely tied with the hash server in bitbake and also means we can't relibably test the hashserv runqueue functionality without OE metadata. Moving this to bitbake as a MixIn class makes most sense and encourages code collaboration and reuse as well as enabling easier and more accurate testing of the APIs. (Bitbake rev: 7bb79099a6c1b463d6ae9226c4cab5e76a965675) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake')
-rw-r--r--bitbake/lib/bb/siggen.py163
1 files changed, 163 insertions, 0 deletions
diff --git a/bitbake/lib/bb/siggen.py b/bitbake/lib/bb/siggen.py
index 6a729f3b1e..0105fa7251 100644
--- a/bitbake/lib/bb/siggen.py
+++ b/bitbake/lib/bb/siggen.py
@@ -354,6 +354,169 @@ class SignatureGeneratorBasicHash(SignatureGeneratorBasic):
354 bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task)) 354 bb.note("Tainting hash to force rebuild of task %s, %s" % (fn, task))
355 bb.build.write_taint(task, d, fn) 355 bb.build.write_taint(task, d, fn)
356 356
357class SignatureGeneratorUniHashMixIn(object):
358 def get_taskdata(self):
359 return (self.server, self.method) + super().get_taskdata()
360
361 def set_taskdata(self, data):
362 self.server, self.method = data[:2]
363 super().set_taskdata(data[2:])
364
365 def __get_task_unihash_key(self, task):
366 # TODO: The key only *needs* to be the taskhash, the task is just
367 # convenient
368 return '%s:%s' % (task, self.taskhash[task])
369
370 def get_stampfile_hash(self, task):
371 if task in self.taskhash:
372 # If a unique hash is reported, use it as the stampfile hash. This
373 # ensures that if a task won't be re-run if the taskhash changes,
374 # but it would result in the same output hash
375 unihash = self.unihashes.get(self.__get_task_unihash_key(task))
376 if unihash is not None:
377 return unihash
378
379 return super().get_stampfile_hash(task)
380
381 def get_unihash(self, task):
382 import urllib
383 import json
384
385 taskhash = self.taskhash[task]
386
387 key = self.__get_task_unihash_key(task)
388
389 # TODO: This cache can grow unbounded. It probably only needs to keep
390 # for each task
391 unihash = self.unihashes.get(key)
392 if unihash is not None:
393 return unihash
394
395 # In the absence of being able to discover a unique hash from the
396 # server, make it be equivalent to the taskhash. The unique "hash" only
397 # really needs to be a unique string (not even necessarily a hash), but
398 # making it match the taskhash has a few advantages:
399 #
400 # 1) All of the sstate code that assumes hashes can be the same
401 # 2) It provides maximal compatibility with builders that don't use
402 # an equivalency server
403 # 3) The value is easy for multiple independent builders to derive the
404 # same unique hash from the same input. This means that if the
405 # independent builders find the same taskhash, but it isn't reported
406 # to the server, there is a better chance that they will agree on
407 # the unique hash.
408 unihash = taskhash
409
410 try:
411 url = '%s/v1/equivalent?%s' % (self.server,
412 urllib.parse.urlencode({'method': self.method, 'taskhash': self.taskhash[task]}))
413
414 request = urllib.request.Request(url)
415 response = urllib.request.urlopen(request)
416 data = response.read().decode('utf-8')
417
418 json_data = json.loads(data)
419
420 if json_data:
421 unihash = json_data['unihash']
422 # A unique hash equal to the taskhash is not very interesting,
423 # so it is reported it at debug level 2. If they differ, that
424 # is much more interesting, so it is reported at debug level 1
425 bb.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, task, self.server))
426 else:
427 bb.debug(2, 'No reported unihash for %s:%s from %s' % (task, taskhash, self.server))
428 except urllib.error.URLError as e:
429 bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
430 except (KeyError, json.JSONDecodeError) as e:
431 bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e)))
432
433 self.unihashes[key] = unihash
434 return unihash
435
436 def report_unihash(self, path, task, d):
437 import urllib
438 import json
439 import tempfile
440 import base64
441 import importlib
442
443 taskhash = d.getVar('BB_TASKHASH')
444 unihash = d.getVar('BB_UNIHASH')
445 report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
446 tempdir = d.getVar('T')
447 fn = d.getVar('BB_FILENAME')
448 key = fn + '.do_' + task + ':' + taskhash
449
450 # Sanity checks
451 cache_unihash = self.unihashes.get(key)
452 if cache_unihash is None:
453 bb.fatal('%s not in unihash cache. Please report this error' % key)
454
455 if cache_unihash != unihash:
456 bb.fatal("Cache unihash %s doesn't match BB_UNIHASH %s" % (cache_unihash, unihash))
457
458 sigfile = None
459 sigfile_name = "depsig.do_%s.%d" % (task, os.getpid())
460 sigfile_link = "depsig.do_%s" % task
461
462 try:
463 sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b')
464
465 locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d}
466
467 (module, method) = self.method.rsplit('.', 1)
468 locs['method'] = getattr(importlib.import_module(module), method)
469
470 outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs)
471
472 try:
473 url = '%s/v1/equivalent' % self.server
474 task_data = {
475 'taskhash': taskhash,
476 'method': self.method,
477 'outhash': outhash,
478 'unihash': unihash,
479 'owner': d.getVar('SSTATE_HASHEQUIV_OWNER')
480 }
481
482 if report_taskdata:
483 sigfile.seek(0)
484
485 task_data['PN'] = d.getVar('PN')
486 task_data['PV'] = d.getVar('PV')
487 task_data['PR'] = d.getVar('PR')
488 task_data['task'] = task
489 task_data['outhash_siginfo'] = sigfile.read().decode('utf-8')
490
491 headers = {'content-type': 'application/json'}
492
493 request = urllib.request.Request(url, json.dumps(task_data).encode('utf-8'), headers)
494 response = urllib.request.urlopen(request)
495 data = response.read().decode('utf-8')
496
497 json_data = json.loads(data)
498 new_unihash = json_data['unihash']
499
500 if new_unihash != unihash:
501 bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
502 else:
503 bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
504 except urllib.error.URLError as e:
505 bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
506 except (KeyError, json.JSONDecodeError) as e:
507 bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e)))
508 finally:
509 if sigfile:
510 sigfile.close()
511
512 sigfile_link_path = os.path.join(tempdir, sigfile_link)
513 bb.utils.remove(sigfile_link_path)
514
515 try:
516 os.symlink(sigfile_name, sigfile_link_path)
517 except OSError:
518 pass
519
357def dump_this_task(outfile, d): 520def dump_this_task(outfile, d):
358 import bb.parse 521 import bb.parse
359 fn = d.getVar("BB_FILENAME") 522 fn = d.getVar("BB_FILENAME")