summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-07-29 14:22:03 +0100
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-08-06 11:24:26 +0100
commit62c8b8c4d96836ba8e23b512318df67b2d055d15 (patch)
tree70133f62db120c257de75dcae804e78f604ce748
parentc49aa783d84eb4dd8241e7baed44775c53e8c511 (diff)
downloadpoky-62c8b8c4d96836ba8e23b512318df67b2d055d15.tar.gz
sstatesig: Move unihash siggen code to bitbake
This code is closely tied with the hash server in bitbake and also means we can't relibably test the hashserv runqueue functionality without OE metadata. Moving this to bitbake as a MixIn class makes most sense and encourages code collaboration and reuse as well as enabling easier and more accurate testing of the APIs. (From OE-Core rev: a2a9c6092d4dde706ed071b08a972d1d87184295) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/lib/oe/sstatesig.py163
1 files changed, 1 insertions, 162 deletions
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index 13af16e473..35d48f328d 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -266,7 +266,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
266 if error_msgs: 266 if error_msgs:
267 bb.fatal("\n".join(error_msgs)) 267 bb.fatal("\n".join(error_msgs))
268 268
269class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): 269class SignatureGeneratorOEEquivHash(bb.siggen.SignatureGeneratorUniHashMixIn, SignatureGeneratorOEBasicHash):
270 name = "OEEquivHash" 270 name = "OEEquivHash"
271 271
272 def init_rundepcheck(self, data): 272 def init_rundepcheck(self, data):
@@ -275,167 +275,6 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash):
275 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') 275 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD')
276 self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data) 276 self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data)
277 277
278 def get_taskdata(self):
279 return (self.server, self.method) + super().get_taskdata()
280
281 def set_taskdata(self, data):
282 self.server, self.method = data[:2]
283 super().set_taskdata(data[2:])
284
285 def __get_task_unihash_key(self, task):
286 # TODO: The key only *needs* to be the taskhash, the task is just
287 # convenient
288 return '%s:%s' % (task, self.taskhash[task])
289
290 def get_stampfile_hash(self, task):
291 if task in self.taskhash:
292 # If a unique hash is reported, use it as the stampfile hash. This
293 # ensures that if a task won't be re-run if the taskhash changes,
294 # but it would result in the same output hash
295 unihash = self.unihashes.get(self.__get_task_unihash_key(task))
296 if unihash is not None:
297 return unihash
298
299 return super().get_stampfile_hash(task)
300
301 def get_unihash(self, task):
302 import urllib
303 import json
304
305 taskhash = self.taskhash[task]
306
307 key = self.__get_task_unihash_key(task)
308
309 # TODO: This cache can grow unbounded. It probably only needs to keep
310 # for each task
311 unihash = self.unihashes.get(key)
312 if unihash is not None:
313 return unihash
314
315 # In the absence of being able to discover a unique hash from the
316 # server, make it be equivalent to the taskhash. The unique "hash" only
317 # really needs to be a unique string (not even necessarily a hash), but
318 # making it match the taskhash has a few advantages:
319 #
320 # 1) All of the sstate code that assumes hashes can be the same
321 # 2) It provides maximal compatibility with builders that don't use
322 # an equivalency server
323 # 3) The value is easy for multiple independent builders to derive the
324 # same unique hash from the same input. This means that if the
325 # independent builders find the same taskhash, but it isn't reported
326 # to the server, there is a better chance that they will agree on
327 # the unique hash.
328 unihash = taskhash
329
330 try:
331 url = '%s/v1/equivalent?%s' % (self.server,
332 urllib.parse.urlencode({'method': self.method, 'taskhash': self.taskhash[task]}))
333
334 request = urllib.request.Request(url)
335 response = urllib.request.urlopen(request)
336 data = response.read().decode('utf-8')
337
338 json_data = json.loads(data)
339
340 if json_data:
341 unihash = json_data['unihash']
342 # A unique hash equal to the taskhash is not very interesting,
343 # so it is reported it at debug level 2. If they differ, that
344 # is much more interesting, so it is reported at debug level 1
345 bb.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, task, self.server))
346 else:
347 bb.debug(2, 'No reported unihash for %s:%s from %s' % (task, taskhash, self.server))
348 except urllib.error.URLError as e:
349 bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
350 except (KeyError, json.JSONDecodeError) as e:
351 bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e)))
352
353 self.unihashes[key] = unihash
354 return unihash
355
356 def report_unihash(self, path, task, d):
357 import urllib
358 import json
359 import tempfile
360 import base64
361 import importlib
362
363 taskhash = d.getVar('BB_TASKHASH')
364 unihash = d.getVar('BB_UNIHASH')
365 report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
366 tempdir = d.getVar('T')
367 fn = d.getVar('BB_FILENAME')
368 key = fn + '.do_' + task + ':' + taskhash
369
370 # Sanity checks
371 cache_unihash = self.unihashes.get(key)
372 if cache_unihash is None:
373 bb.fatal('%s not in unihash cache. Please report this error' % key)
374
375 if cache_unihash != unihash:
376 bb.fatal("Cache unihash %s doesn't match BB_UNIHASH %s" % (cache_unihash, unihash))
377
378 sigfile = None
379 sigfile_name = "depsig.do_%s.%d" % (task, os.getpid())
380 sigfile_link = "depsig.do_%s" % task
381
382 try:
383 sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b')
384
385 locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d}
386
387 (module, method) = self.method.rsplit('.', 1)
388 locs['method'] = getattr(importlib.import_module(module), method)
389
390 outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs)
391
392 try:
393 url = '%s/v1/equivalent' % self.server
394 task_data = {
395 'taskhash': taskhash,
396 'method': self.method,
397 'outhash': outhash,
398 'unihash': unihash,
399 'owner': d.getVar('SSTATE_HASHEQUIV_OWNER')
400 }
401
402 if report_taskdata:
403 sigfile.seek(0)
404
405 task_data['PN'] = d.getVar('PN')
406 task_data['PV'] = d.getVar('PV')
407 task_data['PR'] = d.getVar('PR')
408 task_data['task'] = task
409 task_data['outhash_siginfo'] = sigfile.read().decode('utf-8')
410
411 headers = {'content-type': 'application/json'}
412
413 request = urllib.request.Request(url, json.dumps(task_data).encode('utf-8'), headers)
414 response = urllib.request.urlopen(request)
415 data = response.read().decode('utf-8')
416
417 json_data = json.loads(data)
418 new_unihash = json_data['unihash']
419
420 if new_unihash != unihash:
421 bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
422 else:
423 bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
424 except urllib.error.URLError as e:
425 bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
426 except (KeyError, json.JSONDecodeError) as e:
427 bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e)))
428 finally:
429 if sigfile:
430 sigfile.close()
431
432 sigfile_link_path = os.path.join(tempdir, sigfile_link)
433 bb.utils.remove(sigfile_link_path)
434
435 try:
436 os.symlink(sigfile_name, sigfile_link_path)
437 except OSError:
438 pass
439 278
440# Insert these classes into siggen's namespace so it can see and select them 279# Insert these classes into siggen's namespace so it can see and select them
441bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic 280bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic