summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--meta/lib/oe/sstatesig.py163
1 files changed, 1 insertions, 162 deletions
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py
index 13af16e473..35d48f328d 100644
--- a/meta/lib/oe/sstatesig.py
+++ b/meta/lib/oe/sstatesig.py
@@ -266,7 +266,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash):
266 if error_msgs: 266 if error_msgs:
267 bb.fatal("\n".join(error_msgs)) 267 bb.fatal("\n".join(error_msgs))
268 268
269class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): 269class SignatureGeneratorOEEquivHash(bb.siggen.SignatureGeneratorUniHashMixIn, SignatureGeneratorOEBasicHash):
270 name = "OEEquivHash" 270 name = "OEEquivHash"
271 271
272 def init_rundepcheck(self, data): 272 def init_rundepcheck(self, data):
@@ -275,167 +275,6 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash):
275 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') 275 self.method = data.getVar('SSTATE_HASHEQUIV_METHOD')
276 self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data) 276 self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data)
277 277
278 def get_taskdata(self):
279 return (self.server, self.method) + super().get_taskdata()
280
281 def set_taskdata(self, data):
282 self.server, self.method = data[:2]
283 super().set_taskdata(data[2:])
284
285 def __get_task_unihash_key(self, task):
286 # TODO: The key only *needs* to be the taskhash, the task is just
287 # convenient
288 return '%s:%s' % (task, self.taskhash[task])
289
290 def get_stampfile_hash(self, task):
291 if task in self.taskhash:
292 # If a unique hash is reported, use it as the stampfile hash. This
293 # ensures that if a task won't be re-run if the taskhash changes,
294 # but it would result in the same output hash
295 unihash = self.unihashes.get(self.__get_task_unihash_key(task))
296 if unihash is not None:
297 return unihash
298
299 return super().get_stampfile_hash(task)
300
301 def get_unihash(self, task):
302 import urllib
303 import json
304
305 taskhash = self.taskhash[task]
306
307 key = self.__get_task_unihash_key(task)
308
309 # TODO: This cache can grow unbounded. It probably only needs to keep
310 # for each task
311 unihash = self.unihashes.get(key)
312 if unihash is not None:
313 return unihash
314
315 # In the absence of being able to discover a unique hash from the
316 # server, make it be equivalent to the taskhash. The unique "hash" only
317 # really needs to be a unique string (not even necessarily a hash), but
318 # making it match the taskhash has a few advantages:
319 #
320 # 1) All of the sstate code that assumes hashes can be the same
321 # 2) It provides maximal compatibility with builders that don't use
322 # an equivalency server
323 # 3) The value is easy for multiple independent builders to derive the
324 # same unique hash from the same input. This means that if the
325 # independent builders find the same taskhash, but it isn't reported
326 # to the server, there is a better chance that they will agree on
327 # the unique hash.
328 unihash = taskhash
329
330 try:
331 url = '%s/v1/equivalent?%s' % (self.server,
332 urllib.parse.urlencode({'method': self.method, 'taskhash': self.taskhash[task]}))
333
334 request = urllib.request.Request(url)
335 response = urllib.request.urlopen(request)
336 data = response.read().decode('utf-8')
337
338 json_data = json.loads(data)
339
340 if json_data:
341 unihash = json_data['unihash']
342 # A unique hash equal to the taskhash is not very interesting,
343 # so it is reported it at debug level 2. If they differ, that
344 # is much more interesting, so it is reported at debug level 1
345 bb.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, task, self.server))
346 else:
347 bb.debug(2, 'No reported unihash for %s:%s from %s' % (task, taskhash, self.server))
348 except urllib.error.URLError as e:
349 bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
350 except (KeyError, json.JSONDecodeError) as e:
351 bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e)))
352
353 self.unihashes[key] = unihash
354 return unihash
355
356 def report_unihash(self, path, task, d):
357 import urllib
358 import json
359 import tempfile
360 import base64
361 import importlib
362
363 taskhash = d.getVar('BB_TASKHASH')
364 unihash = d.getVar('BB_UNIHASH')
365 report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1'
366 tempdir = d.getVar('T')
367 fn = d.getVar('BB_FILENAME')
368 key = fn + '.do_' + task + ':' + taskhash
369
370 # Sanity checks
371 cache_unihash = self.unihashes.get(key)
372 if cache_unihash is None:
373 bb.fatal('%s not in unihash cache. Please report this error' % key)
374
375 if cache_unihash != unihash:
376 bb.fatal("Cache unihash %s doesn't match BB_UNIHASH %s" % (cache_unihash, unihash))
377
378 sigfile = None
379 sigfile_name = "depsig.do_%s.%d" % (task, os.getpid())
380 sigfile_link = "depsig.do_%s" % task
381
382 try:
383 sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b')
384
385 locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d}
386
387 (module, method) = self.method.rsplit('.', 1)
388 locs['method'] = getattr(importlib.import_module(module), method)
389
390 outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs)
391
392 try:
393 url = '%s/v1/equivalent' % self.server
394 task_data = {
395 'taskhash': taskhash,
396 'method': self.method,
397 'outhash': outhash,
398 'unihash': unihash,
399 'owner': d.getVar('SSTATE_HASHEQUIV_OWNER')
400 }
401
402 if report_taskdata:
403 sigfile.seek(0)
404
405 task_data['PN'] = d.getVar('PN')
406 task_data['PV'] = d.getVar('PV')
407 task_data['PR'] = d.getVar('PR')
408 task_data['task'] = task
409 task_data['outhash_siginfo'] = sigfile.read().decode('utf-8')
410
411 headers = {'content-type': 'application/json'}
412
413 request = urllib.request.Request(url, json.dumps(task_data).encode('utf-8'), headers)
414 response = urllib.request.urlopen(request)
415 data = response.read().decode('utf-8')
416
417 json_data = json.loads(data)
418 new_unihash = json_data['unihash']
419
420 if new_unihash != unihash:
421 bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server))
422 else:
423 bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server))
424 except urllib.error.URLError as e:
425 bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e)))
426 except (KeyError, json.JSONDecodeError) as e:
427 bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e)))
428 finally:
429 if sigfile:
430 sigfile.close()
431
432 sigfile_link_path = os.path.join(tempdir, sigfile_link)
433 bb.utils.remove(sigfile_link_path)
434
435 try:
436 os.symlink(sigfile_name, sigfile_link_path)
437 except OSError:
438 pass
439 278
440# Insert these classes into siggen's namespace so it can see and select them 279# Insert these classes into siggen's namespace so it can see and select them
441bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic 280bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic