diff options
-rw-r--r-- | meta/lib/oe/sstatesig.py | 163 |
1 files changed, 1 insertions, 162 deletions
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py index 13af16e473..35d48f328d 100644 --- a/meta/lib/oe/sstatesig.py +++ b/meta/lib/oe/sstatesig.py | |||
@@ -266,7 +266,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): | |||
266 | if error_msgs: | 266 | if error_msgs: |
267 | bb.fatal("\n".join(error_msgs)) | 267 | bb.fatal("\n".join(error_msgs)) |
268 | 268 | ||
269 | class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): | 269 | class SignatureGeneratorOEEquivHash(bb.siggen.SignatureGeneratorUniHashMixIn, SignatureGeneratorOEBasicHash): |
270 | name = "OEEquivHash" | 270 | name = "OEEquivHash" |
271 | 271 | ||
272 | def init_rundepcheck(self, data): | 272 | def init_rundepcheck(self, data): |
@@ -275,167 +275,6 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): | |||
275 | self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') | 275 | self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') |
276 | self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data) | 276 | self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data) |
277 | 277 | ||
278 | def get_taskdata(self): | ||
279 | return (self.server, self.method) + super().get_taskdata() | ||
280 | |||
281 | def set_taskdata(self, data): | ||
282 | self.server, self.method = data[:2] | ||
283 | super().set_taskdata(data[2:]) | ||
284 | |||
285 | def __get_task_unihash_key(self, task): | ||
286 | # TODO: The key only *needs* to be the taskhash, the task is just | ||
287 | # convenient | ||
288 | return '%s:%s' % (task, self.taskhash[task]) | ||
289 | |||
290 | def get_stampfile_hash(self, task): | ||
291 | if task in self.taskhash: | ||
292 | # If a unique hash is reported, use it as the stampfile hash. This | ||
293 | # ensures that if a task won't be re-run if the taskhash changes, | ||
294 | # but it would result in the same output hash | ||
295 | unihash = self.unihashes.get(self.__get_task_unihash_key(task)) | ||
296 | if unihash is not None: | ||
297 | return unihash | ||
298 | |||
299 | return super().get_stampfile_hash(task) | ||
300 | |||
301 | def get_unihash(self, task): | ||
302 | import urllib | ||
303 | import json | ||
304 | |||
305 | taskhash = self.taskhash[task] | ||
306 | |||
307 | key = self.__get_task_unihash_key(task) | ||
308 | |||
309 | # TODO: This cache can grow unbounded. It probably only needs to keep | ||
310 | # for each task | ||
311 | unihash = self.unihashes.get(key) | ||
312 | if unihash is not None: | ||
313 | return unihash | ||
314 | |||
315 | # In the absence of being able to discover a unique hash from the | ||
316 | # server, make it be equivalent to the taskhash. The unique "hash" only | ||
317 | # really needs to be a unique string (not even necessarily a hash), but | ||
318 | # making it match the taskhash has a few advantages: | ||
319 | # | ||
320 | # 1) All of the sstate code that assumes hashes can be the same | ||
321 | # 2) It provides maximal compatibility with builders that don't use | ||
322 | # an equivalency server | ||
323 | # 3) The value is easy for multiple independent builders to derive the | ||
324 | # same unique hash from the same input. This means that if the | ||
325 | # independent builders find the same taskhash, but it isn't reported | ||
326 | # to the server, there is a better chance that they will agree on | ||
327 | # the unique hash. | ||
328 | unihash = taskhash | ||
329 | |||
330 | try: | ||
331 | url = '%s/v1/equivalent?%s' % (self.server, | ||
332 | urllib.parse.urlencode({'method': self.method, 'taskhash': self.taskhash[task]})) | ||
333 | |||
334 | request = urllib.request.Request(url) | ||
335 | response = urllib.request.urlopen(request) | ||
336 | data = response.read().decode('utf-8') | ||
337 | |||
338 | json_data = json.loads(data) | ||
339 | |||
340 | if json_data: | ||
341 | unihash = json_data['unihash'] | ||
342 | # A unique hash equal to the taskhash is not very interesting, | ||
343 | # so it is reported it at debug level 2. If they differ, that | ||
344 | # is much more interesting, so it is reported at debug level 1 | ||
345 | bb.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, task, self.server)) | ||
346 | else: | ||
347 | bb.debug(2, 'No reported unihash for %s:%s from %s' % (task, taskhash, self.server)) | ||
348 | except urllib.error.URLError as e: | ||
349 | bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) | ||
350 | except (KeyError, json.JSONDecodeError) as e: | ||
351 | bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e))) | ||
352 | |||
353 | self.unihashes[key] = unihash | ||
354 | return unihash | ||
355 | |||
356 | def report_unihash(self, path, task, d): | ||
357 | import urllib | ||
358 | import json | ||
359 | import tempfile | ||
360 | import base64 | ||
361 | import importlib | ||
362 | |||
363 | taskhash = d.getVar('BB_TASKHASH') | ||
364 | unihash = d.getVar('BB_UNIHASH') | ||
365 | report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1' | ||
366 | tempdir = d.getVar('T') | ||
367 | fn = d.getVar('BB_FILENAME') | ||
368 | key = fn + '.do_' + task + ':' + taskhash | ||
369 | |||
370 | # Sanity checks | ||
371 | cache_unihash = self.unihashes.get(key) | ||
372 | if cache_unihash is None: | ||
373 | bb.fatal('%s not in unihash cache. Please report this error' % key) | ||
374 | |||
375 | if cache_unihash != unihash: | ||
376 | bb.fatal("Cache unihash %s doesn't match BB_UNIHASH %s" % (cache_unihash, unihash)) | ||
377 | |||
378 | sigfile = None | ||
379 | sigfile_name = "depsig.do_%s.%d" % (task, os.getpid()) | ||
380 | sigfile_link = "depsig.do_%s" % task | ||
381 | |||
382 | try: | ||
383 | sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b') | ||
384 | |||
385 | locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} | ||
386 | |||
387 | (module, method) = self.method.rsplit('.', 1) | ||
388 | locs['method'] = getattr(importlib.import_module(module), method) | ||
389 | |||
390 | outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs) | ||
391 | |||
392 | try: | ||
393 | url = '%s/v1/equivalent' % self.server | ||
394 | task_data = { | ||
395 | 'taskhash': taskhash, | ||
396 | 'method': self.method, | ||
397 | 'outhash': outhash, | ||
398 | 'unihash': unihash, | ||
399 | 'owner': d.getVar('SSTATE_HASHEQUIV_OWNER') | ||
400 | } | ||
401 | |||
402 | if report_taskdata: | ||
403 | sigfile.seek(0) | ||
404 | |||
405 | task_data['PN'] = d.getVar('PN') | ||
406 | task_data['PV'] = d.getVar('PV') | ||
407 | task_data['PR'] = d.getVar('PR') | ||
408 | task_data['task'] = task | ||
409 | task_data['outhash_siginfo'] = sigfile.read().decode('utf-8') | ||
410 | |||
411 | headers = {'content-type': 'application/json'} | ||
412 | |||
413 | request = urllib.request.Request(url, json.dumps(task_data).encode('utf-8'), headers) | ||
414 | response = urllib.request.urlopen(request) | ||
415 | data = response.read().decode('utf-8') | ||
416 | |||
417 | json_data = json.loads(data) | ||
418 | new_unihash = json_data['unihash'] | ||
419 | |||
420 | if new_unihash != unihash: | ||
421 | bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server)) | ||
422 | else: | ||
423 | bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server)) | ||
424 | except urllib.error.URLError as e: | ||
425 | bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) | ||
426 | except (KeyError, json.JSONDecodeError) as e: | ||
427 | bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e))) | ||
428 | finally: | ||
429 | if sigfile: | ||
430 | sigfile.close() | ||
431 | |||
432 | sigfile_link_path = os.path.join(tempdir, sigfile_link) | ||
433 | bb.utils.remove(sigfile_link_path) | ||
434 | |||
435 | try: | ||
436 | os.symlink(sigfile_name, sigfile_link_path) | ||
437 | except OSError: | ||
438 | pass | ||
439 | 278 | ||
440 | # Insert these classes into siggen's namespace so it can see and select them | 279 | # Insert these classes into siggen's namespace so it can see and select them |
441 | bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic | 280 | bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic |