diff options
Diffstat (limited to 'meta/lib/oe/sstatesig.py')
| -rw-r--r-- | meta/lib/oe/sstatesig.py | 163 |
1 files changed, 1 insertions, 162 deletions
diff --git a/meta/lib/oe/sstatesig.py b/meta/lib/oe/sstatesig.py index 13af16e473..35d48f328d 100644 --- a/meta/lib/oe/sstatesig.py +++ b/meta/lib/oe/sstatesig.py | |||
| @@ -266,7 +266,7 @@ class SignatureGeneratorOEBasicHash(bb.siggen.SignatureGeneratorBasicHash): | |||
| 266 | if error_msgs: | 266 | if error_msgs: |
| 267 | bb.fatal("\n".join(error_msgs)) | 267 | bb.fatal("\n".join(error_msgs)) |
| 268 | 268 | ||
| 269 | class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): | 269 | class SignatureGeneratorOEEquivHash(bb.siggen.SignatureGeneratorUniHashMixIn, SignatureGeneratorOEBasicHash): |
| 270 | name = "OEEquivHash" | 270 | name = "OEEquivHash" |
| 271 | 271 | ||
| 272 | def init_rundepcheck(self, data): | 272 | def init_rundepcheck(self, data): |
| @@ -275,167 +275,6 @@ class SignatureGeneratorOEEquivHash(SignatureGeneratorOEBasicHash): | |||
| 275 | self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') | 275 | self.method = data.getVar('SSTATE_HASHEQUIV_METHOD') |
| 276 | self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data) | 276 | self.unihashes = bb.persist_data.persist('SSTATESIG_UNIHASH_CACHE_v1_' + self.method.replace('.', '_'), data) |
| 277 | 277 | ||
| 278 | def get_taskdata(self): | ||
| 279 | return (self.server, self.method) + super().get_taskdata() | ||
| 280 | |||
| 281 | def set_taskdata(self, data): | ||
| 282 | self.server, self.method = data[:2] | ||
| 283 | super().set_taskdata(data[2:]) | ||
| 284 | |||
| 285 | def __get_task_unihash_key(self, task): | ||
| 286 | # TODO: The key only *needs* to be the taskhash, the task is just | ||
| 287 | # convenient | ||
| 288 | return '%s:%s' % (task, self.taskhash[task]) | ||
| 289 | |||
| 290 | def get_stampfile_hash(self, task): | ||
| 291 | if task in self.taskhash: | ||
| 292 | # If a unique hash is reported, use it as the stampfile hash. This | ||
| 293 | # ensures that if a task won't be re-run if the taskhash changes, | ||
| 294 | # but it would result in the same output hash | ||
| 295 | unihash = self.unihashes.get(self.__get_task_unihash_key(task)) | ||
| 296 | if unihash is not None: | ||
| 297 | return unihash | ||
| 298 | |||
| 299 | return super().get_stampfile_hash(task) | ||
| 300 | |||
| 301 | def get_unihash(self, task): | ||
| 302 | import urllib | ||
| 303 | import json | ||
| 304 | |||
| 305 | taskhash = self.taskhash[task] | ||
| 306 | |||
| 307 | key = self.__get_task_unihash_key(task) | ||
| 308 | |||
| 309 | # TODO: This cache can grow unbounded. It probably only needs to keep | ||
| 310 | # for each task | ||
| 311 | unihash = self.unihashes.get(key) | ||
| 312 | if unihash is not None: | ||
| 313 | return unihash | ||
| 314 | |||
| 315 | # In the absence of being able to discover a unique hash from the | ||
| 316 | # server, make it be equivalent to the taskhash. The unique "hash" only | ||
| 317 | # really needs to be a unique string (not even necessarily a hash), but | ||
| 318 | # making it match the taskhash has a few advantages: | ||
| 319 | # | ||
| 320 | # 1) All of the sstate code that assumes hashes can be the same | ||
| 321 | # 2) It provides maximal compatibility with builders that don't use | ||
| 322 | # an equivalency server | ||
| 323 | # 3) The value is easy for multiple independent builders to derive the | ||
| 324 | # same unique hash from the same input. This means that if the | ||
| 325 | # independent builders find the same taskhash, but it isn't reported | ||
| 326 | # to the server, there is a better chance that they will agree on | ||
| 327 | # the unique hash. | ||
| 328 | unihash = taskhash | ||
| 329 | |||
| 330 | try: | ||
| 331 | url = '%s/v1/equivalent?%s' % (self.server, | ||
| 332 | urllib.parse.urlencode({'method': self.method, 'taskhash': self.taskhash[task]})) | ||
| 333 | |||
| 334 | request = urllib.request.Request(url) | ||
| 335 | response = urllib.request.urlopen(request) | ||
| 336 | data = response.read().decode('utf-8') | ||
| 337 | |||
| 338 | json_data = json.loads(data) | ||
| 339 | |||
| 340 | if json_data: | ||
| 341 | unihash = json_data['unihash'] | ||
| 342 | # A unique hash equal to the taskhash is not very interesting, | ||
| 343 | # so it is reported it at debug level 2. If they differ, that | ||
| 344 | # is much more interesting, so it is reported at debug level 1 | ||
| 345 | bb.debug((1, 2)[unihash == taskhash], 'Found unihash %s in place of %s for %s from %s' % (unihash, taskhash, task, self.server)) | ||
| 346 | else: | ||
| 347 | bb.debug(2, 'No reported unihash for %s:%s from %s' % (task, taskhash, self.server)) | ||
| 348 | except urllib.error.URLError as e: | ||
| 349 | bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) | ||
| 350 | except (KeyError, json.JSONDecodeError) as e: | ||
| 351 | bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e))) | ||
| 352 | |||
| 353 | self.unihashes[key] = unihash | ||
| 354 | return unihash | ||
| 355 | |||
| 356 | def report_unihash(self, path, task, d): | ||
| 357 | import urllib | ||
| 358 | import json | ||
| 359 | import tempfile | ||
| 360 | import base64 | ||
| 361 | import importlib | ||
| 362 | |||
| 363 | taskhash = d.getVar('BB_TASKHASH') | ||
| 364 | unihash = d.getVar('BB_UNIHASH') | ||
| 365 | report_taskdata = d.getVar('SSTATE_HASHEQUIV_REPORT_TASKDATA') == '1' | ||
| 366 | tempdir = d.getVar('T') | ||
| 367 | fn = d.getVar('BB_FILENAME') | ||
| 368 | key = fn + '.do_' + task + ':' + taskhash | ||
| 369 | |||
| 370 | # Sanity checks | ||
| 371 | cache_unihash = self.unihashes.get(key) | ||
| 372 | if cache_unihash is None: | ||
| 373 | bb.fatal('%s not in unihash cache. Please report this error' % key) | ||
| 374 | |||
| 375 | if cache_unihash != unihash: | ||
| 376 | bb.fatal("Cache unihash %s doesn't match BB_UNIHASH %s" % (cache_unihash, unihash)) | ||
| 377 | |||
| 378 | sigfile = None | ||
| 379 | sigfile_name = "depsig.do_%s.%d" % (task, os.getpid()) | ||
| 380 | sigfile_link = "depsig.do_%s" % task | ||
| 381 | |||
| 382 | try: | ||
| 383 | sigfile = open(os.path.join(tempdir, sigfile_name), 'w+b') | ||
| 384 | |||
| 385 | locs = {'path': path, 'sigfile': sigfile, 'task': task, 'd': d} | ||
| 386 | |||
| 387 | (module, method) = self.method.rsplit('.', 1) | ||
| 388 | locs['method'] = getattr(importlib.import_module(module), method) | ||
| 389 | |||
| 390 | outhash = bb.utils.better_eval('method(path, sigfile, task, d)', locs) | ||
| 391 | |||
| 392 | try: | ||
| 393 | url = '%s/v1/equivalent' % self.server | ||
| 394 | task_data = { | ||
| 395 | 'taskhash': taskhash, | ||
| 396 | 'method': self.method, | ||
| 397 | 'outhash': outhash, | ||
| 398 | 'unihash': unihash, | ||
| 399 | 'owner': d.getVar('SSTATE_HASHEQUIV_OWNER') | ||
| 400 | } | ||
| 401 | |||
| 402 | if report_taskdata: | ||
| 403 | sigfile.seek(0) | ||
| 404 | |||
| 405 | task_data['PN'] = d.getVar('PN') | ||
| 406 | task_data['PV'] = d.getVar('PV') | ||
| 407 | task_data['PR'] = d.getVar('PR') | ||
| 408 | task_data['task'] = task | ||
| 409 | task_data['outhash_siginfo'] = sigfile.read().decode('utf-8') | ||
| 410 | |||
| 411 | headers = {'content-type': 'application/json'} | ||
| 412 | |||
| 413 | request = urllib.request.Request(url, json.dumps(task_data).encode('utf-8'), headers) | ||
| 414 | response = urllib.request.urlopen(request) | ||
| 415 | data = response.read().decode('utf-8') | ||
| 416 | |||
| 417 | json_data = json.loads(data) | ||
| 418 | new_unihash = json_data['unihash'] | ||
| 419 | |||
| 420 | if new_unihash != unihash: | ||
| 421 | bb.debug(1, 'Task %s unihash changed %s -> %s by server %s' % (taskhash, unihash, new_unihash, self.server)) | ||
| 422 | else: | ||
| 423 | bb.debug(1, 'Reported task %s as unihash %s to %s' % (taskhash, unihash, self.server)) | ||
| 424 | except urllib.error.URLError as e: | ||
| 425 | bb.warn('Failure contacting Hash Equivalence Server %s: %s' % (self.server, str(e))) | ||
| 426 | except (KeyError, json.JSONDecodeError) as e: | ||
| 427 | bb.warn('Poorly formatted response from %s: %s' % (self.server, str(e))) | ||
| 428 | finally: | ||
| 429 | if sigfile: | ||
| 430 | sigfile.close() | ||
| 431 | |||
| 432 | sigfile_link_path = os.path.join(tempdir, sigfile_link) | ||
| 433 | bb.utils.remove(sigfile_link_path) | ||
| 434 | |||
| 435 | try: | ||
| 436 | os.symlink(sigfile_name, sigfile_link_path) | ||
| 437 | except OSError: | ||
| 438 | pass | ||
| 439 | 278 | ||
| 440 | # Insert these classes into siggen's namespace so it can see and select them | 279 | # Insert these classes into siggen's namespace so it can see and select them |
| 441 | bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic | 280 | bb.siggen.SignatureGeneratorOEBasic = SignatureGeneratorOEBasic |
