diff options
author | Richard Purdie <richard.purdie@linuxfoundation.org> | 2020-03-23 12:43:25 +0000 |
---|---|---|
committer | Richard Purdie <richard.purdie@linuxfoundation.org> | 2020-03-24 22:01:03 +0000 |
commit | d53958766e2c766b95b3947f69b64c409ed433d8 (patch) | |
tree | 45e943ca7b1cce938d730cf17cdbb4e472082258 /bitbake/lib/bb | |
parent | 89705a60f3b566743703c3a7597ccc4507293dd9 (diff) | |
download | poky-d53958766e2c766b95b3947f69b64c409ed433d8.tar.gz |
bitbake: tinfoil: Simplify remote datastore connections
The current approach to remote datastores used in tinfoil is breaking. For
example, adding a devupstream extension to a recipe with a git upstream,
making it the preferred version and then running "devtool modify" on it
causes get_srcrev() circular dependency issues. The problem is the override
handling in the datastore is broken.
This gets broken since remotedata:recieve_datastore() sets d.dict but doesn't
update d.overridedata (or d.inchistory or d.varhistory). We could play
whack-a-mole but the current implementation seems to be flawed to me. It
also doesn't cover, or only partially covers some datastore operations and
each needs new dedicated command API.
Instead, step back and reimplement the way the datastore connector works.
With this change, the datastore is either remote or local but the data is not
spread on two sides of the connection. All the API is proxied over the connection
by a single function for the datastore (and two to support variable history
and include history).
This code does not support using the datastore as a parameter to any data store
functions. We did have one case of that but its just bad code and can be
replaced.
The result is something which is much simpler and less invasive to the datastore
code itself, meaning its behaviour should be much more consistent. The existing
tests for the remote data no longer make any sense and are removed.
The one bug this code would have is if key/value pairs are returned over the IPC
and those values contained a DataSmart object since we don't recurse into return
values to find such things. Nothing appears to do that currently so lets worry
about it if its ever an issue. This change should simplfy a ton of other issues
and avoid a ton of other bugs so is a huge net gain.
Tested with bitbake's and OE's selftests.
(Bitbake rev: 85e03a64dd0a4ebe71009ec4bdf4192c04a9786e)
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'bitbake/lib/bb')
-rw-r--r-- | bitbake/lib/bb/command.py | 91 | ||||
-rw-r--r-- | bitbake/lib/bb/data_smart.py | 53 | ||||
-rw-r--r-- | bitbake/lib/bb/remotedata.py | 49 | ||||
-rw-r--r-- | bitbake/lib/bb/tests/data.py | 139 | ||||
-rw-r--r-- | bitbake/lib/bb/tinfoil.py | 115 |
5 files changed, 87 insertions, 360 deletions
diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py index b38c151b3d..2879950933 100644 --- a/bitbake/lib/bb/command.py +++ b/bitbake/lib/bb/command.py | |||
@@ -450,54 +450,38 @@ class CommandsSync: | |||
450 | return all_p, best | 450 | return all_p, best |
451 | getRuntimeProviders.readonly = True | 451 | getRuntimeProviders.readonly = True |
452 | 452 | ||
453 | def dataStoreConnectorFindVar(self, command, params): | 453 | def dataStoreConnectorCmd(self, command, params): |
454 | dsindex = params[0] | 454 | dsindex = params[0] |
455 | name = params[1] | 455 | method = params[1] |
456 | datastore = command.remotedatastores[dsindex] | 456 | args = params[2] |
457 | value, overridedata = datastore._findVar(name) | 457 | kwargs = params[3] |
458 | |||
459 | if value: | ||
460 | content = value.get('_content', None) | ||
461 | if isinstance(content, bb.data_smart.DataSmart): | ||
462 | # Value is a datastore (e.g. BB_ORIGENV) - need to handle this carefully | ||
463 | idx = command.remotedatastores.check_store(content, True) | ||
464 | return {'_content': DataStoreConnectionHandle(idx), | ||
465 | '_connector_origtype': 'DataStoreConnectionHandle', | ||
466 | '_connector_overrides': overridedata} | ||
467 | elif isinstance(content, set): | ||
468 | return {'_content': list(content), | ||
469 | '_connector_origtype': 'set', | ||
470 | '_connector_overrides': overridedata} | ||
471 | else: | ||
472 | value['_connector_overrides'] = overridedata | ||
473 | else: | ||
474 | value = {} | ||
475 | value['_connector_overrides'] = overridedata | ||
476 | return value | ||
477 | dataStoreConnectorFindVar.readonly = True | ||
478 | 458 | ||
479 | def dataStoreConnectorGetKeys(self, command, params): | 459 | d = command.remotedatastores[dsindex] |
480 | dsindex = params[0] | 460 | ret = getattr(d, method)(*args, **kwargs) |
481 | datastore = command.remotedatastores[dsindex] | 461 | |
482 | return list(datastore.keys()) | 462 | if isinstance(ret, bb.data_smart.DataSmart): |
483 | dataStoreConnectorGetKeys.readonly = True | 463 | idx = command.remotedatastores.store(ret) |
464 | return DataStoreConnectionHandle(idx) | ||
484 | 465 | ||
485 | def dataStoreConnectorGetVarHistory(self, command, params): | 466 | return ret |
467 | |||
468 | def dataStoreConnectorVarHistCmd(self, command, params): | ||
486 | dsindex = params[0] | 469 | dsindex = params[0] |
487 | name = params[1] | 470 | method = params[1] |
488 | datastore = command.remotedatastores[dsindex] | 471 | args = params[2] |
489 | return datastore.varhistory.variable(name) | 472 | kwargs = params[3] |
490 | dataStoreConnectorGetVarHistory.readonly = True | ||
491 | 473 | ||
492 | def dataStoreConnectorExpandPythonRef(self, command, params): | 474 | d = command.remotedatastores[dsindex].varhistory |
493 | config_data_dict = params[0] | 475 | return getattr(d, method)(*args, **kwargs) |
494 | varname = params[1] | ||
495 | expr = params[2] | ||
496 | 476 | ||
497 | config_data = command.remotedatastores.receive_datastore(config_data_dict) | 477 | def dataStoreConnectorIncHistCmd(self, command, params): |
478 | dsindex = params[0] | ||
479 | method = params[1] | ||
480 | args = params[2] | ||
481 | kwargs = params[3] | ||
498 | 482 | ||
499 | varparse = bb.data_smart.VariableParse(varname, config_data) | 483 | d = command.remotedatastores[dsindex].inchistory |
500 | return varparse.python_sub(expr) | 484 | return getattr(d, method)(*args, **kwargs) |
501 | 485 | ||
502 | def dataStoreConnectorRelease(self, command, params): | 486 | def dataStoreConnectorRelease(self, command, params): |
503 | dsindex = params[0] | 487 | dsindex = params[0] |
@@ -505,31 +489,6 @@ class CommandsSync: | |||
505 | raise CommandError('dataStoreConnectorRelease: invalid index %d' % dsindex) | 489 | raise CommandError('dataStoreConnectorRelease: invalid index %d' % dsindex) |
506 | command.remotedatastores.release(dsindex) | 490 | command.remotedatastores.release(dsindex) |
507 | 491 | ||
508 | def dataStoreConnectorSetVarFlag(self, command, params): | ||
509 | dsindex = params[0] | ||
510 | name = params[1] | ||
511 | flag = params[2] | ||
512 | value = params[3] | ||
513 | datastore = command.remotedatastores[dsindex] | ||
514 | datastore.setVarFlag(name, flag, value) | ||
515 | |||
516 | def dataStoreConnectorDelVar(self, command, params): | ||
517 | dsindex = params[0] | ||
518 | name = params[1] | ||
519 | datastore = command.remotedatastores[dsindex] | ||
520 | if len(params) > 2: | ||
521 | flag = params[2] | ||
522 | datastore.delVarFlag(name, flag) | ||
523 | else: | ||
524 | datastore.delVar(name) | ||
525 | |||
526 | def dataStoreConnectorRenameVar(self, command, params): | ||
527 | dsindex = params[0] | ||
528 | name = params[1] | ||
529 | newname = params[2] | ||
530 | datastore = command.remotedatastores[dsindex] | ||
531 | datastore.renameVar(name, newname) | ||
532 | |||
533 | def parseRecipeFile(self, command, params): | 492 | def parseRecipeFile(self, command, params): |
534 | """ | 493 | """ |
535 | Parse the specified recipe file (with or without bbappends) | 494 | Parse the specified recipe file (with or without bbappends) |
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py index b2dc9d9fd5..70257ab7f8 100644 --- a/bitbake/lib/bb/data_smart.py +++ b/bitbake/lib/bb/data_smart.py | |||
@@ -107,10 +107,6 @@ class VariableParse: | |||
107 | else: | 107 | else: |
108 | code = match.group()[3:-1] | 108 | code = match.group()[3:-1] |
109 | 109 | ||
110 | if "_remote_data" in self.d: | ||
111 | connector = self.d["_remote_data"] | ||
112 | return connector.expandPythonRef(self.varname, code, self.d) | ||
113 | |||
114 | if self.varname: | 110 | if self.varname: |
115 | varname = 'Var <%s>' % self.varname | 111 | varname = 'Var <%s>' % self.varname |
116 | else: | 112 | else: |
@@ -268,12 +264,7 @@ class VariableHistory(object): | |||
268 | self.variables[newvar].append(i.copy()) | 264 | self.variables[newvar].append(i.copy()) |
269 | 265 | ||
270 | def variable(self, var): | 266 | def variable(self, var): |
271 | remote_connector = self.dataroot.getVar('_remote_data', False) | 267 | varhistory = [] |
272 | if remote_connector: | ||
273 | varhistory = remote_connector.getVarHistory(var) | ||
274 | else: | ||
275 | varhistory = [] | ||
276 | |||
277 | if var in self.variables: | 268 | if var in self.variables: |
278 | varhistory.extend(self.variables[var]) | 269 | varhistory.extend(self.variables[var]) |
279 | return varhistory | 270 | return varhistory |
@@ -471,10 +462,6 @@ class DataSmart(MutableMapping): | |||
471 | if var in dest: | 462 | if var in dest: |
472 | return dest[var], self.overridedata.get(var, None) | 463 | return dest[var], self.overridedata.get(var, None) |
473 | 464 | ||
474 | if "_remote_data" in dest: | ||
475 | connector = dest["_remote_data"]["_content"] | ||
476 | return connector.getVar(var) | ||
477 | |||
478 | if "_data" not in dest: | 465 | if "_data" not in dest: |
479 | break | 466 | break |
480 | dest = dest["_data"] | 467 | dest = dest["_data"] |
@@ -499,12 +486,6 @@ class DataSmart(MutableMapping): | |||
499 | if 'parsing' in loginfo: | 486 | if 'parsing' in loginfo: |
500 | parsing=True | 487 | parsing=True |
501 | 488 | ||
502 | if '_remote_data' in self.dict: | ||
503 | connector = self.dict["_remote_data"]["_content"] | ||
504 | res = connector.setVar(var, value) | ||
505 | if not res: | ||
506 | return | ||
507 | |||
508 | if 'op' not in loginfo: | 489 | if 'op' not in loginfo: |
509 | loginfo['op'] = "set" | 490 | loginfo['op'] = "set" |
510 | 491 | ||
@@ -612,12 +593,6 @@ class DataSmart(MutableMapping): | |||
612 | bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key) | 593 | bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key) |
613 | return | 594 | return |
614 | 595 | ||
615 | if '_remote_data' in self.dict: | ||
616 | connector = self.dict["_remote_data"]["_content"] | ||
617 | res = connector.renameVar(key, newkey) | ||
618 | if not res: | ||
619 | return | ||
620 | |||
621 | val = self.getVar(key, 0, parsing=True) | 596 | val = self.getVar(key, 0, parsing=True) |
622 | if val is not None: | 597 | if val is not None: |
623 | self.varhistory.rename_variable_hist(key, newkey) | 598 | self.varhistory.rename_variable_hist(key, newkey) |
@@ -663,11 +638,6 @@ class DataSmart(MutableMapping): | |||
663 | 638 | ||
664 | def delVar(self, var, **loginfo): | 639 | def delVar(self, var, **loginfo): |
665 | self.expand_cache = {} | 640 | self.expand_cache = {} |
666 | if '_remote_data' in self.dict: | ||
667 | connector = self.dict["_remote_data"]["_content"] | ||
668 | res = connector.delVar(var) | ||
669 | if not res: | ||
670 | return | ||
671 | 641 | ||
672 | loginfo['detail'] = "" | 642 | loginfo['detail'] = "" |
673 | loginfo['op'] = 'del' | 643 | loginfo['op'] = 'del' |
@@ -695,11 +665,6 @@ class DataSmart(MutableMapping): | |||
695 | 665 | ||
696 | def setVarFlag(self, var, flag, value, **loginfo): | 666 | def setVarFlag(self, var, flag, value, **loginfo): |
697 | self.expand_cache = {} | 667 | self.expand_cache = {} |
698 | if '_remote_data' in self.dict: | ||
699 | connector = self.dict["_remote_data"]["_content"] | ||
700 | res = connector.setVarFlag(var, flag, value) | ||
701 | if not res: | ||
702 | return | ||
703 | 668 | ||
704 | if 'op' not in loginfo: | 669 | if 'op' not in loginfo: |
705 | loginfo['op'] = "set" | 670 | loginfo['op'] = "set" |
@@ -850,11 +815,6 @@ class DataSmart(MutableMapping): | |||
850 | 815 | ||
851 | def delVarFlag(self, var, flag, **loginfo): | 816 | def delVarFlag(self, var, flag, **loginfo): |
852 | self.expand_cache = {} | 817 | self.expand_cache = {} |
853 | if '_remote_data' in self.dict: | ||
854 | connector = self.dict["_remote_data"]["_content"] | ||
855 | res = connector.delVarFlag(var, flag) | ||
856 | if not res: | ||
857 | return | ||
858 | 818 | ||
859 | local_var, _ = self._findVar(var) | 819 | local_var, _ = self._findVar(var) |
860 | if not local_var: | 820 | if not local_var: |
@@ -972,7 +932,7 @@ class DataSmart(MutableMapping): | |||
972 | 932 | ||
973 | def localkeys(self): | 933 | def localkeys(self): |
974 | for key in self.dict: | 934 | for key in self.dict: |
975 | if key not in ['_data', '_remote_data']: | 935 | if key not in ['_data']: |
976 | yield key | 936 | yield key |
977 | 937 | ||
978 | def __iter__(self): | 938 | def __iter__(self): |
@@ -981,7 +941,7 @@ class DataSmart(MutableMapping): | |||
981 | def keylist(d): | 941 | def keylist(d): |
982 | klist = set() | 942 | klist = set() |
983 | for key in d: | 943 | for key in d: |
984 | if key in ["_data", "_remote_data"]: | 944 | if key in ["_data"]: |
985 | continue | 945 | continue |
986 | if key in deleted: | 946 | if key in deleted: |
987 | continue | 947 | continue |
@@ -995,13 +955,6 @@ class DataSmart(MutableMapping): | |||
995 | if "_data" in d: | 955 | if "_data" in d: |
996 | klist |= keylist(d["_data"]) | 956 | klist |= keylist(d["_data"]) |
997 | 957 | ||
998 | if "_remote_data" in d: | ||
999 | connector = d["_remote_data"]["_content"] | ||
1000 | for key in connector.getKeys(): | ||
1001 | if key in deleted: | ||
1002 | continue | ||
1003 | klist.add(key) | ||
1004 | |||
1005 | return klist | 958 | return klist |
1006 | 959 | ||
1007 | self.need_overrides() | 960 | self.need_overrides() |
diff --git a/bitbake/lib/bb/remotedata.py b/bitbake/lib/bb/remotedata.py index 7391e1b45c..6c9864dd6b 100644 --- a/bitbake/lib/bb/remotedata.py +++ b/bitbake/lib/bb/remotedata.py | |||
@@ -17,16 +17,16 @@ class RemoteDatastores: | |||
17 | self.cooker = cooker | 17 | self.cooker = cooker |
18 | self.datastores = {} | 18 | self.datastores = {} |
19 | self.locked = [] | 19 | self.locked = [] |
20 | self.datastores[0] = self.cooker.data | ||
20 | self.nextindex = 1 | 21 | self.nextindex = 1 |
21 | 22 | ||
22 | def __len__(self): | 23 | def __len__(self): |
23 | return len(self.datastores) | 24 | return len(self.datastores) |
24 | 25 | ||
25 | def __getitem__(self, key): | 26 | def __getitem__(self, key): |
26 | if key is None: | 27 | # Cooker could have changed its datastore from under us |
27 | return self.cooker.data | 28 | self.datastores[0] = self.cooker.data |
28 | else: | 29 | return self.datastores[key] |
29 | return self.datastores[key] | ||
30 | 30 | ||
31 | def items(self): | 31 | def items(self): |
32 | return self.datastores.items() | 32 | return self.datastores.items() |
@@ -63,44 +63,3 @@ class RemoteDatastores: | |||
63 | raise Exception('Tried to release locked datastore %d' % idx) | 63 | raise Exception('Tried to release locked datastore %d' % idx) |
64 | del self.datastores[idx] | 64 | del self.datastores[idx] |
65 | 65 | ||
66 | def receive_datastore(self, remote_data): | ||
67 | """Receive a datastore object sent from the client (as prepared by transmit_datastore())""" | ||
68 | dct = dict(remote_data) | ||
69 | d = bb.data_smart.DataSmart() | ||
70 | d.dict = dct | ||
71 | while True: | ||
72 | if '_remote_data' in dct: | ||
73 | dsindex = dct['_remote_data']['_content'] | ||
74 | del dct['_remote_data'] | ||
75 | if dsindex is None: | ||
76 | dct['_data'] = self.cooker.data.dict | ||
77 | else: | ||
78 | dct['_data'] = self.datastores[dsindex].dict | ||
79 | break | ||
80 | elif '_data' in dct: | ||
81 | idct = dict(dct['_data']) | ||
82 | dct['_data'] = idct | ||
83 | dct = idct | ||
84 | else: | ||
85 | break | ||
86 | return d | ||
87 | |||
88 | @staticmethod | ||
89 | def transmit_datastore(d): | ||
90 | """Prepare a datastore object for sending over IPC from the client end""" | ||
91 | # FIXME content might be a dict, need to turn that into a list as well | ||
92 | def copy_dicts(dct): | ||
93 | if '_remote_data' in dct: | ||
94 | dsindex = dct['_remote_data']['_content'].dsindex | ||
95 | newdct = dct.copy() | ||
96 | newdct['_remote_data'] = {'_content': dsindex} | ||
97 | return list(newdct.items()) | ||
98 | elif '_data' in dct: | ||
99 | newdct = dct.copy() | ||
100 | newdata = copy_dicts(dct['_data']) | ||
101 | if newdata: | ||
102 | newdct['_data'] = newdata | ||
103 | return list(newdct.items()) | ||
104 | return None | ||
105 | main_dict = copy_dicts(d.dict) | ||
106 | return main_dict | ||
diff --git a/bitbake/lib/bb/tests/data.py b/bitbake/lib/bb/tests/data.py index 3e49984c93..2b137706dd 100644 --- a/bitbake/lib/bb/tests/data.py +++ b/bitbake/lib/bb/tests/data.py | |||
@@ -544,142 +544,3 @@ class Serialize(unittest.TestCase): | |||
544 | self.assertEqual(newd.getVarFlag('HELLO', 'other'), 'planet') | 544 | self.assertEqual(newd.getVarFlag('HELLO', 'other'), 'planet') |
545 | 545 | ||
546 | 546 | ||
547 | # Remote datastore tests | ||
548 | # These really only test the interface, since in actual usage we have a | ||
549 | # tinfoil connector that does everything over RPC, and this doesn't test | ||
550 | # that. | ||
551 | |||
552 | class TestConnector: | ||
553 | d = None | ||
554 | def __init__(self, d): | ||
555 | self.d = d | ||
556 | def getVar(self, name): | ||
557 | return self.d._findVar(name) | ||
558 | def getKeys(self): | ||
559 | return set(self.d.keys()) | ||
560 | def getVarHistory(self, name): | ||
561 | return self.d.varhistory.variable(name) | ||
562 | def expandPythonRef(self, varname, expr, d): | ||
563 | localdata = self.d.createCopy() | ||
564 | for key in d.localkeys(): | ||
565 | localdata.setVar(d.getVar(key)) | ||
566 | varparse = bb.data_smart.VariableParse(varname, localdata) | ||
567 | return varparse.python_sub(expr) | ||
568 | def setVar(self, name, value): | ||
569 | self.d.setVar(name, value) | ||
570 | def setVarFlag(self, name, flag, value): | ||
571 | self.d.setVarFlag(name, flag, value) | ||
572 | def delVar(self, name): | ||
573 | self.d.delVar(name) | ||
574 | return False | ||
575 | def delVarFlag(self, name, flag): | ||
576 | self.d.delVarFlag(name, flag) | ||
577 | return False | ||
578 | def renameVar(self, name, newname): | ||
579 | self.d.renameVar(name, newname) | ||
580 | return False | ||
581 | |||
582 | class Remote(unittest.TestCase): | ||
583 | def test_remote(self): | ||
584 | |||
585 | d1 = bb.data.init() | ||
586 | d1.enableTracking() | ||
587 | d2 = bb.data.init() | ||
588 | d2.enableTracking() | ||
589 | connector = TestConnector(d1) | ||
590 | |||
591 | d2.setVar('_remote_data', connector) | ||
592 | |||
593 | d1.setVar('HELLO', 'world') | ||
594 | d1.setVarFlag('OTHER', 'flagname', 'flagvalue') | ||
595 | self.assertEqual(d2.getVar('HELLO'), 'world') | ||
596 | self.assertEqual(d2.expand('${HELLO}'), 'world') | ||
597 | self.assertEqual(d2.expand('${@d.getVar("HELLO")}'), 'world') | ||
598 | self.assertIn('flagname', d2.getVarFlags('OTHER')) | ||
599 | self.assertEqual(d2.getVarFlag('OTHER', 'flagname'), 'flagvalue') | ||
600 | self.assertEqual(d1.varhistory.variable('HELLO'), d2.varhistory.variable('HELLO')) | ||
601 | # Test setVar on client side affects server | ||
602 | d2.setVar('HELLO', 'other-world') | ||
603 | self.assertEqual(d1.getVar('HELLO'), 'other-world') | ||
604 | # Test setVarFlag on client side affects server | ||
605 | d2.setVarFlag('HELLO', 'flagname', 'flagvalue') | ||
606 | self.assertEqual(d1.getVarFlag('HELLO', 'flagname'), 'flagvalue') | ||
607 | # Test client side data is incorporated in python expansion (which is done on server) | ||
608 | d2.setVar('FOO', 'bar') | ||
609 | self.assertEqual(d2.expand('${@d.getVar("FOO")}'), 'bar') | ||
610 | # Test overrides work | ||
611 | d1.setVar('FOO_test', 'baz') | ||
612 | d1.appendVar('OVERRIDES', ':test') | ||
613 | self.assertEqual(d2.getVar('FOO'), 'baz') | ||
614 | |||
615 | |||
616 | # Remote equivalents of local test classes | ||
617 | # Note that these aren't perfect since we only test in one direction | ||
618 | |||
619 | class RemoteDataExpansions(DataExpansions): | ||
620 | def setUp(self): | ||
621 | self.d1 = bb.data.init() | ||
622 | self.d = bb.data.init() | ||
623 | self.d1["foo"] = "value_of_foo" | ||
624 | self.d1["bar"] = "value_of_bar" | ||
625 | self.d1["value_of_foo"] = "value_of_'value_of_foo'" | ||
626 | connector = TestConnector(self.d1) | ||
627 | self.d.setVar('_remote_data', connector) | ||
628 | |||
629 | class TestRemoteNestedExpansions(TestNestedExpansions): | ||
630 | def setUp(self): | ||
631 | self.d1 = bb.data.init() | ||
632 | self.d = bb.data.init() | ||
633 | self.d1["foo"] = "foo" | ||
634 | self.d1["bar"] = "bar" | ||
635 | self.d1["value_of_foobar"] = "187" | ||
636 | connector = TestConnector(self.d1) | ||
637 | self.d.setVar('_remote_data', connector) | ||
638 | |||
639 | class TestRemoteConcat(TestConcat): | ||
640 | def setUp(self): | ||
641 | self.d1 = bb.data.init() | ||
642 | self.d = bb.data.init() | ||
643 | self.d1.setVar("FOO", "foo") | ||
644 | self.d1.setVar("VAL", "val") | ||
645 | self.d1.setVar("BAR", "bar") | ||
646 | connector = TestConnector(self.d1) | ||
647 | self.d.setVar('_remote_data', connector) | ||
648 | |||
649 | class TestRemoteConcatOverride(TestConcatOverride): | ||
650 | def setUp(self): | ||
651 | self.d1 = bb.data.init() | ||
652 | self.d = bb.data.init() | ||
653 | self.d1.setVar("FOO", "foo") | ||
654 | self.d1.setVar("VAL", "val") | ||
655 | self.d1.setVar("BAR", "bar") | ||
656 | connector = TestConnector(self.d1) | ||
657 | self.d.setVar('_remote_data', connector) | ||
658 | |||
659 | class TestRemoteOverrides(TestOverrides): | ||
660 | def setUp(self): | ||
661 | self.d1 = bb.data.init() | ||
662 | self.d = bb.data.init() | ||
663 | self.d1.setVar("OVERRIDES", "foo:bar:local") | ||
664 | self.d1.setVar("TEST", "testvalue") | ||
665 | connector = TestConnector(self.d1) | ||
666 | self.d.setVar('_remote_data', connector) | ||
667 | |||
668 | class TestRemoteKeyExpansion(TestKeyExpansion): | ||
669 | def setUp(self): | ||
670 | self.d1 = bb.data.init() | ||
671 | self.d = bb.data.init() | ||
672 | self.d1.setVar("FOO", "foo") | ||
673 | self.d1.setVar("BAR", "foo") | ||
674 | connector = TestConnector(self.d1) | ||
675 | self.d.setVar('_remote_data', connector) | ||
676 | |||
677 | class TestRemoteFlags(TestFlags): | ||
678 | def setUp(self): | ||
679 | self.d1 = bb.data.init() | ||
680 | self.d = bb.data.init() | ||
681 | self.d1.setVar("foo", "value of foo") | ||
682 | self.d1.setVarFlag("foo", "flag1", "value of flag1") | ||
683 | self.d1.setVarFlag("foo", "flag2", "value of flag2") | ||
684 | connector = TestConnector(self.d1) | ||
685 | self.d.setVar('_remote_data', connector) | ||
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py index 70b381e351..4fbad0774e 100644 --- a/bitbake/lib/bb/tinfoil.py +++ b/bitbake/lib/bb/tinfoil.py | |||
@@ -13,6 +13,7 @@ import sys | |||
13 | import atexit | 13 | import atexit |
14 | import re | 14 | import re |
15 | from collections import OrderedDict, defaultdict | 15 | from collections import OrderedDict, defaultdict |
16 | from functools import partial | ||
16 | 17 | ||
17 | import bb.cache | 18 | import bb.cache |
18 | import bb.cooker | 19 | import bb.cooker |
@@ -44,66 +45,64 @@ class TinfoilUIException(Exception): | |||
44 | class TinfoilCommandFailed(Exception): | 45 | class TinfoilCommandFailed(Exception): |
45 | """Exception raised when run_command fails""" | 46 | """Exception raised when run_command fails""" |
46 | 47 | ||
48 | class TinfoilDataStoreConnectorVarHistory: | ||
49 | def __init__(self, tinfoil, dsindex): | ||
50 | self.tinfoil = tinfoil | ||
51 | self.dsindex = dsindex | ||
52 | |||
53 | def remoteCommand(self, cmd, *args, **kwargs): | ||
54 | return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs) | ||
55 | |||
56 | def __getattr__(self, name): | ||
57 | if not hasattr(bb.data_smart.VariableHistory, name): | ||
58 | raise AttributeError("VariableHistory has no such method %s" % name) | ||
59 | |||
60 | newfunc = partial(self.remoteCommand, name) | ||
61 | setattr(self, name, newfunc) | ||
62 | return newfunc | ||
63 | |||
64 | class TinfoilDataStoreConnectorIncHistory: | ||
65 | def __init__(self, tinfoil, dsindex): | ||
66 | self.tinfoil = tinfoil | ||
67 | self.dsindex = dsindex | ||
68 | |||
69 | def remoteCommand(self, cmd, *args, **kwargs): | ||
70 | return self.tinfoil.run_command('dataStoreConnectorIncHistCmd', self.dsindex, cmd, args, kwargs) | ||
71 | |||
72 | def __getattr__(self, name): | ||
73 | if not hasattr(bb.data_smart.IncludeHistory, name): | ||
74 | raise AttributeError("IncludeHistory has no such method %s" % name) | ||
75 | |||
76 | newfunc = partial(self.remoteCommand, name) | ||
77 | setattr(self, name, newfunc) | ||
78 | return newfunc | ||
79 | |||
47 | class TinfoilDataStoreConnector: | 80 | class TinfoilDataStoreConnector: |
48 | """Connector object used to enable access to datastore objects via tinfoil""" | 81 | """ |
82 | Connector object used to enable access to datastore objects via tinfoil | ||
83 | Method calls are transmitted to the remote datastore for processing, if a datastore is | ||
84 | returned we return a connector object for the new store | ||
85 | """ | ||
49 | 86 | ||
50 | def __init__(self, tinfoil, dsindex): | 87 | def __init__(self, tinfoil, dsindex): |
51 | self.tinfoil = tinfoil | 88 | self.tinfoil = tinfoil |
52 | self.dsindex = dsindex | 89 | self.dsindex = dsindex |
53 | def getVar(self, name): | 90 | self.varhistory = TinfoilDataStoreConnectorVarHistory(tinfoil, dsindex) |
54 | value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name) | 91 | self.inchistory = TinfoilDataStoreConnectorIncHistory(tinfoil, dsindex) |
55 | overrides = None | 92 | |
56 | if isinstance(value, dict): | 93 | def remoteCommand(self, cmd, *args, **kwargs): |
57 | if '_connector_origtype' in value: | 94 | ret = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, cmd, args, kwargs) |
58 | value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype']) | 95 | if isinstance(ret, bb.command.DataStoreConnectionHandle): |
59 | del value['_connector_origtype'] | 96 | return TinfoilDataStoreConnector(self.tinfoil, ret.dsindex) |
60 | if '_connector_overrides' in value: | ||
61 | overrides = value['_connector_overrides'] | ||
62 | del value['_connector_overrides'] | ||
63 | return value, overrides | ||
64 | def getKeys(self): | ||
65 | return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex)) | ||
66 | def getVarHistory(self, name): | ||
67 | return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name) | ||
68 | def expandPythonRef(self, varname, expr, d): | ||
69 | ds = bb.remotedata.RemoteDatastores.transmit_datastore(d) | ||
70 | ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', ds, varname, expr) | ||
71 | return ret | 97 | return ret |
72 | def setVar(self, varname, value): | 98 | |
73 | if self.dsindex is None: | 99 | def __getattr__(self, name): |
74 | self.tinfoil.run_command('setVariable', varname, value) | 100 | if not hasattr(bb.data._dict_type, name): |
75 | else: | 101 | raise AttributeError("Data store has no such method %s" % name) |
76 | # Not currently implemented - indicate that setting should | 102 | |
77 | # be redirected to local side | 103 | newfunc = partial(self.remoteCommand, name) |
78 | return True | 104 | setattr(self, name, newfunc) |
79 | def setVarFlag(self, varname, flagname, value): | 105 | return newfunc |
80 | if self.dsindex is None: | ||
81 | self.tinfoil.run_command('dataStoreConnectorSetVarFlag', self.dsindex, varname, flagname, value) | ||
82 | else: | ||
83 | # Not currently implemented - indicate that setting should | ||
84 | # be redirected to local side | ||
85 | return True | ||
86 | def delVar(self, varname): | ||
87 | if self.dsindex is None: | ||
88 | self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname) | ||
89 | else: | ||
90 | # Not currently implemented - indicate that setting should | ||
91 | # be redirected to local side | ||
92 | return True | ||
93 | def delVarFlag(self, varname, flagname): | ||
94 | if self.dsindex is None: | ||
95 | self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname, flagname) | ||
96 | else: | ||
97 | # Not currently implemented - indicate that setting should | ||
98 | # be redirected to local side | ||
99 | return True | ||
100 | def renameVar(self, name, newname): | ||
101 | if self.dsindex is None: | ||
102 | self.tinfoil.run_command('dataStoreConnectorRenameVar', self.dsindex, name, newname) | ||
103 | else: | ||
104 | # Not currently implemented - indicate that setting should | ||
105 | # be redirected to local side | ||
106 | return True | ||
107 | 106 | ||
108 | class TinfoilCookerAdapter: | 107 | class TinfoilCookerAdapter: |
109 | """ | 108 | """ |
@@ -412,9 +411,7 @@ class Tinfoil: | |||
412 | self.run_actions(config_params) | 411 | self.run_actions(config_params) |
413 | self.recipes_parsed = True | 412 | self.recipes_parsed = True |
414 | 413 | ||
415 | self.config_data = bb.data.init() | 414 | self.config_data = TinfoilDataStoreConnector(self, 0) |
416 | connector = TinfoilDataStoreConnector(self, None) | ||
417 | self.config_data.setVar('_remote_data', connector) | ||
418 | self.cooker = TinfoilCookerAdapter(self) | 415 | self.cooker = TinfoilCookerAdapter(self) |
419 | self.cooker_data = self.cooker.recipecaches[''] | 416 | self.cooker_data = self.cooker.recipecaches[''] |
420 | else: | 417 | else: |
@@ -842,9 +839,7 @@ class Tinfoil: | |||
842 | newobj = origtype(obj) | 839 | newobj = origtype(obj) |
843 | 840 | ||
844 | if isinstance(newobj, bb.command.DataStoreConnectionHandle): | 841 | if isinstance(newobj, bb.command.DataStoreConnectionHandle): |
845 | connector = TinfoilDataStoreConnector(self, newobj.dsindex) | 842 | newobj = TinfoilDataStoreConnector(self, newobj.dsindex) |
846 | newobj = bb.data.init() | ||
847 | newobj.setVar('_remote_data', connector) | ||
848 | 843 | ||
849 | return newobj | 844 | return newobj |
850 | 845 | ||