5 files changed, 87 insertions, 360 deletions

diff --git a/bitbake/lib/bb/command.py b/bitbake/lib/bb/command.py
index b38c151b3d..2879950933 100644
--- a/bitbake/lib/bb/command.py
+++ b/bitbake/lib/bb/command.py
@@ -450,54 +450,38 @@ class CommandsSync:
         return all_p, best
     getRuntimeProviders.readonly = True
 
-    def dataStoreConnectorFindVar(self, command, params):
+    def dataStoreConnectorCmd(self, command, params):
         dsindex = params[0]
-        name = params[1]
-        datastore = command.remotedatastores[dsindex]
-        value, overridedata = datastore._findVar(name)
-
-        if value:
-            content = value.get('_content', None)
-            if isinstance(content, bb.data_smart.DataSmart):
-                # Value is a datastore (e.g. BB_ORIGENV) - need to handle this carefully
-                idx = command.remotedatastores.check_store(content, True)
-                return {'_content': DataStoreConnectionHandle(idx),
-                        '_connector_origtype': 'DataStoreConnectionHandle',
-                        '_connector_overrides': overridedata}
-            elif isinstance(content, set):
-                return {'_content': list(content),
-                        '_connector_origtype': 'set',
-                        '_connector_overrides': overridedata}
-            else:
-                value['_connector_overrides'] = overridedata
-        else:
-            value = {}
-            value['_connector_overrides'] = overridedata
-        return value
-    dataStoreConnectorFindVar.readonly = True
+        method = params[1]
+        args = params[2]
+        kwargs = params[3]
 
-    def dataStoreConnectorGetKeys(self, command, params):
-        dsindex = params[0]
-        datastore = command.remotedatastores[dsindex]
-        return list(datastore.keys())
-    dataStoreConnectorGetKeys.readonly = True
+        d = command.remotedatastores[dsindex]
+        ret = getattr(d, method)(*args, **kwargs)
+
+        if isinstance(ret, bb.data_smart.DataSmart):
+            idx = command.remotedatastores.store(ret)
+            return DataStoreConnectionHandle(idx)
 
-    def dataStoreConnectorGetVarHistory(self, command, params):
+        return ret
+
+    def dataStoreConnectorVarHistCmd(self, command, params):
         dsindex = params[0]
-        name = params[1]
-        datastore = command.remotedatastores[dsindex]
-        return datastore.varhistory.variable(name)
-    dataStoreConnectorGetVarHistory.readonly = True
+        method = params[1]
+        args = params[2]
+        kwargs = params[3]
 
-    def dataStoreConnectorExpandPythonRef(self, command, params):
-        config_data_dict = params[0]
-        varname = params[1]
-        expr = params[2]
+        d = command.remotedatastores[dsindex].varhistory
+        return getattr(d, method)(*args, **kwargs)
 
-        config_data = command.remotedatastores.receive_datastore(config_data_dict)
+    def dataStoreConnectorIncHistCmd(self, command, params):
+        dsindex = params[0]
+        method = params[1]
+        args = params[2]
+        kwargs = params[3]
 
-        varparse = bb.data_smart.VariableParse(varname, config_data)
-        return varparse.python_sub(expr)
+        d = command.remotedatastores[dsindex].inchistory
+        return getattr(d, method)(*args, **kwargs)
 
     def dataStoreConnectorRelease(self, command, params):
         dsindex = params[0]
@@ -505,31 +489,6 @@ class CommandsSync:
             raise CommandError('dataStoreConnectorRelease: invalid index %d' % dsindex)
         command.remotedatastores.release(dsindex)
 
-    def dataStoreConnectorSetVarFlag(self, command, params):
-        dsindex = params[0]
-        name = params[1]
-        flag = params[2]
-        value = params[3]
-        datastore = command.remotedatastores[dsindex]
-        datastore.setVarFlag(name, flag, value)
-
-    def dataStoreConnectorDelVar(self, command, params):
-        dsindex = params[0]
-        name = params[1]
-        datastore = command.remotedatastores[dsindex]
-        if len(params) > 2:
-            flag = params[2]
-            datastore.delVarFlag(name, flag)
-        else:
-            datastore.delVar(name)
-
-    def dataStoreConnectorRenameVar(self, command, params):
-        dsindex = params[0]
-        name = params[1]
-        newname = params[2]
-        datastore = command.remotedatastores[dsindex]
-        datastore.renameVar(name, newname)
-
     def parseRecipeFile(self, command, params):
         """
         Parse the specified recipe file (with or without bbappends)
diff --git a/bitbake/lib/bb/data_smart.py b/bitbake/lib/bb/data_smart.py
index b2dc9d9fd5..70257ab7f8 100644
--- a/bitbake/lib/bb/data_smart.py
+++ b/bitbake/lib/bb/data_smart.py
@@ -107,10 +107,6 @@ class VariableParse:
             else:
                 code = match.group()[3:-1]
 
-            if "_remote_data" in self.d:
-                connector = self.d["_remote_data"]
-                return connector.expandPythonRef(self.varname, code, self.d)
-
             if self.varname:
                 varname = 'Var <%s>' % self.varname
             else:
@@ -268,12 +264,7 @@ class VariableHistory(object):
             self.variables[newvar].append(i.copy())
 
     def variable(self, var):
-        remote_connector = self.dataroot.getVar('_remote_data', False)
-        if remote_connector:
-            varhistory = remote_connector.getVarHistory(var)
-        else:
-            varhistory = []
-
+        varhistory = []
         if var in self.variables:
             varhistory.extend(self.variables[var])
         return varhistory
@@ -471,10 +462,6 @@ class DataSmart(MutableMapping):
             if var in dest:
                 return dest[var], self.overridedata.get(var, None)
 
-            if "_remote_data" in dest:
-                connector = dest["_remote_data"]["_content"]
-                return connector.getVar(var)
-
             if "_data" not in dest:
                 break
             dest = dest["_data"]
@@ -499,12 +486,6 @@ class DataSmart(MutableMapping):
         if 'parsing' in loginfo:
             parsing=True
 
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.setVar(var, value)
-            if not res:
-                return
-
         if 'op' not in loginfo:
             loginfo['op'] = "set"
 
@@ -612,12 +593,6 @@ class DataSmart(MutableMapping):
             bb.warn("Calling renameVar with equivalent keys (%s) is invalid" % key)
             return
 
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.renameVar(key, newkey)
-            if not res:
-                return
-
         val = self.getVar(key, 0, parsing=True)
         if val is not None:
             self.varhistory.rename_variable_hist(key, newkey)
@@ -663,11 +638,6 @@ class DataSmart(MutableMapping):
 
     def delVar(self, var, **loginfo):
         self.expand_cache = {}
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.delVar(var)
-            if not res:
-                return
 
         loginfo['detail'] = ""
         loginfo['op'] = 'del'
@@ -695,11 +665,6 @@ class DataSmart(MutableMapping):
 
     def setVarFlag(self, var, flag, value, **loginfo):
         self.expand_cache = {}
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.setVarFlag(var, flag, value)
-            if not res:
-                return
 
         if 'op' not in loginfo:
             loginfo['op'] = "set"
@@ -850,11 +815,6 @@ class DataSmart(MutableMapping):
 
     def delVarFlag(self, var, flag, **loginfo):
         self.expand_cache = {}
-        if '_remote_data' in self.dict:
-            connector = self.dict["_remote_data"]["_content"]
-            res = connector.delVarFlag(var, flag)
-            if not res:
-                return
 
         local_var, _ = self._findVar(var)
         if not local_var:
@@ -972,7 +932,7 @@ class DataSmart(MutableMapping):
 
     def localkeys(self):
         for key in self.dict:
-            if key not in ['_data', '_remote_data']:
+            if key not in ['_data']:
                 yield key
 
     def __iter__(self):
@@ -981,7 +941,7 @@ class DataSmart(MutableMapping):
         def keylist(d):        
             klist = set()
             for key in d:
-                if key in ["_data", "_remote_data"]:
+                if key in ["_data"]:
                     continue
                 if key in deleted:
                     continue
@@ -995,13 +955,6 @@ class DataSmart(MutableMapping):
             if "_data" in d:
                 klist |= keylist(d["_data"])
 
-            if "_remote_data" in d:
-                connector = d["_remote_data"]["_content"]
-                for key in connector.getKeys():
-                    if key in deleted:
-                        continue
-                    klist.add(key)
-
             return klist
 
         self.need_overrides()
diff --git a/bitbake/lib/bb/remotedata.py b/bitbake/lib/bb/remotedata.py
index 7391e1b45c..6c9864dd6b 100644
--- a/bitbake/lib/bb/remotedata.py
+++ b/bitbake/lib/bb/remotedata.py
@@ -17,16 +17,16 @@ class RemoteDatastores:
         self.cooker = cooker
         self.datastores = {}
         self.locked = []
+        self.datastores[0] = self.cooker.data
         self.nextindex = 1
 
     def __len__(self):
         return len(self.datastores)
 
     def __getitem__(self, key):
-        if key is None:
-            return self.cooker.data
-        else:
-            return self.datastores[key]
+        # Cooker could have changed its datastore from under us
+        self.datastores[0] = self.cooker.data
+        return self.datastores[key]
 
     def items(self):
         return self.datastores.items()
@@ -63,44 +63,3 @@ class RemoteDatastores:
             raise Exception('Tried to release locked datastore %d' % idx)
         del self.datastores[idx]
 
-    def receive_datastore(self, remote_data):
-        """Receive a datastore object sent from the client (as prepared by transmit_datastore())"""
-        dct = dict(remote_data)
-        d = bb.data_smart.DataSmart()
-        d.dict = dct
-        while True:
-            if '_remote_data' in dct:
-                dsindex = dct['_remote_data']['_content']
-                del dct['_remote_data']
-                if dsindex is None:
-                    dct['_data'] = self.cooker.data.dict
-                else:
-                    dct['_data'] = self.datastores[dsindex].dict
-                break
-            elif '_data' in dct:
-                idct = dict(dct['_data'])
-                dct['_data'] = idct
-                dct = idct
-            else:
-                break
-        return d
-
-    @staticmethod
-    def transmit_datastore(d):
-        """Prepare a datastore object for sending over IPC from the client end"""
-        # FIXME content might be a dict, need to turn that into a list as well
-        def copy_dicts(dct):
-            if '_remote_data' in dct:
-                dsindex = dct['_remote_data']['_content'].dsindex
-                newdct = dct.copy()
-                newdct['_remote_data'] = {'_content': dsindex}
-                return list(newdct.items())
-            elif '_data' in dct:
-                newdct = dct.copy()
-                newdata = copy_dicts(dct['_data'])
-                if newdata:
-                    newdct['_data'] = newdata
-                return list(newdct.items())
-            return None
-        main_dict = copy_dicts(d.dict)
-        return main_dict
diff --git a/bitbake/lib/bb/tests/data.py b/bitbake/lib/bb/tests/data.py
index 3e49984c93..2b137706dd 100644
--- a/bitbake/lib/bb/tests/data.py
+++ b/bitbake/lib/bb/tests/data.py
@@ -544,142 +544,3 @@ class Serialize(unittest.TestCase):
         self.assertEqual(newd.getVarFlag('HELLO', 'other'), 'planet')
 
 
-# Remote datastore tests
-# These really only test the interface, since in actual usage we have a
-# tinfoil connector that does everything over RPC, and this doesn't test
-# that.
-
-class TestConnector:
-    d = None
-    def __init__(self, d):
-        self.d = d
-    def getVar(self, name):
-        return self.d._findVar(name)
-    def getKeys(self):
-        return set(self.d.keys())
-    def getVarHistory(self, name):
-        return self.d.varhistory.variable(name)
-    def expandPythonRef(self, varname, expr, d):
-        localdata = self.d.createCopy()
-        for key in d.localkeys():
-            localdata.setVar(d.getVar(key))
-        varparse = bb.data_smart.VariableParse(varname, localdata)
-        return varparse.python_sub(expr)
-    def setVar(self, name, value):
-        self.d.setVar(name, value)
-    def setVarFlag(self, name, flag, value):
-        self.d.setVarFlag(name, flag, value)
-    def delVar(self, name):
-        self.d.delVar(name)
-        return False
-    def delVarFlag(self, name, flag):
-        self.d.delVarFlag(name, flag)
-        return False
-    def renameVar(self, name, newname):
-        self.d.renameVar(name, newname)
-        return False
-
-class Remote(unittest.TestCase):
-    def test_remote(self):
-
-        d1 = bb.data.init()
-        d1.enableTracking()
-        d2 = bb.data.init()
-        d2.enableTracking()
-        connector = TestConnector(d1)
-
-        d2.setVar('_remote_data', connector)
-
-        d1.setVar('HELLO', 'world')
-        d1.setVarFlag('OTHER', 'flagname', 'flagvalue')
-        self.assertEqual(d2.getVar('HELLO'), 'world')
-        self.assertEqual(d2.expand('${HELLO}'), 'world')
-        self.assertEqual(d2.expand('${@d.getVar("HELLO")}'), 'world')
-        self.assertIn('flagname', d2.getVarFlags('OTHER'))
-        self.assertEqual(d2.getVarFlag('OTHER', 'flagname'), 'flagvalue')
-        self.assertEqual(d1.varhistory.variable('HELLO'), d2.varhistory.variable('HELLO'))
-        # Test setVar on client side affects server
-        d2.setVar('HELLO', 'other-world')
-        self.assertEqual(d1.getVar('HELLO'), 'other-world')
-        # Test setVarFlag on client side affects server
-        d2.setVarFlag('HELLO', 'flagname', 'flagvalue')
-        self.assertEqual(d1.getVarFlag('HELLO', 'flagname'), 'flagvalue')
-        # Test client side data is incorporated in python expansion (which is done on server)
-        d2.setVar('FOO', 'bar')
-        self.assertEqual(d2.expand('${@d.getVar("FOO")}'), 'bar')
-        # Test overrides work
-        d1.setVar('FOO_test', 'baz')
-        d1.appendVar('OVERRIDES', ':test')
-        self.assertEqual(d2.getVar('FOO'), 'baz')
-
-
-# Remote equivalents of local test classes
-# Note that these aren't perfect since we only test in one direction
-
-class RemoteDataExpansions(DataExpansions):
-    def setUp(self):
-        self.d1 = bb.data.init()
-        self.d = bb.data.init()
-        self.d1["foo"] = "value_of_foo"
-        self.d1["bar"] = "value_of_bar"
-        self.d1["value_of_foo"] = "value_of_'value_of_foo'"
-        connector = TestConnector(self.d1)
-        self.d.setVar('_remote_data', connector)
-
-class TestRemoteNestedExpansions(TestNestedExpansions):
-    def setUp(self):
-        self.d1 = bb.data.init()
-        self.d = bb.data.init()
-        self.d1["foo"] = "foo"
-        self.d1["bar"] = "bar"
-        self.d1["value_of_foobar"] = "187"
-        connector = TestConnector(self.d1)
-        self.d.setVar('_remote_data', connector)
-
-class TestRemoteConcat(TestConcat):
-    def setUp(self):
-        self.d1 = bb.data.init()
-        self.d = bb.data.init()
-        self.d1.setVar("FOO", "foo")
-        self.d1.setVar("VAL", "val")
-        self.d1.setVar("BAR", "bar")
-        connector = TestConnector(self.d1)
-        self.d.setVar('_remote_data', connector)
-
-class TestRemoteConcatOverride(TestConcatOverride):
-    def setUp(self):
-        self.d1 = bb.data.init()
-        self.d = bb.data.init()
-        self.d1.setVar("FOO", "foo")
-        self.d1.setVar("VAL", "val")
-        self.d1.setVar("BAR", "bar")
-        connector = TestConnector(self.d1)
-        self.d.setVar('_remote_data', connector)
-
-class TestRemoteOverrides(TestOverrides):
-    def setUp(self):
-        self.d1 = bb.data.init()
-        self.d = bb.data.init()
-        self.d1.setVar("OVERRIDES", "foo:bar:local")
-        self.d1.setVar("TEST", "testvalue")
-        connector = TestConnector(self.d1)
-        self.d.setVar('_remote_data', connector)
-
-class TestRemoteKeyExpansion(TestKeyExpansion):
-    def setUp(self):
-        self.d1 = bb.data.init()
-        self.d = bb.data.init()
-        self.d1.setVar("FOO", "foo")
-        self.d1.setVar("BAR", "foo")
-        connector = TestConnector(self.d1)
-        self.d.setVar('_remote_data', connector)
-
-class TestRemoteFlags(TestFlags):
-    def setUp(self):
-        self.d1 = bb.data.init()
-        self.d = bb.data.init()
-        self.d1.setVar("foo", "value of foo")
-        self.d1.setVarFlag("foo", "flag1", "value of flag1")
-        self.d1.setVarFlag("foo", "flag2", "value of flag2")
-        connector = TestConnector(self.d1)
-        self.d.setVar('_remote_data', connector)
diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py
index 70b381e351..4fbad0774e 100644
--- a/bitbake/lib/bb/tinfoil.py
+++ b/bitbake/lib/bb/tinfoil.py
@@ -13,6 +13,7 @@ import sys
 import atexit
 import re
 from collections import OrderedDict, defaultdict
+from functools import partial
 
 import bb.cache
 import bb.cooker
@@ -44,66 +45,64 @@ class TinfoilUIException(Exception):
 class TinfoilCommandFailed(Exception):
     """Exception raised when run_command fails"""
 
+class TinfoilDataStoreConnectorVarHistory:
+    def __init__(self, tinfoil, dsindex):
+        self.tinfoil = tinfoil
+        self.dsindex = dsindex
+
+    def remoteCommand(self, cmd, *args, **kwargs):
+        return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs)
+
+    def __getattr__(self, name):
+        if not hasattr(bb.data_smart.VariableHistory, name):
+            raise AttributeError("VariableHistory has no such method %s" % name)
+
+        newfunc = partial(self.remoteCommand, name)
+        setattr(self, name, newfunc)
+        return newfunc
+
+class TinfoilDataStoreConnectorIncHistory:
+    def __init__(self, tinfoil, dsindex):
+        self.tinfoil = tinfoil
+        self.dsindex = dsindex
+
+    def remoteCommand(self, cmd, *args, **kwargs):
+        return self.tinfoil.run_command('dataStoreConnectorIncHistCmd', self.dsindex, cmd, args, kwargs)
+
+    def __getattr__(self, name):
+        if not hasattr(bb.data_smart.IncludeHistory, name):
+            raise AttributeError("IncludeHistory has no such method %s" % name)
+
+        newfunc = partial(self.remoteCommand, name)
+        setattr(self, name, newfunc)
+        return newfunc
+
 class TinfoilDataStoreConnector:
-    """Connector object used to enable access to datastore objects via tinfoil"""
+    """
+    Connector object used to enable access to datastore objects via tinfoil
+    Method calls are transmitted to the remote datastore for processing, if a datastore is
+    returned we return a connector object for the new store
+    """
 
     def __init__(self, tinfoil, dsindex):
         self.tinfoil = tinfoil
         self.dsindex = dsindex
-    def getVar(self, name):
-        value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name)
-        overrides = None
-        if isinstance(value, dict):
-            if '_connector_origtype' in value:
-                value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype'])
-                del value['_connector_origtype']
-            if '_connector_overrides' in value:
-                overrides = value['_connector_overrides']
-                del value['_connector_overrides']
-        return value, overrides
-    def getKeys(self):
-        return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex))
-    def getVarHistory(self, name):
-        return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name)
-    def expandPythonRef(self, varname, expr, d):
-        ds = bb.remotedata.RemoteDatastores.transmit_datastore(d)
-        ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', ds, varname, expr)
+        self.varhistory = TinfoilDataStoreConnectorVarHistory(tinfoil, dsindex)
+        self.inchistory = TinfoilDataStoreConnectorIncHistory(tinfoil, dsindex)
+
+    def remoteCommand(self, cmd, *args, **kwargs):
+        ret = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, cmd, args, kwargs)
+        if isinstance(ret, bb.command.DataStoreConnectionHandle):
+            return TinfoilDataStoreConnector(self.tinfoil, ret.dsindex)
         return ret
-    def setVar(self, varname, value):
-        if self.dsindex is None:
-            self.tinfoil.run_command('setVariable', varname, value)
-        else:
-            # Not currently implemented - indicate that setting should
-            # be redirected to local side
-            return True
-    def setVarFlag(self, varname, flagname, value):
-        if self.dsindex is None:
-            self.tinfoil.run_command('dataStoreConnectorSetVarFlag', self.dsindex, varname, flagname, value)
-        else:
-            # Not currently implemented - indicate that setting should
-            # be redirected to local side
-            return True
-    def delVar(self, varname):
-        if self.dsindex is None:
-            self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname)
-        else:
-            # Not currently implemented - indicate that setting should
-            # be redirected to local side
-            return True
-    def delVarFlag(self, varname, flagname):
-        if self.dsindex is None:
-            self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname, flagname)
-        else:
-            # Not currently implemented - indicate that setting should
-            # be redirected to local side
-            return True
-    def renameVar(self, name, newname):
-        if self.dsindex is None:
-            self.tinfoil.run_command('dataStoreConnectorRenameVar', self.dsindex, name, newname)
-        else:
-            # Not currently implemented - indicate that setting should
-            # be redirected to local side
-            return True
+
+    def __getattr__(self, name):
+        if not hasattr(bb.data._dict_type, name):
+            raise AttributeError("Data store has no such method %s" % name)
+
+        newfunc = partial(self.remoteCommand, name)
+        setattr(self, name, newfunc)
+        return newfunc
 
 class TinfoilCookerAdapter:
     """
@@ -412,9 +411,7 @@ class Tinfoil:
                 self.run_actions(config_params)
                 self.recipes_parsed = True
 
-            self.config_data = bb.data.init()
-            connector = TinfoilDataStoreConnector(self, None)
-            self.config_data.setVar('_remote_data', connector)
+            self.config_data = TinfoilDataStoreConnector(self, 0)
             self.cooker = TinfoilCookerAdapter(self)
             self.cooker_data = self.cooker.recipecaches['']
         else:
@@ -842,9 +839,7 @@ class Tinfoil:
             newobj = origtype(obj)
 
         if isinstance(newobj, bb.command.DataStoreConnectionHandle):
-            connector = TinfoilDataStoreConnector(self, newobj.dsindex)
-            newobj = bb.data.init()
-            newobj.setVar('_remote_data', connector)
+            newobj = TinfoilDataStoreConnector(self, newobj.dsindex)
 
         return newobj