bitbake: tinfoil: Simplify remote datastore connections

The current approach to remote datastores used in tinfoil is breaking. For
example, adding a devupstream extension to a recipe with a git upstream,
making it the preferred version and then running "devtool modify" on it
causes get_srcrev() circular dependency issues. The problem is the override
handling in the datastore is broken.

This gets broken since remotedata:recieve_datastore() sets d.dict but doesn't
update d.overridedata (or d.inchistory or d.varhistory). We could play
whack-a-mole but the current implementation seems to be flawed to me. It
also doesn't cover, or only partially covers some datastore operations and
each needs new dedicated command API.

Instead, step back and reimplement the way the datastore connector works.

With this change, the datastore is either remote or local but the data is not
spread on two sides of the connection. All the API is proxied over the connection
by a single function for the datastore (and two to support variable history
and include history).

This code does not support using the datastore as a parameter to any data store
functions. We did have one case of that but its just bad code and can be
replaced.

The result is something which is much simpler and less invasive to the datastore
code itself, meaning its behaviour should be much more consistent. The existing
tests for the remote data no longer make any sense and are removed.

The one bug this code would have is if key/value pairs are returned over the IPC
and those values contained a DataSmart object since we don't recurse into return
values to find such things. Nothing appears to do that currently so lets worry
about it if its ever an issue. This change should simplfy a ton of other issues
and avoid a ton of other bugs so is a huge net gain.

Tested with bitbake's and OE's selftests.

(Bitbake rev: 85e03a64dd0a4ebe71009ec4bdf4192c04a9786e)

Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 55 insertions, 60 deletions

diff --git a/bitbake/lib/bb/tinfoil.py b/bitbake/lib/bb/tinfoil.py
index 70b381e351..4fbad0774e 100644
--- a/bitbake/lib/bb/tinfoil.py
+++ b/bitbake/lib/bb/tinfoil.py
@@ -13,6 +13,7 @@ import sys
 import atexit
 import re
 from collections import OrderedDict, defaultdict
+from functools import partial
 
 import bb.cache
 import bb.cooker
@@ -44,66 +45,64 @@ class TinfoilUIException(Exception):
 class TinfoilCommandFailed(Exception):
     """Exception raised when run_command fails"""
 
+class TinfoilDataStoreConnectorVarHistory:
+    def __init__(self, tinfoil, dsindex):
+        self.tinfoil = tinfoil
+        self.dsindex = dsindex
+
+    def remoteCommand(self, cmd, *args, **kwargs):
+        return self.tinfoil.run_command('dataStoreConnectorVarHistCmd', self.dsindex, cmd, args, kwargs)
+
+    def __getattr__(self, name):
+        if not hasattr(bb.data_smart.VariableHistory, name):
+            raise AttributeError("VariableHistory has no such method %s" % name)
+
+        newfunc = partial(self.remoteCommand, name)
+        setattr(self, name, newfunc)
+        return newfunc
+
+class TinfoilDataStoreConnectorIncHistory:
+    def __init__(self, tinfoil, dsindex):
+        self.tinfoil = tinfoil
+        self.dsindex = dsindex
+
+    def remoteCommand(self, cmd, *args, **kwargs):
+        return self.tinfoil.run_command('dataStoreConnectorIncHistCmd', self.dsindex, cmd, args, kwargs)
+
+    def __getattr__(self, name):
+        if not hasattr(bb.data_smart.IncludeHistory, name):
+            raise AttributeError("IncludeHistory has no such method %s" % name)
+
+        newfunc = partial(self.remoteCommand, name)
+        setattr(self, name, newfunc)
+        return newfunc
+
 class TinfoilDataStoreConnector:
-    """Connector object used to enable access to datastore objects via tinfoil"""
+    """
+    Connector object used to enable access to datastore objects via tinfoil
+    Method calls are transmitted to the remote datastore for processing, if a datastore is
+    returned we return a connector object for the new store
+    """
 
     def __init__(self, tinfoil, dsindex):
         self.tinfoil = tinfoil
         self.dsindex = dsindex
-    def getVar(self, name):
-        value = self.tinfoil.run_command('dataStoreConnectorFindVar', self.dsindex, name)
-        overrides = None
-        if isinstance(value, dict):
-            if '_connector_origtype' in value:
-                value['_content'] = self.tinfoil._reconvert_type(value['_content'], value['_connector_origtype'])
-                del value['_connector_origtype']
-            if '_connector_overrides' in value:
-                overrides = value['_connector_overrides']
-                del value['_connector_overrides']
-        return value, overrides
-    def getKeys(self):
-        return set(self.tinfoil.run_command('dataStoreConnectorGetKeys', self.dsindex))
-    def getVarHistory(self, name):
-        return self.tinfoil.run_command('dataStoreConnectorGetVarHistory', self.dsindex, name)
-    def expandPythonRef(self, varname, expr, d):
-        ds = bb.remotedata.RemoteDatastores.transmit_datastore(d)
-        ret = self.tinfoil.run_command('dataStoreConnectorExpandPythonRef', ds, varname, expr)
+        self.varhistory = TinfoilDataStoreConnectorVarHistory(tinfoil, dsindex)
+        self.inchistory = TinfoilDataStoreConnectorIncHistory(tinfoil, dsindex)
+
+    def remoteCommand(self, cmd, *args, **kwargs):
+        ret = self.tinfoil.run_command('dataStoreConnectorCmd', self.dsindex, cmd, args, kwargs)
+        if isinstance(ret, bb.command.DataStoreConnectionHandle):
+            return TinfoilDataStoreConnector(self.tinfoil, ret.dsindex)
         return ret
-    def setVar(self, varname, value):
-        if self.dsindex is None:
-            self.tinfoil.run_command('setVariable', varname, value)
-        else:
-            # Not currently implemented - indicate that setting should
-            # be redirected to local side
-            return True
-    def setVarFlag(self, varname, flagname, value):
-        if self.dsindex is None:
-            self.tinfoil.run_command('dataStoreConnectorSetVarFlag', self.dsindex, varname, flagname, value)
-        else:
-            # Not currently implemented - indicate that setting should
-            # be redirected to local side
-            return True
-    def delVar(self, varname):
-        if self.dsindex is None:
-            self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname)
-        else:
-            # Not currently implemented - indicate that setting should
-            # be redirected to local side
-            return True
-    def delVarFlag(self, varname, flagname):
-        if self.dsindex is None:
-            self.tinfoil.run_command('dataStoreConnectorDelVar', self.dsindex, varname, flagname)
-        else:
-            # Not currently implemented - indicate that setting should
-            # be redirected to local side
-            return True
-    def renameVar(self, name, newname):
-        if self.dsindex is None:
-            self.tinfoil.run_command('dataStoreConnectorRenameVar', self.dsindex, name, newname)
-        else:
-            # Not currently implemented - indicate that setting should
-            # be redirected to local side
-            return True
+
+    def __getattr__(self, name):
+        if not hasattr(bb.data._dict_type, name):
+            raise AttributeError("Data store has no such method %s" % name)
+
+        newfunc = partial(self.remoteCommand, name)
+        setattr(self, name, newfunc)
+        return newfunc
 
 class TinfoilCookerAdapter:
     """
@@ -412,9 +411,7 @@ class Tinfoil:
                 self.run_actions(config_params)
                 self.recipes_parsed = True
 
-            self.config_data = bb.data.init()
-            connector = TinfoilDataStoreConnector(self, None)
-            self.config_data.setVar('_remote_data', connector)
+            self.config_data = TinfoilDataStoreConnector(self, 0)
             self.cooker = TinfoilCookerAdapter(self)
             self.cooker_data = self.cooker.recipecaches['']
         else:
@@ -842,9 +839,7 @@ class Tinfoil:
             newobj = origtype(obj)
 
         if isinstance(newobj, bb.command.DataStoreConnectionHandle):
-            connector = TinfoilDataStoreConnector(self, newobj.dsindex)
-            newobj = bb.data.init()
-            newobj.setVar('_remote_data', connector)
+            newobj = TinfoilDataStoreConnector(self, newobj.dsindex)
 
         return newobj