bitbake: layerindexlib: Initial layer index processing module implementation

The layer index module is expected to be used by various parts of the system
in order to access a layerindex-web (such as layers.openembedded.org) and
perform basic processing on the information, such as dependency scanning.

Along with the layerindex implementation are associated tests.  The tests
properly honor BB_SKIP_NETTESTS='yes' to prevent test failures.

Tests Implemented:
   - Branch, LayerItem, LayerBranch, LayerDependency, Recipe, Machine and
      Distro objects
   - LayerIndex setup using the layers.openembedded.org restapi
   - LayerIndex storing and retrieving from a file
   - LayerIndex verify dependency resolution ordering
   - LayerIndex setup using simulated cooker data

(Bitbake rev: fd0ee6c10dbb5592731e56f4c592fe687682a3e6)

Signed-off-by: Mark Hatle <mark.hatle@windriver.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

1 files changed, 398 insertions, 0 deletions

diff --git a/bitbake/lib/layerindexlib/restapi.py b/bitbake/lib/layerindexlib/restapi.py
new file mode 100644
index 0000000000..d08eb20555
--- /dev/null
+++ b/bitbake/lib/layerindexlib/restapi.py
@@ -0,0 +1,398 @@
+# Copyright (C) 2016-2018 Wind River Systems, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+import logging
+import json
+from urllib.parse import unquote
+from urllib.parse import urlparse
+
+import layerindexlib
+import layerindexlib.plugin
+
+logger = logging.getLogger('BitBake.layerindexlib.restapi')
+
+def plugin_init(plugins):
+    return RestApiPlugin()
+
+class RestApiPlugin(layerindexlib.plugin.IndexPlugin):
+    def __init__(self):
+        self.type = "restapi"
+
+    def load_index(self, url, load):
+        """
+            Fetches layer information from a local or remote layer index.
+
+            The return value is a LayerIndexObj.
+
+            url is the url to the rest api of the layer index, such as:
+            http://layers.openembedded.org/layerindex/api/
+
+            Or a local file...
+        """
+
+        up = urlparse(url)
+
+        if up.scheme == 'file':
+            return self.load_index_file(up, url, load)
+
+        if up.scheme == 'http' or up.scheme == 'https':
+            return self.load_index_web(up, url, load)
+
+        raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
+
+
+    def load_index_file(self, up, url, load):
+        """
+            Fetches layer information from a local file or directory.
+
+            The return value is a LayerIndexObj.
+
+            ud is the parsed url to the local file or directory.
+        """
+        if not os.path.exists(up.path):
+            raise FileNotFoundError(up.path)
+
+        index = layerindexlib.LayerIndexObj()
+
+        index.config = {}
+        index.config['TYPE'] = self.type
+        index.config['URL'] = url
+
+        params = self.layerindex._parse_params(up.params)
+
+        if 'desc' in params:
+            index.config['DESCRIPTION'] = unquote(params['desc'])
+        else:
+            index.config['DESCRIPTION'] = up.path
+
+        if 'cache' in params:
+            index.config['CACHE'] = params['cache']
+
+        if 'branch' in params:
+            branches = params['branch'].split(',')
+            index.config['BRANCH'] = branches
+        else:
+            branches = ['*']
+
+
+        def load_cache(path, index, branches=[]):
+            logger.debug(1, 'Loading json file %s' % path)
+            with open(path, 'rt', encoding='utf-8') as f:
+                pindex = json.load(f)
+
+            # Filter the branches on loaded files...
+            newpBranch = []
+            for branch in branches:
+                if branch != '*':
+                    if 'branches' in pindex:
+                        for br in pindex['branches']:
+                            if br['name'] == branch:
+                                newpBranch.append(br)
+                else:
+                    if 'branches' in pindex:
+                        for br in pindex['branches']:
+                            newpBranch.append(br)
+
+            if newpBranch:
+                index.add_raw_element('branches', layerindexlib.Branch, newpBranch)
+            else:
+                logger.debug(1, 'No matching branches (%s) in index file(s)' % branches)
+                # No matching branches.. return nothing...
+                return
+
+            for (lName, lType) in [("layerItems", layerindexlib.LayerItem),
+                                   ("layerBranches", layerindexlib.LayerBranch),
+                                   ("layerDependencies", layerindexlib.LayerDependency),
+                                   ("recipes", layerindexlib.Recipe),
+                                   ("machines", layerindexlib.Machine),
+                                   ("distros", layerindexlib.Distro)]:
+                if lName in pindex:
+                    index.add_raw_element(lName, lType, pindex[lName])
+
+
+        if not os.path.isdir(up.path):
+            load_cache(up.path, index, branches)
+            return index
+
+        logger.debug(1, 'Loading from dir %s...' % (up.path))
+        for (dirpath, _, filenames) in os.walk(up.path):
+            for filename in filenames:
+                if not filename.endswith('.json'):
+                    continue
+                fpath = os.path.join(dirpath, filename)
+                load_cache(fpath, index, branches)
+
+        return index
+
+
+    def load_index_web(self, up, url, load):
+        """
+            Fetches layer information from a remote layer index.
+
+            The return value is a LayerIndexObj.
+
+            ud is the parsed url to the rest api of the layer index, such as:
+            http://layers.openembedded.org/layerindex/api/
+        """
+
+        def _get_json_response(apiurl=None, username=None, password=None, retry=True):
+            assert apiurl is not None
+
+            logger.debug(1, "fetching %s" % apiurl)
+
+            up = urlparse(apiurl)
+
+            username=up.username
+            password=up.password
+
+            # Strip username/password and params
+            if up.port:
+                up_stripped = up._replace(params="", netloc="%s:%s" % (up.hostname, up.port))
+            else:
+                up_stripped = up._replace(params="", netloc=up.hostname)
+
+            res = self.layerindex._fetch_url(up_stripped.geturl(), username=username, password=password)
+
+            try:
+                parsed = json.loads(res.read().decode('utf-8'))
+            except ConnectionResetError:
+                if retry:
+                    logger.debug(1, "%s: Connection reset by peer.  Retrying..." % url)
+                    parsed = _get_json_response(apiurl=up_stripped.geturl(), username=username, password=password, retry=False)
+                    logger.debug(1, "%s: retry successful.")
+                else:
+                    raise LayerIndexFetchError('%s: Connection reset by peer.  Is there a firewall blocking your connection?' % apiurl)
+
+            return parsed
+
+        index = layerindexlib.LayerIndexObj()
+
+        index.config = {}
+        index.config['TYPE'] = self.type
+        index.config['URL'] = url
+
+        params = self.layerindex._parse_params(up.params)
+
+        if 'desc' in params:
+            index.config['DESCRIPTION'] = unquote(params['desc'])
+        else:
+            index.config['DESCRIPTION'] = up.hostname
+
+        if 'cache' in params:
+            index.config['CACHE'] = params['cache']
+
+        if 'branch' in params:
+            branches = params['branch'].split(',')
+            index.config['BRANCH'] = branches
+        else:
+            branches = ['*']
+
+        try:
+            index.apilinks = _get_json_response(apiurl=url, username=up.username, password=up.password)
+        except Exception as e:
+            raise layerindexlib.LayerIndexFetchError(url, e)
+
+        # Local raw index set...
+        pindex = {}
+
+        # Load all the requested branches at the same time time,
+        # a special branch of '*' means load all branches
+        filter = ""
+        if "*" not in branches:
+            filter = "?filter=name:%s" % "OR".join(branches)
+
+        logger.debug(1, "Loading %s from %s" % (branches, index.apilinks['branches']))
+
+        # The link won't include username/password, so pull it from the original url
+        pindex['branches'] = _get_json_response(index.apilinks['branches'] + filter,
+                                                    username=up.username, password=up.password)
+        if not pindex['branches']:
+            logger.debug(1, "No valid branches (%s) found at url %s." % (branch, url))
+            return index
+        index.add_raw_element("branches", layerindexlib.Branch, pindex['branches'])
+
+        # Load all of the layerItems (these can not be easily filtered)
+        logger.debug(1, "Loading %s from %s" % ('layerItems', index.apilinks['layerItems']))
+
+
+        # The link won't include username/password, so pull it from the original url
+        pindex['layerItems'] = _get_json_response(index.apilinks['layerItems'],
+                                                  username=up.username, password=up.password)
+        if not pindex['layerItems']:
+            logger.debug(1, "No layers were found at url %s." % (url))
+            return index
+        index.add_raw_element("layerItems", layerindexlib.LayerItem, pindex['layerItems'])
+
+
+        # From this point on load the contents for each branch.  Otherwise we
+        # could run into a timeout.
+        for branch in index.branches:
+            filter = "?filter=branch__name:%s" % index.branches[branch].name
+
+            logger.debug(1, "Loading %s from %s" % ('layerBranches', index.apilinks['layerBranches']))
+
+            # The link won't include username/password, so pull it from the original url
+            pindex['layerBranches'] = _get_json_response(index.apilinks['layerBranches'] + filter,
+                                                  username=up.username, password=up.password)
+            if not pindex['layerBranches']:
+                logger.debug(1, "No valid layer branches (%s) found at url %s." % (branches or "*", url))
+                return index
+            index.add_raw_element("layerBranches", layerindexlib.LayerBranch, pindex['layerBranches'])
+
+
+            # Load the rest, they all have a similar format
+            # Note: the layer index has a few more items, we can add them if necessary
+            # in the future.
+            filter = "?filter=layerbranch__branch__name:%s" % index.branches[branch].name
+            for (lName, lType) in [("layerDependencies", layerindexlib.LayerDependency),
+                                   ("recipes", layerindexlib.Recipe),
+                                   ("machines", layerindexlib.Machine),
+                                   ("distros", layerindexlib.Distro)]:
+                if lName not in load:
+                    continue
+                logger.debug(1, "Loading %s from %s" % (lName, index.apilinks[lName]))
+
+                # The link won't include username/password, so pull it from the original url
+                pindex[lName] = _get_json_response(index.apilinks[lName] + filter,
+                                            username=up.username, password=up.password)
+                index.add_raw_element(lName, lType, pindex[lName])
+
+        return index
+
+    def store_index(self, url, index):
+        """
+            Store layer information into a local file/dir.
+
+            The return value is a dictionary containing API,
+            layer, branch, dependency, recipe, machine, distro, information.
+
+            ud is a parsed url to a directory or file.  If the path is a
+            directory, we will split the files into one file per layer.
+            If the path is to a file (exists or not) the entire DB will be
+            dumped into that one file.
+        """
+
+        up = urlparse(url)
+
+        if up.scheme != 'file':
+            raise layerindexlib.plugin.LayerIndexPluginUrlError(self.type, url)
+
+        logger.debug(1, "Storing to %s..." % up.path)
+
+        try:
+            layerbranches = index.layerBranches
+        except KeyError:
+            logger.error('No layerBranches to write.')
+            return
+
+
+        def filter_item(layerbranchid, objects):
+            filtered = []
+            for obj in getattr(index, objects, None):
+                try:
+                    if getattr(index, objects)[obj].layerbranch_id == layerbranchid:
+                       filtered.append(getattr(index, objects)[obj]._data)
+                except AttributeError:
+                    logger.debug(1, 'No obj.layerbranch_id: %s' % objects)
+                    # No simple filter method, just include it...
+                    try:
+                        filtered.append(getattr(index, objects)[obj]._data)
+                    except AttributeError:
+                        logger.debug(1, 'No obj._data: %s %s' % (objects, type(obj)))
+                        filtered.append(obj)
+            return filtered
+
+
+        # Write out to a single file.
+        # Filter out unnecessary items, then sort as we write for determinism
+        if not os.path.isdir(up.path):
+            pindex = {}
+
+            pindex['branches'] = []
+            pindex['layerItems'] = []
+            pindex['layerBranches'] = []
+
+            for layerbranchid in layerbranches:
+                if layerbranches[layerbranchid].branch._data not in pindex['branches']:
+                    pindex['branches'].append(layerbranches[layerbranchid].branch._data)
+
+                if layerbranches[layerbranchid].layer._data not in pindex['layerItems']:
+                    pindex['layerItems'].append(layerbranches[layerbranchid].layer._data)
+
+                if layerbranches[layerbranchid]._data not in pindex['layerBranches']:
+                    pindex['layerBranches'].append(layerbranches[layerbranchid]._data)
+
+                for entry in index._index:
+                    # Skip local items, apilinks and items already processed
+                    if entry in index.config['local'] or \
+                       entry == 'apilinks' or \
+                       entry == 'branches' or \
+                       entry == 'layerBranches' or \
+                       entry == 'layerItems':
+                        continue
+                    if entry not in pindex:
+                        pindex[entry] = []
+                    pindex[entry].extend(filter_item(layerbranchid, entry))
+
+            bb.debug(1, 'Writing index to %s' % up.path)
+            with open(up.path, 'wt') as f:
+                json.dump(layerindexlib.sort_entry(pindex), f, indent=4)
+            return
+
+
+        # Write out to a directory one file per layerBranch
+        # Prepare all layer related items, to create a minimal file.
+        # We have to sort the entries as we write so they are deterministic
+        for layerbranchid in layerbranches:
+            pindex = {}
+
+            for entry in index._index:
+                # Skip local items, apilinks and items already processed
+                if entry in index.config['local'] or \
+                   entry == 'apilinks' or \
+                   entry == 'branches' or \
+                   entry == 'layerBranches' or \
+                   entry == 'layerItems':
+                    continue
+                pindex[entry] = filter_item(layerbranchid, entry)
+
+            # Add the layer we're processing as the first one...
+            pindex['branches'] = [layerbranches[layerbranchid].branch._data]
+            pindex['layerItems'] = [layerbranches[layerbranchid].layer._data]
+            pindex['layerBranches'] = [layerbranches[layerbranchid]._data]
+
+            # We also need to include the layerbranch for any dependencies...
+            for layerdep in pindex['layerDependencies']:
+                layerdependency = layerindexlib.LayerDependency(index, layerdep)
+
+                layeritem = layerdependency.dependency
+                layerbranch = layerdependency.dependency_layerBranch
+
+                # We need to avoid duplicates...
+                if layeritem._data not in pindex['layerItems']:
+                    pindex['layerItems'].append(layeritem._data)
+
+                if layerbranch._data not in pindex['layerBranches']:
+                    pindex['layerBranches'].append(layerbranch._data)
+
+            # apply mirroring adjustments here....
+
+            fname = index.config['DESCRIPTION'] + '__' + pindex['branches'][0]['name'] + '__' + pindex['layerItems'][0]['name']
+            fname = fname.translate(str.maketrans('/ ', '__'))
+            fpath = os.path.join(up.path, fname)
+
+            bb.debug(1, 'Writing index to %s' % fpath + '.json')
+            with open(fpath + '.json', 'wt') as f:
+                json.dump(layerindexlib.sort_entry(pindex), f, indent=4)