Merge pull request #351 from advancedtelematic/feat/better-dependencies

Feat/better dependencies

3 files changed, 237 insertions, 125 deletions

diff --git a/scripts/find_aktualizr_dependencies.sh b/scripts/find_aktualizr_dependencies.sh
new file mode 100755
index 0000000..786d8a9
--- /dev/null
+++ b/scripts/find_aktualizr_dependencies.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+set -euo pipefail
+
+parentdir="$(dirname "$0")"
+
+# Does NOT include garage-sign, anything used only for testing (i.e. strace and
+# gtest), any of the git submodules, all of which are also only used for
+# testing (tuf-test-vectors, isotp-c, ostreesysroot, and HdrHistogram_c), or
+# any other third party modules included directly into the source tree
+# (jsoncpp, open62541, picojson). Also check libp11, dpkg, and systemd since
+# those are common dependencies not enabled by default.
+${parentdir}/find_dependencies.py aktualizr
+${parentdir}/find_dependencies.py aktualizr-auto-prov
+${parentdir}/find_dependencies.py aktualizr-implicit-prov
+${parentdir}/find_dependencies.py aktualizr-ca-implicit-prov
+${parentdir}/find_dependencies.py aktualizr-hsm-prov
+${parentdir}/find_dependencies.py aktualizr-disable-send-ip
+${parentdir}/find_dependencies.py aktualizr-example-interface
+${parentdir}/find_dependencies.py aktualizr-log-debug
+${parentdir}/find_dependencies.py libp11
+${parentdir}/find_dependencies.py dpkg
+${parentdir}/find_dependencies.py systemd
+
diff --git a/scripts/find_dependencies.py b/scripts/find_dependencies.py
new file mode 100755
index 0000000..cffe32b
--- /dev/null
+++ b/scripts/find_dependencies.py
@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+
+from argparse import ArgumentParser
+import os.path
+import sys
+
+scripts_path = os.path.dirname(os.path.realpath(__file__))
+bb_lib_path = os.path.abspath(scripts_path + '/../../poky/bitbake/lib')
+sys.path = sys.path + [bb_lib_path]
+
+import bb.fetch2
+import bb.tinfoil
+
+
+PRINT_PROGRESS = True
+SKIP_BUILD_TOOLS = True
+KNOWN_BUILD_TOOLS = ['virtual/x86_64-poky-linux-gcc', # gcc-cross-x86_64
+                     'virtual/x86_64-poky-linux-compilerlibs', # gcc-runtime
+                     'virtual/i586-poky-linux-gcc', # gcc-cross-i586
+                     'virtual/i586-poky-linux-compilerlibs', # gcc-runtime
+                     'virtual/libc', # glibc
+                     'virtual/libintl', # glibc
+                     'virtual/libiconv', # glibc
+                     'virtual/crypt', # glibc
+                     'autoconf-native',
+                     'automake-native',
+                     'libtool-native',
+                     'gnu-config-native',
+                     'm4-native',
+                     'texinfo-dummy-native',
+                     'gettext-minimal-native',
+                     'libtool-cross',
+                     'gettext-native',
+                     'util-linux-native',
+                     'pkgconfig-native',
+                     'makedepend-native']
+
+
+def get_recipe_info(tinfoil, rn):
+    try:
+        info = tinfoil.get_recipe_info(rn)
+    except Exception:
+        print('Failed to get recipe info for: %s' % rn)
+        return []
+    if not info:
+        print('No recipe info found for: %s' % rn)
+        return []
+    append_files = tinfoil.get_file_appends(info.fn)
+    appends = True
+    data = tinfoil.parse_recipe_file(info.fn, appends, append_files)
+    data.pn = info.pn
+    data.pv = info.pv
+    return data
+
+
+def print_package(manifest_file, data, is_project):
+    src_uri = data.getVar('SRC_URI').split()
+    lic = data.getVar('LICENSE')
+    summary = data.getVar('SUMMARY')
+    homepage = data.getVar('HOMEPAGE')
+    srcrev = data.getVar('SRCREV')
+    branch = data.getVar('BRANCH')
+
+    if is_project:
+        manifest_file.write('  id:\n')
+    else:
+        manifest_file.write('- id:\n')
+    manifest_file.write('    package_manager: "Yocto"\n')
+    manifest_file.write('    namespace: ""\n')
+    manifest_file.write('    name: "%s"\n' % data.pn)
+    manifest_file.write('    version: "%s"\n' % data.pv)
+    manifest_file.write('  declared_lics:\n')
+    manifest_file.write('  - "%s"\n' % lic)
+    if is_project:
+        manifest_file.write('  aliases: []\n')
+    if summary:
+        manifest_file.write('  description: "%s"\n' % summary)
+    else:
+        description = data.getVar('DESCRIPTION')
+        manifest_file.write('  description: "%s"\n' % description)
+    manifest_file.write('  homepage_url: "%s"\n' % homepage)
+    # Binary artifacts almost never exist in Yocto.
+    manifest_file.write('  binary_artifact:\n')
+    manifest_file.write('    url: ""\n')
+    manifest_file.write('    hash: ""\n')
+    manifest_file.write('    hash_algorithm: ""\n')
+    manifest_file.write('  source_artifact:\n')
+    repos = []
+    for src in src_uri:
+        # Strip options.
+        # TODO: ignore files with apply=false?
+        src = src.split(';', maxsplit=1)[0]
+        src_type = src.split('://', maxsplit=1)[0]
+        if src_type == 'file':
+            # TODO: Get full path of patches and other files within the source
+            # repo, not just the filesystem?
+            fetch = bb.fetch2.Fetch([], data)
+            local = fetch.localpath(src)
+            manifest_file.write('  - "%s"\n' % local)
+        else:
+            manifest_file.write('  - "%s"\n' % src)
+            if src_type != 'http' and src_type != 'https' and src_type != 'ftp' and src_type != 'ssh':
+                repos.append(src)
+    if len(repos) > 1:
+        print('Multiple repos for one package are not supported. Package: %s' % info.pn)
+    for repo in repos:
+        vcs_type, url = repo.split('://', maxsplit=1)
+        manifest_file.write('  vcs:\n')
+        if vcs_type == 'gitsm':
+            vcs_type = 'git'
+        manifest_file.write('    type: "%s"\n' % vcs_type)
+        manifest_file.write('    url: "%s"\n' % url)
+        # TODO: Actually support multiple repos here:
+        # TODO: catch and replace AUTOINC?
+        manifest_file.write('    revision: "%s"\n' % srcrev)
+        manifest_file.write('    branch: "%s"\n' % branch)
+
+
+def find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order):
+    data = recipe_info[rn]
+    # Filter out packages from the assume_provided list.
+    depends = []
+    for dep in data.depends:
+        if dep not in assume_provided:
+            depends.append(dep)
+
+    if PRINT_PROGRESS:
+        # Print high-order dependencies as a form of logging/progress notifcation.
+        if order == 2:
+            print(rn)
+        if order == 3:
+            print('  ' + rn)
+
+    # order == 1 is for the initial recipe. We've already printed its
+    # information, so skip it.
+    if order > 1:
+        spaces = '  ' * order
+        manifest_file.write('%s- namespace: ""\n' % spaces)
+        manifest_file.write('%s  name: "%s"\n' % (spaces, data.pn))
+        manifest_file.write('%s  version: "%s"\n' % (spaces, data.pv))
+        if not depends:
+            manifest_file.write('%s  dependencies: []\n' % spaces)
+        else:
+            manifest_file.write('%s  dependencies:\n' % spaces)
+
+    # First find all dependencies not seen yet to our master list.
+    for dep in depends:
+        if dep not in packages:
+            packages.append(dep)
+            dep_data = get_recipe_info(tinfoil, dep)
+            # Do this once now to reduce the number of bitbake calls.
+            dep_data.depends = dep_data.getVar('DEPENDS').split()
+            recipe_info[dep] = dep_data
+
+    # Then recursively analyze all of the dependencies for the current recipe.
+    for dep in depends:
+        find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, dep, order + 1)
+
+    if order > 1:
+        manifest_file.write('%s  errors: []\n' % spaces)
+
+
+def main():
+    parser = ArgumentParser(description='Find all dependencies of a recipe.')
+    parser.add_argument('recipe', metavar='recipe', help='a recipe to investigate')
+    args = parser.parse_args()
+    rn = args.recipe
+    with bb.tinfoil.Tinfoil() as tinfoil:
+        tinfoil.prepare()
+        # These are the packages that bitbake assumes are provided by the host
+        # system. They do not have recipes, so searching tinfoil for them will
+        # not work. Anyway, by nature they are only build tools and will not be
+        # distributed in an image.
+        assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split()
+        if SKIP_BUILD_TOOLS:
+            assume_provided.extend(KNOWN_BUILD_TOOLS)
+
+        data = get_recipe_info(tinfoil, rn)
+        if not data:
+            print('Nothing to do!')
+            return
+
+        with open(rn + '-dependencies.yml', "w") as manifest_file:
+            manifest_file.write('project:\n')
+            data.depends = []
+            depends = data.getVar('DEPENDS').split()
+            for dep in depends:
+                if dep not in assume_provided:
+                    data.depends.append(dep)
+            print_package(manifest_file, data, is_project=True)
+            manifest_file.write('  scopes:\n')
+            manifest_file.write('  - name: "all"\n')
+            manifest_file.write('    delivered: true\n')
+            if not data.depends:
+                manifest_file.write('    dependencies: []\n')
+            else:
+                manifest_file.write('    dependencies:\n')
+
+            recipe_info = dict([(rn, data)])
+            packages = []
+            find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order=1)
+
+            manifest_file.write('packages:\n')
+
+            # Iterate through the list of packages found to print out their full
+            # information. Skip the initial recipe since we already printed it out.
+            for p in packages:
+                if p is not rn:
+                    data = recipe_info[p]
+                    print_package(manifest_file, data, is_project=False)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/scripts/find_packages.py b/scripts/find_packages.py
deleted file mode 100755
index 7a60a17..0000000
--- a/scripts/find_packages.py
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/usr/bin/env python3
-
-import os.path
-import sys
-
-scripts_path = os.path.dirname(os.path.realpath(__file__))
-bb_lib_path = os.path.abspath(scripts_path + '/../../poky/bitbake/lib')
-sys.path = sys.path + [bb_lib_path]
-
-import bb.fetch2
-import bb.tinfoil
-
-
-def print_deps(tinfoil, abcd_file, rn):
-    try:
-        info = tinfoil.get_recipe_info(rn)
-    except Exception:
-        print('Failed to get recipe info for: %s' % rn)
-        return []
-    if not info:
-        print('No recipe info found for: %s' % rn)
-        return []
-    append_files = tinfoil.get_file_appends(info.fn)
-    appends = True
-    data = tinfoil.parse_recipe_file(info.fn, appends, append_files)
-    src_uri = data.getVar('SRC_URI').split()
-    lic = data.getVar('LICENSE')
-    summary = data.getVar('SUMMARY')
-    description = data.getVar('DESCRIPTION')
-    homepage = data.getVar('HOMEPAGE')
-    srcrev = data.getVar('SRCREV')
-    branch = data.getVar('BRANCH')
-    depends = data.getVar('DEPENDS').split()
-
-    abcd_file.write('- id:\n')
-    abcd_file.write('    package_manager: "Yocto"\n')
-    abcd_file.write('    name: "%s"\n' % info.pn)
-    abcd_file.write('    version: "%s"\n' % info.pv)
-    abcd_file.write('  declared_lics:\n')
-    abcd_file.write('  - "%s"\n' % lic)
-    if summary:
-        abcd_file.write('  description: "%s"\n' % summary)
-    else:
-        abcd_file.write('  description: "%s"\n' % description)
-    abcd_file.write('  homepage_url: "%s"\n' % homepage)
-    abcd_file.write('  source_artifact:\n')
-    repos = []
-    for src in src_uri:
-        # Strip options.
-        # TODO: ignore files with apply=false?
-        src = src.split(';', maxsplit=1)[0]
-        src_type = src.split('://', maxsplit=1)[0]
-        if src_type == 'file':
-            # TODO: Get full path of patches and other files within the source
-            # repo, not just the filesystem?
-            fetch = bb.fetch2.Fetch([], data)
-            local = fetch.localpath(src)
-            abcd_file.write('  - "%s"\n' % local)
-        else:
-            abcd_file.write('  - "%s"\n' % src)
-            if src_type != 'http' and src_type != 'https' and src_type != 'ftp' and src_type != 'ssh':
-                repos.append(src)
-    if len(repos) > 1:
-        print('Multiple repos not fully supported yet. Pacakge: %s' % info.pn)
-    for repo in repos:
-        vcs_type, url = repo.split('://', maxsplit=1)
-        abcd_file.write('  vcs:\n')
-        if vcs_type == 'gitsm':
-            vcs_type = 'git'
-        abcd_file.write('    type: "%s"\n' % vcs_type)
-        abcd_file.write('    url: "%s"\n' % url)
-        # TODO: Actually support multiple repos here:
-        abcd_file.write('    revision: "%s"\n' % srcrev)
-        abcd_file.write('    branch: "%s"\n' % branch)
-
-    abcd_file.write('  dependencies:\n')
-    for dep in depends:
-        abcd_file.write('  - "%s"\n' % dep)
-        # TODO: search for transitive dependencies here? Each dependency will
-        # get checked for its own dependencies sooner or later.
-
-    return depends
-
-
-def main():
-    abcd_manifest = 'manifest.abcd'
-    with open(abcd_manifest, "w") as abcd_file, bb.tinfoil.Tinfoil() as tinfoil:
-        tinfoil.prepare()
-        # These are the packages that bitbake assumes are provided by the host
-        # system. They do not have recipes, so searching tinfoil for them will
-        # not work. Anyway, by nature they are not included in code we release,
-        # only used by it.
-        assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split()
-        abcd_file.write('packages:\n')
-
-        # Does NOT include garage-sign, anything used only for testing (i.e.
-        # strace and gtest), any of the git submodules, all of which are also
-        # only used for testing (tuf-test-vectors, isotp-c, ostreesysroot,
-        # and HdrHistogram_c), or any other third party modules included
-        # directly into the source tree (jsoncpp, open62541, picojson)
-        recipes_to_check = ['aktualizr',
-                            'aktualizr-native',
-                            'aktualizr-auto-prov',
-                            'aktualizr-implicit-prov',
-                            'aktualizr-ca-implicit-prov',
-                            'aktualizr-hsm-prov',
-                            'aktualizr-disable-send-ip',
-                            'aktualizr-example-interface',
-                            'aktualizr-log-debug',
-                            'libp11', # BUILD_P11 (HSM) only
-                            'dpkg', # BUILD_DEB only
-                            'systemd'] # BUILD_SYSTEMD only
-
-        # Iterate through the list of recipes to check. Append any dependencies
-        # found that aren't already in the list. As long as we only add to the
-        # list, it should be safe.
-        for recipe in recipes_to_check:
-            depends = print_deps(tinfoil, abcd_file, recipe)
-            for dep in depends:
-                if dep not in recipes_to_check and dep not in assume_provided:
-                    recipes_to_check.append(dep)
-
-
-if __name__ == "__main__":
-    main()