From c3d3a0d42416867d92c77e9b5e3e8879b64b3409 Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Tue, 22 May 2018 17:56:35 +0200 Subject: find_packages.py: Initial version. Finds aktualizr package information for OSS Review Toolkit. --- scripts/find_packages.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100755 scripts/find_packages.py (limited to 'scripts') diff --git a/scripts/find_packages.py b/scripts/find_packages.py new file mode 100755 index 0000000..4b1680b --- /dev/null +++ b/scripts/find_packages.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +import os.path +import sys + +scripts_path = os.path.dirname(os.path.realpath(__file__)) +lib_path = os.path.abspath(scripts_path + '/../../poky/bitbake/lib') +sys.path = sys.path + [lib_path] + +import bb.tinfoil + + +def print_deps(tinfoil, abcd_file, rn): + try: + info = tinfoil.get_recipe_info(rn) + except: + # fails on hostperl-runtime-native, virtual/libintl-native, grep-native, virtual/libiconv-native + print('failing on: %s' % rn) + return [] + append_files = tinfoil.get_file_appends(info.fn) + appends = True + data = tinfoil.parse_recipe_file(info.fn, appends, append_files) + src_uri = data.getVar('SRC_URI').split() + lic = data.getVar('LICENSE') + summary = data.getVar('SUMMARY') + description = data.getVar('DESCRIPTION') + homepage = data.getVar('HOMEPAGE') + srcrev = data.getVar('SRCREV') + branch = data.getVar('BRANCH') + depends = data.getVar('DEPENDS').split() + + abcd_file.write('- id:\n') + abcd_file.write(' package_manager: "Yocto"\n') + abcd_file.write(' name: "%s"\n' % info.pn) + abcd_file.write(' version: "%s"\n' % info.pv) + abcd_file.write(' declared_lics:\n') + abcd_file.write(' - "%s"\n' % lic) + if summary: + abcd_file.write(' description: "%s"\n' % summary) + else: + abcd_file.write(' description: "%s"\n' % description) + abcd_file.write(' homepage_url: "%s"\n' % homepage) + abcd_file.write(' source_artifact:\n') + for src in src_uri: + # TODO: Get full path of patches? + abcd_file.write(' - "%s"\n' % src) + # TODO: Check more than the first and not just git + if src_uri and 'git' in src_uri[0]: + abcd_file.write(' vcs:\n') + abcd_file.write(' type: "git"\n') + abcd_file.write(' url: "%s"\n' % src_uri[0]) + abcd_file.write(' revision: "%s"\n' % srcrev) + abcd_file.write(' branch: "%s"\n' % branch) + + abcd_file.write(' dependencies:\n') + for dep in depends: + abcd_file.write(' - "%s"\n' % dep) + # TODO: continue nesting here? + + return depends + + +def main(): + abcd_manifest = 'manifest.abcd' + with open(abcd_manifest, "w") as abcd_file, bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare() + abcd_file.write('packages:\n') + + recipes_to_check = ['aktualizr', + 'aktualizr-native', + 'aktualizr-auto-prov', + 'aktualizr-implicit-prov', + 'aktualizr-ca-implicit-prov', + 'aktualizr-hsm-prov', + 'aktualizr-disable-send-ip', + 'aktualizr-example-interface', + 'aktualizr-log-debug'] + + for recipe in recipes_to_check: + depends = print_deps(tinfoil, abcd_file, recipe) + for dep in depends: + if dep not in recipes_to_check: + recipes_to_check.append(dep) + + +if __name__ == "__main__": + main() -- cgit v1.2.3-54-g00ecf From eae056623a6e82d5a1147ce5c0205fdd04fa8f92 Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Tue, 22 May 2018 18:05:00 +0200 Subject: find_packages.py: Catch empty info and return early. Master branch returns empty instead of throwing an exception. --- scripts/find_packages.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/find_packages.py b/scripts/find_packages.py index 4b1680b..b297d7d 100755 --- a/scripts/find_packages.py +++ b/scripts/find_packages.py @@ -15,7 +15,11 @@ def print_deps(tinfoil, abcd_file, rn): info = tinfoil.get_recipe_info(rn) except: # fails on hostperl-runtime-native, virtual/libintl-native, grep-native, virtual/libiconv-native - print('failing on: %s' % rn) + print('Failed to get recipe info for: %s' % rn) + return [] + if not info: + # fails on the above and virtual/crypt-native + print('No recipe info found for: %s' % rn) return [] append_files = tinfoil.get_file_appends(info.fn) appends = True -- cgit v1.2.3-54-g00ecf From 79f5e6ade9e4a9f9438077ba71610bf3f54afd9b Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Thu, 24 May 2018 17:04:41 +0200 Subject: find_packages.py: Check for a few unlisted aktualizr dependencies. Not all possible dependencies are specified in DEPENDS, since we don't build with everything enabled by default, so search for additional potential dependencies explicitly. Also add a comment about the things that we do not have visibility into from this process. --- scripts/find_packages.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/find_packages.py b/scripts/find_packages.py index b297d7d..f3b3a85 100755 --- a/scripts/find_packages.py +++ b/scripts/find_packages.py @@ -70,6 +70,11 @@ def main(): tinfoil.prepare() abcd_file.write('packages:\n') + # Does NOT include garage-sign, anything used only for testing (i.e. + # strace and gtest), any of the git submodules, all of which are also + # only used for testing (tuf-test-vectors, isotp-c, ostreesysroot, + # and HdrHistogram_c), or any other third party modules included + # directly into the source tree (jsoncpp, open62541, picojson) recipes_to_check = ['aktualizr', 'aktualizr-native', 'aktualizr-auto-prov', @@ -78,7 +83,10 @@ def main(): 'aktualizr-hsm-prov', 'aktualizr-disable-send-ip', 'aktualizr-example-interface', - 'aktualizr-log-debug'] + 'aktualizr-log-debug', + 'libp11', # BUILD_P11 (HSM) only + 'dpkg', # BUILD_DEB only + 'systemd'] # BUILD_SYSTEMD only for recipe in recipes_to_check: depends = print_deps(tinfoil, abcd_file, recipe) -- cgit v1.2.3-54-g00ecf From 39665ee8c87853f93d56c56754470261a0d19d1d Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Fri, 25 May 2018 12:16:46 +0200 Subject: find_packages.py: Get full path for patches and other local files. It might be even better to get the path relative to the root of the repo the files live in, but this is at least a step in the right direction. --- scripts/find_packages.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'scripts') diff --git a/scripts/find_packages.py b/scripts/find_packages.py index f3b3a85..ddd9034 100755 --- a/scripts/find_packages.py +++ b/scripts/find_packages.py @@ -4,9 +4,10 @@ import os.path import sys scripts_path = os.path.dirname(os.path.realpath(__file__)) -lib_path = os.path.abspath(scripts_path + '/../../poky/bitbake/lib') -sys.path = sys.path + [lib_path] +bb_lib_path = os.path.abspath(scripts_path + '/../../poky/bitbake/lib') +sys.path = sys.path + [bb_lib_path] +import bb.fetch2 import bb.tinfoil @@ -46,8 +47,14 @@ def print_deps(tinfoil, abcd_file, rn): abcd_file.write(' homepage_url: "%s"\n' % homepage) abcd_file.write(' source_artifact:\n') for src in src_uri: - # TODO: Get full path of patches? - abcd_file.write(' - "%s"\n' % src) + if src[0:7] == 'file://': + # TODO: Get full path of patches and other files within the source + # repo, not just the filesystem? + fetch = bb.fetch2.Fetch([], data) + local = fetch.localpath(src) + abcd_file.write(' - "%s"\n' % local) + else: + abcd_file.write(' - "%s"\n' % src) # TODO: Check more than the first and not just git if src_uri and 'git' in src_uri[0]: abcd_file.write(' vcs:\n') -- cgit v1.2.3-54-g00ecf From 6fef541a1410493881689a4b2a60d1d6b49831cd Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Fri, 25 May 2018 14:13:01 +0200 Subject: find_packages.py: Fix repo name parsing logic. Also expand/explain TODOs. --- scripts/find_packages.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/find_packages.py b/scripts/find_packages.py index ddd9034..6bad742 100755 --- a/scripts/find_packages.py +++ b/scripts/find_packages.py @@ -46,7 +46,13 @@ def print_deps(tinfoil, abcd_file, rn): abcd_file.write(' description: "%s"\n' % description) abcd_file.write(' homepage_url: "%s"\n' % homepage) abcd_file.write(' source_artifact:\n') + repos = [] for src in src_uri: + # Strip options. + # TODO: ignore files with apply=false? + semi_pos = src.find(';') + if semi_pos > 0: + src = src[:semi_pos] if src[0:7] == 'file://': # TODO: Get full path of patches and other files within the source # repo, not just the filesystem? @@ -55,18 +61,27 @@ def print_deps(tinfoil, abcd_file, rn): abcd_file.write(' - "%s"\n' % local) else: abcd_file.write(' - "%s"\n' % src) - # TODO: Check more than the first and not just git - if src_uri and 'git' in src_uri[0]: + if src[0:7] != 'http://' and src[0:8] != 'https://' and src[0:6] != 'ftp://' and src[0:6] != 'ssh://': + repos.append(src) + if len(repos) > 1: + print('Multiple repos not fully supported yet. Pacakge: %s' % info.pn) + for repo in repos: + colon_pos = repo.find(':') abcd_file.write(' vcs:\n') - abcd_file.write(' type: "git"\n') - abcd_file.write(' url: "%s"\n' % src_uri[0]) + vcs_type = repo[:colon_pos] + if vcs_type == 'gitsm': + vcs_type = 'git' + abcd_file.write(' type: "%s"\n' % vcs_type) + abcd_file.write(' url: "%s"\n' % repo[colon_pos + 3:]) + # TODO: Actually support multiple repos here: abcd_file.write(' revision: "%s"\n' % srcrev) abcd_file.write(' branch: "%s"\n' % branch) abcd_file.write(' dependencies:\n') for dep in depends: abcd_file.write(' - "%s"\n' % dep) - # TODO: continue nesting here? + # TODO: search for transitive dependencies here? Each dependency will + # get checked for its own dependencies sooner or later. return depends -- cgit v1.2.3-54-g00ecf From 87d769fc52592bc6ef90bd8287716ebbca4e1252 Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Mon, 28 May 2018 13:18:49 +0200 Subject: find_packages.py: Add explanatory comment about list appending. --- scripts/find_packages.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts') diff --git a/scripts/find_packages.py b/scripts/find_packages.py index 6bad742..50453c1 100755 --- a/scripts/find_packages.py +++ b/scripts/find_packages.py @@ -110,6 +110,9 @@ def main(): 'dpkg', # BUILD_DEB only 'systemd'] # BUILD_SYSTEMD only + # Iterate through the list of recipes to check. Append any dependencies + # found that aren't already in the list. As long as we only add to the + # list, it should be safe. for recipe in recipes_to_check: depends = print_deps(tinfoil, abcd_file, recipe) for dep in depends: -- cgit v1.2.3-54-g00ecf From c3c4896f3ce4d89516e46991aed5ad4977e4ea9c Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Mon, 28 May 2018 17:34:49 +0200 Subject: find_packages.py: Fix python style issues. --- scripts/find_packages.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) (limited to 'scripts') diff --git a/scripts/find_packages.py b/scripts/find_packages.py index 50453c1..970a25a 100755 --- a/scripts/find_packages.py +++ b/scripts/find_packages.py @@ -14,7 +14,7 @@ import bb.tinfoil def print_deps(tinfoil, abcd_file, rn): try: info = tinfoil.get_recipe_info(rn) - except: + except Exception: # fails on hostperl-runtime-native, virtual/libintl-native, grep-native, virtual/libiconv-native print('Failed to get recipe info for: %s' % rn) return [] @@ -50,10 +50,9 @@ def print_deps(tinfoil, abcd_file, rn): for src in src_uri: # Strip options. # TODO: ignore files with apply=false? - semi_pos = src.find(';') - if semi_pos > 0: - src = src[:semi_pos] - if src[0:7] == 'file://': + src = src.split(';', maxsplit=1)[0] + src_type = src.split('://', maxsplit=1)[0] + if src_type == 'file': # TODO: Get full path of patches and other files within the source # repo, not just the filesystem? fetch = bb.fetch2.Fetch([], data) @@ -61,18 +60,17 @@ def print_deps(tinfoil, abcd_file, rn): abcd_file.write(' - "%s"\n' % local) else: abcd_file.write(' - "%s"\n' % src) - if src[0:7] != 'http://' and src[0:8] != 'https://' and src[0:6] != 'ftp://' and src[0:6] != 'ssh://': + if src_type != 'http' and src_type != 'https' and src_type != 'ftp' and src_type != 'ssh': repos.append(src) if len(repos) > 1: print('Multiple repos not fully supported yet. Pacakge: %s' % info.pn) for repo in repos: - colon_pos = repo.find(':') + vcs_type, url = repo.split('://', maxsplit=1) abcd_file.write(' vcs:\n') - vcs_type = repo[:colon_pos] if vcs_type == 'gitsm': vcs_type = 'git' abcd_file.write(' type: "%s"\n' % vcs_type) - abcd_file.write(' url: "%s"\n' % repo[colon_pos + 3:]) + abcd_file.write(' url: "%s"\n' % url) # TODO: Actually support multiple repos here: abcd_file.write(' revision: "%s"\n' % srcrev) abcd_file.write(' branch: "%s"\n' % branch) -- cgit v1.2.3-54-g00ecf