From 9107684a89676ca298322e71df25465a831ea7e5 Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Wed, 27 Jun 2018 10:07:42 +0200 Subject: find_packages.py: input package names to search. Use find_aktualizr_dependencies.sh to get the previous output for aktualizr-related projects. --- scripts/find_aktualizr_dependencies.sh | 24 ++++++++++++++++++++++++ scripts/find_packages.py | 34 ++++++++++++++-------------------- 2 files changed, 38 insertions(+), 20 deletions(-) create mode 100755 scripts/find_aktualizr_dependencies.sh diff --git a/scripts/find_aktualizr_dependencies.sh b/scripts/find_aktualizr_dependencies.sh new file mode 100755 index 0000000..f011e0a --- /dev/null +++ b/scripts/find_aktualizr_dependencies.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +parentdir="$(dirname "$0")" + +# Does NOT include garage-sign, anything used only for testing (i.e. strace and +# gtest), any of the git submodules, all of which are also only used for +# testing (tuf-test-vectors, isotp-c, ostreesysroot, and HdrHistogram_c), or +# any other third party modules included directly into the source tree +# (jsoncpp, open62541, picojson). Also check libp11, dpkg, and systemd since +# those are common dependencies not enabled by default. +${parentdir}/find_packages.py aktualizr \ + aktualizr-native \ + aktualizr-auto-prov \ + aktualizr-implicit-prov \ + aktualizr-ca-implicit-prov \ + aktualizr-hsm-prov \ + aktualizr-disable-send-ip \ + aktualizr-example-interface \ + aktualizr-log-debug \ + libp11 \ + dpkg \ + systemd + diff --git a/scripts/find_packages.py b/scripts/find_packages.py index 7a60a17..ad74be5 100755 --- a/scripts/find_packages.py +++ b/scripts/find_packages.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +from argparse import ArgumentParser import os.path import sys @@ -34,6 +35,7 @@ def print_deps(tinfoil, abcd_file, rn): abcd_file.write('- id:\n') abcd_file.write(' package_manager: "Yocto"\n') + abcd_file.write(' namespace: ""\n') abcd_file.write(' name: "%s"\n' % info.pn) abcd_file.write(' version: "%s"\n' % info.pv) abcd_file.write(' declared_lics:\n') @@ -43,6 +45,11 @@ def print_deps(tinfoil, abcd_file, rn): else: abcd_file.write(' description: "%s"\n' % description) abcd_file.write(' homepage_url: "%s"\n' % homepage) + # Binary artifacts almost never exist in Yocto. + abcd_file.write(' binary_artifact:\n') + abcd_file.write(' url: ""\n') + abcd_file.write(' hash: ""\n') + abcd_file.write(' hash_algorithm: ""\n') abcd_file.write(' source_artifact:\n') repos = [] for src in src_uri: @@ -61,7 +68,7 @@ def print_deps(tinfoil, abcd_file, rn): if src_type != 'http' and src_type != 'https' and src_type != 'ftp' and src_type != 'ssh': repos.append(src) if len(repos) > 1: - print('Multiple repos not fully supported yet. Pacakge: %s' % info.pn) + print('Multiple repos for one package are not supported. Package: %s' % info.pn) for repo in repos: vcs_type, url = repo.split('://', maxsplit=1) abcd_file.write(' vcs:\n') @@ -83,7 +90,12 @@ def print_deps(tinfoil, abcd_file, rn): def main(): - abcd_manifest = 'manifest.abcd' + parser = ArgumentParser(description='Find all dependencies of one or more packages.') + parser.add_argument('packages', metavar='package', nargs='+', + help='a package to investigate') + args = parser.parse_args() + recipes_to_check = args.packages + abcd_manifest = 'manifest.yml' with open(abcd_manifest, "w") as abcd_file, bb.tinfoil.Tinfoil() as tinfoil: tinfoil.prepare() # These are the packages that bitbake assumes are provided by the host @@ -93,24 +105,6 @@ def main(): assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split() abcd_file.write('packages:\n') - # Does NOT include garage-sign, anything used only for testing (i.e. - # strace and gtest), any of the git submodules, all of which are also - # only used for testing (tuf-test-vectors, isotp-c, ostreesysroot, - # and HdrHistogram_c), or any other third party modules included - # directly into the source tree (jsoncpp, open62541, picojson) - recipes_to_check = ['aktualizr', - 'aktualizr-native', - 'aktualizr-auto-prov', - 'aktualizr-implicit-prov', - 'aktualizr-ca-implicit-prov', - 'aktualizr-hsm-prov', - 'aktualizr-disable-send-ip', - 'aktualizr-example-interface', - 'aktualizr-log-debug', - 'libp11', # BUILD_P11 (HSM) only - 'dpkg', # BUILD_DEB only - 'systemd'] # BUILD_SYSTEMD only - # Iterate through the list of recipes to check. Append any dependencies # found that aren't already in the list. As long as we only add to the # list, it should be safe. -- cgit v1.2.3-54-g00ecf From ce151d09046c990a152e1f0f7a794fe5caf9df5a Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Wed, 27 Jun 2018 11:41:26 +0200 Subject: find_packages.py: Substantial rewrite based on OSS team feedback. * Only one recipe is investigated at a time. * Output is hopefully closer to the desired format. * Renamed the output file. * Skip some basic native and virtual (cross-compilation) build tools so that the output is not monstrously large. --- scripts/find_packages.py | 179 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 130 insertions(+), 49 deletions(-) diff --git a/scripts/find_packages.py b/scripts/find_packages.py index ad74be5..53b3ca7 100755 --- a/scripts/find_packages.py +++ b/scripts/find_packages.py @@ -12,7 +12,29 @@ import bb.fetch2 import bb.tinfoil -def print_deps(tinfoil, abcd_file, rn): +PRINT_PROGRESS = True +SKIP_BUILD_TOOLS = True +KNOWN_BUILD_TOOLS = ['virtual/x86_64-poky-linux-gcc', # gcc-cross-x86_64 + 'virtual/x86_64-poky-linux-compilerlibs', # gcc-runtime + 'virtual/libc', # glibc + 'virtual/libintl', # glibc + 'virtual/libiconv', # glibc + 'virtual/crypt', # glibc + 'autoconf-native', + 'automake-native', + 'libtool-native', + 'gnu-config-native', + 'm4-native', + 'texinfo-dummy-native', + 'gettext-minimal-native', + 'libtool-cross', + 'gettext-native', + 'util-linux-native', + 'pkgconfig-native', + 'makedepend-native'] + + +def get_recipe_info(tinfoil, rn): try: info = tinfoil.get_recipe_info(rn) except Exception: @@ -24,33 +46,43 @@ def print_deps(tinfoil, abcd_file, rn): append_files = tinfoil.get_file_appends(info.fn) appends = True data = tinfoil.parse_recipe_file(info.fn, appends, append_files) + data.pn = info.pn + data.pv = info.pv + return data + + +def print_package(manifest_file, data, is_project): src_uri = data.getVar('SRC_URI').split() lic = data.getVar('LICENSE') summary = data.getVar('SUMMARY') - description = data.getVar('DESCRIPTION') homepage = data.getVar('HOMEPAGE') srcrev = data.getVar('SRCREV') branch = data.getVar('BRANCH') - depends = data.getVar('DEPENDS').split() - - abcd_file.write('- id:\n') - abcd_file.write(' package_manager: "Yocto"\n') - abcd_file.write(' namespace: ""\n') - abcd_file.write(' name: "%s"\n' % info.pn) - abcd_file.write(' version: "%s"\n' % info.pv) - abcd_file.write(' declared_lics:\n') - abcd_file.write(' - "%s"\n' % lic) + + if is_project: + manifest_file.write(' id:\n') + else: + manifest_file.write('- id:\n') + manifest_file.write(' package_manager: "Yocto"\n') + manifest_file.write(' namespace: ""\n') + manifest_file.write(' name: "%s"\n' % data.pn) + manifest_file.write(' version: "%s"\n' % data.pv) + manifest_file.write(' declared_lics:\n') + manifest_file.write(' - "%s"\n' % lic) + if is_project: + manifest_file.write(' aliases: []\n') if summary: - abcd_file.write(' description: "%s"\n' % summary) + manifest_file.write(' description: "%s"\n' % summary) else: - abcd_file.write(' description: "%s"\n' % description) - abcd_file.write(' homepage_url: "%s"\n' % homepage) + description = data.getVar('DESCRIPTION') + manifest_file.write(' description: "%s"\n' % description) + manifest_file.write(' homepage_url: "%s"\n' % homepage) # Binary artifacts almost never exist in Yocto. - abcd_file.write(' binary_artifact:\n') - abcd_file.write(' url: ""\n') - abcd_file.write(' hash: ""\n') - abcd_file.write(' hash_algorithm: ""\n') - abcd_file.write(' source_artifact:\n') + manifest_file.write(' binary_artifact:\n') + manifest_file.write(' url: ""\n') + manifest_file.write(' hash: ""\n') + manifest_file.write(' hash_algorithm: ""\n') + manifest_file.write(' source_artifact:\n') repos = [] for src in src_uri: # Strip options. @@ -62,57 +94,106 @@ def print_deps(tinfoil, abcd_file, rn): # repo, not just the filesystem? fetch = bb.fetch2.Fetch([], data) local = fetch.localpath(src) - abcd_file.write(' - "%s"\n' % local) + manifest_file.write(' - "%s"\n' % local) else: - abcd_file.write(' - "%s"\n' % src) + manifest_file.write(' - "%s"\n' % src) if src_type != 'http' and src_type != 'https' and src_type != 'ftp' and src_type != 'ssh': repos.append(src) if len(repos) > 1: print('Multiple repos for one package are not supported. Package: %s' % info.pn) for repo in repos: vcs_type, url = repo.split('://', maxsplit=1) - abcd_file.write(' vcs:\n') + manifest_file.write(' vcs:\n') if vcs_type == 'gitsm': vcs_type = 'git' - abcd_file.write(' type: "%s"\n' % vcs_type) - abcd_file.write(' url: "%s"\n' % url) + manifest_file.write(' type: "%s"\n' % vcs_type) + manifest_file.write(' url: "%s"\n' % url) # TODO: Actually support multiple repos here: - abcd_file.write(' revision: "%s"\n' % srcrev) - abcd_file.write(' branch: "%s"\n' % branch) + # TODO: catch and replace AUTOINC? + manifest_file.write(' revision: "%s"\n' % srcrev) + manifest_file.write(' branch: "%s"\n' % branch) + + +def find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order): + data = recipe_info[rn] + depends = data.depends - abcd_file.write(' dependencies:\n') + # order == 1 is for the initial recipe. We've already printed its + # information, so skip it. + if order > 1: + spaces = ' ' * order + manifest_file.write('%s- namespace: ""\n' % spaces) + manifest_file.write('%s name: "%s"\n' % (spaces, data.pn)) + manifest_file.write('%s version: "%s"\n' % (spaces, data.pv)) + if not depends: + manifest_file.write('%s dependencies: []\n' % spaces) + else: + manifest_file.write('%s dependencies:\n' % spaces) + + if PRINT_PROGRESS: + # Print high-order dependencies as a form of logging/progress notifcation. + if order == 2: + print(rn) + if order == 3: + print(' ' + rn) + + # First find all dependencies not seen yet to our master list. + for dep in depends: + if dep not in packages and dep not in assume_provided: + packages.append(dep) + dep_data = get_recipe_info(tinfoil, dep) + # Do this once now to reduce the number of bitbake calls. + dep_data.depends = dep_data.getVar('DEPENDS').split() + recipe_info[dep] = dep_data + # Then recursively analyze all of the dependencies for the current recipe. for dep in depends: - abcd_file.write(' - "%s"\n' % dep) - # TODO: search for transitive dependencies here? Each dependency will - # get checked for its own dependencies sooner or later. + if dep not in assume_provided: + find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, dep, order + 1) - return depends + if order > 1: + manifest_file.write('%s errors: []\n' % spaces) def main(): - parser = ArgumentParser(description='Find all dependencies of one or more packages.') - parser.add_argument('packages', metavar='package', nargs='+', - help='a package to investigate') + parser = ArgumentParser(description='Find all dependencies of a recipe.') + parser.add_argument('recipe', metavar='recipe', help='a recipe to investigate') args = parser.parse_args() - recipes_to_check = args.packages - abcd_manifest = 'manifest.yml' - with open(abcd_manifest, "w") as abcd_file, bb.tinfoil.Tinfoil() as tinfoil: + rn = args.recipe + with open(rn + '-dependencies.yml', "w") as manifest_file, bb.tinfoil.Tinfoil() as tinfoil: tinfoil.prepare() # These are the packages that bitbake assumes are provided by the host # system. They do not have recipes, so searching tinfoil for them will - # not work. Anyway, by nature they are not included in code we release, - # only used by it. + # not work. Anyway, by nature they are only build tools and will not be + # distributed in an image. assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split() - abcd_file.write('packages:\n') - - # Iterate through the list of recipes to check. Append any dependencies - # found that aren't already in the list. As long as we only add to the - # list, it should be safe. - for recipe in recipes_to_check: - depends = print_deps(tinfoil, abcd_file, recipe) - for dep in depends: - if dep not in recipes_to_check and dep not in assume_provided: - recipes_to_check.append(dep) + if SKIP_BUILD_TOOLS: + assume_provided.extend(KNOWN_BUILD_TOOLS) + + manifest_file.write('project:\n') + data = get_recipe_info(tinfoil, rn) + data.depends = [] + depends = data.getVar('DEPENDS').split() + for dep in depends: + if dep not in assume_provided: + data.depends.append(dep) + print_package(manifest_file, data, is_project=True) + manifest_file.write(' scopes:\n') + manifest_file.write(' - name: "all"\n') + manifest_file.write(' delivered: true\n') + manifest_file.write(' dependencies:\n') + + recipe_info = dict([(rn, data)]) + packages = [] + find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order=1) + + manifest_file.write('packages:\n') + + # Iterate through the list of packages found to print out their full + # information. Skip the initial recipe since we already printed it out. + for p in packages: + if p is not rn: + data = recipe_info[p] + print_package(manifest_file, data, is_project=False) if __name__ == "__main__": -- cgit v1.2.3-54-g00ecf From d7c9805b8c6e3d05167d1ac8aeb6b2ea3a8afc0d Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Thu, 28 Jun 2018 13:55:10 +0200 Subject: find_packages.py: Do not write anything if recipe is not found. --- scripts/find_packages.py | 55 ++++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/scripts/find_packages.py b/scripts/find_packages.py index 53b3ca7..893ad85 100755 --- a/scripts/find_packages.py +++ b/scripts/find_packages.py @@ -159,7 +159,7 @@ def main(): parser.add_argument('recipe', metavar='recipe', help='a recipe to investigate') args = parser.parse_args() rn = args.recipe - with open(rn + '-dependencies.yml', "w") as manifest_file, bb.tinfoil.Tinfoil() as tinfoil: + with bb.tinfoil.Tinfoil() as tinfoil: tinfoil.prepare() # These are the packages that bitbake assumes are provided by the host # system. They do not have recipes, so searching tinfoil for them will @@ -169,31 +169,36 @@ def main(): if SKIP_BUILD_TOOLS: assume_provided.extend(KNOWN_BUILD_TOOLS) - manifest_file.write('project:\n') data = get_recipe_info(tinfoil, rn) - data.depends = [] - depends = data.getVar('DEPENDS').split() - for dep in depends: - if dep not in assume_provided: - data.depends.append(dep) - print_package(manifest_file, data, is_project=True) - manifest_file.write(' scopes:\n') - manifest_file.write(' - name: "all"\n') - manifest_file.write(' delivered: true\n') - manifest_file.write(' dependencies:\n') - - recipe_info = dict([(rn, data)]) - packages = [] - find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order=1) - - manifest_file.write('packages:\n') - - # Iterate through the list of packages found to print out their full - # information. Skip the initial recipe since we already printed it out. - for p in packages: - if p is not rn: - data = recipe_info[p] - print_package(manifest_file, data, is_project=False) + if not data: + print('Nothing to do!') + return + + with open(rn + '-dependencies.yml', "w") as manifest_file: + manifest_file.write('project:\n') + data.depends = [] + depends = data.getVar('DEPENDS').split() + for dep in depends: + if dep not in assume_provided: + data.depends.append(dep) + print_package(manifest_file, data, is_project=True) + manifest_file.write(' scopes:\n') + manifest_file.write(' - name: "all"\n') + manifest_file.write(' delivered: true\n') + manifest_file.write(' dependencies:\n') + + recipe_info = dict([(rn, data)]) + packages = [] + find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order=1) + + manifest_file.write('packages:\n') + + # Iterate through the list of packages found to print out their full + # information. Skip the initial recipe since we already printed it out. + for p in packages: + if p is not rn: + data = recipe_info[p] + print_package(manifest_file, data, is_project=False) if __name__ == "__main__": -- cgit v1.2.3-54-g00ecf From 4302f526c8f2ced7dadbb5f2e5c5de770b4cd12d Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Thu, 28 Jun 2018 14:03:00 +0200 Subject: find_packages.py: Hide some i586-specific virtual packages. --- scripts/find_packages.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/find_packages.py b/scripts/find_packages.py index 893ad85..0192912 100755 --- a/scripts/find_packages.py +++ b/scripts/find_packages.py @@ -16,6 +16,8 @@ PRINT_PROGRESS = True SKIP_BUILD_TOOLS = True KNOWN_BUILD_TOOLS = ['virtual/x86_64-poky-linux-gcc', # gcc-cross-x86_64 'virtual/x86_64-poky-linux-compilerlibs', # gcc-runtime + 'virtual/i586-poky-linux-gcc', # gcc-cross-i586 + 'virtual/i586-poky-linux-compilerlibs', # gcc-runtime 'virtual/libc', # glibc 'virtual/libintl', # glibc 'virtual/libiconv', # glibc -- cgit v1.2.3-54-g00ecf From 8109f0a7555566d37a8f10f3ee97887a47a438e6 Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Thu, 28 Jun 2018 14:03:59 +0200 Subject: find_dependencies.py: rename from find_packages.py. More generic, hopefully a bit clearer. The input must be a recipe, but recipes depend on packages. --- scripts/find_dependencies.py | 207 +++++++++++++++++++++++++++++++++++++++++++ scripts/find_packages.py | 207 ------------------------------------------- 2 files changed, 207 insertions(+), 207 deletions(-) create mode 100755 scripts/find_dependencies.py delete mode 100755 scripts/find_packages.py diff --git a/scripts/find_dependencies.py b/scripts/find_dependencies.py new file mode 100755 index 0000000..0192912 --- /dev/null +++ b/scripts/find_dependencies.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 + +from argparse import ArgumentParser +import os.path +import sys + +scripts_path = os.path.dirname(os.path.realpath(__file__)) +bb_lib_path = os.path.abspath(scripts_path + '/../../poky/bitbake/lib') +sys.path = sys.path + [bb_lib_path] + +import bb.fetch2 +import bb.tinfoil + + +PRINT_PROGRESS = True +SKIP_BUILD_TOOLS = True +KNOWN_BUILD_TOOLS = ['virtual/x86_64-poky-linux-gcc', # gcc-cross-x86_64 + 'virtual/x86_64-poky-linux-compilerlibs', # gcc-runtime + 'virtual/i586-poky-linux-gcc', # gcc-cross-i586 + 'virtual/i586-poky-linux-compilerlibs', # gcc-runtime + 'virtual/libc', # glibc + 'virtual/libintl', # glibc + 'virtual/libiconv', # glibc + 'virtual/crypt', # glibc + 'autoconf-native', + 'automake-native', + 'libtool-native', + 'gnu-config-native', + 'm4-native', + 'texinfo-dummy-native', + 'gettext-minimal-native', + 'libtool-cross', + 'gettext-native', + 'util-linux-native', + 'pkgconfig-native', + 'makedepend-native'] + + +def get_recipe_info(tinfoil, rn): + try: + info = tinfoil.get_recipe_info(rn) + except Exception: + print('Failed to get recipe info for: %s' % rn) + return [] + if not info: + print('No recipe info found for: %s' % rn) + return [] + append_files = tinfoil.get_file_appends(info.fn) + appends = True + data = tinfoil.parse_recipe_file(info.fn, appends, append_files) + data.pn = info.pn + data.pv = info.pv + return data + + +def print_package(manifest_file, data, is_project): + src_uri = data.getVar('SRC_URI').split() + lic = data.getVar('LICENSE') + summary = data.getVar('SUMMARY') + homepage = data.getVar('HOMEPAGE') + srcrev = data.getVar('SRCREV') + branch = data.getVar('BRANCH') + + if is_project: + manifest_file.write(' id:\n') + else: + manifest_file.write('- id:\n') + manifest_file.write(' package_manager: "Yocto"\n') + manifest_file.write(' namespace: ""\n') + manifest_file.write(' name: "%s"\n' % data.pn) + manifest_file.write(' version: "%s"\n' % data.pv) + manifest_file.write(' declared_lics:\n') + manifest_file.write(' - "%s"\n' % lic) + if is_project: + manifest_file.write(' aliases: []\n') + if summary: + manifest_file.write(' description: "%s"\n' % summary) + else: + description = data.getVar('DESCRIPTION') + manifest_file.write(' description: "%s"\n' % description) + manifest_file.write(' homepage_url: "%s"\n' % homepage) + # Binary artifacts almost never exist in Yocto. + manifest_file.write(' binary_artifact:\n') + manifest_file.write(' url: ""\n') + manifest_file.write(' hash: ""\n') + manifest_file.write(' hash_algorithm: ""\n') + manifest_file.write(' source_artifact:\n') + repos = [] + for src in src_uri: + # Strip options. + # TODO: ignore files with apply=false? + src = src.split(';', maxsplit=1)[0] + src_type = src.split('://', maxsplit=1)[0] + if src_type == 'file': + # TODO: Get full path of patches and other files within the source + # repo, not just the filesystem? + fetch = bb.fetch2.Fetch([], data) + local = fetch.localpath(src) + manifest_file.write(' - "%s"\n' % local) + else: + manifest_file.write(' - "%s"\n' % src) + if src_type != 'http' and src_type != 'https' and src_type != 'ftp' and src_type != 'ssh': + repos.append(src) + if len(repos) > 1: + print('Multiple repos for one package are not supported. Package: %s' % info.pn) + for repo in repos: + vcs_type, url = repo.split('://', maxsplit=1) + manifest_file.write(' vcs:\n') + if vcs_type == 'gitsm': + vcs_type = 'git' + manifest_file.write(' type: "%s"\n' % vcs_type) + manifest_file.write(' url: "%s"\n' % url) + # TODO: Actually support multiple repos here: + # TODO: catch and replace AUTOINC? + manifest_file.write(' revision: "%s"\n' % srcrev) + manifest_file.write(' branch: "%s"\n' % branch) + + +def find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order): + data = recipe_info[rn] + depends = data.depends + + # order == 1 is for the initial recipe. We've already printed its + # information, so skip it. + if order > 1: + spaces = ' ' * order + manifest_file.write('%s- namespace: ""\n' % spaces) + manifest_file.write('%s name: "%s"\n' % (spaces, data.pn)) + manifest_file.write('%s version: "%s"\n' % (spaces, data.pv)) + if not depends: + manifest_file.write('%s dependencies: []\n' % spaces) + else: + manifest_file.write('%s dependencies:\n' % spaces) + + if PRINT_PROGRESS: + # Print high-order dependencies as a form of logging/progress notifcation. + if order == 2: + print(rn) + if order == 3: + print(' ' + rn) + + # First find all dependencies not seen yet to our master list. + for dep in depends: + if dep not in packages and dep not in assume_provided: + packages.append(dep) + dep_data = get_recipe_info(tinfoil, dep) + # Do this once now to reduce the number of bitbake calls. + dep_data.depends = dep_data.getVar('DEPENDS').split() + recipe_info[dep] = dep_data + # Then recursively analyze all of the dependencies for the current recipe. + for dep in depends: + if dep not in assume_provided: + find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, dep, order + 1) + + if order > 1: + manifest_file.write('%s errors: []\n' % spaces) + + +def main(): + parser = ArgumentParser(description='Find all dependencies of a recipe.') + parser.add_argument('recipe', metavar='recipe', help='a recipe to investigate') + args = parser.parse_args() + rn = args.recipe + with bb.tinfoil.Tinfoil() as tinfoil: + tinfoil.prepare() + # These are the packages that bitbake assumes are provided by the host + # system. They do not have recipes, so searching tinfoil for them will + # not work. Anyway, by nature they are only build tools and will not be + # distributed in an image. + assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split() + if SKIP_BUILD_TOOLS: + assume_provided.extend(KNOWN_BUILD_TOOLS) + + data = get_recipe_info(tinfoil, rn) + if not data: + print('Nothing to do!') + return + + with open(rn + '-dependencies.yml', "w") as manifest_file: + manifest_file.write('project:\n') + data.depends = [] + depends = data.getVar('DEPENDS').split() + for dep in depends: + if dep not in assume_provided: + data.depends.append(dep) + print_package(manifest_file, data, is_project=True) + manifest_file.write(' scopes:\n') + manifest_file.write(' - name: "all"\n') + manifest_file.write(' delivered: true\n') + manifest_file.write(' dependencies:\n') + + recipe_info = dict([(rn, data)]) + packages = [] + find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order=1) + + manifest_file.write('packages:\n') + + # Iterate through the list of packages found to print out their full + # information. Skip the initial recipe since we already printed it out. + for p in packages: + if p is not rn: + data = recipe_info[p] + print_package(manifest_file, data, is_project=False) + + +if __name__ == "__main__": + main() diff --git a/scripts/find_packages.py b/scripts/find_packages.py deleted file mode 100755 index 0192912..0000000 --- a/scripts/find_packages.py +++ /dev/null @@ -1,207 +0,0 @@ -#!/usr/bin/env python3 - -from argparse import ArgumentParser -import os.path -import sys - -scripts_path = os.path.dirname(os.path.realpath(__file__)) -bb_lib_path = os.path.abspath(scripts_path + '/../../poky/bitbake/lib') -sys.path = sys.path + [bb_lib_path] - -import bb.fetch2 -import bb.tinfoil - - -PRINT_PROGRESS = True -SKIP_BUILD_TOOLS = True -KNOWN_BUILD_TOOLS = ['virtual/x86_64-poky-linux-gcc', # gcc-cross-x86_64 - 'virtual/x86_64-poky-linux-compilerlibs', # gcc-runtime - 'virtual/i586-poky-linux-gcc', # gcc-cross-i586 - 'virtual/i586-poky-linux-compilerlibs', # gcc-runtime - 'virtual/libc', # glibc - 'virtual/libintl', # glibc - 'virtual/libiconv', # glibc - 'virtual/crypt', # glibc - 'autoconf-native', - 'automake-native', - 'libtool-native', - 'gnu-config-native', - 'm4-native', - 'texinfo-dummy-native', - 'gettext-minimal-native', - 'libtool-cross', - 'gettext-native', - 'util-linux-native', - 'pkgconfig-native', - 'makedepend-native'] - - -def get_recipe_info(tinfoil, rn): - try: - info = tinfoil.get_recipe_info(rn) - except Exception: - print('Failed to get recipe info for: %s' % rn) - return [] - if not info: - print('No recipe info found for: %s' % rn) - return [] - append_files = tinfoil.get_file_appends(info.fn) - appends = True - data = tinfoil.parse_recipe_file(info.fn, appends, append_files) - data.pn = info.pn - data.pv = info.pv - return data - - -def print_package(manifest_file, data, is_project): - src_uri = data.getVar('SRC_URI').split() - lic = data.getVar('LICENSE') - summary = data.getVar('SUMMARY') - homepage = data.getVar('HOMEPAGE') - srcrev = data.getVar('SRCREV') - branch = data.getVar('BRANCH') - - if is_project: - manifest_file.write(' id:\n') - else: - manifest_file.write('- id:\n') - manifest_file.write(' package_manager: "Yocto"\n') - manifest_file.write(' namespace: ""\n') - manifest_file.write(' name: "%s"\n' % data.pn) - manifest_file.write(' version: "%s"\n' % data.pv) - manifest_file.write(' declared_lics:\n') - manifest_file.write(' - "%s"\n' % lic) - if is_project: - manifest_file.write(' aliases: []\n') - if summary: - manifest_file.write(' description: "%s"\n' % summary) - else: - description = data.getVar('DESCRIPTION') - manifest_file.write(' description: "%s"\n' % description) - manifest_file.write(' homepage_url: "%s"\n' % homepage) - # Binary artifacts almost never exist in Yocto. - manifest_file.write(' binary_artifact:\n') - manifest_file.write(' url: ""\n') - manifest_file.write(' hash: ""\n') - manifest_file.write(' hash_algorithm: ""\n') - manifest_file.write(' source_artifact:\n') - repos = [] - for src in src_uri: - # Strip options. - # TODO: ignore files with apply=false? - src = src.split(';', maxsplit=1)[0] - src_type = src.split('://', maxsplit=1)[0] - if src_type == 'file': - # TODO: Get full path of patches and other files within the source - # repo, not just the filesystem? - fetch = bb.fetch2.Fetch([], data) - local = fetch.localpath(src) - manifest_file.write(' - "%s"\n' % local) - else: - manifest_file.write(' - "%s"\n' % src) - if src_type != 'http' and src_type != 'https' and src_type != 'ftp' and src_type != 'ssh': - repos.append(src) - if len(repos) > 1: - print('Multiple repos for one package are not supported. Package: %s' % info.pn) - for repo in repos: - vcs_type, url = repo.split('://', maxsplit=1) - manifest_file.write(' vcs:\n') - if vcs_type == 'gitsm': - vcs_type = 'git' - manifest_file.write(' type: "%s"\n' % vcs_type) - manifest_file.write(' url: "%s"\n' % url) - # TODO: Actually support multiple repos here: - # TODO: catch and replace AUTOINC? - manifest_file.write(' revision: "%s"\n' % srcrev) - manifest_file.write(' branch: "%s"\n' % branch) - - -def find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order): - data = recipe_info[rn] - depends = data.depends - - # order == 1 is for the initial recipe. We've already printed its - # information, so skip it. - if order > 1: - spaces = ' ' * order - manifest_file.write('%s- namespace: ""\n' % spaces) - manifest_file.write('%s name: "%s"\n' % (spaces, data.pn)) - manifest_file.write('%s version: "%s"\n' % (spaces, data.pv)) - if not depends: - manifest_file.write('%s dependencies: []\n' % spaces) - else: - manifest_file.write('%s dependencies:\n' % spaces) - - if PRINT_PROGRESS: - # Print high-order dependencies as a form of logging/progress notifcation. - if order == 2: - print(rn) - if order == 3: - print(' ' + rn) - - # First find all dependencies not seen yet to our master list. - for dep in depends: - if dep not in packages and dep not in assume_provided: - packages.append(dep) - dep_data = get_recipe_info(tinfoil, dep) - # Do this once now to reduce the number of bitbake calls. - dep_data.depends = dep_data.getVar('DEPENDS').split() - recipe_info[dep] = dep_data - # Then recursively analyze all of the dependencies for the current recipe. - for dep in depends: - if dep not in assume_provided: - find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, dep, order + 1) - - if order > 1: - manifest_file.write('%s errors: []\n' % spaces) - - -def main(): - parser = ArgumentParser(description='Find all dependencies of a recipe.') - parser.add_argument('recipe', metavar='recipe', help='a recipe to investigate') - args = parser.parse_args() - rn = args.recipe - with bb.tinfoil.Tinfoil() as tinfoil: - tinfoil.prepare() - # These are the packages that bitbake assumes are provided by the host - # system. They do not have recipes, so searching tinfoil for them will - # not work. Anyway, by nature they are only build tools and will not be - # distributed in an image. - assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split() - if SKIP_BUILD_TOOLS: - assume_provided.extend(KNOWN_BUILD_TOOLS) - - data = get_recipe_info(tinfoil, rn) - if not data: - print('Nothing to do!') - return - - with open(rn + '-dependencies.yml', "w") as manifest_file: - manifest_file.write('project:\n') - data.depends = [] - depends = data.getVar('DEPENDS').split() - for dep in depends: - if dep not in assume_provided: - data.depends.append(dep) - print_package(manifest_file, data, is_project=True) - manifest_file.write(' scopes:\n') - manifest_file.write(' - name: "all"\n') - manifest_file.write(' delivered: true\n') - manifest_file.write(' dependencies:\n') - - recipe_info = dict([(rn, data)]) - packages = [] - find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order=1) - - manifest_file.write('packages:\n') - - # Iterate through the list of packages found to print out their full - # information. Skip the initial recipe since we already printed it out. - for p in packages: - if p is not rn: - data = recipe_info[p] - print_package(manifest_file, data, is_project=False) - - -if __name__ == "__main__": - main() -- cgit v1.2.3-54-g00ecf From 820466ca924b7f633dd44197c1fbc1569261a371 Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Thu, 28 Jun 2018 14:21:51 +0200 Subject: find_aktualizr_dependencies.sh: Run once per recipe. --- scripts/find_aktualizr_dependencies.sh | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/scripts/find_aktualizr_dependencies.sh b/scripts/find_aktualizr_dependencies.sh index f011e0a..786d8a9 100755 --- a/scripts/find_aktualizr_dependencies.sh +++ b/scripts/find_aktualizr_dependencies.sh @@ -9,16 +9,15 @@ parentdir="$(dirname "$0")" # any other third party modules included directly into the source tree # (jsoncpp, open62541, picojson). Also check libp11, dpkg, and systemd since # those are common dependencies not enabled by default. -${parentdir}/find_packages.py aktualizr \ - aktualizr-native \ - aktualizr-auto-prov \ - aktualizr-implicit-prov \ - aktualizr-ca-implicit-prov \ - aktualizr-hsm-prov \ - aktualizr-disable-send-ip \ - aktualizr-example-interface \ - aktualizr-log-debug \ - libp11 \ - dpkg \ - systemd +${parentdir}/find_dependencies.py aktualizr +${parentdir}/find_dependencies.py aktualizr-auto-prov +${parentdir}/find_dependencies.py aktualizr-implicit-prov +${parentdir}/find_dependencies.py aktualizr-ca-implicit-prov +${parentdir}/find_dependencies.py aktualizr-hsm-prov +${parentdir}/find_dependencies.py aktualizr-disable-send-ip +${parentdir}/find_dependencies.py aktualizr-example-interface +${parentdir}/find_dependencies.py aktualizr-log-debug +${parentdir}/find_dependencies.py libp11 +${parentdir}/find_dependencies.py dpkg +${parentdir}/find_dependencies.py systemd -- cgit v1.2.3-54-g00ecf From a9e7ea37fd7a86f6cd687908f8d95747a07c6765 Mon Sep 17 00:00:00 2001 From: Patrick Vacek Date: Thu, 28 Jun 2018 14:22:12 +0200 Subject: find_dependencies.py: Fix cases with no meaningful dependencies. --- scripts/find_dependencies.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/scripts/find_dependencies.py b/scripts/find_dependencies.py index 0192912..cffe32b 100755 --- a/scripts/find_dependencies.py +++ b/scripts/find_dependencies.py @@ -118,7 +118,18 @@ def print_package(manifest_file, data, is_project): def find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order): data = recipe_info[rn] - depends = data.depends + # Filter out packages from the assume_provided list. + depends = [] + for dep in data.depends: + if dep not in assume_provided: + depends.append(dep) + + if PRINT_PROGRESS: + # Print high-order dependencies as a form of logging/progress notifcation. + if order == 2: + print(rn) + if order == 3: + print(' ' + rn) # order == 1 is for the initial recipe. We've already printed its # information, so skip it. @@ -132,25 +143,18 @@ def find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, pack else: manifest_file.write('%s dependencies:\n' % spaces) - if PRINT_PROGRESS: - # Print high-order dependencies as a form of logging/progress notifcation. - if order == 2: - print(rn) - if order == 3: - print(' ' + rn) - # First find all dependencies not seen yet to our master list. for dep in depends: - if dep not in packages and dep not in assume_provided: + if dep not in packages: packages.append(dep) dep_data = get_recipe_info(tinfoil, dep) # Do this once now to reduce the number of bitbake calls. dep_data.depends = dep_data.getVar('DEPENDS').split() recipe_info[dep] = dep_data + # Then recursively analyze all of the dependencies for the current recipe. for dep in depends: - if dep not in assume_provided: - find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, dep, order + 1) + find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, dep, order + 1) if order > 1: manifest_file.write('%s errors: []\n' % spaces) @@ -187,7 +191,10 @@ def main(): manifest_file.write(' scopes:\n') manifest_file.write(' - name: "all"\n') manifest_file.write(' delivered: true\n') - manifest_file.write(' dependencies:\n') + if not data.depends: + manifest_file.write(' dependencies: []\n') + else: + manifest_file.write(' dependencies:\n') recipe_info = dict([(rn, data)]) packages = [] -- cgit v1.2.3-54-g00ecf