summaryrefslogtreecommitdiffstats
path: root/scripts
diff options
context:
space:
mode:
authorPatrick Vacek <patrickvacek@gmail.com>2018-06-27 11:41:26 +0200
committerPatrick Vacek <patrickvacek@gmail.com>2018-06-28 12:33:56 +0200
commitce151d09046c990a152e1f0f7a794fe5caf9df5a (patch)
tree10aed7009d0c07acc7ba0b6dcf1fd48032cf72e0 /scripts
parent9107684a89676ca298322e71df25465a831ea7e5 (diff)
downloadmeta-updater-ce151d09046c990a152e1f0f7a794fe5caf9df5a.tar.gz
find_packages.py: Substantial rewrite based on OSS team feedback.
* Only one recipe is investigated at a time. * Output is hopefully closer to the desired format. * Renamed the output file. * Skip some basic native and virtual (cross-compilation) build tools so that the output is not monstrously large.
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/find_packages.py179
1 files changed, 130 insertions, 49 deletions
diff --git a/scripts/find_packages.py b/scripts/find_packages.py
index ad74be5..53b3ca7 100755
--- a/scripts/find_packages.py
+++ b/scripts/find_packages.py
@@ -12,7 +12,29 @@ import bb.fetch2
12import bb.tinfoil 12import bb.tinfoil
13 13
14 14
15def print_deps(tinfoil, abcd_file, rn): 15PRINT_PROGRESS = True
16SKIP_BUILD_TOOLS = True
17KNOWN_BUILD_TOOLS = ['virtual/x86_64-poky-linux-gcc', # gcc-cross-x86_64
18 'virtual/x86_64-poky-linux-compilerlibs', # gcc-runtime
19 'virtual/libc', # glibc
20 'virtual/libintl', # glibc
21 'virtual/libiconv', # glibc
22 'virtual/crypt', # glibc
23 'autoconf-native',
24 'automake-native',
25 'libtool-native',
26 'gnu-config-native',
27 'm4-native',
28 'texinfo-dummy-native',
29 'gettext-minimal-native',
30 'libtool-cross',
31 'gettext-native',
32 'util-linux-native',
33 'pkgconfig-native',
34 'makedepend-native']
35
36
37def get_recipe_info(tinfoil, rn):
16 try: 38 try:
17 info = tinfoil.get_recipe_info(rn) 39 info = tinfoil.get_recipe_info(rn)
18 except Exception: 40 except Exception:
@@ -24,33 +46,43 @@ def print_deps(tinfoil, abcd_file, rn):
24 append_files = tinfoil.get_file_appends(info.fn) 46 append_files = tinfoil.get_file_appends(info.fn)
25 appends = True 47 appends = True
26 data = tinfoil.parse_recipe_file(info.fn, appends, append_files) 48 data = tinfoil.parse_recipe_file(info.fn, appends, append_files)
49 data.pn = info.pn
50 data.pv = info.pv
51 return data
52
53
54def print_package(manifest_file, data, is_project):
27 src_uri = data.getVar('SRC_URI').split() 55 src_uri = data.getVar('SRC_URI').split()
28 lic = data.getVar('LICENSE') 56 lic = data.getVar('LICENSE')
29 summary = data.getVar('SUMMARY') 57 summary = data.getVar('SUMMARY')
30 description = data.getVar('DESCRIPTION')
31 homepage = data.getVar('HOMEPAGE') 58 homepage = data.getVar('HOMEPAGE')
32 srcrev = data.getVar('SRCREV') 59 srcrev = data.getVar('SRCREV')
33 branch = data.getVar('BRANCH') 60 branch = data.getVar('BRANCH')
34 depends = data.getVar('DEPENDS').split() 61
35 62 if is_project:
36 abcd_file.write('- id:\n') 63 manifest_file.write(' id:\n')
37 abcd_file.write(' package_manager: "Yocto"\n') 64 else:
38 abcd_file.write(' namespace: ""\n') 65 manifest_file.write('- id:\n')
39 abcd_file.write(' name: "%s"\n' % info.pn) 66 manifest_file.write(' package_manager: "Yocto"\n')
40 abcd_file.write(' version: "%s"\n' % info.pv) 67 manifest_file.write(' namespace: ""\n')
41 abcd_file.write(' declared_lics:\n') 68 manifest_file.write(' name: "%s"\n' % data.pn)
42 abcd_file.write(' - "%s"\n' % lic) 69 manifest_file.write(' version: "%s"\n' % data.pv)
70 manifest_file.write(' declared_lics:\n')
71 manifest_file.write(' - "%s"\n' % lic)
72 if is_project:
73 manifest_file.write(' aliases: []\n')
43 if summary: 74 if summary:
44 abcd_file.write(' description: "%s"\n' % summary) 75 manifest_file.write(' description: "%s"\n' % summary)
45 else: 76 else:
46 abcd_file.write(' description: "%s"\n' % description) 77 description = data.getVar('DESCRIPTION')
47 abcd_file.write(' homepage_url: "%s"\n' % homepage) 78 manifest_file.write(' description: "%s"\n' % description)
79 manifest_file.write(' homepage_url: "%s"\n' % homepage)
48 # Binary artifacts almost never exist in Yocto. 80 # Binary artifacts almost never exist in Yocto.
49 abcd_file.write(' binary_artifact:\n') 81 manifest_file.write(' binary_artifact:\n')
50 abcd_file.write(' url: ""\n') 82 manifest_file.write(' url: ""\n')
51 abcd_file.write(' hash: ""\n') 83 manifest_file.write(' hash: ""\n')
52 abcd_file.write(' hash_algorithm: ""\n') 84 manifest_file.write(' hash_algorithm: ""\n')
53 abcd_file.write(' source_artifact:\n') 85 manifest_file.write(' source_artifact:\n')
54 repos = [] 86 repos = []
55 for src in src_uri: 87 for src in src_uri:
56 # Strip options. 88 # Strip options.
@@ -62,57 +94,106 @@ def print_deps(tinfoil, abcd_file, rn):
62 # repo, not just the filesystem? 94 # repo, not just the filesystem?
63 fetch = bb.fetch2.Fetch([], data) 95 fetch = bb.fetch2.Fetch([], data)
64 local = fetch.localpath(src) 96 local = fetch.localpath(src)
65 abcd_file.write(' - "%s"\n' % local) 97 manifest_file.write(' - "%s"\n' % local)
66 else: 98 else:
67 abcd_file.write(' - "%s"\n' % src) 99 manifest_file.write(' - "%s"\n' % src)
68 if src_type != 'http' and src_type != 'https' and src_type != 'ftp' and src_type != 'ssh': 100 if src_type != 'http' and src_type != 'https' and src_type != 'ftp' and src_type != 'ssh':
69 repos.append(src) 101 repos.append(src)
70 if len(repos) > 1: 102 if len(repos) > 1:
71 print('Multiple repos for one package are not supported. Package: %s' % info.pn) 103 print('Multiple repos for one package are not supported. Package: %s' % info.pn)
72 for repo in repos: 104 for repo in repos:
73 vcs_type, url = repo.split('://', maxsplit=1) 105 vcs_type, url = repo.split('://', maxsplit=1)
74 abcd_file.write(' vcs:\n') 106 manifest_file.write(' vcs:\n')
75 if vcs_type == 'gitsm': 107 if vcs_type == 'gitsm':
76 vcs_type = 'git' 108 vcs_type = 'git'
77 abcd_file.write(' type: "%s"\n' % vcs_type) 109 manifest_file.write(' type: "%s"\n' % vcs_type)
78 abcd_file.write(' url: "%s"\n' % url) 110 manifest_file.write(' url: "%s"\n' % url)
79 # TODO: Actually support multiple repos here: 111 # TODO: Actually support multiple repos here:
80 abcd_file.write(' revision: "%s"\n' % srcrev) 112 # TODO: catch and replace AUTOINC?
81 abcd_file.write(' branch: "%s"\n' % branch) 113 manifest_file.write(' revision: "%s"\n' % srcrev)
114 manifest_file.write(' branch: "%s"\n' % branch)
115
116
117def find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order):
118 data = recipe_info[rn]
119 depends = data.depends
82 120
83 abcd_file.write(' dependencies:\n') 121 # order == 1 is for the initial recipe. We've already printed its
122 # information, so skip it.
123 if order > 1:
124 spaces = ' ' * order
125 manifest_file.write('%s- namespace: ""\n' % spaces)
126 manifest_file.write('%s name: "%s"\n' % (spaces, data.pn))
127 manifest_file.write('%s version: "%s"\n' % (spaces, data.pv))
128 if not depends:
129 manifest_file.write('%s dependencies: []\n' % spaces)
130 else:
131 manifest_file.write('%s dependencies:\n' % spaces)
132
133 if PRINT_PROGRESS:
134 # Print high-order dependencies as a form of logging/progress notifcation.
135 if order == 2:
136 print(rn)
137 if order == 3:
138 print(' ' + rn)
139
140 # First find all dependencies not seen yet to our master list.
141 for dep in depends:
142 if dep not in packages and dep not in assume_provided:
143 packages.append(dep)
144 dep_data = get_recipe_info(tinfoil, dep)
145 # Do this once now to reduce the number of bitbake calls.
146 dep_data.depends = dep_data.getVar('DEPENDS').split()
147 recipe_info[dep] = dep_data
148 # Then recursively analyze all of the dependencies for the current recipe.
84 for dep in depends: 149 for dep in depends:
85 abcd_file.write(' - "%s"\n' % dep) 150 if dep not in assume_provided:
86 # TODO: search for transitive dependencies here? Each dependency will 151 find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, dep, order + 1)
87 # get checked for its own dependencies sooner or later.
88 152
89 return depends 153 if order > 1:
154 manifest_file.write('%s errors: []\n' % spaces)
90 155
91 156
92def main(): 157def main():
93 parser = ArgumentParser(description='Find all dependencies of one or more packages.') 158 parser = ArgumentParser(description='Find all dependencies of a recipe.')
94 parser.add_argument('packages', metavar='package', nargs='+', 159 parser.add_argument('recipe', metavar='recipe', help='a recipe to investigate')
95 help='a package to investigate')
96 args = parser.parse_args() 160 args = parser.parse_args()
97 recipes_to_check = args.packages 161 rn = args.recipe
98 abcd_manifest = 'manifest.yml' 162 with open(rn + '-dependencies.yml', "w") as manifest_file, bb.tinfoil.Tinfoil() as tinfoil:
99 with open(abcd_manifest, "w") as abcd_file, bb.tinfoil.Tinfoil() as tinfoil:
100 tinfoil.prepare() 163 tinfoil.prepare()
101 # These are the packages that bitbake assumes are provided by the host 164 # These are the packages that bitbake assumes are provided by the host
102 # system. They do not have recipes, so searching tinfoil for them will 165 # system. They do not have recipes, so searching tinfoil for them will
103 # not work. Anyway, by nature they are not included in code we release, 166 # not work. Anyway, by nature they are only build tools and will not be
104 # only used by it. 167 # distributed in an image.
105 assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split() 168 assume_provided = tinfoil.config_data.getVar('ASSUME_PROVIDED').split()
106 abcd_file.write('packages:\n') 169 if SKIP_BUILD_TOOLS:
107 170 assume_provided.extend(KNOWN_BUILD_TOOLS)
108 # Iterate through the list of recipes to check. Append any dependencies 171
109 # found that aren't already in the list. As long as we only add to the 172 manifest_file.write('project:\n')
110 # list, it should be safe. 173 data = get_recipe_info(tinfoil, rn)
111 for recipe in recipes_to_check: 174 data.depends = []
112 depends = print_deps(tinfoil, abcd_file, recipe) 175 depends = data.getVar('DEPENDS').split()
113 for dep in depends: 176 for dep in depends:
114 if dep not in recipes_to_check and dep not in assume_provided: 177 if dep not in assume_provided:
115 recipes_to_check.append(dep) 178 data.depends.append(dep)
179 print_package(manifest_file, data, is_project=True)
180 manifest_file.write(' scopes:\n')
181 manifest_file.write(' - name: "all"\n')
182 manifest_file.write(' delivered: true\n')
183 manifest_file.write(' dependencies:\n')
184
185 recipe_info = dict([(rn, data)])
186 packages = []
187 find_dependencies(manifest_file, tinfoil, assume_provided, recipe_info, packages, rn, order=1)
188
189 manifest_file.write('packages:\n')
190
191 # Iterate through the list of packages found to print out their full
192 # information. Skip the initial recipe since we already printed it out.
193 for p in packages:
194 if p is not rn:
195 data = recipe_info[p]
196 print_package(manifest_file, data, is_project=False)
116 197
117 198
118if __name__ == "__main__": 199if __name__ == "__main__":