summaryrefslogtreecommitdiffstats
path: root/scripts/buildstats-diff
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2025-11-07 13:31:53 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2025-11-07 13:31:53 +0000
commit8c22ff0d8b70d9b12f0487ef696a7e915b9e3173 (patch)
treeefdc32587159d0050a69009bdf2330a531727d95 /scripts/buildstats-diff
parentd412d2747595c1cc4a5e3ca975e3adc31b2f7891 (diff)
downloadpoky-8c22ff0d8b70d9b12f0487ef696a7e915b9e3173.tar.gz
The poky repository master branch is no longer being updated.
You can either: a) switch to individual clones of bitbake, openembedded-core, meta-yocto and yocto-docs b) use the new bitbake-setup You can find information about either approach in our documentation: https://docs.yoctoproject.org/ Note that "poky" the distro setting is still available in meta-yocto as before and we continue to use and maintain that. Long live Poky! Some further information on the background of this change can be found in: https://lists.openembedded.org/g/openembedded-architecture/message/2179 Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/buildstats-diff')
-rwxr-xr-xscripts/buildstats-diff324
1 files changed, 0 insertions, 324 deletions
diff --git a/scripts/buildstats-diff b/scripts/buildstats-diff
deleted file mode 100755
index df1df432f1..0000000000
--- a/scripts/buildstats-diff
+++ /dev/null
@@ -1,324 +0,0 @@
1#!/usr/bin/env python3
2#
3# Script for comparing buildstats from two different builds
4#
5# Copyright (c) 2016, Intel Corporation.
6#
7# SPDX-License-Identifier: GPL-2.0-only
8#
9
10import argparse
11import glob
12import logging
13import math
14import os
15import pathlib
16import sys
17from operator import attrgetter
18
19# Import oe libs
20scripts_path = os.path.dirname(os.path.realpath(__file__))
21sys.path.append(os.path.join(scripts_path, 'lib'))
22from buildstats import BuildStats, diff_buildstats, taskdiff_fields, BSVerDiff
23
24
25# Setup logging
26logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
27log = logging.getLogger()
28
29
30class ScriptError(Exception):
31 """Exception for internal error handling of this script"""
32 pass
33
34
35def read_buildstats(path, multi):
36 """Read buildstats"""
37 if not os.path.exists(path):
38 raise ScriptError("No such file or directory: {}".format(path))
39
40 if os.path.isfile(path):
41 return BuildStats.from_file_json(path)
42
43 if os.path.isfile(os.path.join(path, 'build_stats')):
44 return BuildStats.from_dir(path)
45
46 # Handle a non-buildstat directory
47 subpaths = sorted(glob.glob(path + '/*'))
48 if len(subpaths) > 1:
49 if multi:
50 log.info("Averaging over {} buildstats from {}".format(
51 len(subpaths), path))
52 else:
53 raise ScriptError("Multiple buildstats found in '{}'. Please give "
54 "a single buildstat directory of use the --multi "
55 "option".format(path))
56 bs = None
57 for subpath in subpaths:
58 if os.path.isfile(subpath):
59 _bs = BuildStats.from_file_json(subpath)
60 else:
61 _bs = BuildStats.from_dir(subpath)
62 if bs is None:
63 bs = _bs
64 else:
65 bs.aggregate(_bs)
66 if not bs:
67 raise ScriptError("No buildstats found under {}".format(path))
68
69 return bs
70
71
72def print_ver_diff(bs1, bs2):
73 """Print package version differences"""
74
75 diff = BSVerDiff(bs1, bs2)
76
77 maxlen = max([len(r) for r in set(bs1.keys()).union(set(bs2.keys()))])
78 fmt_str = " {:{maxlen}} ({})"
79
80 if diff.new:
81 print("\nNEW RECIPES:")
82 print("------------")
83 for name, val in sorted(diff.new.items()):
84 print(fmt_str.format(name, val.nevr, maxlen=maxlen))
85
86 if diff.dropped:
87 print("\nDROPPED RECIPES:")
88 print("----------------")
89 for name, val in sorted(diff.dropped.items()):
90 print(fmt_str.format(name, val.nevr, maxlen=maxlen))
91
92 fmt_str = " {0:{maxlen}} {1:<20} ({2})"
93 if diff.rchanged:
94 print("\nREVISION CHANGED:")
95 print("-----------------")
96 for name, val in sorted(diff.rchanged.items()):
97 field1 = "{} -> {}".format(val.left.revision, val.right.revision)
98 field2 = "{} -> {}".format(val.left.nevr, val.right.nevr)
99 print(fmt_str.format(name, field1, field2, maxlen=maxlen))
100
101 if diff.vchanged:
102 print("\nVERSION CHANGED:")
103 print("----------------")
104 for name, val in sorted(diff.vchanged.items()):
105 field1 = "{} -> {}".format(val.left.version, val.right.version)
106 field2 = "{} -> {}".format(val.left.nevr, val.right.nevr)
107 print(fmt_str.format(name, field1, field2, maxlen=maxlen))
108
109 if diff.echanged:
110 print("\nEPOCH CHANGED:")
111 print("--------------")
112 for name, val in sorted(diff.echanged.items()):
113 field1 = "{} -> {}".format(val.left.epoch, val.right.epoch)
114 field2 = "{} -> {}".format(val.left.nevr, val.right.nevr)
115 print(fmt_str.format(name, field1, field2, maxlen=maxlen))
116
117
118def print_task_diff(bs1, bs2, val_type, min_val=0, min_absdiff=0, sort_by=('absdiff',), only_tasks=[]):
119 """Diff task execution times"""
120 def val_to_str(val, human_readable=False):
121 """Convert raw value to printable string"""
122 def hms_time(secs):
123 """Get time in human-readable HH:MM:SS format"""
124 h = int(secs / 3600)
125 m = int((secs % 3600) / 60)
126 s = secs % 60
127 if h == 0:
128 return "{:02d}:{:04.1f}".format(m, s)
129 else:
130 return "{:d}:{:02d}:{:04.1f}".format(h, m, s)
131
132 if 'time' in val_type:
133 if human_readable:
134 return hms_time(val)
135 else:
136 return "{:.1f}s".format(val)
137 elif 'bytes' in val_type and human_readable:
138 prefix = ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi']
139 dec = int(math.log(val, 2) / 10)
140 prec = 1 if dec > 0 else 0
141 return "{:.{prec}f}{}B".format(val / (2 ** (10 * dec)),
142 prefix[dec], prec=prec)
143 elif 'ops' in val_type and human_readable:
144 prefix = ['', 'k', 'M', 'G', 'T', 'P']
145 dec = int(math.log(val, 1000))
146 prec = 1 if dec > 0 else 0
147 return "{:.{prec}f}{}ops".format(val / (1000 ** dec),
148 prefix[dec], prec=prec)
149 return str(int(val))
150
151 def sum_vals(buildstats):
152 """Get cumulative sum of all tasks"""
153 total = 0.0
154 for recipe_data in buildstats.values():
155 for name, bs_task in recipe_data.tasks.items():
156 if not only_tasks or name in only_tasks:
157 total += getattr(bs_task, val_type)
158 return total
159
160 if min_val:
161 print("Ignoring tasks less than {} ({})".format(
162 val_to_str(min_val, True), val_to_str(min_val)))
163 if min_absdiff:
164 print("Ignoring differences less than {} ({})".format(
165 val_to_str(min_absdiff, True), val_to_str(min_absdiff)))
166
167 # Prepare the data
168 tasks_diff = diff_buildstats(bs1, bs2, val_type, min_val, min_absdiff, only_tasks)
169
170 # Sort our list
171 for field in reversed(sort_by):
172 if field.startswith('-'):
173 field = field[1:]
174 reverse = True
175 else:
176 reverse = False
177 tasks_diff = sorted(tasks_diff, key=attrgetter(field), reverse=reverse)
178
179 linedata = [(' ', 'PKG', ' ', 'TASK', 'ABSDIFF', 'RELDIFF',
180 val_type.upper() + '1', val_type.upper() + '2')]
181 field_lens = dict([('len_{}'.format(i), len(f)) for i, f in enumerate(linedata[0])])
182
183 # Prepare fields in string format and measure field lengths
184 for diff in tasks_diff:
185 task_prefix = diff.task_op if diff.pkg_op == ' ' else ' '
186 linedata.append((diff.pkg_op, diff.pkg, task_prefix, diff.task,
187 val_to_str(diff.absdiff),
188 '{:+.1f}%'.format(diff.reldiff),
189 val_to_str(diff.value1),
190 val_to_str(diff.value2)))
191 for i, field in enumerate(linedata[-1]):
192 key = 'len_{}'.format(i)
193 if len(field) > field_lens[key]:
194 field_lens[key] = len(field)
195
196 # Print data
197 print()
198 for fields in linedata:
199 print("{:{len_0}}{:{len_1}} {:{len_2}}{:{len_3}} {:>{len_4}} {:>{len_5}} {:>{len_6}} -> {:{len_7}}".format(
200 *fields, **field_lens))
201
202 # Print summary of the diffs
203 total1 = sum_vals(bs1)
204 total2 = sum_vals(bs2)
205 print("\nCumulative {}:".format(val_type))
206 print (" {} {:+.1f}% {} ({}) -> {} ({})".format(
207 val_to_str(total2 - total1), 100 * (total2-total1) / total1,
208 val_to_str(total1, True), val_to_str(total1),
209 val_to_str(total2, True), val_to_str(total2)))
210
211
212def parse_args(argv):
213 """Parse cmdline arguments"""
214 description="""
215Script for comparing buildstats of two separate builds."""
216 parser = argparse.ArgumentParser(
217 formatter_class=argparse.ArgumentDefaultsHelpFormatter,
218 description=description)
219
220 min_val_defaults = {'cputime': 3.0,
221 'read_bytes': 524288,
222 'write_bytes': 524288,
223 'read_ops': 500,
224 'write_ops': 500,
225 'walltime': 5}
226 min_absdiff_defaults = {'cputime': 1.0,
227 'read_bytes': 131072,
228 'write_bytes': 131072,
229 'read_ops': 50,
230 'write_ops': 50,
231 'walltime': 2}
232
233 parser.add_argument('--debug', '-d', action='store_true',
234 help="Verbose logging")
235 parser.add_argument('--ver-diff', action='store_true',
236 help="Show package version differences and exit")
237 parser.add_argument('--diff-attr', default='cputime',
238 choices=min_val_defaults.keys(),
239 help="Buildstat attribute which to compare")
240 parser.add_argument('--min-val', default=min_val_defaults, type=float,
241 help="Filter out tasks less than MIN_VAL. "
242 "Default depends on --diff-attr.")
243 parser.add_argument('--min-absdiff', default=min_absdiff_defaults, type=float,
244 help="Filter out tasks whose difference is less than "
245 "MIN_ABSDIFF, Default depends on --diff-attr.")
246 parser.add_argument('--sort-by', default='absdiff',
247 help="Comma-separated list of field sort order. "
248 "Prepend the field name with '-' for reversed sort. "
249 "Available fields are: {}".format(', '.join(taskdiff_fields)))
250 parser.add_argument('--multi', action='store_true',
251 help="Read all buildstats from the given paths and "
252 "average over them")
253 parser.add_argument('--only-task', dest='only_tasks', metavar='TASK', action='append', default=[],
254 help="Only include TASK in report. May be specified multiple times")
255 parser.add_argument('buildstats1', metavar='BUILDSTATS1', nargs="?", help="'Left' buildstat")
256 parser.add_argument('buildstats2', metavar='BUILDSTATS2', nargs="?", help="'Right' buildstat")
257
258 args = parser.parse_args(argv)
259
260 if args.buildstats1 and args.buildstats2:
261 # Both paths specified
262 pass
263 elif args.buildstats1 or args.buildstats2:
264 # Just one path specified, this is an error
265 parser.print_usage(sys.stderr)
266 print("Either specify two buildstats paths, or none to use the last two paths.", file=sys.stderr)
267 sys.exit(1)
268 else:
269 # No paths specified, try to find the last two buildstats
270 try:
271 buildstats_dir = pathlib.Path(os.environ["BUILDDIR"]) / "tmp" / "buildstats"
272 paths = sorted(buildstats_dir.iterdir())
273 args.buildstats2 = paths.pop()
274 args.buildstats1 = paths.pop()
275 print(f"Comparing {args.buildstats1} -> {args.buildstats2}\n")
276 except KeyError:
277 parser.print_usage(sys.stderr)
278 print("Build environment has not been configured, cannot find buildstats", file=sys.stderr)
279 sys.exit(1)
280
281 # We do not nedd/want to read all buildstats if we just want to look at the
282 # package versions
283 if args.ver_diff:
284 args.multi = False
285
286 # Handle defaults for the filter arguments
287 if args.min_val is min_val_defaults:
288 args.min_val = min_val_defaults[args.diff_attr]
289 if args.min_absdiff is min_absdiff_defaults:
290 args.min_absdiff = min_absdiff_defaults[args.diff_attr]
291
292 return args
293
294def main(argv=None):
295 """Script entry point"""
296 args = parse_args(argv)
297 if args.debug:
298 log.setLevel(logging.DEBUG)
299
300 # Validate sort fields
301 sort_by = []
302 for field in args.sort_by.split(','):
303 if field.lstrip('-') not in taskdiff_fields:
304 log.error("Invalid sort field '%s' (must be one of: %s)" %
305 (field, ', '.join(taskdiff_fields)))
306 sys.exit(1)
307 sort_by.append(field)
308
309 try:
310 bs1 = read_buildstats(args.buildstats1, args.multi)
311 bs2 = read_buildstats(args.buildstats2, args.multi)
312
313 if args.ver_diff:
314 print_ver_diff(bs1, bs2)
315 else:
316 print_task_diff(bs1, bs2, args.diff_attr, args.min_val,
317 args.min_absdiff, sort_by, args.only_tasks)
318 except ScriptError as err:
319 log.error(str(err))
320 return 1
321 return 0
322
323if __name__ == "__main__":
324 sys.exit(main())