summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-02-20 17:07:56 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-02-21 12:34:00 +0000
commit95bd530b772f97e7329749b403bf9e2dff12ff7f (patch)
treea297a6ce30449839a7d1f2464da4a9a8937a36fa
parentf4a270a519a14652b7c13a151068b4ef255710e7 (diff)
downloadpoky-95bd530b772f97e7329749b403bf9e2dff12ff7f.tar.gz
oe-build-perf-report/gitarchive: Move common useful functions to library
These functions can be reused by the resulttool code so move to the common function library for this purpose. (From OE-Core rev: c66f848938c04e133259c5b6903dc592866ab385) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--meta/lib/oeqa/utils/gitarchive.py71
-rwxr-xr-xscripts/oe-build-perf-report90
2 files changed, 81 insertions, 80 deletions
diff --git a/meta/lib/oeqa/utils/gitarchive.py b/meta/lib/oeqa/utils/gitarchive.py
index fddd608593..ff614d06bb 100644
--- a/meta/lib/oeqa/utils/gitarchive.py
+++ b/meta/lib/oeqa/utils/gitarchive.py
@@ -17,6 +17,8 @@
17import os 17import os
18import re 18import re
19import sys 19import sys
20from operator import attrgetter
21from collections import namedtuple
20from oeqa.utils.git import GitRepo, GitError 22from oeqa.utils.git import GitRepo, GitError
21 23
22class ArchiveError(Exception): 24class ArchiveError(Exception):
@@ -171,3 +173,72 @@ def gitarchive(data_dir, git_dir, no_create, bare, commit_msg_subject, commit_ms
171 log.info("Pushing data to remote") 173 log.info("Pushing data to remote")
172 data_repo.run_cmd(cmd) 174 data_repo.run_cmd(cmd)
173 175
176# Container class for tester revisions
177TestedRev = namedtuple('TestedRev', 'commit commit_number tags')
178
179def get_test_runs(log, repo, tag_name, **kwargs):
180 """Get a sorted list of test runs, matching given pattern"""
181 # First, get field names from the tag name pattern
182 field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)]
183 undef_fields = [f for f in field_names if f not in kwargs.keys()]
184
185 # Fields for formatting tag name pattern
186 str_fields = dict([(f, '*') for f in field_names])
187 str_fields.update(kwargs)
188
189 # Get a list of all matching tags
190 tag_pattern = tag_name.format(**str_fields)
191 tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines()
192 log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)
193
194 # Parse undefined fields from tag names
195 str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names])
196 str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)'
197 str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})'
198 str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})'
199 str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})'
200 # escape parenthesis in fields in order to not messa up the regexp
201 fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()])
202 str_fields.update(fixed_fields)
203 tag_re = re.compile(tag_name.format(**str_fields))
204
205 # Parse fields from tags
206 revs = []
207 for tag in tags:
208 m = tag_re.match(tag)
209 groups = m.groupdict()
210 revs.append([groups[f] for f in undef_fields] + [tag])
211
212 # Return field names and a sorted list of revs
213 return undef_fields, sorted(revs)
214
215def get_test_revs(log, repo, tag_name, **kwargs):
216 """Get list of all tested revisions"""
217 fields, runs = get_test_runs(log, repo, tag_name, **kwargs)
218
219 revs = {}
220 commit_i = fields.index('commit')
221 commit_num_i = fields.index('commit_number')
222 for run in runs:
223 commit = run[commit_i]
224 commit_num = run[commit_num_i]
225 tag = run[-1]
226 if not commit in revs:
227 revs[commit] = TestedRev(commit, commit_num, [tag])
228 else:
229 assert commit_num == revs[commit].commit_number, "Commit numbers do not match"
230 revs[commit].tags.append(tag)
231
232 # Return in sorted table
233 revs = sorted(revs.values(), key=attrgetter('commit_number'))
234 log.debug("Found %d tested revisions:\n %s", len(revs),
235 "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs]))
236 return revs
237
238def rev_find(revs, attr, val):
239 """Search from a list of TestedRev"""
240 for i, rev in enumerate(revs):
241 if getattr(rev, attr) == val:
242 return i
243 raise ValueError("Unable to find '{}' value '{}'".format(attr, val))
244
diff --git a/scripts/oe-build-perf-report b/scripts/oe-build-perf-report
index af4f0469f6..f6fb458c2e 100755
--- a/scripts/oe-build-perf-report
+++ b/scripts/oe-build-perf-report
@@ -37,58 +37,18 @@ from buildstats import BuildStats, diff_buildstats, BSVerDiff
37scriptpath.add_oe_lib_path() 37scriptpath.add_oe_lib_path()
38 38
39from oeqa.utils.git import GitRepo, GitError 39from oeqa.utils.git import GitRepo, GitError
40import oeqa.utils.gitarchive as gitarchive
40 41
41 42
42# Setup logging 43# Setup logging
43logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") 44logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
44log = logging.getLogger('oe-build-perf-report') 45log = logging.getLogger('oe-build-perf-report')
45 46
46
47# Container class for tester revisions
48TestedRev = namedtuple('TestedRev', 'commit commit_number tags')
49
50
51def get_test_runs(repo, tag_name, **kwargs):
52 """Get a sorted list of test runs, matching given pattern"""
53 # First, get field names from the tag name pattern
54 field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)]
55 undef_fields = [f for f in field_names if f not in kwargs.keys()]
56
57 # Fields for formatting tag name pattern
58 str_fields = dict([(f, '*') for f in field_names])
59 str_fields.update(kwargs)
60
61 # Get a list of all matching tags
62 tag_pattern = tag_name.format(**str_fields)
63 tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines()
64 log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)
65
66 # Parse undefined fields from tag names
67 str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names])
68 str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)'
69 str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})'
70 str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})'
71 str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})'
72 # escape parenthesis in fields in order to not messa up the regexp
73 fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()])
74 str_fields.update(fixed_fields)
75 tag_re = re.compile(tag_name.format(**str_fields))
76
77 # Parse fields from tags
78 revs = []
79 for tag in tags:
80 m = tag_re.match(tag)
81 groups = m.groupdict()
82 revs.append([groups[f] for f in undef_fields] + [tag])
83
84 # Return field names and a sorted list of revs
85 return undef_fields, sorted(revs)
86
87def list_test_revs(repo, tag_name, verbosity, **kwargs): 47def list_test_revs(repo, tag_name, verbosity, **kwargs):
88 """Get list of all tested revisions""" 48 """Get list of all tested revisions"""
89 valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None]) 49 valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None])
90 50
91 fields, revs = get_test_runs(repo, tag_name, **valid_kwargs) 51 fields, revs = gitarchive.get_test_runs(log, repo, tag_name, **valid_kwargs)
92 ignore_fields = ['tag_number'] 52 ignore_fields = ['tag_number']
93 if verbosity < 2: 53 if verbosity < 2:
94 extra_fields = ['COMMITS', 'TEST RUNS'] 54 extra_fields = ['COMMITS', 'TEST RUNS']
@@ -133,36 +93,6 @@ def list_test_revs(repo, tag_name, verbosity, **kwargs):
133 93
134 print_table(rows) 94 print_table(rows)
135 95
136def get_test_revs(repo, tag_name, **kwargs):
137 """Get list of all tested revisions"""
138 fields, runs = get_test_runs(repo, tag_name, **kwargs)
139
140 revs = {}
141 commit_i = fields.index('commit')
142 commit_num_i = fields.index('commit_number')
143 for run in runs:
144 commit = run[commit_i]
145 commit_num = run[commit_num_i]
146 tag = run[-1]
147 if not commit in revs:
148 revs[commit] = TestedRev(commit, commit_num, [tag])
149 else:
150 assert commit_num == revs[commit].commit_number, "Commit numbers do not match"
151 revs[commit].tags.append(tag)
152
153 # Return in sorted table
154 revs = sorted(revs.values(), key=attrgetter('commit_number'))
155 log.debug("Found %d tested revisions:\n %s", len(revs),
156 "\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs]))
157 return revs
158
159def rev_find(revs, attr, val):
160 """Search from a list of TestedRev"""
161 for i, rev in enumerate(revs):
162 if getattr(rev, attr) == val:
163 return i
164 raise ValueError("Unable to find '{}' value '{}'".format(attr, val))
165
166def is_xml_format(repo, commit): 96def is_xml_format(repo, commit):
167 """Check if the commit contains xml (or json) data""" 97 """Check if the commit contains xml (or json) data"""
168 if repo.rev_parse(commit + ':results.xml'): 98 if repo.rev_parse(commit + ':results.xml'):
@@ -578,11 +508,11 @@ def main(argv=None):
578 if not args.hostname: 508 if not args.hostname:
579 auto_args(repo, args) 509 auto_args(repo, args)
580 510
581 revs = get_test_revs(repo, args.tag_name, hostname=args.hostname, 511 revs = gitarchive.get_test_revs(log, repo, args.tag_name, hostname=args.hostname,
582 branch=args.branch, machine=args.machine) 512 branch=args.branch, machine=args.machine)
583 if args.branch2: 513 if args.branch2:
584 revs2 = get_test_revs(repo, args.tag_name, hostname=args.hostname, 514 revs2 = gitarchive.get_test_revs(log, repo, args.tag_name, hostname=args.hostname,
585 branch=args.branch2, machine=args.machine) 515 branch=args.branch2, machine=args.machine)
586 if not len(revs2): 516 if not len(revs2):
587 log.error("No revisions found to compare against") 517 log.error("No revisions found to compare against")
588 return 1 518 return 1
@@ -598,9 +528,9 @@ def main(argv=None):
598 if args.commit: 528 if args.commit:
599 if args.commit_number: 529 if args.commit_number:
600 log.warning("Ignoring --commit-number as --commit was specified") 530 log.warning("Ignoring --commit-number as --commit was specified")
601 index1 = rev_find(revs, 'commit', args.commit) 531 index1 = gitarchive.rev_find(revs, 'commit', args.commit)
602 elif args.commit_number: 532 elif args.commit_number:
603 index1 = rev_find(revs, 'commit_number', args.commit_number) 533 index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number)
604 else: 534 else:
605 index1 = len(revs) - 1 535 index1 = len(revs) - 1
606 536
@@ -612,9 +542,9 @@ def main(argv=None):
612 if args.commit2: 542 if args.commit2:
613 if args.commit_number2: 543 if args.commit_number2:
614 log.warning("Ignoring --commit-number2 as --commit2 was specified") 544 log.warning("Ignoring --commit-number2 as --commit2 was specified")
615 index2 = rev_find(revs, 'commit', args.commit2) 545 index2 = gitarchive.rev_find(revs, 'commit', args.commit2)
616 elif args.commit_number2: 546 elif args.commit_number2:
617 index2 = rev_find(revs, 'commit_number', args.commit_number2) 547 index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2)
618 else: 548 else:
619 if index1 > 0: 549 if index1 > 0:
620 index2 = index1 - 1 550 index2 = index1 - 1