summaryrefslogtreecommitdiffstats
path: root/scripts/lib/resulttool/store.py
diff options
context:
space:
mode:
authorRichard Purdie <richard.purdie@linuxfoundation.org>2019-02-16 18:13:00 +0000
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-02-21 12:34:00 +0000
commit47eb3d00e9d6a66aee1283dab29af8117a006d6d (patch)
tree9d574720d23da3edeab5648f05fb5d2ee95c9cd2 /scripts/lib/resulttool/store.py
parentbeed7523b667affea71d37d88d2f5c19c935d159 (diff)
downloadpoky-47eb3d00e9d6a66aee1283dab29af8117a006d6d.tar.gz
resulttool: Improvements to allow integration to the autobuilder
This is a combined patch of the various tweaks and improvements I made to resulttool: * Avoid subprocess.run() as its a python 3.6 feature and we have autobuilder workers with 3.5. * Avoid python keywords as variable names * Simplify dict accesses using .get() * Rename resultsutils -> resultutils to match the resultstool -> resulttool rename * Formalised the handling of "file_name" to "TESTSERIES" which the code will now add into the json configuration data if its not present, based on the directory name. * When we don't have failed test cases, print something saying so instead of an empty table * Tweak the table headers in the report to be more readable (reference "Test Series" instead if file_id and ID instead of results_id) * Improve/simplify the max string length handling * Merge the counts and percentage data into one table in the report since printing two reports of the same data confuses the user * Removed the confusing header in the regression report * Show matches, then regressions, then unmatched runs in the regression report, also remove chatting unneeded output * Try harder to "pair" up matching configurations to reduce noise in the regressions report * Abstracted the "mapping" table concept used to pairing in the regression code to general code in resultutils * Created multiple mappings for results analysis, results storage and 'flattening' results data in a merge * Simplify the merge command to take a source and a destination, letting the destination be a directory or a file, removing the need for an output directory parameter * Add the 'IMAGE_PKGTYPE' and 'DISTRO' config options to the regression mappings * Have the store command place the testresults files in a layout from the mapping, making commits into the git repo for results storage more useful for simple comparison purposes * Set the oe-git-archive tag format appropriately for oeqa results storage (and simplify the commit messages closer to their defaults) * Fix oe-git-archive to use the commit/branch data from the results file * Cleaned up the command option help to match other changes * Follow the model of git branch/tag processing used by oe-build-perf-report and use that to read the data using git show to avoid branch change * Add ptest summary to the report command * Update the tests to match the above changes (From OE-Core rev: ff2c029b568f70aa9960dde04ddd207829812ea0) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Diffstat (limited to 'scripts/lib/resulttool/store.py')
-rw-r--r--scripts/lib/resulttool/store.py136
1 files changed, 59 insertions, 77 deletions
diff --git a/scripts/lib/resulttool/store.py b/scripts/lib/resulttool/store.py
index 2c6fd8492c..6744fb3c05 100644
--- a/scripts/lib/resulttool/store.py
+++ b/scripts/lib/resulttool/store.py
@@ -1,6 +1,7 @@
1# test result tool - store test results 1# resulttool - store test results
2# 2#
3# Copyright (c) 2019, Intel Corporation. 3# Copyright (c) 2019, Intel Corporation.
4# Copyright (c) 2019, Linux Foundation
4# 5#
5# This program is free software; you can redistribute it and/or modify it 6# This program is free software; you can redistribute it and/or modify it
6# under the terms and conditions of the GNU General Public License, 7# under the terms and conditions of the GNU General Public License,
@@ -11,100 +12,81 @@
11# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12# more details. 13# more details.
13# 14#
14import datetime
15import tempfile 15import tempfile
16import os 16import os
17import subprocess 17import subprocess
18import json
19import shutil
18import scriptpath 20import scriptpath
19scriptpath.add_bitbake_lib_path() 21scriptpath.add_bitbake_lib_path()
20scriptpath.add_oe_lib_path() 22scriptpath.add_oe_lib_path()
21from resulttool.resultsutils import checkout_git_dir 23import resulttool.resultutils as resultutils
22try: 24import oeqa.utils.gitarchive as gitarchive
23 import bb
24except ImportError:
25 pass
26 25
27class ResultsGitStore(object):
28 26
29 def _get_output_dir(self): 27def store(args, logger):
30 basepath = os.environ['BUILDDIR'] 28 tempdir = tempfile.mkdtemp(prefix='testresults.')
31 return basepath + '/testresults_%s/' % datetime.datetime.now().strftime("%Y%m%d%H%M%S") 29 try:
32 30 results = {}
33 def _create_temporary_workspace_dir(self): 31 logger.info('Reading files from %s' % args.source)
34 return tempfile.mkdtemp(prefix='testresults.') 32 for root, dirs, files in os.walk(args.source):
35 33 for name in files:
36 def _remove_temporary_workspace_dir(self, workspace_dir): 34 f = os.path.join(root, name)
37 return subprocess.run(["rm", "-rf", workspace_dir]) 35 if name == "testresults.json":
38 36 resultutils.append_resultsdata(results, f)
39 def _oe_copy_files(self, source_dir, destination_dir): 37 elif args.all:
40 from oe.path import copytree 38 dst = f.replace(args.source, tempdir + "/")
41 copytree(source_dir, destination_dir) 39 os.makedirs(os.path.dirname(dst), exist_ok=True)
40 shutil.copyfile(f, dst)
41 resultutils.save_resultsdata(results, tempdir)
42 42
43 def _copy_files(self, source_dir, destination_dir, copy_ignore=None): 43 if not results and not args.all:
44 from shutil import copytree 44 if args.allow_empty:
45 copytree(source_dir, destination_dir, ignore=copy_ignore) 45 logger.info("No results found to store")
46 return 0
47 logger.error("No results found to store")
48 return 1
46 49
47 def _store_files_to_git(self, logger, file_dir, git_dir, git_branch, commit_msg_subject, commit_msg_body): 50 keywords = {'branch': None, 'commit': None, 'commit_count': None}
48 logger.debug('Storing test result into git repository (%s) and branch (%s)'
49 % (git_dir, git_branch))
50 return subprocess.run(["oe-git-archive",
51 file_dir,
52 "-g", git_dir,
53 "-b", git_branch,
54 "--commit-msg-subject", commit_msg_subject,
55 "--commit-msg-body", commit_msg_body])
56 51
57 def store_to_existing(self, logger, source_dir, git_dir, git_branch): 52 # Find the branch/commit/commit_count and ensure they all match
58 logger.debug('Storing files to existing git repository and branch') 53 for suite in results:
59 from shutil import ignore_patterns 54 for result in results[suite]:
60 dest_dir = self._create_temporary_workspace_dir() 55 config = results[suite][result]['configuration']['LAYERS']['meta']
61 dest_top_dir = os.path.join(dest_dir, 'top_dir') 56 for k in keywords:
62 self._copy_files(git_dir, dest_top_dir, copy_ignore=ignore_patterns('.git')) 57 if keywords[k] is None:
63 self._oe_copy_files(source_dir, dest_top_dir) 58 keywords[k] = config.get(k)
64 self._store_files_to_git(logger, dest_top_dir, git_dir, git_branch, 59 if config.get(k) != keywords[k]:
65 'Store as existing git and branch', 'Store as existing git repository and branch') 60 logger.error("Mismatched source commit/branch/count: %s vs %s" % (config.get(k), keywords[k]))
66 self._remove_temporary_workspace_dir(dest_dir) 61 return 1
67 return git_dir
68 62
69 def store_to_existing_with_new_branch(self, logger, source_dir, git_dir, git_branch): 63 logger.info('Storing test result into git repository %s' % args.git_dir)
70 logger.debug('Storing files to existing git repository with new branch')
71 self._store_files_to_git(logger, source_dir, git_dir, git_branch,
72 'Store as existing git with new branch',
73 'Store as existing git repository with new branch')
74 return git_dir
75 64
76 def store_to_new(self, logger, source_dir, git_branch): 65 gitarchive.gitarchive(tempdir, args.git_dir, False, False,
77 logger.debug('Storing files to new git repository') 66 "Results of {branch}:{commit}", "branch: {branch}\ncommit: {commit}", "{branch}",
78 output_dir = self._get_output_dir() 67 False, "{branch}/{commit_count}-g{commit}/{tag_number}",
79 self._store_files_to_git(logger, source_dir, output_dir, git_branch, 68 'Test run #{tag_number} of {branch}:{commit}', '',
80 'Store as new', 'Store as new git repository') 69 [], [], False, keywords, logger)
81 return output_dir
82 70
83 def store(self, logger, source_dir, git_dir, git_branch): 71 finally:
84 if git_dir: 72 subprocess.check_call(["rm", "-rf", tempdir])
85 if checkout_git_dir(git_dir, git_branch):
86 self.store_to_existing(logger, source_dir, git_dir, git_branch)
87 else:
88 self.store_to_existing_with_new_branch(logger, source_dir, git_dir, git_branch)
89 else:
90 self.store_to_new(logger, source_dir, git_branch)
91 73
92def store(args, logger):
93 gitstore = ResultsGitStore()
94 gitstore.store(logger, args.source_dir, args.git_dir, args.git_branch)
95 return 0 74 return 0
96 75
97def register_commands(subparsers): 76def register_commands(subparsers):
98 """Register subcommands from this plugin""" 77 """Register subcommands from this plugin"""
99 parser_build = subparsers.add_parser('store', help='store test result files and directories into git repository', 78 parser_build = subparsers.add_parser('store', help='store test results into a git repository',
100 description='store the testresults.json files and related directories ' 79 description='takes a results file or directory of results files and stores '
101 'from the source directory into the destination git repository ' 80 'them into the destination git repository, splitting out the results '
102 'with the given git branch', 81 'files as configured',
103 group='setup') 82 group='setup')
104 parser_build.set_defaults(func=store) 83 parser_build.set_defaults(func=store)
105 parser_build.add_argument('source_dir', 84 parser_build.add_argument('source',
106 help='source directory that contain the test result files and directories to be stored') 85 help='source file or directory that contain the test result files to be stored')
107 parser_build.add_argument('git_branch', help='git branch used for store') 86 parser_build.add_argument('git_dir',
108 parser_build.add_argument('-d', '--git-dir', default='', 87 help='the location of the git repository to store the results in')
109 help='(optional) default store to new <top_dir>/<build>/<testresults_datetime> ' 88 parser_build.add_argument('-a', '--all', action='store_true',
110 'directory unless provided with existing git repository as destination') 89 help='include all files, not just testresults.json files')
90 parser_build.add_argument('-e', '--allow-empty', action='store_true',
91 help='don\'t error if no results to store are found')
92