summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJoshua Watt <jpewhacker@gmail.com>2019-04-18 21:57:17 -0500
committerRichard Purdie <richard.purdie@linuxfoundation.org>2019-05-22 00:31:49 +0100
commit35fa3e7e188d19a036b443505f21594f5279b24f (patch)
tree39494a48272e8f1270def179ea77c8370e55ea6c
parent47d3b81ad332a8b76b8720c51347c67729b14e01 (diff)
downloadpoky-35fa3e7e188d19a036b443505f21594f5279b24f.tar.gz
resulttool: Load results from URL
Adds support for resulttool to load JSON files directly from a http:// or https:// URL (From OE-Core rev: 3e48404afe27c93fa6ffbd8d66bc52dcd6216005) Signed-off-by: Joshua Watt <JPEWhacker@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org> Signed-off-by: Armin Kuster <akuster808@gmail.com> Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
-rw-r--r--scripts/lib/resulttool/merge.py8
-rw-r--r--scripts/lib/resulttool/regression.py4
-rw-r--r--scripts/lib/resulttool/report.py2
-rw-r--r--scripts/lib/resulttool/resultutils.py23
-rw-r--r--scripts/lib/resulttool/store.py4
5 files changed, 28 insertions, 13 deletions
diff --git a/scripts/lib/resulttool/merge.py b/scripts/lib/resulttool/merge.py
index 3e4b7a38ad..7159463f6e 100644
--- a/scripts/lib/resulttool/merge.py
+++ b/scripts/lib/resulttool/merge.py
@@ -17,7 +17,7 @@ import json
17import resulttool.resultutils as resultutils 17import resulttool.resultutils as resultutils
18 18
19def merge(args, logger): 19def merge(args, logger):
20 if os.path.isdir(args.target_results): 20 if resultutils.is_url(args.target_results) or os.path.isdir(args.target_results):
21 results = resultutils.load_resultsdata(args.target_results, configmap=resultutils.store_map) 21 results = resultutils.load_resultsdata(args.target_results, configmap=resultutils.store_map)
22 resultutils.append_resultsdata(results, args.base_results, configmap=resultutils.store_map) 22 resultutils.append_resultsdata(results, args.base_results, configmap=resultutils.store_map)
23 resultutils.save_resultsdata(results, args.target_results) 23 resultutils.save_resultsdata(results, args.target_results)
@@ -31,12 +31,12 @@ def merge(args, logger):
31 31
32def register_commands(subparsers): 32def register_commands(subparsers):
33 """Register subcommands from this plugin""" 33 """Register subcommands from this plugin"""
34 parser_build = subparsers.add_parser('merge', help='merge test result files/directories', 34 parser_build = subparsers.add_parser('merge', help='merge test result files/directories/URLs',
35 description='merge the results from multiple files/directories into the target file or directory', 35 description='merge the results from multiple files/directories/URLs into the target file or directory',
36 group='setup') 36 group='setup')
37 parser_build.set_defaults(func=merge) 37 parser_build.set_defaults(func=merge)
38 parser_build.add_argument('base_results', 38 parser_build.add_argument('base_results',
39 help='the results file/directory to import') 39 help='the results file/directory/URL to import')
40 parser_build.add_argument('target_results', 40 parser_build.add_argument('target_results',
41 help='the target file or directory to merge the base_results with') 41 help='the target file or directory to merge the base_results with')
42 42
diff --git a/scripts/lib/resulttool/regression.py b/scripts/lib/resulttool/regression.py
index bdf531dedf..aecb9da9ce 100644
--- a/scripts/lib/resulttool/regression.py
+++ b/scripts/lib/resulttool/regression.py
@@ -161,9 +161,9 @@ def register_commands(subparsers):
161 group='analysis') 161 group='analysis')
162 parser_build.set_defaults(func=regression) 162 parser_build.set_defaults(func=regression)
163 parser_build.add_argument('base_result', 163 parser_build.add_argument('base_result',
164 help='base result file/directory for the comparison') 164 help='base result file/directory/URL for the comparison')
165 parser_build.add_argument('target_result', 165 parser_build.add_argument('target_result',
166 help='target result file/directory to compare with') 166 help='target result file/directory/URL to compare with')
167 parser_build.add_argument('-b', '--base-result-id', default='', 167 parser_build.add_argument('-b', '--base-result-id', default='',
168 help='(optional) filter the base results to this result ID') 168 help='(optional) filter the base results to this result ID')
169 parser_build.add_argument('-t', '--target-result-id', default='', 169 parser_build.add_argument('-t', '--target-result-id', default='',
diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py
index 90086209e3..8ae42728e4 100644
--- a/scripts/lib/resulttool/report.py
+++ b/scripts/lib/resulttool/report.py
@@ -143,7 +143,7 @@ def register_commands(subparsers):
143 group='analysis') 143 group='analysis')
144 parser_build.set_defaults(func=report) 144 parser_build.set_defaults(func=report)
145 parser_build.add_argument('source_dir', 145 parser_build.add_argument('source_dir',
146 help='source file/directory that contain the test result files to summarise') 146 help='source file/directory/URL that contain the test result files to summarise')
147 parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in") 147 parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
148 parser_build.add_argument('--commit', help="Revision to report") 148 parser_build.add_argument('--commit', help="Revision to report")
149 parser_build.add_argument('-t', '--tag', default='', 149 parser_build.add_argument('-t', '--tag', default='',
diff --git a/scripts/lib/resulttool/resultutils.py b/scripts/lib/resulttool/resultutils.py
index ad40ac8499..aab312dd17 100644
--- a/scripts/lib/resulttool/resultutils.py
+++ b/scripts/lib/resulttool/resultutils.py
@@ -16,6 +16,8 @@ import os
16import json 16import json
17import scriptpath 17import scriptpath
18import copy 18import copy
19import urllib
20import posixpath
19scriptpath.add_oe_lib_path() 21scriptpath.add_oe_lib_path()
20 22
21flatten_map = { 23flatten_map = {
@@ -40,20 +42,33 @@ store_map = {
40 "manual": ['TEST_TYPE', 'TEST_MODULE', 'MACHINE', 'IMAGE_BASENAME'] 42 "manual": ['TEST_TYPE', 'TEST_MODULE', 'MACHINE', 'IMAGE_BASENAME']
41} 43}
42 44
45def is_url(p):
46 """
47 Helper for determining if the given path is a URL
48 """
49 return p.startswith('http://') or p.startswith('https://')
50
43# 51#
44# Load the json file and append the results data into the provided results dict 52# Load the json file and append the results data into the provided results dict
45# 53#
46def append_resultsdata(results, f, configmap=store_map): 54def append_resultsdata(results, f, configmap=store_map):
47 if type(f) is str: 55 if type(f) is str:
48 with open(f, "r") as filedata: 56 if is_url(f):
49 data = json.load(filedata) 57 with urllib.request.urlopen(f) as response:
58 data = json.loads(response.read().decode('utf-8'))
59 url = urllib.parse.urlparse(f)
60 testseries = posixpath.basename(posixpath.dirname(url.path))
61 else:
62 with open(f, "r") as filedata:
63 data = json.load(filedata)
64 testseries = os.path.basename(os.path.dirname(f))
50 else: 65 else:
51 data = f 66 data = f
52 for res in data: 67 for res in data:
53 if "configuration" not in data[res] or "result" not in data[res]: 68 if "configuration" not in data[res] or "result" not in data[res]:
54 raise ValueError("Test results data without configuration or result section?") 69 raise ValueError("Test results data without configuration or result section?")
55 if "TESTSERIES" not in data[res]["configuration"]: 70 if "TESTSERIES" not in data[res]["configuration"]:
56 data[res]["configuration"]["TESTSERIES"] = os.path.basename(os.path.dirname(f)) 71 data[res]["configuration"]["TESTSERIES"] = testseries
57 testtype = data[res]["configuration"].get("TEST_TYPE") 72 testtype = data[res]["configuration"].get("TEST_TYPE")
58 if testtype not in configmap: 73 if testtype not in configmap:
59 raise ValueError("Unknown test type %s" % testtype) 74 raise ValueError("Unknown test type %s" % testtype)
@@ -69,7 +84,7 @@ def append_resultsdata(results, f, configmap=store_map):
69# 84#
70def load_resultsdata(source, configmap=store_map): 85def load_resultsdata(source, configmap=store_map):
71 results = {} 86 results = {}
72 if os.path.isfile(source): 87 if is_url(source) or os.path.isfile(source):
73 append_resultsdata(results, source, configmap) 88 append_resultsdata(results, source, configmap)
74 return results 89 return results
75 for root, dirs, files in os.walk(source): 90 for root, dirs, files in os.walk(source):
diff --git a/scripts/lib/resulttool/store.py b/scripts/lib/resulttool/store.py
index e4a0807528..acdfbd94fd 100644
--- a/scripts/lib/resulttool/store.py
+++ b/scripts/lib/resulttool/store.py
@@ -29,7 +29,7 @@ def store(args, logger):
29 try: 29 try:
30 results = {} 30 results = {}
31 logger.info('Reading files from %s' % args.source) 31 logger.info('Reading files from %s' % args.source)
32 if os.path.isfile(args.source): 32 if resultutils.is_url(args.source) or os.path.isfile(args.source):
33 resultutils.append_resultsdata(results, args.source) 33 resultutils.append_resultsdata(results, args.source)
34 else: 34 else:
35 for root, dirs, files in os.walk(args.source): 35 for root, dirs, files in os.walk(args.source):
@@ -92,7 +92,7 @@ def register_commands(subparsers):
92 group='setup') 92 group='setup')
93 parser_build.set_defaults(func=store) 93 parser_build.set_defaults(func=store)
94 parser_build.add_argument('source', 94 parser_build.add_argument('source',
95 help='source file or directory that contain the test result files to be stored') 95 help='source file/directory/URL that contain the test result files to be stored')
96 parser_build.add_argument('git_dir', 96 parser_build.add_argument('git_dir',
97 help='the location of the git repository to store the results in') 97 help='the location of the git repository to store the results in')
98 parser_build.add_argument('-a', '--all', action='store_true', 98 parser_build.add_argument('-a', '--all', action='store_true',