resulttool: modify to be multi-machine

Currently, the code will sum all of the different machine results into a
single report of the tests results.  This can lead to confusion as to
which machine may be experiencing issues.  Modify the code to store the
results in a per machine basis and report them accordingly.

(From OE-Core rev: 7ba5a0af31605eaf91eb75b5f673f4caf81361dc)

Signed-off-by: Jon Mason <jdmason@kudzu.us>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
Signed-off-by: Armin Kuster <akuster808@gmail.com>
Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>

3 files changed, 82 insertions, 68 deletions

diff --git a/meta/lib/oeqa/selftest/cases/resulttooltests.py b/meta/lib/oeqa/selftest/cases/resulttooltests.py
index 10eb9c12d9..dac5c46801 100644
--- a/meta/lib/oeqa/selftest/cases/resulttooltests.py
+++ b/meta/lib/oeqa/selftest/cases/resulttooltests.py
@@ -56,7 +56,7 @@ class ResultToolTests(OESelftestTestCase):
                                   'test4': {'status': 'ERROR'},
                                   'test5': {'status': 'SKIPPED'}}}
         report = ResultsTextReport()
-        result_report = report.get_aggregated_test_result(None, result_data)
+        result_report = report.get_aggregated_test_result(None, result_data, 'DummyMachine')
         self.assertTrue(result_report['passed'] == 2, msg="Passed count not correct:%s" % result_report['passed'])
         self.assertTrue(result_report['failed'] == 2, msg="Failed count not correct:%s" % result_report['failed'])
         self.assertTrue(result_report['skipped'] == 1, msg="Skipped count not correct:%s" % result_report['skipped'])
diff --git a/scripts/lib/resulttool/report.py b/scripts/lib/resulttool/report.py
index cb6b1cf948..a48c59f632 100644
--- a/scripts/lib/resulttool/report.py
+++ b/scripts/lib/resulttool/report.py
@@ -24,16 +24,19 @@ class ResultsTextReport(object):
                              'skipped': ['SKIPPED', 'skipped']}
 
 
-    def handle_ptest_result(self, k, status, result):
+    def handle_ptest_result(self, k, status, result, machine):
+        if machine not in self.ptests:
+            self.ptests[machine] = {}
+
         if k == 'ptestresult.sections':
             # Ensure tests without any test results still show up on the report
             for suite in result['ptestresult.sections']:
-                if suite not in self.ptests:
-                    self.ptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
+                if suite not in self.ptests[machine]:
+                    self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
                 if 'duration' in result['ptestresult.sections'][suite]:
-                    self.ptests[suite]['duration'] = result['ptestresult.sections'][suite]['duration']
+                    self.ptests[machine][suite]['duration'] = result['ptestresult.sections'][suite]['duration']
                 if 'timeout' in result['ptestresult.sections'][suite]:
-                    self.ptests[suite]['duration'] += " T"
+                    self.ptests[machine][suite]['duration'] += " T"
             return
         try:
             _, suite, test = k.split(".", 2)
@@ -47,22 +50,25 @@ class ResultsTextReport(object):
                     suite = suite + "." + suite1
             except ValueError:
                 pass
-        if suite not in self.ptests:
-            self.ptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
+        if suite not in self.ptests[machine]:
+            self.ptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
         for tk in self.result_types:
             if status in self.result_types[tk]:
-                self.ptests[suite][tk] += 1
+                self.ptests[machine][suite][tk] += 1
+
+    def handle_ltptest_result(self, k, status, result, machine):
+        if machine not in self.ltptests:
+            self.ltptests[machine] = {}
 
-    def handle_ltptest_result(self, k, status, result):
         if k == 'ltpresult.sections':
             # Ensure tests without any test results still show up on the report
             for suite in result['ltpresult.sections']:
-                if suite not in self.ltptests:
-                    self.ltptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
+                if suite not in self.ltptests[machine]:
+                    self.ltptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
                 if 'duration' in result['ltpresult.sections'][suite]:
-                    self.ltptests[suite]['duration'] = result['ltpresult.sections'][suite]['duration']
+                    self.ltptests[machine][suite]['duration'] = result['ltpresult.sections'][suite]['duration']
                 if 'timeout' in result['ltpresult.sections'][suite]:
-                    self.ltptests[suite]['duration'] += " T"
+                    self.ltptests[machine][suite]['duration'] += " T"
             return
         try:
             _, suite, test = k.split(".", 2)
@@ -77,20 +83,23 @@ class ResultsTextReport(object):
                     suite = suite + "." + suite1
             except ValueError:
                 pass
-        if suite not in self.ltptests:
-            self.ltptests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
+        if suite not in self.ltptests[machine]:
+            self.ltptests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
         for tk in self.result_types:
             if status in self.result_types[tk]:
-                self.ltptests[suite][tk] += 1
+                self.ltptests[machine][suite][tk] += 1
+
+    def handle_ltpposixtest_result(self, k, status, result, machine):
+        if machine not in self.ltpposixtests:
+            self.ltpposixtests[machine] = {}
 
-    def handle_ltpposixtest_result(self, k, status, result):
         if k == 'ltpposixresult.sections':
             # Ensure tests without any test results still show up on the report
             for suite in result['ltpposixresult.sections']:
-                if suite not in self.ltpposixtests:
-                    self.ltpposixtests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
+                if suite not in self.ltpposixtests[machine]:
+                    self.ltpposixtests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
                 if 'duration' in result['ltpposixresult.sections'][suite]:
-                    self.ltpposixtests[suite]['duration'] = result['ltpposixresult.sections'][suite]['duration']
+                    self.ltpposixtests[machine][suite]['duration'] = result['ltpposixresult.sections'][suite]['duration']
             return
         try:
             _, suite, test = k.split(".", 2)
@@ -104,19 +113,13 @@ class ResultsTextReport(object):
                     suite = suite + "." + suite1
             except ValueError:
                 pass
-        if suite not in self.ltpposixtests:
-            self.ltpposixtests[suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
+        if suite not in self.ltpposixtests[machine]:
+            self.ltpposixtests[machine][suite] = {'passed': 0, 'failed': 0, 'skipped': 0, 'duration' : '-', 'failed_testcases': []}
         for tk in self.result_types:
             if status in self.result_types[tk]:
-                self.ltpposixtests[suite][tk] += 1
+                self.ltpposixtests[machine][suite][tk] += 1
 
-    def get_aggregated_test_result(self, logger, testresult):
-        test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
-    def get_aggregated_test_result(self, logger, testresult):
-        test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
-    def get_aggregated_test_result(self, logger, testresult):
-        test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
-    def get_aggregated_test_result(self, logger, testresult):
+    def get_aggregated_test_result(self, logger, testresult, machine):
         test_count_report = {'passed': 0, 'failed': 0, 'skipped': 0, 'failed_testcases': []}
         result = testresult.get('result', [])
         for k in result:
@@ -127,11 +130,11 @@ class ResultsTextReport(object):
             if test_status in self.result_types['failed']:
                 test_count_report['failed_testcases'].append(k)
             if k.startswith("ptestresult."):
-                self.handle_ptest_result(k, test_status, result)
+                self.handle_ptest_result(k, test_status, result, machine)
             if k.startswith("ltpresult."):
-                self.handle_ltptest_result(k, test_status, result)
+                self.handle_ltptest_result(k, test_status, result, machine)
             if k.startswith("ltpposixresult."):
-                self.handle_ltpposixtest_result(k, test_status, result)
+                self.handle_ltpposixtest_result(k, test_status, result, machine)
         return test_count_report
 
     def print_test_report(self, template_file_name, test_count_reports):
@@ -141,10 +144,8 @@ class ResultsTextReport(object):
         env = Environment(loader=file_loader, trim_blocks=True)
         template = env.get_template(template_file_name)
         havefailed = False
-        haveptest = bool(self.ptests)
-        haveltp = bool(self.ltptests)
-        haveltpposix = bool(self.ltpposixtests)
         reportvalues = []
+        machines = []
         cols = ['passed', 'failed', 'skipped']
         maxlen = {'passed' : 0, 'failed' : 0, 'skipped' : 0, 'result_id': 0, 'testseries' : 0, 'ptest' : 0 ,'ltptest': 0, 'ltpposixtest': 0}
         for line in test_count_reports:
@@ -162,21 +163,24 @@ class ResultsTextReport(object):
             reportvalues.append(vals)
             if line['failed_testcases']:
                 havefailed = True
-        for ptest in self.ptests:
-            if len(ptest) > maxlen['ptest']:
-                maxlen['ptest'] = len(ptest)
-        for ltptest in self.ltptests:
-            if len(ltptest) > maxlen['ltptest']:
-                maxlen['ltptest'] = len(ltptest)
-        for ltpposixtest in self.ltpposixtests:
-            if len(ltpposixtest) > maxlen['ltpposixtest']:
-                maxlen['ltpposixtest'] = len(ltpposixtest)
+            if line['machine'] not in machines:
+                machines.append(line['machine'])
+        for (machine, report) in self.ptests.items():
+            for ptest in self.ptests[machine]:
+                if len(ptest) > maxlen['ptest']:
+                    maxlen['ptest'] = len(ptest)
+        for (machine, report) in self.ltptests.items():
+            for ltptest in self.ltptests[machine]:
+                if len(ltptest) > maxlen['ltptest']:
+                    maxlen['ltptest'] = len(ltptest)
+        for (machine, report) in self.ltpposixtests.items():
+            for ltpposixtest in self.ltpposixtests[machine]:
+                if len(ltpposixtest) > maxlen['ltpposixtest']:
+                    maxlen['ltpposixtest'] = len(ltpposixtest)
         output = template.render(reportvalues=reportvalues,
                                  havefailed=havefailed,
-                                 haveptest=haveptest,
+                                 machines=machines,
                                  ptests=self.ptests,
-                                 haveltp=haveltp,
-                                 haveltpposix=haveltpposix,
                                  ltptests=self.ltptests,
                                  ltpposixtests=self.ltpposixtests,
                                  maxlen=maxlen)
@@ -200,7 +204,9 @@ class ResultsTextReport(object):
         for testsuite in testresults:
             for resultid in testresults[testsuite]:
                 result = testresults[testsuite][resultid]
-                test_count_report = self.get_aggregated_test_result(logger, result)
+                machine = result['configuration']['MACHINE']
+                test_count_report = self.get_aggregated_test_result(logger, result, machine)
+                test_count_report['machine'] = machine
                 test_count_report['testseries'] = result['configuration']['TESTSERIES']
                 test_count_report['result_id'] = resultid
                 test_count_reports.append(test_count_report)
diff --git a/scripts/lib/resulttool/template/test_report_full_text.txt b/scripts/lib/resulttool/template/test_report_full_text.txt
index d2725b8d01..6a5a081c6d 100644
--- a/scripts/lib/resulttool/template/test_report_full_text.txt
+++ b/scripts/lib/resulttool/template/test_report_full_text.txt
@@ -9,54 +9,62 @@ Test Result Status Summary (Counts/Percentages sorted by testseries, ID)
 {% endfor %}
 --------------------------------------------------------------------------------------------------------------
 
-{% if haveptest %}
+
+{% for machine in machines %}
+{% if ptests[machine] %}
 ==============================================================================================================
-PTest Result Summary
+{{ machine }} PTest Result Summary
 ==============================================================================================================
 --------------------------------------------------------------------------------------------------------------
 {{ 'Recipe'.ljust(maxlen['ptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
 --------------------------------------------------------------------------------------------------------------
-{% for ptest in ptests |sort %}
-{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[ptest]['duration']|string) }}
+{% for ptest in ptests[machine] |sort %}
+{{ ptest.ljust(maxlen['ptest']) }} | {{ (ptests[machine][ptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ptests[machine][ptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ptests[machine][ptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ptests[machine][ptest]['duration']|string) }}
 {% endfor %}
 --------------------------------------------------------------------------------------------------------------
-
 {% else %}
-There was no ptest data
+There was no ptest data for {{ machine }}
 {% endif %}
 
-{% if haveltp %}
+{% endfor %}
+
+
+{% for machine in machines %}
+{% if ltptests[machine] %}
 ==============================================================================================================
-Ltp Test Result Summary
+{{ machine }} Ltp Test Result Summary
 ==============================================================================================================
 --------------------------------------------------------------------------------------------------------------
 {{ 'Recipe'.ljust(maxlen['ltptest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
 --------------------------------------------------------------------------------------------------------------
-{% for ltptest in ltptests |sort %}
-{{ ltptest.ljust(maxlen['ltptest']) }} | {{ (ltptests[ltptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltptests[ltptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltptests[ltptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltptests[ltptest]['duration']|string) }}
+{% for ltptest in ltptests[machine] |sort %}
+{{ ltptest.ljust(maxlen['ltptest']) }} | {{ (ltptests[machine][ltptest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltptests[machine][ltptest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltptests[machine][ltptest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltptests[machine][ltptest]['duration']|string) }}
 {% endfor %}
 --------------------------------------------------------------------------------------------------------------
-
 {% else %}
-There was no LTP Test data
+There was no LTP Test data for {{ machine }}
 {% endif %}
 
-{% if haveltpposix %}
+{% endfor %}
+
+
+{% for machine in machines %}
+{% if ltpposixtests[machine] %}
 ==============================================================================================================
-Ltp Posix Result Summary
+{{ machine }} Ltp Posix Result Summary
 ==============================================================================================================
 --------------------------------------------------------------------------------------------------------------
 {{ 'Recipe'.ljust(maxlen['ltpposixtest']) }} | {{ 'Passed'.ljust(maxlen['passed']) }} | {{ 'Failed'.ljust(maxlen['failed']) }} | {{ 'Skipped'.ljust(maxlen['skipped']) }} | {{ 'Time(s)'.ljust(10) }}
 --------------------------------------------------------------------------------------------------------------
-{% for ltpposixtest in ltpposixtests |sort %}
-{{ ltpposixtest.ljust(maxlen['ltpposixtest']) }} | {{ (ltpposixtests[ltpposixtest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltpposixtests[ltpposixtest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltpposixtests[ltpposixtest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltpposixtests[ltpposixtest]['duration']|string) }}
+{% for ltpposixtest in ltpposixtests[machine] |sort %}
+{{ ltpposixtest.ljust(maxlen['ltpposixtest']) }} | {{ (ltpposixtests[machine][ltpposixtest]['passed']|string).ljust(maxlen['passed']) }} | {{ (ltpposixtests[machine][ltpposixtest]['failed']|string).ljust(maxlen['failed']) }} | {{ (ltpposixtests[machine][ltpposixtest]['skipped']|string).ljust(maxlen['skipped']) }} | {{ (ltpposixtests[machine][ltpposixtest]['duration']|string) }}
 {% endfor %}
 --------------------------------------------------------------------------------------------------------------
-
 {% else %}
-There was no LTP Posix Test data
+There was no LTP Posix Test data for {{ machine }}
 {% endif %}
 
+{% endfor %}
 
 
 ==============================================================================================================