From b34039d790a1df1c81a16e7762e0da42e3ccbd9a Mon Sep 17 00:00:00 2001 From: dongfangtianyu <7629022+dongfangtianyu@users.noreply.github.com> Date: Thu, 17 Sep 2020 14:01:29 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat=EF=BC=9A=20show=20`expected=5Ffailure`?= =?UTF-8?q?=20and=20`unexpected=5Fsuccess`=20in=20HTMLreport?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- HtmlTestRunner/result.py | 30 ++++++++++++++++---- HtmlTestRunner/template/report_template.html | 29 +++++++++++++++---- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/HtmlTestRunner/result.py b/HtmlTestRunner/result.py index 96fb431..1c2b18c 100644 --- a/HtmlTestRunner/result.py +++ b/HtmlTestRunner/result.py @@ -1,16 +1,15 @@ from __future__ import print_function +import copy import os import sys import time -import copy import traceback from unittest import TestResult, TextTestResult from unittest.result import failfast from jinja2 import Template - DEFAULT_TEMPLATE = os.path.join(os.path.dirname(__file__), "template", "report_template.html") @@ -71,7 +70,7 @@ def strip_module_names(testcase_names): class _TestInfo(object): """" Keeps information about the execution of a test method. """ - (SUCCESS, FAILURE, ERROR, SKIP) = range(4) + (SUCCESS, FAILURE, ERROR, SKIP, XFAIL, XPASS,) = range(6) def __init__(self, test_result, test_method, outcome=SUCCESS, err=None, subTest=None): @@ -86,7 +85,7 @@ def __init__(self, test_result, test_method, outcome=SUCCESS, self.test_description = self.test_result.getDescription(test_method) self.test_exception_info = ( - '' if outcome in (self.SUCCESS, self.SKIP) + '' if outcome in (self.SUCCESS, self.SKIP, self.XPASS) else self.test_result._exc_info_to_string( self.err, test_method)) @@ -235,6 +234,19 @@ def addSkip(self, test, reason): testinfo = self.infoclass(self, test, self.infoclass.SKIP, reason) self._prepare_callback(testinfo, self.skipped, "SKIP", "S") + def addExpectedFailure(self, test, err): + """Called when an expected failure/error occurred.""" + self._save_output_data() + testinfo = self.infoclass(self, test, self.infoclass.XFAIL, err) + self._prepare_callback(testinfo, self.successes, "expected failure", "X") + + @failfast + def addUnexpectedSuccess(self, test): + """Called when a test was expected to fail, but succeed.""" + self._save_output_data() + self._prepare_callback(self.infoclass(self, test, self.infoclass.XPASS), self.successes, "unexpected success", + "U") + def printErrorList(self, flavour, errors): """ Writes information about the FAIL or ERROR to the stream. @@ -293,7 +305,7 @@ def _format_duration(elapsed_time): def get_results_summary(self, tests): """Create a summary of the outcomes of all given tests.""" - failures = errors = skips = successes = 0 + failures = errors = skips = successes = expected_failure = unexpected_success = 0 for test in tests: outcome = test.outcome if outcome == test.ERROR: @@ -304,6 +316,10 @@ def get_results_summary(self, tests): skips += 1 elif outcome == test.SUCCESS: successes += 1 + elif outcome == test.XFAIL: + expected_failure += 1 + elif outcome == test.XPASS: + unexpected_success += 1 elapsed_time = 0 for testinfo in tests: @@ -319,6 +335,8 @@ def get_results_summary(self, tests): "failure": failures, "skip": skips, "success": successes, + "expected_failure": expected_failure, + "unexpected_success": unexpected_success, "duration": self._format_duration(elapsed_time) } @@ -343,7 +361,7 @@ def _get_report_summaries(self, all_results, testRunner): def generate_reports(self, testRunner): """ Generate report(s) for all given test cases that have been run. """ - status_tags = ('success', 'danger', 'warning', 'info') + status_tags = ('success', 'danger', 'warning', 'info', 'info', 'warning') all_results = self._get_info_by_testcase() summaries = self._get_report_summaries(all_results, testRunner) diff --git a/HtmlTestRunner/template/report_template.html b/HtmlTestRunner/template/report_template.html index 4c2256b..ba518b8 100644 --- a/HtmlTestRunner/template/report_template.html +++ b/HtmlTestRunner/template/report_template.html @@ -13,7 +13,14 @@

{{ title }}

Start Time: {{ header_info.start_time.strftime("%Y-%m-%d %H:%M:%S") }}

Duration: {{ header_info.status.duration }}

-

Summary: Total: {{ header_info.status.total }}, Pass: {{ header_info.status.success }}{% if header_info.status.failure %}, Fail: {{ header_info.status.failure }}{% endif %}{% if header_info.status.error %}, Error: {{ header_info.status.error }}{% endif %}{% if header_info.status.skip %}, Skip: {{ header_info.status.skip }}{% endif %}

+

Summary: Total: {{ header_info.status.total }}, Pass: {{ header_info.status.success }} + {% if header_info.status.failure %}, Fail: {{ header_info.status.failure }}{% endif %} + {% if header_info.status.error %}, Error: {{ header_info.status.error }}{% endif %} + {% if header_info.status.skip %}, Skip: {{ header_info.status.skip }}{% endif %} + {% if header_info.status.expected_failure %}, XFail: {{ header_info.status.expected_failure }}{% endif %} + {% if header_info.status.unexpected_success %}, XPass: {{ header_info.status.unexpected_success }}{% endif %} +

+

Tester: {{ tester }}

{%- for test_case_name, tests_results in all_results.items() %} @@ -34,15 +41,21 @@

{{ title }}

{{ test_case.test_id.split(".")[-1] }} - + {%- if test_case.outcome == test_case.SUCCESS -%} Pass {%- elif test_case.outcome == test_case.SKIP -%} Skip {%- elif test_case.outcome == test_case.FAILURE -%} Fail - {%- else -%} + {%- elif test_case.outcome == test_case.ERROR -%} Error + {%- elif test_case.outcome == test_case.XFAIL -%} + XFail + {%- elif test_case.outcome == test_case.XPASS -%} + XPass + {%- else -%} + Other {%- endif -%} @@ -129,7 +142,13 @@

{{ title }}

{%- endfor %} - Total: {{ summaries[test_case_name].total }}, Pass: {{ summaries[test_case_name].success }}{% if summaries[test_case_name].failure %}, Fail: {{ summaries[test_case_name].failure }}{% endif %}{% if summaries[test_case_name].error %}, Error: {{ summaries[test_case_name].error }}{% endif %}{% if summaries[test_case_name].skip %}, Skip: {{ summaries[test_case_name].skip }}{% endif %} -- Duration: {{ summaries[test_case_name].duration }} + Total: {{ summaries[test_case_name].total }}, Pass: {{ summaries[test_case_name].success }} + {% if summaries[test_case_name].failure %}, Fail: {{ summaries[test_case_name].failure }}{% endif %} + {% if summaries[test_case_name].error %}, Error: {{ summaries[test_case_name].error }}{% endif %} + {% if summaries[test_case_name].skip %}, Skip: {{ summaries[test_case_name].skip }}{% endif %} + {% if summaries[test_case_name].expected_failure %}, XFail: {{ summaries[test_case_name].expected_failure }}{% endif %} + {% if summaries[test_case_name].unexpected_success %}, XPass: {{ header_info.status.unexpected_success }}{% endif %} + -- Duration: {{ summaries[test_case_name].duration }} @@ -159,4 +178,4 @@

{{ title }}

}); - \ No newline at end of file From c7e4293635d3cd2b0c51791001ded212ca0bbac6 Mon Sep 17 00:00:00 2001 From: dongfangtianyu <7629022+dongfangtianyu@users.noreply.github.com> Date: Tue, 22 Sep 2020 16:49:57 +0800 Subject: [PATCH 2/2] fix: expectedFailures and unexpectedSuccesses result input error list --- HtmlTestRunner/result.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/HtmlTestRunner/result.py b/HtmlTestRunner/result.py index 1c2b18c..5a8f01c 100644 --- a/HtmlTestRunner/result.py +++ b/HtmlTestRunner/result.py @@ -238,13 +238,13 @@ def addExpectedFailure(self, test, err): """Called when an expected failure/error occurred.""" self._save_output_data() testinfo = self.infoclass(self, test, self.infoclass.XFAIL, err) - self._prepare_callback(testinfo, self.successes, "expected failure", "X") + self._prepare_callback(testinfo, self.expectedFailures, "expected failure", "X") @failfast def addUnexpectedSuccess(self, test): """Called when a test was expected to fail, but succeed.""" self._save_output_data() - self._prepare_callback(self.infoclass(self, test, self.infoclass.XPASS), self.successes, "unexpected success", + self._prepare_callback(self.infoclass(self, test, self.infoclass.XPASS), self.unexpectedSuccesses, "unexpected success", "U") def printErrorList(self, flavour, errors): @@ -273,7 +273,7 @@ def _get_info_by_testcase(self): tests_by_testcase[testcase_name] = [] tests_by_testcase[testcase_name].append(subtest_info) - for tests in (self.successes, self.failures, self.errors, self.skipped): + for tests in (self.successes, self.failures, self.errors, self.skipped, self.expectedFailures, self.unexpectedSuccesses): for test_info in tests: # subtests will be contained by _SubTestInfos objects but there is also the # case where all subtests pass and the method is added as a success as well