Add catch2 support

- Also drops support for Python 2.7 and 3.5.
- Adapt boost tests.

PR #60

Author: salim-runsafe

.github/workflows/main.yml

@@ -53,12 +53,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: ["2.7", "3.5", "3.6", "3.7", "3.8"]
+        python: ["3.6", "3.7", "3.8"]
         include:
-          - python: "2.7"
-            tox_env: "py27-pytestlatest"
-          - python: "3.5"
-            tox_env: "py35-pytestlatest"
           - python: "3.6"
             tox_env: "py36-pytestlatest"
           - python: "3.7"

README.rst

@@ -6,8 +6,9 @@ Use `pytest <https://pypi.python.org/pypi/pytest>`_ runner to discover and execu
 
 |python| |version| |anaconda| |ci| |coverage| |black|
 
-Supports both `Google Test <https://code.google.com/p/googletest>`_ and
-`Boost::Test <http://www.boost.org/doc/libs/release/libs/test>`_:
+Supports `Google Test <https://code.google.com/p/googletest>`_,
+`Boost::Test <http://www.boost.org/doc/libs/release/libs/test>`_,
+and `Catch2 <https://github.com/catchorg/Catch2>`_:
 
 .. image:: https://raw.githubusercontent.com/pytest-dev/pytest-cpp/master/images/screenshot.png
 
@@ -46,7 +47,7 @@ Usage
 
 Once installed, when py.test runs it will search and run tests
 found in executable files, detecting if the suites are
-Google or Boost tests automatically.
+Google, Boost, or Catch2 tests automatically.
 
 Configuration Options
 ~~~~~~~~~~~~~~~~~~~~~

setup.py

@@ -9,6 +9,7 @@
     package_dir={"": "src"},
     entry_points={"pytest11": ["cpp = pytest_cpp.plugin"],},
     install_requires=["pytest !=5.4.0, !=5.4.1", "colorama",],
+    python_requires=">=3.6",
     # metadata for upload to PyPI
     author="Bruno Oliveira",
     author_email="[email protected]",
@@ -23,10 +24,7 @@
         "Intended Audience :: Developers",
         "License :: OSI Approved :: MIT License",
         "Operating System :: OS Independent",
-        "Programming Language :: Python :: 2",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",

src/pytest_cpp/boost.py

@@ -79,14 +79,6 @@ def read_file(name):
             )
             return [failure], stdout
 
-        if report is not None and (
-            report.startswith("Boost.Test framework internal error: ")
-            or report.startswith("Test setup error: ")
-        ):
-            # boost.test doesn't do XML output on fatal-enough errors.
-            failure = BoostTestFailure("unknown location", 0, report)
-            return [failure], stdout
-
         results = self._parse_log(log=log)
         shutil.rmtree(temp_dir)
 
@@ -102,22 +94,23 @@ def _parse_log(self, log):
         This is always a XML file, and from this we produce most of the
         failures possible when running BoostTest.
         """
-        # Fatal errors apparently generate invalid xml in the form:
-        # <FatalError>...</FatalError><TestLog>...</TestLog>
-        # so we have to manually split it into two xmls if that's the case.
+        # Boosttest will sometimes generate unparseable XML
+        # so we surround it with xml tags.
         parsed_elements = []
-        if log.startswith("<FatalError"):
-            fatal, log = log.split("</FatalError>")
-            fatal += "</FatalError>"  # put it back, removed by split()
-            fatal_root = ElementTree.fromstring(fatal)
-            fatal_root.text = "Fatal Error: %s" % fatal_root.text
-            parsed_elements.append(fatal_root)
+        log = "<xml>{}</xml>".format(log)
 
         log_root = ElementTree.fromstring(log)
+        testlog = log_root.find("TestLog")
+
         parsed_elements.extend(log_root.findall("Exception"))
         parsed_elements.extend(log_root.findall("Error"))
         parsed_elements.extend(log_root.findall("FatalError"))
 
+        if testlog is not None:
+            parsed_elements.extend(testlog.findall("Exception"))
+            parsed_elements.extend(testlog.findall("Error"))
+            parsed_elements.extend(testlog.findall("FatalError"))
+
         result = []
         for elem in parsed_elements:
             filename = elem.attrib["file"]

src/pytest_cpp/catch2.py

@@ -0,0 +1,153 @@
+import os
+import subprocess
+import tempfile
+from xml.etree import ElementTree
+
+import pytest
+
+from pytest_cpp.error import CppTestFailure
+
+
+class Catch2Facade(object):
+    """
+    Facade for Catch2.
+    """
+
+    @classmethod
+    def is_test_suite(cls, executable):
+        try:
+            output = subprocess.check_output(
+                [executable, "--help"],
+                stderr=subprocess.STDOUT,
+                universal_newlines=True,
+            )
+        except (subprocess.CalledProcessError, OSError):
+            return False
+        else:
+            return "--list-test-names-only" in output
+
+    def list_tests(self, executable):
+        """
+        Executes test with "--list-test-names-only" and gets list of tests
+        parsing output like this:
+
+        1: All test cases reside in other .cpp files (empty)
+        2: Factorial of 0 is 1 (fail)
+        2: Factorials of 1 and higher are computed (pass)
+        """
+        # This will return an exit code with the number of tests available
+        try:
+            output = subprocess.check_output(
+                [executable, "--list-test-names-only"],
+                stderr=subprocess.STDOUT,
+                universal_newlines=True,
+            )
+        except subprocess.CalledProcessError as e:
+            output = e.output
+
+        result = output.strip().split("\n")
+
+        return result
+
+    def run_test(self, executable, test_id="", test_args=(), harness=None):
+        harness = harness or []
+        xml_filename = self._get_temp_xml_filename()
+        args = harness + [
+            executable,
+            test_id,
+            "--success",
+            "--reporter=xml",
+            "--out %s" % xml_filename,
+        ]
+        args.extend(test_args)
+
+        try:
+            output = subprocess.check_output(
+                args, stderr=subprocess.STDOUT, universal_newlines=True
+            )
+        except subprocess.CalledProcessError as e:
+            output = e.output
+            if e.returncode != 1:
+                msg = (
+                    "Internal Error: calling {executable} "
+                    "for test {test_id} failed (returncode={returncode}):\n"
+                    "{output}"
+                )
+                failure = Catch2Failure(
+                    executable,
+                    0,
+                    msg.format(
+                        executable=executable,
+                        test_id=test_id,
+                        output=e.output,
+                        returncode=e.returncode,
+                    ),
+                )
+
+                return [failure], output
+
+        results = self._parse_xml(xml_filename)
+        os.remove(xml_filename)
+        for (executed_test_id, failures, skipped) in results:
+            if executed_test_id == test_id:
+                if failures:
+                    return (
+                        [
+                            Catch2Failure(filename, linenum, lines)
+                            for (filename, linenum, lines) in failures
+                        ],
+                        output,
+                    )
+                elif skipped:
+                    pytest.skip()
+                else:
+                    return None, output
+
+        msg = "Internal Error: could not find test " "{test_id} in results:\n{results}"
+
+        results_list = "\n".join("\n".join(x) for (n, x, f) in results)
+        failure = Catch2Failure(msg.format(test_id=test_id, results=results_list))
+        return [failure], output
+
+    def _get_temp_xml_filename(self):
+        return tempfile.mktemp()
+
+    def _parse_xml(self, xml_filename):
+        root = ElementTree.parse(xml_filename)
+        result = []
+        for test_suite in root.findall("Group"):
+            test_suite_name = test_suite.attrib["name"]
+            for test_case in test_suite.findall("TestCase"):
+                test_name = test_case.attrib["name"]
+                test_result = test_case.find("OverallResult")
+                failures = []
+                if test_result.attrib["success"] == "false":
+                    test_checks = test_case.findall("Expression")
+                    for check in test_checks:
+                        file_name = check.attrib["filename"]
+                        line_num = check.attrib["line"]
+                        if check.attrib["success"] == "false":
+                            expected = check.find("Original").text
+                            actual = check.find("Expanded").text
+                            fail_msg = "Expected: {expected}\nActual: {actual}".format(
+                                expected=expected, actual=actual
+                            )
+                            failures.append((file_name, line_num, fail_msg,))
+                skipped = False  # TODO: skipped tests don't appear in the results
+                result.append((test_name, failures, skipped))
+
+        return result
+
+
+class Catch2Failure(CppTestFailure):
+    def __init__(self, filename, linenum, lines):
+        self.lines = lines.splitlines()
+        self.filename = filename
+        self.linenum = int(linenum)
+
+    def get_lines(self):
+        m = ("red", "bold")
+        return [(x, m) for x in self.lines]
+
+    def get_file_reference(self):
+        return self.filename, self.linenum

src/pytest_cpp/plugin.py

@@ -7,8 +7,9 @@
 from pytest_cpp.boost import BoostTestFacade
 from pytest_cpp.error import CppFailureRepr, CppFailureError
 from pytest_cpp.google import GoogleTestFacade
+from pytest_cpp.catch2 import Catch2Facade
 
-FACADES = [GoogleTestFacade, BoostTestFacade]
+FACADES = [GoogleTestFacade, BoostTestFacade, Catch2Facade]
 DEFAULT_MASKS = ("test_*", "*_test")
 
 _ARGUMENTS = "cpp_arguments"

tests/.gitignore

@@ -1,5 +1,6 @@
 *
 
+!acceptance/
 !.gitignore
 !*.h
 !*.cpp
@@ -8,3 +9,4 @@
 !*.xml
 !README*
 !SCons*
+!catch.hpp

tests/SConstruct

@@ -19,8 +19,9 @@ if 'CXX' in os.environ:
 
 env = Environment(**kwargs)
 genv = env.Clone(LIBS=['gtest'] + LIBS)
+c2env = env.Clone(CPPPATH=['.'])
 
-Export('env genv')
+Export('env genv c2env')
 
 genv.Program('gtest.cpp')
 genv.Program('gtest_args.cpp')
@@ -38,5 +39,9 @@ boost_files = [
 for filename in boost_files:
     env.Program(filename)
 
+c2env.Program('catch2_success.cpp')
+c2env.Program('catch2_failure.cpp')
+
 SConscript('acceptance/googletest-samples/SConscript')
 SConscript('acceptance/boosttest-samples/SConscript')
+SConscript('acceptance/catch2-samples/SConscript')

tests/acceptance/catch2-samples/000-CatchMain.cpp

@@ -0,0 +1,14 @@
+// 000-CatchMain.cpp
+
+// It is generally recommended to have a single file provide the main
+// of a testing binary, and other test files to link against it.
+
+// Let Catch provide main():
+#define CATCH_CONFIG_MAIN  // This tells Catch to provide a main() - only do this in one cpp file
+#include "catch.hpp"
+
+// That's it
+
+// Compile implementation of Catch for use with files that do contain tests:
+// - g++ -std=c++11 -Wall -I$(CATCH_SINGLE_INCLUDE) -c 000-CatchMain.cpp
+// - cl -EHsc -I%CATCH_SINGLE_INCLUDE% -c 000-CatchMain.cpp

tests/acceptance/catch2-samples/010-TestCase.cpp

@@ -0,0 +1,33 @@
+// 010-TestCase.cpp
+
+#include "catch.hpp"
+
+int Factorial( int number ) {
+   return number <= 1 ? number : Factorial( number - 1 ) * number;  // fail
+// return number <= 1 ? 1      : Factorial( number - 1 ) * number;  // pass
+}
+
+TEST_CASE( "Factorial of 0 is 1 (fail)", "[single-file]" ) {
+    REQUIRE( Factorial(0) == 1 );
+}
+
+TEST_CASE( "Factorials of 1 and higher are computed (pass)", "[single-file]" ) {
+    REQUIRE( Factorial(1) == 1 );
+    REQUIRE( Factorial(2) == 2 );
+    REQUIRE( Factorial(3) == 6 );
+    REQUIRE( Factorial(10) == 3628800 );
+}
+
+// Compile & run:
+// - g++ -std=c++11 -Wall -I$(CATCH_SINGLE_INCLUDE) -o 010-TestCase 010-TestCase.cpp && 010-TestCase --success
+// - cl -EHsc -I%CATCH_SINGLE_INCLUDE% 010-TestCase.cpp && 010-TestCase --success
+
+// Expected compact output (all assertions):
+//
+// prompt> 010-TestCase --reporter compact --success
+// 010-TestCase.cpp:14: failed: Factorial(0) == 1 for: 0 == 1
+// 010-TestCase.cpp:18: passed: Factorial(1) == 1 for: 1 == 1
+// 010-TestCase.cpp:19: passed: Factorial(2) == 2 for: 2 == 2
+// 010-TestCase.cpp:20: passed: Factorial(3) == 6 for: 6 == 6
+// 010-TestCase.cpp:21: passed: Factorial(10) == 3628800 for: 3628800 (0x375f00) == 3628800 (0x375f00)
+// Failed 1 test case, failed 1 assertion.