better support for single test runs, better match catch2 output in TestResult, basic assertion tracking

Author: Tsche

example/always_passes.cpp

@@ -2,8 +2,7 @@
 #include <iostream>
 
 namespace demo {
-  template <typename T= void>
-auto zoinks()  -> T {
+auto zoinks() {
   ASSERT(false, "oh no");
 }
 
@@ -12,8 +11,4 @@ auto zoinks()  -> T {
   std::cerr << "bar\n";
   zoinks();
 }
-
-[[ = rsl::test, = rsl::expect_failure ]] void always_fails() {
-  ASSERT(false, "oh no");
-}
 }  // namespace demo

include/rsl/testing/assert.hpp

@@ -3,20 +3,21 @@
 #include <vector>
 #include <string_view>
 
-namespace rsl::testing::_testing_impl {
+namespace rsl::testing {
 struct AssertionInfo {
   std::string_view raw;
   std::string_view expanded;
   bool success;
 };
-
+namespace _testing_impl {
 struct AssertionTracker {
   std::vector<AssertionInfo> assertions;
   std::string test_name;
 };
 
 AssertionTracker& assertion_counter();
-}  // namespace rsl::testing::_testing_impl
+}  // namespace _testing_impl
+}  // namespace rsl::testing
 
 #define LIBASSERT_ASSERT_MAIN_BODY(expr,                                                       \
                                    name,                                                       \
@@ -38,5 +39,5 @@ AssertionTracker& assertion_counter();
                                            libassert_params LIBASSERT_VA_ARGS(__VA_ARGS__)     \
                                                pretty_function_arg);                           \
   }
-
+#define LIBASSERT_BREAK_ON_FAIL
 #include <libassert/assert.hpp>

include/rsl/testing/test.hpp

@@ -105,12 +105,15 @@ struct TestNamespace {
     bool operator==(iterator const& other) const;
   };
 
+  [[nodiscard]] bool is_empty() const { return tests.empty() && children.empty(); }
   [[nodiscard]] iterator begin() const { return iterator{*this}; }
   [[nodiscard]] static iterator end() { return {}; }
   void insert(const Test& test, size_t i = 0);
 
   [[nodiscard]] std::size_t count() const;
   bool run(Reporter* reporter);
+
+  void filter(std::span<std::string const> parts);
 };
 
 struct TestRoot : TestNamespace {

include/rsl/testing/test_case.hpp

@@ -1,18 +1,22 @@
 #pragma once
 #include <functional>
+#include <string>
+#include "assert.hpp"
 
 namespace rsl::testing {
 
 struct TestResult {
   class Test const* test;
   std::string name;
-
   bool passed;
-  std::string error;
+  double duration_ms;
 
+  std::string failure;
+  std::string exception;
   std::string stdout;
   std::string stderr;
-  double duration_ms;
+  
+  std::vector<AssertionInfo> assertions;
 };
 
 struct TestCase {

src/main/main.cpp

@@ -1,17 +1,71 @@
-#include "rsl/testing/output.hpp"
-#include "rsl/testing/test.hpp"
-#define RSLTEST_SKIP
 #include <memory>
-#include <rsl/test>
+#include <ranges>
 #include <string_view>
 #include <string>
 
 #include <rsl/config>
+#include <rsl/testing/output.hpp>
+#include <rsl/testing/test.hpp>
 #include <rsl/testing/util.hpp>
 #include <rsl/testing/_testing_impl/factory.hpp>
 #include "output.hpp"
 
-class[[= rsl::cli::description("rsl::test (in Catch2 v3.8.1 compatibility mode)")]] TestConfig : public rsl::cli {
+std::string_view base_name(std::string_view name) {
+  auto lt    = name.find('<');
+  auto paren = name.find('(');
+  auto pos   = std::min(lt, paren);
+  return pos == std::string_view::npos ? name : name.substr(0, pos);
+}
+
+std::vector<std::string_view> split_filter_path(std::string_view filter) {
+  if (filter.starts_with("::")) {
+    filter.remove_prefix(2);
+  }
+
+  std::vector<std::string_view> parts;
+
+  while (true) {
+    auto next  = filter.find("::");
+    auto templ = filter.find('<');
+
+    if (templ != std::string_view::npos && (next == std::string_view::npos || templ < next)) {
+      parts.push_back(filter);
+      break;
+    }
+
+    if (next == std::string_view::npos) {
+      parts.push_back(filter);
+      break;
+    }
+
+    parts.push_back(filter.substr(0, next));
+    filter.remove_prefix(next + 2);
+  }
+
+  return parts;
+}
+
+void filter_test_tree(rsl::testing::TestRoot& root,
+                      std::string_view filter,
+                      std::vector<std::string> subfilters) {
+  if (filter.empty() || filter == "[.],*") {
+    return;
+  }
+
+  auto parts = split_filter_path(filter);
+
+  for (auto part : std::ranges::reverse_view(parts)) {
+    subfilters.insert(subfilters.begin(), std::string(part));
+  }
+  if (!subfilters.empty()) {
+    // cut off template arguments and parameters
+    subfilters.back() = base_name(subfilters.back());
+  }
+  root.filter(subfilters);
+}
+
+class[[= rsl::cli::description("rsl::test (in Catch2 v3.8.1 compatibility mode)")]] TestConfig
+    : public rsl::cli {
   rsl::testing::TestRoot tree;
   std::vector<std::string> sections;
   std::unique_ptr<rsl::testing::Output> _output;
@@ -38,35 +92,7 @@ class[[= rsl::cli::description("rsl::test (in Catch2 v3.8.1 compatibility mode)"
       , _output(new rsl::testing::ConsoleOutput()) {}
 
   void apply_filter() {
-    if (filter.empty() || filter == "[.],*") {
-      return;
-    }
-    std::string_view filter_view = filter;
-    std::vector<std::string> names;
-    std::size_t idx = 0;
-    while ((idx = filter_view.find(',')) != filter_view.npos) {
-      names.emplace_back(filter_view.substr(0, idx));
-      filter_view.remove_prefix(idx + 1);
-    }
-    if (!filter_view.empty()) {
-      names.emplace_back(filter_view);
-    }
-
-    rsl::testing::TestRoot new_tree;
-    // rebuild the test tree with filters applied
-    for (auto&& test : tree) {
-      auto full_name = rsl::join_str(test.full_name, "::");
-
-      for (auto const& name : names) {
-        if (name == full_name || name == test.name || name == test.full_name[0]) {
-          new_tree.insert(test);
-        }
-      }
-      // if (!std::regex_search(full_name, std::regex{filter})) {
-      //   continue;
-      // }
-    }
-    tree = new_tree;
+    filter_test_tree(tree, filter, sections);
   }
 
   static void print_tests(rsl::testing::TestNamespace const& current, std::size_t indent = 0) {

src/main/reporters/catch2xml.cpp

@@ -1,3 +1,4 @@
+#include <numeric>
 #include <print>
 #include <string>
 #include <optional>
@@ -101,7 +102,13 @@ struct TestCase {
   void update_results() {
     for (auto& section : sections) {
       section.update_results();
+
+      // TODO this is no good, we need to count skips in sections
+      result.skips += unsigned(section.results.skipped);
     }
+    result.success = std::ranges::none_of(sections, [](Section const& section) {
+      return section.results.failures > section.results.expectedFailures;
+    });
   }
 };
 
@@ -194,7 +201,7 @@ class[[= rename("xml")]] Catch2XmlReporter : public Reporter::Registrar<Catch2Xm
     if (result.passed) {
       ++section->results.successes;
     } else {
-      section->failure = {.value = result.error};
+      section->failure = {.value = result.failure};
       ++section->results.failures;
     }
     section->results.durationInSeconds += result.duration_ms / 1000.;

src/main/reporters/terminal.cpp

@@ -26,7 +26,7 @@ class [[=rename("plain")]] ConsoleReporter : public Reporter::Registrar<ConsoleR
                  reset,
                  result.name,
                  result.duration_ms);
-      std::print("{}ERROR{}: {}\n", color[1], reset, result.error);
+      std::print("{}ERROR{}: {}\n", color[1], reset, result.failure);
       std::print("==== {}stdout{} ====\n{}\n", color[1], reset, result.stdout);
       std::print("==== {}stderr{} ====\n{}\n", color[1], reset, result.stderr);
     }

src/main/reporters/xml.cpp

@@ -23,7 +23,7 @@ class [[=rename("junit")]] JUnitXmlReporter : public Reporter::Registrar<JUnitXm
   void after_test(TestResult const& result) override {
     auto node = testcase{.name=std::string(result.name), .time=result.duration_ms / 1000.};
     if (!result.passed) {
-      node.failure = result.error + "\n";
+      node.failure = result.failure + "\n";
     }
     suite.tests.push_back(node);
   }

src/test.cpp

@@ -1,4 +1,5 @@
 #include <algorithm>
+#include <ranges>
 #include <print>
 #include <chrono>
 
@@ -26,20 +27,21 @@ void cleanup_frames(cpptrace::stacktrace& trace, std::string_view test_name) {
 
 void failure_handler(libassert::assertion_info const& info) {
   // libassert::enable_virtual_terminal_processing_if_needed();  // for terminal colors on windows
-  constexpr bool Colorize = true;
+  constexpr bool Colorize = false;
   auto width              = libassert::terminal_width(libassert::stderr_fileno);
   const auto& scheme  = Colorize ? libassert::get_color_scheme() : libassert::color_scheme::blank;
   std::string message = std::string(info.action()) + " at " + info.location() + ":";
   if (info.message) {
     message += " " + *info.message;
   }
   message += "\n";
-  message += info.statement(scheme) + info.print_binary_diagnostics(width, scheme) +
-             info.print_extra_diagnostics(width, scheme);// + info.print_stacktrace(width, scheme);
+  message +=
+      info.statement(scheme) + info.print_binary_diagnostics(width, scheme) +
+      info.print_extra_diagnostics(width, scheme);  // + info.print_stacktrace(width, scheme);
 
   auto trace = info.get_stacktrace();
-  cleanup_frames(trace,  rsl::testing::_testing_impl::assertion_counter().test_name);
-  message += trace.to_string(true);
+  cleanup_frames(trace, rsl::testing::_testing_impl::assertion_counter().test_name);
+  message += trace.to_string(Colorize);
   throw rsl::testing::assertion_failure(message);
 }
 
@@ -82,15 +84,15 @@ bool TestNamespace::run(Reporter* reporter) {
 
     std::vector<TestResult> results;
     for (auto const& test_run : test.get_tests()) {
-      auto& tracker = _testing_impl::assertion_counter();
+      auto& tracker      = _testing_impl::assertion_counter();
       tracker.assertions = {};
-      tracker.test_name = join_str(test.full_name, "::");
+      tracker.test_name  = join_str(test.full_name, "::");
 
       reporter->before_test(test_run);
       auto result = test_run.run();
       reporter->after_test(result);
 
-      std::println("assertion count: {}", tracker.assertions.size());
+      result.assertions = tracker.assertions;
       results.push_back(result);
     }
 
@@ -116,16 +118,16 @@ TestResult TestCase::run() const {
     ret.duration_ms = std::chrono::duration<double, std::milli>(t1 - t0).count();
     return ret;
   } catch (assertion_failure const& failure) {
-    ret.error += failure.what();
+    ret.failure += failure.what();
   } catch (std::exception const& exc) {  //
-    ret.error += exc.what();
+    ret.exception += exc.what();
   } catch (std::string const& msg) {  //
-    ret.error += msg;
+    ret.failure += msg;
   } catch (std::string_view msg) {  //
-    ret.error += msg;
+    ret.failure += msg;
   } catch (char const* msg) {  //
-    ret.error += msg;
-  } catch (...) { ret.error += "unknown exception thrown"; }
+    ret.failure += msg;
+  } catch (...) { ret.exception += "unknown exception thrown"; }
 
   ret.passed = test->expect_failure;
   return ret;
@@ -197,6 +199,31 @@ std::size_t TestNamespace::count() const {
   return total;
 }
 
+void TestNamespace::filter(std::span<std::string const> parts) {
+  if (parts.empty()) {
+    return;
+  }
+
+  std::string_view current         = parts.front();
+  std::span<std::string const> next = parts.subspan(1);
+
+  auto it = std::ranges::find_if(children, [&](TestNamespace& ns) { return ns.name == current; });
+
+  if (it != children.end()) {
+    tests.clear();
+    it->filter(next);
+    if (it->children.empty() && it->tests.empty()) {
+      children.clear();
+    } else {
+      children = {*it};
+    }
+    return;
+  } else {
+    std::erase_if(tests, [&](const Test& t) { return t.name != current; });
+    children.clear();
+  }
+}
+
 TestRoot get_tests() {
   TestRoot root;
   for (auto test_def : rsl::testing::_testing_impl::registry()) {