Merge pull request #15 from Progmasoft/feature/catch3-runner

Replace Catch2's runner with the Catch3 runner

65 files changed, 3412 insertions, 122909 deletions

diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel
index 1e192a1c..00c39a7d 100644
--- a/tests/BUILD.bazel
+++ b/tests/BUILD.bazel
@@ -22,33 +22,23 @@ cc_test(
     srcs = [
         "SelfTest/IntrospectiveTests/Algorithms.tests.cpp",
         "SelfTest/IntrospectiveTests/AssertionHandler.tests.cpp",
-        "SelfTest/IntrospectiveTests/Clara.tests.cpp",
-        "SelfTest/IntrospectiveTests/CmdLine.tests.cpp",
-        "SelfTest/IntrospectiveTests/CmdLineHelpers.tests.cpp",
-        "SelfTest/IntrospectiveTests/ColourImpl.tests.cpp",
         "SelfTest/IntrospectiveTests/Details.tests.cpp",
         "SelfTest/IntrospectiveTests/FloatingPoint.tests.cpp",
         "SelfTest/IntrospectiveTests/GeneratorsImpl.tests.cpp",
         "SelfTest/IntrospectiveTests/Integer.tests.cpp",
         "SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp",
-        "SelfTest/IntrospectiveTests/Json.tests.cpp",
         "SelfTest/IntrospectiveTests/Parse.tests.cpp",
         "SelfTest/IntrospectiveTests/PartTracker.tests.cpp",
         "SelfTest/IntrospectiveTests/RandomNumberGeneration.tests.cpp",
-        "SelfTest/IntrospectiveTests/Reporters.tests.cpp",
-        "SelfTest/IntrospectiveTests/Sharding.tests.cpp",
-        "SelfTest/IntrospectiveTests/Stream.tests.cpp",
         "SelfTest/IntrospectiveTests/String.tests.cpp",
         "SelfTest/IntrospectiveTests/StringManip.tests.cpp",
         "SelfTest/IntrospectiveTests/Tag.tests.cpp",
         "SelfTest/IntrospectiveTests/TestCaseInfoHasher.tests.cpp",
         "SelfTest/IntrospectiveTests/TestSpec.tests.cpp",
         "SelfTest/IntrospectiveTests/TestSpecParser.tests.cpp",
-        "SelfTest/IntrospectiveTests/TextFlow.tests.cpp",
         "SelfTest/IntrospectiveTests/ToString.tests.cpp",
         "SelfTest/IntrospectiveTests/Traits.tests.cpp",
         "SelfTest/IntrospectiveTests/UniquePtr.tests.cpp",
-        "SelfTest/IntrospectiveTests/Xml.tests.cpp",
         "SelfTest/TestRegistrations.cpp",
         "SelfTest/TimingTests/Sleep.tests.cpp",
         "SelfTest/UsageTests/Approx.tests.cpp",
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 3d4db3b3..71411b53 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -32,7 +32,10 @@ if(CATCH_BUILD_SURROGATES)
     foreach(Source ${targetSources})
       string(REGEX MATCH "^.*\\.hpp$" isHeader ${Source})
       string(REGEX MATCH "_all.hpp" isAllHeader ${Source})
-      if(isHeader AND NOT isAllHeader)
+      # The runner's headers need C++20; the surrogates check Catch2's
+      # headers at the language level of the build.
+      string(REGEX MATCH "/Progmasoft/" isRunnerHeader ${Source})
+      if(isHeader AND NOT isAllHeader AND NOT isRunnerHeader)
         list(APPEND AllHeaders ${Source})
       endif()
     endforeach()
@@ -71,30 +74,20 @@ set(TEST_SOURCES
   ${SELF_TEST_DIR}/TestRegistrations.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/Algorithms.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/AssertionHandler.tests.cpp
-  ${SELF_TEST_DIR}/IntrospectiveTests/Clara.tests.cpp
-  ${SELF_TEST_DIR}/IntrospectiveTests/CmdLine.tests.cpp
-  ${SELF_TEST_DIR}/IntrospectiveTests/CmdLineHelpers.tests.cpp
-  ${SELF_TEST_DIR}/IntrospectiveTests/ColourImpl.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/Details.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/FloatingPoint.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/GeneratorsImpl.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/Integer.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/InternalBenchmark.tests.cpp
-  ${SELF_TEST_DIR}/IntrospectiveTests/Json.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/Parse.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/PartTracker.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/RandomNumberGeneration.tests.cpp
-  ${SELF_TEST_DIR}/IntrospectiveTests/Reporters.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/Tag.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/TestCaseInfoHasher.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/TestSpec.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/TestSpecParser.tests.cpp
-  ${SELF_TEST_DIR}/IntrospectiveTests/TextFlow.tests.cpp
-  ${SELF_TEST_DIR}/IntrospectiveTests/Sharding.tests.cpp
-  ${SELF_TEST_DIR}/IntrospectiveTests/Stream.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/String.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/StringManip.tests.cpp
-  ${SELF_TEST_DIR}/IntrospectiveTests/Xml.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/Traits.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/ToString.tests.cpp
   ${SELF_TEST_DIR}/IntrospectiveTests/UniquePtr.tests.cpp
@@ -162,13 +155,29 @@ add_executable(ProgmasoftSelfTest
   Progmasoft/Property.tests.cpp
   Progmasoft/ForAll.tests.cpp
   Progmasoft/Results.tests.cpp
+  Progmasoft/Runner.tests.cpp
+  Progmasoft/RunnerUnits.tests.cpp
+  Progmasoft/RunnerPlanner.tests.cpp
+  Progmasoft/RunnerOptions.tests.cpp
+  Progmasoft/RunnerAnalysis.tests.cpp
+  Progmasoft/RunnerOutput.tests.cpp
   Progmasoft/Snapshot.tests.cpp
   Progmasoft/Stateful.tests.cpp
   Progmasoft/Strategy.tests.cpp
 )
 target_link_libraries(ProgmasoftSelfTest PRIVATE Progmasoft::Catch3 Catch2WithMain)
 target_compile_features(ProgmasoftSelfTest PRIVATE cxx_std_20)
-add_test(NAME Progmasoft::PropertyTests COMMAND $<TARGET_FILE:ProgmasoftSelfTest>)
+
+# Tests that misbehave on purpose, for the runner's integration tests.
+add_executable(RunnerFixture Progmasoft/RunnerFixture.cpp)
+target_link_libraries(RunnerFixture PRIVATE Catch2WithMain)
+target_compile_features(RunnerFixture PRIVATE cxx_std_20)
+add_dependencies(ProgmasoftSelfTest RunnerFixture)
+
+add_test(NAME Progmasoft::PropertyTests COMMAND $<TARGET_FILE:ProgmasoftSelfTest> --no-history)
+set_tests_properties(Progmasoft::PropertyTests PROPERTIES
+  ENVIRONMENT "CATCH3_RUNNER_FIXTURE=$<TARGET_FILE:RunnerFixture>"
+)
 if(BUILD_SHARED_LIBS AND WIN32)
   add_custom_command(TARGET SelfTest PRE_LINK
     COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Catch2>
@@ -185,203 +194,108 @@ if(CATCH_ENABLE_COVERAGE)
 endif()
 
 # configure unit tests via CTest
-add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest> --order rand --rng-seed time --warn InfiniteGenerators)
-set_tests_properties(RunTests PROPERTIES
-  FAIL_REGULAR_EXPRESSION "Filters:"
-  COST 15
-)
+add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest> --order=random --warn=infinite-generators --no-history)
+set_tests_properties(RunTests PROPERTIES COST 15)
 
 # Because CTest does not allow us to check both return code _and_ expected
 # output in one test, we run these commands twice. First time we check
 # the output, the second time we check the exit code.
-add_test(NAME List::Tests::Output COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high)
+add_test(NAME List::Tests::Output COMMAND $<TARGET_FILE:SelfTest> --list)
 set_tests_properties(List::Tests::Output PROPERTIES
-  PASS_REGULAR_EXPRESSION "[0-9]+ test cases"
+  PASS_REGULAR_EXPRESSION "[0-9]+ matching test\\(s\\)"
   FAIL_REGULAR_EXPRESSION "Hidden Test"
 )
+add_test(NAME List::Tests::ExitCode COMMAND $<TARGET_FILE:SelfTest> --list)
 
-add_test(NAME VerbosityIsPerReporter COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high --reporter console::verbosity=quiet)
-set_tests_properties(VerbosityIsPerReporter
-  PROPERTIES
-    FAIL_REGULAR_EXPRESSION "\.cpp"
-)
-
-# This should be equivalent to the old --list-test-names-only and be usable
-# with --input-file.
-add_test(NAME List::Tests::Quiet COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity quiet)
-# Sadly we cannot ask for start-of-line and end-of-line in a ctest regex,
-# so we fail if we see space/tab at the start...
-set_tests_properties(List::Tests::Quiet PROPERTIES
-  PASS_REGULAR_EXPRESSION "\"#1905 -- test spec parser properly clears internal state between compound tests\"[\r\n]"
-  FAIL_REGULAR_EXPRESSION "[ \t]\"#1905 -- test spec parser properly clears internal state between compound tests\""
-)
-
-add_test(NAME List::Tests::ExitCode COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high)
-add_test(NAME List::Tests::XmlOutput COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high -r xml)
-set_tests_properties(List::Tests::XmlOutput PROPERTIES
-  PASS_REGULAR_EXPRESSION "<Line>[0-9]+</Line>"
-  FAIL_REGULAR_EXPRESSION "[0-9]+ test cases"
+add_test(NAME List::Tests::Names COMMAND $<TARGET_FILE:SelfTest> --list=names)
+# CTest regexes cannot anchor at the start of a line, so a name indented by
+# space or tab fails the test.
+set_tests_properties(List::Tests::Names PROPERTIES
+  PASS_REGULAR_EXPRESSION "#1905 -- test spec parser properly clears internal state between compound tests[\r\n]"
+  FAIL_REGULAR_EXPRESSION "[ \t]#1905 -- test spec parser properly clears internal state between compound tests"
 )
 
-add_test(NAME List::Tests::Json::Normal COMMAND $<TARGET_FILE:SelfTest> --list-tests -r json)
-set_tests_properties(List::Tests::Json::Normal PROPERTIES
-  PASS_REGULAR_EXPRESSION "\"tags\": \\["
-  FAIL_REGULAR_EXPRESSION "\"source-location\": {"
-)
-add_test(NAME List::Tests::Json::Quiet COMMAND $<TARGET_FILE:SelfTest> --list-tests -r json::verbosity=quiet)
-set_tests_properties(List::Tests::Json::Quiet PROPERTIES
-  PASS_REGULAR_EXPRESSION "\"name\": \""
-  FAIL_REGULAR_EXPRESSION "\"tags\": \\["
-)
-add_test(NAME List::Tests::Json::High COMMAND $<TARGET_FILE:SelfTest> --list-tests -r json::verbosity=high)
-set_tests_properties(List::Tests::Json::High PROPERTIES
-  PASS_REGULAR_EXPRESSION "\"source-location\": {"
+add_test(NAME List::Tests::Json COMMAND $<TARGET_FILE:SelfTest> --list=json)
+set_tests_properties(List::Tests::Json PROPERTIES
+  PASS_REGULAR_EXPRESSION "\"format\": \"catch3-tests\""
 )
 
 add_test(NAME List::Tags::Output COMMAND $<TARGET_FILE:SelfTest> --list-tags)
 set_tests_properties(List::Tags::Output PROPERTIES
-  PASS_REGULAR_EXPRESSION "[0-9]+ tags"
+  PASS_REGULAR_EXPRESSION "[0-9]+ tag\\(s\\)"
   FAIL_REGULAR_EXPRESSION "\\[\\.\\]")
 add_test(NAME List::Tags::ExitCode COMMAND $<TARGET_FILE:SelfTest> --list-tags)
-add_test(NAME List::Tags::XmlOutput COMMAND $<TARGET_FILE:SelfTest> --list-tags -r xml)
-set_tests_properties(List::Tags::XmlOutput PROPERTIES
-  PASS_REGULAR_EXPRESSION "<Count>18</Count>"
-  FAIL_REGULAR_EXPRESSION "[0-9]+ tags"
-)
-add_test(NAME List::Tags::JsonOutput COMMAND $<TARGET_FILE:SelfTest> --list-tags -r json)
-set_tests_properties(List::Tags::JsonOutput PROPERTIES
-  PASS_REGULAR_EXPRESSION "\"count\": 18"
-  FAIL_REGULAR_EXPRESSION "[0-9]+ tags"
-)
-
-add_test(NAME List::Reporters::Output COMMAND $<TARGET_FILE:SelfTest> --list-reporters)
-set_tests_properties(List::Reporters::Output PROPERTIES PASS_REGULAR_EXPRESSION "Available reporters:")
-add_test(NAME List::Reporters::ExitCode COMMAND $<TARGET_FILE:SelfTest> --list-reporters)
-add_test(NAME List::Reporters::XmlOutput COMMAND $<TARGET_FILE:SelfTest> --list-reporters -r xml)
-set_tests_properties(List::Reporters::XmlOutput PROPERTIES
-  PASS_REGULAR_EXPRESSION "<Name>compact</Name>"
-  FAIL_REGULAR_EXPRESSION "Available reporters:"
-)
-add_test(NAME List::Reporters::JsonOutput COMMAND $<TARGET_FILE:SelfTest> --list-reporters -r json)
-set_tests_properties(List::Reporters::JsonOutput PROPERTIES
-  PASS_REGULAR_EXPRESSION "\"name\": \"compact\""
-  FAIL_REGULAR_EXPRESSION "Available reporters:"
-)
 
+add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> --no-history --warn=no-assertions "An empty test with no assertions")
+set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "no assertion ran")
 
-add_test(NAME List::Listeners::Output
-  COMMAND
-    $<TARGET_FILE:SelfTest> --list-listeners
-)
-set_tests_properties(List::Listeners::Output
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "Registered listeners:"
-)
-add_test(NAME List::Listeners::ExitCode
-  COMMAND
-    $<TARGET_FILE:SelfTest> --list-listeners
-)
-add_test(NAME List::Listeners::XmlOutput
-  COMMAND
-    $<TARGET_FILE:SelfTest>
-      --list-listeners
-      --reporter xml
-)
-set_tests_properties(List::Listeners::XmlOutput
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "<RegisteredListeners>"
-    FAIL_REGULAR_EXPRESSION "Registered listeners:"
-)
-add_test(NAME List::Listeners::JsonOutput
-  COMMAND
-    $<TARGET_FILE:SelfTest>
-      --list-listeners
-      --reporter json
-)
-set_tests_properties(List::Listeners::JsonOutput
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "\"listeners\": \\["
-    FAIL_REGULAR_EXPRESSION "Registered listeners:"
-)
-
+add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-1 COMMAND $<TARGET_FILE:SelfTest> --no-history Tracker, "___nonexistent_test___")
 
-add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> -w NoAssertions "An empty test with no assertions")
-set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case")
-
-# We cannot combine a regular expression on output with return code check
-# in one test, so we register two instead of making a checking script because
-# the runtime overhead is small enough.
-add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-1 COMMAND $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___")
-
-add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-2 COMMAND $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___")
+add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-2 COMMAND $<TARGET_FILE:SelfTest> --no-history Tracker, "___nonexistent_test___")
 set_tests_properties(TestSpecs::CombiningMatchingAndNonMatchingIsOk-2 PROPERTIES
-  PASS_REGULAR_EXPRESSION "No test cases matched '\"___nonexistent_test___\"'"
-  FAIL_REGULAR_EXPRESSION "No tests ran"
+  PASS_REGULAR_EXPRESSION "no test matches '\"___nonexistent_test___\"'"
+  FAIL_REGULAR_EXPRESSION "none ran"
 )
 
 add_test(NAME TestSpecs::NoMatchedTestsFail
-  COMMAND $<TARGET_FILE:SelfTest> "___nonexistent_test___"
+  COMMAND $<TARGET_FILE:SelfTest> --no-history "___nonexistent_test___"
 )
 set_tests_properties(TestSpecs::NoMatchedTestsFail
   PROPERTIES
     WILL_FAIL ON
 )
 add_test(NAME TestSpecs::OverrideFailureWithNoMatchedTests
-  COMMAND $<TARGET_FILE:SelfTest> "___nonexistent_test___" --allow-running-no-tests
+  COMMAND $<TARGET_FILE:SelfTest> --no-history "___nonexistent_test___" --allow-no-tests
 )
 
 add_test(NAME TestSpecs::OverrideAllSkipFailure
-  COMMAND $<TARGET_FILE:SelfTest> "tests can be skipped dynamically at runtime" --allow-running-no-tests
+  COMMAND $<TARGET_FILE:SelfTest> --no-history "tests can be skipped dynamically at runtime" --allow-no-tests
 )
 
 add_test(NAME TestSpecs::NonMatchingTestSpecIsRoundTrippable
-  COMMAND $<TARGET_FILE:SelfTest> Tracker, "this test does not exist" "[nor does this tag]"
+  COMMAND $<TARGET_FILE:SelfTest> --no-history Tracker, "this test does not exist" "[nor does this tag]"
 )
 set_tests_properties(TestSpecs::NonMatchingTestSpecIsRoundTrippable
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "No test cases matched '\"this test does not exist\" \\[nor does this tag\\]'"
+    PASS_REGULAR_EXPRESSION "no test matches '\"this test does not exist\" \\[nor does this tag\\]'"
 )
 
 add_test(NAME Warnings::UnmatchedTestSpecIsAccepted
-  COMMAND $<TARGET_FILE:SelfTest> Tracker --warn UnmatchedTestSpec
-)
-set_tests_properties(Warnings::UnmatchedTestSpecIsAccepted
-  PROPERTIES
-    FAIL_REGULAR_EXPRESSION "Unrecognised warning option: "
+  COMMAND $<TARGET_FILE:SelfTest> --no-history Tracker --warn=unmatched-specs
 )
 
 add_test(NAME Warnings::MultipleWarningsCanBeSpecified
   COMMAND
-    $<TARGET_FILE:SelfTest> Tracker
-      --warn NoAssertions
-      --warn UnmatchedTestSpec
+    $<TARGET_FILE:SelfTest> --no-history Tracker
+      --warn=no-assertions
+      --warn=unmatched-specs
 )
 
 add_test(NAME TestSpecs::WarnUnmatchedTestSpecFailsWithUnmatchedTestSpec
   COMMAND
-    $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___" --warn UnmatchedTestSpec
+    $<TARGET_FILE:SelfTest> --no-history Tracker, "___nonexistent_test___" --warn=unmatched-specs
 )
 set_tests_properties(TestSpecs::WarnUnmatchedTestSpecFailsWithUnmatchedTestSpec
   PROPERTIES
     WILL_FAIL ON
 )
 
-add_test(NAME UnmatchedOutputFilter COMMAND $<TARGET_FILE:SelfTest> [this-tag-does-not-exist])
+add_test(NAME UnmatchedOutputFilter COMMAND $<TARGET_FILE:SelfTest> --no-history [this-tag-does-not-exist])
 set_tests_properties(UnmatchedOutputFilter
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "No test cases matched '\\[this-tag-does-not-exist\\]'"
+    PASS_REGULAR_EXPRESSION "no test matches '\\[this-tag-does-not-exist\\]'"