Merge pull request #15 from Progmasoft/feature/catch3-runner

Replace Catch2's runner with the Catch3 runner

10 files changed, 171 insertions, 738 deletions

diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt
index 98f050d6..8e8102ee 100644
--- a/tests/ExtraTests/CMakeLists.txt
+++ b/tests/ExtraTests/CMakeLists.txt
@@ -14,74 +14,22 @@ set_tests_properties(TestShardingIntegration
 
 add_test(
   NAME TestSharding::OverlyLargeShardIndex
-  COMMAND $<TARGET_FILE:SelfTest>  --shard-index 5 --shard-count 5
+  COMMAND $<TARGET_FILE:SelfTest> --shard=5/5
 )
 set_tests_properties(TestSharding::OverlyLargeShardIndex
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "The shard count \\(5\\) must be greater than the shard index \\(5\\)"
+    PASS_REGULAR_EXPRESSION "--shard takes INDEX/COUNT with INDEX below COUNT"
 )
 
-set(CATCH_SHARDING_WARNING_REGEX "Warning: using sharding .* with random order")
-
-add_test(
-  NAME TestSharding::WarningOnRandomOrderWithoutFixedSeed
-  COMMAND $<TARGET_FILE:SelfTest> --shard-index 0 --shard-count 2 --list-tests
-)
-set_tests_properties(TestSharding::WarningOnRandomOrderWithoutFixedSeed
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "${CATCH_SHARDING_WARNING_REGEX}"
-)
-
-add_test(
-  NAME TestSharding::NoWarningOnRandomOrderWithFixedSeed
-  COMMAND $<TARGET_FILE:SelfTest> --shard-index 0 --shard-count 2 --rng-seed 12345 --list-tests
-)
-set_tests_properties(TestSharding::NoWarningOnRandomOrderWithFixedSeed
-  PROPERTIES
-    FAIL_REGULAR_EXPRESSION "${CATCH_SHARDING_WARNING_REGEX}"
-)
-
-foreach(shardOrder decl lex)
-  add_test(
-    NAME TestSharding::NoWarningOn${shardOrder}OrderWithoutFixedSeed
-    COMMAND $<TARGET_FILE:SelfTest> --shard-index 0 --shard-count 2 --order ${shardOrder} --list-tests
-  )
-  set_tests_properties(TestSharding::NoWarningOn${shardOrder}OrderWithoutFixedSeed
-    PROPERTIES
-      FAIL_REGULAR_EXPRESSION "${CATCH_SHARDING_WARNING_REGEX}"
-  )
-endforeach()
-
-# The MinDuration reporting tests do not need separate compilation, but
-# they have non-trivial execution time, so they are categorized as
-# extra tests, so that they are run less.
-add_test(NAME MinDuration::SimpleThreshold COMMAND $<TARGET_FILE:SelfTest> --min-duration 0.950 [min_duration_test])
-set_tests_properties(MinDuration::SimpleThreshold
+# The slowest tests are listed with their durations. The tests sleep, so
+# they run less often, as extra tests.
+add_test(NAME Durations::Slowest COMMAND $<TARGET_FILE:SelfTest> --no-history --slowest=1 [min_duration_test])
+set_tests_properties(Durations::Slowest
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "s: sleep_for_1000ms"
-    FAIL_REGULAR_EXPRESSION "sleep_for_100ms"
-    RUN_SERIAL ON  # The test is timing sensitive, so we want to run it
-    # serially to avoid false positives on oversubscribed machines
+    PASS_REGULAR_EXPRESSION "slowest tests:[\r\n]+ +[0-9.]+ s  sleep_for_1000ms"
+    FAIL_REGULAR_EXPRESSION "  sleep_for_100ms"
 )
 
-# -d yes overrides the threshold, so we should see the faster test even
-# with a ridiculous high min duration threshold
-add_test(NAME MinDuration::DurationOverrideYes COMMAND $<TARGET_FILE:SelfTest> --min-duration 1.0 -d yes [min_duration_test])
-set_tests_properties(MinDuration::DurationOverrideYes
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "s: sleep_for_100ms"
-)
-
-# -d no overrides the threshold, so we should never see any tests even
-# with ridiculously low min duration threshold
-add_test(NAME MinDuration::DurationOverrideNo COMMAND $<TARGET_FILE:SelfTest> --min-duration 0.0001 -d no [min_duration_test])
-set_tests_properties(MinDuration::DurationOverrideNo
-  PROPERTIES
-    FAIL_REGULAR_EXPRESSION "sleep_for_250ms"
-)
-
-# ------------ end of duration reporting tests
-
 # define folders used:
 set(TESTS_DIR ${CATCH_DIR}/tests/ExtraTests)
 
@@ -91,7 +39,7 @@ target_compile_definitions(PrefixedMacros PRIVATE CATCH_CONFIG_PREFIX_ALL CATCH_
 # macros, regression test for #3087.
 target_link_libraries(PrefixedMacros Catch2_buildall_interface)
 
-add_test(NAME CATCH_CONFIG_PREFIX_ALL COMMAND PrefixedMacros -s)
+add_test(NAME CATCH_CONFIG_PREFIX_ALL COMMAND PrefixedMacros --no-history --success)
 set_tests_properties(CATCH_CONFIG_PREFIX_ALL
   PROPERTIES
     PASS_REGULAR_EXPRESSION "CATCH_"
@@ -107,40 +55,44 @@ target_compile_definitions(DisabledMacros PRIVATE CATCH_CONFIG_DISABLE)
 # it against the main library
 target_link_libraries(DisabledMacros Catch2WithMain)
 
-add_test(NAME CATCH_CONFIG_DISABLE-1 COMMAND DisabledMacros -s)
+add_test(NAME CATCH_CONFIG_DISABLE-1 COMMAND DisabledMacros --no-history --success)
 set_tests_properties(CATCH_CONFIG_DISABLE-1
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "No tests ran"
+    PASS_REGULAR_EXPRESSION "none ran"
     FAIL_REGULAR_EXPRESSION "This should not happen"
 )
-add_test(NAME CATCH_CONFIG_DISABLE-2 COMMAND DisabledMacros --list-tests)
+add_test(NAME CATCH_CONFIG_DISABLE-2 COMMAND DisabledMacros --list)
 set_tests_properties(CATCH_CONFIG_DISABLE-2
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "0 test cases"
+    PASS_REGULAR_EXPRESSION "0 matching test\\(s\\)"
 )
 
 add_executable(DisabledExceptions-DefaultHandler ${TESTS_DIR}/X03-DisabledExceptions-DefaultHandler.cpp)
 add_executable(DisabledExceptions-CustomHandler ${TESTS_DIR}/X04-DisabledExceptions-CustomHandler.cpp)
 
+# clang-cl reports the Clang compiler ID but takes MSVC's options, so the
+# front end decides which spelling turns exceptions off.
+if(CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC" OR MSVC)
+  set(CATCH_NO_EXCEPTIONS_OPTIONS /EHs-c- /D_HAS_EXCEPTIONS=0)
+else()
+  set(CATCH_NO_EXCEPTIONS_OPTIONS -fno-exceptions)
+endif()
 foreach(target DisabledExceptions-DefaultHandler DisabledExceptions-CustomHandler)
-  target_compile_options(${target}
-    PUBLIC
-      $<$<CXX_COMPILER_ID:MSVC>:/EHs-c-;/D_HAS_EXCEPTIONS=0>
-      $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:GNU>,$<CXX_COMPILER_ID:QCC>,$<CXX_COMPILER_ID:AppleClang>>:-fno-exceptions>
-  )
+  target_compile_options(${target} PUBLIC ${CATCH_NO_EXCEPTIONS_OPTIONS})
   target_link_libraries(${target} Catch2_buildall_interface)
 endforeach()
 target_compile_definitions(DisabledExceptions-CustomHandler PUBLIC CATCH_CONFIG_DISABLE_EXCEPTIONS_CUSTOM_HANDLER)
 
 
-add_test(NAME CATCH_CONFIG_DISABLE_EXCEPTIONS-1 COMMAND DisabledExceptions-DefaultHandler "Tests that run")
+add_test(NAME CATCH_CONFIG_DISABLE_EXCEPTIONS-1 COMMAND DisabledExceptions-DefaultHandler --no-history --retries=off "Tests that run")
 set_tests_properties(CATCH_CONFIG_DISABLE_EXCEPTIONS-1
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "assertions: 4 \| 2 passed \| 2 failed"
+    PASS_REGULAR_EXPRESSION "4 assertion\\(s\\)"
     FAIL_REGULAR_EXPRESSION "abort;terminate;fatal"
 )
 
-# We must now test this works without the build flag.
+# Bazel passes its wishes in the environment: a JUnit file, a filter, a
+# shard, a seed.
 add_executable(BazelReporterNoCatchConfig ${TESTS_DIR}/X30-BazelReporter.cpp)
 target_link_libraries(BazelReporterNoCatchConfig Catch2WithMain)
 add_test(NAME NO_CATCH_CONFIG_BAZEL_REPORTER-1
@@ -160,7 +112,7 @@ add_test(NAME BazelEnv::TESTBRIDGE_TEST_ONLY
 set_tests_properties(BazelEnv::TESTBRIDGE_TEST_ONLY
   PROPERTIES
     ENVIRONMENT "BAZEL_TEST=1;TESTBRIDGE_TEST_ONLY=Passing test case"
-    PASS_REGULAR_EXPRESSION "All tests passed \\(1 assertion in 1 test case\\)"
+    PASS_REGULAR_EXPRESSION "1 test\\(s\\): 1 passed"
 )
 
 add_test(NAME BazelEnv::Sharding
@@ -178,21 +130,21 @@ set_tests_properties(BazelEnv::Sharding
 # which makes this test unsuitable for CI and headless runs, as it opens
 # up an interactive dialog.
 if(NOT WIN32)
-  add_test(NAME CATCH_CONFIG_DISABLE_EXCEPTIONS-2 COMMAND DisabledExceptions-DefaultHandler "Tests that abort")
+  add_test(NAME CATCH_CONFIG_DISABLE_EXCEPTIONS-2 COMMAND DisabledExceptions-DefaultHandler --no-history --retries=off "Tests that abort")
   set_tests_properties(CATCH_CONFIG_DISABLE_EXCEPTIONS-2
     PROPERTIES
       PASS_REGULAR_EXPRESSION "Catch will terminate"
   )
 endif()
 
-add_test(NAME CATCH_CONFIG_DISABLE_EXCEPTIONS-3 COMMAND DisabledExceptions-CustomHandler "Tests that run")
+add_test(NAME CATCH_CONFIG_DISABLE_EXCEPTIONS-3 COMMAND DisabledExceptions-CustomHandler --no-history --retries=off "Tests that run")
 set_tests_properties(CATCH_CONFIG_DISABLE_EXCEPTIONS-3
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "assertions: 4 \| 2 passed \| 2 failed"
+    PASS_REGULAR_EXPRESSION "4 assertion\\(s\\)"
     FAIL_REGULAR_EXPRESSION "====== CUSTOM HANDLER ======"
 )
 
-add_test(NAME CATCH_CONFIG_DISABLE_EXCEPTIONS-4 COMMAND DisabledExceptions-CustomHandler "Tests that abort")
+add_test(NAME CATCH_CONFIG_DISABLE_EXCEPTIONS-4 COMMAND DisabledExceptions-CustomHandler --no-history --retries=off "Tests that abort")
 set_tests_properties(CATCH_CONFIG_DISABLE_EXCEPTIONS-4
   PROPERTIES
     PASS_REGULAR_EXPRESSION "====== CUSTOM HANDLER ======"
@@ -202,10 +154,10 @@ add_executable(DeferredStaticChecks ${TESTS_DIR}/X05-DeferredStaticChecks.cpp)
 target_link_libraries(DeferredStaticChecks PRIVATE Catch2WithMain)
 target_compile_definitions(DeferredStaticChecks PRIVATE "CATCH_CONFIG_RUNTIME_STATIC_REQUIRE")
 
-add_test(NAME DeferredStaticChecks COMMAND DeferredStaticChecks -r compact)
+add_test(NAME DeferredStaticChecks COMMAND DeferredStaticChecks --no-history --retries=off)
 set_tests_properties(DeferredStaticChecks
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "test cases: 1 \\| 1 failed\nassertions: 3 \\| 3 failed;test cases: 1 \\| 1 failed\nassertions: 4 \\| 4 failed"
+    PASS_REGULAR_EXPRESSION "1 test\\(s\\): 1 failed[\r\n]+[34] assertion\\(s\\)"
 )
 
 add_executable(MixingClearedAndUnclearedMessages ${TESTS_DIR}/X06-MixingClearedAndUnclearedMessages.cpp)
@@ -214,26 +166,27 @@ target_link_libraries(MixingClearedAndUnclearedMessages PRIVATE Catch2WithMain)
 add_test(NAME MixingClearedAndUnclearedMessages
   COMMAND
     MixingClearedAndUnclearedMessages
-    -r compact)
+    --no-history --success)
 set_tests_properties(MixingClearedAndUnclearedMessages
   PROPERTIES
-    PASS_REGULAR_EXPRESSION ": false with 1 message: 'b'"
+    PASS_REGULAR_EXPRESSION "FAILED: REQUIRE\\( false \\)[\r\n]+ +with context: b[\r\n]"
+    FAIL_REGULAR_EXPRESSION "FAILED: REQUIRE\\( false \\)[\r\n]+ +with context: a"
 )
 
 add_executable(FallbackStringifier ${TESTS_DIR}/X10-FallbackStringifier.cpp)
 target_compile_definitions(FallbackStringifier PRIVATE CATCH_CONFIG_FALLBACK_STRINGIFIER=fallbackStringifier)
 target_link_libraries(FallbackStringifier Catch2WithMain)
 
-add_test(NAME FallbackStringifier COMMAND FallbackStringifier -r compact -s)
+add_test(NAME FallbackStringifier COMMAND FallbackStringifier --no-history --success)
 set_tests_properties(FallbackStringifier
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "foo{} for: { !!! }"
+    PASS_REGULAR_EXPRESSION "foo{} \\) for: { !!! }"
 )
 
 add_executable(DisableStringification ${TESTS_DIR}/X11-DisableStringification.cpp)
 target_compile_definitions(DisableStringification PRIVATE CATCH_CONFIG_DISABLE_STRINGIFICATION)
 target_link_libraries(DisableStringification Catch2WithMain)
-add_test(NAME CATCH_CONFIG_DISABLE_STRINGIFICATION COMMAND DisableStringification -r compact -s)
+add_test(NAME CATCH_CONFIG_DISABLE_STRINGIFICATION COMMAND DisableStringification --no-history --success)
 set_tests_properties(CATCH_CONFIG_DISABLE_STRINGIFICATION
   PROPERTIES
     PASS_REGULAR_EXPRESSION "Disabled by CATCH_CONFIG_DISABLE_STRINGIFICATION"
@@ -245,7 +198,7 @@ if(MSVC)
   # This test fails if it does not compile and succeeds otherwise
   add_executable(WindowsHeader ${TESTS_DIR}/X90-WindowsHeaderInclusion.cpp)
   target_link_libraries(WindowsHeader Catch2WithMain)
-  add_test(NAME WindowsHeader COMMAND WindowsHeader -r compact)
+  add_test(NAME WindowsHeader COMMAND WindowsHeader --no-history)
   list(APPEND CATCH_TEST_TARGETS ${EXTRA_TEST_BINARIES} WindowsHeader)
 endif()
 
@@ -260,144 +213,61 @@ set_tests_properties(PartialTestCaseEvents
     LABELS "uses-python"
 )
 
-add_executable(BenchmarksInCumulativeReporter ${TESTS_DIR}/X22-BenchmarksInCumulativeReporter.cpp)
-target_link_libraries(BenchmarksInCumulativeReporter PRIVATE Catch2::Catch2WithMain)
-add_test(
-  NAME BenchmarksInCumulativeReporter
-  COMMAND BenchmarksInCumulativeReporter --reporter testReporter
-)
-set_tests_properties(BenchmarksInCumulativeReporter
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "1\n2\n3\n4\n5\n"
-    COST 30
-)
-
-add_executable(CasingInReporterNames ${TESTS_DIR}/X23-CasingInReporterNames.cpp)
-target_link_libraries(CasingInReporterNames PRIVATE Catch2::Catch2WithMain)
-add_test(
-  NAME Reporters::registration-is-case-preserving
-  COMMAND CasingInReporterNames --list-reporters
-)
-set_tests_properties(Reporters::registration-is-case-preserving
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "testReporterCASED"
-)
-add_test(
-  NAME Reporters::selection-is-case-insensitive
-  COMMAND  CasingInReporterNames -r testReportercased
-)
-set_tests_properties(Reporters::selection-is-case-insensitive
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "TestReporter constructed"
-)
-
 add_executable(CapturedStdoutInTestCaseEvents ${TESTS_DIR}/X27-CapturedStdoutInTestCaseEvents.cpp)
 target_link_libraries(CapturedStdoutInTestCaseEvents PRIVATE Catch2::Catch2WithMain)
 add_test(
-  NAME Reporters::CapturedStdOutInEvents
-  COMMAND CapturedStdoutInTestCaseEvents
-    --reporter test-reporter
+  NAME Listeners::CapturedStdOutInEvents
+  COMMAND CapturedStdoutInTestCaseEvents --no-history
 )
-set_tests_properties(Reporters::CapturedStdOutInEvents
+set_tests_properties(Listeners::CapturedStdOutInEvents
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "X27 - TestReporter constructed"
-    FAIL_REGULAR_EXPRESSION "X27 ERROR"
+    PASS_REGULAR_EXPRESSION "X27 - listener saw 0 error\\(s\\)"
 )
 
-if(WIN32)
-  set(_NullFile "NUL")
-else()
-  set(_NullFile "/dev/null")
-endif()
-
 add_executable(ListenerStdoutCaptureInMultireporter ${TESTS_DIR}/X24-ListenerStdoutCaptureInMultireporter.cpp)
 target_link_libraries(ListenerStdoutCaptureInMultireporter PRIVATE Catch2::Catch2WithMain)
 
-# This test checks that there is nothing written out from the process,
-# but if CMake is running the tests under Valgrind or similar tool, then
-# that will write its own output to stdout and the test would fail.
-if(NOT MEMORYCHECK_COMMAND)
-  add_test(
-    NAME MultiReporter::NoncapturingListenerDoesntCauseStdoutPassThrough
-    COMMAND ListenerStdoutCaptureInMultireporter
-      --reporter xml::out=${_NullFile}
-      --reporter junit::out=${_NullFile}
-  )
-
-  set_tests_properties(MultiReporter::NoncapturingListenerDoesntCauseStdoutPassThrough
-    PROPERTIES
-      PASS_REGULAR_EXPRESSION "X24 - NonCapturingListener initialized"
-      FAIL_REGULAR_EXPRESSION "X24 - FooBarBaz"
-  )
-endif()
-
-add_executable(ListenerCanAskForCapturedStdout ${TESTS_DIR}/X25-ListenerCanAskForCapturedStdout.cpp)
-target_link_libraries(ListenerCanAskForCapturedStdout PRIVATE Catch2::Catch2WithMain)
+# The runner keeps what a passing test prints, whatever a listener prefers;
+# what the listener prints outside a test reaches the output.
 add_test(
-  NAME MultiReporter::CapturingListenerCausesStdoutCapture
-  COMMAND ListenerCanAskForCapturedStdout
-    --reporter compact::out=${_NullFile}
-    --reporter console::out=${_NullFile}
+  NAME Listeners::NoncapturingListenerDoesntCauseStdoutPassThrough
+  COMMAND ListenerStdoutCaptureInMultireporter --no-history
 )
-set_tests_properties(MultiReporter::CapturingListenerCausesStdoutCapture
+set_tests_properties(Listeners::NoncapturingListenerDoesntCauseStdoutPassThrough
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "CapturingListener initialized"
-    FAIL_REGULAR_EXPRESSION "X25 - ERROR"
+    PASS_REGULAR_EXPRESSION "X24 - NonCapturingListener initialized"
+    FAIL_REGULAR_EXPRESSION "X24 - FooBarBaz"
 )
 
-add_executable(ReporterPreferencesForPassingAssertionsIsRespected ${TESTS_DIR}/X26-ReporterPreferencesForPassingAssertionsIsRespected.cpp)
-target_link_libraries(ReporterPreferencesForPassingAssertionsIsRespected PRIVATE Catch2::Catch2WithMain)
-add_test(
-  NAME Reporters::PreferencesForPassingAssertionsIsRespected
-  COMMAND ReporterPreferencesForPassingAssertionsIsRespected
-    --reporter test-reporter
-)
-set_tests_properties(Reporters::PreferencesForPassingAssertionsIsRespected
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "X26 - TestReporter constructed"
-    FAIL_REGULAR_EXPRESSION "X26 - assertionEnded"
-)
+add_executable(ListenerCanAskForCapturedStdout ${TESTS_DIR}/X25-ListenerCanAskForCapturedStdout.cpp)
+target_link_libraries(ListenerCanAskForCapturedStdout PRIVATE Catch2::Catch2WithMain)
 add_test(
-  NAME MultiReporter::PreferencesForPassingAssertionsIsRespected
-  COMMAND ReporterPreferencesForPassingAssertionsIsRespected
-    --reporter test-reporter
-    --reporter console::out=${_NullFile}
+  NAME Listeners::CapturingListenerCausesStdoutCapture
+  COMMAND ListenerCanAskForCapturedStdout --no-history
 )
-set_tests_properties(MultiReporter::PreferencesForPassingAssertionsIsRespected
+set_tests_properties(Listeners::CapturingListenerCausesStdoutCapture
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "X26 - TestReporter constructed"
-    FAIL_REGULAR_EXPRESSION "X26 - assertionEnded"
+    PASS_REGULAR_EXPRESSION "CapturingListener initialized"
+    FAIL_REGULAR_EXPRESSION "X25 - ERROR"
 )
 
 add_executable(ListenersGetEventsBeforeReporters ${TESTS_DIR}/X28-ListenersGetEventsBeforeReporters.cpp)
 target_link_libraries(ListenersGetEventsBeforeReporters PRIVATE Catch2::Catch2WithMain)
 add_test(
-  NAME ListenersGetEventsBeforeReporters
-  COMMAND ListenersGetEventsBeforeReporters --reporter test-reporter
+  NAME ListenersGetRunEvents
+  COMMAND ListenersGetEventsBeforeReporters --no-history
 )
-set_tests_properties(ListenersGetEventsBeforeReporters
+set_tests_properties(ListenersGetRunEvents
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "X28 - TestReporter constructed"
+    PASS_REGULAR_EXPRESSION "X28 - listener received every event"
     FAIL_REGULAR_EXPRESSION "X28 - ERROR"
 )
 
-add_executable(CustomArgumentsForReporters ${TESTS_DIR}/X29-CustomArgumentsForReporters.cpp)
-target_link_libraries(CustomArgumentsForReporters PRIVATE Catch2::Catch2WithMain)
-add_test(
-  NAME CustomArgumentsForReporters
-  COMMAND CustomArgumentsForReporters
-    --reporter "test-reporter::Xa b=c 1::Xz:e = 1234"
-)
-set_tests_properties(CustomArgumentsForReporters
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "Xa b=c 1::Xz:e = 1234"
-)
-
 add_executable(DuplicatedTestCases-SameNameAndTags ${TESTS_DIR}/X31-DuplicatedTestCases.cpp)
 target_link_libraries(DuplicatedTestCases-SameNameAndTags PRIVATE Catch2::Catch2WithMain)
 add_test(
   NAME DuplicatedTestCases::SameNameAndTags
-  COMMAND $<TARGET_FILE:DuplicatedTestCases-SameNameAndTags>
+  COMMAND $<TARGET_FILE:DuplicatedTestCases-SameNameAndTags> --no-history
 )
 set_tests_properties(DuplicatedTestCases::SameNameAndTags
   PROPERTIES
@@ -408,7 +278,7 @@ add_executable(DuplicatedTestCases-SameNameDifferentTags ${TESTS_DIR}/X32-Duplic
 target_link_libraries(DuplicatedTestCases-SameNameDifferentTags PRIVATE Catch2::Catch2WithMain)
 add_test(
   NAME DuplicatedTestCases::SameNameDifferentTags
-  COMMAND $<TARGET_FILE:DuplicatedTestCases-SameNameDifferentTags>
+  COMMAND $<TARGET_FILE:DuplicatedTestCases-SameNameDifferentTags> --no-history
 )
 set_tests_properties(DuplicatedTestCases::SameNameDifferentTags
   PROPERTIES
@@ -419,7 +289,7 @@ add_executable(DuplicatedTestCases-DuplicatedTestCaseMethods ${TESTS_DIR}/X33-Du
 target_link_libraries(DuplicatedTestCases-DuplicatedTestCaseMethods PRIVATE Catch2::Catch2WithMain)
 add_test(
   NAME DuplicatedTestCases::DuplicatedTestCaseMethods
-  COMMAND $<TARGET_FILE:DuplicatedTestCases-DuplicatedTestCaseMethods>
+  COMMAND $<TARGET_FILE:DuplicatedTestCases-DuplicatedTestCaseMethods> --no-history
 )
 set_tests_properties(DuplicatedTestCases::DuplicatedTestCaseMethods
   PROPERTIES
@@ -430,33 +300,22 @@ add_executable(DuplicatedTestCases-DifferentFixtures ${TESTS_DIR}/X34-Duplicated
 target_link_libraries(DuplicatedTestCases-DifferentFixtures PRIVATE Catch2::Catch2WithMain)
 add_test(
   NAME DuplicatedTestCases::DuplicatedTestCaseMethodsDifferentFixtures
-  COMMAND $<TARGET_FILE:DuplicatedTestCases-DifferentFixtures>
+  COMMAND $<TARGET_FILE:DuplicatedTestCases-DifferentFixtures> --no-history
 )
 set_tests_properties(DuplicatedTestCases::DuplicatedTestCaseMethodsDifferentFixtures
   PROPERTIES
     FAIL_REGULAR_EXPRESSION "error: .* already defined\\."
 )
 
-add_executable(DuplicatedReporters ${TESTS_DIR}/X35-DuplicatedReporterNames.cpp)
-target_link_libraries(DuplicatedReporters PRIVATE Catch2::Catch2WithMain)
-add_test(
-  NAME Reporters::RegistrationErrorsAreCaught
-  COMMAND $<TARGET_FILE:DuplicatedReporters>
-)
-set_tests_properties(Reporters::RegistrationErrorsAreCaught
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "Errors occurred during startup!"
-)
-
 add_executable(ReportingCrashWithJunitReporter ${TESTS_DIR}/X36-ReportingCrashWithJunitReporter.cpp)
 target_link_libraries(ReportingCrashWithJunitReporter PRIVATE Catch2::Catch2WithMain)
 add_test(
-  NAME Reporters::CrashInJunitReporter
-  COMMAND ${CMAKE_COMMAND} -E env $<TARGET_FILE:ReportingCrashWithJunitReporter> --reporter JUnit
+  NAME Runner::CrashIsReported
+  COMMAND $<TARGET_FILE:ReportingCrashWithJunitReporter> --no-history --retries=off
 )
-set_tests_properties(Reporters::CrashInJunitReporter
+set_tests_properties(Runner::CrashIsReported
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "</testsuites>"
+    PASS_REGULAR_EXPRESSION "CRASH raises signal"
     LABELS "uses-signals"
 )
 
@@ -465,7 +324,7 @@ add_executable(ReportingInvalidTestCaseTags ${TESTS_DIR}/X37-ReportingInvalidTes
 target_link_libraries(ReportingInvalidTestCaseTags PRIVATE Catch2::Catch2WithMain)
 add_test(
   NAME Validation::TestCaseTags
-  COMMAND $<TARGET_FILE:ReportingInvalidTestCaseTags>
+  COMMAND $<TARGET_FILE:ReportingInvalidTestCaseTags> --no-history
 )
 # Passing multiple PASS_REGULAR_EXPRESSION does not work, so we join multiple ones with '.*'
 set(regexp
@@ -482,79 +341,27 @@ set_tests_properties(Validation::TestCaseTags
 )
 
 
-add_executable(ThrowingReporter ${TESTS_DIR}/X38-ThrowingReporter.cpp)
-target_link_libraries(ThrowingReporter PRIVATE Catch2::Catch2WithMain)
-add_test(
-  NAME Reporters::ThrowInConstructor
-  COMMAND
-    $<TARGET_FILE:ThrowingReporter>
-      --reporter ThrowingReporter::Xthrow-in-constructor=true
-)
-set_tests_properties(Reporters::ThrowInConstructor
-  PROPERTIES
-    PASS_REGULAR_EXPRESSION "Error when creating reporter: Exception in constructor"
-)
-
-foreach(reporterName   "Automake"
-                       "compact"
-                       "console"
-                       "JUnit"
-                       "SonarQube"
-                       "TAP"
-                       "TeamCity"
-                       "XML"
-                       "JSON")
-  add_test(NAME "Reporters::RejectUnknownArgs::${reporterName}"
-    COMMAND
-      $<TARGET_FILE:SelfTest>
-        --reporter ${reporterName}::Xfoo=bar
-        "[comparisons]" # Limit selection to avoid running full test binary if the error check doesn't trigger
-  )
-  set_tests_properties("Reporters::RejectUnknownArgs::${reporterName}"
-    PROPERTIES
-      PASS_REGULAR_EXPRESSION "Error when creating reporter: Unknown reporter argument: Xfoo"
-  )
-endforeach()
-
-
 add_executable(QuickExitInTest ${TESTS_DIR}/X40-QuickExit.cpp)
 target_link_libraries(QuickExitInTest PRIVATE Catch2::Catch2WithMain)
+# A test that ends its process early cannot pass: the exit of its worker is
+# reported as a crash of the test.
 add_test(
-  NAME BazelEnv::TEST_PREMATURE_EXIT_FILE
-  COMMAND
-    Python3::Interpreter
-    "${CATCH_DIR}/tests/TestScripts/testBazelExitGuardFile.py"
-    $<TARGET_FILE:QuickExitInTest>
-    "${CMAKE_CURRENT_BINARY_DIR}"
-    "bazel"
+  NAME Runner::ExitInTestIsACrash
+  COMMAND $<TARGET_FILE:QuickExitInTest> --no-history --retries=off
 )
-set_tests_properties(BazelEnv::TEST_PREMATURE_EXIT_FILE
+set_tests_properties(Runner::ExitInTestIsACrash
   PROPERTIES
-    LABELS "uses-python"
+    PASS_REGULAR_EXPRESSION "crashed with exit code 0"
 )
 add_test(
-  NAME PrematureExitGuardFileCanBeUsedFromCLI::CheckAfterCrash
-  COMMAND
-    Python3::Interpreter
-    "${CATCH_DIR}/tests/TestScripts/testBazelExitGuardFile.py"
-    $<TARGET_FILE:QuickExitInTest>
-    "${CMAKE_CURRENT_BINARY_DIR}"
-    "cli"
-)
-set_tests_properties(PrematureExitGuardFileCanBeUsedFromCLI::CheckAfterCrash
-  PROPERTIES
-    LABELS "uses-python"
-)
-add_test(
-  NAME PrematureExitGuardFileCanBeUsedFromCLI::CheckWithoutCrash
+  NAME BazelEnv::TEST_PREMATURE_EXIT_FILE
   COMMAND
     Python3::Interpreter
     "${CATCH_DIR}/tests/TestScripts/testBazelExitGuardFile.py"
     $<TARGET_FILE:QuickExitInTest>
     "${CMAKE_CURRENT_BINARY_DIR}"
-    "no-crash"
 )
-set_tests_properties(PrematureExitGuardFileCanBeUsedFromCLI::CheckWithoutCrash
+set_tests_properties(BazelEnv::TEST_PREMATURE_EXIT_FILE
   PROPERTIES
     LABELS "uses-python"
 )
@@ -568,7 +375,7 @@ target_link_libraries(AssertionStartingEventGoesBeforeAssertionIsEvaluated
 )
 add_test(
   NAME ReporterEvents::AssertionStartingHappensBeforeAssertionIsEvaluated
-  COMMAND $<TARGET_FILE:AssertionStartingEventGoesBeforeAssertionIsEvaluated>
+  COMMAND $<TARGET_FILE:AssertionStartingEventGoesBeforeAssertionIsEvaluated> --no-history
 )
 
 #add_executable(DebugBreakMacros ${TESTS_DIR}/X12-CustomDebugBreakMacro.cpp)
@@ -584,7 +391,7 @@ target_link_libraries(NoTests PRIVATE Catch2::Catch2WithMain)
 
 add_test(
   NAME TestSpecs::EmptySpecWithNoTestsFails
-  COMMAND $<TARGET_FILE:NoTests>
+  COMMAND $<TARGET_FILE:NoTests> --no-history
 )
 set_tests_properties(TestSpecs::EmptySpecWithNoTestsFails
   PROPERTIES
@@ -593,12 +400,7 @@ set_tests_properties(TestSpecs::EmptySpecWithNoTestsFails
 
 add_test(
   NAME TestSpecs::OverrideFailureWithEmptySpec
-  COMMAND $<TARGET_FILE:NoTests> --allow-running-no-tests
-)
-
-add_test(
-  NAME List::Listeners::WorksWithoutRegisteredListeners
-  COMMAND $<TARGET_FILE:NoTests> --list-listeners
+  COMMAND $<TARGET_FILE:NoTests> --no-history --allow-no-tests
 )
 
 add_executable(AllSkipped ${TESTS_DIR}/X93-AllSkipped.cpp)
@@ -606,7 +408,7 @@ target_link_libraries(AllSkipped PRIVATE Catch2::Catch2WithMain)
 
 add_test(
   NAME TestSpecs::SkippingAllTestsFails
-  COMMAND $<TARGET_FILE:AllSkipped>
+  COMMAND $<TARGET_FILE:AllSkipped> --no-history
 )
 set_tests_properties(TestSpecs::SkippingAllTestsFails
   PROPERTIES
@@ -619,11 +421,11 @@ target_compile_definitions(FastCompileMacros PRIVATE CATCH_CONFIG_FAST_COMPILE)
 
 add_test(
   NAME CompileConfiguration::FastCompile
-  COMMAND $<TARGET_FILE:FastCompileMacros>
+  COMMAND $<TARGET_FILE:FastCompileMacros> --no-history --retries=off
 )
 set_tests_properties(CompileConfiguration::FastCompile
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "test cases:  6 \\| 1 passed \\| 1 failed \\| 4 failed as expected\nassertions: 13 \\| 6 passed \\| 2 failed \\| 5 failed as expected"
+    PASS_REGULAR_EXPRESSION "6 test\\(s\\): 1 passed, 1 failed, 4 failed as expected[\r\n]+13 assertion\\(s\\)"
 )
 
 # This sets up a one-off executable that compiles against the amalgamated
@@ -635,12 +437,13 @@ add_executable(AmalgamatedTestCompilation
   ${CATCH_DIR}/extras/catch_amalgamated.cpp
 )
 target_include_directories(AmalgamatedTestCompilation PRIVATE ${CATCH_DIR}/extras)
-target_compile_features(AmalgamatedTestCompilation PRIVATE cxx_std_14)
+# The amalgamated files hold the runner, which needs C++20.
+target_compile_features(AmalgamatedTestCompilation PRIVATE cxx_std_20)
 
-add_test(NAME AmalgamatedFileTest COMMAND AmalgamatedTestCompilation)
+add_test(NAME AmalgamatedFileTest COMMAND AmalgamatedTestCompilation --no-history)
 set_tests_properties(AmalgamatedFileTest
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "All tests passed \\(14 assertions in 3 test cases\\)"
+    PASS_REGULAR_EXPRESSION "3 test\\(s\\): 3 passed[\r\n]+14 assertion\\(s\\)"
 )
 
 add_executable(ThreadSafetyTests
@@ -650,19 +453,19 @@ target_link_libraries(ThreadSafetyTests Catch2_buildall_interface)
 target_compile_definitions(ThreadSafetyTests PUBLIC CATCH_CONFIG_THREAD_SAFE_ASSERTIONS)
 
 add_test(NAME ThreadSafetyTests::ScopedMessagesAndAssertions
-  COMMAND ThreadSafetyTests -r compact "Failed REQUIRE in the main thread is fine"
+  COMMAND ThreadSafetyTests --no-history "Failed REQUIRE in the main thread is fine"
 )
 set_tests_properties(ThreadSafetyTests::ScopedMessagesAndAssertions
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "assertions: 801 \\| 400 passed \\| 401 failed as expected"
+    PASS_REGULAR_EXPRESSION "1 failed as expected[\r\n]+801 assertion\\(s\\)"
     RUN_SERIAL ON
 )
 add_test(NAME ThreadSafetyTests::UnscopedMessagesAndAssertions
-  COMMAND ThreadSafetyTests -r compact "Using unscoped messages in sibling threads"
+  COMMAND ThreadSafetyTests --no-history "Using unscoped messages in sibling threads"
 )
 set_tests_properties(ThreadSafetyTests::UnscopedMessagesAndAssertions
   PROPERTIES
-    PASS_REGULAR_EXPRESSION "assertions: 401 \\| 401 failed as expected"
+    PASS_REGULAR_EXPRESSION "1 failed as expected[\r\n]+401 assertion\\(s\\)"
     RUN_SERIAL ON
 )
 
@@ -671,52 +474,54 @@ target_link_libraries(InfiniteGenerators PRIVATE Catch2::Catch2WithMain)
 
 add_test(
   NAME Warnings::InfiniteGenerators::NoFilterWarns
-  COMMAND $<TARGET_FILE:InfiniteGenerators> --warn InfiniteGenerators
+  COMMAND $<TARGET_FILE:InfiniteGenerators> --no-history --retries=off --warn=infinite-generators
 )
 set_tests_properties(Warnings::InfiniteGenerators::NoFilterWarns
   PROPERTIES
     # One test case fails with infinite generator, but the other one runs
-    PASS_REGULAR_EXPRESSION "test cases: 2 \\| 1 passed \\| 1 failed"
-    TIMEOUT 5
+    PASS_REGULAR_EXPRESSION "2 test\\(s\\): 1 passed, 1 failed"
+    TIMEOUT 10
 )
 
 add_test(
   NAME Warnings::InfiniteGenerators::MatchAllFilterWarns
   COMMAND $<TARGET_FILE:InfiniteGenerators>
-    --warn InfiniteGenerators
-    --path-filter g:*
+    --no-history --retries=off
+    --warn=infinite-generators
+    --path-filter=g:*
 )
 set_tests_properties(Warnings::InfiniteGenerators::MatchAllFilterWarns
   PROPERTIES
     # One test case fails with infinite generator, but the other one runs
-    PASS_REGULAR_EXPRESSION "test cases: 2 \\| 1 passed \\| 1 failed"
-    TIMEOUT 5
+    PASS_REGULAR_EXPRESSION "2 test\\(s\\): 1 passed, 1 failed"
+    TIMEOUT 10
 )
 
 add_test(
   NAME Warnings::InfiniteGenerators::MatchOneFilterDoesntWarn
   COMMAND $<TARGET_FILE:InfiniteGenerators>
-    --warn InfiniteGenerators
-    --path-filter g:1
+    --no-history --retries=off
+    --warn=infinite-generators
+    --path-filter=g:1
 )
 set_tests_properties(Warnings::InfiniteGenerators::MatchOneFilterDoesntWarn
   PROPERTIES
-    # One test case fails with infinite generator, but the other one runs
-    PASS_REGULAR_EXPRESSION "All tests passed \\(1 assertion in 2 test cases\\)"
-    TIMEOUT 5
+    PASS_REGULAR_EXPRESSION "2 test\\(s\\): 2 passed"
+    TIMEOUT 10
 )
 
 add_test(
   NAME Warnings::InfiniteGenerators::SectionFilterWarns
   COMMAND $<TARGET_FILE:InfiniteGenerators>
-    --warn InfiniteGenerators
-    --section FooBarBaz
+    --no-history --retries=off
+    --warn=infinite-generators
+    --section=FooBarBaz
 )
 set_tests_properties(Warnings::InfiniteGenerators::SectionFilterWarns
   PROPERTIES
     # One test case fails with infinite generator, but the other one runs
-    PASS_REGULAR_EXPRESSION "test cases: 2 \\| 1 passed \\| 1 failed"
-    TIMEOUT 5
+    PASS_REGULAR_EXPRESSION "2 test\\(s\\): 1 passed, 1 failed"
+    TIMEOUT 10
 )
 
 
diff --git a/tests/ExtraTests/X21-PartialTestCaseEvents.cpp b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp
index aa5204c9..99002822 100644
--- a/tests/ExtraTests/X21-PartialTestCaseEvents.cpp
+++ b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp
@@ -7,59 +7,61 @@
 // SPDX-License-Identifier: BSL-1.0
 
 /**\file
- * Registers custom reporter that reports testCase* events
+ * Registers a listener that records the testCase* events and prints them
+ * when the test run ends, outside any test, where the runner passes the
+ * output on.
  *
  * The resulting executable can then be used by an external Python script
  * to verify that testCase{Starting,Ended} and testCasePartial{Starting,Ended}
  * events are properly nested.
  */
 
-
 #include <catch2/catch_test_macros.hpp>
-#include <catch2/reporters/catch_reporter_streaming_base.hpp>
 #include <catch2/catch_test_case_info.hpp>
+#include <catch2/reporters/catch_reporter_event_listener.hpp>
 #include <catch2/reporters/catch_reporter_registrars.hpp>
 #include <catch2/generators/catch_generators.hpp>
 
-
 #include <iostream>
+#include <string>
 
 using Catch::TestCaseInfo;
 using Catch::TestCaseStats;
 
-class PartialReporter : public Catch::StreamingReporterBase {
+namespace {
+class PartialListener : public Catch::EventListenerBase {
 public:
-    using StreamingReporterBase::StreamingReporterBase;
-
-    ~PartialReporter() override; // = default
-
-    static std::string getDescription() {
-        return "Special reporter for testing TestCasePartialStarting/Ended events";
-    }
+    using EventListenerBase::EventListenerBase;
 
     //! Called _once_ for each TEST_CASE, no matter how many times it is entered
     void testCaseStarting(TestCaseInfo const& testInfo) override {
-        s