diff options
| author | Leitwolf11 | 2026-10-23 23:25:41 +0300 |
|---|---|---|
| committer | Leitwolf11 | 2026-10-23 23:25:41 +0300 |
| commit | 93d4e7dbebab867e0a7f59fc25308460a7a76280 (patch) | |
| tree | 54c568a3063d49e21f039b6d38f40f51be7991f0 /tests | |
| parent | 83c6fc3ac13abe83fa39f7b5f8559ccaf4412233 (diff) | |
| download | catch3-93d4e7dbebab867e0a7f59fc25308460a7a76280.tar.gz | |
Replace Catch2's runner with the Catch3 runner
Catch3 keeps Catch2's API for writing tests (test cases, sections,
assertions, matchers, generators, listeners, benchmarks) and replaces its
session, command line and reporters with a runner of its own.
The runner starts the test binary again as worker processes that take the
next test when they are free. A crash, an exit or a hang ends one test, not
the run. On top of that it adds:
- retries that tell deterministic, flaky and order-dependent failures
apart, and a bisection that names the test leaving state behind;
- a history of verdicts and durations, kept under a file lock, that drives
automatic quarantine, adaptive time limits, predictive ordering,
incremental runs, failure replay and analytics;
- selection by change, past failure or prediction, dependency closure,
and sharding by a stable hash or by recorded durations;
- tags for serial and isolated tests, shared resources, dependencies,
time limits and retries;
- a failure category, a signature, an analysis and a replay command for
every failure;
- console, JUnit XML (with flakyFailure and rerunFailure), JSON and Chrome
trace reports, and a versioned discovery listing (--list=json);
- a temporary directory per test (TMPDIR, TMP, TEMP and
TestTemporaryDirectory());
- Bazel's test environment: XML output file, test filter, sharding, seed,
premature exit file and undeclared outputs.
The command line is parsed by a hand-written, typed parser. Catch::Session
remains as a thin facade over the runner for custom main functions.
The result model in Progmasoft/Catch3/Results.hpp was rewritten around
tests and attempts, and the JUnit writer now writes it. The property
engine's example database is updated under a file lock with an atomic
rename, so parallel workers can write it at the same time.
Removed: Clara, the command-line and reporter-spec parsers, every reporter
and CATCH_REGISTER_REPORTER, listing, text flow, console colour and width,
the JSON and XML writers, sharding in the test registry, approval tests and
their baselines, Conan packaging, and the unused third_party/clara.hpp.
Listeners keep working; reporters' list and skip events are gone.
The Catch2 library is now compiled as C++20; code that includes only
Catch2's headers still needs C++14. Duplicate test cases are still refused
at start-up. Version 0.3.0.
ExtraTests now turn exceptions off for clang-cl with MSVC-style options,
which -fno-exceptions did not do.
Diffstat (limited to 'tests')
60 files changed, 1803 insertions, 122909 deletions
diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index 1e192a1c..00c39a7d 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -22,33 +22,23 @@ cc_test( srcs = [ "SelfTest/IntrospectiveTests/Algorithms.tests.cpp", "SelfTest/IntrospectiveTests/AssertionHandler.tests.cpp", - "SelfTest/IntrospectiveTests/Clara.tests.cpp", - "SelfTest/IntrospectiveTests/CmdLine.tests.cpp", - "SelfTest/IntrospectiveTests/CmdLineHelpers.tests.cpp", - "SelfTest/IntrospectiveTests/ColourImpl.tests.cpp", "SelfTest/IntrospectiveTests/Details.tests.cpp", "SelfTest/IntrospectiveTests/FloatingPoint.tests.cpp", "SelfTest/IntrospectiveTests/GeneratorsImpl.tests.cpp", "SelfTest/IntrospectiveTests/Integer.tests.cpp", "SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp", - "SelfTest/IntrospectiveTests/Json.tests.cpp", "SelfTest/IntrospectiveTests/Parse.tests.cpp", "SelfTest/IntrospectiveTests/PartTracker.tests.cpp", "SelfTest/IntrospectiveTests/RandomNumberGeneration.tests.cpp", - "SelfTest/IntrospectiveTests/Reporters.tests.cpp", - "SelfTest/IntrospectiveTests/Sharding.tests.cpp", - "SelfTest/IntrospectiveTests/Stream.tests.cpp", "SelfTest/IntrospectiveTests/String.tests.cpp", "SelfTest/IntrospectiveTests/StringManip.tests.cpp", "SelfTest/IntrospectiveTests/Tag.tests.cpp", "SelfTest/IntrospectiveTests/TestCaseInfoHasher.tests.cpp", "SelfTest/IntrospectiveTests/TestSpec.tests.cpp", "SelfTest/IntrospectiveTests/TestSpecParser.tests.cpp", - "SelfTest/IntrospectiveTests/TextFlow.tests.cpp", "SelfTest/IntrospectiveTests/ToString.tests.cpp", "SelfTest/IntrospectiveTests/Traits.tests.cpp", "SelfTest/IntrospectiveTests/UniquePtr.tests.cpp", - "SelfTest/IntrospectiveTests/Xml.tests.cpp", "SelfTest/TestRegistrations.cpp", "SelfTest/TimingTests/Sleep.tests.cpp", "SelfTest/UsageTests/Approx.tests.cpp", diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3d4db3b3..14f2ed56 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -32,7 +32,10 @@ if(CATCH_BUILD_SURROGATES) foreach(Source ${targetSources}) string(REGEX MATCH "^.*\\.hpp$" isHeader ${Source}) string(REGEX MATCH "_all.hpp" isAllHeader ${Source}) - if(isHeader AND NOT isAllHeader) + # The runner's headers need C++20; the surrogates check Catch2's + # headers at the language level of the build. + string(REGEX MATCH "/Progmasoft/" isRunnerHeader ${Source}) + if(isHeader AND NOT isAllHeader AND NOT isRunnerHeader) list(APPEND AllHeaders ${Source}) endif() endforeach() @@ -71,30 +74,20 @@ set(TEST_SOURCES ${SELF_TEST_DIR}/TestRegistrations.cpp ${SELF_TEST_DIR}/IntrospectiveTests/Algorithms.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/AssertionHandler.tests.cpp - ${SELF_TEST_DIR}/IntrospectiveTests/Clara.tests.cpp - ${SELF_TEST_DIR}/IntrospectiveTests/CmdLine.tests.cpp - ${SELF_TEST_DIR}/IntrospectiveTests/CmdLineHelpers.tests.cpp - ${SELF_TEST_DIR}/IntrospectiveTests/ColourImpl.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/Details.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/FloatingPoint.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/GeneratorsImpl.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/Integer.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/InternalBenchmark.tests.cpp - ${SELF_TEST_DIR}/IntrospectiveTests/Json.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/Parse.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/PartTracker.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/RandomNumberGeneration.tests.cpp - ${SELF_TEST_DIR}/IntrospectiveTests/Reporters.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/Tag.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/TestCaseInfoHasher.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/TestSpec.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/TestSpecParser.tests.cpp - ${SELF_TEST_DIR}/IntrospectiveTests/TextFlow.tests.cpp - ${SELF_TEST_DIR}/IntrospectiveTests/Sharding.tests.cpp - ${SELF_TEST_DIR}/IntrospectiveTests/Stream.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/String.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/StringManip.tests.cpp - ${SELF_TEST_DIR}/IntrospectiveTests/Xml.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/Traits.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/ToString.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/UniquePtr.tests.cpp @@ -162,13 +155,25 @@ add_executable(ProgmasoftSelfTest Progmasoft/Property.tests.cpp Progmasoft/ForAll.tests.cpp Progmasoft/Results.tests.cpp + Progmasoft/Runner.tests.cpp + Progmasoft/RunnerUnits.tests.cpp Progmasoft/Snapshot.tests.cpp Progmasoft/Stateful.tests.cpp Progmasoft/Strategy.tests.cpp ) target_link_libraries(ProgmasoftSelfTest PRIVATE Progmasoft::Catch3 Catch2WithMain) target_compile_features(ProgmasoftSelfTest PRIVATE cxx_std_20) -add_test(NAME Progmasoft::PropertyTests COMMAND $<TARGET_FILE:ProgmasoftSelfTest>) + +# Tests that misbehave on purpose, for the runner's integration tests. +add_executable(RunnerFixture Progmasoft/RunnerFixture.cpp) +target_link_libraries(RunnerFixture PRIVATE Catch2WithMain) +target_compile_features(RunnerFixture PRIVATE cxx_std_20) +add_dependencies(ProgmasoftSelfTest RunnerFixture) + +add_test(NAME Progmasoft::PropertyTests COMMAND $<TARGET_FILE:ProgmasoftSelfTest> --no-history) +set_tests_properties(Progmasoft::PropertyTests PROPERTIES + ENVIRONMENT "CATCH3_RUNNER_FIXTURE=$<TARGET_FILE:RunnerFixture>" +) if(BUILD_SHARED_LIBS AND WIN32) add_custom_command(TARGET SelfTest PRE_LINK COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Catch2> @@ -185,203 +190,108 @@ if(CATCH_ENABLE_COVERAGE) endif() # configure unit tests via CTest -add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest> --order rand --rng-seed time --warn InfiniteGenerators) -set_tests_properties(RunTests PROPERTIES - FAIL_REGULAR_EXPRESSION "Filters:" - COST 15 -) +add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest> --order=random --warn=infinite-generators --no-history) +set_tests_properties(RunTests PROPERTIES COST 15) # Because CTest does not allow us to check both return code _and_ expected # output in one test, we run these commands twice. First time we check # the output, the second time we check the exit code. -add_test(NAME List::Tests::Output COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high) +add_test(NAME List::Tests::Output COMMAND $<TARGET_FILE:SelfTest> --list) set_tests_properties(List::Tests::Output PROPERTIES - PASS_REGULAR_EXPRESSION "[0-9]+ test cases" + PASS_REGULAR_EXPRESSION "[0-9]+ matching test\\(s\\)" FAIL_REGULAR_EXPRESSION "Hidden Test" ) +add_test(NAME List::Tests::ExitCode COMMAND $<TARGET_FILE:SelfTest> --list) -add_test(NAME VerbosityIsPerReporter COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high --reporter console::verbosity=quiet) -set_tests_properties(VerbosityIsPerReporter - PROPERTIES - FAIL_REGULAR_EXPRESSION "\.cpp" -) - -# This should be equivalent to the old --list-test-names-only and be usable -# with --input-file. -add_test(NAME List::Tests::Quiet COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity quiet) -# Sadly we cannot ask for start-of-line and end-of-line in a ctest regex, -# so we fail if we see space/tab at the start... -set_tests_properties(List::Tests::Quiet PROPERTIES - PASS_REGULAR_EXPRESSION "\"#1905 -- test spec parser properly clears internal state between compound tests\"[\r\n]" - FAIL_REGULAR_EXPRESSION "[ \t]\"#1905 -- test spec parser properly clears internal state between compound tests\"" -) - -add_test(NAME List::Tests::ExitCode COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high) -add_test(NAME List::Tests::XmlOutput COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high -r xml) -set_tests_properties(List::Tests::XmlOutput PROPERTIES - PASS_REGULAR_EXPRESSION "<Line>[0-9]+</Line>" - FAIL_REGULAR_EXPRESSION "[0-9]+ test cases" +add_test(NAME List::Tests::Names COMMAND $<TARGET_FILE:SelfTest> --list=names) +# CTest regexes cannot anchor at the start of a line, so a name indented by +# space or tab fails the test. +set_tests_properties(List::Tests::Names PROPERTIES + PASS_REGULAR_EXPRESSION "#1905 -- test spec parser properly clears internal state between compound tests[\r\n]" + FAIL_REGULAR_EXPRESSION "[ \t]#1905 -- test spec parser properly clears internal state between compound tests" ) -add_test(NAME List::Tests::Json::Normal COMMAND $<TARGET_FILE:SelfTest> --list-tests -r json) -set_tests_properties(List::Tests::Json::Normal PROPERTIES - PASS_REGULAR_EXPRESSION "\"tags\": \\[" - FAIL_REGULAR_EXPRESSION "\"source-location\": {" -) -add_test(NAME List::Tests::Json::Quiet COMMAND $<TARGET_FILE:SelfTest> --list-tests -r json::verbosity=quiet) -set_tests_properties(List::Tests::Json::Quiet PROPERTIES - PASS_REGULAR_EXPRESSION "\"name\": \"" - FAIL_REGULAR_EXPRESSION "\"tags\": \\[" -) -add_test(NAME List::Tests::Json::High COMMAND $<TARGET_FILE:SelfTest> --list-tests -r json::verbosity=high) -set_tests_properties(List::Tests::Json::High PROPERTIES - PASS_REGULAR_EXPRESSION "\"source-location\": {" +add_test(NAME List::Tests::Json COMMAND $<TARGET_FILE:SelfTest> --list=json) +set_tests_properties(List::Tests::Json PROPERTIES + PASS_REGULAR_EXPRESSION "\"format\": \"catch3-tests\"" ) add_test(NAME List::Tags::Output COMMAND $<TARGET_FILE:SelfTest> --list-tags) set_tests_properties(List::Tags::Output PROPERTIES - PASS_REGULAR_EXPRESSION "[0-9]+ tags" + PASS_REGULAR_EXPRESSION "[0-9]+ tag\\(s\\)" FAIL_REGULAR_EXPRESSION "\\[\\.\\]") add_test(NAME List::Tags::ExitCode COMMAND $<TARGET_FILE:SelfTest> --list-tags) -add_test(NAME List::Tags::XmlOutput COMMAND $<TARGET_FILE:SelfTest> --list-tags -r xml) -set_tests_properties(List::Tags::XmlOutput PROPERTIES - PASS_REGULAR_EXPRESSION "<Count>18</Count>" - FAIL_REGULAR_EXPRESSION "[0-9]+ tags" -) -add_test(NAME List::Tags::JsonOutput COMMAND $<TARGET_FILE:SelfTest> --list-tags -r json) -set_tests_properties(List::Tags::JsonOutput PROPERTIES - PASS_REGULAR_EXPRESSION "\"count\": 18" - FAIL_REGULAR_EXPRESSION "[0-9]+ tags" -) - -add_test(NAME List::Reporters::Output COMMAND $<TARGET_FILE:SelfTest> --list-reporters) -set_tests_properties(List::Reporters::Output PROPERTIES PASS_REGULAR_EXPRESSION "Available reporters:") -add_test(NAME List::Reporters::ExitCode COMMAND $<TARGET_FILE:SelfTest> --list-reporters) -add_test(NAME List::Reporters::XmlOutput COMMAND $<TARGET_FILE:SelfTest> --list-reporters -r xml) -set_tests_properties(List::Reporters::XmlOutput PROPERTIES - PASS_REGULAR_EXPRESSION "<Name>compact</Name>" - FAIL_REGULAR_EXPRESSION "Available reporters:" -) -add_test(NAME List::Reporters::JsonOutput COMMAND $<TARGET_FILE:SelfTest> --list-reporters -r json) -set_tests_properties(List::Reporters::JsonOutput PROPERTIES - PASS_REGULAR_EXPRESSION "\"name\": \"compact\"" - FAIL_REGULAR_EXPRESSION "Available reporters:" -) +add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> --no-history --warn=no-assertions "An empty test with no assertions") +set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "no assertion ran") -add_test(NAME List::Listeners::Output - COMMAND - $<TARGET_FILE:SelfTest> --list-listeners -) -set_tests_properties(List::Listeners::Output - PROPERTIES - PASS_REGULAR_EXPRESSION "Registered listeners:" -) -add_test(NAME List::Listeners::ExitCode - COMMAND - $<TARGET_FILE:SelfTest> --list-listeners -) -add_test(NAME List::Listeners::XmlOutput - COMMAND - $<TARGET_FILE:SelfTest> - --list-listeners - --reporter xml -) -set_tests_properties(List::Listeners::XmlOutput - PROPERTIES - PASS_REGULAR_EXPRESSION "<RegisteredListeners>" - FAIL_REGULAR_EXPRESSION "Registered listeners:" -) -add_test(NAME List::Listeners::JsonOutput - COMMAND - $<TARGET_FILE:SelfTest> - --list-listeners - --reporter json -) -set_tests_properties(List::Listeners::JsonOutput - PROPERTIES - PASS_REGULAR_EXPRESSION "\"listeners\": \\[" - FAIL_REGULAR_EXPRESSION "Registered listeners:" -) - +add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-1 COMMAND $<TARGET_FILE:SelfTest> --no-history Tracker, "___nonexistent_test___") -add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> -w NoAssertions "An empty test with no assertions") -set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case") - -# We cannot combine a regular expression on output with return code check -# in one test, so we register two instead of making a checking script because -# the runtime overhead is small enough. -add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-1 COMMAND $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___") - -add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-2 COMMAND $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___") +add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-2 COMMAND $<TARGET_FILE:SelfTest> --no-history Tracker, "___nonexistent_test___") set_tests_properties(TestSpecs::CombiningMatchingAndNonMatchingIsOk-2 PROPERTIES - PASS_REGULAR_EXPRESSION "No test cases matched '\"___nonexistent_test___\"'" - FAIL_REGULAR_EXPRESSION "No tests ran" + PASS_REGULAR_EXPRESSION "no test matches '\"___nonexistent_test___\"'" + FAIL_REGULAR_EXPRESSION "none ran" ) add_test(NAME TestSpecs::NoMatchedTestsFail - COMMAND $<TARGET_FILE:SelfTest> "___nonexistent_test___" + COMMAND $<TARGET_FILE:SelfTest> --no-history "___nonexistent_test___" ) set_tests_properties(TestSpecs::NoMatchedTestsFail PROPERTIES WILL_FAIL ON ) add_test(NAME TestSpecs::OverrideFailureWithNoMatchedTests - COMMAND $<TARGET_FILE:SelfTest> "___nonexistent_test___" --allow-running-no-tests + COMMAND $<TARGET_FILE:SelfTest> --no-history "___nonexistent_test___" --allow-no-tests ) add_test(NAME TestSpecs::OverrideAllSkipFailure - COMMAND $<TARGET_FILE:SelfTest> "tests can be skipped dynamically at runtime" --allow-running-no-tests + COMMAND $<TARGET_FILE:SelfTest> --no-history "tests can be skipped dynamically at runtime" --allow-no-tests ) add_test(NAME TestSpecs::NonMatchingTestSpecIsRoundTrippable - COMMAND $<TARGET_FILE:SelfTest> Tracker, "this test does not exist" "[nor does this tag]" + COMMAND $<TARGET_FILE:SelfTest> --no-history Tracker, "this test does not exist" "[nor does this tag]" ) set_tests_properties(TestSpecs::NonMatchingTestSpecIsRoundTrippable PROPERTIES - PASS_REGULAR_EXPRESSION "No test cases matched '\"this test does not exist\" \\[nor does this tag\\]'" + PASS_REGULAR_EXPRESSION "no test matches '\"this test does not exist\" \\[nor does this tag\\]'" ) add_test(NAME Warnings::UnmatchedTestSpecIsAccepted - COMMAND $<TARGET_FILE:SelfTest> Tracker --warn UnmatchedTestSpec -) -set_tests_properties(Warnings::UnmatchedTestSpecIsAccepted - PROPERTIES - FAIL_REGULAR_EXPRESSION "Unrecognised warning option: " + COMMAND $<TARGET_FILE:SelfTest> --no-history Tracker --warn=unmatched-specs ) add_test(NAME Warnings::MultipleWarningsCanBeSpecified COMMAND - $<TARGET_FILE:SelfTest> Tracker - --warn NoAssertions - --warn UnmatchedTestSpec + $<TARGET_FILE:SelfTest> --no-history Tracker + --warn=no-assertions + --warn=unmatched-specs ) add_test(NAME TestSpecs::WarnUnmatchedTestSpecFailsWithUnmatchedTestSpec COMMAND - $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___" --warn UnmatchedTestSpec + $<TARGET_FILE:SelfTest> --no-history Tracker, "___nonexistent_test___" --warn=unmatched-specs ) set_tests_properties(TestSpecs::WarnUnmatchedTestSpecFailsWithUnmatchedTestSpec PROPERTIES WILL_FAIL ON ) -add_test(NAME UnmatchedOutputFilter COMMAND $<TARGET_FILE:SelfTest> [this-tag-does-not-exist]) +add_test(NAME UnmatchedOutputFilter COMMAND $<TARGET_FILE:SelfTest> --no-history [this-tag-does-not-exist]) set_tests_properties(UnmatchedOutputFilter PROPERTIES - PASS_REGULAR_EXPRESSION "No test cases matched '\\[this-tag-does-not-exist\\]'" + PASS_REGULAR_EXPRESSION "no test matches '\\[this-tag-does-not-exist\\]'" ) -add_test(NAME FilteredSections::SimpleExample::1 COMMAND $<TARGET_FILE:SelfTest> \#1394 -c RunSection) -set_tests_properties(FilteredSections::SimpleExample::1 PROPERTIES FAIL_REGULAR_EXPRESSION "No tests ran") -add_test(NAME FilteredSections::SimpleExample::2 COMMAND $<TARGET_FILE:SelfTest> \#1394\ nested -c NestedRunSection -c s1) -set_tests_properties(FilteredSections::SimpleExample::2 PROPERTIES FAI |