catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
Added FAIL_CHECK
- Commit
5dd06395206e6abbfea2271507e1e01bb1bdc428- Author
- Phil Nash ·
- Parents
a2515755c329
Changes
docs/logging.md | 4 ++++ include/catch.hpp | 20 ++++++++++++-------- projects/SelfTest/Baselines/console.std.approved.txt | 18 ++++++++++++++++-- projects/SelfTest/Baselines/console.sw.approved.txt | 20 +++++++++++++++++--- .../SelfTest/Baselines/console.swa4.approved.txt | 9 ++++++--- projects/SelfTest/Baselines/junit.sw.approved.txt | 8 +++++++- projects/SelfTest/Baselines/xml.sw.approved.txt | 19 ++++++++++++++----- projects/SelfTest/MessageTests.cpp | 7 +++++++ 8 files changed, 83 insertions(+), 22 deletions(-) diff --git a/docs/logging.md b/docs/logging.md index a659b3e8..01c6fbc7 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -26,6 +26,10 @@ The message is always reported but does not fail the test. The message is reported and the test case fails. +**FAIL_CHECK(** _message expression_ **)** + +AS `FAIL`, but does not abort the test + ## Quickly capture a variable value **CAPTURE(** _expression_ **)** diff --git a/include/catch.hpp b/include/catch.hpp index 5d009ee0..91ef93b7 100644 --- a/include/catch.hpp +++ b/include/catch.hpp @@ -126,6 +126,7 @@ LeakDetector leakDetector; #define CATCH_REGISTER_TEST_CASE( Function, ... ) INTERNAL_CATCH_REGISTER_TESTCASE( Function, __VA_ARGS__ ) #define CATCH_SECTION( ... ) INTERNAL_CATCH_SECTION( __VA_ARGS__ ) #define CATCH_FAIL( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, "CATCH_FAIL", __VA_ARGS__ ) + #define CATCH_FAIL_CHECK( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, "CATCH_FAIL_CHECK", __VA_ARGS__ ) #define CATCH_SUCCEED( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, "CATCH_SUCCEED", __VA_ARGS__ ) #else #define CATCH_TEST_CASE( name, description ) INTERNAL_CATCH_TESTCASE( name, description ) @@ -134,6 +135,7 @@ LeakDetector leakDetector; #define CATCH_REGISTER_TEST_CASE( function, name, description ) INTERNAL_CATCH_REGISTER_TESTCASE( function, name, description ) #define CATCH_SECTION( name, description ) INTERNAL_CATCH_SECTION( name, description ) #define CATCH_FAIL( msg ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, "CATCH_FAIL", msg ) + #define CATCH_FAIL_CHECK( msg ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, "CATCH_FAIL_CHECK", msg ) #define CATCH_SUCCEED( msg ) INTERNAL_CATCH_MSG( Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, "CATCH_SUCCEED", msg ) #endif #define CATCH_ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE( "", "" ) @@ -189,20 +191,22 @@ LeakDetector leakDetector; #define SCOPED_CAPTURE( msg ) INTERNAL_CATCH_INFO( #msg " := " << Catch::toString(msg), "CAPTURE" ) #ifdef CATCH_CONFIG_VARIADIC_MACROS - #define TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE( __VA_ARGS__ ) - #define TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TEST_CASE_METHOD( className, __VA_ARGS__ ) - #define METHOD_AS_TEST_CASE( method, ... ) INTERNAL_CATCH_METHOD_AS_TEST_CASE( method, __VA_ARGS__ ) - #define REGISTER_TEST_CASE( Function, ... ) INTERNAL_CATCH_REGISTER_TESTCASE( Function, __VA_ARGS__ ) - #define SECTION( ... ) INTERNAL_CATCH_SECTION( __VA_ARGS__ ) - #define FAIL( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, "FAIL", __VA_ARGS__ ) - #define SUCCEED( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, "SUCCEED", __VA_ARGS__ ) +#define TEST_CASE( ... ) INTERNAL_CATCH_TESTCASE( __VA_ARGS__ ) +#define TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TEST_CASE_METHOD( className, __VA_ARGS__ ) +#define METHOD_AS_TEST_CASE( method, ... ) INTERNAL_CATCH_METHOD_AS_TEST_CASE( method, __VA_ARGS__ ) +#define REGISTER_TEST_CASE( Function, ... ) INTERNAL_CATCH_REGISTER_TESTCASE( Function, __VA_ARGS__ ) +#define SECTION( ... ) INTERNAL_CATCH_SECTION( __VA_ARGS__ ) +#define FAIL( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, "FAIL", __VA_ARGS__ ) +#define FAIL_CHECK( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, "FAIL_CHECK", __VA_ARGS__ ) +#define SUCCEED( ... ) INTERNAL_CATCH_MSG( Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, "SUCCEED", __VA_ARGS__ ) #else - #define TEST_CASE( name, description ) INTERNAL_CATCH_TESTCASE( name, description ) +#define TEST_CASE( name, description ) INTERNAL_CATCH_TESTCASE( name, description ) #define TEST_CASE_METHOD( className, name, description ) INTERNAL_CATCH_TEST_CASE_METHOD( className, name, description ) #define METHOD_AS_TEST_CASE( method, name, description ) INTERNAL_CATCH_METHOD_AS_TEST_CASE( method, name, description ) #define REGISTER_TEST_CASE( method, name, description ) INTERNAL_CATCH_REGISTER_TESTCASE( method, name, description ) #define SECTION( name, description ) INTERNAL_CATCH_SECTION( name, description ) #define FAIL( msg ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::Normal, "FAIL", msg ) + #define FAIL_CHECK( msg ) INTERNAL_CATCH_MSG( Catch::ResultWas::ExplicitFailure, Catch::ResultDisposition::ContinueOnFailure, "FAIL_CHECK", msg ) #define SUCCEED( msg ) INTERNAL_CATCH_MSG( Catch::ResultWas::Ok, Catch::ResultDisposition::ContinueOnFailure, "SUCCEED", msg ) #endif #define ANON_TEST_CASE() INTERNAL_CATCH_TESTCASE( "", "" ) diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt index fe0c0863..ef6948d2 100644 --- a/projects/SelfTest/Baselines/console.std.approved.txt +++ b/projects/SelfTest/Baselines/console.std.approved.txt @@ -287,6 +287,20 @@ MessageTests.cpp:<line number> MessageTests.cpp:<line number>: FAILED: +------------------------------------------------------------------------------- +FAIL_CHECK does not abort the test +------------------------------------------------------------------------------- +MessageTests.cpp:<line number> +............................................................................... + +MessageTests.cpp:<line number>: FAILED: +explicitly with message: + This is a failure + +MessageTests.cpp:<line number>: +warning: + This message appears in the output + ------------------------------------------------------------------------------- INFO and WARN do not abort tests ------------------------------------------------------------------------------- @@ -901,6 +915,6 @@ with expansion: "first" == "second" =============================================================================== -test cases: 165 | 119 passed | 44 failed | 2 failed as expected -assertions: 957 | 852 passed | 87 failed | 18 failed as expected +test cases: 166 | 119 passed | 44 failed | 3 failed as expected +assertions: 958 | 852 passed | 87 failed | 19 failed as expected diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index 4c92d229..7558e0c3 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -61,7 +61,7 @@ with expansion: MiscTests.cpp:<line number>: PASSED: - REQUIRE( (*__errno_location ()) == 1 ) + REQUIRE( (*__error()) == 1 ) with expansion: 1 == 1 @@ -1216,6 +1216,20 @@ MessageTests.cpp:<line number> MessageTests.cpp:<line number>: FAILED: +------------------------------------------------------------------------------- +FAIL_CHECK does not abort the test +------------------------------------------------------------------------------- +MessageTests.cpp:<line number> +............................................................................... + +MessageTests.cpp:<line number>: FAILED: +explicitly with message: + This is a failure + +MessageTests.cpp:<line number>: +warning: + This message appears in the output + ------------------------------------------------------------------------------- Factorials are computed ------------------------------------------------------------------------------- @@ -9367,6 +9381,6 @@ MiscTests.cpp:<line number>: PASSED: =============================================================================== -test cases: 165 | 118 passed | 45 failed | 2 failed as expected -assertions: 959 | 852 passed | 89 failed | 18 failed as expected +test cases: 166 | 118 passed | 45 failed | 3 failed as expected +assertions: 960 | 852 passed | 89 failed | 19 failed as expected diff --git a/projects/SelfTest/Baselines/console.swa4.approved.txt b/projects/SelfTest/Baselines/console.swa4.approved.txt index 6c4f19a0..aec6b675 100644 --- a/projects/SelfTest/Baselines/console.swa4.approved.txt +++ b/projects/SelfTest/Baselines/console.swa4.approved.txt @@ -61,7 +61,7 @@ with expansion: MiscTests.cpp:<line number>: PASSED: - REQUIRE( (*__errno_location ()) == 1 ) + REQUIRE( (*__error()) == 1 ) with expansion: 1 == 1 @@ -82,7 +82,10 @@ ConditionTests.cpp:<line number>: FAILED: with expansion: false +ConditionTests.cpp:<line number>: FAILED: + CHECK_FALSE( true ) + =============================================================================== -test cases: 5 | 3 passed | 2 failed -assertions: 11 | 7 passed | 4 failed +test cases: 5 | 3 passed | 1 failed | 1 failed as expected +assertions: 12 | 7 passed | 4 failed | 1 failed as expected diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 2bec8e6b..44a6e56a 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <testsuitesspanner> - <testsuite name="<exe-name>" errors="13" failures="77" tests="960" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> + <testsuite name="<exe-name>" errors="13" failures="77" tests="961" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> <testcase classname="global" name="# A test name that starts with a #" time="{duration}"/> <testcase classname="global" name="#542" time="{duration}"/> <testcase classname="global" name="#809" time="{duration}"/> @@ -199,6 +199,12 @@ MessageTests.cpp:<line number> </testcase> <testcase classname="global" name="FAIL does not require an argument" time="{duration}"> <failure type="FAIL"> +MessageTests.cpp:<line number> + </failure> + </testcase> + <testcase classname="global" name="FAIL_CHECK does not abort the test" time="{duration}"> + <failure type="FAIL_CHECK"> +This is a failure MessageTests.cpp:<line number> </failure> </testcase> diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index f77b7e52..5db8eff2 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -50,7 +50,7 @@ </Expression> <OverallResult success="true"/> </TestCase> - <TestCase name="#835 -- errno should not be touched by Catch" filename="projects/<exe-name>/MiscTests.cpp" > + <TestCase name="#835 -- errno should not be touched by Catch" tags="[!shouldfail]" filename="projects/<exe-name>/MiscTests.cpp" > <Expression success="false" type="CHECK" filename="projects/<exe-name>/MiscTests.cpp" > <Original> f() == 0 @@ -61,13 +61,13 @@ </Expression> <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/MiscTests.cpp" > <Original> - (*__errno_location ()) == 1 + (*__error()) == 1 </Original> <Expanded> 1 == 1 </Expanded> </Expression> - <OverallResult success="false"/> + <OverallResult success="true"/> </TestCase> <TestCase name="'Not' checks that should fail" tags="[.][failing][hide]" filename="projects/<exe-name>/ConditionTests.cpp" > <Expression success="false" type="CHECK" filename="projects/<exe-name>/ConditionTests.cpp" > @@ -1378,6 +1378,15 @@ <Failure filename="projects/<exe-name>/MessageTests.cpp" /> <OverallResult success="false"/> </TestCase> + <TestCase name="FAIL_CHECK does not abort the test" tags="[.][failing][hide][messages]" filename="projects/<exe-name>/MessageTests.cpp" > + <Failure filename="projects/<exe-name>/MessageTests.cpp" > + This is a failure + </Failure> + <Warning> + This message appears in the output + </Warning> + <OverallResult success="false"/> + </TestCase> <TestCase name="Factorials are computed" tags="[factorial]" filename="projects/<exe-name>/MiscTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/MiscTests.cpp" > <Original> @@ -10007,7 +10016,7 @@ spanner <OverallResult success="true"/> </Section> <OverallResult success="true"/> </TestCase> - <OverallResults successes="852" failures="90" expectedFailures="18"/> + <OverallResults successes="852" failures="90" expectedFailures="19"/> </Group> - <OverallResults successes="852" failures="89" expectedFailures="18"/> + <OverallResults successes="852" failures="89" expectedFailures="19"/> </Catch> diff --git a/projects/SelfTest/MessageTests.cpp b/projects/SelfTest/MessageTests.cpp index 87a85a82..bed9c137 100644 --- a/projects/SelfTest/MessageTests.cpp +++ b/projects/SelfTest/MessageTests.cpp @@ -53,6 +53,13 @@ TEST_CASE( "INFO gets logged on failure, even if captured before successful asse TEST_CASE( "FAIL aborts the test", "[failing][messages][.]" ) { FAIL( "This is a " << "failure" ); // This should output the message and abort + WARN( "We should never see this"); +} + +TEST_CASE( "FAIL_CHECK does not abort the test", "[failing][messages][.]" ) +{ + FAIL_CHECK( "This is a " << "failure" ); // This should output the message then continue + WARN( "This message appears in the output"); } #ifdef CATCH_CONFIG_VARIADIC_MACROS