catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
Don’t include tests that rely on varying two-phase-lookup behaviour in approval tests
- Commit
225e90d8baf78d6b76d059d5e81aeb2faffa5dde- Author
- Phil Nash ·
- Parents
31c23b9489c2
Changes
.../SelfTest/Baselines/console.std.approved.txt | 4 +-- .../SelfTest/Baselines/console.sw.approved.txt | 30 ++-------------------- projects/SelfTest/Baselines/junit.sw.approved.txt | 4 +-- projects/SelfTest/Baselines/xml.sw.approved.txt | 28 ++------------------ projects/SelfTest/ToStringWhich.cpp | 11 +++++--- scripts/approvalTests.py | 10 ++++---- 6 files changed, 19 insertions(+), 68 deletions(-) diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt index f5c53cd6..61b33905 100644 --- a/projects/SelfTest/Baselines/console.std.approved.txt +++ b/projects/SelfTest/Baselines/console.std.approved.txt @@ -830,6 +830,6 @@ with expansion: "first" == "second" =============================================================================== -test cases: 159 | 115 passed | 42 failed | 2 failed as expected -assertions: 915 | 819 passed | 78 failed | 18 failed as expected +test cases: 157 | 113 passed | 42 failed | 2 failed as expected +assertions: 913 | 817 passed | 78 failed | 18 failed as expected diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index 4ac83ca2..2222058a 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -8757,32 +8757,6 @@ with expansion: == "{ StringMaker<has_maker> }" -------------------------------------------------------------------------------- -toString( vectors<has_maker_and_toString ) -------------------------------------------------------------------------------- -ToStringWhich.cpp:<line number> -............................................................................... - -ToStringWhich.cpp:<line number>: -PASSED: - REQUIRE( Catch::toString( v ) == "{ StringMaker<has_maker_and_toString> }" ) -with expansion: - "{ StringMaker<has_maker_and_toString> }" - == - "{ StringMaker<has_maker_and_toString> }" - -------------------------------------------------------------------------------- -toString( vectors<has_toString ) -------------------------------------------------------------------------------- -ToStringWhich.cpp:<line number> -............................................................................... - -ToStringWhich.cpp:<line number>: -PASSED: - REQUIRE( Catch::toString( v ) == "{ {?} }" ) -with expansion: - "{ {?} }" == "{ {?} }" - ------------------------------------------------------------------------------- toString(enum w/operator<<) ------------------------------------------------------------------------------- @@ -9052,6 +9026,6 @@ MiscTests.cpp:<line number>: PASSED: =============================================================================== -test cases: 159 | 114 passed | 43 failed | 2 failed as expected -assertions: 917 | 819 passed | 80 failed | 18 failed as expected +test cases: 157 | 112 passed | 43 failed | 2 failed as expected +assertions: 915 | 817 passed | 80 failed | 18 failed as expected diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 43b808a2..81eb0975 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,6 +1,6 @@ <?xml version="1.1" encoding="UTF-8"?> <testsuites> - <testsuite name="<exe-name>" errors="13" failures="68" tests="918" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> + <testsuite name="<exe-name>" errors="13" failures="68" tests="916" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> <testcase classname="global" name="# A test name that starts with a #" time="{duration}"/> <testcase classname="global" name="'Not' checks that should fail" time="{duration}"> <failure message="false != false" type="CHECK"> @@ -654,8 +654,6 @@ TrickyTests.cpp:<line number> <testcase classname="global" name="toString( has_maker_and_toString )" time="{duration}"/> <testcase classname="global" name="toString( has_toString )" time="{duration}"/> <testcase classname="global" name="toString( vectors<has_maker )" time="{duration}"/> - <testcase classname="global" name="toString( vectors<has_maker_and_toString )" time="{duration}"/> - <testcase classname="global" name="toString( vectors<has_toString )" time="{duration}"/> <testcase classname="global" name="toString(enum w/operator<<)" time="{duration}"/> <testcase classname="global" name="toString(enum)" time="{duration}"/> <testcase classname="global" name="vector<int> -> toString" time="{duration}"/> diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index 33989783..0af58602 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -9246,30 +9246,6 @@ there" </Expression> <OverallResult success="true"/> </TestCase> - <TestCase name="toString( vectors<has_maker_and_toString )"> - <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ToStringWhich.cpp" > - <Original> - Catch::toString( v ) == "{ StringMaker<has_maker_and_toString> }" - </Original> - <Expanded> - "{ StringMaker<has_maker_and_toString> }" -== -"{ StringMaker<has_maker_and_toString> }" - </Expanded> - </Expression> - <OverallResult success="true"/> - </TestCase> - <TestCase name="toString( vectors<has_toString )"> - <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ToStringWhich.cpp" > - <Original> - Catch::toString( v ) == "{ {?} }" - </Original> - <Expanded> - "{ {?} }" == "{ {?} }" - </Expanded> - </Expression> - <OverallResult success="true"/> - </TestCase> <TestCase name="toString(enum w/operator<<)"> <Expression success="true" type="CHECK" filename="projects/<exe-name>/EnumToString.cpp" > <Original> @@ -9527,7 +9503,7 @@ there" </Section> <OverallResult success="true"/> </TestCase> - <OverallResults successes="819" failures="81" expectedFailures="18"/> + <OverallResults successes="817" failures="81" expectedFailures="18"/> </Group> - <OverallResults successes="819" failures="80" expectedFailures="18"/> + <OverallResults successes="817" failures="80" expectedFailures="18"/> </Catch> diff --git a/projects/SelfTest/ToStringWhich.cpp b/projects/SelfTest/ToStringWhich.cpp index 1d4aa892..a1d2a609 100644 --- a/projects/SelfTest/ToStringWhich.cpp +++ b/projects/SelfTest/ToStringWhich.cpp @@ -37,19 +37,21 @@ TEST_CASE( "toString( has_toString )", "[toString]" ) { } // Call the overload -TEST_CASE( "toString( has_maker )", "[toString]" ) { +TEST_CASE( "toString( has_maker )", "toString]" ) { has_maker item; REQUIRE( Catch::toString( item ) == "StringMaker<has_maker>" ); } // Call the overload -TEST_CASE( "toString( has_maker_and_toString )", "[toString]" ) { +TEST_CASE( "toString( has_maker_and_toString )", "[.][toString]" ) { has_maker_and_toString item; REQUIRE( Catch::toString( item ) == "toString( has_maker_and_toString )" ); } // Vectors... -TEST_CASE( "toString( vectors<has_toString )", "[toString]" ) { + +// Don't run this in approval tests as it is sensitive to two phase lookup differences +TEST_CASE( "toString( vectors<has_toString )", "[.][toString][!nonportable]" ) { std::vector<has_toString> v(1); // This invokes template<T> toString which actually gives us '{ ? }' REQUIRE( Catch::toString( v ) == "{ {?} }" ); @@ -61,7 +63,8 @@ TEST_CASE( "toString( vectors<has_maker )", "[toString]" ) { } -TEST_CASE( "toString( vectors<has_maker_and_toString )", "[toString]" ) { +// Don't run this in approval tests as it is sensitive to two phase lookup differences +TEST_CASE( "toString( vectors<has_maker_and_toString )", "[.][toString][!nonportable]" ) { std::vector<has_maker_and_toString> v(1); // Note: This invokes the template<T> toString -> StringMaker REQUIRE( Catch::toString( v ) == "{ StringMaker<has_maker_and_toString> }" ); diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py index e57241da..6c51aa4e 100644 --- a/scripts/approvalTests.py +++ b/scripts/approvalTests.py @@ -137,15 +137,15 @@ print("Running approvals against executable:") print(" " + cmdPath) # Standard console reporter -approve("console.std", ["~[c++11]", "--order", "lex"]) +approve("console.std", ["~[c++11]~[!nonportable]", "--order", "lex"]) # console reporter, include passes, warn about No Assertions -approve("console.sw", ["~[c++11]", "-s", "-w", "NoAssertions", "--order", "lex"]) +approve("console.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "--order", "lex"]) # console reporter, include passes, warn about No Assertions, limit failures to first 4 -approve("console.swa4", ["~[c++11]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"]) +approve("console.swa4", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"]) # junit reporter, include passes, warn about No Assertions -approve("junit.sw", ["~[c++11]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"]) +approve("junit.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"]) # xml reporter, include passes, warn about No Assertions -approve("xml.sw", ["~[c++11]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"]) +approve("xml.sw", ["~[c++11]~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"]) if overallResult != 0: print("If these differenecs are expected run approve.py to approve new baselines")