catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
Fix/disable failing approval tests
- Commit
ab5d176195f7b9fc0bee52da9f7643b79fab35e4- Author
- Martin Hořeňovský ·
- Parents
b3a923133d7b
Changes
CMakeLists.txt | 1 + .../SelfTest/Baselines/console.std.approved.txt | 18 +------- .../SelfTest/Baselines/console.sw.approved.txt | 50 ++-------------------- projects/SelfTest/Baselines/junit.sw.approved.txt | 5 +-- projects/SelfTest/Baselines/xml.sw.approved.txt | 45 +++---------------- projects/SelfTest/ConditionTests.cpp | 3 +- projects/SelfTest/MessageTests.cpp | 6 +-- projects/SelfTest/TrickyTests.cpp | 17 +++++--- scripts/approvalTests.py | 12 +++--- 9 files changed, 37 insertions(+), 120 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2903ad9e..c6f4b900 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -317,6 +317,7 @@ if (NOT NO_SELFTEST) add_test(NAME ListTags COMMAND SelfTest --list-tags) set_tests_properties(ListTags PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ tags") + # AppVeyor has a Python 2.7 in path, but doesn't have .py files as autorunnable add_test(NAME ApprovalTests COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/scripts/approvalTests.py $<TARGET_FILE:SelfTest>) set_tests_properties(ApprovalTests PROPERTIES FAIL_REGULAR_EXPRESSION "Results differed") endif() # !NO_SELFTEST diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt index 42d62c9c..dfa2fa92 100644 --- a/projects/SelfTest/Baselines/console.std.approved.txt +++ b/projects/SelfTest/Baselines/console.std.approved.txt @@ -644,20 +644,6 @@ MessageTests.cpp:<line number>: FAILED: explicitly with message: Message from section two -------------------------------------------------------------------------------- -Pointers can be converted to strings -------------------------------------------------------------------------------- -MessageTests.cpp:<line number> -............................................................................... - -MessageTests.cpp:<line number>: -warning: - actual address of p: 0x<hex digits> - -MessageTests.cpp:<line number>: -warning: - toString(p): 0x<hex digits> - ------------------------------------------------------------------------------- Reconstruction should be based on stringification: #914 ------------------------------------------------------------------------------- @@ -1020,6 +1006,6 @@ with expansion: "{?}" == "1" =============================================================================== -test cases: 182 | 131 passed | 47 failed | 4 failed as expected -assertions: 881 | 764 passed | 96 failed | 21 failed as expected +test cases: 179 | 128 passed | 47 failed | 4 failed as expected +assertions: 879 | 762 passed | 96 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index 02ec81d7..bcba36a1 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -955,20 +955,6 @@ PASSED: with expansion: 0x<hex digits> == 0x<hex digits> -------------------------------------------------------------------------------- -Comparing member function pointers -------------------------------------------------------------------------------- -TrickyTests.cpp:<line number> -............................................................................... - -TrickyTests.cpp:<line number>: -PASSED: - CHECK( m == &S::f ) -with expansion: - 0x<hex digits> - == - 0x<hex digits> - ------------------------------------------------------------------------------- Comparison with explicitly convertible types ------------------------------------------------------------------------------- @@ -1182,11 +1168,9 @@ with expansion: ConditionTests.cpp:<line number>: PASSED: - REQUIRE( (std::numeric_limits<unsigned long>::max)() > ul ) + REQUIRE( (std::numeric_limits<uint32_t>::max)() > ul ) with expansion: - 18446744073709551615 (0x<hex digits>) - > - 4 + 4294967295 (0x<hex digits>) > 4 ------------------------------------------------------------------------------- Contains string matcher @@ -2096,18 +2080,6 @@ PASSED: with expansion: !{?} -------------------------------------------------------------------------------- -Operators at different namespace levels not hijacked by Koenig lookup -------------------------------------------------------------------------------- -TrickyTests.cpp:<line number> -............................................................................... - -TrickyTests.cpp:<line number>: -PASSED: - REQUIRE( 0x<hex digits> == o ) -with expansion: - 3221225472 (0x<hex digits>) == {?} - ------------------------------------------------------------------------------- Ordering comparison checks that should fail ------------------------------------------------------------------------------- @@ -3432,20 +3404,6 @@ PASSED: with expansion: 0 != 0x<hex digits> -------------------------------------------------------------------------------- -Pointers can be converted to strings -------------------------------------------------------------------------------- -MessageTests.cpp:<line number> -............................................................................... - -MessageTests.cpp:<line number>: -warning: - actual address of p: 0x<hex digits> - -MessageTests.cpp:<line number>: -warning: - toString(p): 0x<hex digits> - ------------------------------------------------------------------------------- Process can be configured on command line empty args don't cause a crash @@ -7521,6 +7479,6 @@ MiscTests.cpp:<line number>: PASSED: =============================================================================== -test cases: 182 | 128 passed | 50 failed | 4 failed as expected -assertions: 883 | 760 passed | 102 failed | 21 failed as expected +test cases: 179 | 125 passed | 50 failed | 4 failed as expected +assertions: 881 | 758 passed | 102 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 9c693386..13c48c19 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <testsuitesloose text artifact > - <testsuite name="<exe-name>" errors="15" failures="88" tests="884" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> + <testsuite name="<exe-name>" errors="15" failures="88" tests="882" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> <testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/> <testcase classname="<exe-name>.global" name="#1005: Comparing pointer to int and long (NULL can be either on various systems)" time="{duration}"/> <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}"> @@ -116,7 +116,6 @@ ExceptionTests.cpp:<line number> <testcase classname="<exe-name>.global" name="Character pretty printing/Low ASCII" time="{duration}"/> <testcase classname="<exe-name>.global" name="Commas in various macros are allowed" time="{duration}"/> <testcase classname="<exe-name>.global" name="Comparing function pointers" time="{duration}"/> - <testcase classname="<exe-name>.global" name="Comparing member function pointers" time="{duration}"/> <testcase classname="<exe-name>.global" name="Comparison with explicitly convertible types" time="{duration}"/> <testcase classname="<exe-name>.global" name="Comparisons between ints where one side is computed" time="{duration}"/> <testcase classname="<exe-name>.global" name="Comparisons between unsigned ints and negative signed ints match c++ standard behaviour" time="{duration}"/> @@ -335,7 +334,6 @@ ExceptionTests.cpp:<line number> </error> </testcase> <testcase classname="<exe-name>.global" name="Objects that evaluated in boolean contexts can be checked" time="{duration}"/> - <testcase classname="<exe-name>.global" name="Operators at different namespace levels not hijacked by Koenig lookup" time="{duration}"/> <testcase classname="<exe-name>.global" name="Ordering comparison checks that should fail" time="{duration}"> <failure message="7 > 7" type="CHECK"> ConditionTests.cpp:<line number> @@ -441,7 +439,6 @@ MessageTests.cpp:<line number> <testcase classname="<exe-name>.global" name="Parse test names and tags/quoted string followed by tag exclusion" time="{duration}"/> <testcase classname="<exe-name>.global" name="Parsing a std::pair" time="{duration}"/> <testcase classname="<exe-name>.global" name="Pointers can be compared to null" time="{duration}"/> - <testcase classname="<exe-name>.global" name="Pointers can be converted to strings" time="{duration}"/> <testcase classname="<exe-name>.global" name="Process can be configured on command line/empty args don't cause a crash" time="{duration}"/> <testcase classname="<exe-name>.global" name="Process can be configured on command line/default - no arguments" time="{duration}"/> <testcase classname="<exe-name>.global" name="Process can be configured on command line/test lists/1 test" time="{duration}"/> diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index 1acc1d74..9ab6d491 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -1023,19 +1023,6 @@ </Expression> <OverallResult success="true"/> </TestCase> - <TestCase name="Comparing member function pointers" tags="[Tricky][member function pointer]" filename="projects/<exe-name>/TrickyTests.cpp" > - <Expression success="true" type="CHECK" filename="projects/<exe-name>/TrickyTests.cpp" > - <Original> - m == &S::f - </Original> - <Expanded> - 0x<hex digits> -== -0x<hex digits> - </Expanded> - </Expression> - <OverallResult success="true"/> - </TestCase> <TestCase name="Comparison with explicitly convertible types" tags="[Approx]" filename="projects/<exe-name>/ApproxTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ApproxTests.cpp" > <Original> @@ -1296,12 +1283,10 @@ </Expression> <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ConditionTests.cpp" > <Original> - (std::numeric_limits<unsigned long>::max)() > ul + (std::numeric_limits<uint32_t>::max)() > ul </Original> <Expanded> - 18446744073709551615 (0x<hex digits>) -> -4 + 4294967295 (0x<hex digits>) > 4 </Expanded> </Expression> <OverallResult success="true"/> @@ -2388,17 +2373,6 @@ </Expression> <OverallResult success="true"/> </TestCase> - <TestCase name="Operators at different namespace levels not hijacked by Koenig lookup" tags="[Tricky]" filename="projects/<exe-name>/TrickyTests.cpp" > - <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/TrickyTests.cpp" > - <Original> - 0x<hex digits> == o - </Original> - <Expanded> - 3221225472 (0x<hex digits>) == {?} - </Expanded> - </Expression> - <OverallResult success="true"/> - </TestCase> <TestCase name="Ordering comparison checks that should fail" tags="[.][failing]" filename="projects/<exe-name>/ConditionTests.cpp" > <Expression success="false" type="CHECK" filename="projects/<exe-name>/ConditionTests.cpp" > <Original> @@ -3962,15 +3936,6 @@ </Expression> <OverallResult success="true"/> </TestCase> - <TestCase name="Pointers can be converted to strings" tags="[.][messages]" filename="projects/<exe-name>/MessageTests.cpp" > - <Warning> - actual address of p: 0x<hex digits> - </Warning> - <Warning> - toString(p): 0x<hex digits> - </Warning> - <OverallResult success="true"/> - </TestCase> <TestCase name="Process can be configured on command line" tags="[command-line][config]" filename="projects/<exe-name>/TestMain.cpp" > <Section name="empty args don't cause a crash" filename="projects/<exe-name>/TestMain.cpp" > <Expression success="true" type="CHECK" filename="projects/<exe-name>/TestMain.cpp" > @@ -4824,7 +4789,7 @@ A string sent directly to stderr </Expression> <OverallResult success="true"/> </TestCase> - <TestCase name="Standard error is reported and redirected" tags="[.][messages]" filename="projects/<exe-name>/MessageTests.cpp" > + <TestCase name="Standard error is reported and redirected" tags="[.][.approvals][messages]" filename="projects/<exe-name>/MessageTests.cpp" > <Section name="std::cerr" filename="projects/<exe-name>/MessageTests.cpp" > <OverallResults successes="0" failures="1" expectedFailures="0"/> </Section> @@ -8276,7 +8241,7 @@ loose text artifact </Section> <OverallResult success="true"/> </TestCase> - <OverallResults successes="760" failures="103" expectedFailures="21"/> + <OverallResults successes="758" failures="103" expectedFailures="21"/> </Group> - <OverallResults successes="760" failures="102" expectedFailures="21"/> + <OverallResults successes="758" failures="102" expectedFailures="21"/> </Catch> diff --git a/projects/SelfTest/ConditionTests.cpp b/projects/SelfTest/ConditionTests.cpp index 565d9815..00ff6060 100644 --- a/projects/SelfTest/ConditionTests.cpp +++ b/projects/SelfTest/ConditionTests.cpp @@ -13,6 +13,7 @@ #include <string> #include <limits> +#include <cstdint> struct TestData { int int_seven = 7; @@ -184,7 +185,7 @@ TEST_CASE( "Comparisons with int literals don't warn when mixing signed/ unsigne REQUIRE( 5 == c ); REQUIRE( 6 == uc ); - REQUIRE( (std::numeric_limits<unsigned long>::max)() > ul ); + REQUIRE( (std::numeric_limits<uint32_t>::max)() > ul ); } // Disable warnings about sign conversions for the next two tests diff --git a/projects/SelfTest/MessageTests.cpp b/projects/SelfTest/MessageTests.cpp index 437ba53d..6082a4e6 100644 --- a/projects/SelfTest/MessageTests.cpp +++ b/projects/SelfTest/MessageTests.cpp @@ -85,7 +85,7 @@ TEST_CASE( "Standard output from all sections is reported", "[messages][.]" ) { } } -TEST_CASE( "Standard error is reported and redirected", "[messages][.]" ) { +TEST_CASE( "Standard error is reported and redirected", "[messages][.][approvals]" ) { SECTION( "std::cerr" ) { std::cerr << "Write to std::cerr" << std::endl; } @@ -98,7 +98,7 @@ TEST_CASE( "Standard error is reported and redirected", "[messages][.]" ) { std::cerr << ' '; std::clog << "writes"; std::cerr << " to error"; - std::clog << " streams\n"; + std::clog << " streams" << std::endl; } } @@ -130,7 +130,7 @@ TEST_CASE( "sends information to INFO", "[.][failing]" ) { REQUIRE( false ); } -TEST_CASE( "Pointers can be converted to strings", "[messages][.]" ) { +TEST_CASE( "Pointers can be converted to strings", "[messages][.][approvals]" ) { int p; WARN( "actual address of p: " << &p ); WARN( "toString(p): " << ::Catch::Detail::stringify( &p ) ); diff --git a/projects/SelfTest/TrickyTests.cpp b/projects/SelfTest/TrickyTests.cpp index f551d3bf..90abb4a8 100644 --- a/projects/SelfTest/TrickyTests.cpp +++ b/projects/SelfTest/TrickyTests.cpp @@ -175,7 +175,7 @@ namespace ObjectWithConversions TEST_CASE ( "Operators at different namespace levels not hijacked by Koenig lookup", - "[Tricky]" + "[Tricky][approvals]" ) { Object o; @@ -211,9 +211,16 @@ namespace ObjectWithNonConstEqualityOperator namespace EnumBitFieldTests { - enum Bits {bit0 = 0x0001, bit1 = 0x0002, bit2 = 0x0004, bit3 = 0x0008, bit1and2 = 0x0006, - bit30 = 0x40000000, bit31 = 0x80000000, - bit30and31 = 0xc0000000}; + enum Bits : uint32_t { + bit0 = 0x0001, + bit1 = 0x0002, + bit2 = 0x0004, + bit3 = 0x0008, + bit1and2 = bit1 | bit2, + bit30 = 0x40000000, + bit31 = 0x80000000, + bit30and31 = bit30 | bit31 + }; TEST_CASE( "Test enum bit values", "[Tricky]" ) { @@ -365,7 +372,7 @@ struct S }; -TEST_CASE( "Comparing member function pointers", "[Tricky][member function pointer]" ) +TEST_CASE( "Comparing member function pointers", "[Tricky][member function pointer][approvals]" ) { typedef void (S::*MF)(); MF m = &S::f; diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py index 49553349..5c0c63ec 100755 --- a/scripts/approvalTests.py +++ b/scripts/approvalTests.py @@ -41,6 +41,8 @@ errnoParser = re.compile(r''' \(\*__errno_location\ \(\)\) | \(\*__error\(\)\) + | + \(\*_errno\(\)\) ''', re.VERBOSE) if len(sys.argv) == 2: @@ -145,15 +147,15 @@ print("Running approvals against executable:") print(" " + cmdPath) # Standard console reporter -approve("console.std", ["~[!nonportable]~[!benchmark]", "--order", "lex"]) +approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals]", "--order", "lex"]) # console reporter, include passes, warn about No Assertions -approve("console.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "--order", "lex"]) +approve("console.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "--order", "lex"]) # console reporter, include passes, warn about No Assertions, limit failures to first 4 -approve("console.swa4", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"]) +approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"]) # junit reporter, include passes, warn about No Assertions -approve("junit.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"]) +approve("junit.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"]) # xml reporter, include passes, warn about No Assertions -approve("xml.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"]) +approve("xml.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"]) if overallResult != 0: print("If these differences are expected, run approve.py to approve new baselines.")