Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

Fix/disable failing approval tests

Commit
ab5d176195f7b9fc0bee52da9f7643b79fab35e4
Author
Martin Hořeňovský ·
Parents
b3a923133d7b

Files · Download .tar.gz

Changes

 CMakeLists.txt                                     |  1 +
 .../SelfTest/Baselines/console.std.approved.txt    | 18 +-------
 .../SelfTest/Baselines/console.sw.approved.txt     | 50 ++--------------------
 projects/SelfTest/Baselines/junit.sw.approved.txt  |  5 +--
 projects/SelfTest/Baselines/xml.sw.approved.txt    | 45 +++----------------
 projects/SelfTest/ConditionTests.cpp               |  3 +-
 projects/SelfTest/MessageTests.cpp                 |  6 +--
 projects/SelfTest/TrickyTests.cpp                  | 17 +++++---
 scripts/approvalTests.py                           | 12 +++---
 9 files changed, 37 insertions(+), 120 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2903ad9e..c6f4b900 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -317,6 +317,7 @@ if (NOT NO_SELFTEST)
     add_test(NAME ListTags COMMAND SelfTest --list-tags)
     set_tests_properties(ListTags PROPERTIES PASS_REGULAR_EXPRESSION "[0-9]+ tags")
 
+    # AppVeyor has a Python 2.7 in path, but doesn't have .py files as autorunnable
     add_test(NAME ApprovalTests COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/scripts/approvalTests.py $<TARGET_FILE:SelfTest>)
     set_tests_properties(ApprovalTests PROPERTIES FAIL_REGULAR_EXPRESSION "Results differed")
 endif() # !NO_SELFTEST
diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt
index 42d62c9c..dfa2fa92 100644
--- a/projects/SelfTest/Baselines/console.std.approved.txt
+++ b/projects/SelfTest/Baselines/console.std.approved.txt
@@ -644,20 +644,6 @@ MessageTests.cpp:<line number>: FAILED:
 explicitly with message:
   Message from section two
 
--------------------------------------------------------------------------------
-Pointers can be converted to strings
--------------------------------------------------------------------------------
-MessageTests.cpp:<line number>
-...............................................................................
-
-MessageTests.cpp:<line number>:
-warning:
-  actual address of p: 0x<hex digits>
-
-MessageTests.cpp:<line number>:
-warning:
-  toString(p): 0x<hex digits>
-
 -------------------------------------------------------------------------------
 Reconstruction should be based on stringification: #914
 -------------------------------------------------------------------------------
@@ -1020,6 +1006,6 @@ with expansion:
   "{?}" == "1"
 
 ===============================================================================
-test cases: 182 | 131 passed | 47 failed |  4 failed as expected
-assertions: 881 | 764 passed | 96 failed | 21 failed as expected
+test cases: 179 | 128 passed | 47 failed |  4 failed as expected
+assertions: 879 | 762 passed | 96 failed | 21 failed as expected
 
diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt
index 02ec81d7..bcba36a1 100644
--- a/projects/SelfTest/Baselines/console.sw.approved.txt
+++ b/projects/SelfTest/Baselines/console.sw.approved.txt
@@ -955,20 +955,6 @@ PASSED:
 with expansion:
   0x<hex digits> == 0x<hex digits>
 
--------------------------------------------------------------------------------
-Comparing member function pointers
--------------------------------------------------------------------------------
-TrickyTests.cpp:<line number>
-...............................................................................
-
-TrickyTests.cpp:<line number>:
-PASSED:
-  CHECK( m == &S::f )
-with expansion:
-  0x<hex digits>
-  ==
-  0x<hex digits>
-
 -------------------------------------------------------------------------------
 Comparison with explicitly convertible types
 -------------------------------------------------------------------------------
@@ -1182,11 +1168,9 @@ with expansion:
 
 ConditionTests.cpp:<line number>:
 PASSED:
-  REQUIRE( (std::numeric_limits<unsigned long>::max)() > ul )
+  REQUIRE( (std::numeric_limits<uint32_t>::max)() > ul )
 with expansion:
-  18446744073709551615 (0x<hex digits>)
-  >
-  4
+  4294967295 (0x<hex digits>) > 4
 
 -------------------------------------------------------------------------------
 Contains string matcher
@@ -2096,18 +2080,6 @@ PASSED:
 with expansion:
   !{?}
 
--------------------------------------------------------------------------------
-Operators at different namespace levels not hijacked by Koenig lookup
--------------------------------------------------------------------------------
-TrickyTests.cpp:<line number>
-...............................................................................
-
-TrickyTests.cpp:<line number>:
-PASSED:
-  REQUIRE( 0x<hex digits> == o )
-with expansion:
-  3221225472 (0x<hex digits>) == {?}
-
 -------------------------------------------------------------------------------
 Ordering comparison checks that should fail
 -------------------------------------------------------------------------------
@@ -3432,20 +3404,6 @@ PASSED:
 with expansion:
   0 != 0x<hex digits>
 
--------------------------------------------------------------------------------
-Pointers can be converted to strings
--------------------------------------------------------------------------------
-MessageTests.cpp:<line number>
-...............................................................................
-
-MessageTests.cpp:<line number>:
-warning:
-  actual address of p: 0x<hex digits>
-
-MessageTests.cpp:<line number>:
-warning:
-  toString(p): 0x<hex digits>
-
 -------------------------------------------------------------------------------
 Process can be configured on command line
   empty args don't cause a crash
@@ -7521,6 +7479,6 @@ MiscTests.cpp:<line number>:
 PASSED:
 
 ===============================================================================
-test cases: 182 | 128 passed |  50 failed |  4 failed as expected
-assertions: 883 | 760 passed | 102 failed | 21 failed as expected
+test cases: 179 | 125 passed |  50 failed |  4 failed as expected
+assertions: 881 | 758 passed | 102 failed | 21 failed as expected
 
diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt
index 9c693386..13c48c19 100644
--- a/projects/SelfTest/Baselines/junit.sw.approved.txt
+++ b/projects/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesloose text artifact
 >
-  <testsuite name="<exe-name>" errors="15" failures="88" tests="884" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="15" failures="88" tests="882" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#1005: Comparing pointer to int and long (NULL can be either on various systems)" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
@@ -116,7 +116,6 @@ ExceptionTests.cpp:<line number>
     <testcase classname="<exe-name>.global" name="Character pretty printing/Low ASCII" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Commas in various macros are allowed" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Comparing function pointers" time="{duration}"/>
-    <testcase classname="<exe-name>.global" name="Comparing member function pointers" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Comparison with explicitly convertible types" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Comparisons between ints where one side is computed" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Comparisons between unsigned ints and negative signed ints match c++ standard behaviour" time="{duration}"/>
@@ -335,7 +334,6 @@ ExceptionTests.cpp:<line number>
       </error>
     </testcase>
     <testcase classname="<exe-name>.global" name="Objects that evaluated in boolean contexts can be checked" time="{duration}"/>
-    <testcase classname="<exe-name>.global" name="Operators at different namespace levels not hijacked by Koenig lookup" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Ordering comparison checks that should fail" time="{duration}">
       <failure message="7 > 7" type="CHECK">
 ConditionTests.cpp:<line number>
@@ -441,7 +439,6 @@ MessageTests.cpp:<line number>
     <testcase classname="<exe-name>.global" name="Parse test names and tags/quoted string followed by tag exclusion" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Parsing a std::pair" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Pointers can be compared to null" time="{duration}"/>
-    <testcase classname="<exe-name>.global" name="Pointers can be converted to strings" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Process can be configured on command line/empty args don't cause a crash" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Process can be configured on command line/default - no arguments" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="Process can be configured on command line/test lists/1 test" time="{duration}"/>
diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt
index 1acc1d74..9ab6d491 100644
--- a/projects/SelfTest/Baselines/xml.sw.approved.txt
+++ b/projects/SelfTest/Baselines/xml.sw.approved.txt
@@ -1023,19 +1023,6 @@
       </Expression>
       <OverallResult success="true"/>
     </TestCase>
-    <TestCase name="Comparing member function pointers" tags="[Tricky][member function pointer]" filename="projects/<exe-name>/TrickyTests.cpp" >
-      <Expression success="true" type="CHECK" filename="projects/<exe-name>/TrickyTests.cpp" >
-        <Original>
-          m == &amp;S::f
-        </Original>
-        <Expanded>
-          0x<hex digits>
-==
-0x<hex digits>
-        </Expanded>
-      </Expression>
-      <OverallResult success="true"/>
-    </TestCase>
     <TestCase name="Comparison with explicitly convertible types" tags="[Approx]" filename="projects/<exe-name>/ApproxTests.cpp" >
       <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ApproxTests.cpp" >
         <Original>
@@ -1296,12 +1283,10 @@
       </Expression>
       <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/ConditionTests.cpp" >
         <Original>
-          (std::numeric_limits&lt;unsigned long>::max)() > ul
+          (std::numeric_limits&lt;uint32_t>::max)() > ul
         </Original>
         <Expanded>
-          18446744073709551615 (0x<hex digits>)
->
-4
+          4294967295 (0x<hex digits>) > 4
         </Expanded>
       </Expression>
       <OverallResult success="true"/>
@@ -2388,17 +2373,6 @@
       </Expression>
       <OverallResult success="true"/>
     </TestCase>
-    <TestCase name="Operators at different namespace levels not hijacked by Koenig lookup" tags="[Tricky]" filename="projects/<exe-name>/TrickyTests.cpp" >
-      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/TrickyTests.cpp" >
-        <Original>
-          0x<hex digits> == o
-        </Original>
-        <Expanded>
-          3221225472 (0x<hex digits>) == {?}
-        </Expanded>
-      </Expression>
-      <OverallResult success="true"/>
-    </TestCase>
     <TestCase name="Ordering comparison checks that should fail" tags="[.][failing]" filename="projects/<exe-name>/ConditionTests.cpp" >
       <Expression success="false" type="CHECK" filename="projects/<exe-name>/ConditionTests.cpp" >
         <Original>
@@ -3962,15 +3936,6 @@
       </Expression>
       <OverallResult success="true"/>
     </TestCase>
-    <TestCase name="Pointers can be converted to strings" tags="[.][messages]" filename="projects/<exe-name>/MessageTests.cpp" >
-      <Warning>
-        actual address of p: 0x<hex digits>
-      </Warning>
-      <Warning>
-        toString(p): 0x<hex digits>
-      </Warning>
-      <OverallResult success="true"/>
-    </TestCase>
     <TestCase name="Process can be configured on command line" tags="[command-line][config]" filename="projects/<exe-name>/TestMain.cpp" >
       <Section name="empty args don't cause a crash" filename="projects/<exe-name>/TestMain.cpp" >
         <Expression success="true" type="CHECK" filename="projects/<exe-name>/TestMain.cpp" >
@@ -4824,7 +4789,7 @@ A string sent directly to stderr
       </Expression>
       <OverallResult success="true"/>
     </TestCase>
-    <TestCase name="Standard error is reported and redirected" tags="[.][messages]" filename="projects/<exe-name>/MessageTests.cpp" >
+    <TestCase name="Standard error is reported and redirected" tags="[.][.approvals][messages]" filename="projects/<exe-name>/MessageTests.cpp" >
       <Section name="std::cerr" filename="projects/<exe-name>/MessageTests.cpp" >
         <OverallResults successes="0" failures="1" expectedFailures="0"/>
       </Section>
@@ -8276,7 +8241,7 @@ loose text artifact
       </Section>
       <OverallResult success="true"/>
     </TestCase>
-    <OverallResults successes="760" failures="103" expectedFailures="21"/>
+    <OverallResults successes="758" failures="103" expectedFailures="21"/>
   </Group>
-  <OverallResults successes="760" failures="102" expectedFailures="21"/>
+  <OverallResults successes="758" failures="102" expectedFailures="21"/>
 </Catch>
diff --git a/projects/SelfTest/ConditionTests.cpp b/projects/SelfTest/ConditionTests.cpp
index 565d9815..00ff6060 100644
--- a/projects/SelfTest/ConditionTests.cpp
+++ b/projects/SelfTest/ConditionTests.cpp
@@ -13,6 +13,7 @@
 
 #include <string>
 #include <limits>
+#include <cstdint>
 
 struct TestData {
     int int_seven = 7;
@@ -184,7 +185,7 @@ TEST_CASE( "Comparisons with int literals don't warn when mixing signed/ unsigne
     REQUIRE( 5 == c );
     REQUIRE( 6 == uc );
 
-    REQUIRE( (std::numeric_limits<unsigned long>::max)() > ul );
+    REQUIRE( (std::numeric_limits<uint32_t>::max)() > ul );
 }
 
 // Disable warnings about sign conversions for the next two tests
diff --git a/projects/SelfTest/MessageTests.cpp b/projects/SelfTest/MessageTests.cpp
index 437ba53d..6082a4e6 100644
--- a/projects/SelfTest/MessageTests.cpp
+++ b/projects/SelfTest/MessageTests.cpp
@@ -85,7 +85,7 @@ TEST_CASE( "Standard output from all sections is reported", "[messages][.]" ) {
     }
 }
 
-TEST_CASE( "Standard error is reported and redirected", "[messages][.]" ) {
+TEST_CASE( "Standard error is reported and redirected", "[messages][.][approvals]" ) {
     SECTION( "std::cerr" ) {
         std::cerr << "Write to std::cerr" << std::endl;
     }
@@ -98,7 +98,7 @@ TEST_CASE( "Standard error is reported and redirected", "[messages][.]" ) {
         std::cerr << ' ';
         std::clog << "writes";
         std::cerr << " to error";
-        std::clog << " streams\n";
+        std::clog << " streams" << std::endl;
     }
 }
 
@@ -130,7 +130,7 @@ TEST_CASE( "sends information to INFO", "[.][failing]" ) {
     REQUIRE( false );
 }
 
-TEST_CASE( "Pointers can be converted to strings", "[messages][.]" ) {
+TEST_CASE( "Pointers can be converted to strings", "[messages][.][approvals]" ) {
     int p;
     WARN( "actual address of p: " << &p );
     WARN( "toString(p): " << ::Catch::Detail::stringify( &p ) );
diff --git a/projects/SelfTest/TrickyTests.cpp b/projects/SelfTest/TrickyTests.cpp
index f551d3bf..90abb4a8 100644
--- a/projects/SelfTest/TrickyTests.cpp
+++ b/projects/SelfTest/TrickyTests.cpp
@@ -175,7 +175,7 @@ namespace ObjectWithConversions
     TEST_CASE
     (
         "Operators at different namespace levels not hijacked by Koenig lookup",
-        "[Tricky]"
+        "[Tricky][approvals]"
     )
     {
         Object o;
@@ -211,9 +211,16 @@ namespace ObjectWithNonConstEqualityOperator
 
 namespace EnumBitFieldTests
 {
-    enum Bits {bit0 = 0x0001, bit1 = 0x0002, bit2 = 0x0004, bit3 = 0x0008, bit1and2 = 0x0006,
-        bit30 = 0x40000000, bit31 = 0x80000000,
-        bit30and31 = 0xc0000000};
+    enum Bits : uint32_t {
+        bit0 = 0x0001,
+        bit1 = 0x0002,
+        bit2 = 0x0004,
+        bit3 = 0x0008,
+        bit1and2 = bit1 | bit2,
+        bit30 = 0x40000000,
+        bit31 = 0x80000000,
+        bit30and31 = bit30 | bit31
+    };
 
     TEST_CASE( "Test enum bit values", "[Tricky]" )
     {
@@ -365,7 +372,7 @@ struct S
 };
 
 
-TEST_CASE( "Comparing member function pointers", "[Tricky][member function pointer]" )
+TEST_CASE( "Comparing member function pointers", "[Tricky][member function pointer][approvals]" )
 {
     typedef void (S::*MF)();
     MF m = &S::f;
diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py
index 49553349..5c0c63ec 100755
--- a/scripts/approvalTests.py
+++ b/scripts/approvalTests.py
@@ -41,6 +41,8 @@ errnoParser = re.compile(r'''
     \(\*__errno_location\ \(\)\)
     |
     \(\*__error\(\)\)
+    |
+    \(\*_errno\(\)\)
 ''', re.VERBOSE)
 
 if len(sys.argv) == 2:
@@ -145,15 +147,15 @@ print("Running approvals against executable:")
 print("  " + cmdPath)
 
 # Standard console reporter
-approve("console.std", ["~[!nonportable]~[!benchmark]", "--order", "lex"])
+approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals]", "--order", "lex"])
 # console reporter, include passes, warn about No Assertions
-approve("console.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "--order", "lex"])
+approve("console.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "--order", "lex"])
 # console reporter, include passes, warn about No Assertions, limit failures to first 4
-approve("console.swa4", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
+approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
 # junit reporter, include passes, warn about No Assertions
-approve("junit.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
+approve("junit.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
 # xml reporter, include passes, warn about No Assertions
-approve("xml.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
+approve("xml.sw", ["~[!nonportable]~[!benchmark]~[approvals]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
 
 if overallResult != 0:
     print("If these differences are expected, run approve.py to approve new baselines.")