Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

Fixed !shouldfail

Commit
447f53e9e3804e4baaafb61856b65984962ce1b8
Author
Phil Nash ·
Parents
13a887ad241f

Files · Download .tar.gz

Changes

 include/internal/catch_run_context.hpp             |   5 +
 include/reporters/catch_reporter_console.hpp       |   2 +-
 .../SelfTest/Baselines/console.std.approved.txt    |   4 +-
 .../SelfTest/Baselines/console.sw.approved.txt     |  15 ++-
 .../SelfTest/Baselines/console.swa4.approved.txt   | 128 ++++++++++++++++++++-
 projects/SelfTest/Baselines/junit.sw.approved.txt  |   3 +-
 projects/SelfTest/Baselines/xml.sw.approved.txt    |   9 +-
 projects/SelfTest/ConditionTests.cpp               |   2 +-
 projects/SelfTest/MiscTests.cpp                    |   5 +
 9 files changed, 161 insertions(+), 12 deletions(-)

diff --git a/include/internal/catch_run_context.hpp b/include/internal/catch_run_context.hpp
index da5990ad..d37bdba1 100644
--- a/include/internal/catch_run_context.hpp
+++ b/include/internal/catch_run_context.hpp
@@ -109,6 +109,11 @@ namespace Catch {
             while( getCurrentContext().advanceGeneratorsForCurrentTest() && !aborting() );
 
             Totals deltaTotals = m_totals.delta( prevTotals );
+            if( testInfo.expectedToFail() && deltaTotals.testCases.passed > 0 ) {
+                deltaTotals.assertions.failed++;
+                deltaTotals.testCases.passed--;
+                deltaTotals.testCases.failed++;
+            }
             m_totals.testCases += deltaTotals.testCases;
             m_reporter->testCaseEnded( TestCaseStats(   testInfo,
                                                         deltaTotals,
diff --git a/include/reporters/catch_reporter_console.hpp b/include/reporters/catch_reporter_console.hpp
index e7367039..5564d885 100644
--- a/include/reporters/catch_reporter_console.hpp
+++ b/include/reporters/catch_reporter_console.hpp
@@ -349,7 +349,7 @@ namespace Catch {
             if( totals.testCases.total() == 0 ) {
                 stream << Colour( Colour::Warning ) << "No tests ran\n";
             }
-            else if( totals.assertions.total() > 0 && totals.assertions.allPassed() ) {
+            else if( totals.assertions.total() > 0 && totals.testCases.allPassed() ) {
                 stream << Colour( Colour::ResultSuccess ) << "All tests passed";
                 stream << " ("
                         << pluralise( totals.assertions.passed, "assertion" ) << " in "
diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt
index 29870b93..ae45f32d 100644
--- a/projects/SelfTest/Baselines/console.std.approved.txt
+++ b/projects/SelfTest/Baselines/console.std.approved.txt
@@ -830,6 +830,6 @@ with expansion:
   "first" == "second"
 
 ===============================================================================
-test cases: 167 | 124 passed | 42 failed |  1 failed as expected
-assertions: 919 | 823 passed | 83 failed | 13 failed as expected
+test cases: 168 | 124 passed | 42 failed |  2 failed as expected
+assertions: 920 | 824 passed | 78 failed | 18 failed as expected
 
diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt
index cd73c2a7..e0a49030 100644
--- a/projects/SelfTest/Baselines/console.sw.approved.txt
+++ b/projects/SelfTest/Baselines/console.sw.approved.txt
@@ -3951,6 +3951,17 @@ with expansion:
   ==
   9223372036854775807 (0x<hex digits>)
 
+-------------------------------------------------------------------------------
+This test 'should' fail but doesn't
+-------------------------------------------------------------------------------
+MiscTests.cpp:<line number>
+...............................................................................
+
+MiscTests.cpp:<line number>:
+PASSED:
+with message:
+  oops!
+
 -------------------------------------------------------------------------------
 Process can be configured on command line
   default - no arguments
@@ -9093,6 +9104,6 @@ with expansion:
   1 > 0
 
 ===============================================================================
-test cases: 167 | 123 passed | 43 failed |  1 failed as expected
-assertions: 921 | 823 passed | 85 failed | 13 failed as expected
+test cases: 168 | 123 passed | 43 failed |  2 failed as expected
+assertions: 922 | 824 passed | 80 failed | 18 failed as expected
 
diff --git a/projects/SelfTest/Baselines/console.swa4.approved.txt b/projects/SelfTest/Baselines/console.swa4.approved.txt
index 0793f1d1..76af9f55 100644
--- a/projects/SelfTest/Baselines/console.swa4.approved.txt
+++ b/projects/SelfTest/Baselines/console.swa4.approved.txt
@@ -485,7 +485,131 @@ ConditionTests.cpp:<line number>: FAILED:
 with expansion:
   9.1f != Approx( 9.1000003815 )
 
+-------------------------------------------------------------------------------
+Ordering comparison checks that should succeed
+-------------------------------------------------------------------------------
+ConditionTests.cpp:<line number>
+...............................................................................
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.int_seven < 8 )
+with expansion:
+  7 < 8
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.int_seven > 6 )
+with expansion:
+  7 > 6
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.int_seven > 0 )
+with expansion:
+  7 > 0
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.int_seven > -1 )
+with expansion:
+  7 > -1
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.int_seven >= 7 )
+with expansion:
+  7 >= 7
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.int_seven >= 6 )
+with expansion:
+  7 >= 6
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.int_seven <= 7 )
+with expansion:
+  7 <= 7
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.int_seven <= 8 )
+with expansion:
+  7 <= 8
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.float_nine_point_one > 9 )
+with expansion:
+  9.1f > 9
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.float_nine_point_one < 10 )
+with expansion:
+  9.1f < 10
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.float_nine_point_one < 9.2 )
+with expansion:
+  9.1f < 9.2
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.str_hello <= "hello" )
+with expansion:
+  "hello" <= "hello"
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.str_hello >= "hello" )
+with expansion:
+  "hello" >= "hello"
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.str_hello < "hellp" )
+with expansion:
+  "hello" < "hellp"
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.str_hello < "zebra" )
+with expansion:
+  "hello" < "zebra"
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.str_hello > "hellm" )
+with expansion:
+  "hello" > "hellm"
+
+ConditionTests.cpp:<line number>:
+PASSED:
+  REQUIRE( data.str_hello > "a" )
+with expansion:
+  "hello" > "a"
+
+-------------------------------------------------------------------------------
+Ordering comparison checks that should fail
+-------------------------------------------------------------------------------
+ConditionTests.cpp:<line number>
+...............................................................................
+
+ConditionTests.cpp:<line number>: FAILED:
+  CHECK( data.int_seven > 7 )
+with expansion:
+  7 > 7
+
+ConditionTests.cpp:<line number>: FAILED:
+  CHECK( data.int_seven < 7 )
+with expansion:
+  7 < 7
+
 ===============================================================================
-test cases: 19 | 15 passed | 3 failed | 1 failed as expected
-assertions: 62 | 56 passed | 4 failed | 2 failed as expected
+test cases: 21 | 16 passed | 3 failed | 2 failed as expected
+assertions: 81 | 73 passed | 4 failed | 4 failed as expected
 
diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt
index 110ee8cc..e3268bc6 100644
--- a/projects/SelfTest/Baselines/junit.sw.approved.txt
+++ b/projects/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,5 +1,5 @@
 <testsuites>
-  <testsuite name="CatchSelfTest" errors="13" failures="72" tests="921" hostname="tbd" time="{duration}" timestamp="tbd">
+  <testsuite name="CatchSelfTest" errors="13" failures="68" tests="923" hostname="tbd" time="{duration}" timestamp="tbd">
     <testcase classname="global" name="toString(enum)" time="{duration}"/>
     <testcase classname="global" name="toString(enum w/operator&lt;&lt;)" time="{duration}"/>
     <testcase classname="global" name="toString(enum class)" time="{duration}"/>
@@ -499,6 +499,7 @@ MiscTests.cpp:<line number>
     <testcase classname="XmlEncode" name="string with control char (1)" time="{duration}"/>
     <testcase classname="XmlEncode" name="string with control char (x7F)" time="{duration}"/>
     <testcase classname="global" name="long long" time="{duration}"/>
+    <testcase classname="global" name="This test 'should' fail but doesn't" time="{duration}"/>
     <testcase classname="Process can be configured on command line" name="default - no arguments" time="{duration}"/>
     <testcase classname="Process can be configured on command line" name="test lists/1 test" time="{duration}"/>
     <testcase classname="Process can be configured on command line" name="test lists/Specify one test case exclusion using exclude:" time="{duration}"/>
diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt
index c0af02fb..be30de52 100644
--- a/projects/SelfTest/Baselines/xml.sw.approved.txt
+++ b/projects/SelfTest/Baselines/xml.sw.approved.txt
@@ -665,7 +665,7 @@
           5 != 5
         </Expanded>
       </Expression>
-      <OverallResult success="false"/>
+      <OverallResult success="true"/>
     </TestCase>
     <TestCase name="Ordering comparison checks that should succeed">
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
@@ -4085,6 +4085,9 @@
       </Expression>
       <OverallResult success="true"/>
     </TestCase>
+    <TestCase name="This test 'should' fail but doesn't">
+      <OverallResult success="false"/>
+    </TestCase>
     <TestCase name="Process can be configured on command line">
       <Section name="default - no arguments">
         <Expression success="true" type="CHECK_NOTHROW" filename="projects/SelfTest/TestMain.cpp" >
@@ -9563,7 +9566,7 @@ there"
       </Section>
       <OverallResult success="true"/>
     </TestCase>
-    <OverallResults successes="823" failures="85" expectedFailures="13"/>
+    <OverallResults successes="824" failures="81" expectedFailures="18"/>
   </Group>
-  <OverallResults successes="823" failures="85" expectedFailures="13"/>
+  <OverallResults successes="824" failures="80" expectedFailures="18"/>
 </Catch>
diff --git a/projects/SelfTest/ConditionTests.cpp b/projects/SelfTest/ConditionTests.cpp
index d1b7ed24..b1d29106 100644
--- a/projects/SelfTest/ConditionTests.cpp
+++ b/projects/SelfTest/ConditionTests.cpp
@@ -101,7 +101,7 @@ TEST_CASE( "Inequality checks that should succeed", "" )
     REQUIRE( data.str_hello.size() != 6 );
 }
 
-TEST_CASE( "Inequality checks that should fail", "[.][failing]" )
+TEST_CASE( "Inequality checks that should fail", "[.][failing][!shouldfail]" )
 {
     TestData data;
 
diff --git a/projects/SelfTest/MiscTests.cpp b/projects/SelfTest/MiscTests.cpp
index cf7f48cf..9a2a2ab6 100644
--- a/projects/SelfTest/MiscTests.cpp
+++ b/projects/SelfTest/MiscTests.cpp
@@ -478,3 +478,8 @@ TEST_CASE( "long long" ) {
 //    int x = 10/i; // This should cause the signal to fire
 //    CHECK( x == 0 );
 //}
+
+TEST_CASE( "This test 'should' fail but doesn't", "[.][failing][!shouldfail]" )
+{
+    SUCCEED( "oops!" );
+}