Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

Removed benchmark tests from approval tests (for now)

Commit
4421672fb869318d3445a80deb5de09761f8a119
Author
Phil Nash ·
Parents
f45d35c9804b

Files · Download .tar.gz

Changes

 include/internal/catch_test_case_info.cpp          |  2 +
 include/internal/catch_test_case_info.h            |  3 +-
 .../SelfTest/Baselines/console.std.approved.txt    | 20 +------
 .../SelfTest/Baselines/console.sw.approved.txt     | 65 +---------------------
 projects/SelfTest/Baselines/junit.sw.approved.txt  |  3 +-
 projects/SelfTest/Baselines/xml.sw.approved.txt    | 39 +------------
 projects/SelfTest/Benchmark.tests.cpp              |  2 +-
 scripts/approvalTests.py                           | 10 ++--
 8 files changed, 17 insertions(+), 127 deletions(-)

diff --git a/include/internal/catch_test_case_info.cpp b/include/internal/catch_test_case_info.cpp
index 5a8d8c0f..9c707307 100644
--- a/include/internal/catch_test_case_info.cpp
+++ b/include/internal/catch_test_case_info.cpp
@@ -31,6 +31,8 @@ namespace Catch {
             return TestCaseInfo::MayFail;
         else if( tag == "!nonportable" )
             return TestCaseInfo::NonPortable;
+        else if( tag == "!benchmark" )
+            return static_cast<TestCaseInfo::SpecialProperties>( TestCaseInfo::Benchmark | TestCaseInfo::IsHidden );
         else
             return TestCaseInfo::None;
     }
diff --git a/include/internal/catch_test_case_info.h b/include/internal/catch_test_case_info.h
index 626f95cb..2a911b06 100644
--- a/include/internal/catch_test_case_info.h
+++ b/include/internal/catch_test_case_info.h
@@ -30,7 +30,8 @@ namespace Catch {
             ShouldFail = 1 << 2,
             MayFail = 1 << 3,
             Throws = 1 << 4,
-            NonPortable = 1 << 5
+            NonPortable = 1 << 5,
+            Benchmark = 1 << 6
         };
 
         TestCaseInfo(   std::string const& _name,
diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt
index acd58b94..9e41f7f5 100644
--- a/projects/SelfTest/Baselines/console.std.approved.txt
+++ b/projects/SelfTest/Baselines/console.std.approved.txt
@@ -850,22 +850,6 @@ warning:
   Uncomment the code in this test to check that it gives a sensible compiler
   error
 
--------------------------------------------------------------------------------
-benchmarked
--------------------------------------------------------------------------------
-String.tests.cpp:<line number>
-...............................................................................
-
-+-----------------------------------------------------------------------------+
-| benchmark name                           |  iters | elapsed ns |    average |
-+-----------------------------------------------------------------------------+
-
-| Load up a vector                         |      1 |       8459 |    8459 ns |
-| Load up a map                            |      1 |      28774 |   28774 ns |
-| Reserved vector                          |      1 |       2430 |    2430 ns |
-| A fixed size array that should require   |        |            |            |
-| no allocations                           |     10 |       2563 |     256 ns |
-+-----------------------------------------------------------------------------+
 -------------------------------------------------------------------------------
 checkedElse, failing
 -------------------------------------------------------------------------------
@@ -1032,6 +1016,6 @@ with expansion:
   "{?}" == "1"
 
 ===============================================================================
-test cases: 183 | 132 passed | 47 failed |  4 failed as expected
-assertions: 900 | 783 passed | 96 failed | 21 failed as expected
+test cases: 182 | 131 passed | 47 failed |  4 failed as expected
+assertions: 896 | 779 passed | 96 failed | 21 failed as expected
 
diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt
index 7f4861d4..971c0e32 100644
--- a/projects/SelfTest/Baselines/console.sw.approved.txt
+++ b/projects/SelfTest/Baselines/console.sw.approved.txt
@@ -6381,67 +6381,6 @@ PASSED:
 with expansion:
   0 == 0
 
--------------------------------------------------------------------------------
-benchmarked
--------------------------------------------------------------------------------
-String.tests.cpp:<line number>
-...............................................................................
-
-+-----------------------------------------------------------------------------+
-| benchmark name                           |  iters | elapsed ns |    average |
-+-----------------------------------------------------------------------------+
-
-| Load up a vector                         |      1 |       6957 |    6957 ns |
-+-----------------------------------------------------------------------------+
-String.tests.cpp:<line number>:
-PASSED:
-  REQUIRE( v.size() == size )
-with expansion:
-  100 == 100
-
-
-+-----------------------------------------------------------------------------+
-
-| benchmark name                           |  iters | elapsed ns |    average |
-+-----------------------------------------------------------------------------+
-
-| Load up a map                            |      1 |      27868 |   27868 ns |
-+-----------------------------------------------------------------------------+
-String.tests.cpp:<line number>:
-PASSED:
-  REQUIRE( m.size() == size )
-with expansion:
-  100 == 100
-
-
-+-----------------------------------------------------------------------------+
-
-| benchmark name                           |  iters | elapsed ns |    average |
-+-----------------------------------------------------------------------------+
-
-| Reserved vector                          |      1 |       2711 |    2711 ns |
-+-----------------------------------------------------------------------------+
-String.tests.cpp:<line number>:
-PASSED:
-  REQUIRE( v.size() == size )
-with expansion:
-  100 == 100
-
-
-+-----------------------------------------------------------------------------+
-
-| benchmark name                           |  iters | elapsed ns |    average |
-+-----------------------------------------------------------------------------+
-
-| A fixed size array that should require   |        |            |            |
-| no allocations                           |      1 |        530 |     530 ns |
-+-----------------------------------------------------------------------------+
-String.tests.cpp:<line number>:
-PASSED:
-  REQUIRE( sum > size )
-with expansion:
-  4950 (0x<hex digits>) > 100
-
 -------------------------------------------------------------------------------
 boolean member
 -------------------------------------------------------------------------------
@@ -7651,6 +7590,6 @@ MiscTests.cpp:<line number>:
 PASSED:
 
 ===============================================================================
-test cases: 183 | 131 passed | 48 failed |  4 failed as expected
-assertions: 902 | 783 passed | 98 failed | 21 failed as expected
+test cases: 182 | 130 passed | 48 failed |  4 failed as expected
+assertions: 898 | 779 passed | 98 failed | 21 failed as expected
 
diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt
index 31ad3c5a..58807c18 100644
--- a/projects/SelfTest/Baselines/junit.sw.approved.txt
+++ b/projects/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <testsuitesspanner>
-  <testsuite name="<exe-name>" errors="15" failures="84" tests="903" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+  <testsuite name="<exe-name>" errors="15" failures="84" tests="899" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
     <testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}">
       <error type="TEST_CASE">
@@ -641,7 +641,6 @@ ExceptionTests.cpp:<line number>
     <testcase classname="<exe-name>.global" name="XmlEncode/string with control char (1)" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="XmlEncode/string with control char (x7F)" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="atomic if" time="{duration}"/>
-    <testcase classname="<exe-name>.global" name="benchmarked" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="boolean member" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="checkedElse" time="{duration}"/>
     <testcase classname="<exe-name>.global" name="checkedElse, failing" time="{duration}">
diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt
index 01ec013c..e70ad12e 100644
--- a/projects/SelfTest/Baselines/xml.sw.approved.txt
+++ b/projects/SelfTest/Baselines/xml.sw.approved.txt
@@ -7201,41 +7201,6 @@ Message from section two
       </Expression>
       <OverallResult success="true"/>
     </TestCase>
-    <TestCase name="benchmarked" tags="[benchmark]" filename="projects/<exe-name>/String.tests.cpp" >
-      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
-        <Original>
-          v.size() == size
-        </Original>
-        <Expanded>
-          100 == 100
-        </Expanded>
-      </Expression>
-      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
-        <Original>
-          m.size() == size
-        </Original>
-        <Expanded>
-          100 == 100
-        </Expanded>
-      </Expression>
-      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
-        <Original>
-          v.size() == size
-        </Original>
-        <Expanded>
-          100 == 100
-        </Expanded>
-      </Expression>
-      <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" >
-        <Original>
-          sum > size
-        </Original>
-        <Expanded>
-          4950 (0x<hex digits>) > 100
-        </Expanded>
-      </Expression>
-      <OverallResult success="true"/>
-    </TestCase>
     <TestCase name="boolean member" tags="[Tricky]" filename="projects/<exe-name>/TrickyTests.cpp" >
       <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/TrickyTests.cpp" >
         <Original>
@@ -8476,7 +8441,7 @@ spanner      <OverallResult success="true"/>
       </Section>
       <OverallResult success="true"/>
     </TestCase>
-    <OverallResults successes="783" failures="99" expectedFailures="21"/>
+    <OverallResults successes="779" failures="99" expectedFailures="21"/>
   </Group>
-  <OverallResults successes="783" failures="98" expectedFailures="21"/>
+  <OverallResults successes="779" failures="98" expectedFailures="21"/>
 </Catch>
diff --git a/projects/SelfTest/Benchmark.tests.cpp b/projects/SelfTest/Benchmark.tests.cpp
index ff1ec479..ddf69504 100644
--- a/projects/SelfTest/Benchmark.tests.cpp
+++ b/projects/SelfTest/Benchmark.tests.cpp
@@ -2,7 +2,7 @@
 
 #include <map>
 
-TEST_CASE( "benchmarked", "[.][benchmark]" ) {
+TEST_CASE( "benchmarked", "[!benchmark]" ) {
 
     static const int size = 100;
 
diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py
index 016997cc..49553349 100755
--- a/scripts/approvalTests.py
+++ b/scripts/approvalTests.py
@@ -145,15 +145,15 @@ print("Running approvals against executable:")
 print("  " + cmdPath)
 
 # Standard console reporter
-approve("console.std", ["~[!nonportable]", "--order", "lex"])
+approve("console.std", ["~[!nonportable]~[!benchmark]", "--order", "lex"])
 # console reporter, include passes, warn about No Assertions
-approve("console.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "--order", "lex"])
+approve("console.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "--order", "lex"])
 # console reporter, include passes, warn about No Assertions, limit failures to first 4
-approve("console.swa4", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
+approve("console.swa4", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
 # junit reporter, include passes, warn about No Assertions
-approve("junit.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
+approve("junit.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
 # xml reporter, include passes, warn about No Assertions
-approve("xml.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
+approve("xml.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
 
 if overallResult != 0:
     print("If these differences are expected, run approve.py to approve new baselines.")