catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
Removed benchmark tests from approval tests (for now)
- Commit
4421672fb869318d3445a80deb5de09761f8a119- Author
- Phil Nash ·
- Parents
f45d35c9804b
Changes
include/internal/catch_test_case_info.cpp | 2 + include/internal/catch_test_case_info.h | 3 +- .../SelfTest/Baselines/console.std.approved.txt | 20 +------ .../SelfTest/Baselines/console.sw.approved.txt | 65 +--------------------- projects/SelfTest/Baselines/junit.sw.approved.txt | 3 +- projects/SelfTest/Baselines/xml.sw.approved.txt | 39 +------------ projects/SelfTest/Benchmark.tests.cpp | 2 +- scripts/approvalTests.py | 10 ++-- 8 files changed, 17 insertions(+), 127 deletions(-) diff --git a/include/internal/catch_test_case_info.cpp b/include/internal/catch_test_case_info.cpp index 5a8d8c0f..9c707307 100644 --- a/include/internal/catch_test_case_info.cpp +++ b/include/internal/catch_test_case_info.cpp @@ -31,6 +31,8 @@ namespace Catch { return TestCaseInfo::MayFail; else if( tag == "!nonportable" ) return TestCaseInfo::NonPortable; + else if( tag == "!benchmark" ) + return static_cast<TestCaseInfo::SpecialProperties>( TestCaseInfo::Benchmark | TestCaseInfo::IsHidden ); else return TestCaseInfo::None; } diff --git a/include/internal/catch_test_case_info.h b/include/internal/catch_test_case_info.h index 626f95cb..2a911b06 100644 --- a/include/internal/catch_test_case_info.h +++ b/include/internal/catch_test_case_info.h @@ -30,7 +30,8 @@ namespace Catch { ShouldFail = 1 << 2, MayFail = 1 << 3, Throws = 1 << 4, - NonPortable = 1 << 5 + NonPortable = 1 << 5, + Benchmark = 1 << 6 }; TestCaseInfo( std::string const& _name, diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt index acd58b94..9e41f7f5 100644 --- a/projects/SelfTest/Baselines/console.std.approved.txt +++ b/projects/SelfTest/Baselines/console.std.approved.txt @@ -850,22 +850,6 @@ warning: Uncomment the code in this test to check that it gives a sensible compiler error -------------------------------------------------------------------------------- -benchmarked -------------------------------------------------------------------------------- -String.tests.cpp:<line number> -............................................................................... - -+-----------------------------------------------------------------------------+ -| benchmark name | iters | elapsed ns | average | -+-----------------------------------------------------------------------------+ - -| Load up a vector | 1 | 8459 | 8459 ns | -| Load up a map | 1 | 28774 | 28774 ns | -| Reserved vector | 1 | 2430 | 2430 ns | -| A fixed size array that should require | | | | -| no allocations | 10 | 2563 | 256 ns | -+-----------------------------------------------------------------------------+ ------------------------------------------------------------------------------- checkedElse, failing ------------------------------------------------------------------------------- @@ -1032,6 +1016,6 @@ with expansion: "{?}" == "1" =============================================================================== -test cases: 183 | 132 passed | 47 failed | 4 failed as expected -assertions: 900 | 783 passed | 96 failed | 21 failed as expected +test cases: 182 | 131 passed | 47 failed | 4 failed as expected +assertions: 896 | 779 passed | 96 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index 7f4861d4..971c0e32 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -6381,67 +6381,6 @@ PASSED: with expansion: 0 == 0 -------------------------------------------------------------------------------- -benchmarked -------------------------------------------------------------------------------- -String.tests.cpp:<line number> -............................................................................... - -+-----------------------------------------------------------------------------+ -| benchmark name | iters | elapsed ns | average | -+-----------------------------------------------------------------------------+ - -| Load up a vector | 1 | 6957 | 6957 ns | -+-----------------------------------------------------------------------------+ -String.tests.cpp:<line number>: -PASSED: - REQUIRE( v.size() == size ) -with expansion: - 100 == 100 - - -+-----------------------------------------------------------------------------+ - -| benchmark name | iters | elapsed ns | average | -+-----------------------------------------------------------------------------+ - -| Load up a map | 1 | 27868 | 27868 ns | -+-----------------------------------------------------------------------------+ -String.tests.cpp:<line number>: -PASSED: - REQUIRE( m.size() == size ) -with expansion: - 100 == 100 - - -+-----------------------------------------------------------------------------+ - -| benchmark name | iters | elapsed ns | average | -+-----------------------------------------------------------------------------+ - -| Reserved vector | 1 | 2711 | 2711 ns | -+-----------------------------------------------------------------------------+ -String.tests.cpp:<line number>: -PASSED: - REQUIRE( v.size() == size ) -with expansion: - 100 == 100 - - -+-----------------------------------------------------------------------------+ - -| benchmark name | iters | elapsed ns | average | -+-----------------------------------------------------------------------------+ - -| A fixed size array that should require | | | | -| no allocations | 1 | 530 | 530 ns | -+-----------------------------------------------------------------------------+ -String.tests.cpp:<line number>: -PASSED: - REQUIRE( sum > size ) -with expansion: - 4950 (0x<hex digits>) > 100 - ------------------------------------------------------------------------------- boolean member ------------------------------------------------------------------------------- @@ -7651,6 +7590,6 @@ MiscTests.cpp:<line number>: PASSED: =============================================================================== -test cases: 183 | 131 passed | 48 failed | 4 failed as expected -assertions: 902 | 783 passed | 98 failed | 21 failed as expected +test cases: 182 | 130 passed | 48 failed | 4 failed as expected +assertions: 898 | 779 passed | 98 failed | 21 failed as expected diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 31ad3c5a..58807c18 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <testsuitesspanner> - <testsuite name="<exe-name>" errors="15" failures="84" tests="903" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> + <testsuite name="<exe-name>" errors="15" failures="84" tests="899" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> <testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/> <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}"> <error type="TEST_CASE"> @@ -641,7 +641,6 @@ ExceptionTests.cpp:<line number> <testcase classname="<exe-name>.global" name="XmlEncode/string with control char (1)" time="{duration}"/> <testcase classname="<exe-name>.global" name="XmlEncode/string with control char (x7F)" time="{duration}"/> <testcase classname="<exe-name>.global" name="atomic if" time="{duration}"/> - <testcase classname="<exe-name>.global" name="benchmarked" time="{duration}"/> <testcase classname="<exe-name>.global" name="boolean member" time="{duration}"/> <testcase classname="<exe-name>.global" name="checkedElse" time="{duration}"/> <testcase classname="<exe-name>.global" name="checkedElse, failing" time="{duration}"> diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index 01ec013c..e70ad12e 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -7201,41 +7201,6 @@ Message from section two </Expression> <OverallResult success="true"/> </TestCase> - <TestCase name="benchmarked" tags="[benchmark]" filename="projects/<exe-name>/String.tests.cpp" > - <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" > - <Original> - v.size() == size - </Original> - <Expanded> - 100 == 100 - </Expanded> - </Expression> - <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" > - <Original> - m.size() == size - </Original> - <Expanded> - 100 == 100 - </Expanded> - </Expression> - <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" > - <Original> - v.size() == size - </Original> - <Expanded> - 100 == 100 - </Expanded> - </Expression> - <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/String.tests.cpp" > - <Original> - sum > size - </Original> - <Expanded> - 4950 (0x<hex digits>) > 100 - </Expanded> - </Expression> - <OverallResult success="true"/> - </TestCase> <TestCase name="boolean member" tags="[Tricky]" filename="projects/<exe-name>/TrickyTests.cpp" > <Expression success="true" type="REQUIRE" filename="projects/<exe-name>/TrickyTests.cpp" > <Original> @@ -8476,7 +8441,7 @@ spanner <OverallResult success="true"/> </Section> <OverallResult success="true"/> </TestCase> - <OverallResults successes="783" failures="99" expectedFailures="21"/> + <OverallResults successes="779" failures="99" expectedFailures="21"/> </Group> - <OverallResults successes="783" failures="98" expectedFailures="21"/> + <OverallResults successes="779" failures="98" expectedFailures="21"/> </Catch> diff --git a/projects/SelfTest/Benchmark.tests.cpp b/projects/SelfTest/Benchmark.tests.cpp index ff1ec479..ddf69504 100644 --- a/projects/SelfTest/Benchmark.tests.cpp +++ b/projects/SelfTest/Benchmark.tests.cpp @@ -2,7 +2,7 @@ #include <map> -TEST_CASE( "benchmarked", "[.][benchmark]" ) { +TEST_CASE( "benchmarked", "[!benchmark]" ) { static const int size = 100; diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py index 016997cc..49553349 100755 --- a/scripts/approvalTests.py +++ b/scripts/approvalTests.py @@ -145,15 +145,15 @@ print("Running approvals against executable:") print(" " + cmdPath) # Standard console reporter -approve("console.std", ["~[!nonportable]", "--order", "lex"]) +approve("console.std", ["~[!nonportable]~[!benchmark]", "--order", "lex"]) # console reporter, include passes, warn about No Assertions -approve("console.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "--order", "lex"]) +approve("console.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "--order", "lex"]) # console reporter, include passes, warn about No Assertions, limit failures to first 4 -approve("console.swa4", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"]) +approve("console.swa4", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"]) # junit reporter, include passes, warn about No Assertions -approve("junit.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"]) +approve("junit.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"]) # xml reporter, include passes, warn about No Assertions -approve("xml.sw", ["~[!nonportable]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"]) +approve("xml.sw", ["~[!nonportable]~[!benchmark]", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"]) if overallResult != 0: print("If these differences are expected, run approve.py to approve new baselines.")