Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/projects/SelfTest
diff options
context:
space:
mode:
authorMartin Hořeňovský2019-06-22 20:26:03 +0200
committerMartin Hořeňovský2019-10-29 14:07:18 +0100
commit4f47d1c6c1bf21e047fc4dd8a946740a2da2f673 (patch)
tree3a82692b0821ab44cf69f54f40d5e0aaa7731cb9 /projects/SelfTest
parent2bcf1b3db6e1a61328f1efe093d749ef40e0bbd6 (diff)
downloadcatch3-4f47d1c6c1bf21e047fc4dd8a946740a2da2f673.tar.gz
Hidden tests now require positive filter match to be selected
This also required some refactoring of how the pattern matching works. This means that the concepts of include and exclude patterns are no longer unified, with exclusion patterns working as just negation of an inclusion patterns (which led to including hidden tags by default, as they did not match the exclusion), but rather both include and exclude patterns are handled separately. The new logic is that given a filter and a test case, the test case must match _all_ include patterns and _no_ exclude patterns to be included by the filter. Furthermore, if the test case is hidden, then the filter must have at least one include pattern for the test case to be used. Closes #1184
Diffstat (limited to 'projects/SelfTest')
-rw-r--r--projects/SelfTest/Baselines/compact.sw.approved.txt4
-rw-r--r--projects/SelfTest/Baselines/console.std.approved.txt2
-rw-r--r--projects/SelfTest/Baselines/console.sw.approved.txt10
-rw-r--r--projects/SelfTest/Baselines/console.swa4.approved.txt2
-rw-r--r--projects/SelfTest/Baselines/junit.sw.approved.txt2
-rw-r--r--projects/SelfTest/Baselines/xml.sw.approved.txt10
-rw-r--r--projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp6
7 files changed, 18 insertions, 18 deletions
diff --git a/projects/SelfTest/Baselines/compact.sw.approved.txt b/projects/SelfTest/Baselines/compact.sw.approved.txt
index a65f15ab..bf3d16c6 100644
--- a/projects/SelfTest/Baselines/compact.sw.approved.txt
+++ b/projects/SelfTest/Baselines/compact.sw.approved.txt
@@ -962,7 +962,7 @@ CmdLine.tests.cpp:<line number>: passed: spec.matches( tcD ) == false for: false
CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcB ) == false for: false == false
-CmdLine.tests.cpp:<line number>: passed: spec.matches( tcC ) == true for: true == true
+CmdLine.tests.cpp:<line number>: passed: spec.matches( tcC ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcB ) == true for: true == true
@@ -980,7 +980,7 @@ CmdLine.tests.cpp:<line number>: passed: spec.matches( tcD ) == true for: true =
CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcB ) == false for: false == false
-CmdLine.tests.cpp:<line number>: passed: spec.matches( tcC ) == true for: true == true
+CmdLine.tests.cpp:<line number>: passed: spec.matches( tcC ) == false for: false == false
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcD ) == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.hasFilters() == true for: true == true
CmdLine.tests.cpp:<line number>: passed: spec.matches( tcA ) == true for: true == true
diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt
index d43063fe..2fb24b6f 100644
--- a/projects/SelfTest/Baselines/console.std.approved.txt
+++ b/projects/SelfTest/Baselines/console.std.approved.txt
@@ -1,4 +1,4 @@
-Filters: ~[!nonportable]~[!benchmark]~[approvals]
+Filters: ~[!nonportable]~[!benchmark]~[approvals] *
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt
index d321bd91..b7167e98 100644
--- a/projects/SelfTest/Baselines/console.sw.approved.txt
+++ b/projects/SelfTest/Baselines/console.sw.approved.txt
@@ -1,4 +1,4 @@
-Filters: ~[!nonportable]~[!benchmark]~[approvals]
+Filters: ~[!nonportable]~[!benchmark]~[approvals] *
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
@@ -6946,9 +6946,9 @@ with expansion:
false == false
CmdLine.tests.cpp:<line number>: PASSED:
- CHECK( spec.matches( tcC ) == true )
+ CHECK( spec.matches( tcC ) == false )
with expansion:
- true == true
+ false == false
-------------------------------------------------------------------------------
Parse test names and tags
@@ -7064,9 +7064,9 @@ with expansion:
false == false
CmdLine.tests.cpp:<line number>: PASSED:
- CHECK( spec.matches( tcC ) == true )
+ CHECK( spec.matches( tcC ) == false )
with expansion:
- true == true
+ false == false
CmdLine.tests.cpp:<line number>: PASSED:
CHECK( spec.matches( tcD ) == true )
diff --git a/projects/SelfTest/Baselines/console.swa4.approved.txt b/projects/SelfTest/Baselines/console.swa4.approved.txt
index 322b8158..0c1ab7b7 100644
--- a/projects/SelfTest/Baselines/console.swa4.approved.txt
+++ b/projects/SelfTest/Baselines/console.swa4.approved.txt
@@ -1,4 +1,4 @@
-Filters: ~[!nonportable]~[!benchmark]~[approvals]
+Filters: ~[!nonportable]~[!benchmark]~[approvals] *
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt
index 588c3a21..814d05ee 100644
--- a/projects/SelfTest/Baselines/junit.sw.approved.txt
+++ b/projects/SelfTest/Baselines/junit.sw.approved.txt
@@ -3,7 +3,7 @@
>
<testsuite name="<exe-name>" errors="17" failures="132" tests="1713" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
- <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/>
+ <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals] *"/>
<property name="random-seed" value="1"/>
</properties>
<testcase classname="<exe-name>.global" name="# A test name that starts with a #" time="{duration}"/>
diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt
index 35640250..8c95f95b 100644
--- a/projects/SelfTest/Baselines/xml.sw.approved.txt
+++ b/projects/SelfTest/Baselines/xml.sw.approved.txt
@@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
-<Catch name="<exe-name>" filters="~[!nonportable]~[!benchmark]~[approvals]">
+<Catch name="<exe-name>" filters="~[!nonportable]~[!benchmark]~[approvals] *">
<Randomness seed="1"/>
<Group name="<exe-name>">
<TestCase name="# A test name that starts with a #" filename="projects/<exe-name>/UsageTests/Misc.tests.cpp" >
@@ -8720,10 +8720,10 @@ Nor would this
</Expression>
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
- spec.matches( tcC ) == true
+ spec.matches( tcC ) == false
</Original>
<Expanded>
- true == true
+ false == false
</Expanded>
</Expression>
<OverallResults successes="4" failures="0" expectedFailures="0"/>
@@ -8876,10 +8876,10 @@ Nor would this
</Expression>
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
- spec.matches( tcC ) == true
+ spec.matches( tcC ) == false
</Original>
<Expanded>
- true == true
+ false == false
</Expanded>
</Expression>
<Expression success="true" type="CHECK" filename="projects/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
diff --git a/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
index 60eb97af..8fccc0ab 100644
--- a/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
+++ b/projects/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
@@ -173,7 +173,7 @@ TEST_CASE( "Parse test names and tags" ) {
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == false );
- CHECK( spec.matches( tcC ) == true );
+ CHECK( spec.matches( tcC ) == false );
}
SECTION( "One tag exclusion and one tag inclusion" ) {
TestSpec spec = parseTestSpec( "~[two][x]" );
@@ -203,7 +203,7 @@ TEST_CASE( "Parse test names and tags" ) {
CHECK( spec.hasFilters() == true );
CHECK( spec.matches( tcA ) == true );
CHECK( spec.matches( tcB ) == false );
- CHECK( spec.matches( tcC ) == true );
+ CHECK( spec.matches( tcC ) == false );
CHECK( spec.matches( tcD ) == true );
}
SECTION( "wildcarded name exclusion" ) {
@@ -486,7 +486,7 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
REQUIRE(config.benchmarkSamples == 200);
}
-
+
SECTION("resamples") {
CHECK(cli.parse({ "test", "--benchmark-resamples=20000" }));