Support filtering on both sections and generators (#3069)

Not being able to filter generators to specific element has been regularly
causing problems. It was possible to use a dynamic section to run tests
for specific element in a generator, at least if the element had a nice
string representation, but the test case would still run once per element
in the generator.

With this change, it is possible to have the generator return only one
specific element, and do so based on the index, rather than the string
representation of the element. This enables simple debugging of tests
that fail for specific generator element.

1 files changed, 77 insertions, 0 deletions

diff --git a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
index c797ce5c..154d8ed9 100644
--- a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
+++ b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
@@ -483,3 +483,80 @@ TEST_CASE( "Parse rng seed in different formats", "[approvals][cli][rng-seed]" )
         REQUIRE_FALSE( cli.parse( { "tests", "--rng-seed", seed_string } ) );
     }
 }
+
+TEST_CASE( "Parsing path filter specs",
+           "[cli][section-spec]" ) {
+    using Catch::PathFilter;
+
+    Catch::ConfigData config;
+    auto cli = Catch::makeCommandLineParser( config );
+    SECTION( "Only section specs leads to old filter behaviour" ) {
+        auto result = cli.parse( { "tests", "-c", "1", "--section", "a section" } );
+        REQUIRE( result );
+        REQUIRE_FALSE( config.useNewPathFilteringBehaviour );
+    }
+    SECTION( "Generator specs enable new filter behaviour" ) {
+        auto result =
+            cli.parse( { "tests", "-g", "1", "--generator-index", "2" } );
+        REQUIRE( result );
+        REQUIRE( config.useNewPathFilteringBehaviour );
+    }
+    SECTION("Generator specs do not accept stringish arguments") {
+        auto result = cli.parse( { "tests", "--generator-index", "foo-baz" } );
+        REQUIRE_FALSE( result );
+    }
+    SECTION( "Generator specs accept star as argument" ) {
+        auto result = cli.parse( { "tests", "--generator-index", "*" } );
+        REQUIRE( result );
+
+        REQUIRE( config.pathFilters[0] ==
+                 PathFilter( PathFilter::For::Generator, "*" ) );
+    }
+    SECTION( "Generator specs do not accept negative numbers" ) {
+        auto result = cli.parse( { "tests", "--generator-index", "-2" } );
+        REQUIRE_FALSE( result );
+    }
+    SECTION( "Generic path spec enables new filter behaviour" ) {
+        auto result =
+            cli.parse( { "tests", "-p", "g:1", "--path-filter", "c:foobar" } );
+        REQUIRE( result );
+        REQUIRE( config.useNewPathFilteringBehaviour );
+
+        REQUIRE( config.pathFilters[0] ==
+                 PathFilter( PathFilter::For::Generator, "1" ) );
+        REQUIRE( config.pathFilters[1] ==
+                 PathFilter( PathFilter::For::Section, "foobar" ) );
+    }
+    SECTION( "Generic path spec for generator is validated" ) {
+        auto result = cli.parse( { "tests", "-p", "g:foo-bar" } );
+        REQUIRE_FALSE( result );
+    }
+    SECTION( "Generic path spec without colon is rejected" ) {
+        auto result1 = cli.parse( { "tests", "-p", "g123" } );
+        REQUIRE_FALSE( result1 );
+        auto result2 = cli.parse( { "tests", "-p", "carp" } );
+        REQUIRE_FALSE( result2 );
+    }
+    SECTION( "Using both section and generator filters creates filter stack" ) {
+        auto result = cli.parse( { "tests",
+                                   "--section", "foo-bar",
+                                   "--generator-index", "3",
+                                   "-g", "123",
+                                   "-c", "baz"
+        });
+        REQUIRE( result );
+        REQUIRE( config.pathFilters[0] == PathFilter( PathFilter::For::Section, "foo-bar" ) );
+        REQUIRE( config.pathFilters[1] == PathFilter( PathFilter::For::Generator, "3" ) );
+        REQUIRE( config.pathFilters[2] == PathFilter( PathFilter::For::Generator, "123" ) );
+        REQUIRE( config.pathFilters[3] == PathFilter( PathFilter::For::Section, "baz" ) );
+    }
+    SECTION( "Section/generator filters are whitespace trimmed" ) {
+        auto result = cli.parse( { "tests",
+                                   "--section", "  untrimmed  ",
+                                   "--generator-index", "  42  "
+        });
+        REQUIRE( result );
+        REQUIRE( config.pathFilters[0] == PathFilter(PathFilter::For::Section, "untrimmed" ) );
+        REQUIRE( config.pathFilters[1] == PathFilter(PathFilter::For::Generator, "42" ) );
+    }
+}