Support filtering on both sections and generators (#3069)

Not being able to filter generators to specific element has been regularly
causing problems. It was possible to use a dynamic section to run tests
for specific element in a generator, at least if the element had a nice
string representation, but the test case would still run once per element
in the generator.

With this change, it is possible to have the generator return only one
specific element, and do so based on the index, rather than the string
representation of the element. This enables simple debugging of tests
that fail for specific generator element.

1 files changed, 67 insertions, 6 deletions

diff --git a/tests/TestScripts/testSectionFiltering.py b/tests/TestScripts/testSectionFiltering.py
index e4f33ad0..4f56df53 100755
--- a/tests/TestScripts/testSectionFiltering.py
+++ b/tests/TestScripts/testSectionFiltering.py
@@ -22,20 +22,34 @@ import sys
 from typing import Tuple, List
 import xml.etree.ElementTree as ET
 
-def make_cli_filter(section_names: Tuple[str, ...]) -> List[str]:
+def make_old_filters(section_names: Tuple[str, ...]) -> List[str]:
     final = []
     for name in section_names:
         final.append('--section')
         final.append(name)
     return final
 
-def run_one_test(binary_path: str, test_name: str, section_names: Tuple[str, ...], expected_assertions: int):
+def make_new_filters(path_filters: Tuple[str, ...]) -> List[str]:
+    final = []
+    for component in path_filters:
+        final.append('--path-filter')
+        final.append(component)
+    return final
+
+def run_one_test(binary_path: str,
+                 test_name: str,
+                 section_names: Tuple[str, ...],
+                 expected_assertions: int,
+                 new_filters: bool):
     cmd = [
       binary_path,
       '--reporter', 'xml',
       test_name
     ]
-    cmd.extend(make_cli_filter(section_names))
+    if new_filters:
+        cmd.extend(make_new_filters(section_names))
+    else:
+        cmd.extend(make_old_filters(section_names))
     try:
         ret = subprocess.run(
             cmd,
@@ -70,7 +84,7 @@ def run_one_test(binary_path: str, test_name: str, section_names: Tuple[str, ...
 
 
 # Inputs taken from issue #3038
-tests = {
+old_section_tests = {
   'foo': (
     ((), 3),
     (('A',), 3),
@@ -117,12 +131,59 @@ tests = {
   ),
 }
 
+new_filter_tests = {
+  'grault': (
+    ((), 12),
+    (('c:B', 'g:1'), 3),
+    (('c:B', 'g:1', 'c:i=2'), 3),
+    # We filter the generator to different element than the dynamic section
+    (('c:B', 'g:1', 'c:i=1'), 2),
+  ),
+  'garply': (
+    ((), 3),
+    (('g:1',), 1),
+    (('g:*',), 3),
+    # Followup section filters do not change results (there are no sections underneath)
+    (('g:1', 'c:abcdef'), 1),
+    (('g:*', 'c:abcdef'), 3),
+  ),
+  'waldo': (
+    ((), 9),
+    (('g:1',), 3),
+    (('g:*',), 9),
+    (('g:1', 'g:2'), 1),
+    (('g:*', 'g:1'), 3),
+    (('g:0', 'g:*'), 3),
+    (('g:*', 'g:*'), 9),
+    (('g:*', 'g:1', 'c:abcdef'), 3),
+  ),
+  'fred': (
+    ((), 24),
+    # No sections entered, but we still hit assertion at the start of test
+    (('c:abcdef',), 1),
+    (('c:A', ), 12),
+    (('c:A', 'g:*', 'c:A1'), 6),
+    (('c:A', 'g:1', 'c:A1'), 2),
+    (('c:A', 'g:1'), 4),
+    # Section 'A3' does not exist
+    (('c:A', 'g:*', 'c:A3'), 3),
+    (('c:A', 'g:1', 'c:A3'), 1),
+    # We would like something like 'c:*', 'g:2', but we do not support globs
+    # for sections (yet?)
+  ),
+}
+
+
 if len(sys.argv) != 2:
     print("Wrong number of arguments, expected just the path to Catch2 SelfTest binary")
     exit(1)
 
 bin_path = os.path.abspath(sys.argv[1])
 
-for test_filter, specs in tests.items():
+for test_filter, specs in old_section_tests.items():
+    for section_path, expected_assertions in specs:
+        run_one_test(bin_path, test_filter, section_path, expected_assertions, new_filters=False)
+
+for test_filter, specs in new_filter_tests.items():
     for section_path, expected_assertions in specs:
-        run_one_test(bin_path, test_filter, section_path, expected_assertions)
+        run_one_test(bin_path, test_filter, section_path, expected_assertions, new_filters=True)