Support adding test tags as CTest labels in catch_discover_tests

We also bump the minimum CMake version to 3.20 as per #2943

3 files changed, 52 insertions, 16 deletions

diff --git a/tests/TestScripts/DiscoverTests/CMakeLists.txt b/tests/TestScripts/DiscoverTests/CMakeLists.txt
index 5105cddb..64eb251f 100644
--- a/tests/TestScripts/DiscoverTests/CMakeLists.txt
+++ b/tests/TestScripts/DiscoverTests/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.20)
 
 project(discover-tests-test
   LANGUAGES CXX
@@ -19,4 +19,11 @@ if (CMAKE_VERSION GREATER_EQUAL 3.27)
             DL_PATHS "${CMAKE_CURRENT_LIST_DIR};${CMAKE_CURRENT_LIST_DIR}/.."
     )
 endif ()
-catch_discover_tests(tests ${extra_args})
+catch_discover_tests(
+  tests
+  ADD_TAGS_AS_LABELS
+  DISCOVERY_MODE PRE_TEST
+  ${extra_args}
+)
+
+# DISCOVERY_MODE <POST_BUILD|PRE_TEST>
diff --git a/tests/TestScripts/DiscoverTests/VerifyRegistration.py b/tests/TestScripts/DiscoverTests/VerifyRegistration.py
index 9800674f..33150cae 100644
--- a/tests/TestScripts/DiscoverTests/VerifyRegistration.py
+++ b/tests/TestScripts/DiscoverTests/VerifyRegistration.py
@@ -12,6 +12,10 @@ import subprocess
 import sys
 import re
 import json
+from collections import namedtuple
+from typing import List
+
+TestInfo = namedtuple('TestInfo', ['name', 'tags'])
 
 cmake_version_regex = re.compile('cmake version (\d+)\.(\d+)\.(\d+)')
 
@@ -61,7 +65,7 @@ def build_project(sources_dir, output_base_path, catch2_path):
 
 
 
-def get_test_names(build_path):
+def get_test_names(build_path: str) -> List[TestInfo]:
     # For now we assume that Windows builds are done using MSBuild under
     # Debug configuration. This means that we need to add "Debug" folder
     # to the path when constructing it. On Linux, we don't add anything.
@@ -69,15 +73,23 @@ def get_test_names(build_path):
     full_path = os.path.join(build_path, config_path, 'tests')
 
 
-    cmd = [full_path, '--reporter', 'xml', '--list-tests']
+    cmd = [full_path, '--reporter', 'json', '--list-tests']
     result = subprocess.run(cmd,
                             capture_output = True,
                             check = True,
                             text = True)
 
-    import xml.etree.ElementTree as ET
-    root = ET.fromstring(result.stdout)
-    return [tc.text for tc in root.findall('TestCase/Name')]
+    test_listing = json.loads(result.stdout)
+
+    assert test_listing['version'] == 1
+
+    tests = []
+    for test in test_listing['listings']['tests']:
+        test_name = test['name']
+        tags = test['tags']
+        tests.append(TestInfo(test_name, tags))
+
+    return tests
 
 def get_ctest_listing(build_path):
     old_path = os.getcwd()
@@ -91,20 +103,25 @@ def get_ctest_listing(build_path):
     os.chdir(old_path)
     return result.stdout
 
-def extract_tests_from_ctest(ctest_output):
+def extract_tests_from_ctest(ctest_output) -> List[TestInfo]:
     ctest_response = json.loads(ctest_output)
     tests = ctest_response['tests']
-    test_names = []
+    test_infos = []
     for test in tests:
         test_command = test['command']
         # First part of the command is the binary, second is the filter.
         # If there are less, registration has failed. If there are more,
         # registration has changed and the script needs updating.
         assert len(test_command) == 2
-        test_names.append(test_command[1])
         test_name = test_command[1]
+        labels = []
+        for prop in test['properties']:
+            if prop['name'] == 'LABELS':
+                labels = prop['value']
+
+        test_infos.append(TestInfo(test_name, labels))
 
-    return test_names
+    return test_infos
 
 def check_DL_PATHS(ctest_output):
     ctest_response = json.loads(ctest_output)
@@ -115,10 +132,14 @@ def check_DL_PATHS(ctest_output):
             if property['name'] == 'ENVIRONMENT_MODIFICATION':
                 assert len(property['value']) == 2, f"The test provides 2 arguments to DL_PATHS, but instead found {len(property['value'])}"
 
-def escape_catch2_test_name(name):
-    for char in ('\\', ',', '[', ']'):
-        name = name.replace(char, f"\\{char}")
-    return name
+def escape_catch2_test_names(infos: List[TestInfo]):
+    escaped = []
+    for info in infos:
+      name = info.name
+      for char in ('\\', ',', '[', ']'):
+          name = name.replace(char, f"\\{char}")
+      escaped.append(TestInfo(name, info.tags))
+    return escaped
 
 if __name__ == '__main__':
     if len(sys.argv) != 3:
@@ -130,7 +151,7 @@ if __name__ == '__main__':
 
     build_path = build_project(sources_dir, output_base_path, catch2_path)
 
-    catch_test_names = [escape_catch2_test_name(name) for name in get_test_names(build_path)]
+    catch_test_names = escape_catch2_test_names(get_test_names(build_path))
     ctest_output = get_ctest_listing(build_path)
     ctest_test_names = extract_tests_from_ctest(ctest_output)
 
@@ -147,6 +168,7 @@ if __name__ == '__main__':
     if mismatched:
         print(f"Found {mismatched} mismatched tests catch test names and ctest test commands!")
         exit(1)
+    print(f"{len(catch_test_names)} tests matched")
 
     cmake_version = get_cmake_version()
     if cmake_version >= (3, 27):
diff --git a/tests/TestScripts/DiscoverTests/register-tests.cpp b/tests/TestScripts/DiscoverTests/register-tests.cpp
index aa603df1..be533ab6 100644
--- a/tests/TestScripts/DiscoverTests/register-tests.cpp
+++ b/tests/TestScripts/DiscoverTests/register-tests.cpp
@@ -14,3 +14,10 @@ TEST_CASE( "Let's have a test case with a long name. Longer. No, even longer. "
            "Really looooooooooooong. Even longer than that. Multiple lines "
            "worth of test name. Yep, like this." ) {}
 TEST_CASE( "And now a test case with weird tags.", "[tl;dr][tl;dw][foo,bar]" ) {}
+// Also check that we handle tests on class, which have name in output as 'class-name', not 'name'.
+class TestCaseFixture {
+public:
+    int m_a;
+};
+
+TEST_CASE_METHOD(TestCaseFixture, "A test case as method", "[tagstagstags]") {}