Support adding test tags as CTest labels in catch_discover_tests

We also bump the minimum CMake version to 3.20 as per #2943

2 files changed, 76 insertions, 42 deletions

diff --git a/extras/Catch.cmake b/extras/Catch.cmake
index cef73def..3d93fe20 100644
--- a/extras/Catch.cmake
+++ b/extras/Catch.cmake
@@ -39,6 +39,7 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
                          [OUTPUT_SUFFIX suffix]
                          [DISCOVERY_MODE <POST_BUILD|PRE_TEST>]
                          [SKIP_IS_FAILURE]
+                         [ADD_TAGS_AS_LABELS]
     )
 
   ``catch_discover_tests`` sets up a post-build command on the test executable
@@ -148,6 +149,9 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``.
   ``SKIP_IS_FAILURE``
     Disables skipped test detection.
 
+  ``ADD_TAGS_AS_LABELS``
+    Adds all test tags as CTest labels.
+
 #]=======================================================================]
 
 #------------------------------------------------------------------------------
@@ -155,12 +159,16 @@ function(catch_discover_tests TARGET)
 
   cmake_parse_arguments(
     ""
-    "SKIP_IS_FAILURE"
+    "SKIP_IS_FAILURE;ADD_TAGS_AS_LABELS"
     "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;REPORTER;OUTPUT_DIR;OUTPUT_PREFIX;OUTPUT_SUFFIX;DISCOVERY_MODE"
     "TEST_SPEC;EXTRA_ARGS;PROPERTIES;DL_PATHS;DL_FRAMEWORK_PATHS"
     ${ARGN}
   )
 
+  if (${CMAKE_VERSION} VERSION_LESS "3.19")
+    message(FATAL_ERROR "This script requires JSON support from CMake version 3.19 or greater.")
+  endif()
+
   if(NOT _WORKING_DIRECTORY)
     set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
   endif()
@@ -222,6 +230,7 @@ function(catch_discover_tests TARGET)
               -D "TEST_DL_PATHS=${_DL_PATHS}"
               -D "TEST_DL_FRAMEWORK_PATHS=${_DL_FRAMEWORK_PATHS}"
               -D "CTEST_FILE=${ctest_tests_file}"
+              -D "ADD_TAGS_AS_LABELS=${_ADD_TAGS_AS_LABELS}"
               -P "${_CATCH_DISCOVER_TESTS_SCRIPT}"
       VERBATIM
     )
@@ -267,6 +276,7 @@ function(catch_discover_tests TARGET)
       "      CTEST_FILE"             " [==[" "${ctest_tests_file}"        "]==]"   "\n"
       "      TEST_DL_PATHS"          " [==[" "${_DL_PATHS}"               "]==]"   "\n"
       "      TEST_DL_FRAMEWORK_PATHS" " [==[" "${_DL_FRAMEWORK_PATHS}"     "]==]"   "\n"
+      "      ADD_TAGS_AS_LABELS"     " [==[" "${_ADD_TAGS_AS_LABELS}"     "]==]"   "\n"
       "    )"                                                                      "\n"
       "  endif()"                                                                  "\n"
       "  include(\"${ctest_tests_file}\")"                                         "\n"
@@ -293,22 +303,10 @@ function(catch_discover_tests TARGET)
     endif()
   endif()
 
-  if(NOT ${CMAKE_VERSION} VERSION_LESS "3.10.0")
-    # Add discovered tests to directory TEST_INCLUDE_FILES
-    set_property(DIRECTORY
-      APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
-    )
-  else()
-    # Add discovered tests as directory TEST_INCLUDE_FILE if possible
-    get_property(test_include_file_set DIRECTORY PROPERTY TEST_INCLUDE_FILE SET)
-    if (NOT ${test_include_file_set})
-      set_property(DIRECTORY
-        PROPERTY TEST_INCLUDE_FILE "${ctest_include_file}"
-      )
-    else()
-      message(FATAL_ERROR "Cannot set more than one TEST_INCLUDE_FILE")
-    endif()
-  endif()
+  # Add discovered tests to directory TEST_INCLUDE_FILES
+  set_property(DIRECTORY
+    APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}"
+  )
 
 endfunction()
 
diff --git a/extras/CatchAddTests.cmake b/extras/CatchAddTests.cmake
index 399a839d..ceaf8652 100644
--- a/extras/CatchAddTests.cmake
+++ b/extras/CatchAddTests.cmake
@@ -22,10 +22,11 @@ function(catch_discover_tests_impl)
     ""
     ""
     "TEST_EXECUTABLE;TEST_WORKING_DIR;TEST_OUTPUT_DIR;TEST_OUTPUT_PREFIX;TEST_OUTPUT_SUFFIX;TEST_PREFIX;TEST_REPORTER;TEST_SPEC;TEST_SUFFIX;TEST_LIST;CTEST_FILE"
-    "TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS"
+    "TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS;ADD_TAGS_AS_LABELS"
     ${ARGN}
   )
 
+  set(add_tags "${_ADD_TAGS_AS_LABELS}")
   set(prefix "${_TEST_PREFIX}")
   set(suffix "${_TEST_SUFFIX}")
   set(spec ${_TEST_SPEC})
@@ -72,25 +73,19 @@ function(catch_discover_tests_impl)
   endif()
 
   execute_process(
-    COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests --verbosity quiet
-    OUTPUT_VARIABLE output
+    COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list-tests --reporter json
+    OUTPUT_VARIABLE listing_output
     RESULT_VARIABLE result
     WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
   )
   if(NOT ${result} EQUAL 0)
     message(FATAL_ERROR
-      "Error running test executable '${_TEST_EXECUTABLE}':\n"
+      "Error listing tests from executable '${_TEST_EXECUTABLE}':\n"
       "  Result: ${result}\n"
-      "  Output: ${output}\n"
+      "  Output: ${listing_output}\n"
     )
   endif()
 
-  # Make sure to escape ; (semicolons) in test names first, because
-  # that'd break the foreach loop for "Parse output" later and create
-  # wrongly splitted and thus failing test cases (false positives)
-  string(REPLACE ";" "\;" output "${output}")
-  string(REPLACE "\n" ";" output "${output}")
-
   # Prepare reporter
   if(reporter)
     set(reporter_arg "--reporter ${reporter}")
@@ -110,7 +105,7 @@ function(catch_discover_tests_impl)
       )
     elseif(NOT ${reporter_check_result} EQUAL 0)
       message(FATAL_ERROR
-        "Error running test executable '${_TEST_EXECUTABLE}':\n"
+        "Error checking for reporter in test executable '${_TEST_EXECUTABLE}':\n"
         "  Result: ${reporter_check_result}\n"
         "  Output: ${reporter_check_output}\n"
       )
@@ -139,46 +134,86 @@ function(catch_discover_tests_impl)
     endforeach()
   endif()
 
-  # Parse output
-  foreach(line ${output})
-    set(test "${line}")
+  # Parse JSON output for list of tests/class names/tags
+  string(JSON version GET "${listing_output}" "version")
+  if (NOT version STREQUAL "1")
+    message(FATAL_ERROR "Unsupported catch output version: '${version}'")
+  endif()
+
+  # Speed-up reparsing by cutting away unneeded parts of JSON.
+  string(JSON test_listing GET "${listing_output}" "listings" "tests")
+  string(JSON num_tests LENGTH "${test_listing}")
+  # CMake's foreach-RANGE is inclusive, so we have to subtract 1
+  math(EXPR num_tests "${num_tests} - 1")
+
+  foreach(idx RANGE ${num_tests})
+    string(JSON single_test GET ${test_listing} ${idx})
+    string(JSON test_tags GET "${single_test}" "tags")
+    string(JSON plain_name GET "${single_test}" "name")
+
     # Escape characters in test case names that would be parsed by Catch2
     # Note that the \ escaping must happen FIRST! Do not change the order.
-    set(test_name "${test}")
-    foreach(char \\ , [ ])
-      string(REPLACE ${char} "\\${char}" test_name "${test_name}")
+    set(escaped_name "${plain_name}")
+    foreach(char \\ , [ ] ;)
+      string(REPLACE ${char} "\\${char}" escaped_name "${escaped_name}")
     endforeach(char)
     # ...add output dir
     if(output_dir)
-      string(REGEX REPLACE "[^A-Za-z0-9_]" "_" test_name_clean "${test_name}")
-      set(output_dir_arg "--out ${output_dir}/${output_prefix}${test_name_clean}${output_suffix}")
+      string(REGEX REPLACE "[^A-Za-z0-9_]" "_" escaped_name_clean "${escaped_name}")
+      set(output_dir_arg "--out ${output_dir}/${output_prefix}${escaped_name_clean}${output_suffix}")
     endif()
 
     # ...and add to script
     add_command(add_test
-      "${prefix}${test}${suffix}"
+      "${prefix}${plain_name}${suffix}"
       ${_TEST_EXECUTOR}
       "${_TEST_EXECUTABLE}"
-      "${test_name}"
+      "${escaped_name}"
       ${extra_args}
       "${reporter_arg}"
       "${output_dir_arg}"
     )
     add_command(set_tests_properties
-      "${prefix}${test}${suffix}"
+      "${prefix}${plain_name}${suffix}"
       PROPERTIES
       WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
       ${properties}
     )
 
+    if (add_tags)
+      string(JSON num_tags LENGTH "${test_tags}")
+      math(EXPR num_tags "${num_tags} - 1")
+      set(tag_list "")
+      if (num_tags GREATER_EQUAL "0")
+        foreach(tag_idx RANGE ${num_tags})
+          string(JSON a_tag GET "${test_tags}" "${tag_idx}")
+          # Catch2's tags can contain semicolons, which are list element separators
+          # in CMake, so we have to escape them. Ideally we could use the [=[...]=]
+          # syntax for this, but CTest currently keeps the square quotes in the label
+          # name. So we add 2 backslashes to escape it instead.
+          # **IMPORTANT**: The number of backslashes depends on how many layers
+          #                of CMake the tag goes. If this script is changed, the
+          #                number of backslashes to escape may change as well.
+          string(REPLACE ";" "\\;" a_tag "${a_tag}")
+          list(APPEND tag_list "${a_tag}")
+        endforeach()
+
+        add_command(set_tests_properties
+          "${prefix}${plain_name}${suffix}"
+          PROPERTIES
+          LABELS "${tag_list}"
+        )
+      endif()
+    endif(add_tags)
+
     if(environment_modifications)
       add_command(set_tests_properties
-        "${prefix}${test}${suffix}"
+        "${prefix}${plain_name}${suffix}"
         PROPERTIES
         ENVIRONMENT_MODIFICATION "${environment_modifications}")
     endif()
 
-    list(APPEND tests "${prefix}${test}${suffix}")
+    list(APPEND tests "${prefix}${plain_name}${suffix}")
   endforeach()
 
   # Create a list of all discovered tests, which users may use to e.g. set
@@ -207,5 +242,6 @@ if(CMAKE_SCRIPT_MODE_FILE)
     TEST_DL_PATHS ${TEST_DL_PATHS}
     TEST_DL_FRAMEWORK_PATHS ${TEST_DL_FRAMEWORK_PATHS}
     CTEST_FILE ${CTEST_FILE}
+    ADD_TAGS_AS_LABELS ${ADD_TAGS_AS_LABELS}
   )
 endif()