Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

develroot / extras / Catch3AddTests.cmake

20.9 KiB · Raw · Log

# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
# file Copyright.txt or https://cmake.org/licensing for details.

# Because natively using CMake's JSON processing for arrays leads to quadratic
# running times, we do terrible hack and split JSON array by object boundary
# + commas and try to reconstruct valid JSON objects. During this, we need
# to replace CMake characters that could be in the test name/tags with
# placeholder, so it doesn't affect CMake's processing of the strings/lists
# we create during the parsing.
#
# We use 0x01, 0x02, 0x03, and 0x04 as placeholder bytes, as those cannot
# exist in JSON unescaped.
#
#   0x01  <=> ';'   (CMake list separator)
#   0x02  <-> '['   (opens a CMake bracket-argument context)
#   0x03  <-> ']'   (closes a CMake bracket-argument context)
#   0x04  ==  element boundary marker used while splitting the tests array
#
string(ASCII 1 _SemicolonEscape)
string(ASCII 2 _OpenBracketEscape)
string(ASCII 3 _CloseBracketEscape)
string(ASCII 4 _BoundaryEscape)


# As far as I can tell, the only way of having unclosed '[' or ']' in the
# list of test names we define for CTest, is to escape them into completely
# different characters. Otherwise, they will break parsing of unrelated
# semicolons (inserted by CMake after parsing the list from the user-facing
# space separated format). This means that the consumers of the list have
# to unescape them back, but that's CMake :v
#
#   0x02  <-> '['
#   0x03  <-> ']'
#
string(ASCII 2 _OpenBracketListingEscape)
string(ASCII 3 _CloseBracketListingEscape)


# Placeholder bytes in the listing would break our parsing hack, so
# we check they don't exist. They shouldn't exist in valid JSON, but
# the reporter might not be escaping them properly.
function(catch3_validate_input_noescapes listing_var)
  foreach(byte "${_SemicolonEscape}" "${_BoundaryEscape}" "${_OpenBracketEscape}" "${_CloseBracketEscape}")
    string(FIND "${${listing_var}}" "${byte}" found)
    if(NOT found EQUAL -1)
      message(FATAL_ERROR
        "The test listing contains raw control byte (0x01-0x04) which should not "
        "be there. This means either bad escaping in JSON reporter, or corrupted file. "
      )
    endif()
  endforeach()
endfunction()


# Replaces relevant characters with their placeholders, see the top of this file.
# Modifies argument `var` in place.
function(catch3_magic_escape_chars var)
  set(value "${${var}}")
  string(REPLACE ";" "${_SemicolonEscape}" value "${value}")
  string(REPLACE "[" "${_OpenBracketEscape}" value "${value}")
  string(REPLACE "]" "${_CloseBracketEscape}" value "${value}")
  set(${var} "${value}" PARENT_SCOPE)
endfunction()


# Turns placeholders back into original characters, see the top of this file.
# Modifies argument `var` in place.
function(catch3_magic_unescape_chars var)
  set(value "${${var}}")
  string(REPLACE "${_SemicolonEscape}" ";" value "${value}")
  string(REPLACE "${_OpenBracketEscape}" "[" value "${value}")
  string(REPLACE "${_CloseBracketEscape}" "]" value "${value}")
  set(${var} "${value}" PARENT_SCOPE)
endfunction()


# Abuses knowledge of the runner's JSON listing (`--list=json`) to
# split JSON array of the test listings into a CMake list of strings,
# with each element being the JSON string of one array entry.
#
# This avoids the terrible quadratic running time of using CMake's JSON
# support to parse the JSON reporter output "properly", where the whole
# JSON array of tests is parsed again for every element. Instead, we can
# use the CMake's API to only parse the individual test's objects, which
# are usually small and only have to be reparsed fixed number of times
# (once for test names, once for labels).
#
# We process the string representing the JSON array by splitting it on
# `}<ws>*,<ws>*{` and then checking for each chunk whether it is a valid
# JSON object representing Catch2's test. If not (e.g. because we split
# on the presence of `}<ws>*,<ws>*{` inside a test name), then we append
# the next chunk to the current one and check again. And again, until
# we get back to a valid JSON.
#
# Note that to support passing the object strings back from the function,
# they will still contain the placeholders and need to be unescaped before
# further processing (e.g. sending them into CMake's JSON parsing API).
function(catch3_split_json_array json_array_var out_var)
  # We have to pass the input by var name to avoid CMake processing
  # the input as an arg.
  set(json_in "${${json_array_var}}")

  # Strip the array brackets at the start and end of the JSON array.
  # Must happen before we escape the other [] instances below from the
  # actual array data.
  string(REGEX REPLACE "^[ \t\r\n]*\\[" "" json_in "${json_in}")
  string(REGEX REPLACE "\\][ \t\r\n]*$" "" json_in "${json_in}")

  catch3_magic_escape_chars(json_in)

  # We need to keep the whitespace around comma around, so that if we
  # split inside the test object, we can reconstruct it losslessly.
  string(REGEX REPLACE "(}[ \t\r\n]*)[,]([ \t\r\n]*{)" "\\1${_BoundaryEscape}\\2" json_in "${json_in}")

  # We escaped all list separators above, so now we can turn the JSON
  # string into a CMake list of fragments in single pass.
  string(REPLACE "${_BoundaryEscape}" ";" fragments "${json_in}")

  # And now we have to reconstruct the actual JSON structure from fragments.
  set(array_elements "")
  set(accumulator "")
  foreach(next_fragment IN LISTS fragments)
    if(accumulator)
      set(accumulator "${accumulator},${next_fragment}")
    else()
      set(accumulator "${next_fragment}")
    endif()

    # Because the fragments (might) contain invalid JSON characters due
    # to escaping, we have to unescape it before checking if we can parse it.
    set(maybe_json "${accumulator}")
    catch3_magic_unescape_chars(maybe_json)
    string(JSON unused ERROR_VARIABLE err GET "${maybe_json}" "name")
    if(err STREQUAL "NOTFOUND")
      list(APPEND array_elements "${accumulator}")
      set(accumulator "")
    endif()
  endforeach()

  set(${out_var} "${array_elements}" PARENT_SCOPE)
endfunction()


# Prepare (a part of) command with bracketed arguments and return it via `out_var`.
#
# This allows the registration script to escape parts of the test script
# only once, instead of escaping the unchanged arguments over and over again.
function(catch3_prepare_command_fragment out_var)
  set(_args "")
  math(EXPR _last_arg ${ARGC}-1)
  if(_last_arg GREATER_EQUAL 1)
    foreach(_n RANGE 1 ${_last_arg})
      set(_arg "${ARGV${_n}}")
      if(_arg MATCHES "[^-./:a-zA-Z0-9_]")
        set(_args "${_args} [==[${_arg}]==]") # form a bracket_argument
      else()
        set(_args "${_args} ${_arg}")
      endif()
    endforeach()
  endif()
  set(${out_var} "${_args}" PARENT_SCOPE)
endfunction()


# Generates random filename in the temp folder.
# Temp folder is retrieved by checking env vars from various platforms.
function(catch3_make_temp_file_path OUT_VARIABLE FALLBACK_PATH)
  set(TEMP_DIR "")
  set(ENV_VARS
    # From XDG base dir specification
    XDG_RUNTIME_DIR
    # From POSIX standard
    TMPDIR
    # From Windows
    TMP
    TEMP
  )

  foreach(var ${ENV_VARS})
    if(DEFINED ENV{${var}} AND NOT "$ENV{${var}}" STREQUAL "")
        set(TEMP_DIR "$ENV{${var}}")
        break()
    endif()
  endforeach()

  # If all checks fail, we use the fallback path
  if(TEMP_DIR STREQUAL "")
    set(TEMP_DIR "${FALLBACK_PATH}")
  endif()

  file(TO_CMAKE_PATH "${TEMP_DIR}" TEMP_DIR)

  # Generate the random file name
  string(RANDOM LENGTH 8 RAND_ID)
  set(FINAL_TEMP_PATH "${TEMP_DIR}/Catch3-test-listing.${RAND_ID}.json")

  set(${OUT_VARIABLE} "${FINAL_TEMP_PATH}" PARENT_SCOPE)
endfunction()

# Computes the path of the test-list file based on the path for the main
# CTest script file (passed in `CTEST_FILE`).
#
# It works by replacing the `_tests` part of the original name with
# `_test-list`, or failing that, it appends `-list.cmake` instead.
#
#   <base>_tests.cmake       -> <base>_test-list.cmake
#   <base>_tests-Debug.cmake -> <base>_test-list-Debug.cmake
function(catch3_make_test_list_file_path CTEST_FILE OUT_VARIABLE)
  if("${CTEST_FILE}" MATCHES "_tests(.*)\\.cmake$")
    string(REGEX REPLACE "_tests(.*)\\.cmake$" "_test-list\\1.cmake" list_file "${CTEST_FILE}")
  else()
    set(list_file "${CTEST_FILE}-list.cmake")
  endif()
  set(${OUT_VARIABLE} "${list_file}" PARENT_SCOPE)
endfunction()

function(catch3_discover_tests_impl)
  cmake_parse_arguments(
    ""
    ""
    "TEST_EXECUTABLE;TEST_WORKING_DIR;TEST_OUTPUT_DIR;TEST_OUTPUT_PREFIX;TEST_OUTPUT_SUFFIX;TEST_PREFIX;TEST_REPORTER;TEST_SPEC;TEST_SUFFIX;TEST_LIST;CTEST_FILE"
    "TEST_EXTRA_ARGS;TEST_PROPERTIES;TEST_EXECUTOR;TEST_DL_PATHS;TEST_DL_FRAMEWORK_PATHS;ADD_TAGS_AS_LABELS"
    ${ARGN}
  )

  # We write the list of all discovered tests into a separate file
  catch3_make_test_list_file_path("${_CTEST_FILE}" _CTEST_LIST_FILE)

  # We periodically append to the output files below, so we have to ensure
  # that it is empty at the start, or we get duplicated test scripts.
  file(REMOVE "${_CTEST_FILE}")
  file(REMOVE "${_CTEST_LIST_FILE}")

  # Size (in Bytes) at which the intermediate `script` var is dumped to file.
  set(_WriteToFileThreshold 50000)

  set(add_tags "${_ADD_TAGS_AS_LABELS}")
  # TEST_{PREFIX,SUFFIX} is enclosed in single quotes to keep ensure
  # leading/trailing whitespace isn't trimmed by CMake's argument passing.
  string(REGEX REPLACE "^'(.*)'$" "\\1" prefix "${_TEST_PREFIX}")
  string(REGEX REPLACE "^'(.*)'$" "\\1" suffix "${_TEST_SUFFIX}")
  set(spec ${_TEST_SPEC})
  set(extra_args ${_TEST_EXTRA_ARGS})
  set(properties ${_TEST_PROPERTIES})
  set(reporter ${_TEST_REPORTER})
  set(output_dir ${_TEST_OUTPUT_DIR})
  set(output_prefix ${_TEST_OUTPUT_PREFIX})
  set(output_suffix ${_TEST_OUTPUT_SUFFIX})
  set(dl_paths ${_TEST_DL_PATHS})
  set(dl_framework_paths ${_TEST_DL_FRAMEWORK_PATHS})
  set(environment_modifications "")
  set(script)
  set(suite)
  set(tests)
  # Holds the list of all registered test names **as a string**, not list.
  # This avoids issue of CMake removing semicolon escapes even inside
  # bracket-quoted strings.
  set(_test_names)

  if(WIN32)
    set(dl_paths_variable_name PATH)
  elseif(APPLE)
    set(dl_paths_variable_name DYLD_LIBRARY_PATH)
  else()
    set(dl_paths_variable_name LD_LIBRARY_PATH)
  endif()

  # Run test executable to get list of available tests
  if(NOT EXISTS "${_TEST_EXECUTABLE}")
    message(FATAL_ERROR
      "Specified test executable '${_TEST_EXECUTABLE}' does not exist"
    )
  endif()

  if(dl_paths)
    cmake_path(CONVERT "$ENV{${dl_paths_variable_name}}" TO_NATIVE_PATH_LIST env_dl_paths)
    list(PREPEND env_dl_paths "${dl_paths}")
    cmake_path(CONVERT "${env_dl_paths}" TO_NATIVE_PATH_LIST paths)
    set(ENV{${dl_paths_variable_name}} "${paths}")
  endif()

  if(APPLE AND dl_framework_paths)
    cmake_path(CONVERT "$ENV{DYLD_FRAMEWORK_PATH}" TO_NATIVE_PATH_LIST env_dl_framework_paths)
    list(PREPEND env_dl_framework_paths "${dl_framework_paths}")
    cmake_path(CONVERT "${env_dl_framework_paths}" TO_NATIVE_PATH_LIST paths)
    set(ENV{DYLD_FRAMEWORK_PATH} "${paths}")
  endif()

  # The discovery protocol of the Catch3 runner: `--list=json` prints one
  # JSON document, "format": "catch3-tests", in registration order.
  execute_process(
    COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list=json
    OUTPUT_VARIABLE listing_output
    RESULT_VARIABLE result
    WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
  )
  if(NOT ${result} EQUAL 0)
    message(FATAL_ERROR
      "Error listing tests from executable '${_TEST_EXECUTABLE}':\n"
      "  Result: ${result}\n"
      "  Output: ${listing_output}\n"
    )
  endif()
  # Whatever the binary printed before the document is not part of it.
  string(FIND "${listing_output}" "{" listing_start)
  if(listing_start GREATER 0)
    string(SUBSTRING "${listing_output}" ${listing_start} -1 listing_output)
  endif()
  catch3_validate_input_noescapes(listing_output)

  # The runner writes reports itself: REPORTER chooses which, OUTPUT_DIR where.
  set(report_option "")
  if(reporter)
    string(TOLOWER "${reporter}" reporter_lower)
    if(reporter_lower STREQUAL "junit")
      set(report_option "--junit=")
    elseif(reporter_lower STREQUAL "json")
      set(report_option "--json=")
    elseif(NOT reporter_lower STREQUAL "console")
      message(FATAL_ERROR
        "\"${reporter}\" is not a report of the Catch3 runner; use junit, json or console\n"
      )
    endif()
    if(report_option AND NOT output_dir)
      message(FATAL_ERROR
        "REPORTER ${reporter} writes a file and needs OUTPUT_DIR\n"
      )
    endif()
  endif()

  # Prepare output dir
  if(output_dir AND NOT IS_ABSOLUTE ${output_dir})
    set(output_dir "${_TEST_WORKING_DIR}/${output_dir}")
    if(NOT EXISTS ${output_dir})
      file(MAKE_DIRECTORY ${output_dir})
    endif()
  endif()

  if(dl_paths)
    foreach(path ${dl_paths})
      cmake_path(NATIVE_PATH path native_path)
      list(PREPEND environment_modifications "${dl_paths_variable_name}=path_list_prepend:${native_path}")
    endforeach()
  endif()

  if(APPLE AND dl_framework_paths)
    foreach(path ${dl_framework_paths})
      cmake_path(NATIVE_PATH path native_path)
      list(PREPEND environment_modifications "DYLD_FRAMEWORK_PATH=path_list_prepend:${native_path}")
    endforeach()
  endif()

  # Parse JSON output for list of tests/class names/tags
  string(JSON format GET "${listing_output}" "format")
  string(JSON version GET "${listing_output}" "version")
  if(NOT format STREQUAL "catch3-tests" OR NOT version STREQUAL "1")
    message(FATAL_ERROR "Unsupported test listing: '${format}' version '${version}'")
  endif()

  # Extract just the JSON array with tests and then split them into
  # individual objects.
  string(JSON test_listing GET "${listing_output}" "tests")
  catch3_split_json_array(test_listing tests)

  # Exit early if no tests are detected
  if(NOT tests)
    # Still emit an (empty) test list file and have the main script include
    # it, so that consumers relying on the `${_TEST_LIST}` variable and on the
    # include structure get consistent behavior regardless of test count.
    file(WRITE "${_CTEST_LIST_FILE}" "set(${_TEST_LIST})\n")
    file(WRITE "${_CTEST_FILE}" "include(\"${_CTEST_LIST_FILE}\")\n")
    return()
  endif()


  # The 'set(VAR` header for the test list has to be written separately,
  # so that each test name can be appended file without further processing.
  set(test_names "set(${_TEST_LIST}")

  # Most of the commands/arguments in the CTest script are identical
  # for every test registered with one `catch3_discover_tests` call.
  # To avoid repeating the work in escaping them, we escape them before
  # the per-test loop and reuse the escaped fragments.
  #
  # `add_test` calls are
  #   add_test(<name><exec><exe><escaped_name><extra_args><reporter><out_dir>)
  # Of these, <exec>,<exe>,<extra_args>, and <reporter> are the same between
  # all tests.
  catch3_prepare_command_fragment(_exec_exe_fragment
    ${_TEST_EXECUTOR}
    "${_TEST_EXECUTABLE}"
  )
  catch3_prepare_command_fragment(_args_reporter_fragment
    "${extra_args}"
  )

  # `set_tests_properties` calls are
  #   set_tests_properties(<name> PROPERTIES WORKING_DIRECTORY <dir> <properties>)
  #   set_tests_properties(<name> PROPERTIES ENVIRONMENT_MODIFICATION <env_mod>)
  # Of these, only the <name> changes between tests.
  catch3_prepare_command_fragment(_properties_fragment
    PROPERTIES
    WORKING_DIRECTORY "${_TEST_WORKING_DIR}"
    ${properties}
  )
  # Env modification is optional, so we prepare it in a separate command
  if(environment_modifications)
    catch3_prepare_command_fragment(_env_modification_fragment
      PROPERTIES
      ENVIRONMENT_MODIFICATION "${environment_modifications}"
    )
  endif()

  # Each element in the tests is JSON-string representing one test object.
  # We have to parse it and then turn it into CTest script commands.
  foreach(single_test IN LISTS tests)
    # Because appending to the same string in CMake has quadratic runtime,
    # we flush the script into the file periodically to avoid the worst case.
    string(LENGTH "${script}" script_len)
    if (script_len GREATER _WriteToFileThreshold)
      file(APPEND "${_CTEST_FILE}" "${script}")
      set(script "")
    endif()
    string(LENGTH "${test_names}" names_len)
    if (names_len GREATER _WriteToFileThreshold)
      file(APPEND "${_CTEST_LIST_FILE}" "${test_names}")
      set(test_names "")
    endif()

    # The elements are still escaped and contain JSON-invalid characters,
    # they have to be unescaped before parsing them as JSON.
    catch3_magic_unescape_chars(single_test)
    if(add_tags)
      string(JSON test_tags GET "${single_test}" "tags")
    endif()
    string(JSON plain_name GET "${single_test}" "name")

    # Escape characters in test case names that would be parsed by Catch2
    # Note that the \ escaping must happen FIRST! Do not change the order.
    set(escaped_name "${plain_name}")
    foreach(char \\ , [ ] ;)
      string(REPLACE ${char} "\\${char}" escaped_name "${escaped_name}")
    endforeach(char)
    # ...add output dir
    if(output_dir)
      string(REGEX REPLACE "[^A-Za-z0-9_]" "_" escaped_name_clean "${escaped_name}")
      if(report_option)
        set(output_dir_arg "${report_option}${output_dir}/${output_prefix}${escaped_name_clean}${output_suffix}")
      endif()
    endif()

    set(full_name "${prefix}${plain_name}${suffix}")
    catch3_prepare_command_fragment(_full_name_fragment "${full_name}")
    catch3_prepare_command_fragment(_escaped_name_fragment "${escaped_name}")
    catch3_prepare_command_fragment(_outdir_fragment "${output_dir_arg}")

    string(APPEND script
      "add_test(${_full_name_fragment}${_exec_exe_fragment}${_escaped_name_fragment}${_args_reporter_fragment}${_outdir_fragment})\n")
    string(APPEND script
      "set_tests_properties(${_full_name_fragment}${_properties_fragment})\n")

    if(add_tags)
      string(JSON num_tags LENGTH "${test_tags}")
      math(EXPR num_tags "${num_tags} - 1")
      set(tag_list "")
      if(num_tags GREATER_EQUAL "0")
        foreach(tag_idx RANGE ${num_tags})
          string(JSON a_tag GET "${test_tags}" "${tag_idx}")
          # Catch2's tags can contain semicolons, which are list element separators
          # in CMake, so we have to escape them. Ideally we could use the [=[...]=]
          # syntax for this, but CTest currently keeps the square quotes in the label
          # name. So we add 2 backslashes to escape it instead.
          # **IMPORTANT**: The number of backslashes depends on how many layers
          #                of CMake the tag goes. If this script is changed, the
          #                number of backslashes to escape may change as well.
          string(REPLACE ";" "\\;" a_tag "${a_tag}")
          list(APPEND tag_list "${a_tag}")
        endforeach()

        catch3_prepare_command_fragment(_labels_fragment
          PROPERTIES
          LABELS "${tag_list}"
        )
        string(APPEND script "set_tests_properties(${_full_name_fragment}${_labels_fragment})\n")
      endif()
    endif(add_tags)

    if(environment_modifications)
      string(APPEND script "set_tests_properties(${_full_name_fragment}${_env_modification_fragment})\n")
    endif()

    # The test name has to be escaped using the same rules as prepare_command
    # uses for the arguments, so that it keeps being single element in list
    # even with weird characters and semicolons.
    set(full_list_name "${prefix}${plain_name}${suffix}")
    string(REPLACE ";" "\\;" full_list_name "${full_list_name}")
    string(REPLACE "[" "${_OpenBracketListingEscape}" full_list_name "${full_list_name}")
    string(REPLACE "]" "${_CloseBracketListingEscape}" full_list_name "${full_list_name}")
    if(full_list_name MATCHES "[^-./:a-zA-Z0-9_]")
      # The space before the start of quote is important, so that we get
      # space-separated list in the final file.
      string(APPEND test_names " [==[${full_list_name}]==]")
    else()
      string(APPEND test_names " ${full_list_name}")
    endif()
  endforeach()

  # Write any test names leftovers we have
  file(APPEND "${_CTEST_LIST_FILE}" "${test_names})\n")
  # Write any main script leftovers we have, and append the include of test names
  file(APPEND "${_CTEST_FILE}" "${script}")
  file(APPEND "${_CTEST_FILE}" "include(\"${_CTEST_LIST_FILE}\")\n")
endfunction()

# To enable `include`ing this file in the unit test scripts, we only run
# the impl if an actual `TEST_EXECUTABLE` is provided.
if(CMAKE_SCRIPT_MODE_FILE AND DEFINED TEST_EXECUTABLE)
  catch3_discover_tests_impl(
    TEST_EXECUTABLE ${TEST_EXECUTABLE}
    TEST_EXECUTOR ${TEST_EXECUTOR}
    TEST_WORKING_DIR ${TEST_WORKING_DIR}
    TEST_SPEC ${TEST_SPEC}
    TEST_EXTRA_ARGS ${TEST_EXTRA_ARGS}
    TEST_PROPERTIES ${TEST_PROPERTIES}
    TEST_PREFIX ${TEST_PREFIX}
    TEST_SUFFIX ${TEST_SUFFIX}
    TEST_LIST ${TEST_LIST}
    TEST_REPORTER ${TEST_REPORTER}
    TEST_OUTPUT_DIR ${TEST_OUTPUT_DIR}
    TEST_OUTPUT_PREFIX ${TEST_OUTPUT_PREFIX}
    TEST_OUTPUT_SUFFIX ${TEST_OUTPUT_SUFFIX}
    TEST_DL_PATHS ${TEST_DL_PATHS}
    TEST_DL_FRAMEWORK_PATHS ${TEST_DL_FRAMEWORK_PATHS}
    CTEST_FILE ${CTEST_FILE}
    ADD_TAGS_AS_LABELS ${ADD_TAGS_AS_LABELS}
  )
endif()