diff options
| author | Martin Hořeňovský | 2022-02-19 21:27:53 +0100 |
|---|---|---|
| committer | Martin Hořeňovský | 2022-02-20 19:32:30 +0100 |
| commit | 72f037266415fee521adc598bd324fea8001cd77 (patch) | |
| tree | 2dd55c68f97c5e6752914045ed5764cf00dc565d /tests | |
| parent | 4396a9119fa32abf966f189c6f6d9c54698d89c8 (diff) | |
| download | catch3-72f037266415fee521adc598bd324fea8001cd77.tar.gz | |
Add tests for CMake configure toggles passing to Catch2 build
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/CMakeLists.txt | 18 | ||||
| -rw-r--r-- | tests/TestScripts/ConfigureTestsCommon.py | 75 | ||||
| -rw-r--r-- | tests/TestScripts/testConfigureDefaultReporter.py | 44 | ||||
| -rw-r--r-- | tests/TestScripts/testConfigureDisable.py | 48 | ||||
| -rw-r--r-- | tests/TestScripts/testConfigureDisableStringification.py | 44 | ||||
| -rw-r--r-- | tests/TestScripts/testConfigureExperimentalRedirect.py | 49 |
6 files changed, 278 insertions, 0 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0f25bfe5..2115a690 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -471,6 +471,24 @@ set_tests_properties("Reporters::DashAsLocationInReporterSpecSendsOutputToStdout PASS_REGULAR_EXPRESSION "All tests passed \\(5 assertions in 1 test case\\)" ) +if (CATCH_ENABLE_CONFIGURE_TESTS) + add_test(NAME "CMakeConfig::DefaultReporter" + COMMAND + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDefaultReporter.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" + ) + add_test(NAME "CMakeConfig::Disable" + COMMAND + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDisable.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" + ) + add_test(NAME "CMakeConfig::DisableStringification" + COMMAND + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureDisableStringification.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" + ) + add_test(NAME "CMakeConfig::ExperimentalRedirect" + COMMAND + "${PYTHON_EXECUTABLE}" "${CMAKE_CURRENT_LIST_DIR}/TestScripts/testConfigureExperimentalRedirect.py" "${CATCH_DIR}" "${CMAKE_CURRENT_BINARY_DIR}" + ) +endif() if (CATCH_USE_VALGRIND) add_test(NAME ValgrindRunTests COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest>) diff --git a/tests/TestScripts/ConfigureTestsCommon.py b/tests/TestScripts/ConfigureTestsCommon.py new file mode 100644 index 00000000..ad5b358b --- /dev/null +++ b/tests/TestScripts/ConfigureTestsCommon.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +from typing import List, Tuple + +import os +import subprocess + +def configure_and_build(source_path: str, project_path: str, options: List[Tuple[str, str]]): + base_configure_cmd = ['cmake', + '-B{}'.format(project_path), + '-H{}'.format(source_path), + '-DCMAKE_BUILD_TYPE=Debug', + '-DCATCH_DEVELOPMENT_BUILD=ON'] + for option, value in options: + base_configure_cmd.append('-D{}={}'.format(option, value)) + try: + subprocess.run(base_configure_cmd, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + check = True) + except subprocess.SubprocessError as ex: + print("Could not configure build to '{}' from '{}'".format(project_path, source_path)) + print("Return code: {}".format(ex.returncode)) + print("output: {}".format(ex.output)) + raise + print('Configuring {} finished'.format(project_path)) + + build_cmd = ['cmake', + '--build', '{}'.format(project_path), + # For now we assume that we only need Debug config + '--config', 'Debug'] + try: + subprocess.run(build_cmd, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + check = True) + except subprocess.SubprocessError as ex: + print("Could not build project in '{}'".format(project_path)) + print("Return code: {}".format(ex.returncode)) + print("output: {}".format(ex.output)) + raise + print('Building {} finished'.format(project_path)) + +def run_and_return_output(base_path: str, binary_name: str, other_options: List[str]) -> Tuple[str, str]: + # For now we assume that Windows builds are done using MSBuild under + # Debug configuration. This means that we need to add "Debug" folder + # to the path when constructing it. On Linux, we don't add anything. + config_path = "Debug" if os.name == 'nt' else "" + full_path = os.path.join(base_path, config_path, binary_name) + + base_cmd = [full_path] + base_cmd.extend(other_options) + + try: + ret = subprocess.run(base_cmd, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE, + check = True, + universal_newlines = True) + except subprocess.SubprocessError as ex: + print('Could not run "{}"'.format(base_cmd)) + print('Args: "{}"'.format(other_options)) + print('Return code: {}'.format(ex.returncode)) + print('stdout: {}'.format(ex.stdout)) + print('stderr: {}'.format(ex.stdout)) + raise + + return (ret.stdout, ret.stderr) diff --git a/tests/TestScripts/testConfigureDefaultReporter.py b/tests/TestScripts/testConfigureDefaultReporter.py new file mode 100644 index 00000000..24ec8228 --- /dev/null +++ b/tests/TestScripts/testConfigureDefaultReporter.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +from ConfigureTestsCommon import configure_and_build, run_and_return_output + +import os +import re +import sys + +""" +Tests the CMake configure option for CATCH_CONFIG_DEFAULT_REPORTER + +Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt +exists, and path to where the output files should be stored. +""" + +if len(sys.argv) != 3: + print('Wrong number of arguments: {}'.format(len(sys.argv))) + print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0])) + exit(1) + +catch2_source_path = os.path.abspath(sys.argv[1]) +build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DefaultReporter') + +configure_and_build(catch2_source_path, + build_dir_path, + [("CATCH_CONFIG_DEFAULT_REPORTER", "compact")]) + +stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTest', ['[approx][custom]']) + + +# This matches the summary line made by compact reporter, console reporter's +# summary line does not match the regex. +summary_regex = 'Passed \d+ test case with \d+ assertions.' +if not re.match(summary_regex, stdout): + print("Could not find '{}' in the stdout".format(summary_regex)) + print('stdout: "{}"'.format(stdout)) + exit(2) diff --git a/tests/TestScripts/testConfigureDisable.py b/tests/TestScripts/testConfigureDisable.py new file mode 100644 index 00000000..92946d8d --- /dev/null +++ b/tests/TestScripts/testConfigureDisable.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +from ConfigureTestsCommon import configure_and_build, run_and_return_output + +import os +import re +import sys + +""" +Tests the CMake configure option for CATCH_CONFIG_DISABLE + +Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt +exists, and path to where the output files should be stored. +""" + +if len(sys.argv) != 3: + print('Wrong number of arguments: {}'.format(len(sys.argv))) + print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0])) + exit(1) + +catch2_source_path = os.path.abspath(sys.argv[1]) +build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'Disable') + +configure_and_build(catch2_source_path, + build_dir_path, + [("CATCH_CONFIG_DISABLE", "ON"), + # We need to turn off WERROR, because the compilers + # can see that the various variables inside test cases + # are set but unused. + ("CATCH_ENABLE_WERROR", "OFF")]) + +stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), + 'SelfTest', + ['--allow-running-no-tests']) + + +summary_line = 'No tests ran' +if not summary_line in stdout: + print("Could not find '{}' in the stdout".format(summary_line)) + print('stdout: "{}"'.format(stdout)) + exit(2) diff --git a/tests/TestScripts/testConfigureDisableStringification.py b/tests/TestScripts/testConfigureDisableStringification.py new file mode 100644 index 00000000..a8a53e4f --- /dev/null +++ b/tests/TestScripts/testConfigureDisableStringification.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +from ConfigureTestsCommon import configure_and_build, run_and_return_output + +import os +import re +import sys + +""" +Tests the CMake configure option for CATCH_CONFIG_DISABLE_STRINGIFICATION + +Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt +exists, and path to where the output files should be stored. +""" + +if len(sys.argv) != 3: + print('Wrong number of arguments: {}'.format(len(sys.argv))) + print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0])) + exit(1) + +catch2_source_path = os.path.abspath(sys.argv[1]) +build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DisableStringification') + +configure_and_build(catch2_source_path, + build_dir_path, + [("CATCH_CONFIG_DISABLE_STRINGIFICATION", "ON")]) + +stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), + 'SelfTest', + ['-s', '[approx][custom]']) + + +required_output = 'Disabled by CATCH_CONFIG_DISABLE_STRINGIFICATION' +if not required_output in stdout: + print("Could not find '{}' in the stdout".format(required_output)) + print('stdout: "{}"'.format(stdout)) + exit(2) diff --git a/tests/TestScripts/testConfigureExperimentalRedirect.py b/tests/TestScripts/testConfigureExperimentalRedirect.py new file mode 100644 index 00000000..b5313fe9 --- /dev/null +++ b/tests/TestScripts/testConfigureExperimentalRedirect.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# Copyright Catch2 Authors +# Distributed under the Boost Software License, Version 1.0. +# (See accompanying file LICENSE_1_0.txt or copy at +# https://www.boost.org/LICENSE_1_0.txt) + +# SPDX-License-Identifier: BSL-1.0 + +from ConfigureTestsCommon import configure_and_build, run_and_return_output + +import os +import re +import sys + +""" +Tests the CMake configure option for CATCH_CONFIG_EXPERIMENTAL_REDIRECT + +Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt +exists, and path to where the output files should be stored. +""" + +if len(sys.argv) != 3: + print('Wrong number of arguments: {}'.format(len(sys.argv))) + print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0])) + exit(1) + +catch2_source_path = os.path.abspath(sys.argv[1]) +build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'ExperimentalRedirect') + +configure_and_build(catch2_source_path, + build_dir_path, + [("CATCH_CONFIG_EXPERIMENTAL_REDIRECT", "ON")]) + +stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), + 'SelfTest', + ['-r', 'xml', '"has printf"']) + + +# The print from printf must be within the XML's reporter stdout tag. +required_output = '''\ + <StdOut> +loose text artifact + </StdOut> +''' +if not required_output in stdout: + print("Could not find '{}' in the stdout".format(required_output)) + print('stdout: "{}"'.format(stdout)) + exit(2) |