Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/tests/TestScripts
diff options
context:
space:
mode:
authorMartin Hořeňovský2022-02-19 21:27:53 +0100
committerMartin Hořeňovský2022-02-20 19:32:30 +0100
commit72f037266415fee521adc598bd324fea8001cd77 (patch)
tree2dd55c68f97c5e6752914045ed5764cf00dc565d /tests/TestScripts
parent4396a9119fa32abf966f189c6f6d9c54698d89c8 (diff)
downloadcatch3-72f037266415fee521adc598bd324fea8001cd77.tar.gz
Add tests for CMake configure toggles passing to Catch2 build
Diffstat (limited to 'tests/TestScripts')
-rw-r--r--tests/TestScripts/ConfigureTestsCommon.py75
-rw-r--r--tests/TestScripts/testConfigureDefaultReporter.py44
-rw-r--r--tests/TestScripts/testConfigureDisable.py48
-rw-r--r--tests/TestScripts/testConfigureDisableStringification.py44
-rw-r--r--tests/TestScripts/testConfigureExperimentalRedirect.py49
5 files changed, 260 insertions, 0 deletions
diff --git a/tests/TestScripts/ConfigureTestsCommon.py b/tests/TestScripts/ConfigureTestsCommon.py
new file mode 100644
index 00000000..ad5b358b
--- /dev/null
+++ b/tests/TestScripts/ConfigureTestsCommon.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+# Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from typing import List, Tuple
+
+import os
+import subprocess
+
+def configure_and_build(source_path: str, project_path: str, options: List[Tuple[str, str]]):
+ base_configure_cmd = ['cmake',
+ '-B{}'.format(project_path),
+ '-H{}'.format(source_path),
+ '-DCMAKE_BUILD_TYPE=Debug',
+ '-DCATCH_DEVELOPMENT_BUILD=ON']
+ for option, value in options:
+ base_configure_cmd.append('-D{}={}'.format(option, value))
+ try:
+ subprocess.run(base_configure_cmd,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.STDOUT,
+ check = True)
+ except subprocess.SubprocessError as ex:
+ print("Could not configure build to '{}' from '{}'".format(project_path, source_path))
+ print("Return code: {}".format(ex.returncode))
+ print("output: {}".format(ex.output))
+ raise
+ print('Configuring {} finished'.format(project_path))
+
+ build_cmd = ['cmake',
+ '--build', '{}'.format(project_path),
+ # For now we assume that we only need Debug config
+ '--config', 'Debug']
+ try:
+ subprocess.run(build_cmd,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.STDOUT,
+ check = True)
+ except subprocess.SubprocessError as ex:
+ print("Could not build project in '{}'".format(project_path))
+ print("Return code: {}".format(ex.returncode))
+ print("output: {}".format(ex.output))
+ raise
+ print('Building {} finished'.format(project_path))
+
+def run_and_return_output(base_path: str, binary_name: str, other_options: List[str]) -> Tuple[str, str]:
+ # For now we assume that Windows builds are done using MSBuild under
+ # Debug configuration. This means that we need to add "Debug" folder
+ # to the path when constructing it. On Linux, we don't add anything.
+ config_path = "Debug" if os.name == 'nt' else ""
+ full_path = os.path.join(base_path, config_path, binary_name)
+
+ base_cmd = [full_path]
+ base_cmd.extend(other_options)
+
+ try:
+ ret = subprocess.run(base_cmd,
+ stdout = subprocess.PIPE,
+ stderr = subprocess.PIPE,
+ check = True,
+ universal_newlines = True)
+ except subprocess.SubprocessError as ex:
+ print('Could not run "{}"'.format(base_cmd))
+ print('Args: "{}"'.format(other_options))
+ print('Return code: {}'.format(ex.returncode))
+ print('stdout: {}'.format(ex.stdout))
+ print('stderr: {}'.format(ex.stdout))
+ raise
+
+ return (ret.stdout, ret.stderr)
diff --git a/tests/TestScripts/testConfigureDefaultReporter.py b/tests/TestScripts/testConfigureDefaultReporter.py
new file mode 100644
index 00000000..24ec8228
--- /dev/null
+++ b/tests/TestScripts/testConfigureDefaultReporter.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+# Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from ConfigureTestsCommon import configure_and_build, run_and_return_output
+
+import os
+import re
+import sys
+
+"""
+Tests the CMake configure option for CATCH_CONFIG_DEFAULT_REPORTER
+
+Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
+exists, and path to where the output files should be stored.
+"""
+
+if len(sys.argv) != 3:
+ print('Wrong number of arguments: {}'.format(len(sys.argv)))
+ print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
+ exit(1)
+
+catch2_source_path = os.path.abspath(sys.argv[1])
+build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DefaultReporter')
+
+configure_and_build(catch2_source_path,
+ build_dir_path,
+ [("CATCH_CONFIG_DEFAULT_REPORTER", "compact")])
+
+stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTest', ['[approx][custom]'])
+
+
+# This matches the summary line made by compact reporter, console reporter's
+# summary line does not match the regex.
+summary_regex = 'Passed \d+ test case with \d+ assertions.'
+if not re.match(summary_regex, stdout):
+ print("Could not find '{}' in the stdout".format(summary_regex))
+ print('stdout: "{}"'.format(stdout))
+ exit(2)
diff --git a/tests/TestScripts/testConfigureDisable.py b/tests/TestScripts/testConfigureDisable.py
new file mode 100644
index 00000000..92946d8d
--- /dev/null
+++ b/tests/TestScripts/testConfigureDisable.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+
+# Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from ConfigureTestsCommon import configure_and_build, run_and_return_output
+
+import os
+import re
+import sys
+
+"""
+Tests the CMake configure option for CATCH_CONFIG_DISABLE
+
+Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
+exists, and path to where the output files should be stored.
+"""
+
+if len(sys.argv) != 3:
+ print('Wrong number of arguments: {}'.format(len(sys.argv)))
+ print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
+ exit(1)
+
+catch2_source_path = os.path.abspath(sys.argv[1])
+build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'Disable')
+
+configure_and_build(catch2_source_path,
+ build_dir_path,
+ [("CATCH_CONFIG_DISABLE", "ON"),
+ # We need to turn off WERROR, because the compilers
+ # can see that the various variables inside test cases
+ # are set but unused.
+ ("CATCH_ENABLE_WERROR", "OFF")])
+
+stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
+ 'SelfTest',
+ ['--allow-running-no-tests'])
+
+
+summary_line = 'No tests ran'
+if not summary_line in stdout:
+ print("Could not find '{}' in the stdout".format(summary_line))
+ print('stdout: "{}"'.format(stdout))
+ exit(2)
diff --git a/tests/TestScripts/testConfigureDisableStringification.py b/tests/TestScripts/testConfigureDisableStringification.py
new file mode 100644
index 00000000..a8a53e4f
--- /dev/null
+++ b/tests/TestScripts/testConfigureDisableStringification.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+# Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from ConfigureTestsCommon import configure_and_build, run_and_return_output
+
+import os
+import re
+import sys
+
+"""
+Tests the CMake configure option for CATCH_CONFIG_DISABLE_STRINGIFICATION
+
+Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
+exists, and path to where the output files should be stored.
+"""
+
+if len(sys.argv) != 3:
+ print('Wrong number of arguments: {}'.format(len(sys.argv)))
+ print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
+ exit(1)
+
+catch2_source_path = os.path.abspath(sys.argv[1])
+build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DisableStringification')
+
+configure_and_build(catch2_source_path,
+ build_dir_path,
+ [("CATCH_CONFIG_DISABLE_STRINGIFICATION", "ON")])
+
+stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
+ 'SelfTest',
+ ['-s', '[approx][custom]'])
+
+
+required_output = 'Disabled by CATCH_CONFIG_DISABLE_STRINGIFICATION'
+if not required_output in stdout:
+ print("Could not find '{}' in the stdout".format(required_output))
+ print('stdout: "{}"'.format(stdout))
+ exit(2)
diff --git a/tests/TestScripts/testConfigureExperimentalRedirect.py b/tests/TestScripts/testConfigureExperimentalRedirect.py
new file mode 100644
index 00000000..b5313fe9
--- /dev/null
+++ b/tests/TestScripts/testConfigureExperimentalRedirect.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+# Copyright Catch2 Authors
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# https://www.boost.org/LICENSE_1_0.txt)
+
+# SPDX-License-Identifier: BSL-1.0
+
+from ConfigureTestsCommon import configure_and_build, run_and_return_output
+
+import os
+import re
+import sys
+
+"""
+Tests the CMake configure option for CATCH_CONFIG_EXPERIMENTAL_REDIRECT
+
+Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
+exists, and path to where the output files should be stored.
+"""
+
+if len(sys.argv) != 3:
+ print('Wrong number of arguments: {}'.format(len(sys.argv)))
+ print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
+ exit(1)
+
+catch2_source_path = os.path.abspath(sys.argv[1])
+build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'ExperimentalRedirect')
+
+configure_and_build(catch2_source_path,
+ build_dir_path,
+ [("CATCH_CONFIG_EXPERIMENTAL_REDIRECT", "ON")])
+
+stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
+ 'SelfTest',
+ ['-r', 'xml', '"has printf"'])
+
+
+# The print from printf must be within the XML's reporter stdout tag.
+required_output = '''\
+ <StdOut>
+loose text artifact
+ </StdOut>
+'''
+if not required_output in stdout:
+ print("Could not find '{}' in the stdout".format(required_output))
+ print('stdout: "{}"'.format(stdout))
+ exit(2)