diff options
| author | John Bytheway | 2020-04-13 08:34:27 -0400 |
|---|---|---|
| committer | Martin Hořeňovský | 2020-07-26 10:47:53 +0200 |
| commit | 80b0d6975c1582d03436ba82146983d557b0a1a6 (patch) | |
| tree | e9b9cc1e4eb1f5fb6a298eed1c89c16f6076a5d5 /tests | |
| parent | 36131f7ffa7f6a3e5a54de9fcf4f622e7b97a5a5 (diff) | |
| download | catch3-80b0d6975c1582d03436ba82146983d557b0a1a6.tar.gz | |
Add --min-duration option
A test runner already has a --durations option to print durations.
However, this isn't entirely satisfactory.
When there are many tests, this produces output spam which makes it hard
to find the test failure output. Nevertheless, it is helpful to be
informed of tests which are unusually slow.
Therefore, introduce a new option --min-duration that causes all
durations above a certain threshold to be printed. This allows slow
tests to be visible without mentioning every test.
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/CMakeLists.txt | 4 | ||||
| -rw-r--r-- | tests/SelfTest/TimingTests/Sleep.tests.cpp | 23 | ||||
| -rw-r--r-- | tests/TestScripts/testTimeThreshold.py | 41 |
3 files changed, 68 insertions, 0 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a5fdf802..72ece878 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -28,6 +28,7 @@ set(TEST_SOURCES ${SELF_TEST_DIR}/IntrospectiveTests/Xml.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/ToString.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/UniquePtr.tests.cpp + ${SELF_TEST_DIR}/TimingTests/Sleep.tests.cpp ${SELF_TEST_DIR}/UsageTests/Approx.tests.cpp ${SELF_TEST_DIR}/UsageTests/BDD.tests.cpp ${SELF_TEST_DIR}/UsageTests/Benchmark.tests.cpp @@ -201,6 +202,9 @@ set_tests_properties(TagAlias PROPERTIES add_test(NAME RandomTestOrdering COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tests/TestScripts/testRandomOrder.py $<TARGET_FILE:SelfTest>) +add_test(NAME TestTimeThreshold COMMAND ${PYTHON_EXECUTABLE} + ${CATCH_DIR}/tests/TestScripts/testTimeThreshold.py $<TARGET_FILE:SelfTest>) + add_test(NAME CheckConvenienceHeaders COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tools/scripts/checkConvenienceHeaders.py diff --git a/tests/SelfTest/TimingTests/Sleep.tests.cpp b/tests/SelfTest/TimingTests/Sleep.tests.cpp new file mode 100644 index 00000000..6ecc5fd2 --- /dev/null +++ b/tests/SelfTest/TimingTests/Sleep.tests.cpp @@ -0,0 +1,23 @@ +/* + * Copyright 2011 Two Blue Cubes Ltd. All rights reserved. + * + * Distributed under the Boost Software License, Version 1.0. (See accompanying + * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) + */ + +#include <catch2/catch_test_macros.hpp> + +#include <chrono> +#include <thread> + +TEST_CASE( "sleep_for_100ms", "[.min_duration_test][approvals]" ) +{ + std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); + CHECK( true ); +} + +TEST_CASE( "sleep_for_200ms", "[.min_duration_test][approvals]" ) +{ + std::this_thread::sleep_for( std::chrono::milliseconds( 200 ) ); + CHECK( true ); +} diff --git a/tests/TestScripts/testTimeThreshold.py b/tests/TestScripts/testTimeThreshold.py new file mode 100644 index 00000000..51804aae --- /dev/null +++ b/tests/TestScripts/testTimeThreshold.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +import subprocess +import sys + +def run_tests_with_threshold(self_test_exe, threshold): + cmd = [self_test_exe, '--min-duration', str(threshold), + '[min_duration_test]'] + process = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if stderr: + raise RuntimeError("Unexpected error output:\n" + + stderr.decode()) + if process.returncode != 0: + raise RuntimeError("Unexpected failure to run tests\n") + result = stdout.split(b'\n') + report_lines = [s.split() for s in result if b' s: ' in s] + tests_reported = [l[2] for l in report_lines] + times_reported = [float(l[0]) for l in report_lines] + return tests_reported, times_reported + +def check_times_at_least(times_reported, minimum): + for time in times_reported: + assert time >= minimum, ( + 'Time {} was less that requested minimum {}' .format( + time, minimum)) + +def main(): + self_test_exe, = sys.argv[1:] + tests, times = run_tests_with_threshold(self_test_exe, '0.15') + assert tests == [b'sleep_for_200ms'], ( + "Unexpected tests reported %s" % tests) + check_times_at_least(times, 0.15) + tests,times = run_tests_with_threshold(self_test_exe, '0') + assert tests == [b'sleep_for_100ms', b'sleep_for_200ms'], ( + "Unexpected tests reported %s" % tests) + check_times_at_least(times, 0) + +if __name__ == '__main__': + sys.exit(main()) |