Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/tests
diff options
context:
space:
mode:
authorMartin Hořeňovský2021-12-13 15:15:23 +0100
committerMartin Hořeňovský2021-12-18 20:50:00 +0100
commit9f2dca5384a6a8f9f37a7dc2d998cac616545592 (patch)
tree181d46f4eb905903e08af58a8a35108f87a24505 /tests
parent602e484f027ccdfb59cfde852e20283ad1635f81 (diff)
downloadcatch3-9f2dca5384a6a8f9f37a7dc2d998cac616545592.tar.gz
Exit with non-0 return code if no tests were run
A new flag, `--allow-running-no-tests` was added to override this behaviour if exit code of 0 was desired. This change also made `-w NoTests` obsolete, and so it has been removed.
Diffstat (limited to 'tests')
-rw-r--r--tests/CMakeLists.txt22
-rw-r--r--tests/ExtraTests/CMakeLists.txt17
-rw-r--r--tests/ExtraTests/X92-NoTests.cpp11
-rw-r--r--tests/SelfTest/WarnAboutNoTests.cmake19
4 files changed, 46 insertions, 23 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 6a7563ec..4a84ad8e 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -207,15 +207,29 @@ set_tests_properties(List::Reporters::XmlOutput PROPERTIES
add_test(NAME NoAssertions COMMAND $<TARGET_FILE:SelfTest> -w NoAssertions "An empty test with no assertions")
set_tests_properties(NoAssertions PROPERTIES PASS_REGULAR_EXPRESSION "No assertions in test case")
-add_test(NAME NoTest COMMAND $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___")
-set_tests_properties(NoTest PROPERTIES
+# We cannot combine a regular expression on output with return code check
+# in one test, so we register two instead of making a checking script because
+# the runtime overhead is small enough.
+add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-1 COMMAND $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___")
+
+add_test(NAME TestSpecs::CombiningMatchingAndNonMatchingIsOk-2 COMMAND $<TARGET_FILE:SelfTest> Tracker, "___nonexistent_test___")
+set_tests_properties(TestSpecs::CombiningMatchingAndNonMatchingIsOk-2 PROPERTIES
PASS_REGULAR_EXPRESSION "No test cases matched '___nonexistent_test___'"
FAIL_REGULAR_EXPRESSION "No tests ran"
)
-add_test(NAME WarnAboutNoTests COMMAND ${CMAKE_COMMAND} -P ${SELF_TEST_DIR}/WarnAboutNoTests.cmake $<TARGET_FILE:SelfTest>)
+add_test(NAME TestSpecs::NoMatchedTestsFail
+ COMMAND $<TARGET_FILE:SelfTest> "___nonexistent_test___"
+)
+set_tests_properties(TestSpecs::NoMatchedTestsFail
+ PROPERTIES
+ WILL_FAIL ON
+)
+add_test(NAME TestSpecs::OverrideFailureWithNoMatchedTests
+ COMMAND $<TARGET_FILE:SelfTest> "___nonexistent_test___" --allow-running-no-tests
+)
-add_test(NAME UnmatchedOutputFilter COMMAND $<TARGET_FILE:SelfTest> [this-tag-does-not-exist] -w NoTests)
+add_test(NAME UnmatchedOutputFilter COMMAND $<TARGET_FILE:SelfTest> [this-tag-does-not-exist])
set_tests_properties(UnmatchedOutputFilter
PROPERTIES
PASS_REGULAR_EXPRESSION "No test cases matched '\\[this-tag-does-not-exist\\]'"
diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt
index f7e9ec18..b3b8dbec 100644
--- a/tests/ExtraTests/CMakeLists.txt
+++ b/tests/ExtraTests/CMakeLists.txt
@@ -299,6 +299,22 @@ set_tests_properties(
# PASS_REGULAR_EXPRESSION "Pretty please, break into debugger"
#)
+add_executable(NoTests ${TESTS_DIR}/X92-NoTests.cpp)
+target_link_libraries(NoTests PRIVATE Catch2::Catch2WithMain)
+
+add_test(
+ NAME TestSpecs::EmptySpecWithNoTestsFails
+ COMMAND $<TARGET_FILE:NoTests>
+)
+set_tests_properties(TestSpecs::EmptySpecWithNoTestsFails
+ PROPERTIES
+ WILL_FAIL ON
+)
+add_test(
+ NAME TestSpecs::OverrideFailureWithEmptySpec
+ COMMAND $<TARGET_FILE:NoTests> --allow-running-no-tests
+)
+
set( EXTRA_TEST_BINARIES
PrefixedMacros
DisabledMacros
@@ -310,6 +326,7 @@ set( EXTRA_TEST_BINARIES
DuplicatedTestCases-SameNameAndTags
DuplicatedTestCases-SameNameDifferentTags
DuplicatedTestCases-DuplicatedTestCaseMethods
+ NoTests
# DebugBreakMacros
)
diff --git a/tests/ExtraTests/X92-NoTests.cpp b/tests/ExtraTests/X92-NoTests.cpp
new file mode 100644
index 00000000..fc6bf597
--- /dev/null
+++ b/tests/ExtraTests/X92-NoTests.cpp
@@ -0,0 +1,11 @@
+
+// Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE_1_0.txt or copy at
+// https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+/**\file
+ * Links into executable with no tests which should fail when run
+ */
diff --git a/tests/SelfTest/WarnAboutNoTests.cmake b/tests/SelfTest/WarnAboutNoTests.cmake
deleted file mode 100644
index 4637e3f3..00000000
--- a/tests/SelfTest/WarnAboutNoTests.cmake
+++ /dev/null
@@ -1,19 +0,0 @@
-# Workaround for a peculiarity where CTest disregards the return code from a
-# test command if a PASS_REGULAR_EXPRESSION is also set
-execute_process(
- COMMAND ${CMAKE_ARGV3} -w NoTests "___nonexistent_test___"
- RESULT_VARIABLE ret
- OUTPUT_VARIABLE out
-)
-
-message("${out}")
-
-if(NOT ${ret} MATCHES "^[0-9]+$")
- message(FATAL_ERROR "${ret}")
-endif()
-
-if(${ret} EQUAL 0)
- message(FATAL_ERROR "Expected nonzero return code")
-elseif(${out} MATCHES "Helper failed with")
- message(FATAL_ERROR "Helper failed")
-endif()