Add new SKIP macro for skipping tests at runtime (#2360)

* Add new SKIP macro for skipping tests at runtime

This adds a new `SKIP` macro for dynamically skipping tests at runtime.
The "skipped" status of a test case is treated as a first-class citizen,
like "succeeded" or "failed", and is reported with a new color on the
console.

* Don't show "skipped assertions" in console/compact reporters

Also extend skip tests to cover a few more use cases.

* Return exit code 4 if all test cases are skipped

* Use LightGrey for the skip colour

This isn't great, but is better than the deep blue that was borderline
invisible on dark backgrounds. The fix is to redo the colouring
a bit, including introducing light-blue that is actually visible.

* Add support for explicit skips in all reporters

* --allow-running-no-tests also allows all tests to be skipped

* Add docs for SKIP macro, deprecate IEventListener::skipTest

Co-authored-by: Martin Hořeňovský <martin.horenovsky@gmail.com>

7 files changed, 28 insertions, 1 deletions

diff --git a/src/catch2/internal/catch_assertion_handler.cpp b/src/catch2/internal/catch_assertion_handler.cpp
index f051314c..0b14e0bb 100644
--- a/src/catch2/internal/catch_assertion_handler.cpp
+++ b/src/catch2/internal/catch_assertion_handler.cpp
@@ -50,6 +50,13 @@ namespace Catch {
         if (m_reaction.shouldThrow) {
             throw_test_failure_exception();
         }
+        if ( m_reaction.shouldSkip ) {
+#if !defined( CATCH_CONFIG_DISABLE_EXCEPTIONS )
+            throw Catch::TestSkipException();
+#else
+            CATCH_ERROR( "Explicitly skipping tests during runtime requires exceptions" );
+#endif
+        }
     }
     void AssertionHandler::setCompleted() {
         m_completed = true;
diff --git a/src/catch2/internal/catch_assertion_handler.hpp b/src/catch2/internal/catch_assertion_handler.hpp
index 36b55243..ae7776d8 100644
--- a/src/catch2/internal/catch_assertion_handler.hpp
+++ b/src/catch2/internal/catch_assertion_handler.hpp
@@ -22,6 +22,7 @@ namespace Catch {
     struct AssertionReaction {
         bool shouldDebugBreak = false;
         bool shouldThrow = false;
+        bool shouldSkip = false;
     };
 
     class AssertionHandler {
diff --git a/src/catch2/internal/catch_console_colour.hpp b/src/catch2/internal/catch_console_colour.hpp
index 9aa6a163..d9144315 100644
--- a/src/catch2/internal/catch_console_colour.hpp
+++ b/src/catch2/internal/catch_console_colour.hpp
@@ -47,6 +47,7 @@ namespace Catch {
 
             Error = BrightRed,
             Success = Green,
+            Skip = LightGrey,
 
             OriginalExpression = Cyan,
             ReconstructedExpression = BrightYellow,
diff --git a/src/catch2/internal/catch_exception_translator_registry.cpp b/src/catch2/internal/catch_exception_translator_registry.cpp
index 2a240a9b..0645c6ce 100644
--- a/src/catch2/internal/catch_exception_translator_registry.cpp
+++ b/src/catch2/internal/catch_exception_translator_registry.cpp
@@ -44,6 +44,9 @@ namespace Catch {
         catch( TestFailureException& ) {
             std::rethrow_exception(std::current_exception());
         }
+        catch( TestSkipException& ) {
+            std::rethrow_exception(std::current_exception());
+        }
         catch( std::exception const& ex ) {
             return ex.what();
         }
diff --git a/src/catch2/internal/catch_result_type.hpp b/src/catch2/internal/catch_result_type.hpp
index faf0683d..e66afaff 100644
--- a/src/catch2/internal/catch_result_type.hpp
+++ b/src/catch2/internal/catch_result_type.hpp
@@ -16,6 +16,8 @@ namespace Catch {
         Ok = 0,
         Info = 1,
         Warning = 2,
+        // TODO: Should explicit skip be considered "not OK" (cf. isOk)? I.e., should it have the failure bit?
+        ExplicitSkip = 4,
 
         FailureBit = 0x10,
 
diff --git a/src/catch2/internal/catch_run_context.cpp b/src/catch2/internal/catch_run_context.cpp
index d2e8fb8c..e1b81d0b 100644
--- a/src/catch2/internal/catch_run_context.cpp
+++ b/src/catch2/internal/catch_run_context.cpp
@@ -270,6 +270,9 @@ namespace Catch {
         if (result.getResultType() == ResultWas::Ok) {
             m_totals.assertions.passed++;
             m_lastAssertionPassed = true;
+        } else if (result.getResultType() == ResultWas::ExplicitSkip) {
+            m_totals.assertions.skipped++;
+            m_lastAssertionPassed = true;
         } else if (!result.succeeded()) {
             m_lastAssertionPassed = false;
             if (result.isOk()) {
@@ -475,6 +478,8 @@ namespace Catch {
             duration = timer.getElapsedSeconds();
         } CATCH_CATCH_ANON (TestFailureException&) {
             // This just means the test was aborted due to failure
+        } CATCH_CATCH_ANON (TestSkipException&) {
+            // This just means the test was explicitly skipped
         } CATCH_CATCH_ALL {
             // Under CATCH_CONFIG_FAST_COMPILE, unexpected exceptions under REQUIRE assertions
             // are reported without translation at the point of origin.
@@ -571,8 +576,13 @@ namespace Catch {
         data.message = static_cast<std::string>(message);
         AssertionResult assertionResult{ m_lastAssertionInfo, data };
         assertionEnded( assertionResult );
-        if( !assertionResult.isOk() )
+        if ( !assertionResult.isOk() ) {
             populateReaction( reaction );
+        } else if ( resultType == ResultWas::ExplicitSkip ) {
+            // TODO: Need to handle this explicitly, as ExplicitSkip is
+            // considered "OK"
+            reaction.shouldSkip = true;
+        }
     }
     void RunContext::handleUnexpectedExceptionNotThrown(
             AssertionInfo const& info,
diff --git a/src/catch2/internal/catch_test_failure_exception.hpp b/src/catch2/internal/catch_test_failure_exception.hpp
index 810a81c9..13c5fc08 100644
--- a/src/catch2/internal/catch_test_failure_exception.hpp
+++ b/src/catch2/internal/catch_test_failure_exception.hpp
@@ -20,6 +20,9 @@ namespace Catch {
      */
     [[noreturn]] void throw_test_failure_exception();
 
+    //! Used to signal that the remainder of a test should be skipped
+    struct TestSkipException{};
+
 } // namespace Catch
 
 #endif // CATCH_TEST_FAILURE_EXCEPTION_HPP_INCLUDED