Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

Add new reporter event called for each test case enter/exit

This means that e.g. for `TEST_CASE` with two sibling `SECTION`s
the event will fire twice, because the `TEST_CASE` will be entered
twice.

Closes #2107 (the event mentioned there already exists, but this
is its counterpart that we also want to provide to users)
Commit
10fb93cce8cb878fdfe8faa118b531258db96a26
Author
Martin Hořeňovský ·
Parents
4dcf8382c70c

Files · Download .tar.gz

Changes

 .../interfaces/catch_interfaces_reporter.hpp       |  6 ++
 src/catch2/internal/catch_run_context.cpp          | 19 +++++-
 .../reporters/catch_reporter_combined_tu.cpp       |  2 +
 .../reporters/catch_reporter_cumulative_base.hpp   |  3 +-
 .../reporters/catch_reporter_event_listener.hpp    |  5 ++
 src/catch2/reporters/catch_reporter_listening.cpp  | 17 +++++
 src/catch2/reporters/catch_reporter_listening.hpp  |  2 +
 .../reporters/catch_reporter_streaming_base.hpp    |  2 +
 tests/ExtraTests/CMakeLists.txt                    |  8 +++
 tests/ExtraTests/X21-PartialTestCaseEvents.cpp     | 74 ++++++++++++++++++++++
 tests/TestScripts/testPartialTestCaseEvent.py      | 72 +++++++++++++++++++++
 11 files changed, 207 insertions(+), 3 deletions(-)

diff --git a/src/catch2/interfaces/catch_interfaces_reporter.hpp b/src/catch2/interfaces/catch_interfaces_reporter.hpp
index 09480f04..0056c8ad 100644
--- a/src/catch2/interfaces/catch_interfaces_reporter.hpp
+++ b/src/catch2/interfaces/catch_interfaces_reporter.hpp
@@ -177,7 +177,10 @@ namespace Catch {
 
         virtual void testRunStarting( TestRunInfo const& testRunInfo ) = 0;
 
+        //! Called _once_ for each TEST_CASE, no matter how many times it is entered
         virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0;
+        //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections)
+        virtual void testCasePartialStarting( TestCaseInfo const& testInfo, uint64_t partNumber ) = 0;
         virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0;
 
         virtual void benchmarkPreparing( StringRef ) {}
@@ -191,6 +194,9 @@ namespace Catch {
         virtual bool assertionEnded( AssertionStats const& assertionStats ) = 0;
 
         virtual void sectionEnded( SectionStats const& sectionStats ) = 0;
+        //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections)
+        virtual void testCasePartialEnded(TestCaseStats const& testCaseStats, uint64_t partNumber ) = 0;
+        //! Called _once_ for each TEST_CASE, no matter how many times it is entered
         virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0;
         virtual void testRunEnded( TestRunStats const& testRunStats ) = 0;
 
diff --git a/src/catch2/internal/catch_run_context.cpp b/src/catch2/internal/catch_run_context.cpp
index 222ee932..1e4cc6b1 100644
--- a/src/catch2/internal/catch_run_context.cpp
+++ b/src/catch2/internal/catch_run_context.cpp
@@ -176,7 +176,7 @@ namespace Catch {
     }
 
     Totals RunContext::runTest(TestCaseHandle const& testCase) {
-        Totals prevTotals = m_totals;
+        const Totals prevTotals = m_totals;
 
         std::string redirectedCout;
         std::string redirectedCerr;
@@ -191,10 +191,25 @@ namespace Catch {
         ITracker& rootTracker = m_trackerContext.startRun();
         assert(rootTracker.isSectionTracker());
         static_cast<SectionTracker&>(rootTracker).addInitialFilters(m_config->getSectionsToRun());
+
+        uint64_t testRuns = 0;
         do {
             m_trackerContext.startCycle();
             m_testCaseTracker = &SectionTracker::acquire(m_trackerContext, TestCaseTracking::NameAndLocation(testInfo.name, testInfo.lineInfo));
-            runCurrentTest(redirectedCout, redirectedCerr);
+
+            m_reporter->testCasePartialStarting(testInfo, testRuns);
+
+            const auto beforeRunTotals = m_totals;
+            std::string oneRunCout, oneRunCerr;
+            runCurrentTest(oneRunCout, oneRunCerr);
+            redirectedCout += oneRunCout;
+            redirectedCerr += oneRunCerr;
+
+            const auto singleRunTotals = m_totals.delta(beforeRunTotals);
+            auto statsForOneRun = TestCaseStats(testInfo, singleRunTotals, redirectedCout, oneRunCerr, aborting());
+
+            m_reporter->testCasePartialEnded(statsForOneRun, testRuns);
+            ++testRuns;
         } while (!m_testCaseTracker->isSuccessfullyCompleted() && !aborting());
 
         Totals deltaTotals = m_totals.delta(prevTotals);
diff --git a/src/catch2/reporters/catch_reporter_combined_tu.cpp b/src/catch2/reporters/catch_reporter_combined_tu.cpp
index 084f46c5..dc5dc427 100644
--- a/src/catch2/reporters/catch_reporter_combined_tu.cpp
+++ b/src/catch2/reporters/catch_reporter_combined_tu.cpp
@@ -232,8 +232,10 @@ namespace Catch {
     void EventListenerBase::noMatchingTestCases( std::string const& ) {}
     void EventListenerBase::testRunStarting( TestRunInfo const& ) {}
     void EventListenerBase::testCaseStarting( TestCaseInfo const& ) {}
+    void EventListenerBase::testCasePartialStarting(TestCaseInfo const&, uint64_t) {}
     void EventListenerBase::sectionStarting( SectionInfo const& ) {}
     void EventListenerBase::sectionEnded( SectionStats const& ) {}
+    void EventListenerBase::testCasePartialEnded(TestCaseStats const&, uint64_t) {}
     void EventListenerBase::testCaseEnded( TestCaseStats const& ) {}
     void EventListenerBase::testRunEnded( TestRunStats const& ) {}
     void EventListenerBase::skipTest( TestCaseInfo const& ) {}
diff --git a/src/catch2/reporters/catch_reporter_cumulative_base.hpp b/src/catch2/reporters/catch_reporter_cumulative_base.hpp
index 8340aadb..0c7f7def 100644
--- a/src/catch2/reporters/catch_reporter_cumulative_base.hpp
+++ b/src/catch2/reporters/catch_reporter_cumulative_base.hpp
@@ -52,13 +52,14 @@ namespace Catch {
         void testRunStarting( TestRunInfo const& ) override {}
 
         void testCaseStarting( TestCaseInfo const& ) override {}
-
+        void testCasePartialStarting( TestCaseInfo const&, uint64_t ) override {}
         void sectionStarting( SectionInfo const& sectionInfo ) override;
 
         void assertionStarting( AssertionInfo const& ) override {}
 
         bool assertionEnded( AssertionStats const& assertionStats ) override;
         void sectionEnded( SectionStats const& sectionStats ) override;
+        void testCasePartialEnded( TestCaseStats const&, uint64_t ) override {}
         void testCaseEnded( TestCaseStats const& testCaseStats ) override;
         void testRunEnded( TestRunStats const& testRunStats ) override;
         //! Customization point: called after last test finishes (testRunEnded has been handled)
diff --git a/src/catch2/reporters/catch_reporter_event_listener.hpp b/src/catch2/reporters/catch_reporter_event_listener.hpp
index 034c6f3c..855acc43 100644
--- a/src/catch2/reporters/catch_reporter_event_listener.hpp
+++ b/src/catch2/reporters/catch_reporter_event_listener.hpp
@@ -35,11 +35,16 @@ namespace Catch {
         void noMatchingTestCases( std::string const& spec ) override;
         void testRunStarting( TestRunInfo const& testRunInfo ) override;
         void testCaseStarting( TestCaseInfo const& testInfo ) override;
+        void testCasePartialStarting( TestCaseInfo const& testInfo,
+                                      uint64_t partNumber ) override;
         void sectionStarting( SectionInfo const& sectionInfo ) override;
         void sectionEnded( SectionStats const& sectionStats ) override;
+        void testCasePartialEnded( TestCaseStats const& testCaseStats,
+                                   uint64_t partNumber ) override;
         void testCaseEnded( TestCaseStats const& testCaseStats ) override;
         void testRunEnded( TestRunStats const& testRunStats ) override;
         void skipTest( TestCaseInfo const& testInfo ) override;
+
     };
 
 } // end namespace Catch
diff --git a/src/catch2/reporters/catch_reporter_listening.cpp b/src/catch2/reporters/catch_reporter_listening.cpp
index 74cfd0aa..62f6af55 100644
--- a/src/catch2/reporters/catch_reporter_listening.cpp
+++ b/src/catch2/reporters/catch_reporter_listening.cpp
@@ -76,6 +76,15 @@ namespace Catch {
         m_reporter->testCaseStarting( testInfo );
     }
 
+    void
+    ListeningReporter::testCasePartialStarting( TestCaseInfo const& testInfo,
+                                                uint64_t partNumber ) {
+        for ( auto& listener : m_listeners ) {
+            listener->testCasePartialStarting( testInfo, partNumber );
+        }
+        m_reporter->testCasePartialStarting( testInfo, partNumber );
+    }
+
     void ListeningReporter::sectionStarting( SectionInfo const& sectionInfo ) {
         for ( auto& listener : m_listeners ) {
             listener->sectionStarting( sectionInfo );
@@ -105,6 +114,14 @@ namespace Catch {
         m_reporter->sectionEnded( sectionStats );
     }
 
+    void ListeningReporter::testCasePartialEnded( TestCaseStats const& testInfo,
+                                                  uint64_t partNumber ) {
+        for ( auto& listener : m_listeners ) {
+            listener->testCasePartialEnded( testInfo, partNumber );
+        }
+        m_reporter->testCasePartialEnded( testInfo, partNumber );
+    }
+
     void ListeningReporter::testCaseEnded( TestCaseStats const& testCaseStats ) {
         for ( auto& listener : m_listeners ) {
             listener->testCaseEnded( testCaseStats );
diff --git a/src/catch2/reporters/catch_reporter_listening.hpp b/src/catch2/reporters/catch_reporter_listening.hpp
index d00320b0..0e317653 100644
--- a/src/catch2/reporters/catch_reporter_listening.hpp
+++ b/src/catch2/reporters/catch_reporter_listening.hpp
@@ -41,12 +41,14 @@ namespace Catch {
 
         void testRunStarting( TestRunInfo const& testRunInfo ) override;
         void testCaseStarting( TestCaseInfo const& testInfo ) override;
+        void testCasePartialStarting(TestCaseInfo const& testInfo, uint64_t partNumber) override;
         void sectionStarting( SectionInfo const& sectionInfo ) override;
         void assertionStarting( AssertionInfo const& assertionInfo ) override;
 
         // The return value indicates if the messages buffer should be cleared:
         bool assertionEnded( AssertionStats const& assertionStats ) override;
         void sectionEnded( SectionStats const& sectionStats ) override;
+        void testCasePartialEnded(TestCaseStats const& testInfo, uint64_t partNumber) override;
         void testCaseEnded( TestCaseStats const& testCaseStats ) override;
         void testRunEnded( TestRunStats const& testRunStats ) override;
 
diff --git a/src/catch2/reporters/catch_reporter_streaming_base.hpp b/src/catch2/reporters/catch_reporter_streaming_base.hpp
index c51595e6..3051294e 100644
--- a/src/catch2/reporters/catch_reporter_streaming_base.hpp
+++ b/src/catch2/reporters/catch_reporter_streaming_base.hpp
@@ -51,6 +51,7 @@ namespace Catch {
         void testCaseStarting(TestCaseInfo const& _testInfo) override  {
             currentTestCaseInfo = &_testInfo;
         }
+        void testCasePartialStarting( TestCaseInfo const&, uint64_t ) override {}
         void sectionStarting(SectionInfo const& _sectionInfo) override {
             m_sectionStack.push_back(_sectionInfo);
         }
@@ -61,6 +62,7 @@ namespace Catch {
         void sectionEnded(SectionStats const& /* _sectionStats */) override {
             m_sectionStack.pop_back();
         }
+        void testCasePartialEnded( TestCaseStats const&, uint64_t ) override {}
         void testCaseEnded(TestCaseStats const& /* _testCaseStats */) override {
             currentTestCaseInfo = nullptr;
         }
diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt
index a861345d..bd7a575c 100644
--- a/tests/ExtraTests/CMakeLists.txt
+++ b/tests/ExtraTests/CMakeLists.txt
@@ -172,6 +172,14 @@ if (MSVC)
     list(APPEND CATCH_WARNING_TARGETS ${EXTRA_TEST_BINARIES} WindowsHeader)
 endif()
 
+
+add_executable(PartialTestCaseEvents ${TESTS_DIR}/X21-PartialTestCaseEvents.cpp)
+target_link_libraries(PartialTestCaseEvents PRIVATE Catch2WithMain)
+add_test(
+  NAME PartialTestCaseEvents
+  COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tests/TestScripts/testPartialTestCaseEvent.py $<TARGET_FILE:PartialTestCaseEvents>
+)
+
 #add_executable(DebugBreakMacros ${TESTS_DIR}/X12-CustomDebugBreakMacro.cpp)
 #target_link_libraries(DebugBreakMacros Catch2)
 #add_test(NAME DebugBreakMacros COMMAND DebugBreakMacros --break)
diff --git a/tests/ExtraTests/X21-PartialTestCaseEvents.cpp b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp
new file mode 100644
index 00000000..997b9b9f
--- /dev/null
+++ b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp
@@ -0,0 +1,74 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE_1_0.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+/**\file
+ * Registers custom reporter that reports testCase* events
+ *
+ * The resulting executable can then be used by an external Python script
+ * to verify that testCase{Starting,Ended} and testCasePartial{Starting,Ended}
+ * events are properly nested.
+ */
+
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/reporters/catch_reporter_streaming_base.hpp>
+#include <catch2/catch_test_case_info.hpp>
+#include <catch2/catch_reporter_registrars.hpp>
+#include <catch2/generators/catch_generators.hpp>
+
+
+#include <iostream>
+
+using Catch::TestCaseInfo;
+using Catch::TestCaseStats;
+
+class PartialReporter : public Catch::StreamingReporterBase {
+public:
+    using StreamingReporterBase::StreamingReporterBase;
+
+    ~PartialReporter() override; // = default
+
+    static std::string getDescription() {
+        return "Special reporter for testing TestCasePartialStarting/Ended events";
+    }
+
+    //! Called _once_ for each TEST_CASE, no matter how many times it is entered
+    void testCaseStarting(TestCaseInfo const& testInfo) override {
+        std::cout << "TestCaseStarting: " << testInfo.name << '\n';
+    }
+    //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections)
+    void testCasePartialStarting(TestCaseInfo const& testInfo, uint64_t partNumber) override {
+        std::cout << "TestCaseStartingPartial: " << testInfo.name << '#' << partNumber << '\n';
+    }
+
+
+    //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections)
+    void testCasePartialEnded(TestCaseStats const& testCaseStats, uint64_t partNumber) override {
+        std::cout << "TestCasePartialEnded: " << testCaseStats.testInfo->name << '#' << partNumber << '\n';
+    }
+    //! Called _once_ for each TEST_CASE, no matter how many times it is entered
+    void testCaseEnded(TestCaseStats const& testCaseStats) override {
+        std::cout << "TestCaseEnded: " << testCaseStats.testInfo->name << '\n';
+    }
+};
+PartialReporter::~PartialReporter() = default;
+
+
+CATCH_REGISTER_REPORTER("partial", PartialReporter)
+
+TEST_CASE("section") {
+    SECTION("A") {}
+    SECTION("B") {}
+    SECTION("C") {}
+    SECTION("D") {}
+}
+
+TEST_CASE("generator") {
+    auto _ = GENERATE(1, 2, 3, 4);
+    (void)_;
+}
diff --git a/tests/TestScripts/testPartialTestCaseEvent.py b/tests/TestScripts/testPartialTestCaseEvent.py
new file mode 100755
index 00000000..d119e6da
--- /dev/null
+++ b/tests/TestScripts/testPartialTestCaseEvent.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+"""
+This test script verifies that the testCasePartial{Starting,Ended} reporter
+events fire properly. This is done by calling a test binary compiled with
+reporter that reports specifically testCase* events, and verifying the
+outputs match what we expect.
+"""
+
+import subprocess
+import sys
+
+expected_section_output = '''\
+TestCaseStarting: section
+TestCaseStartingPartial: section#0
+TestCasePartialEnded: section#0
+TestCaseStartingPartial: section#1
+TestCasePartialEnded: section#1
+TestCaseStartingPartial: section#2
+TestCasePartialEnded: section#2
+TestCaseStartingPartial: section#3
+TestCasePartialEnded: section#3
+TestCaseEnded: section
+'''
+
+expected_generator_output = '''\
+TestCaseStarting: generator
+TestCaseStartingPartial: generator#0
+TestCasePartialEnded: generator#0
+TestCaseStartingPartial: generator#1
+TestCasePartialEnded: generator#1
+TestCaseStartingPartial: generator#2
+TestCasePartialEnded: generator#2
+TestCaseStartingPartial: generator#3
+TestCasePartialEnded: generator#3
+TestCaseEnded: generator
+'''
+
+
+from typing import List
+
+def get_test_output(test_exe: str, sections: bool) -> List[str]:
+    cmd = [test_exe, '--reporter', 'partial']
+    if sections:
+        cmd.append('section')
+    else:
+        cmd.append('generator')
+
+    ret = subprocess.run(cmd,
+                         stdout = subprocess.PIPE,
+                         stderr = subprocess.PIPE,
+                         timeout = 10,
+                         check = True,
+                         universal_newlines = True)
+
+    return ret.stdout
+
+def main():
+    test_exe, = sys.argv[1:]
+    actual_section_output = get_test_output(test_exe, sections = True)
+
+    assert actual_section_output == expected_section_output, (
+    'Sections\nActual:\n{}\nExpected:\n{}\n'.format(actual_section_output, expected_section_output))
+
+    actual_generator_output = get_test_output(test_exe, sections = False)
+    assert actual_generator_output == expected_generator_output, (
+    'Generators\nActual:\n{}\nExpected:\n{}\n'.format(actual_generator_output, expected_generator_output))
+
+
+
+if __name__ == '__main__':
+    sys.exit(main())