Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorMartin Hořeňovský2021-09-15 21:46:07 +0200
committerMartin Hořeňovský2021-09-15 23:38:43 +0200
commit3f8cae8025f4f5e804383f44bb393a63bcef90a4 (patch)
tree6968a937b713cc21820c3ae32ca45239dd0a809a
parent2c82f82ee211b011460f2f44647b69e0feaf3d4e (diff)
downloadcatch3-3f8cae8025f4f5e804383f44bb393a63bcef90a4.tar.gz
Add consistency-checking event listener to SelfTest
This event listener performs basic consistency checks (akin to matching braces) on events that are passed to the listeners when the `SelfTest` test binary is run. The current checks are about nesting events (e.g. `testCaseStarting` cannot be received before `testRunStarting`, `sectionStarting` can only be received when a test case is active, etc), and matching up counts of starting/ended events. The simplicity means that it could be confused by starting/ended events matching up but being out of order, e.g. ``` * test case A starting * test case B ended * test case B starting * test case A ended ``` would be accepted, even though it is wrong. However, doing full order checking would be much more implementation work, for relatively little benefit, so it is left out for now.
-rw-r--r--tests/SelfTest/TestRegistrations.cpp171
1 files changed, 161 insertions, 10 deletions
diff --git a/tests/SelfTest/TestRegistrations.cpp b/tests/SelfTest/TestRegistrations.cpp
index cb8376bd..45a01c02 100644
--- a/tests/SelfTest/TestRegistrations.cpp
+++ b/tests/SelfTest/TestRegistrations.cpp
@@ -1,14 +1,21 @@
-/*
- * Distributed under the Boost Software License, Version 1.0. (See accompanying
- * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
- */
+
+// Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+// (See accompanying file LICENSE_1_0.txt or copy at
+// https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
#include <catch2/catch_tag_alias_autoregistrar.hpp>
#include <catch2/reporters/catch_reporter_event_listener.hpp>
+#include <catch2/internal/catch_enforce.hpp>
+#include <catch2/catch_test_case_info.hpp>
+#include <catch2/catch_reporter_registrars.hpp>
+
// Some example tag aliases
-CATCH_REGISTER_TAG_ALIAS( "[@nhf]", "[failing]~[.]" )
-CATCH_REGISTER_TAG_ALIAS( "[@tricky]", "[tricky]~[.]" )
+CATCH_REGISTER_TAG_ALIAS("[@nhf]", "[failing]~[.]")
+CATCH_REGISTER_TAG_ALIAS("[@tricky]", "[tricky]~[.]")
#ifdef __clang__
# pragma clang diagnostic ignored "-Wpadded"
@@ -16,11 +23,155 @@ CATCH_REGISTER_TAG_ALIAS( "[@tricky]", "[tricky]~[.]" )
# pragma clang diagnostic ignored "-Wc++98-compat"
#endif
+/**
+ * Event listener that internally counts and validates received events.
+ *
+ * Currently only performs validation by counting received events, rather
+ * than performing full matching. This means that it won't fail if the *Ended
+ * events are provided in wrong order, as long as they come in the right amount
+ * and with the right nesting.
+ */
+class ValidatingTestListener : public Catch::EventListenerBase {
+ struct EventCounter {
+ int starting = 0;
+ int ended = 0;
+
+ bool hasActiveEvent() const {
+ return starting > ended;
+ }
+ bool hasSingleActiveEvent() const {
+ return starting - 1 == ended;
+ }
+ bool allEventsEnded() const {
+ return starting == ended;
+ }
+ };
+
+public:
+ ValidatingTestListener(Catch::ReporterConfig const& config) :
+ EventListenerBase(config) {
+ m_preferences.shouldReportAllAssertions = true;
+ }
+
+ void testRunStarting( Catch::TestRunInfo const& ) override {
+ CATCH_ENFORCE( m_testRunCounter.starting == 0,
+ "Test run can only start once" );
+ ++m_testRunCounter.starting;
+ }
+ void testCaseStarting(Catch::TestCaseInfo const&) override {
+ CATCH_ENFORCE( m_testRunCounter.hasActiveEvent(),
+ "Test case can only be started if the test run has already started" );
+ CATCH_ENFORCE( m_testCaseCounter.allEventsEnded(),
+ "Test case cannot start if there is an unfinished one" );
+
+ ++m_testCaseCounter.starting;
+
+ // Reset the part tracking for partial test case events
+ m_lastSeenPartNumber = -1;
+ }
+
+ void testCasePartialStarting(Catch::TestCaseInfo const&,
+ uint64_t partNumber) override {
+ CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(),
+ "Test case can only be partially started if the test case has fully started already" );
+ CATCH_ENFORCE( m_lastSeenPartNumber + 1 == partNumber,
+ "Partial test case started out of order" );
+
+ ++m_testCasePartialCounter.starting;
+ m_lastSeenPartNumber = partNumber;
+ }
+
+ void sectionStarting(Catch::SectionInfo const&) override {
+ CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(),
+ "Section can only start in a test case" );
+ CATCH_ENFORCE( m_testCasePartialCounter.hasSingleActiveEvent(),
+ "Section can only start in a test case" );
+
+ ++m_sectionCounter.starting;
+ }
+
+ void assertionStarting(Catch::AssertionInfo const&) override {
+ CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(),
+ "Assertion can only start if test case is started" );
-struct TestListener : Catch::EventListenerBase {
- using EventListenerBase::EventListenerBase;
+ ++m_assertionCounter.starting;
+ }
+ void assertionEnded(Catch::AssertionStats const&) override {
+ // todo:
+ // * Check that assertions are balanced
+ // * Check that assertions has started
+ ++m_assertionCounter.ended;
+ }
+
+ void sectionEnded(Catch::SectionStats const&) override {
+ CATCH_ENFORCE( m_sectionCounter.hasActiveEvent(),
+ "Section ended without corresponding start" );
+ // TODO: Check that all assertions ended
+
+ ++m_sectionCounter.ended;
+ }
+
+
+ void testCasePartialEnded(Catch::TestCaseStats const&,
+ uint64_t partNumber) override {
+ CATCH_ENFORCE( m_lastSeenPartNumber == partNumber,
+ "Partial test case ended out of order" );
+ CATCH_ENFORCE( m_testCasePartialCounter.hasSingleActiveEvent(),
+ "Partial test case ended without corresponding start" );
+ CATCH_ENFORCE( m_sectionCounter.allEventsEnded(),
+ "Partial test case ended with unbalanced sections" );
+ // TODO: Check that all assertions ended
+
+ ++m_testCasePartialCounter.ended;
+ }
+
+
+ void testCaseEnded(Catch::TestCaseStats const&) override {
+ CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(),
+ "Test case end is not matched with test case start" );
+ CATCH_ENFORCE( m_testCasePartialCounter.allEventsEnded(),
+ "A partial test case has not ended" );
+ CATCH_ENFORCE( m_sectionCounter.allEventsEnded(),
+ "Test case ended with unbalanced sections" );
+
+ // TODO: Check that all assertions ended
+
+ ++m_testCaseCounter.ended;
+ }
+ void testRunEnded( Catch::TestRunStats const& ) override {
+ CATCH_ENFORCE( m_testRunCounter.hasSingleActiveEvent(),
+ "Test run end is not matched with test run start" );
+ CATCH_ENFORCE( m_testRunCounter.ended == 0,
+ "Test run can only end once" );
+
+ ++m_testRunCounter.ended;
+ }
+
+ ~ValidatingTestListener() override;
+
+private:
+ EventCounter m_testRunCounter;
+ EventCounter m_testCaseCounter;
+ EventCounter m_testCasePartialCounter;
+ uint64_t m_lastSeenPartNumber = 0;
+ EventCounter m_sectionCounter;
+ EventCounter m_assertionCounter;
};
-#include <catch2/catch_reporter_registrars.hpp>
-CATCH_REGISTER_LISTENER( TestListener )
+ValidatingTestListener::~ValidatingTestListener() {
+ // Throwing from noexcept destructor terminates, but we don't mind
+ // because this is test-only check and we don't need to try and recover
+ // from assumption violation here.
+
+ CATCH_ENFORCE( m_testRunCounter.ended < 2,
+ "Test run should be started at most once" );
+ CATCH_ENFORCE( m_testRunCounter.allEventsEnded(),
+ "The test run has not finished" );
+ CATCH_ENFORCE( m_testCaseCounter.allEventsEnded(),
+ "A test case did not finish" );
+
+ // TODO: other counters being balanced?
+}
+
+CATCH_REGISTER_LISTENER( ValidatingTestListener )