diff options
| author | Martin Hořeňovský | 2021-09-15 21:46:07 +0200 |
|---|---|---|
| committer | Martin Hořeňovský | 2021-09-15 23:38:43 +0200 |
| commit | 3f8cae8025f4f5e804383f44bb393a63bcef90a4 (patch) | |
| tree | 6968a937b713cc21820c3ae32ca45239dd0a809a | |
| parent | 2c82f82ee211b011460f2f44647b69e0feaf3d4e (diff) | |
| download | catch3-3f8cae8025f4f5e804383f44bb393a63bcef90a4.tar.gz | |
Add consistency-checking event listener to SelfTest
This event listener performs basic consistency checks (akin to
matching braces) on events that are passed to the listeners
when the `SelfTest` test binary is run.
The current checks are about nesting events (e.g. `testCaseStarting`
cannot be received before `testRunStarting`, `sectionStarting`
can only be received when a test case is active, etc), and matching
up counts of starting/ended events.
The simplicity means that it could be confused by starting/ended
events matching up but being out of order, e.g.
```
* test case A starting
* test case B ended
* test case B starting
* test case A ended
```
would be accepted, even though it is wrong. However, doing full
order checking would be much more implementation work, for relatively
little benefit, so it is left out for now.
| -rw-r--r-- | tests/SelfTest/TestRegistrations.cpp | 171 |
1 files changed, 161 insertions, 10 deletions
diff --git a/tests/SelfTest/TestRegistrations.cpp b/tests/SelfTest/TestRegistrations.cpp index cb8376bd..45a01c02 100644 --- a/tests/SelfTest/TestRegistrations.cpp +++ b/tests/SelfTest/TestRegistrations.cpp @@ -1,14 +1,21 @@ -/* - * Distributed under the Boost Software License, Version 1.0. (See accompanying - * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - */ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 #include <catch2/catch_tag_alias_autoregistrar.hpp> #include <catch2/reporters/catch_reporter_event_listener.hpp> +#include <catch2/internal/catch_enforce.hpp> +#include <catch2/catch_test_case_info.hpp> +#include <catch2/catch_reporter_registrars.hpp> + // Some example tag aliases -CATCH_REGISTER_TAG_ALIAS( "[@nhf]", "[failing]~[.]" ) -CATCH_REGISTER_TAG_ALIAS( "[@tricky]", "[tricky]~[.]" ) +CATCH_REGISTER_TAG_ALIAS("[@nhf]", "[failing]~[.]") +CATCH_REGISTER_TAG_ALIAS("[@tricky]", "[tricky]~[.]") #ifdef __clang__ # pragma clang diagnostic ignored "-Wpadded" @@ -16,11 +23,155 @@ CATCH_REGISTER_TAG_ALIAS( "[@tricky]", "[tricky]~[.]" ) # pragma clang diagnostic ignored "-Wc++98-compat" #endif +/** + * Event listener that internally counts and validates received events. + * + * Currently only performs validation by counting received events, rather + * than performing full matching. This means that it won't fail if the *Ended + * events are provided in wrong order, as long as they come in the right amount + * and with the right nesting. + */ +class ValidatingTestListener : public Catch::EventListenerBase { + struct EventCounter { + int starting = 0; + int ended = 0; + + bool hasActiveEvent() const { + return starting > ended; + } + bool hasSingleActiveEvent() const { + return starting - 1 == ended; + } + bool allEventsEnded() const { + return starting == ended; + } + }; + +public: + ValidatingTestListener(Catch::ReporterConfig const& config) : + EventListenerBase(config) { + m_preferences.shouldReportAllAssertions = true; + } + + void testRunStarting( Catch::TestRunInfo const& ) override { + CATCH_ENFORCE( m_testRunCounter.starting == 0, + "Test run can only start once" ); + ++m_testRunCounter.starting; + } + void testCaseStarting(Catch::TestCaseInfo const&) override { + CATCH_ENFORCE( m_testRunCounter.hasActiveEvent(), + "Test case can only be started if the test run has already started" ); + CATCH_ENFORCE( m_testCaseCounter.allEventsEnded(), + "Test case cannot start if there is an unfinished one" ); + + ++m_testCaseCounter.starting; + + // Reset the part tracking for partial test case events + m_lastSeenPartNumber = -1; + } + + void testCasePartialStarting(Catch::TestCaseInfo const&, + uint64_t partNumber) override { + CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(), + "Test case can only be partially started if the test case has fully started already" ); + CATCH_ENFORCE( m_lastSeenPartNumber + 1 == partNumber, + "Partial test case started out of order" ); + + ++m_testCasePartialCounter.starting; + m_lastSeenPartNumber = partNumber; + } + + void sectionStarting(Catch::SectionInfo const&) override { + CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(), + "Section can only start in a test case" ); + CATCH_ENFORCE( m_testCasePartialCounter.hasSingleActiveEvent(), + "Section can only start in a test case" ); + + ++m_sectionCounter.starting; + } + + void assertionStarting(Catch::AssertionInfo const&) override { + CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(), + "Assertion can only start if test case is started" ); -struct TestListener : Catch::EventListenerBase { - using EventListenerBase::EventListenerBase; + ++m_assertionCounter.starting; + } + void assertionEnded(Catch::AssertionStats const&) override { + // todo: + // * Check that assertions are balanced + // * Check that assertions has started + ++m_assertionCounter.ended; + } + + void sectionEnded(Catch::SectionStats const&) override { + CATCH_ENFORCE( m_sectionCounter.hasActiveEvent(), + "Section ended without corresponding start" ); + // TODO: Check that all assertions ended + + ++m_sectionCounter.ended; + } + + + void testCasePartialEnded(Catch::TestCaseStats const&, + uint64_t partNumber) override { + CATCH_ENFORCE( m_lastSeenPartNumber == partNumber, + "Partial test case ended out of order" ); + CATCH_ENFORCE( m_testCasePartialCounter.hasSingleActiveEvent(), + "Partial test case ended without corresponding start" ); + CATCH_ENFORCE( m_sectionCounter.allEventsEnded(), + "Partial test case ended with unbalanced sections" ); + // TODO: Check that all assertions ended + + ++m_testCasePartialCounter.ended; + } + + + void testCaseEnded(Catch::TestCaseStats const&) override { + CATCH_ENFORCE( m_testCaseCounter.hasSingleActiveEvent(), + "Test case end is not matched with test case start" ); + CATCH_ENFORCE( m_testCasePartialCounter.allEventsEnded(), + "A partial test case has not ended" ); + CATCH_ENFORCE( m_sectionCounter.allEventsEnded(), + "Test case ended with unbalanced sections" ); + + // TODO: Check that all assertions ended + + ++m_testCaseCounter.ended; + } + void testRunEnded( Catch::TestRunStats const& ) override { + CATCH_ENFORCE( m_testRunCounter.hasSingleActiveEvent(), + "Test run end is not matched with test run start" ); + CATCH_ENFORCE( m_testRunCounter.ended == 0, + "Test run can only end once" ); + + ++m_testRunCounter.ended; + } + + ~ValidatingTestListener() override; + +private: + EventCounter m_testRunCounter; + EventCounter m_testCaseCounter; + EventCounter m_testCasePartialCounter; + uint64_t m_lastSeenPartNumber = 0; + EventCounter m_sectionCounter; + EventCounter m_assertionCounter; }; -#include <catch2/catch_reporter_registrars.hpp> -CATCH_REGISTER_LISTENER( TestListener ) +ValidatingTestListener::~ValidatingTestListener() { + // Throwing from noexcept destructor terminates, but we don't mind + // because this is test-only check and we don't need to try and recover + // from assumption violation here. + + CATCH_ENFORCE( m_testRunCounter.ended < 2, + "Test run should be started at most once" ); + CATCH_ENFORCE( m_testRunCounter.allEventsEnded(), + "The test run has not finished" ); + CATCH_ENFORCE( m_testCaseCounter.allEventsEnded(), + "A test case did not finish" ); + + // TODO: other counters being balanced? +} + +CATCH_REGISTER_LISTENER( ValidatingTestListener ) |