diff options
| author | Martin Hořeňovský | 2021-09-12 00:00:47 +0200 |
|---|---|---|
| committer | Martin Hořeňovský | 2021-09-14 13:19:45 +0200 |
| commit | 10fb93cce8cb878fdfe8faa118b531258db96a26 (patch) | |
| tree | 41eadf4802932051def7770b52a4a7fe94260c80 /tests | |
| parent | 4dcf8382c70c25a3ccfbe5d9e37276adf7c6058f (diff) | |
| download | catch3-10fb93cce8cb878fdfe8faa118b531258db96a26.tar.gz | |
Add new reporter event called for each test case enter/exit
This means that e.g. for `TEST_CASE` with two sibling `SECTION`s
the event will fire twice, because the `TEST_CASE` will be entered
twice.
Closes #2107 (the event mentioned there already exists, but this
is its counterpart that we also want to provide to users)
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/ExtraTests/CMakeLists.txt | 8 | ||||
| -rw-r--r-- | tests/ExtraTests/X21-PartialTestCaseEvents.cpp | 74 | ||||
| -rwxr-xr-x | tests/TestScripts/testPartialTestCaseEvent.py | 72 |
3 files changed, 154 insertions, 0 deletions
diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt index a861345d..bd7a575c 100644 --- a/tests/ExtraTests/CMakeLists.txt +++ b/tests/ExtraTests/CMakeLists.txt @@ -172,6 +172,14 @@ if (MSVC) list(APPEND CATCH_WARNING_TARGETS ${EXTRA_TEST_BINARIES} WindowsHeader) endif() + +add_executable(PartialTestCaseEvents ${TESTS_DIR}/X21-PartialTestCaseEvents.cpp) +target_link_libraries(PartialTestCaseEvents PRIVATE Catch2WithMain) +add_test( + NAME PartialTestCaseEvents + COMMAND ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tests/TestScripts/testPartialTestCaseEvent.py $<TARGET_FILE:PartialTestCaseEvents> +) + #add_executable(DebugBreakMacros ${TESTS_DIR}/X12-CustomDebugBreakMacro.cpp) #target_link_libraries(DebugBreakMacros Catch2) #add_test(NAME DebugBreakMacros COMMAND DebugBreakMacros --break) diff --git a/tests/ExtraTests/X21-PartialTestCaseEvents.cpp b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp new file mode 100644 index 00000000..997b9b9f --- /dev/null +++ b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp @@ -0,0 +1,74 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +/**\file + * Registers custom reporter that reports testCase* events + * + * The resulting executable can then be used by an external Python script + * to verify that testCase{Starting,Ended} and testCasePartial{Starting,Ended} + * events are properly nested. + */ + + +#include <catch2/catch_test_macros.hpp> +#include <catch2/reporters/catch_reporter_streaming_base.hpp> +#include <catch2/catch_test_case_info.hpp> +#include <catch2/catch_reporter_registrars.hpp> +#include <catch2/generators/catch_generators.hpp> + + +#include <iostream> + +using Catch::TestCaseInfo; +using Catch::TestCaseStats; + +class PartialReporter : public Catch::StreamingReporterBase { +public: + using StreamingReporterBase::StreamingReporterBase; + + ~PartialReporter() override; // = default + + static std::string getDescription() { + return "Special reporter for testing TestCasePartialStarting/Ended events"; + } + + //! Called _once_ for each TEST_CASE, no matter how many times it is entered + void testCaseStarting(TestCaseInfo const& testInfo) override { + std::cout << "TestCaseStarting: " << testInfo.name << '\n'; + } + //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections) + void testCasePartialStarting(TestCaseInfo const& testInfo, uint64_t partNumber) override { + std::cout << "TestCaseStartingPartial: " << testInfo.name << '#' << partNumber << '\n'; + } + + + //! Called _every time_ a TEST_CASE is entered, including repeats (due to sections) + void testCasePartialEnded(TestCaseStats const& testCaseStats, uint64_t partNumber) override { + std::cout << "TestCasePartialEnded: " << testCaseStats.testInfo->name << '#' << partNumber << '\n'; + } + //! Called _once_ for each TEST_CASE, no matter how many times it is entered + void testCaseEnded(TestCaseStats const& testCaseStats) override { + std::cout << "TestCaseEnded: " << testCaseStats.testInfo->name << '\n'; + } +}; +PartialReporter::~PartialReporter() = default; + + +CATCH_REGISTER_REPORTER("partial", PartialReporter) + +TEST_CASE("section") { + SECTION("A") {} + SECTION("B") {} + SECTION("C") {} + SECTION("D") {} +} + +TEST_CASE("generator") { + auto _ = GENERATE(1, 2, 3, 4); + (void)_; +} diff --git a/tests/TestScripts/testPartialTestCaseEvent.py b/tests/TestScripts/testPartialTestCaseEvent.py new file mode 100755 index 00000000..d119e6da --- /dev/null +++ b/tests/TestScripts/testPartialTestCaseEvent.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +""" +This test script verifies that the testCasePartial{Starting,Ended} reporter +events fire properly. This is done by calling a test binary compiled with +reporter that reports specifically testCase* events, and verifying the +outputs match what we expect. +""" + +import subprocess +import sys + +expected_section_output = '''\ +TestCaseStarting: section +TestCaseStartingPartial: section#0 +TestCasePartialEnded: section#0 +TestCaseStartingPartial: section#1 +TestCasePartialEnded: section#1 +TestCaseStartingPartial: section#2 +TestCasePartialEnded: section#2 +TestCaseStartingPartial: section#3 +TestCasePartialEnded: section#3 +TestCaseEnded: section +''' + +expected_generator_output = '''\ +TestCaseStarting: generator +TestCaseStartingPartial: generator#0 +TestCasePartialEnded: generator#0 +TestCaseStartingPartial: generator#1 +TestCasePartialEnded: generator#1 +TestCaseStartingPartial: generator#2 +TestCasePartialEnded: generator#2 +TestCaseStartingPartial: generator#3 +TestCasePartialEnded: generator#3 +TestCaseEnded: generator +''' + + +from typing import List + +def get_test_output(test_exe: str, sections: bool) -> List[str]: + cmd = [test_exe, '--reporter', 'partial'] + if sections: + cmd.append('section') + else: + cmd.append('generator') + + ret = subprocess.run(cmd, + stdout = subprocess.PIPE, + stderr = subprocess.PIPE, + timeout = 10, + check = True, + universal_newlines = True) + + return ret.stdout + +def main(): + test_exe, = sys.argv[1:] + actual_section_output = get_test_output(test_exe, sections = True) + + assert actual_section_output == expected_section_output, ( + 'Sections\nActual:\n{}\nExpected:\n{}\n'.format(actual_section_output, expected_section_output)) + + actual_generator_output = get_test_output(test_exe, sections = False) + assert actual_generator_output == expected_generator_output, ( + 'Generators\nActual:\n{}\nExpected:\n{}\n'.format(actual_generator_output, expected_generator_output)) + + + +if __name__ == '__main__': + sys.exit(main()) |