catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
--list-* flags write to target specified by the -o flag
- Commit
971b1fc32a857caa10eb166f39442cee5237267b- Author
- Martin Hořeňovský ·
- Parents
6798c139a65f
Changes
.../interfaces/catch_interfaces_reporter.cpp | 16 - .../interfaces/catch_interfaces_reporter.hpp | 6 +- .../reporters/catch_reporter_cumulative_base.cpp | 16 + .../reporters/catch_reporter_cumulative_base.hpp | 5 + .../reporters/catch_reporter_streaming_base.cpp | 16 + .../reporters/catch_reporter_streaming_base.hpp | 4 + tests/CMakeLists.txt | 1 + tests/SelfTest/Baselines/automake.sw.approved.txt | 2 + tests/SelfTest/Baselines/compact.sw.approved.txt | 178 +++++ tests/SelfTest/Baselines/console.std.approved.txt | 4 +- tests/SelfTest/Baselines/console.sw.approved.txt | 767 ++++++++++++++++++++- tests/SelfTest/Baselines/junit.sw.approved.txt | 30 +- tests/SelfTest/Baselines/sonarqube.sw.approved.txt | 30 + tests/SelfTest/Baselines/tap.sw.approved.txt | 104 ++- tests/SelfTest/Baselines/teamcity.sw.approved.txt | 4 + tests/SelfTest/Baselines/xml.sw.approved.txt | 702 ++++++++++++++++++- .../IntrospectiveTests/Reporters.tests.cpp | 109 +++ 17 files changed, 1965 insertions(+), 29 deletions(-) diff --git a/src/catch2/interfaces/catch_interfaces_reporter.cpp b/src/catch2/interfaces/catch_interfaces_reporter.cpp index 29fafecf..19416130 100644 --- a/src/catch2/interfaces/catch_interfaces_reporter.cpp +++ b/src/catch2/interfaces/catch_interfaces_reporter.cpp @@ -108,20 +108,4 @@ namespace Catch { void IStreamingReporter::fatalErrorEncountered( StringRef ) {} - void IStreamingReporter::listReporters(std::vector<ReporterDescription> const& descriptions) { - defaultListReporters( - Catch::cout(), descriptions, m_config->verbosity() ); - } - - void IStreamingReporter::listTests(std::vector<TestCaseHandle> const& tests) { - defaultListTests( Catch::cout(), - tests, - m_config->hasTestFilters(), - m_config->verbosity() ); - } - - void IStreamingReporter::listTags(std::vector<TagInfo> const& tags) { - defaultListTags( Catch::cout(), tags, m_config->hasTestFilters() ); - } - } // end namespace Catch diff --git a/src/catch2/interfaces/catch_interfaces_reporter.hpp b/src/catch2/interfaces/catch_interfaces_reporter.hpp index 5449cc22..5a4c31a8 100644 --- a/src/catch2/interfaces/catch_interfaces_reporter.hpp +++ b/src/catch2/interfaces/catch_interfaces_reporter.hpp @@ -222,11 +222,11 @@ namespace Catch { virtual void fatalErrorEncountered( StringRef name ); //! Writes out information about provided reporters using reporter-specific format - virtual void listReporters(std::vector<ReporterDescription> const& descriptions); + virtual void listReporters(std::vector<ReporterDescription> const& descriptions) = 0; //! Writes out information about provided tests using reporter-specific format - virtual void listTests(std::vector<TestCaseHandle> const& tests); + virtual void listTests(std::vector<TestCaseHandle> const& tests) = 0; //! Writes out information about the provided tags using reporter-specific format - virtual void listTags(std::vector<TagInfo> const& tags); + virtual void listTags(std::vector<TagInfo> const& tags) = 0; }; using IStreamingReporterPtr = Detail::unique_ptr<IStreamingReporter>; diff --git a/src/catch2/reporters/catch_reporter_cumulative_base.cpp b/src/catch2/reporters/catch_reporter_cumulative_base.cpp index bba6f67d..a83a9ad0 100644 --- a/src/catch2/reporters/catch_reporter_cumulative_base.cpp +++ b/src/catch2/reporters/catch_reporter_cumulative_base.cpp @@ -6,6 +6,7 @@ // SPDX-License-Identifier: BSL-1.0 #include <catch2/reporters/catch_reporter_cumulative_base.hpp> +#include <catch2/reporters/catch_reporter_helpers.hpp> #include <algorithm> #include <cassert> @@ -110,4 +111,19 @@ namespace Catch { testRunEndedCumulative(); } + void CumulativeReporterBase::listReporters(std::vector<ReporterDescription> const& descriptions) { + defaultListReporters(stream, descriptions, m_config->verbosity()); + } + + void CumulativeReporterBase::listTests(std::vector<TestCaseHandle> const& tests) { + defaultListTests(stream, + tests, + m_config->hasTestFilters(), + m_config->verbosity()); + } + + void CumulativeReporterBase::listTags(std::vector<TagInfo> const& tags) { + defaultListTags( stream, tags, m_config->hasTestFilters() ); + } + } // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_cumulative_base.hpp b/src/catch2/reporters/catch_reporter_cumulative_base.hpp index 0729567c..2820e9f6 100644 --- a/src/catch2/reporters/catch_reporter_cumulative_base.hpp +++ b/src/catch2/reporters/catch_reporter_cumulative_base.hpp @@ -69,6 +69,11 @@ namespace Catch { void skipTest(TestCaseInfo const&) override {} + void listReporters( std::vector<ReporterDescription> const& descriptions ) override; + void listTests( std::vector<TestCaseHandle> const& tests ) override; + void listTags( std::vector<TagInfo> const& tags ) override; + + std::ostream& stream; // Note: We rely on pointer identity being stable, which is why // which is why we store around pointers rather than values. diff --git a/src/catch2/reporters/catch_reporter_streaming_base.cpp b/src/catch2/reporters/catch_reporter_streaming_base.cpp index e4bbb261..d0b8ae94 100644 --- a/src/catch2/reporters/catch_reporter_streaming_base.cpp +++ b/src/catch2/reporters/catch_reporter_streaming_base.cpp @@ -6,6 +6,7 @@ // SPDX-License-Identifier: BSL-1.0 #include <catch2/reporters/catch_reporter_streaming_base.hpp> +#include <catch2/reporters/catch_reporter_helpers.hpp> namespace Catch { @@ -31,4 +32,19 @@ namespace Catch { currentTestRunInfo.reset(); } + void StreamingReporterBase::listReporters(std::vector<ReporterDescription> const& descriptions) { + defaultListReporters( stream, descriptions, m_config->verbosity() ); + } + + void StreamingReporterBase::listTests(std::vector<TestCaseHandle> const& tests) { + defaultListTests(stream, + tests, + m_config->hasTestFilters(), + m_config->verbosity()); + } + + void StreamingReporterBase::listTags(std::vector<TagInfo> const& tags) { + defaultListTags( stream, tags, m_config->hasTestFilters() ); + } + } // end namespace Catch diff --git a/src/catch2/reporters/catch_reporter_streaming_base.hpp b/src/catch2/reporters/catch_reporter_streaming_base.hpp index ebf06385..b0f1dd49 100644 --- a/src/catch2/reporters/catch_reporter_streaming_base.hpp +++ b/src/catch2/reporters/catch_reporter_streaming_base.hpp @@ -71,6 +71,10 @@ namespace Catch { // It can optionally be overridden in the derived class. } + void listReporters( std::vector<ReporterDescription> const& descriptions ) override; + void listTests( std::vector<TestCaseHandle> const& tests ) override; + void listTags( std::vector<TagInfo> const& tags ) override; + std::ostream& stream; LazyStat<TestRunInfo> currentTestRunInfo; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7a3c6deb..f8bdb3b4 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -23,6 +23,7 @@ set(TEST_SOURCES ${SELF_TEST_DIR}/IntrospectiveTests/InternalBenchmark.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/PartTracker.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/RandomNumberGeneration.tests.cpp + ${SELF_TEST_DIR}/IntrospectiveTests/Reporters.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/Tag.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/String.tests.cpp ${SELF_TEST_DIR}/IntrospectiveTests/StringManip.tests.cpp diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 87f2a189..983d9764 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -185,6 +185,7 @@ Nor would this :test-result: FAIL Reconstruction should be based on stringification: #914 :test-result: FAIL Regex string matcher :test-result: PASS Regression test #1 +:test-result: PASS Reporter's write listings to provided stream :test-result: PASS SUCCEED counts as a test pass :test-result: PASS SUCCEED does not require an argument :test-result: PASS Scenario: BDD tests requiring Fixtures to provide commonly-accessed data or methods @@ -231,6 +232,7 @@ Message from section two :test-result: PASS Test enum bit values :test-result: PASS Test with special, characters "in name :test-result: FAIL The NO_FAIL macro reports a failure but does not fail the test +:test-result: PASS The default listing implementation write to provided stream :test-result: FAIL This test 'should' fail but doesn't :test-result: FAIL Thrown string literals are translated :test-result: PASS Tracker diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 4042f171..3b19a962 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1273,6 +1273,169 @@ Matchers.tests.cpp:<line number>: failed: testStringForMatching(), Matches("this Matchers.tests.cpp:<line number>: failed: testStringForMatching(), Matches("contains 'abc' as a substring") for: "this string contains 'abc' as a substring" matches "contains 'abc' as a substring" case sensitively Matchers.tests.cpp:<line number>: failed: testStringForMatching(), Matches("this string contains 'abc' as a") for: "this string contains 'abc' as a substring" matches "this string contains 'abc' as a" case sensitively Matchers.tests.cpp:<line number>: passed: actual, !UnorderedEquals(expected) for: { 'a', 'b' } not UnorderedEquals: { 'c', 'b' } +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fakeTag"s) for: "All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" with 1 message: 'Tested reporter: automake' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fake reporter"s) for: "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" with 1 message: 'Tested reporter: automake' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: automake' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fakeTag"s) for: "All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" with 1 message: 'Tested reporter: compact' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fake reporter"s) for: "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" with 1 message: 'Tested reporter: compact' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: compact' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fakeTag"s) for: "All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" with 1 message: 'Tested reporter: console' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fake reporter"s) for: "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" with 1 message: 'Tested reporter: console' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: console' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fakeTag"s) for: "<?xml version="1.0" encoding="UTF-8"?> +All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" with 1 message: 'Tested reporter: junit' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fake reporter"s) for: "<?xml version="1.0" encoding="UTF-8"?> +Available reporters: + fake reporter: fake description + +" contains: "fake reporter" with 1 message: 'Tested reporter: junit' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "<?xml version="1.0" encoding="UTF-8"?> +All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: junit' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fakeTag"s) for: "<?xml version="1.0" encoding="UTF-8"?> +All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" with 1 message: 'Tested reporter: sonarqube' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fake reporter"s) for: "<?xml version="1.0" encoding="UTF-8"?> +Available reporters: + fake reporter: fake description + +" contains: "fake reporter" with 1 message: 'Tested reporter: sonarqube' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "<?xml version="1.0" encoding="UTF-8"?> +All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: sonarqube' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fakeTag"s) for: "All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" with 1 message: 'Tested reporter: tap' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fake reporter"s) for: "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" with 1 message: 'Tested reporter: tap' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: tap' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fakeTag"s) for: "All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" with 1 message: 'Tested reporter: teamcity' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fake reporter"s) for: "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" with 1 message: 'Tested reporter: teamcity' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: teamcity' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fakeTag"s) for: "<?xml version="1.0" encoding="UTF-8"?> +<TagsFromMatchingTests> + <Tag> + <Count>1</Count> + <Aliases> + <Alias>fakeTag</Alias> + </Aliases> + </Tag> +</TagsFromMatchingTests>" contains: "fakeTag" with 1 message: 'Tested reporter: xml' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fake reporter"s) for: "<?xml version="1.0" encoding="UTF-8"?> +<AvailableReporters> + <Reporter> + <Name>fake reporter</Name> + <Description>fake description</Description> + </Reporter> +</AvailableReporters>" contains: "fake reporter" with 1 message: 'Tested reporter: xml' +Reporters.tests.cpp:<line number>: passed: !(factories.empty()) for: !false +Reporters.tests.cpp:<line number>: passed: listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "<?xml version="1.0" encoding="UTF-8"?> +<MatchingTests> + <TestCase> + <Name>fake test name</Name> + <ClassName/> + <Tags>[fakeTestTag]</Tags> + <SourceInfo> + <File>fake-file.cpp</File> + <Line>123456789</Line> + </SourceInfo> + </TestCase> +</MatchingTests>" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: xml' Message.tests.cpp:<line number>: passed: with 1 message: 'this is a success' Message.tests.cpp:<line number>: passed: BDD.tests.cpp:<line number>: passed: before == 0 for: 0 == 0 @@ -1563,6 +1726,21 @@ VariadicMacros.tests.cpp:<line number>: passed: with 1 message: 'no assertions' Tricky.tests.cpp:<line number>: passed: 0x<hex digits> == bit30and31 for: 3221225472 (0x<hex digits>) == 3221225472 CmdLine.tests.cpp:<line number>: passed: Message.tests.cpp:<line number>: failed - but was ok: 1 == 2 +Reporters.tests.cpp:<line number>: passed: listingString, Contains("[fakeTag]"s) for: "All available tags: + 1 [fakeTag] +1 tag + +" contains: "[fakeTag]" +Reporters.tests.cpp:<line number>: passed: listingString, Contains("fake reporter"s) for: "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" +Reporters.tests.cpp:<line number>: passed: listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) Misc.tests.cpp:<line number>: passed: with 1 message: 'oops!' Exception.tests.cpp:<line number>: failed: unexpected exception with message: 'For some reason someone is throwing a string literal!' PartTracker.tests.cpp:<line number>: passed: testCase.isOpen() for: true diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index 149bed95..d781a60e 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1380,6 +1380,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 354 | 280 passed | 70 failed | 4 failed as expected -assertions: 2037 | 1885 passed | 131 failed | 21 failed as expected +test cases: 356 | 282 passed | 70 failed | 4 failed as expected +assertions: 2088 | 1936 passed | 131 failed | 21 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 1a8d3ef4..7066d593 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -9399,6 +9399,721 @@ Matchers.tests.cpp:<line number>: PASSED: with expansion: { 'a', 'b' } not UnorderedEquals: { 'c', 'b' } +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + automake reporter lists tags +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fakeTag"s) ) +with expansion: + "All available tags: + 1 [fakeTag] + 1 tag + +" contains: "fakeTag" +with message: + Tested reporter: automake + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + automake reporter lists reporters +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fake reporter"s) ) +with expansion: + "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" +with message: + Tested reporter: automake + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + automake reporter lists tests +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) ) +with expansion: + "All available test cases: + fake test name + [fakeTestTag] + 1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) +with message: + Tested reporter: automake + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + compact reporter lists tags +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fakeTag"s) ) +with expansion: + "All available tags: + 1 [fakeTag] + 1 tag + +" contains: "fakeTag" +with message: + Tested reporter: compact + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + compact reporter lists reporters +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fake reporter"s) ) +with expansion: + "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" +with message: + Tested reporter: compact + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + compact reporter lists tests +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) ) +with expansion: + "All available test cases: + fake test name + [fakeTestTag] + 1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) +with message: + Tested reporter: compact + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + console reporter lists tags +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fakeTag"s) ) +with expansion: + "All available tags: + 1 [fakeTag] + 1 tag + +" contains: "fakeTag" +with message: + Tested reporter: console + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + console reporter lists reporters +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fake reporter"s) ) +with expansion: + "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" +with message: + Tested reporter: console + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + console reporter lists tests +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) ) +with expansion: + "All available test cases: + fake test name + [fakeTestTag] + 1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) +with message: + Tested reporter: console + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + junit reporter lists tags +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fakeTag"s) ) +with expansion: + "<?xml version="1.0" encoding="UTF-8"?> + All available tags: + 1 [fakeTag] + 1 tag + +" contains: "fakeTag" +with message: + Tested reporter: junit + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + junit reporter lists reporters +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fake reporter"s) ) +with expansion: + "<?xml version="1.0" encoding="UTF-8"?> + Available reporters: + fake reporter: fake description + +" contains: "fake reporter" +with message: + Tested reporter: junit + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + junit reporter lists tests +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) ) +with expansion: + "<?xml version="1.0" encoding="UTF-8"?> + All available test cases: + fake test name + [fakeTestTag] + 1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) +with message: + Tested reporter: junit + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + sonarqube reporter lists tags +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fakeTag"s) ) +with expansion: + "<?xml version="1.0" encoding="UTF-8"?> + All available tags: + 1 [fakeTag] + 1 tag + +" contains: "fakeTag" +with message: + Tested reporter: sonarqube + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + sonarqube reporter lists reporters +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fake reporter"s) ) +with expansion: + "<?xml version="1.0" encoding="UTF-8"?> + Available reporters: + fake reporter: fake description + +" contains: "fake reporter" +with message: + Tested reporter: sonarqube + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + sonarqube reporter lists tests +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) ) +with expansion: + "<?xml version="1.0" encoding="UTF-8"?> + All available test cases: + fake test name + [fakeTestTag] + 1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) +with message: + Tested reporter: sonarqube + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + tap reporter lists tags +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fakeTag"s) ) +with expansion: + "All available tags: + 1 [fakeTag] + 1 tag + +" contains: "fakeTag" +with message: + Tested reporter: tap + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + tap reporter lists reporters +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fake reporter"s) ) +with expansion: + "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" +with message: + Tested reporter: tap + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + tap reporter lists tests +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) ) +with expansion: + "All available test cases: + fake test name + [fakeTestTag] + 1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) +with message: + Tested reporter: tap + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + teamcity reporter lists tags +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fakeTag"s) ) +with expansion: + "All available tags: + 1 [fakeTag] + 1 tag + +" contains: "fakeTag" +with message: + Tested reporter: teamcity + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + teamcity reporter lists reporters +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fake reporter"s) ) +with expansion: + "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" +with message: + Tested reporter: teamcity + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + teamcity reporter lists tests +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) ) +with expansion: + "All available test cases: + fake test name + [fakeTestTag] + 1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) +with message: + Tested reporter: teamcity + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + xml reporter lists tags +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fakeTag"s) ) +with expansion: + "<?xml version="1.0" encoding="UTF-8"?> + <TagsFromMatchingTests> + <Tag> + <Count>1</Count> + <Aliases> + <Alias>fakeTag</Alias> + </Aliases> + </Tag> + </TagsFromMatchingTests>" contains: "fakeTag" +with message: + Tested reporter: xml + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + xml reporter lists reporters +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fake reporter"s) ) +with expansion: + "<?xml version="1.0" encoding="UTF-8"?> + <AvailableReporters> + <Reporter> + <Name>fake reporter</Name> + <Description>fake description</Description> + </Reporter> + </AvailableReporters>" contains: "fake reporter" +with message: + Tested reporter: xml + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_FALSE( factories.empty() ) +with expansion: + !false + +------------------------------------------------------------------------------- +Reporter's write listings to provided stream + xml reporter lists tests +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) ) +with expansion: + "<?xml version="1.0" encoding="UTF-8"?> + <MatchingTests> + <TestCase> + <Name>fake test name</Name> + <ClassName/> + <Tags>[fakeTestTag]</Tags> + <SourceInfo> + <File>fake-file.cpp</File> + <Line>123456789</Line> + </SourceInfo> + </TestCase> + </MatchingTests>" ( contains: "fake test name" and contains: "fakeTestTag" ) +with message: + Tested reporter: xml + ------------------------------------------------------------------------------- SUCCEED counts as a test pass ------------------------------------------------------------------------------- @@ -11641,6 +12356,54 @@ Message.tests.cpp:<line number>: FAILED - but was ok: No assertions in test case 'The NO_FAIL macro reports a failure but does not fail the test' +------------------------------------------------------------------------------- +The default listing implementation write to provided stream + Listing tags +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("[fakeTag]"s) ) +with expansion: + "All available tags: + 1 [fakeTag] + 1 tag + +" contains: "[fakeTag]" + +------------------------------------------------------------------------------- +The default listing implementation write to provided stream + Listing reporters +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains("fake reporter"s) ) +with expansion: + "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" + +------------------------------------------------------------------------------- +The default listing implementation write to provided stream + Listing tests +------------------------------------------------------------------------------- +Reporters.tests.cpp:<line number> +............................................................................... + +Reporters.tests.cpp:<line number>: PASSED: + REQUIRE_THAT( listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) ) +with expansion: + "All available test cases: + fake test name + [fakeTestTag] + 1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) + ------------------------------------------------------------------------------- This test 'should' fail but doesn't ------------------------------------------------------------------------------- @@ -16005,6 +16768,6 @@ Misc.tests.cpp:<line number> Misc.tests.cpp:<line number>: PASSED: =============================================================================== -test cases: 354 | 264 passed | 86 failed | 4 failed as expected -assertions: 2054 | 1885 passed | 148 failed | 21 failed as expected +test cases: 356 | 266 passed | 86 failed | 4 failed as expected +assertions: 2105 | 1936 passed | 148 failed | 21 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index c7e3d02c..5206ed2d 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <testsuitesloose text artifact > - <testsuite name="<exe-name>" errors="17" failures="132" tests="2055" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> + <testsuite name="<exe-name>" errors="17" failures="132" tests="2106" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> <properties> <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals] *"/> <property name="random-seed" value="1"/> @@ -1101,6 +1101,31 @@ Matchers.tests.cpp:<line number> </failure> </testcase> <testcase classname="<exe-name>.global" name="Regression test #1" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/automake reporter lists tags" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/automake reporter lists reporters" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/automake reporter lists tests" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/compact reporter lists tags" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/compact reporter lists reporters" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/compact reporter lists tests" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/console reporter lists tags" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/console reporter lists reporters" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/console reporter lists tests" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/junit reporter lists tags" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/junit reporter lists reporters" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/junit reporter lists tests" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/sonarqube reporter lists tags" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/sonarqube reporter lists reporters" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/sonarqube reporter lists tests" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/tap reporter lists tags" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/tap reporter lists reporters" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/tap reporter lists tests" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/teamcity reporter lists tags" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/teamcity reporter lists reporters" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/teamcity reporter lists tests" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/xml reporter lists tags" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/xml reporter lists reporters" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="Reporter's write listings to provided stream/xml reporter lists tests" time="{duration}" status="run"/> <testcase classname="<exe-name>.global" name="SUCCEED counts as a test pass" time="{duration}" status="run"/> <testcase classname="<exe-name>.global" name="SUCCEED does not require an argument" time="{duration}" status="run"/> <testcase classname="<exe-name>.Fixture" name="Scenario: BDD tests requiring Fixtures to provide commonly-accessed data or methods/Given: No operations precede me" time="{duration}" status="run"/> @@ -1252,6 +1277,9 @@ Misc.tests.cpp:<line number> <testcase classname="<exe-name>.global" name="Test enum bit values" time="{duration}" status="run"/> <testcase classname="<exe-name>.global" name="Test with special, characters "in name" time="{duration}" status="run"/> <testcase classname="<exe-name>.global" name="The NO_FAIL macro reports a failure but does not fail the test" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="The default listing implementation write to provided stream/Listing tags" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="The default listing implementation write to provided stream/Listing reporters" time="{duration}" status="run"/> + <testcase classname="<exe-name>.global" name="The default listing implementation write to provided stream/Listing tests" time="{duration}" status="run"/> <testcase classname="<exe-name>.global" name="This test 'should' fail but doesn't" time="{duration}" status="run"/> <testcase classname="<exe-name>.global" name="Thrown string literals are translated" time="{duration}" status="run"> <error type="TEST_CASE"> diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index 736b1b4f..1f0cc5da 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -150,6 +150,36 @@ <testCase name="Our PCG implementation provides expected results for known seeds/Default seeded" duration="{duration}"/> <testCase name="Our PCG implementation provides expected results for known seeds/Specific seed" duration="{duration}"/> </file> + <file path="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp"> + <testCase name="Reporter's write listings to provided stream" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/automake reporter lists tags" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/automake reporter lists reporters" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/automake reporter lists tests" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/compact reporter lists tags" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/compact reporter lists reporters" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/compact reporter lists tests" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/console reporter lists tags" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/console reporter lists reporters" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/console reporter lists tests" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/junit reporter lists tags" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/junit reporter lists reporters" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/junit reporter lists tests" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/sonarqube reporter lists tags" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/sonarqube reporter lists reporters" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/sonarqube reporter lists tests" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/tap reporter lists tags" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/tap reporter lists reporters" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/tap reporter lists tests" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/teamcity reporter lists tags" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/teamcity reporter lists reporters" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/teamcity reporter lists tests" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/xml reporter lists tags" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/xml reporter lists reporters" duration="{duration}"/> + <testCase name="Reporter's write listings to provided stream/xml reporter lists tests" duration="{duration}"/> + <testCase name="The default listing implementation write to provided stream/Listing tags" duration="{duration}"/> + <testCase name="The default listing implementation write to provided stream/Listing reporters" duration="{duration}"/> + <testCase name="The default listing implementation write to provided stream/Listing tests" duration="{duration}"/> + </file> <file path="tests/<exe-name>/IntrospectiveTests/String.tests.cpp"> <testCase name="StringRef/Empty string" duration="{duration}"/> <testCase name="StringRef/From string literal" duration="{duration}"/> diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index b699781e..edf6d1ed 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -2468,6 +2468,102 @@ not ok {test-number} - testStringForMatching(), Matches("contains 'abc' as a sub not ok {test-number} - testStringForMatching(), Matches("this string contains 'abc' as a") for: "this string contains 'abc' as a substring" matches "this string contains 'abc' as a" case sensitively # Regression test #1 ok {test-number} - actual, !UnorderedEquals(expected) for: { 'a', 'b' } not UnorderedEquals: { 'c', 'b' } +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fakeTag"s) for: "All available tags: 1 [fakeTag] 1 tag " contains: "fakeTag" with 1 message: 'Tested reporter: automake' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fake reporter"s) for: "Available reporters: fake reporter: fake description " contains: "fake reporter" with 1 message: 'Tested reporter: automake' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "All available test cases: fake test name [fakeTestTag] 1 test case " ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: automake' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fakeTag"s) for: "All available tags: 1 [fakeTag] 1 tag " contains: "fakeTag" with 1 message: 'Tested reporter: compact' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fake reporter"s) for: "Available reporters: fake reporter: fake description " contains: "fake reporter" with 1 message: 'Tested reporter: compact' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "All available test cases: fake test name [fakeTestTag] 1 test case " ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: compact' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fakeTag"s) for: "All available tags: 1 [fakeTag] 1 tag " contains: "fakeTag" with 1 message: 'Tested reporter: console' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fake reporter"s) for: "Available reporters: fake reporter: fake description " contains: "fake reporter" with 1 message: 'Tested reporter: console' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "All available test cases: fake test name [fakeTestTag] 1 test case " ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: console' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fakeTag"s) for: "<?xml version="1.0" encoding="UTF-8"?> All available tags: 1 [fakeTag] 1 tag " contains: "fakeTag" with 1 message: 'Tested reporter: junit' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fake reporter"s) for: "<?xml version="1.0" encoding="UTF-8"?> Available reporters: fake reporter: fake description " contains: "fake reporter" with 1 message: 'Tested reporter: junit' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "<?xml version="1.0" encoding="UTF-8"?> All available test cases: fake test name [fakeTestTag] 1 test case " ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: junit' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fakeTag"s) for: "<?xml version="1.0" encoding="UTF-8"?> All available tags: 1 [fakeTag] 1 tag " contains: "fakeTag" with 1 message: 'Tested reporter: sonarqube' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fake reporter"s) for: "<?xml version="1.0" encoding="UTF-8"?> Available reporters: fake reporter: fake description " contains: "fake reporter" with 1 message: 'Tested reporter: sonarqube' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "<?xml version="1.0" encoding="UTF-8"?> All available test cases: fake test name [fakeTestTag] 1 test case " ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: sonarqube' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fakeTag"s) for: "All available tags: 1 [fakeTag] 1 tag " contains: "fakeTag" with 1 message: 'Tested reporter: tap' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fake reporter"s) for: "Available reporters: fake reporter: fake description " contains: "fake reporter" with 1 message: 'Tested reporter: tap' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "All available test cases: fake test name [fakeTestTag] 1 test case " ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: tap' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fakeTag"s) for: "All available tags: 1 [fakeTag] 1 tag " contains: "fakeTag" with 1 message: 'Tested reporter: teamcity' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fake reporter"s) for: "Available reporters: fake reporter: fake description " contains: "fake reporter" with 1 message: 'Tested reporter: teamcity' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "All available test cases: fake test name [fakeTestTag] 1 test case " ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: teamcity' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fakeTag"s) for: "<?xml version="1.0" encoding="UTF-8"?> <TagsFromMatchingTests> <Tag> <Count>1</Count> <Aliases> <Alias>fakeTag</Alias> </Aliases> </Tag> </TagsFromMatchingTests>" contains: "fakeTag" with 1 message: 'Tested reporter: xml' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains("fake reporter"s) for: "<?xml version="1.0" encoding="UTF-8"?> <AvailableReporters> <Reporter> <Name>fake reporter</Name> <Description>fake description</Description> </Reporter> </AvailableReporters>" contains: "fake reporter" with 1 message: 'Tested reporter: xml' +# Reporter's write listings to provided stream +ok {test-number} - !(factories.empty()) for: !false +# Reporter's write listings to provided stream +ok {test-number} - listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "<?xml version="1.0" encoding="UTF-8"?> <MatchingTests> <TestCase> <Name>fake test name</Name> <ClassName/> <Tags>[fakeTestTag]</Tags> <SourceInfo> <File>fake-file.cpp</File> <Line>123456789</Line> </SourceInfo> </TestCase> </MatchingTests>" ( contains: "fake test name" and contains: "fakeTestTag" ) with 1 message: 'Tested reporter: xml' # SUCCEED counts as a test pass ok {test-number} - with 1 message: 'this is a success' # SUCCEED does not require an argument @@ -3001,6 +3097,12 @@ ok {test-number} - 0x<hex digits> == bit30and31 for: 3221225472 (0x<hex digits>) ok {test-number} - # The NO_FAIL macro reports a failure but does not fail the test ok {test-number} - 1 == 2 # TODO +# The default listing implementation write to provided stream +ok {test-number} - listingString, Contains("[fakeTag]"s) for: "All available tags: 1 [fakeTag] 1 tag " contains: "[fakeTag]" +# The default listing implementation write to provided stream +ok {test-number} - listingString, Contains("fake reporter"s) for: "Available reporters: fake reporter: fake description " contains: "fake reporter" +# The default listing implementation write to provided stream +ok {test-number} - listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) for: "All available test cases: fake test name [fakeTestTag] 1 test case " ( contains: "fake test name" and contains: "fakeTestTag" ) # This test 'should' fail but doesn't ok {test-number} - with 1 message: 'oops!' # Thrown string literals are translated @@ -4100,5 +4202,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..2054 +1..2105 diff --git a/tests/SelfTest/Baselines/teamcity.sw.approved.txt b/tests/SelfTest/Baselines/teamcity.sw.approved.txt index 413e5153..e64c984f 100644 --- a/tests/SelfTest/Baselines/teamcity.sw.approved.txt +++ b/tests/SelfTest/Baselines/teamcity.sw.approved.txt @@ -475,6 +475,8 @@ Matchers.tests.cpp:<line number>|nexpression failed|n CHECK_THAT( testStringFor ##teamcity[testFinished name='Regex string matcher' duration="{duration}"] ##teamcity[testStarted name='Regression test #1'] ##teamcity[testFinished name='Regression test #1' duration="{duration}"] +##teamcity[testStarted name='Reporter|'s write listings to provided stream'] +##teamcity[testFinished name='Reporter|'s write listings to provided stream' duration="{duration}"] ##teamcity[testStarted name='SUCCEED counts as a test pass'] ##teamcity[testFinished name='SUCCEED counts as a test pass' duration="{duration}"] ##teamcity[testStarted name='SUCCEED does not require an argument'] @@ -563,6 +565,8 @@ Misc.tests.cpp:<line number>|nexpression failed|n CHECK( s1 == s2 )|nwith expan ##teamcity[testFinished name='Test with special, characters "in name' duration="{duration}"] ##teamcity[testStarted name='The NO_FAIL macro reports a failure but does not fail the test'] ##teamcity[testFinished name='The NO_FAIL macro reports a failure but does not fail the test' duration="{duration}"] +##teamcity[testStarted name='The default listing implementation write to provided stream'] +##teamcity[testFinished name='The default listing implementation write to provided stream' duration="{duration}"] ##teamcity[testStarted name='This test |'should|' fail but doesn|'t'] ##teamcity[testFinished name='This test |'should|' fail but doesn|'t' duration="{duration}"] ##teamcity[testStarted name='Thrown string literals are translated'] diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index e13949a0..ceaa32bc 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -11483,6 +11483,652 @@ Nor would this </Expression> <OverallResult success="true"/> </TestCase> + <TestCase name="Reporter's write listings to provided stream" tags="[reporters]" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="automake reporter lists tags" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: automake + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fakeTag"s) + </Original> + <Expanded> + "All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="automake reporter lists reporters" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: automake + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fake reporter"s) + </Original> + <Expanded> + "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="automake reporter lists tests" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: automake + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) + </Original> + <Expanded> + "All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="compact reporter lists tags" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: compact + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fakeTag"s) + </Original> + <Expanded> + "All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="compact reporter lists reporters" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: compact + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fake reporter"s) + </Original> + <Expanded> + "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="compact reporter lists tests" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: compact + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) + </Original> + <Expanded> + "All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="console reporter lists tags" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: console + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fakeTag"s) + </Original> + <Expanded> + "All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="console reporter lists reporters" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: console + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fake reporter"s) + </Original> + <Expanded> + "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="console reporter lists tests" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: console + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) + </Original> + <Expanded> + "All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="junit reporter lists tags" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: junit + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fakeTag"s) + </Original> + <Expanded> + "<?xml version="1.0" encoding="UTF-8"?> +All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="junit reporter lists reporters" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: junit + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fake reporter"s) + </Original> + <Expanded> + "<?xml version="1.0" encoding="UTF-8"?> +Available reporters: + fake reporter: fake description + +" contains: "fake reporter" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="junit reporter lists tests" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: junit + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) + </Original> + <Expanded> + "<?xml version="1.0" encoding="UTF-8"?> +All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="sonarqube reporter lists tags" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: sonarqube + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fakeTag"s) + </Original> + <Expanded> + "<?xml version="1.0" encoding="UTF-8"?> +All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="sonarqube reporter lists reporters" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: sonarqube + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fake reporter"s) + </Original> + <Expanded> + "<?xml version="1.0" encoding="UTF-8"?> +Available reporters: + fake reporter: fake description + +" contains: "fake reporter" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="sonarqube reporter lists tests" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: sonarqube + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) + </Original> + <Expanded> + "<?xml version="1.0" encoding="UTF-8"?> +All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="tap reporter lists tags" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: tap + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fakeTag"s) + </Original> + <Expanded> + "All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="tap reporter lists reporters" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: tap + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fake reporter"s) + </Original> + <Expanded> + "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="tap reporter lists tests" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: tap + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) + </Original> + <Expanded> + "All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="teamcity reporter lists tags" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: teamcity + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fakeTag"s) + </Original> + <Expanded> + "All available tags: + 1 [fakeTag] +1 tag + +" contains: "fakeTag" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="teamcity reporter lists reporters" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: teamcity + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fake reporter"s) + </Original> + <Expanded> + "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="teamcity reporter lists tests" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: teamcity + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) + </Original> + <Expanded> + "All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="xml reporter lists tags" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: xml + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fakeTag"s) + </Original> + <Expanded> + "<?xml version="1.0" encoding="UTF-8"?> +<TagsFromMatchingTests> + <Tag> + <Count>1</Count> + <Aliases> + <Alias>fakeTag</Alias> + </Aliases> + </Tag> +</TagsFromMatchingTests>" contains: "fakeTag" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="xml reporter lists reporters" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: xml + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fake reporter"s) + </Original> + <Expanded> + "<?xml version="1.0" encoding="UTF-8"?> +<AvailableReporters> + <Reporter> + <Name>fake reporter</Name> + <Description>fake description</Description> + </Reporter> +</AvailableReporters>" contains: "fake reporter" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Expression success="true" type="REQUIRE_FALSE" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + !(factories.empty()) + </Original> + <Expanded> + !false + </Expanded> + </Expression> + <Section name="xml reporter lists tests" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Info> + Tested reporter: xml + </Info> + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) + </Original> + <Expanded> + "<?xml version="1.0" encoding="UTF-8"?> +<MatchingTests> + <TestCase> + <Name>fake test name</Name> + <ClassName/> + <Tags>[fakeTestTag]</Tags> + <SourceInfo> + <File>fake-file.cpp</File> + <Line>123456789</Line> + </SourceInfo> + </TestCase> +</MatchingTests>" ( contains: "fake test name" and contains: "fakeTestTag" ) + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <OverallResult success="true"/> + </TestCase> <TestCase name="SUCCEED counts as a test pass" tags="[messages]" filename="tests/<exe-name>/UsageTests/Message.tests.cpp" > <OverallResult success="true"/> </TestCase> @@ -13838,6 +14484,54 @@ Message from section two </Expression> <OverallResult success="false"/> </TestCase> + <TestCase name="The default listing implementation write to provided stream" tags="[reporter-helpers][reporters]" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Section name="Listing tags" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("[fakeTag]"s) + </Original> + <Expanded> + "All available tags: + 1 [fakeTag] +1 tag + +" contains: "[fakeTag]" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Section name="Listing reporters" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains("fake reporter"s) + </Original> + <Expanded> + "Available reporters: + fake reporter: fake description + +" contains: "fake reporter" + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <Section name="Listing tests" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Expression success="true" type="REQUIRE_THAT" filename="tests/<exe-name>/IntrospectiveTests/Reporters.tests.cpp" > + <Original> + listingString, Contains( "fake test name"s ) && Contains( "fakeTestTag"s ) + </Original> + <Expanded> + "All available test cases: + fake test name + [fakeTestTag] +1 test case + +" ( contains: "fake test name" and contains: "fakeTestTag" ) + </Expanded> + </Expression> + <OverallResults successes="1" failures="0" expectedFailures="0"/> + </Section> + <OverallResult success="true"/> + </TestCase> <TestCase name="This test 'should' fail but doesn't" tags="[!shouldfail][.][failing]" filename="tests/<exe-name>/UsageTests/Misc.tests.cpp" > <OverallResult success="false"/> </TestCase> @@ -19021,9 +19715,9 @@ loose text artifact </Section> <OverallResult success="true"/> </TestCase> - <OverallResults successes="1885" failures="149" expectedFailures="21"/> - <OverallResultsCases successes="264" failures="86" expectedFailures="4"/> + <OverallResults successes="1936" failures="149" expectedFailures="21"/> + <OverallResultsCases successes="266" failures="86" expectedFailures="4"/> </Group> - <OverallResults successes="1885" failures="148" expectedFailures="21"/> - <OverallResultsCases successes="264" failures="86" expectedFailures="4"/> + <OverallResults successes="1936" failures="148" expectedFailures="21"/> + <OverallResultsCases successes="266" failures="86" expectedFailures="4"/> </Catch> diff --git a/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp b/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp new file mode 100644 index 00000000..c5c97998 --- /dev/null +++ b/tests/SelfTest/IntrospectiveTests/Reporters.tests.cpp @@ -0,0 +1,109 @@ + +// Copyright Catch2 Authors +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE_1_0.txt or copy at +// https://www.boost.org/LICENSE_1_0.txt) + +// SPDX-License-Identifier: BSL-1.0 + +#include <catch2/catch_test_macros.hpp> + +#include <catch2/catch_test_case_info.hpp> +#include <catch2/catch_config.hpp> +#include <catch2/interfaces/catch_interfaces_reporter.hpp> +#include <catch2/interfaces/catch_interfaces_reporter_factory.hpp> +#include <catch2/interfaces/catch_interfaces_reporter_registry.hpp> +#include <catch2/internal/catch_list.hpp> +#include <catch2/matchers/catch_matchers_string.hpp> +#include <catch2/reporters/catch_reporter_helpers.hpp> + +#include <sstream> + +TEST_CASE( "The default listing implementation write to provided stream", + "[reporters][reporter-helpers]" ) { + using Catch::Matchers::Contains; + using namespace std::string_literals; + + std::stringstream sstream; + SECTION( "Listing tags" ) { + std::vector<Catch::TagInfo> tags(1); + tags[0].add("fakeTag"_catch_sr); + Catch::defaultListTags(sstream, tags, false); + + auto listingString = sstream.str(); + REQUIRE_THAT(listingString, Contains("[fakeTag]"s)); + } + SECTION( "Listing reporters" ) { + std::vector<Catch::ReporterDescription> reporters( + { { "fake reporter", "fake description" } } ); + Catch::defaultListReporters(sstream, reporters, Catch::Verbosity::Normal); + + auto listingString = sstream.str(); + REQUIRE_THAT(listingString, Contains("fake reporter"s)); + } + SECTION( "Listing tests" ) { + Catch::TestCaseInfo fakeInfo{ + ""s, + { "fake test name"_catch_sr, "[fakeTestTag]"_catch_sr }, + { "fake-file.cpp", 123456789 } }; + std::vector<Catch::TestCaseHandle> tests({ {&fakeInfo, nullptr} }); + Catch::defaultListTests(sstream, tests, false, Catch::Verbosity::Normal); + + auto listingString = sstream.str(); + REQUIRE_THAT( listingString, + Contains( "fake test name"s ) && + Contains( "fakeTestTag"s ) ); + } +} + +TEST_CASE( "Reporter's write listings to provided stream", "[reporters]" ) { + using Catch::Matchers::Contains; + using namespace std::string_literals; + + auto const& factories = Catch::getRegistryHub().getReporterRegistry().getFactories(); + // If there are no reporters, the test would pass falsely + // while there is something obviously broken + REQUIRE_FALSE(factories.empty()); + + for (auto const& factory : factories) { + INFO("Tested reporter: " << factory.first); + std::stringstream sstream; + + Catch::ConfigData config_data; + Catch::Config config( config_data ); + Catch::ReporterConfig rep_config( &config, sstream ); + auto reporter = factory.second->create( rep_config ); + + DYNAMIC_SECTION( factory.first << " reporter lists tags" ) { + std::vector<Catch::TagInfo> tags(1); + tags[0].add("fakeTag"_catch_sr); + reporter->listTags(tags); + + auto listingString = sstream.str(); + REQUIRE_THAT(listingString, Contains("fakeTag"s)); + } + + DYNAMIC_SECTION( factory.first << " reporter lists reporters" ) { + std::vector<Catch::ReporterDescription> reporters( + { { "fake reporter", "fake description" } } ); + reporter->listReporters(reporters); + + auto listingString = sstream.str(); + REQUIRE_THAT(listingString, Contains("fake reporter"s)); + } + + DYNAMIC_SECTION( factory.first << " reporter lists tests" ) { + Catch::TestCaseInfo fakeInfo{ + ""s, + { "fake test name"_catch_sr, "[fakeTestTag]"_catch_sr }, + { "fake-file.cpp", 123456789 } }; + std::vector<Catch::TestCaseHandle> tests({ {&fakeInfo, nullptr} }); + reporter->listTests(tests); + + auto listingString = sstream.str(); + REQUIRE_THAT( listingString, + Contains( "fake test name"s ) && + Contains( "fakeTestTag"s ) ); + } + } +}