Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorMartin Hořeňovský2022-04-19 10:32:13 +0200
committerMartin Hořeňovský2022-04-19 10:32:13 +0200
commit17a04f88d90d3687bd561e89dab40e4ba6a55b83 (patch)
tree76ccd8e074c7ed3e63cc7fa34be219cdb8a86627
parent90e69050505ff210ca29e1c9e8feaf598de4fc74 (diff)
downloadcatch3-17a04f88d90d3687bd561e89dab40e4ba6a55b83.tar.gz
Always report rng seed from builtin reporters
Not all reporters use a format that supports this, so TeamCity and Automake reporters still do not report it. The console reporter now reports it even on successful runs, where before it only reported the rng seed in the header, which was showed either for failed run, or for run with `-s`. CLoses #2065
-rw-r--r--src/catch2/reporters/catch_reporter_compact.cpp11
-rw-r--r--src/catch2/reporters/catch_reporter_compact.hpp2
-rw-r--r--src/catch2/reporters/catch_reporter_console.cpp16
-rw-r--r--src/catch2/reporters/catch_reporter_console.hpp1
-rw-r--r--src/catch2/reporters/catch_reporter_sonarqube.cpp12
-rw-r--r--src/catch2/reporters/catch_reporter_tap.cpp5
-rw-r--r--src/catch2/reporters/catch_reporter_tap.hpp2
-rw-r--r--tests/CMakeLists.txt22
-rw-r--r--tests/SelfTest/Baselines/compact.sw.approved.txt2
-rw-r--r--tests/SelfTest/Baselines/compact.sw.multi.approved.txt2
-rw-r--r--tests/SelfTest/Baselines/console.std.approved.txt3
-rw-r--r--tests/SelfTest/Baselines/console.sw.approved.txt3
-rw-r--r--tests/SelfTest/Baselines/console.sw.multi.approved.txt3
-rw-r--r--tests/SelfTest/Baselines/console.swa4.approved.txt3
-rw-r--r--tests/SelfTest/Baselines/sonarqube.sw.approved.txt1
-rw-r--r--tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt1
-rw-r--r--tests/SelfTest/Baselines/tap.sw.approved.txt1
-rw-r--r--tests/SelfTest/Baselines/tap.sw.multi.approved.txt1
-rw-r--r--tests/TestScripts/testConfigureDefaultReporter.py2
19 files changed, 73 insertions, 20 deletions
diff --git a/src/catch2/reporters/catch_reporter_compact.cpp b/src/catch2/reporters/catch_reporter_compact.cpp
index 1d3d0329..3f036ec5 100644
--- a/src/catch2/reporters/catch_reporter_compact.cpp
+++ b/src/catch2/reporters/catch_reporter_compact.cpp
@@ -7,6 +7,7 @@
// SPDX-License-Identifier: BSL-1.0
#include <catch2/reporters/catch_reporter_compact.hpp>
+#include <catch2/catch_test_spec.hpp>
#include <catch2/reporters/catch_reporter_helpers.hpp>
#include <catch2/interfaces/catch_interfaces_config.hpp>
#include <catch2/internal/catch_platform.hpp>
@@ -253,6 +254,16 @@ private:
m_stream << "No test cases matched '" << unmatchedSpec << "'\n";
}
+ void CompactReporter::testRunStarting( TestRunInfo const& ) {
+ if ( m_config->testSpec().hasFilters() ) {
+ m_stream << m_colour->guardColour( Colour::BrightYellow )
+ << "Filters: "
+ << serializeFilters( m_config->getTestsOrTags() )
+ << '\n';
+ }
+ m_stream << "RNG seed: " << m_config->rngSeed() << '\n';
+ }
+
void CompactReporter::assertionEnded( AssertionStats const& _assertionStats ) {
AssertionResult const& result = _assertionStats.assertionResult;
diff --git a/src/catch2/reporters/catch_reporter_compact.hpp b/src/catch2/reporters/catch_reporter_compact.hpp
index 4232138c..6362fc29 100644
--- a/src/catch2/reporters/catch_reporter_compact.hpp
+++ b/src/catch2/reporters/catch_reporter_compact.hpp
@@ -24,6 +24,8 @@ namespace Catch {
void noMatchingTestCases( StringRef unmatchedSpec ) override;
+ void testRunStarting( TestRunInfo const& _testInfo ) override;
+
void assertionEnded(AssertionStats const& _assertionStats) override;
void sectionEnded(SectionStats const& _sectionStats) override;
diff --git a/src/catch2/reporters/catch_reporter_console.cpp b/src/catch2/reporters/catch_reporter_console.cpp
index 6b564b15..bb755ed7 100644
--- a/src/catch2/reporters/catch_reporter_console.cpp
+++ b/src/catch2/reporters/catch_reporter_console.cpp
@@ -496,7 +496,11 @@ void ConsoleReporter::testRunEnded(TestRunStats const& _testRunStats) {
}
void ConsoleReporter::testRunStarting(TestRunInfo const& _testInfo) {
StreamingReporterBase::testRunStarting(_testInfo);
- printTestFilters();
+ if ( m_config->testSpec().hasFilters() ) {
+ m_stream << m_colour->guardColour( Colour::BrightYellow ) << "Filters: "
+ << serializeFilters( m_config->getTestsOrTags() ) << '\n';
+ }
+ m_stream << "Randomness seeded to: " << m_config->rngSeed() << '\n';
}
void ConsoleReporter::lazyPrint() {
@@ -521,8 +525,7 @@ void ConsoleReporter::lazyPrintRunInfo() {
<< m_colour->guardColour( Colour::SecondaryText )
<< currentTestRunInfo.name << " is a Catch2 v" << libraryVersion()
<< " host application.\n"
- << "Run with -? for options\n\n"
- << "Randomness seeded to: " << m_config->rngSeed() << "\n\n";
+ << "Run with -? for options\n\n";
m_testRunInfoPrinted = true;
}
@@ -701,13 +704,6 @@ void ConsoleReporter::printSummaryDivider() {
m_stream << lineOfChars('-') << '\n';
}
-void ConsoleReporter::printTestFilters() {
- if (m_config->testSpec().hasFilters()) {
- m_stream << m_colour->guardColour( Colour::BrightYellow ) << "Filters: "
- << serializeFilters( m_config->getTestsOrTags() ) << '\n';
- }
-}
-
} // end namespace Catch
#if defined(_MSC_VER)
diff --git a/src/catch2/reporters/catch_reporter_console.hpp b/src/catch2/reporters/catch_reporter_console.hpp
index 12bfc0e2..719dcc44 100644
--- a/src/catch2/reporters/catch_reporter_console.hpp
+++ b/src/catch2/reporters/catch_reporter_console.hpp
@@ -63,7 +63,6 @@ namespace Catch {
void printTotalsDivider(Totals const& totals);
void printSummaryDivider();
- void printTestFilters();
bool m_headerPrinted = false;
bool m_testRunInfoPrinted = false;
diff --git a/src/catch2/reporters/catch_reporter_sonarqube.cpp b/src/catch2/reporters/catch_reporter_sonarqube.cpp
index d45bbfe1..8119c3b8 100644
--- a/src/catch2/reporters/catch_reporter_sonarqube.cpp
+++ b/src/catch2/reporters/catch_reporter_sonarqube.cpp
@@ -9,13 +9,25 @@
#include <catch2/internal/catch_string_manip.hpp>
#include <catch2/catch_test_case_info.hpp>
+#include <catch2/internal/catch_reusable_string_stream.hpp>
+#include <catch2/interfaces/catch_interfaces_config.hpp>
#include <map>
namespace Catch {
+ namespace {
+ std::string createRngSeedString(uint32_t seed) {
+ ReusableStringStream sstr;
+ sstr << "rng-seed=" << seed;
+ return sstr.str();
+ }
+ }
+
void SonarQubeReporter::testRunStarting(TestRunInfo const& testRunInfo) {
CumulativeReporterBase::testRunStarting(testRunInfo);
+
+ xml.writeComment( createRngSeedString( m_config->rngSeed() ) );
xml.startElement("testExecutions");
xml.writeAttribute("version"_sr, '1');
}
diff --git a/src/catch2/reporters/catch_reporter_tap.cpp b/src/catch2/reporters/catch_reporter_tap.cpp
index 71249f87..8ecf3928 100644
--- a/src/catch2/reporters/catch_reporter_tap.cpp
+++ b/src/catch2/reporters/catch_reporter_tap.cpp
@@ -9,6 +9,7 @@
#include <catch2/internal/catch_console_colour.hpp>
#include <catch2/internal/catch_string_manip.hpp>
#include <catch2/catch_test_case_info.hpp>
+#include <catch2/interfaces/catch_interfaces_config.hpp>
#include <algorithm>
#include <iterator>
@@ -189,6 +190,10 @@ namespace Catch {
} // End anonymous namespace
+ void TAPReporter::testRunStarting( TestRunInfo const& ) {
+ m_stream << "# rng-seed: " << m_config->rngSeed() << '\n';
+ }
+
void TAPReporter::noMatchingTestCases( StringRef unmatchedSpec ) {
m_stream << "# No test cases matched '" << unmatchedSpec << "'\n";
}
diff --git a/src/catch2/reporters/catch_reporter_tap.hpp b/src/catch2/reporters/catch_reporter_tap.hpp
index b8c3208f..0c4b14b4 100644
--- a/src/catch2/reporters/catch_reporter_tap.hpp
+++ b/src/catch2/reporters/catch_reporter_tap.hpp
@@ -26,6 +26,8 @@ namespace Catch {
return "Reports test results in TAP format, suitable for test harnesses"s;
}
+ void testRunStarting( TestRunInfo const& testInfo ) override;
+
void noMatchingTestCases( StringRef unmatchedSpec ) override;
void assertionEnded(AssertionStats const& _assertionStats) override;
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 5ea6b7cd..911f3857 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -523,5 +523,27 @@ if (CATCH_ENABLE_CONFIGURE_TESTS)
endforeach()
endif()
+foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata
+ "compact"
+ "console"
+ "JUnit"
+ "SonarQube"
+ "TAP"
+ # "TeamCity" - does not seem to support test suite-level metadata/comments
+ "XML")
+ add_test(NAME "Reporters:RngSeed:${reporterName}"
+ COMMAND
+ $<TARGET_FILE:SelfTest> "Factorials are computed"
+ --reporter ${reporterName}
+ --rng-seed 18181818
+ )
+ set_tests_properties("Reporters:RngSeed:${reporterName}"
+ PROPERTIES
+ PASS_REGULAR_EXPRESSION "18181818"
+ )
+
+endforeach()
+
+
list(APPEND CATCH_WARNING_TARGETS SelfTest)
set(CATCH_WARNING_TARGETS ${CATCH_WARNING_TARGETS} PARENT_SCOPE)
diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt
index 03a07f53..bf9482c6 100644
--- a/tests/SelfTest/Baselines/compact.sw.approved.txt
+++ b/tests/SelfTest/Baselines/compact.sw.approved.txt
@@ -1,3 +1,5 @@
+Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+RNG seed: 1
Misc.tests.cpp:<line number>: passed: with 1 message: 'yay'
Compilation.tests.cpp:<line number>: passed: y.v == 0 for: 0 == 0
Compilation.tests.cpp:<line number>: passed: 0 == y.v for: 0 == 0
diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt
index 105789c9..a94192eb 100644
--- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt
@@ -1,3 +1,5 @@
+Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+RNG seed: 1
Misc.tests.cpp:<line number>: passed: with 1 message: 'yay'
Compilation.tests.cpp:<line number>: passed: y.v == 0 for: 0 == 0
Compilation.tests.cpp:<line number>: passed: 0 == y.v for: 0 == 0
diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt
index a6847cdf..c85a9203 100644
--- a/tests/SelfTest/Baselines/console.std.approved.txt
+++ b/tests/SelfTest/Baselines/console.std.approved.txt
@@ -1,11 +1,10 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
Run with -? for options
-Randomness seeded to: 1
-
-------------------------------------------------------------------------------
#1455 - INFO and WARN can start with a linebreak
-------------------------------------------------------------------------------
diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt
index 66706e13..8bb1c3db 100644
--- a/tests/SelfTest/Baselines/console.sw.approved.txt
+++ b/tests/SelfTest/Baselines/console.sw.approved.txt
@@ -1,11 +1,10 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
Run with -? for options
-Randomness seeded to: 1
-
-------------------------------------------------------------------------------
# A test name that starts with a #
-------------------------------------------------------------------------------
diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt
index 08b16228..a64c56a5 100644
--- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt
@@ -1,11 +1,10 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
Run with -? for options
-Randomness seeded to: 1
-
-------------------------------------------------------------------------------
# A test name that starts with a #
-------------------------------------------------------------------------------
diff --git a/tests/SelfTest/Baselines/console.swa4.approved.txt b/tests/SelfTest/Baselines/console.swa4.approved.txt
index 8b4b4c13..c25719d8 100644
--- a/tests/SelfTest/Baselines/console.swa4.approved.txt
+++ b/tests/SelfTest/Baselines/console.swa4.approved.txt
@@ -1,11 +1,10 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
Run with -? for options
-Randomness seeded to: 1
-
-------------------------------------------------------------------------------
# A test name that starts with a #
-------------------------------------------------------------------------------
diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
index 83b5d3cb..f2ec6f49 100644
--- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
+++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
@@ -1,4 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!-- rng-seed=1 -->
<testExecutions version="1"loose text artifact
>
<file path="tests/<exe-name>/IntrospectiveTests/Clara.tests.cpp">
diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt
index 2723afe8..98cb8b7d 100644
--- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt
@@ -1,4 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!-- rng-seed=1 -->
<testExecutions version="1">
<file path="tests/<exe-name>/IntrospectiveTests/Clara.tests.cpp">
<testCase name="Clara::Arg supports single-arg parse the way Opt does" duration="{duration}"/>
diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt
index 2f375905..a2e69782 100644
--- a/tests/SelfTest/Baselines/tap.sw.approved.txt
+++ b/tests/SelfTest/Baselines/tap.sw.approved.txt
@@ -1,3 +1,4 @@
+# rng-seed: 1
# # A test name that starts with a #
ok {test-number} - with 1 message: 'yay'
# #1027: Bitfields can be captured
diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt
index 2c5790b4..69d85c7a 100644
--- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt
@@ -1,3 +1,4 @@
+# rng-seed: 1
# # A test name that starts with a #
ok {test-number} - with 1 message: 'yay'
# #1027: Bitfields can be captured
diff --git a/tests/TestScripts/testConfigureDefaultReporter.py b/tests/TestScripts/testConfigureDefaultReporter.py
index 24ec8228..66c88da0 100644
--- a/tests/TestScripts/testConfigureDefaultReporter.py
+++ b/tests/TestScripts/testConfigureDefaultReporter.py
@@ -38,7 +38,7 @@ stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTe
# This matches the summary line made by compact reporter, console reporter's
# summary line does not match the regex.
summary_regex = 'Passed \d+ test case with \d+ assertions.'
-if not re.match(summary_regex, stdout):
+if not re.search(summary_regex, stdout):
print("Could not find '{}' in the stdout".format(summary_regex))
print('stdout: "{}"'.format(stdout))
exit(2)