Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/tests
diff options
context:
space:
mode:
authorMartin Hořeňovský2022-04-19 10:32:13 +0200
committerMartin Hořeňovský2022-04-19 10:32:13 +0200
commit17a04f88d90d3687bd561e89dab40e4ba6a55b83 (patch)
tree76ccd8e074c7ed3e63cc7fa34be219cdb8a86627 /tests
parent90e69050505ff210ca29e1c9e8feaf598de4fc74 (diff)
downloadcatch3-17a04f88d90d3687bd561e89dab40e4ba6a55b83.tar.gz
Always report rng seed from builtin reporters
Not all reporters use a format that supports this, so TeamCity and Automake reporters still do not report it. The console reporter now reports it even on successful runs, where before it only reported the rng seed in the header, which was showed either for failed run, or for run with `-s`. CLoses #2065
Diffstat (limited to 'tests')
-rw-r--r--tests/CMakeLists.txt22
-rw-r--r--tests/SelfTest/Baselines/compact.sw.approved.txt2
-rw-r--r--tests/SelfTest/Baselines/compact.sw.multi.approved.txt2
-rw-r--r--tests/SelfTest/Baselines/console.std.approved.txt3
-rw-r--r--tests/SelfTest/Baselines/console.sw.approved.txt3
-rw-r--r--tests/SelfTest/Baselines/console.sw.multi.approved.txt3
-rw-r--r--tests/SelfTest/Baselines/console.swa4.approved.txt3
-rw-r--r--tests/SelfTest/Baselines/sonarqube.sw.approved.txt1
-rw-r--r--tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt1
-rw-r--r--tests/SelfTest/Baselines/tap.sw.approved.txt1
-rw-r--r--tests/SelfTest/Baselines/tap.sw.multi.approved.txt1
-rw-r--r--tests/TestScripts/testConfigureDefaultReporter.py2
12 files changed, 35 insertions, 9 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 5ea6b7cd..911f3857 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -523,5 +523,27 @@ if (CATCH_ENABLE_CONFIGURE_TESTS)
endforeach()
endif()
+foreach (reporterName # "Automake" - the simple .trs format does not support any kind of comments/metadata
+ "compact"
+ "console"
+ "JUnit"
+ "SonarQube"
+ "TAP"
+ # "TeamCity" - does not seem to support test suite-level metadata/comments
+ "XML")
+ add_test(NAME "Reporters:RngSeed:${reporterName}"
+ COMMAND
+ $<TARGET_FILE:SelfTest> "Factorials are computed"
+ --reporter ${reporterName}
+ --rng-seed 18181818
+ )
+ set_tests_properties("Reporters:RngSeed:${reporterName}"
+ PROPERTIES
+ PASS_REGULAR_EXPRESSION "18181818"
+ )
+
+endforeach()
+
+
list(APPEND CATCH_WARNING_TARGETS SelfTest)
set(CATCH_WARNING_TARGETS ${CATCH_WARNING_TARGETS} PARENT_SCOPE)
diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt
index 03a07f53..bf9482c6 100644
--- a/tests/SelfTest/Baselines/compact.sw.approved.txt
+++ b/tests/SelfTest/Baselines/compact.sw.approved.txt
@@ -1,3 +1,5 @@
+Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+RNG seed: 1
Misc.tests.cpp:<line number>: passed: with 1 message: 'yay'
Compilation.tests.cpp:<line number>: passed: y.v == 0 for: 0 == 0
Compilation.tests.cpp:<line number>: passed: 0 == y.v for: 0 == 0
diff --git a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt
index 105789c9..a94192eb 100644
--- a/tests/SelfTest/Baselines/compact.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/compact.sw.multi.approved.txt
@@ -1,3 +1,5 @@
+Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+RNG seed: 1
Misc.tests.cpp:<line number>: passed: with 1 message: 'yay'
Compilation.tests.cpp:<line number>: passed: y.v == 0 for: 0 == 0
Compilation.tests.cpp:<line number>: passed: 0 == y.v for: 0 == 0
diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt
index a6847cdf..c85a9203 100644
--- a/tests/SelfTest/Baselines/console.std.approved.txt
+++ b/tests/SelfTest/Baselines/console.std.approved.txt
@@ -1,11 +1,10 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
Run with -? for options
-Randomness seeded to: 1
-
-------------------------------------------------------------------------------
#1455 - INFO and WARN can start with a linebreak
-------------------------------------------------------------------------------
diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt
index 66706e13..8bb1c3db 100644
--- a/tests/SelfTest/Baselines/console.sw.approved.txt
+++ b/tests/SelfTest/Baselines/console.sw.approved.txt
@@ -1,11 +1,10 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
Run with -? for options
-Randomness seeded to: 1
-
-------------------------------------------------------------------------------
# A test name that starts with a #
-------------------------------------------------------------------------------
diff --git a/tests/SelfTest/Baselines/console.sw.multi.approved.txt b/tests/SelfTest/Baselines/console.sw.multi.approved.txt
index 08b16228..a64c56a5 100644
--- a/tests/SelfTest/Baselines/console.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/console.sw.multi.approved.txt
@@ -1,11 +1,10 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
Run with -? for options
-Randomness seeded to: 1
-
-------------------------------------------------------------------------------
# A test name that starts with a #
-------------------------------------------------------------------------------
diff --git a/tests/SelfTest/Baselines/console.swa4.approved.txt b/tests/SelfTest/Baselines/console.swa4.approved.txt
index 8b4b4c13..c25719d8 100644
--- a/tests/SelfTest/Baselines/console.swa4.approved.txt
+++ b/tests/SelfTest/Baselines/console.swa4.approved.txt
@@ -1,11 +1,10 @@
Filters: ~[!nonportable]~[!benchmark]~[approvals] *
+Randomness seeded to: 1
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
<exe-name> is a <version> host application.
Run with -? for options
-Randomness seeded to: 1
-
-------------------------------------------------------------------------------
# A test name that starts with a #
-------------------------------------------------------------------------------
diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
index 83b5d3cb..f2ec6f49 100644
--- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
+++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
@@ -1,4 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!-- rng-seed=1 -->
<testExecutions version="1"loose text artifact
>
<file path="tests/<exe-name>/IntrospectiveTests/Clara.tests.cpp">
diff --git a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt
index 2723afe8..98cb8b7d 100644
--- a/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/sonarqube.sw.multi.approved.txt
@@ -1,4 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
+<!-- rng-seed=1 -->
<testExecutions version="1">
<file path="tests/<exe-name>/IntrospectiveTests/Clara.tests.cpp">
<testCase name="Clara::Arg supports single-arg parse the way Opt does" duration="{duration}"/>
diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt
index 2f375905..a2e69782 100644
--- a/tests/SelfTest/Baselines/tap.sw.approved.txt
+++ b/tests/SelfTest/Baselines/tap.sw.approved.txt
@@ -1,3 +1,4 @@
+# rng-seed: 1
# # A test name that starts with a #
ok {test-number} - with 1 message: 'yay'
# #1027: Bitfields can be captured
diff --git a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt
index 2c5790b4..69d85c7a 100644
--- a/tests/SelfTest/Baselines/tap.sw.multi.approved.txt
+++ b/tests/SelfTest/Baselines/tap.sw.multi.approved.txt
@@ -1,3 +1,4 @@
+# rng-seed: 1
# # A test name that starts with a #
ok {test-number} - with 1 message: 'yay'
# #1027: Bitfields can be captured
diff --git a/tests/TestScripts/testConfigureDefaultReporter.py b/tests/TestScripts/testConfigureDefaultReporter.py
index 24ec8228..66c88da0 100644
--- a/tests/TestScripts/testConfigureDefaultReporter.py
+++ b/tests/TestScripts/testConfigureDefaultReporter.py
@@ -38,7 +38,7 @@ stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTe
# This matches the summary line made by compact reporter, console reporter's
# summary line does not match the regex.
summary_regex = 'Passed \d+ test case with \d+ assertions.'
-if not re.match(summary_regex, stdout):
+if not re.search(summary_regex, stdout):
print("Could not find '{}' in the stdout".format(summary_regex))
print('stdout: "{}"'.format(stdout))
exit(2)