Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/tests
diff options
context:
space:
mode:
authorkhyperia2020-01-27 15:43:27 +0100
committerMartin Hořeňovský2020-02-16 15:44:23 +0100
commit3c7e737a7bcf2d2059b1db2a5a221f76806ae8b0 (patch)
treebcbf40fe8e4914c8ecc3a585066933bf871652c7 /tests
parente880da93bd335e59a0d4b00bfcb9f06ab4b7c847 (diff)
downloadcatch3-3c7e737a7bcf2d2059b1db2a5a221f76806ae8b0.tar.gz
Allow configuring of benchmark warmup time
Diffstat (limited to 'tests')
-rw-r--r--tests/SelfTest/Baselines/compact.sw.approved.txt2
-rw-r--r--tests/SelfTest/Baselines/console.std.approved.txt2
-rw-r--r--tests/SelfTest/Baselines/console.sw.approved.txt24
-rw-r--r--tests/SelfTest/Baselines/junit.sw.approved.txt7
-rw-r--r--tests/SelfTest/Baselines/sonarqube.sw.approved.txt5
-rw-r--r--tests/SelfTest/Baselines/tap.sw.approved.txt6
-rw-r--r--tests/SelfTest/Baselines/xml.sw.approved.txt30
-rw-r--r--tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp10
8 files changed, 70 insertions, 16 deletions
diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt
index 0d6cd160..48007822 100644
--- a/tests/SelfTest/Baselines/compact.sw.approved.txt
+++ b/tests/SelfTest/Baselines/compact.sw.approved.txt
@@ -1152,6 +1152,8 @@ CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-confid
CmdLine.tests.cpp:<line number>: passed: config.benchmarkConfidenceInterval == Catch::Approx(0.99) for: 0.99 == Approx( 0.99 )
CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-no-analysis" }) for: {?}
CmdLine.tests.cpp:<line number>: passed: config.benchmarkNoAnalysis for: true
+CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?}
+CmdLine.tests.cpp:<line number>: passed: config.benchmarkWarmupTime == 10 for: 10 == 10
Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 3 >= 1
Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 2 >= 1
Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 1 >= 1
diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt
index 7056db0b..69ba4035 100644
--- a/tests/SelfTest/Baselines/console.std.approved.txt
+++ b/tests/SelfTest/Baselines/console.std.approved.txt
@@ -1381,5 +1381,5 @@ due to unexpected exception with message:
===============================================================================
test cases: 328 | 254 passed | 70 failed | 4 failed as expected
-assertions: 1827 | 1675 passed | 131 failed | 21 failed as expected
+assertions: 1829 | 1677 passed | 131 failed | 21 failed as expected
diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt
index 595acde6..d6285a81 100644
--- a/tests/SelfTest/Baselines/console.sw.approved.txt
+++ b/tests/SelfTest/Baselines/console.sw.approved.txt
@@ -8284,7 +8284,7 @@ with expansion:
-------------------------------------------------------------------------------
Process can be configured on command line
Benchmark options
- resamples
+ confidence-interval
-------------------------------------------------------------------------------
CmdLine.tests.cpp:<line number>
...............................................................................
@@ -8302,7 +8302,7 @@ with expansion:
-------------------------------------------------------------------------------
Process can be configured on command line
Benchmark options
- resamples
+ no-analysis
-------------------------------------------------------------------------------
CmdLine.tests.cpp:<line number>
...............................................................................
@@ -8318,6 +8318,24 @@ with expansion:
true
-------------------------------------------------------------------------------
+Process can be configured on command line
+ Benchmark options
+ warmup-time
+-------------------------------------------------------------------------------
+CmdLine.tests.cpp:<line number>
+...............................................................................
+
+CmdLine.tests.cpp:<line number>: PASSED:
+ CHECK( cli.parse({ "test", "--benchmark-warmup-time=10" }) )
+with expansion:
+ {?}
+
+CmdLine.tests.cpp:<line number>: PASSED:
+ REQUIRE( config.benchmarkWarmupTime == 10 )
+with expansion:
+ 10 == 10
+
+-------------------------------------------------------------------------------
Product with differing arities - std::tuple<int, double, float>
-------------------------------------------------------------------------------
Misc.tests.cpp:<line number>
@@ -14283,5 +14301,5 @@ Misc.tests.cpp:<line number>: PASSED:
===============================================================================
test cases: 328 | 238 passed | 86 failed | 4 failed as expected
-assertions: 1844 | 1675 passed | 148 failed | 21 failed as expected
+assertions: 1846 | 1677 passed | 148 failed | 21 failed as expected
diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt
index f7214ed7..0be5cc07 100644
--- a/tests/SelfTest/Baselines/junit.sw.approved.txt
+++ b/tests/SelfTest/Baselines/junit.sw.approved.txt
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuitesloose text artifact
>
- <testsuite name="<exe-name>" errors="17" failures="132" tests="1845" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
+ <testsuite name="<exe-name>" errors="17" failures="132" tests="1847" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}">
<properties>
<property name="filters" value="~[!nonportable]~[!benchmark]~[approvals] *"/>
<property name="random-seed" value="1"/>
@@ -1024,8 +1024,9 @@ Message.tests.cpp:<line number>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/use-colour/error" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/samples" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/>
- <testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/>
- <testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/>
+ <testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/confidence-interval" time="{duration}"/>
+ <testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/no-analysis" time="{duration}"/>
+ <testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/warmup-time" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple&lt;int, double, float>" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple&lt;int, double>" time="{duration}"/>
<testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple&lt;int>" time="{duration}"/>
diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
index 517c5f0b..07c512f0 100644
--- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
+++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt
@@ -64,8 +64,9 @@
<testCase name="Process can be configured on command line/use-colour/error" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/samples" duration="{duration}"/>
<testCase name="Process can be configured on command line/Benchmark options/resamples" duration="{duration}"/>
- <testCase name="Process can be configured on command line/Benchmark options/resamples" duration="{duration}"/>
- <testCase name="Process can be configured on command line/Benchmark options/resamples" duration="{duration}"/>
+ <testCase name="Process can be configured on command line/Benchmark options/confidence-interval" duration="{duration}"/>
+ <testCase name="Process can be configured on command line/Benchmark options/no-analysis" duration="{duration}"/>
+ <testCase name="Process can be configured on command line/Benchmark options/warmup-time" duration="{duration}"/>
<testCase name="Test with special, characters &quot;in name" duration="{duration}"/>
</file>
<file path="tests/<exe-name>/IntrospectiveTests/GeneratorsImpl.tests.cpp">
diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt
index f6c53961..b22d58c5 100644
--- a/tests/SelfTest/Baselines/tap.sw.approved.txt
+++ b/tests/SelfTest/Baselines/tap.sw.approved.txt
@@ -2226,6 +2226,10 @@ ok {test-number} - config.benchmarkConfidenceInterval == Catch::Approx(0.99) for
ok {test-number} - cli.parse({ "test", "--benchmark-no-analysis" }) for: {?}
# Process can be configured on command line
ok {test-number} - config.benchmarkNoAnalysis for: true
+# Process can be configured on command line
+ok {test-number} - cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?}
+# Process can be configured on command line
+ok {test-number} - config.benchmarkWarmupTime == 10 for: 10 == 10
# Product with differing arities - std::tuple<int, double, float>
ok {test-number} - std::tuple_size<TestType>::value >= 1 for: 3 >= 1
# Product with differing arities - std::tuple<int, double>
@@ -3680,5 +3684,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0
ok {test-number} -
# xmlentitycheck
ok {test-number} -
-1..1836
+1..1838
diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt
index 6abee4b7..d2d0ce54 100644
--- a/tests/SelfTest/Baselines/xml.sw.approved.txt
+++ b/tests/SelfTest/Baselines/xml.sw.approved.txt
@@ -10354,7 +10354,7 @@ Nor would this
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<Section name="Benchmark options" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
- <Section name="resamples" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+ <Section name="confidence-interval" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cli.parse({ "test", "--benchmark-confidence-interval=0.99" })
@@ -10376,7 +10376,7 @@ Nor would this
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
<Section name="Benchmark options" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
- <Section name="resamples" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+ <Section name="no-analysis" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
<Original>
cli.parse({ "test", "--benchmark-no-analysis" })
@@ -10397,6 +10397,28 @@ Nor would this
</Section>
<OverallResults successes="2" failures="0" expectedFailures="0"/>
</Section>
+ <Section name="Benchmark options" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+ <Section name="warmup-time" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+ <Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+ <Original>
+ cli.parse({ "test", "--benchmark-warmup-time=10" })
+ </Original>
+ <Expanded>
+ {?}
+ </Expanded>
+ </Expression>
+ <Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" >
+ <Original>
+ config.benchmarkWarmupTime == 10
+ </Original>
+ <Expanded>
+ 10 == 10
+ </Expanded>
+ </Expression>
+ <OverallResults successes="2" failures="0" expectedFailures="0"/>
+ </Section>
+ <OverallResults successes="2" failures="0" expectedFailures="0"/>
+ </Section>
<OverallResult success="true"/>
</TestCase>
<TestCase name="Product with differing arities - std::tuple&lt;int, double, float>" tags="[product][template]" filename="tests/<exe-name>/UsageTests/Misc.tests.cpp" >
@@ -17117,7 +17139,7 @@ loose text artifact
</Section>
<OverallResult success="true"/>
</TestCase>
- <OverallResults successes="1675" failures="149" expectedFailures="21"/>
+ <OverallResults successes="1677" failures="149" expectedFailures="21"/>
</Group>
- <OverallResults successes="1675" failures="148" expectedFailures="21"/>
+ <OverallResults successes="1677" failures="148" expectedFailures="21"/>
</Catch>
diff --git a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
index 68dafad1..b7d5c9c2 100644
--- a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
+++ b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
@@ -497,17 +497,23 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]"
REQUIRE(config.benchmarkResamples == 20000);
}
- SECTION("resamples") {
+ SECTION("confidence-interval") {
CHECK(cli.parse({ "test", "--benchmark-confidence-interval=0.99" }));
REQUIRE(config.benchmarkConfidenceInterval == Catch::Approx(0.99));
}
- SECTION("resamples") {
+ SECTION("no-analysis") {
CHECK(cli.parse({ "test", "--benchmark-no-analysis" }));
REQUIRE(config.benchmarkNoAnalysis);
}
+
+ SECTION("warmup-time") {
+ CHECK(cli.parse({ "test", "--benchmark-warmup-time=10" }));
+
+ REQUIRE(config.benchmarkWarmupTime == 10);
+ }
}
}