diff options
| author | khyperia | 2020-01-27 15:43:27 +0100 |
|---|---|---|
| committer | Martin Hořeňovský | 2020-02-16 15:44:23 +0100 |
| commit | 3c7e737a7bcf2d2059b1db2a5a221f76806ae8b0 (patch) | |
| tree | bcbf40fe8e4914c8ecc3a585066933bf871652c7 /tests/SelfTest | |
| parent | e880da93bd335e59a0d4b00bfcb9f06ab4b7c847 (diff) | |
| download | catch3-3c7e737a7bcf2d2059b1db2a5a221f76806ae8b0.tar.gz | |
Allow configuring of benchmark warmup time
Diffstat (limited to 'tests/SelfTest')
| -rw-r--r-- | tests/SelfTest/Baselines/compact.sw.approved.txt | 2 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/console.std.approved.txt | 2 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/console.sw.approved.txt | 24 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/junit.sw.approved.txt | 7 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/sonarqube.sw.approved.txt | 5 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/tap.sw.approved.txt | 6 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/xml.sw.approved.txt | 30 | ||||
| -rw-r--r-- | tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp | 10 |
8 files changed, 70 insertions, 16 deletions
diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 0d6cd160..48007822 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1152,6 +1152,8 @@ CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-confid CmdLine.tests.cpp:<line number>: passed: config.benchmarkConfidenceInterval == Catch::Approx(0.99) for: 0.99 == Approx( 0.99 ) CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-no-analysis" }) for: {?} CmdLine.tests.cpp:<line number>: passed: config.benchmarkNoAnalysis for: true +CmdLine.tests.cpp:<line number>: passed: cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?} +CmdLine.tests.cpp:<line number>: passed: config.benchmarkWarmupTime == 10 for: 10 == 10 Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 3 >= 1 Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 2 >= 1 Misc.tests.cpp:<line number>: passed: std::tuple_size<TestType>::value >= 1 for: 1 >= 1 diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index 7056db0b..69ba4035 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1381,5 +1381,5 @@ due to unexpected exception with message: =============================================================================== test cases: 328 | 254 passed | 70 failed | 4 failed as expected -assertions: 1827 | 1675 passed | 131 failed | 21 failed as expected +assertions: 1829 | 1677 passed | 131 failed | 21 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 595acde6..d6285a81 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -8284,7 +8284,7 @@ with expansion: ------------------------------------------------------------------------------- Process can be configured on command line Benchmark options - resamples + confidence-interval ------------------------------------------------------------------------------- CmdLine.tests.cpp:<line number> ............................................................................... @@ -8302,7 +8302,7 @@ with expansion: ------------------------------------------------------------------------------- Process can be configured on command line Benchmark options - resamples + no-analysis ------------------------------------------------------------------------------- CmdLine.tests.cpp:<line number> ............................................................................... @@ -8318,6 +8318,24 @@ with expansion: true ------------------------------------------------------------------------------- +Process can be configured on command line + Benchmark options + warmup-time +------------------------------------------------------------------------------- +CmdLine.tests.cpp:<line number> +............................................................................... + +CmdLine.tests.cpp:<line number>: PASSED: + CHECK( cli.parse({ "test", "--benchmark-warmup-time=10" }) ) +with expansion: + {?} + +CmdLine.tests.cpp:<line number>: PASSED: + REQUIRE( config.benchmarkWarmupTime == 10 ) +with expansion: + 10 == 10 + +------------------------------------------------------------------------------- Product with differing arities - std::tuple<int, double, float> ------------------------------------------------------------------------------- Misc.tests.cpp:<line number> @@ -14283,5 +14301,5 @@ Misc.tests.cpp:<line number>: PASSED: =============================================================================== test cases: 328 | 238 passed | 86 failed | 4 failed as expected -assertions: 1844 | 1675 passed | 148 failed | 21 failed as expected +assertions: 1846 | 1677 passed | 148 failed | 21 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index f7214ed7..0be5cc07 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <testsuitesloose text artifact > - <testsuite name="<exe-name>" errors="17" failures="132" tests="1845" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> + <testsuite name="<exe-name>" errors="17" failures="132" tests="1847" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> <properties> <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals] *"/> <property name="random-seed" value="1"/> @@ -1024,8 +1024,9 @@ Message.tests.cpp:<line number> <testcase classname="<exe-name>.global" name="Process can be configured on command line/use-colour/error" time="{duration}"/> <testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/samples" time="{duration}"/> <testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/> - <testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/> - <testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/resamples" time="{duration}"/> + <testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/confidence-interval" time="{duration}"/> + <testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/no-analysis" time="{duration}"/> + <testcase classname="<exe-name>.global" name="Process can be configured on command line/Benchmark options/warmup-time" time="{duration}"/> <testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple<int, double, float>" time="{duration}"/> <testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple<int, double>" time="{duration}"/> <testcase classname="<exe-name>.global" name="Product with differing arities - std::tuple<int>" time="{duration}"/> diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index 517c5f0b..07c512f0 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -64,8 +64,9 @@ <testCase name="Process can be configured on command line/use-colour/error" duration="{duration}"/> <testCase name="Process can be configured on command line/Benchmark options/samples" duration="{duration}"/> <testCase name="Process can be configured on command line/Benchmark options/resamples" duration="{duration}"/> - <testCase name="Process can be configured on command line/Benchmark options/resamples" duration="{duration}"/> - <testCase name="Process can be configured on command line/Benchmark options/resamples" duration="{duration}"/> + <testCase name="Process can be configured on command line/Benchmark options/confidence-interval" duration="{duration}"/> + <testCase name="Process can be configured on command line/Benchmark options/no-analysis" duration="{duration}"/> + <testCase name="Process can be configured on command line/Benchmark options/warmup-time" duration="{duration}"/> <testCase name="Test with special, characters "in name" duration="{duration}"/> </file> <file path="tests/<exe-name>/IntrospectiveTests/GeneratorsImpl.tests.cpp"> diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index f6c53961..b22d58c5 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -2226,6 +2226,10 @@ ok {test-number} - config.benchmarkConfidenceInterval == Catch::Approx(0.99) for ok {test-number} - cli.parse({ "test", "--benchmark-no-analysis" }) for: {?} # Process can be configured on command line ok {test-number} - config.benchmarkNoAnalysis for: true +# Process can be configured on command line +ok {test-number} - cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?} +# Process can be configured on command line +ok {test-number} - config.benchmarkWarmupTime == 10 for: 10 == 10 # Product with differing arities - std::tuple<int, double, float> ok {test-number} - std::tuple_size<TestType>::value >= 1 for: 3 >= 1 # Product with differing arities - std::tuple<int, double> @@ -3680,5 +3684,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..1836 +1..1838 diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 6abee4b7..d2d0ce54 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -10354,7 +10354,7 @@ Nor would this <OverallResults successes="2" failures="0" expectedFailures="0"/> </Section> <Section name="Benchmark options" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > - <Section name="resamples" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > + <Section name="confidence-interval" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > <Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > <Original> cli.parse({ "test", "--benchmark-confidence-interval=0.99" }) @@ -10376,7 +10376,7 @@ Nor would this <OverallResults successes="2" failures="0" expectedFailures="0"/> </Section> <Section name="Benchmark options" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > - <Section name="resamples" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > + <Section name="no-analysis" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > <Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > <Original> cli.parse({ "test", "--benchmark-no-analysis" }) @@ -10397,6 +10397,28 @@ Nor would this </Section> <OverallResults successes="2" failures="0" expectedFailures="0"/> </Section> + <Section name="Benchmark options" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > + <Section name="warmup-time" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > + <Expression success="true" type="CHECK" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > + <Original> + cli.parse({ "test", "--benchmark-warmup-time=10" }) + </Original> + <Expanded> + {?} + </Expanded> + </Expression> + <Expression success="true" type="REQUIRE" filename="tests/<exe-name>/IntrospectiveTests/CmdLine.tests.cpp" > + <Original> + config.benchmarkWarmupTime == 10 + </Original> + <Expanded> + 10 == 10 + </Expanded> + </Expression> + <OverallResults successes="2" failures="0" expectedFailures="0"/> + </Section> + <OverallResults successes="2" failures="0" expectedFailures="0"/> + </Section> <OverallResult success="true"/> </TestCase> <TestCase name="Product with differing arities - std::tuple<int, double, float>" tags="[product][template]" filename="tests/<exe-name>/UsageTests/Misc.tests.cpp" > @@ -17117,7 +17139,7 @@ loose text artifact </Section> <OverallResult success="true"/> </TestCase> - <OverallResults successes="1675" failures="149" expectedFailures="21"/> + <OverallResults successes="1677" failures="149" expectedFailures="21"/> </Group> - <OverallResults successes="1675" failures="148" expectedFailures="21"/> + <OverallResults successes="1677" failures="148" expectedFailures="21"/> </Catch> diff --git a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp index 68dafad1..b7d5c9c2 100644 --- a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp @@ -497,17 +497,23 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" REQUIRE(config.benchmarkResamples == 20000); } - SECTION("resamples") { + SECTION("confidence-interval") { CHECK(cli.parse({ "test", "--benchmark-confidence-interval=0.99" })); REQUIRE(config.benchmarkConfidenceInterval == Catch::Approx(0.99)); } - SECTION("resamples") { + SECTION("no-analysis") { CHECK(cli.parse({ "test", "--benchmark-no-analysis" })); REQUIRE(config.benchmarkNoAnalysis); } + + SECTION("warmup-time") { + CHECK(cli.parse({ "test", "--benchmark-warmup-time=10" })); + + REQUIRE(config.benchmarkWarmupTime == 10); + } } } |