diff options
| author | Martin Hořeňovský | 2020-02-03 18:08:15 +0100 |
|---|---|---|
| committer | Martin Hořeňovský | 2020-02-06 11:36:46 +0100 |
| commit | cd7d7a1c67f1b915d883fa7979cbd2162dde7d46 (patch) | |
| tree | 671c0f0cd2cae0273a91d2e7b8b2bac3ae1a9a8e /tests | |
| parent | 86e19b952dc4a4a2526037602abcc96a80607acc (diff) | |
| download | catch3-cd7d7a1c67f1b915d883fa7979cbd2162dde7d46.tar.gz | |
Remove CATCH_CONFIG_ENABLE_BENCHMARKING compilation toggle
Now that Catch2 is a proper library, we can always build the full
library (comparatively minor slowdown) and the user can avoid
including benchmarking headers to avoid the compilation slowdown.
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/ExtraTests/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/automake.sw.approved.txt | 15 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/compact.sw.approved.txt | 122 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/console.std.approved.txt | 4 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/console.sw.approved.txt | 748 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/junit.sw.approved.txt | 23 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/sonarqube.sw.approved.txt | 23 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/tap.sw.approved.txt | 250 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/teamcity.sw.approved.txt | 30 | ||||
| -rw-r--r-- | tests/SelfTest/Baselines/xml.sw.approved.txt | 1049 | ||||
| -rw-r--r-- | tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp | 23 | ||||
| -rw-r--r-- | tests/SelfTest/UsageTests/Benchmark.tests.cpp | 6 |
12 files changed, 2273 insertions, 21 deletions
diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt index 0001d940..b534fc8d 100644 --- a/tests/ExtraTests/CMakeLists.txt +++ b/tests/ExtraTests/CMakeLists.txt @@ -120,7 +120,6 @@ set_tests_properties( add_executable(BenchmarkingMacros ${TESTS_DIR}/X20-BenchmarkingMacros.cpp) -target_compile_definitions( BenchmarkingMacros PUBLIC CATCH_CONFIG_ENABLE_BENCHMARKING ) target_link_libraries( BenchmarkingMacros Catch2_buildall_interface ) add_test(NAME BenchmarkingMacros COMMAND BenchmarkingMacros -r console -s) diff --git a/tests/SelfTest/Baselines/automake.sw.approved.txt b/tests/SelfTest/Baselines/automake.sw.approved.txt index 96a0a795..035a92f2 100644 --- a/tests/SelfTest/Baselines/automake.sw.approved.txt +++ b/tests/SelfTest/Baselines/automake.sw.approved.txt @@ -229,15 +229,20 @@ Message from section two :test-result: PASS X/level/1/a :test-result: PASS X/level/1/b :test-result: PASS XmlEncode +:test-result: PASS analyse no analysis :test-result: PASS array<int, N> -> toString :test-result: PASS atomic if +:test-result: PASS benchmark function call :test-result: PASS boolean member :test-result: PASS checkedElse :test-result: FAIL checkedElse, failing :test-result: PASS checkedIf :test-result: FAIL checkedIf, failing +:test-result: PASS classify_outliers :test-result: PASS comparisons between const int variables :test-result: PASS comparisons between int variables +:test-result: PASS erfc_inv +:test-result: PASS estimate_clock_resolution :test-result: PASS even more nested SECTION tests :test-result: FAIL first tag loose text artifact @@ -249,11 +254,15 @@ loose text artifact :test-result: PASS long long :test-result: FAIL looped SECTION tests :test-result: FAIL looped tests +:test-result: PASS mean +:test-result: PASS measure :test-result: FAIL mix info, unscoped info and warning :test-result: FAIL more nested SECTION tests :test-result: PASS nested SECTION tests :test-result: PASS non streamable - with conv. op :test-result: PASS non-copyable objects +:test-result: PASS normal_cdf +:test-result: PASS normal_quantile :test-result: PASS not allowed :test-result: FAIL not prints unscoped info from previous failures :test-result: PASS null strings @@ -266,6 +275,9 @@ loose text artifact :test-result: FAIL prints unscoped info only for the first assertion :test-result: PASS random SECTION tests :test-result: PASS replaceInPlace +:test-result: PASS resolution +:test-result: PASS run_for_at_least, chronometer +:test-result: PASS run_for_at_least, int :test-result: FAIL second tag :test-result: FAIL send a single char to INFO :test-result: FAIL sends information to INFO @@ -304,10 +316,13 @@ loose text artifact :test-result: PASS tuple<0,int,const char *> :test-result: PASS tuple<string,string> :test-result: PASS tuple<tuple<int>,tuple<>,float> +:test-result: PASS uniform samples :test-result: PASS vec<vec<string,alloc>> -> toString :test-result: PASS vector<bool> -> toString :test-result: PASS vector<int,allocator> -> toString :test-result: PASS vector<int> -> toString :test-result: PASS vector<string> -> toString :test-result: PASS vectors can be sized and resized +:test-result: PASS warmup +:test-result: PASS weighted_average_quantile :test-result: PASS xmlentitycheck diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 7716196d..5f9e6a9f 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1562,10 +1562,33 @@ Xml.tests.cpp:<line number>: passed: encode( stringWithQuotes, Catch::XmlEncode: "don't "quote" me on that" Xml.tests.cpp:<line number>: passed: encode( "[\x01]" ) == "[\\x01]" for: "[\x01]" == "[\x01]" Xml.tests.cpp:<line number>: passed: encode( "[\x7F]" ) == "[\\x7F]" for: "[\x7F]" == "[\x7F]" +InternalBenchmark.tests.cpp:<line number>: passed: analysis.mean.point.count() == 23 for: 23.0 == 23 +InternalBenchmark.tests.cpp:<line number>: passed: analysis.mean.lower_bound.count() == 23 for: 23.0 == 23 +InternalBenchmark.tests.cpp:<line number>: passed: analysis.mean.upper_bound.count() == 23 for: 23.0 == 23 +InternalBenchmark.tests.cpp:<line number>: passed: analysis.standard_deviation.point.count() == 0 for: 0.0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: analysis.standard_deviation.lower_bound.count() == 0 for: 0.0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: analysis.standard_deviation.upper_bound.count() == 0 for: 0.0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: analysis.outliers.total() == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: analysis.outliers.low_mild == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: analysis.outliers.low_severe == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: analysis.outliers.high_mild == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: analysis.outliers.high_severe == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: analysis.outliers.samples_seen == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: analysis.outlier_variance == 0 for: 0.0 == 0 ToStringVector.tests.cpp:<line number>: passed: Catch::Detail::stringify( empty ) == "{ }" for: "{ }" == "{ }" ToStringVector.tests.cpp:<line number>: passed: Catch::Detail::stringify( oneValue ) == "{ 42 }" for: "{ 42 }" == "{ 42 }" ToStringVector.tests.cpp:<line number>: passed: Catch::Detail::stringify( twoValues ) == "{ 42, 250 }" for: "{ 42, 250 }" == "{ 42, 250 }" Misc.tests.cpp:<line number>: passed: x == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: model.started == 1 for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: model.finished == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: model.started == 1 for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: model.finished == 1 for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: called == 1 for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: model.started == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: model.finished == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: model.started == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: model.finished == 0 for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: called == 1 for: 1 == 1 Tricky.tests.cpp:<line number>: passed: obj.prop != 0 for: 0x<hex digits> != 0 Misc.tests.cpp:<line number>: passed: flag for: true Misc.tests.cpp:<line number>: passed: testCheckedElse( true ) for: true @@ -1575,6 +1598,42 @@ Misc.tests.cpp:<line number>: passed: flag for: true Misc.tests.cpp:<line number>: passed: testCheckedIf( true ) for: true Misc.tests.cpp:<line number>: failed: flag for: false Misc.tests.cpp:<line number>: failed: testCheckedIf( false ) for: false +InternalBenchmark.tests.cpp:<line number>: passed: o.samples_seen == static_cast<int>(x.size()) for: 6 == 6 +InternalBenchmark.tests.cpp:<line number>: passed: o.low_severe == los for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.low_mild == lom for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.high_mild == him for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.high_severe == his for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.total() == los + lom + him + his for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.samples_seen == static_cast<int>(x.size()) for: 6 == 6 +InternalBenchmark.tests.cpp:<line number>: passed: o.low_severe == los for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: o.low_mild == lom for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.high_mild == him for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.high_severe == his for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.total() == los + lom + him + his for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: o.samples_seen == static_cast<int>(x.size()) for: 6 == 6 +InternalBenchmark.tests.cpp:<line number>: passed: o.low_severe == los for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.low_mild == lom for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: o.high_mild == him for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.high_severe == his for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.total() == los + lom + him + his for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: o.samples_seen == static_cast<int>(x.size()) for: 6 == 6 +InternalBenchmark.tests.cpp:<line number>: passed: o.low_severe == los for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.low_mild == lom for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.high_mild == him for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: o.high_severe == his for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.total() == los + lom + him + his for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: o.samples_seen == static_cast<int>(x.size()) for: 6 == 6 +InternalBenchmark.tests.cpp:<line number>: passed: o.low_severe == los for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.low_mild == lom for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.high_mild == him for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.high_severe == his for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: o.total() == los + lom + him + his for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: o.samples_seen == static_cast<int>(x.size()) for: 6 == 6 +InternalBenchmark.tests.cpp:<line number>: passed: o.low_severe == los for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: o.low_mild == lom for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.high_mild == him for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: o.high_severe == his for: 0 == 0 +InternalBenchmark.tests.cpp:<line number>: passed: o.total() == los + lom + him + his for: 2 == 2 Condition.tests.cpp:<line number>: passed: unsigned_char_var == 1 for: 1 == 1 Condition.tests.cpp:<line number>: passed: unsigned_short_var == 1 for: 1 == 1 Condition.tests.cpp:<line number>: passed: unsigned_int_var == 1 for: 1 == 1 @@ -1583,6 +1642,11 @@ Condition.tests.cpp:<line number>: passed: long_var == unsigned_char_var for: 1 Condition.tests.cpp:<line number>: passed: long_var == unsigned_short_var for: 1 == 1 Condition.tests.cpp:<line number>: passed: long_var == unsigned_int_var for: 1 == 1 Condition.tests.cpp:<line number>: passed: long_var == unsigned_long_var for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: erfc_inv(1.103560) == Approx(-0.09203687623843015) for: -0.0920368762 == Approx( -0.0920368762 ) +InternalBenchmark.tests.cpp:<line number>: passed: erfc_inv(1.067400) == Approx(-0.05980291115763361) for: -0.0598029112 == Approx( -0.0598029112 ) +InternalBenchmark.tests.cpp:<line number>: passed: erfc_inv(0.050000) == Approx(1.38590382434967796) for: 1.3859038243 == Approx( 1.3859038243 ) +InternalBenchmark.tests.cpp:<line number>: passed: res.mean.count() == rate for: 2000.0 == 2000 (0x<hex digits>) +InternalBenchmark.tests.cpp:<line number>: passed: res.outliers.total() == 0 for: 0 == 0 Misc.tests.cpp:<line number>: passed: Misc.tests.cpp:<line number>: passed: Misc.tests.cpp:<line number>: passed: @@ -1610,6 +1674,15 @@ Misc.tests.cpp:<line number>: failed: ( fib[i] % 2 ) == 0 for: 1 == 0 with 1 mes Misc.tests.cpp:<line number>: passed: ( fib[i] % 2 ) == 0 for: 0 == 0 with 1 message: 'Testing if fib[5] (8) is even' Misc.tests.cpp:<line number>: failed: ( fib[i] % 2 ) == 0 for: 1 == 0 with 1 message: 'Testing if fib[6] (13) is even' Misc.tests.cpp:<line number>: failed: ( fib[i] % 2 ) == 0 for: 1 == 0 with 1 message: 'Testing if fib[7] (21) is even' +InternalBenchmark.tests.cpp:<line number>: passed: m == 19. for: 19.0 == 19.0 +InternalBenchmark.tests.cpp:<line number>: passed: x == 17 for: 17 == 17 +InternalBenchmark.tests.cpp:<line number>: passed: x == 23 for: 23 == 23 +InternalBenchmark.tests.cpp:<line number>: passed: r.elapsed.count() == 42 for: 42 == 42 +InternalBenchmark.tests.cpp:<line number>: passed: r.result == 23 for: 23 == 23 +InternalBenchmark.tests.cpp:<line number>: passed: r.iterations == 1 for: 1 == 1 +InternalBenchmark.tests.cpp:<line number>: passed: s.elapsed.count() == 69 for: 69 == 69 +InternalBenchmark.tests.cpp:<line number>: passed: s.result == 17 for: 17 == 17 +InternalBenchmark.tests.cpp:<line number>: passed: s.iterations == 1 for: 1 == 1 Message.tests.cpp:<line number>: warning: 'info' with 2 messages: 'unscoped info' and 'and warn may mix' Message.tests.cpp:<line number>: warning: 'info' with 2 messages: 'unscoped info' and 'they are not cleared after warnings' Misc.tests.cpp:<line number>: failed: a == b for: 1 == 2 @@ -1620,6 +1693,14 @@ Misc.tests.cpp:<line number>: passed: b != a for: 2 != 1 Misc.tests.cpp:<line number>: passed: a != b for: 1 != 2 Tricky.tests.cpp:<line number>: passed: s == "7" for: "7" == "7" Tricky.tests.cpp:<line number>: passed: ti == typeid(int) for: {?} == {?} +InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(0.000000) == Approx(0.50000000000000000) for: 0.5 == Approx( 0.5 ) +InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(1.000000) == Approx(0.84134474606854293) for: 0.8413447461 == Approx( 0.8413447461 ) +InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(-1.000000) == Approx(0.15865525393145705) for: 0.1586552539 == Approx( 0.1586552539 ) +InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(2.809729) == Approx(0.99752083845315409) for: 0.9975208385 == Approx( 0.9975208385 ) +InternalBenchmark.tests.cpp:<line number>: passed: normal_cdf(-1.352570) == Approx(0.08809652095066035) for: 0.088096521 == Approx( 0.088096521 ) +InternalBenchmark.tests.cpp:<line number>: passed: normal_quantile(0.551780) == Approx(0.13015979861484198) for: 0.1301597986 == Approx( 0.1301597986 ) +InternalBenchmark.tests.cpp:<line number>: passed: normal_quantile(0.533700) == Approx(0.08457408802851875) for: 0.084574088 == Approx( 0.084574088 ) +InternalBenchmark.tests.cpp:<line number>: passed: normal_quantile(0.025000) == Approx(-1.95996398454005449) for: -1.9599639845 == Approx( -1.9599639845 ) Misc.tests.cpp:<line number>: passed: Message.tests.cpp:<line number>: passed: true with 1 message: 'this MAY be seen only for the FIRST assertion IF info is printed for passing assertions' Message.tests.cpp:<line number>: passed: true with 1 message: 'this MAY be seen only for the SECOND assertion IF info is printed for passing assertions' @@ -1661,6 +1742,38 @@ StringManip.tests.cpp:<line number>: passed: !(Catch::replaceInPlace(letters, "x StringManip.tests.cpp:<line number>: passed: letters == letters for: "abcdefcg" == "abcdefcg" StringManip.tests.cpp:<line number>: passed: Catch::replaceInPlace(s, "'", "|'") for: true StringManip.tests.cpp:<line number>: passed: s == "didn|'t" for: "didn|'t" == "didn|'t" +InternalBenchmark.tests.cpp:<line number>: passed: res.size() == count for: 10 == 10 +InternalBenchmark.tests.cpp:<line number>: passed: res[i] == rate for: 1000.0 == 1000 (0x<hex digits>) +InternalBenchmark.tests.cpp:<line number>: passed: res[i] == rate for: 1000.0 == 1000 (0x<hex digits>) +InternalBenchmark.tests.cpp:<line number>: passed: res[i] == rate for: 1000.0 == 1000 (0x<hex digits>) +InternalBenchmark.tests.cpp:<line number>: passed: res[i] == rate for: 1000.0 == 1000 (0x<hex digits>) +InternalBenchmark.tests.cpp:<line number>: passed: res[i] == rate for: 1000.0 == 1000 (0x<hex digits>) +InternalBenchmark.tests.cpp:<line number>: passed: res[i] == rate for: 1000.0 == 1000 (0x<hex digits>) +InternalBenchmark.tests.cpp:<line number>: passed: res[i] == rate for: 1000.0 == 1000 (0x<hex digits>) +InternalBenchmark.tests.cpp:<line number>: passed: res[i] == rate for: 1000.0 == 1000 (0x<hex digits>) +InternalBenchmark.tests.cpp:<line number>: passed: res[i] == rate for: 1000.0 == 1000 (0x<hex digits>) +InternalBenchmark.tests.cpp:<line number>: passed: meter.runs() >= old_runs for: 1 >= 1 +InternalBenchmark.tests.cpp:<line number>: passed: meter.runs() >= old_runs for: 2 >= 1 +InternalBenchmark.tests.cpp:<line number>: passed: meter.runs() >= old_runs for: 4 >= 2 +InternalBenchmark.tests.cpp:<line number>: passed: meter.runs() >= old_runs for: 8 >= 4 +InternalBenchmark.tests.cpp:<line number>: passed: meter.runs() >= old_runs for: 16 >= 8 +InternalBenchmark.tests.cpp:<line number>: passed: meter.runs() >= old_runs for: 32 >= 16 +InternalBenchmark.tests.cpp:<line number>: passed: meter.runs() >= old_runs for: 64 >= 32 +InternalBenchmark.tests.cpp:<line number>: passed: meter.runs() >= old_runs for: 128 >= 64 +InternalBenchmark.tests.cpp:<line number>: passed: Timing.elapsed >= time for: 128 ns >= 100 ns +InternalBenchmark.tests.cpp:<line number>: passed: Timing.result == Timing.iterations + 17 for: 145 == 145 +InternalBenchmark.tests.cpp:<line number>: passed: Timing.iterations >= time.count() for: 128 >= 100 +InternalBenchmark.tests.cpp:<line number>: passed: x >= old_x for: 1 >= 1 +InternalBenchmark.tests.cpp:<line number>: passed: x >= old_x for: 2 >= 1 +InternalBenchmark.tests.cpp:<line number>: passed: x >= old_x for: 4 >= 2 +InternalBenchmark.tests.cpp:<line number>: passed: x >= old_x for: 8 >= 4 +InternalBenchmark.tests.cpp:<line number>: passed: x >= old_x for: 16 >= 8 +InternalBenchmark.tests.cpp:<line number>: passed: x >= old_x for: 32 >= 16 +InternalBenchmark.tests.cpp:<line number>: passed: x >= old_x for: 64 >= 32 +InternalBenchmark.tests.cpp:<line number>: passed: x >= old_x for: 128 >= 64 +InternalBenchmark.tests.cpp:<line number>: passed: Timing.elapsed >= time for: 128 ns >= 100 ns +InternalBenchmark.tests.cpp:<line number>: passed: Timing.result == Timing.iterations + 17 for: 145 == 145 +InternalBenchmark.tests.cpp:<line number>: passed: Timing.iterations >= time.count() for: 128 >= 100 Misc.tests.cpp:<line number>: failed: false with 1 message: '3' Message.tests.cpp:<line number>: failed: false with 2 messages: 'hi' and 'i := 7' Tag.tests.cpp:<line number>: passed: tags, Catch::VectorContains("magic-tag"_catch_sr) && Catch::VectorContains("."_catch_sr) for: { ., magic-tag } ( Contains: magic-tag and Contains: . ) @@ -1753,6 +1866,10 @@ ToStringTuple.tests.cpp:<line number>: passed: "{ \"hello\", \"world\" }" == ::C ToStringTuple.tests.cpp:<line number>: passed: "{ { 42 }, { }, 1.2f }" == ::Catch::Detail::stringify(value) for: "{ { 42 }, { }, 1.2f }" == "{ { 42 }, { }, 1.2f }" +InternalBenchmark.tests.cpp:<line number>: passed: e.point == 23 for: 23.0 == 23 +InternalBenchmark.tests.cpp:<line number>: passed: e.upper_bound == 23 for: 23.0 == 23 +InternalBenchmark.tests.cpp:<line number>: passed: e.lower_bound == 23 for: 23.0 == 23 +InternalBenchmark.tests.cpp:<line number>: passed: e.confidence_interval == 0.95 for: 0.95 == 0.95 ToStringVector.tests.cpp:<line number>: passed: ::Catch::Detail::stringify(v) == "{ }" for: "{ }" == "{ }" ToStringVector.tests.cpp:<line number>: passed: ::Catch::Detail::stringify(v) == "{ { \"hello\" }, { \"world\" } }" for: "{ { "hello" }, { "world" } }" == @@ -1788,6 +1905,11 @@ Misc.tests.cpp:<line number>: passed: v.size() == 5 for: 5 == 5 Misc.tests.cpp:<line number>: passed: v.capacity() >= 5 for: 5 >= 5 Misc.tests.cpp:<line number>: passed: v.size() == 5 for: 5 == 5 Misc.tests.cpp:<line number>: passed: v.capacity() >= 5 for: 5 >= 5 +InternalBenchmark.tests.cpp:<line number>: passed: (iterations * rate) > Catch::Benchmark::Detail::warmup_time.count() for: 160000000 (0x<hex digits>) > 100 +InternalBenchmark.tests.cpp:<line number>: passed: (end - start) > Catch::Benchmark::Detail::warmup_time for: 310016000 ns > 100 ms +InternalBenchmark.tests.cpp:<line number>: passed: q1 == 14.5 for: 14.5 == 14.5 +InternalBenchmark.tests.cpp:<line number>: passed: med == 18. for: 18.0 == 18.0 +InternalBenchmark.tests.cpp:<line number>: passed: q3 == 23. for: 23.0 == 23.0 Misc.tests.cpp:<line number>: passed: Misc.tests.cpp:<line number>: passed: Failed 86 test cases, failed 148 assertions. diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index 3ae07e23..140a0b8a 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1380,6 +1380,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 305 | 231 passed | 70 failed | 4 failed as expected -assertions: 1654 | 1502 passed | 131 failed | 21 failed as expected +test cases: 320 | 246 passed | 70 failed | 4 failed as expected +assertions: 1776 | 1624 passed | 131 failed | 21 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 3dc7166c..ea8481d2 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -11431,6 +11431,77 @@ with expansion: "[\x7F]" == "[\x7F]" ------------------------------------------------------------------------------- +analyse no analysis +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp:<line number> +............................................................................... + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.mean.point.count() == 23 ) +with expansion: + 23.0 == 23 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.mean.lower_bound.count() == 23 ) +with expansion: + 23.0 == 23 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.mean.upper_bound.count() == 23 ) +with expansion: + 23.0 == 23 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.standard_deviation.point.count() == 0 ) +with expansion: + 0.0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.standard_deviation.lower_bound.count() == 0 ) +with expansion: + 0.0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.standard_deviation.upper_bound.count() == 0 ) +with expansion: + 0.0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.outliers.total() == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.outliers.low_mild == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.outliers.low_severe == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.outliers.high_mild == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.outliers.high_severe == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.outliers.samples_seen == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( analysis.outlier_variance == 0 ) +with expansion: + 0.0 == 0 + +------------------------------------------------------------------------------- array<int, N> -> toString ------------------------------------------------------------------------------- ToStringVector.tests.cpp:<line number> @@ -11463,6 +11534,70 @@ with expansion: 0 == 0 ------------------------------------------------------------------------------- +benchmark function call + without chronometer +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp:<line number> +............................................................................... + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( model.started == 1 ) +with expansion: + 1 == 1 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( model.finished == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( model.started == 1 ) +with expansion: + 1 == 1 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( model.finished == 1 ) +with expansion: + 1 == 1 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( called == 1 ) +with expansion: + 1 == 1 + +------------------------------------------------------------------------------- +benchmark function call + with chronometer +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp:<line number> +............................................................................... + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( model.started == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( model.finished == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( model.started == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( model.finished == 0 ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + CHECK( called == 1 ) +with expansion: + 1 == 1 + +------------------------------------------------------------------------------- boolean member ------------------------------------------------------------------------------- Tricky.tests.cpp:<line number> @@ -11538,6 +11673,228 @@ with expansion: false ------------------------------------------------------------------------------- +classify_outliers + none +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp:<line number> +............................................................................... + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.samples_seen == static_cast<int>(x.size()) ) +with expansion: + 6 == 6 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.low_severe == los ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.low_mild == lom ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.high_mild == him ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.high_severe == his ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.total() == los + lom + him + his ) +with expansion: + 0 == 0 + +------------------------------------------------------------------------------- +classify_outliers + low severe +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp:<line number> +............................................................................... + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.samples_seen == static_cast<int>(x.size()) ) +with expansion: + 6 == 6 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.low_severe == los ) +with expansion: + 1 == 1 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.low_mild == lom ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.high_mild == him ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.high_severe == his ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.total() == los + lom + him + his ) +with expansion: + 1 == 1 + +------------------------------------------------------------------------------- +classify_outliers + low mild +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp:<line number> +............................................................................... + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.samples_seen == static_cast<int>(x.size()) ) +with expansion: + 6 == 6 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.low_severe == los ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.low_mild == lom ) +with expansion: + 1 == 1 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.high_mild == him ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.high_severe == his ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.total() == los + lom + him + his ) +with expansion: + 1 == 1 + +------------------------------------------------------------------------------- +classify_outliers + high mild +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp:<line number> +............................................................................... + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.samples_seen == static_cast<int>(x.size()) ) +with expansion: + 6 == 6 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.low_severe == los ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.low_mild == lom ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.high_mild == him ) +with expansion: + 1 == 1 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.high_severe == his ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.total() == los + lom + him + his ) +with expansion: + 1 == 1 + +------------------------------------------------------------------------------- +classify_outliers + high severe +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp:<line number> +............................................................................... + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.samples_seen == static_cast<int>(x.size()) ) +with expansion: + 6 == 6 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.low_severe == los ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.low_mild == lom ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.high_mild == him ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.high_severe == his ) +with expansion: + 1 == 1 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.total() == los + lom + him + his ) +with expansion: + 1 == 1 + +------------------------------------------------------------------------------- +classify_outliers + mixed +------------------------------------------------------------------------------- +InternalBenchmark.tests.cpp:<line number> +............................................................................... + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.samples_seen == static_cast<int>(x.size()) ) +with expansion: + 6 == 6 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.low_severe == los ) +with expansion: + 1 == 1 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.low_mild == lom ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.high_mild == him ) +with expansion: + 1 == 1 + +InternalBenchmark.tests.cpp:<line number>: PASSED: + REQUIRE( o.high_severe == his ) +with expansion: + 0 == 0 + +InternalBenchmark.tests.cpp:<line number>: PASSED: |