diff options
| author | Martin Hořeňovský | 2021-06-09 20:48:58 +0200 |
|---|---|---|
| committer | Martin Hořeňovský | 2021-08-07 15:16:11 +0200 |
| commit | 0a3f511cfed071c9c4ad512341ae7569383922b7 (patch) | |
| tree | 9bb28fd5a87cd1fffc96832fafeb244c6bef29a7 /tests | |
| parent | 9ef510b7694c7199a78d13a8cba8ddbdeac7cb16 (diff) | |
| download | catch3-0a3f511cfed071c9c4ad512341ae7569383922b7.tar.gz | |
Fix inconsistencies in reporting benchmarking failures
With these changes, all these benchmarks
```cpp
BENCHMARK("Empty benchmark") {};
BENCHMARK("Throwing benchmark") {
throw "just a plain literal, bleh";
};
BENCHMARK("Asserting benchmark") {
REQUIRE(1 == 2);
};
BENCHMARK("FAIL'd benchmark") {
FAIL("This benchmark only fails, nothing else");
};
```
report the respective failure and mark the outer `TEST_CASE` as
failed. Previously, the first two would not fail the `TEST_CASE`,
and the latter two would break xml reporter's formatting, because
`benchmarkFailed`, `benchmarkEnded` etc would not be be called
properly in failure cases.
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/CMakeLists.txt | 44 | ||||
| -rw-r--r-- | tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp | 21 |
2 files changed, 65 insertions, 0 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a1e34226..750547c0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -315,6 +315,50 @@ add_test(NAME CheckConvenienceHeaders ${PYTHON_EXECUTABLE} ${CATCH_DIR}/tools/scripts/checkConvenienceHeaders.py ) + +add_test(NAME "Benchmarking::FailureReporting::OptimizedOut" + COMMAND + $<TARGET_FILE:SelfTest> "Failing benchmarks" -c "empty" -r xml + # This test only makes sense with the optimizer being enabled when + # the tests are being compiled. + CONFIGURATIONS Release +) +set_tests_properties("Benchmarking::FailureReporting::OptimizedOut" + PROPERTIES + PASS_REGULAR_EXPRESSION "could not measure benchmark\, maybe it was optimized away" + FAIL_REGULAR_EXPRESSION "successes=\"1\"" +) + +add_test(NAME "Benchmarking::FailureReporting::ThrowingBenchmark" + COMMAND + $<TARGET_FILE:SelfTest> "Failing benchmarks" -c "throw" -r xml +) +set_tests_properties("Benchmarking::FailureReporting::ThrowingBenchmark" + PROPERTIES + PASS_REGULAR_EXPRESSION "<failed message=\"just a plain literal" + FAIL_REGULAR_EXPRESSION "successes=\"1\"" +) + +add_test(NAME "Benchmarking::FailureReporting::FailedAssertion" + COMMAND + $<TARGET_FILE:SelfTest> "Failing benchmarks" -c "assert" -r xml +) +set_tests_properties("Benchmarking::FailureReporting::FailedAssertion" + PROPERTIES + PASS_REGULAR_EXPRESSION "<Expression success=\"false\"" + FAIL_REGULAR_EXPRESSION "successes=\"1\"" +) + +add_test(NAME "Benchmarking::FailureReporting::FailMacro" + COMMAND + $<TARGET_FILE:SelfTest> "Failing benchmarks" -c "fail" -r xml +) +set_tests_properties("Benchmarking::FailureReporting::FailMacro" + PROPERTIES + PASS_REGULAR_EXPRESSION "This benchmark only fails\, nothing else" + FAIL_REGULAR_EXPRESSION "successes=\"1\"" +) + if (CATCH_USE_VALGRIND) add_test(NAME ValgrindRunTests COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest>) add_test(NAME ValgrindListTests COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest> --list-tests --verbosity high) diff --git a/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp b/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp index 0a3895a8..b2250415 100644 --- a/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp @@ -412,3 +412,24 @@ TEST_CASE("run benchmark", "[benchmark][approvals]") { CHECK((end - start).count() == 2867251000); } + +TEST_CASE("Failing benchmarks", "[!benchmark][.approvals]") { + SECTION("empty", "Benchmark that has been optimized away (because it is empty)") { + BENCHMARK("Empty benchmark") {}; + } + SECTION("throw", "Benchmark that throws an exception") { + BENCHMARK("Throwing benchmark") { + throw "just a plain literal, bleh"; + }; + } + SECTION("assert", "Benchmark that asserts inside") { + BENCHMARK("Asserting benchmark") { + REQUIRE(1 == 2); + }; + } + SECTION("fail", "Benchmark that fails inside") { + BENCHMARK("FAIL'd benchmark") { + FAIL("This benchmark only fails, nothing else"); + }; + } +} |