Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

Add simple runtime benchmarks

For now we add just two binaries, one with assertions taking the
fast path, one with assertions taking the slow path, and the ability
to run 1 of `REQUIRE(true)`, `REQUIRE_NOTHROW`, `REQUIRE_THROWS`
in a loop.

I also split off a CMake preset which enables more tests than the
basic `simple-tests` preset, but does not enable the most expensive
tests which force recompilation of Catch2 multiple times.
Commit
b626e4c7aedb953e79784265e12283a62d5e94d3
Author
Martin Hořeňovský ·
Parents
434bf55d4727

Files · Download .tar.gz

Changes

 CMakeLists.txt                           |  6 ++++++
 CMakePresets.json                        | 13 +++++++++++--
 benchmarks/CMakeLists.txt                | 16 ++++++++++++++++
 benchmarks/assertion_listener.cpp        | 28 ++++++++++++++++++++++++++++
 benchmarks/runtime_assertion_benches.cpp | 27 +++++++++++++++++++++++++++
 5 files changed, 88 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 43fc78b8..37c48ca5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -20,6 +20,7 @@ cmake_dependent_option(CATCH_BUILD_TESTING "Build the SelfTest project" ON "CATC
 cmake_dependent_option(CATCH_BUILD_EXAMPLES "Build code examples" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
 cmake_dependent_option(CATCH_BUILD_EXTRA_TESTS "Build extra tests" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
 cmake_dependent_option(CATCH_BUILD_FUZZERS "Build fuzzers" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
+cmake_dependent_option(CATCH_BUILD_BENCHMARKS "Build the benchmarks" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
 cmake_dependent_option(CATCH_ENABLE_COVERAGE "Generate coverage for codecov.io" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
 cmake_dependent_option(CATCH_ENABLE_WERROR "Enables Werror during build" ON "CATCH_DEVELOPMENT_BUILD" OFF)
 cmake_dependent_option(CATCH_BUILD_SURROGATES "Enable generating and building surrogate TUs for the main headers" OFF "CATCH_DEVELOPMENT_BUILD" OFF)
@@ -77,6 +78,11 @@ set(SELF_TEST_DIR ${CATCH_DIR}/tests/SelfTest)
 # We need to bring-in the variables defined there to this scope
 add_subdirectory(src)
 
+if (CATCH_BUILD_BENCHMARKS)
+  set(CMAKE_FOLDER "benchmarks")
+  add_subdirectory(benchmarks)
+endif()
+
 # Build tests only if requested
 if(BUILD_TESTING AND CATCH_BUILD_TESTING AND NOT_SUBPROJECT)
   find_package(Python3 REQUIRED COMPONENTS Interpreter)
diff --git a/CMakePresets.json b/CMakePresets.json
index e9ce875b..ffc6e240 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -15,14 +15,23 @@
             }
         },
         {
-            "name": "all-tests",
+            "name": "most-tests",
             "inherits": "basic-tests",
             "displayName": "Full development build",
-            "description": "Enables development build with examples and ALL tests",
+            "description": "Enables development build with extended set of tests (still relatively cheap to build)",
             "cacheVariables": {
                 "CATCH_BUILD_EXAMPLES": "ON",
                 "CATCH_BUILD_EXTRA_TESTS": "ON",
                 "CATCH_BUILD_SURROGATES": "ON",
+                "CATCH_BUILD_BENCHMARKS": "ON"
+            }
+        },
+        {
+            "name": "all-tests",
+            "inherits": "most-tests",
+            "displayName": "Full development build",
+            "description": "Enables development build with examples and ALL tests",
+            "cacheVariables": {
                 "CATCH_ENABLE_CONFIGURE_TESTS": "ON",
                 "CATCH_ENABLE_CMAKE_HELPER_TESTS": "ON"
             }
diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt
new file mode 100644
index 00000000..68837783
--- /dev/null
+++ b/benchmarks/CMakeLists.txt
@@ -0,0 +1,16 @@
+include(CatchMiscFunctions)
+
+add_executable(AssertionsFastPath
+  runtime_assertion_benches.cpp
+)
+
+add_executable(AssertionsSlowPath
+  runtime_assertion_benches.cpp
+  assertion_listener.cpp
+)
+
+target_link_libraries(AssertionsFastPath PRIVATE Catch2::Catch2WithMain)
+target_link_libraries(AssertionsSlowPath PRIVATE Catch2::Catch2WithMain)
+
+list(APPEND CATCH_TEST_TARGETS AssertionsFastPath AssertionsSlowPath)
+set(CATCH_TEST_TARGETS ${CATCH_TEST_TARGETS} PARENT_SCOPE)
diff --git a/benchmarks/assertion_listener.cpp b/benchmarks/assertion_listener.cpp
new file mode 100644
index 00000000..2aa65b45
--- /dev/null
+++ b/benchmarks/assertion_listener.cpp
@@ -0,0 +1,28 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+#include <catch2/reporters/catch_reporter_event_listener.hpp>
+#include <catch2/reporters/catch_reporter_registrars.hpp>
+
+/**
+ * Event listener that listens to all assertions, forcing assertion slow path
+ */
+class AssertionSlowPathListener : public Catch::EventListenerBase {
+public:
+    static std::string getDescription() {
+        return "Validates ordering of Catch2's listener events";
+    }
+
+    AssertionSlowPathListener(Catch::IConfig const* config) :
+        EventListenerBase(config) {
+        m_preferences.shouldReportAllAssertions = true;
+        m_preferences.shouldReportAllAssertionStarts = true;
+    }
+};
+
+CATCH_REGISTER_LISTENER( AssertionSlowPathListener )
diff --git a/benchmarks/runtime_assertion_benches.cpp b/benchmarks/runtime_assertion_benches.cpp
new file mode 100644
index 00000000..6a28e3fa
--- /dev/null
+++ b/benchmarks/runtime_assertion_benches.cpp
@@ -0,0 +1,27 @@
+
+//              Copyright Catch2 Authors
+// Distributed under the Boost Software License, Version 1.0.
+//   (See accompanying file LICENSE.txt or copy at
+//        https://www.boost.org/LICENSE_1_0.txt)
+
+// SPDX-License-Identifier: BSL-1.0
+
+#include <catch2/catch_test_macros.hpp>
+
+TEST_CASE("Simple REQUIRE - 10M") {
+    for (size_t i = 0; i < 10'000'000; ++i) {
+        REQUIRE(true);
+    }
+}
+
+TEST_CASE("Simple NOTHROW - 10M") {
+    for (size_t i = 0; i < 10'000'000; ++i) {
+        REQUIRE_NOTHROW([](){}());
+    }
+}
+
+TEST_CASE("Simple THROWS - 10M") {
+    for (size_t i = 0; i < 10'000'000; ++i) {
+        REQUIRE_THROWS([]() { throw 1; }());
+    }
+}