catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
Replace Catch2's runner with the Catch3 runner
- Commit
93d4e7dbebab867e0a7f59fc25308460a7a76280- Author
- Leitwolf11 ·
- Parents
83c6fc3ac13a
Changes
.conan/build.py | 94 - .conan/test_package/CMakeLists.txt | 7 - .conan/test_package/conanfile.py | 40 - .conan/test_package/test_package.cpp | 13 - .github/workflows/package-manager-builds.yaml | 46 - .gitignore | 1 - BUILD.bazel | 27 +- CMake/CatchConfigOptions.cmake | 4 +- README.md | 25 +- conanfile.py | 129 - docs/Readme.md | 10 +- docs/ci-and-misc.md | 50 +- docs/cmake-integration.md | 21 +- docs/command-line.md | 785 +- docs/configuration.md | 41 +- docs/event-listeners.md | 21 +- docs/filtering-execution-path.md | 10 +- docs/own-main.md | 126 +- docs/progmasoft-catch3.md | 44 +- docs/progmasoft-roadmap.md | 44 +- docs/progmasoft-runner.md | 324 + docs/reporter-events.md | 48 +- docs/reporters.md | 243 +- docs/skipping-passing-failing.md | 2 +- docs/tutorial.md | 2 +- docs/usage-tips.md | 84 +- docs/why-catch3.md | 18 +- examples/210-Evt-EventListeners.cpp | 8 - examples/232-Cfg-CustomMain.cpp | 40 +- extras/Catch.cmake | 20 +- extras/CatchAddTests.cmake | 69 +- extras/CatchShardTests.cmake | 2 +- extras/CatchShardTestsImpl.cmake | 17 +- extras/catch_amalgamated.cpp | 22058 ++++++++++------- extras/catch_amalgamated.hpp | 3095 +-- fuzzing/CMakeLists.txt | 6 +- fuzzing/fuzz_RunnerJson.cpp | 29 + fuzzing/fuzz_RunnerOptions.cpp | 50 + fuzzing/fuzz_XmlWriter.cpp | 22 - fuzzing/fuzz_textflow.cpp | 54 - premake5.lua | 15 +- requirements-conan.txt | 4 - src/CMakeLists.txt | 115 +- src/Progmasoft/BUILD.bazel | 32 +- src/Progmasoft/Catch3/Results.hpp | 288 +- src/Progmasoft/Catch3/Runner.hpp | 33 + src/Progmasoft/Catch3/Version.hpp | 9 + src/Progmasoft/Catch3/XmlWriter.hpp | 27 +- src/Progmasoft/PropertyEngine.cpp | 71 +- src/Progmasoft/Results.cpp | 359 +- src/Progmasoft/Runner/.clang-format | 112 + src/Progmasoft/Runner/Analysis.cpp | 365 + src/Progmasoft/Runner/Analysis.hpp | 41 + src/Progmasoft/Runner/Catalog.cpp | 272 + src/Progmasoft/Runner/Catalog.hpp | 104 + src/Progmasoft/Runner/Console.cpp | 519 + src/Progmasoft/Runner/Console.hpp | 63 + src/Progmasoft/Runner/Execution.cpp | 588 + src/Progmasoft/Runner/Execution.hpp | 64 + src/Progmasoft/Runner/Expected.hpp | 139 + src/Progmasoft/Runner/Files.cpp | 205 + src/Progmasoft/Runner/Files.hpp | 58 + src/Progmasoft/Runner/Git.cpp | 172 + src/Progmasoft/Runner/Git.hpp | 39 + src/Progmasoft/Runner/Hash.cpp | 218 + src/Progmasoft/Runner/Hash.hpp | 61 + src/Progmasoft/Runner/History.cpp | 486 + src/Progmasoft/Runner/History.hpp | 167 + src/Progmasoft/Runner/Json.cpp | 918 + src/Progmasoft/Runner/Json.hpp | 183 + src/Progmasoft/Runner/Options.cpp | 1143 + src/Progmasoft/Runner/Options.hpp | 181 + src/Progmasoft/Runner/Planner.cpp | 620 + src/Progmasoft/Runner/Planner.hpp | 82 + src/Progmasoft/Runner/Process.cpp | 1289 + src/Progmasoft/Runner/Process.hpp | 262 + src/Progmasoft/Runner/Protocol.cpp | 64 + src/Progmasoft/Runner/Protocol.hpp | 76 + src/Progmasoft/Runner/ResultsJson.cpp | 399 + src/Progmasoft/Runner/ResultsJson.hpp | 51 + src/Progmasoft/Runner/Runner.cpp | 846 + src/Progmasoft/Runner/Scheduler.cpp | 1426 ++ src/Progmasoft/Runner/Scheduler.hpp | 53 + src/Progmasoft/Runner/SchedulerModel.hpp | 163 + src/Progmasoft/Runner/TemporaryDirectory.cpp | 111 + src/Progmasoft/Runner/TemporaryDirectory.hpp | 29 + src/Progmasoft/XmlWriter.cpp | 497 +- src/Progmasoft/meson.build | 8 +- src/catch2/catch_all.hpp | 12 - src/catch2/catch_config.cpp | 200 - src/catch2/catch_config.hpp | 57 +- src/catch2/catch_registry_hub.cpp | 3 - src/catch2/catch_session.cpp | 488 +- src/catch2/catch_session.hpp | 37 +- src/catch2/catch_test_case_info.cpp | 21 + src/catch2/catch_user_config.hpp.in | 2 - .../interfaces/catch_interfaces_registry_hub.hpp | 3 - .../interfaces/catch_interfaces_reporter.cpp | 28 - .../interfaces/catch_interfaces_reporter.hpp | 46 - .../catch_interfaces_reporter_factory.cpp | 1 - .../catch_interfaces_reporter_factory.hpp | 12 - src/catch2/internal/catch_clara.cpp | 464 - src/catch2/internal/catch_clara.hpp | 749 - src/catch2/internal/catch_commandline.cpp | 379 - src/catch2/internal/catch_commandline.hpp | 21 - src/catch2/internal/catch_console_colour.cpp | 288 - src/catch2/internal/catch_console_colour.hpp | 141 - src/catch2/internal/catch_console_width.hpp | 19 - src/catch2/internal/catch_debug_console.cpp | 45 - src/catch2/internal/catch_debug_console.hpp | 17 - src/catch2/internal/catch_istream.cpp | 152 - src/catch2/internal/catch_istream.hpp | 52 - src/catch2/internal/catch_jsonwriter.cpp | 214 - src/catch2/internal/catch_jsonwriter.hpp | 129 - src/catch2/internal/catch_list.cpp | 120 - src/catch2/internal/catch_list.hpp | 43 - src/catch2/internal/catch_reporter_registry.cpp | 61 +- src/catch2/internal/catch_reporter_registry.hpp | 14 - src/catch2/internal/catch_reporter_spec_parser.cpp | 196 - src/catch2/internal/catch_reporter_spec_parser.hpp | 90 - .../internal/catch_reusable_string_stream.hpp | 1 + src/catch2/internal/catch_run_context.cpp | 8 + src/catch2/internal/catch_run_context.hpp | 4 + src/catch2/internal/catch_sharding.hpp | 44 - .../internal/catch_test_case_registry_impl.cpp | 4 +- src/catch2/internal/catch_textflow.cpp | 430 - src/catch2/internal/catch_textflow.hpp | 309 - src/catch2/internal/catch_xmlwriter.cpp | 344 - src/catch2/internal/catch_xmlwriter.hpp | 164 - src/catch2/meson.build | 79 +- src/catch2/reporters/catch_reporter_automake.cpp | 46 - src/catch2/reporters/catch_reporter_automake.hpp | 36 - .../reporters/catch_reporter_common_base.cpp | 50 - .../reporters/catch_reporter_common_base.hpp | 81 - src/catch2/reporters/catch_reporter_compact.cpp | 262 - src/catch2/reporters/catch_reporter_compact.hpp | 38 - src/catch2/reporters/catch_reporter_console.cpp | 671 - src/catch2/reporters/catch_reporter_console.hpp | 65 - .../reporters/catch_reporter_cumulative_base.cpp | 157 - .../reporters/catch_reporter_cumulative_base.hpp | 152 - .../reporters/catch_reporter_event_listener.cpp | 7 - .../reporters/catch_reporter_event_listener.hpp | 7 - src/catch2/reporters/catch_reporter_helpers.cpp | 393 - src/catch2/reporters/catch_reporter_helpers.hpp | 106 - src/catch2/reporters/catch_reporter_json.cpp | 381 - src/catch2/reporters/catch_reporter_json.hpp | 91 - src/catch2/reporters/catch_reporter_junit.cpp | 319 - src/catch2/reporters/catch_reporter_junit.hpp | 56 - src/catch2/reporters/catch_reporter_multi.cpp | 31 - src/catch2/reporters/catch_reporter_multi.hpp | 5 - src/catch2/reporters/catch_reporter_registrars.cpp | 13 - src/catch2/reporters/catch_reporter_registrars.hpp | 37 - src/catch2/reporters/catch_reporter_sonarqube.cpp | 172 - src/catch2/reporters/catch_reporter_sonarqube.hpp | 53 - .../reporters/catch_reporter_streaming_base.cpp | 23 - .../reporters/catch_reporter_streaming_base.hpp | 73 - src/catch2/reporters/catch_reporter_tap.cpp | 237 - src/catch2/reporters/catch_reporter_tap.hpp | 39 - src/catch2/reporters/catch_reporter_teamcity.cpp | 185 - src/catch2/reporters/catch_reporter_teamcity.hpp | 61 - src/catch2/reporters/catch_reporter_xml.cpp | 334 - src/catch2/reporters/catch_reporter_xml.hpp | 64 - src/catch2/reporters/catch_reporters_all.hpp | 26 +- tests/BUILD.bazel | 10 - tests/CMakeLists.txt | 471 +- tests/ExtraTests/CMakeLists.txt | 415 +- tests/ExtraTests/X21-PartialTestCaseEvents.cpp | 40 +- .../X22-BenchmarksInCumulativeReporter.cpp | 79 - tests/ExtraTests/X23-CasingInReporterNames.cpp | 41 - ...rPreferencesForPassingAssertionsIsRespected.cpp | 52 - .../X27-CapturedStdoutInTestCaseEvents.cpp | 38 +- .../X28-ListenersGetEventsBeforeReporters.cpp | 70 +- .../ExtraTests/X29-CustomArgumentsForReporters.cpp | 60 - tests/ExtraTests/X35-DuplicatedReporterNames.cpp | 31 - tests/ExtraTests/X38-ThrowingReporter.cpp | 79 - tests/Progmasoft/BUILD.bazel | 21 +- tests/Progmasoft/Results.tests.cpp | 360 +- tests/Progmasoft/Runner.tests.cpp | 454 + tests/Progmasoft/RunnerFixture.cpp | 190 + tests/Progmasoft/RunnerUnits.tests.cpp | 424 + tests/SelfTest/Baselines/automake.std.approved.txt | 168 - tests/SelfTest/Baselines/automake.sw.approved.txt | 466 - .../Baselines/automake.sw.multi.approved.txt | 455 - tests/SelfTest/Baselines/compact.sw.approved.txt | 3036 --- .../Baselines/compact.sw.multi.approved.txt | 3025 --- tests/SelfTest/Baselines/console.std.approved.txt | 1748 -- tests/SelfTest/Baselines/console.sw.approved.txt | 20332 ---------------- .../Baselines/console.sw.multi.approved.txt | 20321 ---------------- tests/SelfTest/Baselines/console.swa4.approved.txt | 973 - .../Baselines/default.sw.multi.approved.txt | 11 - tests/SelfTest/Baselines/junit.sw.approved.txt | 1948 -- .../SelfTest/Baselines/junit.sw.multi.approved.txt | 1947 -- tests/SelfTest/Baselines/sonarqube.sw.approved.txt | 2491 -- .../Baselines/sonarqube.sw.multi.approved.txt | 2490 -- tests/SelfTest/Baselines/tap.sw.approved.txt | 4899 ---- tests/SelfTest/Baselines/tap.sw.multi.approved.txt | 4888 ---- tests/SelfTest/Baselines/teamcity.sw.approved.txt | 1110 - .../Baselines/teamcity.sw.multi.approved.txt | 1109 - tests/SelfTest/Baselines/xml.sw.approved.txt | 23600 ------------------- tests/SelfTest/Baselines/xml.sw.multi.approved.txt | 23599 ------------------ tests/SelfTest/IntrospectiveTests/Clara.tests.cpp | 88 - .../SelfTest/IntrospectiveTests/CmdLine.tests.cpp | 582 - .../IntrospectiveTests/CmdLineHelpers.tests.cpp | 138 - .../IntrospectiveTests/ColourImpl.tests.cpp | 64 - tests/SelfTest/IntrospectiveTests/Json.tests.cpp | 250 - .../IntrospectiveTests/Reporters.tests.cpp | 346 - .../SelfTest/IntrospectiveTests/Sharding.tests.cpp | 45 - tests/SelfTest/IntrospectiveTests/Stream.tests.cpp | 32 - .../SelfTest/IntrospectiveTests/TestSpec.tests.cpp | 6 +- .../SelfTest/IntrospectiveTests/TextFlow.tests.cpp | 586 - tests/SelfTest/IntrospectiveTests/Xml.tests.cpp | 203 - .../DiscoverTests/VerifyRegistration.py | 30 +- tests/TestScripts/testBazelExitGuardFile.py | 167 +- tests/TestScripts/testBazelReporter.py | 66 +- tests/TestScripts/testBazelSharding.py | 60 +- tests/TestScripts/testConfigureDefaultReporter.py | 50 - tests/TestScripts/testConfigureDisable.py | 4 +- .../testConfigureExperimentalRedirect.py | 28 +- tests/TestScripts/testPartialTestCaseEvent.py | 23 +- tests/TestScripts/testRandomOrder.py | 77 - tests/TestScripts/testSectionFiltering.py | 189 - tests/TestScripts/testSharding.py | 196 +- tests/meson.build | 31 +- third_party/clara.hpp | 1267 - tools/scripts/approvalTests.py | 243 - tools/scripts/approve.py | 31 - tools/scripts/generateAmalgamatedFiles.py | 54 +- xmake.lua | 17 +- 228 files changed, 29760 insertions(+), 147082 deletions(-) diff --git a/.conan/build.py b/.conan/build.py deleted file mode 100644 index e163d5f4..00000000 --- a/.conan/build.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -import os -import re -from cpt.packager import ConanMultiPackager -from cpt.ci_manager import CIManager -from cpt.printer import Printer - - -class BuilderSettings(object): - @property - def username(self): - """ Set catchorg as package's owner - """ - return os.getenv("CONAN_USERNAME", "catchorg") - - @property - def login_username(self): - """ Set Bintray login username - """ - return os.getenv("CONAN_LOGIN_USERNAME", "horenmar") - - @property - def upload(self): - """ Set Catch2 repository to be used on upload. - The upload server address could be customized by env var - CONAN_UPLOAD. If not defined, the method will check the branch name. - Only devel or CONAN_STABLE_BRANCH_PATTERN will be accepted. - The devel branch will be pushed to testing channel, because it does - not match the stable pattern. Otherwise it will upload to stable - channel. - """ - return os.getenv("CONAN_UPLOAD", "https://api.bintray.com/conan/catchorg/catch2") - - @property - def upload_only_when_stable(self): - """ Force to upload when running over tag branch - """ - return os.getenv("CONAN_UPLOAD_ONLY_WHEN_STABLE", "True").lower() in ["true", "1", "yes"] - - @property - def stable_branch_pattern(self): - """ Only upload the package the branch name is like a tag - """ - return os.getenv("CONAN_STABLE_BRANCH_PATTERN", r"v\d+\.\d+\.\d+") - - @property - def reference(self): - """ Read project version from branch create Conan reference - """ - return os.getenv("CONAN_REFERENCE", "catch2/{}".format(self._version)) - - @property - def channel(self): - """ Default Conan package channel when not stable - """ - return os.getenv("CONAN_CHANNEL", "testing") - - @property - def _version(self): - """ Get version name from cmake file - """ - pattern = re.compile(r"project\(Catch2 LANGUAGES CXX VERSION (\d+\.\d+\.\d+)\)") - version = "latest" - with open("CMakeLists.txt") as file: - for line in file: - result = pattern.search(line) - if result: - version = result.group(1) - return version - - @property - def _branch(self): - """ Get branch name from CI manager - """ - printer = Printer(None) - ci_manager = CIManager(printer) - return ci_manager.get_branch() - - -if __name__ == "__main__": - settings = BuilderSettings() - builder = ConanMultiPackager( - reference=settings.reference, - channel=settings.channel, - upload=settings.upload, - upload_only_when_stable=False, - stable_branch_pattern=settings.stable_branch_pattern, - login_username=settings.login_username, - username=settings.username, - test_folder=os.path.join(".conan", "test_package")) - builder.add() - builder.run() diff --git a/.conan/test_package/CMakeLists.txt b/.conan/test_package/CMakeLists.txt deleted file mode 100644 index 04d638aa..00000000 --- a/.conan/test_package/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -cmake_minimum_required(VERSION 3.16) -project(PackageTest LANGUAGES CXX) - -find_package(Catch2 CONFIG REQUIRED) - -add_executable(test_package test_package.cpp) -target_link_libraries(test_package Catch2::Catch2WithMain) diff --git a/.conan/test_package/conanfile.py b/.conan/test_package/conanfile.py deleted file mode 100644 index dc038764..00000000 --- a/.conan/test_package/conanfile.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -from conan import ConanFile -from conan.tools.cmake import CMake, cmake_layout -from conan.tools.build import can_run -from conan.tools.files import save, load -import os - - -class TestPackageConan(ConanFile): - settings = "os", "compiler", "build_type", "arch" - generators = "CMakeToolchain", "CMakeDeps", "VirtualRunEnv" - test_type = "explicit" - - def requirements(self): - self.requires(self.tested_reference_str) - - def layout(self): - cmake_layout(self) - - def generate(self): - save(self, os.path.join(self.build_folder, "package_folder"), - self.dependencies[self.tested_reference_str].package_folder) - save(self, os.path.join(self.build_folder, "license"), - self.dependencies[self.tested_reference_str].license) - - def build(self): - cmake = CMake(self) - cmake.configure() - cmake.build() - - def test(self): - if can_run(self): - cmd = os.path.join(self.cpp.build.bindir, "test_package") - self.run(cmd, env="conanrun") - - package_folder = load(self, os.path.join(self.build_folder, "package_folder")) - license = load(self, os.path.join(self.build_folder, "license")) - assert os.path.isfile(os.path.join(package_folder, "licenses", "LICENSE.txt")) - assert license == 'BSL-1.0' diff --git a/.conan/test_package/test_package.cpp b/.conan/test_package/test_package.cpp deleted file mode 100644 index 3c08090b..00000000 --- a/.conan/test_package/test_package.cpp +++ /dev/null @@ -1,13 +0,0 @@ -#include <catch2/catch_test_macros.hpp> - -int Factorial( int number ) { - return number <= 1 ? 1 : Factorial( number - 1 ) * number; -} - -TEST_CASE( "Factorial Tests", "[single-file]" ) { - REQUIRE( Factorial(0) == 1 ); - REQUIRE( Factorial(1) == 1 ); - REQUIRE( Factorial(2) == 2 ); - REQUIRE( Factorial(3) == 6 ); - REQUIRE( Factorial(10) == 3628800 ); -} \ No newline at end of file diff --git a/.github/workflows/package-manager-builds.yaml b/.github/workflows/package-manager-builds.yaml deleted file mode 100644 index 1c5bb2cc..00000000 --- a/.github/workflows/package-manager-builds.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: Package Manager Builds - -on: [push, pull_request] - -permissions: - contents: read - -jobs: - conan_builds: - name: Conan on ${{ matrix.os }} - runs-on: ${{matrix.os}} - strategy: - fail-fast: false - matrix: - # Conan's detected MSVC runtime targets the VS 2022 (v143) toolset. - os: [macos-15, windows-2022] - - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - with: - persist-credentials: false - - - name: Install conan - run: python -m pip install --requirement requirements-conan.txt - - - name: Detect a Clang-based Conan profile - run: conan profile detect --force - env: - CC: ${{ runner.os == 'Windows' && 'clang-cl' || 'clang' }} - CXX: ${{ runner.os == 'Windows' && 'clang-cl' || 'clang++' }} - - - name: Run conan package create on macOS - if: runner.os == 'macOS' - run: conan create . -tf .conan/test_package - env: - CC: clang - CXX: clang++ - - - name: Run conan package create with Visual Studio ClangCL - if: runner.os == 'Windows' - # The VS 2022 image ships the v144 CRT. Conan's generic clang profile - # otherwise picks v143 and MinGW Makefiles, which cannot link that CRT. - run: conan create . -tf .conan/test_package -s compiler.runtime_version=v144 -c "tools.cmake.cmaketoolchain:generator=Visual Studio 17 2022" - env: - CC: clang-cl - CXX: clang-cl diff --git a/.gitignore b/.gitignore index ac7ff3f3..0c36403a 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,6 @@ Build .vscode cmake-build-* benchmark-dir -.conan/test_package/build **/CMakeUserPresets.json /bazel-* build-fuzzers diff --git a/BUILD.bazel b/BUILD.bazel index 4d4bfac9..fce4ad82 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -8,6 +8,7 @@ package( exports_files([ "LICENSE.txt", + "src/catch2/catch_session.cpp", ]) license( @@ -20,8 +21,6 @@ expand_template( name = "catch_user_config", out = "catch2/catch_user_config.hpp", substitutions = { - "@CATCH_CONFIG_CONSOLE_WIDTH@": "80", - "@CATCH_CONFIG_DEFAULT_REPORTER@": "console", "#cmakedefine CATCH_CONFIG_ANDROID_LOGWRITE": "", "#cmakedefine CATCH_CONFIG_BAZEL_SUPPORT": "#define CATCH_CONFIG_BAZEL_SUPPORT", "#cmakedefine CATCH_CONFIG_COLOUR_WIN32": "", @@ -92,20 +91,36 @@ cc_library( visibility = ["//visibility:public"], ) -# Static library, without main. +# Catch2's engine: everything but the entry points. The runner is in +# //src/Progmasoft:runner. cc_library( - name = "catch2", + name = "catch2_core", srcs = glob( ["src/catch2/**/*.cpp"], - exclude = ["src/catch2/internal/catch_main.cpp"], + exclude = [ + "src/catch2/internal/catch_main.cpp", + "src/catch2/catch_session.cpp", + ], ), hdrs = glob(["src/catch2/**/*.hpp"]), includes = ["src/"], linkstatic = True, - visibility = ["//visibility:public"], + visibility = ["//src/Progmasoft:__pkg__"], deps = [":catch2_generated"], ) +# Static library, without main: Catch2's engine with the Catch3 runner, which +# replaced Catch2's session, command line and reporters. +cc_library( + name = "catch2", + linkstatic = True, + visibility = ["//visibility:public"], + deps = [ + ":catch2_core", + "//src/Progmasoft:runner", + ], +) + # Static library, with main. cc_library( name = "catch2_main", diff --git a/CMake/CatchConfigOptions.cmake b/CMake/CatchConfigOptions.cmake index 96725066..43243485 100644 --- a/CMake/CatchConfigOptions.cmake +++ b/CMake/CatchConfigOptions.cmake @@ -80,10 +80,8 @@ else() set(CATCH_CONFIG_SHARED_LIBRARY "") endif() -set(CATCH_CONFIG_DEFAULT_REPORTER "console" CACHE STRING "Read docs/configuration.md for details. The name of the reporter should be without quotes.") -set(CATCH_CONFIG_CONSOLE_WIDTH "80" CACHE STRING "Read docs/configuration.md for details. Must form a valid integer literal.") -mark_as_advanced(CATCH_CONFIG_SHARED_LIBRARY CATCH_CONFIG_DEFAULT_REPORTER CATCH_CONFIG_CONSOLE_WIDTH) +mark_as_advanced(CATCH_CONFIG_SHARED_LIBRARY) # There is no good way to both turn this into a CMake cache variable, # and keep reasonable default semantics inside the project. Thus we do diff --git a/README.md b/README.md index d1ef75e0..9397daa6 100644 --- a/README.md +++ b/README.md @@ -17,22 +17,28 @@ serialization. ## What Catch3 provides -- **Familiar test authoring and execution.** The existing test runner, - assertions, sections, reporters, benchmarks, and command-line behavior remain - available through the Catch2 compatibility surface. +- **Familiar test authoring.** Catch2's test cases, assertions, sections, + matchers, generators, listeners and benchmarks keep working unchanged. +- **A runner of its own.** The [Catch3 runner](docs/progmasoft-runner.md) + runs tests in parallel worker processes, survives crashes and hangs, retries + failures to tell flaky and order-dependent tests apart, quarantines flaky + tests, keeps a history that drives time limits, ordering, incremental runs + and replay, selects and shards tests, and writes console, JUnit, JSON and + trace reports. - **Deterministic property testing.** Generate values from a reproducible seed, check a predicate over multiple trials, and shrink a failing input to a smaller counterexample. - **Text snapshots.** Compare generated output against explicitly located snapshot files, with opt-in creation or replacement modes. -- **Structured results and XML output.** Build a runner-neutral result model and - serialize it as deterministic JUnit-style XML. +- **Structured results and XML output.** The runner's result model, with + every attempt, and its deterministic JUnit XML writer are public. - **A compiled library, not a header-only package.** Snapshot I/O and result/XML serialization are compiled C++20 implementations; templates and assertion macros remain in headers where needed. -Catch3 is in an early development stage. The Catch2-compatible runner is still -its execution engine; Catch3 does not yet provide an independent runner. The +Catch3 is in an early development stage. Catch2's command line and reporters +were replaced by the Catch3 runner; see +[the command line](docs/command-line.md) before moving a project over. The property API currently checks one generated value per property and uses bounded, deterministic greedy shrinking. See [current capabilities and limitations](docs/why-catch3.md) before adopting it in a production project. @@ -81,7 +87,7 @@ target_link_libraries(MyTests PRIVATE Catch2::Catch2WithMain) ``` -The Catch2 target supplies the compatibility runner and `main`; link +The Catch2 target supplies the Catch3 runner and `main`; link `Progmasoft::Catch3` for the compiled Catch3 additions. The two targets retain their separate language baselines and licensing boundaries. @@ -89,7 +95,8 @@ their separate language baselines and licensing boundaries. - [Catch3 additions: properties, snapshots, results, and XML](docs/progmasoft-catch3.md) - [Project scope, compatibility policy, and limitations](docs/why-catch3.md) -- [Catch2-compatible runner tutorial](docs/tutorial.md) +- [The Catch3 runner](docs/progmasoft-runner.md) +- [Tutorial](docs/tutorial.md) - [Catch2-compatible API reference](docs/Readme.md) - [Catch2 upstream project and source history](https://github.com/catchorg/Catch2) diff --git a/conanfile.py b/conanfile.py deleted file mode 100755 index 7a3ac7c3..00000000 --- a/conanfile.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python -from conan import ConanFile -from conan.tools.cmake import CMake, CMakeToolchain, CMakeDeps, cmake_layout -from conan.tools.files import copy, rmdir -from conan.tools.build import check_min_cppstd -from conan.tools.scm import Version -from conan.errors import ConanInvalidConfiguration -import os -import re - -required_conan_version = ">=1.53.0" - -class CatchConan(ConanFile): - name = "catch2" - description = "A modern, C++-native, framework for unit-tests, TDD and BDD" - topics = ("conan", "catch2", "unit-test", "tdd", "bdd") - url = "https://github.com/catchorg/Catch2" - homepage = url - license = "BSL-1.0" - version = "latest" - settings = "os", "compiler", "build_type", "arch" - extension_properties = {"compatibility_cppstd": False} - - options = { - "shared": [True, False], - "fPIC": [True, False], - } - default_options = { - "shared": False, - "fPIC": True, - } - - @property - def _min_cppstd(self): - return "14" - - @property - def _compilers_minimum_version(self): - return { - "gcc": "7", - "Visual Studio": "15", - "msvc": "191", - "clang": "5", - "apple-clang": "10", - } - - - def set_version(self): - pattern = re.compile(r"\w*VERSION (\d+\.\d+\.\d+) # CML version placeholder, don't delete") - with open("CMakeLists.txt") as file: - for line in file: - result = pattern.search(line) - if result: - self.version = result.group(1) - - self.output.info(f'Using version: {self.version}') - - def export(self): - copy(self, "LICENSE.txt", src=self.recipe_folder, dst=self.export_folder) - - def export_sources(self): - copy(self, "CMakeLists.txt", src=self.recipe_folder, dst=self.export_sources_folder) - copy(self, "src/*", src=self.recipe_folder, dst=self.export_sources_folder) - copy(self, "extras/*", src=self.recipe_folder, dst=self.export_sources_folder) - copy(self, "CMake/*", src=self.recipe_folder, dst=self.export_sources_folder) - - def config_options(self): - if self.settings.os == "Windows": - del self.options.fPIC - - def configure(self): - if self.options.shared: - self.options.rm_safe("fPIC") - - def layout(self): - cmake_layout(self) - - def validate(self): - if self.settings.compiler.get_safe("cppstd"): - check_min_cppstd(self, self._min_cppstd) - # INFO: Conan 1.x does not specify cppstd by default, so we need to check the compiler version instead. - minimum_version = self._compilers_minimum_version.get(str(self.settings.compiler), False) - if minimum_version and Version(self.settings.compiler.version) < minimum_version: - raise ConanInvalidConfiguration(f"{self.ref} requires C++{self._min_cppstd}, which your compiler doesn't support") - - def generate(self): - tc = CMakeToolchain(self) - tc.cache_variables["BUILD_TESTING"] = False - tc.cache_variables["CATCH_INSTALL_DOCS"] = False - tc.cache_variables["CATCH_INSTALL_EXTRAS"] = True - tc.generate() - - deps = CMakeDeps(self) - deps.generate() - - def build(self): - cmake = CMake(self) - cmake.configure() - cmake.build() - - def package(self): - copy(self, "LICENSE.txt", src=str(self.recipe_folder), dst=os.path.join(self.package_folder, "licenses")) - cmake = CMake(self) - cmake.install() - rmdir(self, os.path.join(self.package_folder, "share")) - rmdir(self, os.path.join(self.package_folder, "lib", "cmake")) - copy(self, "*.cmake", src=os.path.join(self.export_sources_folder, "extras"), - dst=os.path.join(self.package_folder, "lib", "cmake", "Catch2")) - - def package_info(self): - lib_suffix = "d" if self.settings.build_type == "Debug" else "" - - self.cpp_info.set_property("cmake_file_name", "Catch2") - self.cpp_info.set_property("cmake_target_name", "Catch2::Catch2WithMain") - self.cpp_info.set_property("pkg_config_name", "catch2-with-main") - - # Catch2 - self.cpp_info.components["catch2base"].set_property("cmake_file_name", "Catch2::Catch2") - self.cpp_info.components["catch2base"].set_property("cmake_target_name", "Catch2::Catch2") - self.cpp_info.components["catch2base"].set_property("pkg_config_name", "catch2") - self.cpp_info.components["catch2base"].libs = ["Catch2" + lib_suffix] - self.cpp_info.components["catch2base"].builddirs.append("lib/cmake/Catch2") - - # Catch2WithMain - self.cpp_info.components["catch2main"].set_property("cmake_file_name", "Catch2::Catch2WithMain") - self.cpp_info.components["catch2main"].set_property("cmake_target_name", "Catch2::Catch2WithMain") - self.cpp_info.components["catch2main"].set_property("pkg_config_name", "catch2-with-main") - self.cpp_info.components["catch2main"].libs = ["Catch2Main" + lib_suffix] - self.cpp_info.components["catch2main"].requires = ["catch2base"] diff --git a/docs/Readme.md b/docs/Readme.md index d8e9730e..5d21ef7f 100644 --- a/docs/Readme.md +++ b/docs/Readme.md @@ -1,9 +1,10 @@ <a id="top"></a> # Catch3 documentation -Catch3 combines the upstream Catch2-compatible runner and API with -Progmasoft's C++20 extensions. Start with the [tutorial](tutorial.md#top), -then read [what Catch3 adds](progmasoft-catch3.md), +Catch3 keeps Catch2's API for writing tests, runs them with +[its own runner](progmasoft-runner.md#top) and adds Progmasoft's C++20 +extensions. Start with the [tutorial](tutorial.md#top), then read +[the runner](progmasoft-runner.md#top), [what Catch3 adds](progmasoft-catch3.md), [native property testing](progmasoft-properties.md), [the roadmap](progmasoft-roadmap.md) and [why the two API families remain distinct](why-catch3.md). The compatibility pages below retain real `catch2/` include paths, `Catch2::` CMake targets, @@ -24,7 +25,7 @@ engine, not a claim that Catch3 is an official Catch2 release. * [Test cases and sections](test-cases-and-sections.md#top) * [Test fixtures](test-fixtures.md#top) * [Explicitly skipping, passing, and failing tests at runtime](skipping-passing-failing.md#top) -* [Reporters (output customization)](reporters.md#top) +* [Reports](reporters.md#top) * [Event Listeners](event-listeners.md#top) * [Data Generators (value parameterized tests)](generators.md#top) * [Other macros](other-macros.md#top) @@ -36,6 +37,7 @@ engine, not a claim that Catch3 is an official Catch2 release. * [String Conversions](tostring.md#top) **Running:** +* [The Catch3 runner](progmasoft-runner.md#top) * [Command line reference](command-line.md#top) * [Running specific section/generator](filtering-execution-path.md#top) diff --git a/docs/ci-and-misc.md b/docs/ci-and-misc.md index 4fa033fd..9932ccbc 100644 --- a/docs/ci-and-misc.md +++ b/docs/ci-and-misc.md @@ -16,50 +16,12 @@ like Continuous Integration and 3rd party test runners. ## Continuous Integration systems -Probably the most important aspect to using Catch with a build server is the use of different reporters. Catch comes bundled with three reporters that should cover the majority of build servers out there - although adding more for better integration with some is always a possibility (currently we also offer TeamCity, TAP, Automake and SonarQube reporters). - -Two of these reporters are built in (XML and JUnit) and the third (TeamCity) is included as a separate header. It's possible that the other two may be split out in the future too - as that would make the core of Catch smaller for those that don't need them. - -### XML Reporter -```-r xml``` - -The XML Reporter writes in an XML format that is specific to Catch. - -The advantage of this format is that it corresponds well to the way Catch works (especially the more unusual features, such as nested sections) and is a fully streaming format - that is it writes output as it goes, without having to store up all its results before it can start writing. - -The disadvantage is that, being specific to Catch, no existing build servers understand the format natively. It can be used as input to an XSLT transformation that could convert it to, say, HTML - although this loses the streaming advantage, of course. - -### JUnit Reporter -```-r junit``` - -The JUnit Reporter writes in an XML format that mimics the JUnit ANT schema. - -The advantage of this format is that the JUnit Ant schema is widely understood by most build servers and so can usually be consumed with no additional work. - -The disadvantage is that this schema was designed to correspond to how JUnit works - and there is a significant mismatch with how Catch works. Additionally the format is not streamable (because opening elements hold counts of failed and passing tests as attributes) - so the whole test run must complete before it can be written. - - -### TeamCity Reporter -```-r teamcity``` - -The TeamCity Reporter writes TeamCity service messages to stdout. In order to be able to use this reporter an additional header must also be included. - -Being specific to TeamCity this is the best reporter to use with it - but it is completely unsuitable for any other purpose. It is a streaming format (it writes as it goes) - although test results don't appear in the TeamCity interface until the completion of a suite (usually the whole test run). - -### Automake Reporter -```-r automake``` - -The Automake Reporter writes out the [meta tags](https://www.gnu.org/software/automake/manual/html_node/Log-files-generation-and-test-results-recording.html#Log-files-generation-and-test-results-recording) expected by automake via `make check`. - -### TAP (Test Anything Protocol) Reporter -```-r tap``` - -Because of the incremental nature of Catch's test suites and ability to run specific tests, our implementation of TAP reporter writes out the number of tests in a suite last. - -### SonarQube Reporter -```-r sonarqube``` -[SonarQube Generic Test Data](https://docs.sonarqube.org/latest/analysis/generic-test/) XML format for tests metrics. - +The [Catch3 runner](progmasoft-runner.md#top) writes the reports build +servers read: `--junit=FILE` for JUnit XML, which most CI systems display, +with flaky tests and retries in the `<flakyFailure>` and `<rerunFailure>` +elements Jenkins and GitLab understand, and `--json=FILE` for everything +else. Under `bazel test` the JUnit report goes to the file Bazel names. See +[reports](reporters.md#top). ## Bazel test runner integration diff --git a/docs/cmake-integration.md b/docs/cmake-integration.md index bf68ae92..d3d45dfe 100644 --- a/docs/cmake-integration.md +++ b/docs/cmake-integration.md @@ -166,28 +166,27 @@ Note that this variable is only available in CTest. * `REPORTER reporter` -Use the specified reporter when running the test case. The reporter will -be passed to the test runner as `--reporter reporter`. +Write a report of each test: `junit` passes `--junit=<file>` and `json` +passes `--json=<file>` to the test executable; `console` writes no file. +`junit` and `json` need `OUTPUT_DIR`. * `OUTPUT_DIR dir` -If specified, the parameter is passed along as -`--out dir/<test_name>` to test executable. The actual file name is the -same as the test name. This should be used instead of -`EXTRA_ARGS --out foo` to avoid race conditions writing the result output -when using parallel test execution. +The directory the reports of `REPORTER` go to, one file per test, named +after the test. Use it instead of `EXTRA_ARGS --junit=foo` so that tests +that run at the same time do not write the same file. * `OUTPUT_PREFIX prefix` May be used in conjunction with `OUTPUT_DIR`. If specified, `prefix` is added to each output file name, like so -`--out dir/prefix<test_name>`. +`dir/prefix<test_name>`. * `OUTPUT_SUFFIX suffix` May be used in conjunction with `OUTPUT_DIR`. If specified, `suffix` is added to each output file name, like so -`--out dir/<test_name>suffix`. This can be used to add a file extension to +`dir/<test_name>suffix`. This can be used to add a file extension to the output file name e.g. ".xml". * `DISCOVERY_MODE mode` @@ -294,7 +293,7 @@ and the seed changes every invocation of CTest. Currently there are 3 customization points for this script: * SHARD_COUNT - number of shards to split target's tests into - * REPORTER - reporter spec to use for tests + * REPORTER - report options for the shards, such as --junit=FILE * TEST_SPEC - test spec used for filtering tests Example usage: @@ -304,13 +303,11 @@ include(CatchShardTests) catch_add_sharded_tests(foo-tests SHARD_COUNT 4 - REPORTER "xml::out=-" TEST_SPEC "A" ) catch_add_sharded_tests(tests SHARD_COUNT 8 - REPORTER "xml::out=-" TEST_SPEC "B" ) ``` diff --git a/docs/command-line.md b/docs/command-line.md index 829eba5a..29abc11a 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -1,657 +1,146 @@ <a id="top"></a> - -> **Catch3 compatibility reference:** This page describes the bundled Catch2-compatible API. Names such as `Catch2`, `catch2/`, and `Catch2::Catch2WithMain` remain where they are actual compatibility interfaces. See [Catch3 additions](progmasoft-catch3.md) for Progmasoft APIs. - # Command line **Contents**<br> -[Specifying which tests to run](#specifying-which-tests-to-run)<br> -[Choosing a reporter to use](#choosing-a-reporter-to-use)<br> -[Breaking into the debugger](#breaking-into-the-debugger)<br> -[Showing results for successful tests](#showing-results-for-successful-tests)<br> -[Aborting after a certain number of failures](#aborting-after-a-certain-number-of-failures)<br> -[Listing available tests, tags or reporters](#listing-available-tests-tags-or-reporters)<br> -[Sending output to a file](#sending-output-to-a-file)<br> -[Naming a test run](#naming-a-test-run)<br> -[Eliding assertions expected to throw](#eliding-assertions-expected-to-throw)<br> -[Make whitespace visible](#make-whitespace-visible)<br> -[Warnings](#warnings)<br> -[Reporting timings](#reporting-timings)<br> -[Load test names to run from a file](#load-test-names-to-run-from-a-file)<br> -[Specify the order test cases are run](#specify-the-order-test-cases-are-run)<br> -[Specify a seed for the Random Number Generator](#specify-a-seed-for-the-random-number-generator)<br> -[Identify framework and version according to the libIdentify standard](#identify-framework-and-version-according-to-the-libidentify-standard)<br> -[Wait for key before continuing](#wait-for-key-before-continuing)<br> -[Skip all benchmarks](#skip-all-benchmarks)<br> -[Specify the number of benchmark samples to collect](#specify-the-number-of-benchmark-samples-to-collect)<br> -[Specify the number of resamples for bootstrapping](#specify-the-number-of-resamples-for-bootstrapping)<br> -[Specify the confidence-interval for bootstrapping](#specify-the-confidence-interval-for-bootstrapping)<br> -[Disable statistical analysis of collected benchmark samples](#disable-statistical-analysis-of-collected-benchmark-samples)<br> -[Specify the amount of time in milliseconds spent on warming up each test](#specify-the-amount-of-time-in-milliseconds-spent-on-warming-up-each-test)<br> -[Usage](#usage)<br> -[Specify the section/generator element to run](#specify-the-sectiongenerator-element-to-run)<br> -[Filenames as tags](#filenames-as-tags)<br> -[Override output colouring](#override-output-colouring)<br> -[Test Sharding](#test-sharding)<br> -[Allow running the binary without tests](#allow-running-the-binary-without-tests)<br> -[Output verbosity](#output-verbosity)<br> -[Create file to guard against silent early termination](#create-file-to-guard-against-silent-early-termination)<br> - -Catch works quite nicely without any command line options at all - but for those times when you want greater control the following options are available. -Click one of the following links to take you straight to that option - or scroll on to browse the available options. - -<a href="#specifying-which-tests-to-run"> ` <test-spec> ...`</a><br /> -<a href="#usage"> ` -h, -?, --help`</a><br /> -<a href="#showing-results-for-successful-tests"> ` -s, --success`</a><br /> -<a href="#breaking-into-the-debugger"> ` -b, --break`</a><br /> -<a href="#eliding-assertions-expected-to-throw"> ` -e, --nothrow`</a><br /> -<a href="#invisibles"> ` -i, --invisibles`</a><br /> -<a href="#sending-output-to-a-file"> ` -o, --out`</a><br /> -<a href="#choosing-a-reporter-to-use"> ` -r, --reporter`</a><br /> -<a href="#naming-a-test-run"> ` -n, --name`</a><br /> -<a href="#aborting-after-a-certain-number-of-failures"> ` -a, --abort`</a><br /> -<a href="#aborting-after-a-certain-number-of-failures"> ` -x, --abortx`</a><br /> -<a href="#warnings"> ` -w, --warn`</a><br /> -<a href="#reporting-timings"> ` -d, --durations`</a><br /> -<a href="#input-file"> ` -f, --input-file`</a><br /> -<a href="#run-section"> ` -c, --section`</a><br /> -<a href="#path-filtering"> ` -g, --generator-index`</a><br /> -<a href="#path-filtering"> ` -p, --path-filter`</a><br /> -<a href="#filenames-as-tags"> ` -#, --filenames-as-tags`</a><br /> - - -</br> - -<a href="#listing-available-tests-tags-or-reporters"> ` --list-tests`</a><br /> -<a href="#listing-available-tests-tags-or-reporters"> ` --list-tags`</a><br /> -<a href="#listing-available-tests-tags-or-reporters"> ` --list-reporters`</a><br /> -<a href="#listing-available-tests-tags-or-reporters"> ` --list-listeners`</a><br /> -<a href="#order"> ` --order`</a><br /> -<a href="#rng-seed"> ` --rng-seed`</a><br /> -<a href="#libidentify"> ` --libidentify`</a><br /> -<a href="#wait-for-keypress"> ` --wait-for-keypress`</a><br /> -<a href="#skip-benchmarks"> ` --skip-benchmarks`</a><br /> -<a href="#benchmark-samples"> ` --benchmark-samples`</a><br /> -<a href="#benchmark-resamples"> ` --benchmark-resamples`</a><br /> -<a href="#benchmark-confidence-interval"> ` --benchmark-confidence-interval`</a><br /> -<a href="#benchmark-no-analysis"> ` --benchmark-no-analysis`</a><br /> -<a href="#benchmark-warmup-time"> ` --benchmark-warmup-time`</a><br /> -<a href="#colour-mode"> ` --colour-mode`</a><br /> -<a href="#test-sharding"> ` --shard-count`</a><br /> -<a href="#test-sharding"> ` --shard-index`</a><br /> -<a href=#no-tests-override> ` --allow-running-no-tests`</a><br /> -<a href=#output-verbosity> ` --verbosity`</a><br /> - -</br> - - - -<a id="specifying-which-tests-to-run"></a> -## Specifying which tests to run - -<pre><test-spec> ...</pre> - -By providing a test spec, you filter which tests will be run. If you call -Catch2 without any test spec, then it will run all non-hidden test -cases. A test case is hidden if it has the `[!benchmark]` tag, any tag -with a dot at the start, e.g. `[.]` or `[.foo]`. - -There are three basic test specs that can then be combined into more -complex specs: - - * Full test name, e.g. `"Test 1"`. - - This allows only test cases whose name is "Test 1". - - * Wildcarded test name, e.g. `"*Test"`, or `"Test*"`, or `"*Test*"`. - - This allows any test case whose name ends with, starts with, or contains - in the middle the string "Test". Note that the wildcard can only be at - the start or end. - - * Tag name, e.g. `[some-tag]`. - - This allows any test case tagged with "[some-tag]". Remember that some - tags are special, e.g. those that start with "." or with "!". - - -You can also combine the basic test specs to create more complex test -specs. You can: - - * Concatenate specs to apply all of them, e.g. `[some-tag][other-tag]`. - - This allows test cases that are tagged with **both** "[some-tag]" **and** - "[other-tag]". A test case with just "[some-tag]" will not pass the filter, - nor will test case with just "[other-tag]". - - * Comma-join specs to apply any of them, e.g. `[some-tag],[other-tag]`. - - This allows test cases that are tagged with **either** "[some-tag]" **or** - "[other-tag]". A test case with both will obviously also pass the filter. - - Note that commas take precedence over simple concatenation. This means - that `[a][b],[c]` accepts tests that are tagged with either both "[a]" and - "[b]", or tests that are tagged with just "[c]". - - * Negate the spec by prepending it with `~`, e.g. `~[some-tag]`. - - This rejects any test case that is tagged with "[some-tag]". Note that - rejection takes precedence over other filters. - - Note that negations always binds to the following _basic_ test spec. - This means that `~[foo][bar]` negates only the "[foo]" tag and not the - "[bar]" tag. - -Note that when Catch2 is deciding whether to include a test, first it -checks whether the test matches any negative filters. If it does, -the test is rejected. After that, the behaviour depends on whether there -are positive filters as well. If there are no positive filters, all -remaining non-hidden tests are included. If there are positive filters, -only tests that match the positive filters are included. - -You can also match test names with special characters by escaping them -with a backslash (`"\"`), e.g. a test named `"Do A, then B"` is matched -by `"Do A\, then B"` test spec. Backslash also escapes itself. - - -### Examples - -Given these TEST_CASEs, -``` -TEST_CASE("Test 1") {} - -TEST_CASE("Test 2", "[.foo]") {} - -TEST_CASE("Test 3", "[.bar]") {} - -TEST_CASE("Test 4", "[.][foo][bar]") {} -``` - -this is the result of these filters -``` -./tests # Selects only the first test, others are hidden -./tests "Test 1" # Selects only the first test, other do not match -./tests ~"Test 1" # Selects no tests. Test 1 is rejected, other tests are hidden -./tests "Test *" # Selects all tests. -./tests [bar] # Selects tests 3 and 4. Other tests are not tagged [bar] -./tests ~[foo] # Selects test 1, because it is the only non-hidden test without [foo] tag -./tests [foo][bar] # Selects test 4. -./tests [foo],[bar] # Selects tests 2, 3, 4. -./tests ~[foo][bar] # Selects test 3. 2 and 4 are rejected due to having [foo] tag -./tests ~"Test 2"[foo] # Selects test 4, because test 2 is explicitly rejected -./tests [foo][bar],"Test 1" # Selects tests 1 and 4. -./tests "Test 1*" # Selects test 1, wildcard can match zero characters -``` - -_Note: Using plain asterisk on a command line can cause issues with shell -expansion. Make sure that the asterisk is passed to Catch2 and is not -interpreted by the shell._ - - -<a id="choosing-a-reporter-to-use"></a> -## Choosing a reporter to use - -<pre>-r, --reporter <reporter[::key=value]*></pre> - -Reporters are how the output from Catch2 (results of assertions, tests, -benchmarks and so on) is formatted and written out. The default reporter -is called the "Console" reporter and is intended to provide relatively -verbose and human-friendly output. - -Reporters are also individually configurable. To pass configuration options -to the reporter, you append `::key=value` to the reporter specification -as many times as you want, e.g. `--reporter xml::out=someFile.xml` or -`--reporter custom::colour-mode=ansi::Xoption=2`. - -The keys must either be prefixed by "X", in which case they are not parsed -by Catch2 and are only passed down to the reporter, or one of options -hardcoded into Catch2. Currently there are 3 supported options: - -* ["out"](#sending-output-to-a-file) -* ["colour-mode"](#colour-mode) -* ["verbosity"](#output-verbosity) - -> Support for per-reporter verbosity option was added in Catch2 3.16.0 - -_Note that the reporter might still check the X-prefixed options for -validity, and throw an error if they are wrong._ - -> Support for passing arguments to reporters through the `-r`, `--reporter` flag was introduced in Catch2 3.0.1 - -There are multiple built-in reporters, you can see what they do by using the -[`--list-reporters`](command-line.md#listing-available-tests-tags-or-reporters) -flag. If you need a reporter providing custom format outside of the already -provided ones, look at the ["write your own reporter" part of the reporter -documentation](reporters.md#writing-your-own-reporter). - -This option may be passed multiple times to use multiple (different) -reporters at the same time. See the [reporter documentation](reporters.md#multiple-reporters) -for details on what the resulting behaviour is. Also note that at most one -reporter can be provided without the output-file part of reporter spec. -This reporter will use the "default" output destination, based on -the [`-o`, `--out`](#sending-output-to-a-file) option. - -> Support for using multiple different reporters at the same time was [introduced](https://github.com/catchorg/Catch2/pull/2183) in Catch2 3.0.1 - - -_Note: There is currently no way to escape `::` in the reporter spec, -and thus the reporter names, or configuration keys and values, cannot -contain `::`. As `::` in paths is relatively obscure (unlike ':'), we do -not consider this an issue._ - - -<a id="breaking-into-the-debugger"></a> -## Breaking into the debugger -<pre>-b, --break</pre> - -Under most debuggers Catch2 is capable of automatically breaking on a test -failure. This allows the user to see the current state of the test during -failure. - -<a id="showing-results-for-successful-tests"></a> -## Showing results for successful tests -<pre>-s, --success</pre> - -Usually you only want to see reporting for failed tests. Sometimes it's useful to see *all* the output (especially when you don't trust that that test you just added worked first time!). -To see successful, as well as failing, test results just pass this option. Note that each reporter may treat this option differently. The Junit reporter, for example, logs all results regardless. - -<a id="aborting-after-a-certain-number-of-failures"></a> -## Aborting after a certain number of failures -<pre>-a, --abort --x, --abortx [<failure threshold>] -</pre> - -If a ```REQUIRE``` assertion fails the test case aborts, but subsequent test cases are still run. -If a ```CHECK``` assertion fails even the current test case is not aborted. - -Sometimes this results in a flood of failure messages and you'd rather just see the first few. Specifying ```-a``` or ```--abort``` on its own will abort the whole test run on the first failed assertion of any kind. Use ```-x``` or ```--abortx``` followed by a number to abort after that number of assertion failures. - -<a id="listing-available-tests-tags-or-reporters"></a> -## Listing available tests, tags or reporters -``` ---list-tests ---list-tags ---list-reporters ---list-listeners -``` - -> The `--list*` options became customizable through reporters in Catch2 3.0.1 - -> The `--list-listeners` option was added in Catch2 3.0.1 - -`--list-tests` lists all registered tests matching specified test spec. -Usually this listing also includes tags, and potentially also other -information, like source location, based on verbosity and reporter's design. - -`--list-tags` lists all tags from registered tests matching specified test -spec. Usually this also includes number of tests cases they match and -similar information. - -`--list-reporters` lists all available reporters and their descriptions. - -`--list-listeners` lists all registered listeners and their descriptions. - -The [`--verbosity` argument](#output-verbosity) modifies the level of detail provided by the default `--list*` options -as follows: - -| Option | `normal` (default) | `quiet` | `high` | -|--------------------|---------------------------------|---------------------|-----------------------------------------| -| `--list-tests` | Test names and tags | Test names only | Same as `normal`, plus source code line | -| `--list-tags` | Tags and counts | Tags only | Same as `normal` | -| `--list-reporters` | Reporter names and descriptions | Reporter names only | Same as `normal` | -| `--list-listeners` | Listener names and descriptions | Listener names only | Same as `normal` | - - -<a id="sending-output-to-a-file"></a> -## Sending output to a file -<pre>-o, --out <filename> -</pre> - -Use this option to send all output to a file, instead of stdout. You can -use `-` as the filename to explicitly send the output to stdout (this is -useful e.g. when using multiple reporters). - -> Support for `-` as the filename was introduced in Catch2 3.0.1 - -Filenames starting with "%" (percent symbol) are reserved by Catch2 for -meta purposes, e.g. using `%debug` as the filename opens stream that -writes to platform specific debugging/logging mechanism. - -Catch2 currently recognizes 3 meta streams: - -* `%debug` - writes to platform specific debugging/logging output -* `%stdout` - writes to stdout -* `%stderr` - writes to stderr - -> Support for `%stdout` and `%stderr` was introduced in Catch2 3.0.1 - - -<a id="naming-a-test-run"></a> -## Naming a test run -<pre>-n, --name <name for test run></pre> - -If a name is supplied it will be used by the reporter to provide an overall name for the test run. This can be useful if you are sending to a file, for example, and need to distinguish different test runs - either from different Catch executables or runs of the same executable with different options. If not supplied the name is defaulted to the name of the executable. - -<a id="eliding-assertions-expected-to-throw"></a> -## Eliding assertions expected to throw -<pre>-e, --nothrow</pre> - -Skips all assertions that test that an exception is thrown, e.g. ```REQUIRE_THROWS```. - -These can be a nuisance in certain debugging environments that may break when exceptions are thrown (while this is usually optional for handled exceptions, it can be useful to have enabled if you are trying to track down something unexpected). - -Sometimes exceptions are expected outside of one of the assertions that tests for them (perhaps thrown and caught within the code-under-test). The whole test case can be skipped when using ```-e``` by marking it with the ```[!throws]``` tag. - -When running with this option any throw checking assertions are skipped so as not to contribute additional noise. Be careful if this affects the behaviour of subsequent tests. - -<a id="invisibles"></a> -## Make whitespace visible -<pre>-i, --invisibles</pre> - -If a string comparison fails due to differences in whitespace - especially leading or trailing whitespace - it can be hard to see what's going on. -This option transforms tabs and newline characters into ```\t``` and ```\n``` respectively when printing. - -<a id="warnings"></a> -## Warnings -<pre>-w, --warn <warning name></pre> - -You can think of Catch2's warnings as the equivalent of `-Werror` (`/WX`) -flag for C++ compilers. It turns some suspicious occurrences, like a section -without assertions, into errors. Because these might be intended, warnings -are not enabled by default, but user can opt in. - -You can enable multiple warnings at the same time. - -There are currently two warnings implemented: - -``` - NoAssertions // Fail test case / leaf section if no assertions - // (e.g. `REQUIRE`) is encountered. - UnmatchedTestSpec // Fail test run if any of the CLI test specs did - // not match any tests. - InfiniteGenerators // Fail if GENERATE would run infinitely -``` - -> `UnmatchedTestSpec` was introduced in Catch2 3.0.1. - -> `InfiniteGenerators` was introduced in Catch2 3.13.0 - - -<a id="reporting-timings"></a> -## Reporting timings -<pre>-d, --durations <yes/no></pre> - -When set to ```yes``` Catch will report the duration of each test case, in seconds with millisecond precision. Note that it does this regardless of whether a test case passes or fails. Note, also, the certain reporters (e.g. Junit) always report test case durations regardless of this option being set or not. - -<pre>-D, --min-duration <value></pre> - -> `--min-duration` was [introduced](https://github.com/catchorg/Catch2/pull/1910) in Catch2 2.13.0 - -When set, Catch will report the duration of each test case that took more -than <value> seconds, in seconds with millisecond precision. This option is overridden by both -`-d yes` and `-d no`, so that either all durations are reported, or none -are. - - -<a id="input-file"></a> -## Load test names to run from a file -<pre>-f, --input-file <filename></pre> - -Provide the name of a file that contains a list of test case names, -one per line. Blank lines are skipped. - -A useful way to generate an initial instance of this file is to combine -the [`--list-tests`](#listing-available-tests-tags-or-reporters) flag with -the [`--verbosity quiet`](#output-verbosity) option. You can also -use test specs to filter this list down to what you want first. - - -<a id="order"></a> -## Specify the order test cases are run -<pre>--order <decl|lex|rand></pre> - -Test cases are ordered one of three ways: - -### decl -Declaration order. - -Tests in the same translation unit are sorted using their declaration orders, -different TUs are sorted in an implementation (linking) dependent order. - - -### lex -Lexicographic order. - -Tests are sorted by their name, their tags are ignored. - - -### rand -Randomized order. The default order. - -> Randomized order has been made default in Catch2 3.9.0 - -The order is dependent on Catch2's random seed (see -[`--rng-seed`](#rng-seed)), and is subset invariant. What this means -is that as long as the random seed is fixed, running only some tests -(e.g. via tag) does not change their relative order. - -> The subset stability was introduced in Catch2 v2.12.0 - -Since the random order was made subset stable, we promise that given -the same random seed, the order of test cases will be the same across -different platforms, as long as the tests were compiled against identical -version of Catch2. We reserve the right to change the relative order -of tests cases between Catch2 versions, but it is unlikely to happen often. - - -<a id="rng-seed"></a> -## Specify a seed for the Random Number Generator -<pre>--rng-seed <'time'|'random-device'|number></pre> - -Sets the seed for random number generators used by Catch2. These are used -e.g. to shuffle tests when user asks for tests to be in random order. - -Using `time` as the argument asks Catch2 generate the seed through call -to `std::time(nullptr)`. This provides very weak randomness and multiple -runs of the binary can generate the same seed if they are started close -to each other. - -Using `random-device` asks for `std::random_device` to be used instead. -If your implementation provides working `std::random_device`, it should -be preferred to using `time`. Catch2 uses `std::random_device` by default. - - -<a id="libidentify"></a> -## Identify framework and version according to the libIdentify standard -<pre>--libidentify</pre> - -See [The LibIdentify repo for more information and examples](https://github.com/janwilmans/LibIdentify). - -<a id="wait-for-keypress"></a> -## Wait for key before continuing -<pre>--wait-for-keypress <never|start|exit|both></pre> - -Will cause the executable to print a message and wait until the return/ enter key is pressed before continuing - -either before running any tests, after running all tests - or both, depending on the argument. - -<a id="skip-benchmarks"></a> -## Skip all benchmarks -<pre>--skip-benchmarks</pre> - -> [Introduced](https://github.com/catchorg/Catch2/issues/2408) in Catch2 3.0.1. - -This flag tells Catch2 to skip running all benchmarks. Benchmarks in this -case mean code blocks in `BENCHMARK` and `BENCHMARK_ADVANCED` macros, not -test cases with the `[!benchmark]` tag. - -<a id="benchmark-samples"></a> -## Specify the number of benchmark samples to collect -<pre>--benchmark-samples <# of samples></pre> - -> [Introduced](https://github.com/catchorg/Catch2/issues/1616) in Catch2 2.9.0. - -When running benchmarks a number of "samples" is collected. This is the base data for later statistical analysis. -Per sample a clock resolution dependent number of iterations of the user code is run, which is independent of the number of samples. Defaults to 100. - -<a id="benchmark-resamples"></a> -## Specify the number of resamples for bootstrapping -<pre>--benchmark-resamples <# of resamples></pre> - -> [Introduced](https://github.com/catchorg/Catch2/issues/1616) in Catch2 2.9.0. - -After the measurements are performed, statistical [bootstrapping] is performed -on the samples. The number of resamples for that bootstrapping is configurable -but defaults to 100000. Due to the bootstrapping it is possible to give -estimates for the mean and standard deviation. The estimates come with a lower -bound and an upper bound, and the confidence interval (which is configurable but -defaults to 95%). - - [bootstrapping]: http://en.wikipedia.org/wiki/Bootstrapping_%28statistics%29 - -<a id="benchmark-confidence-interval"></a> -## Specify the confidence-interval for bootstrapping -<pre>--benchmark-confidence-interval <confidence-interval></pre> - -> [Introduced](https://github.com/catchorg/Catch2/issues/1616) in Catch2 2.9.0. - -The confidence-interval is used for statistical bootstrapping on the samples to -calculate the upper and lower bounds of mean and standard deviation. -Must be between 0 and 1 and defaults to 0.95. - -<a id="benchmark-no-analysis"></a> -## Disable statistical analysis of collected benchmark samples -<pre>--benchmark-no-analysis</pre> - -> [Introduced](https://github.com/catchorg/Catch2/issues/1616) in Catch2 2.9.0. - -When this flag is specified no bootstrapping or any other statistical analysis is performed. -Instead the user code is only measured and the plain mean from the samples is reported. - -<a id="benchmark-warmup-time"></a> -## Specify the amount of time in milliseconds spent on warming up each test -<pre>--benchmark-warmup-time</pre> - -> [Introduced](https://github.com/catchorg/Catch2/pull/1844) in Catch2 2.11.2. - -Configure the amount of time spent warming up each test. - -<a id="usage"></a> -## Usage -<pre>-h, -?, --help</pre> - -Prints the command line arguments to stdout - - -<a id="run-section"></a> -<a id="path-filtering"></a> -## Specify the section/generator element to run -<pre>-c, --section <section name></pre> -<pre>-g, --generator-index <index in generator></pre> -<pre>-p, --path-filter <path filter spec></pre> - -> The generator and generic path filtering was added in Catch2 3.13.0 - -These arguments allow you to run specific section(s) in a test case, or -only get specific element from a generator. All the variants form a shared -stack of filters. - -[See the full documentation of path filtering for more details](filtering-execution-path.md#top) - - -<a id="filenames-as-tags"></a> -## Filenames as tags -<pre>-#, --filenames-as-tags</pre> - -This option adds an extra tag to all test cases. The tag is `#` followed -by the unqualified filename the test case is defined in, with the _last_ -extension stripped out. - -For example, tests within the file `tests\SelfTest\UsageTests\BDD.tests.cpp` -will be given the `[#BDD.tests]` tag. - - -<a id="colour-mode"></a> -## Override output colouring -<pre>--colour-mode <ansi|win32|none|default></pre> - -> The `--colour-mode` option replaced the old `--colour` option in Catch2 3.0.1 - - -Catch2 support two different ways of colouring terminal output, and by -default it attempts to make a good guess on which implementation to use -(and whether to even use it, e.g. Catch2 tries to avoid writing colour -codes when writing the results into a file). - -`--colour-mode` allows the user to explicitly select what happens. - -* `--colour-mode ansi` tells Catch2 to always use ANSI colour codes, even -when writing to a file -* `--colour-mode win32` tells Catch2 to use colour implementation based - on Win32 terminal API -* `--colour-mode none` tells Catch2 to disable colours completely -* `--colour-mode default` lets Catch2 decide - -`--colour-mode default` is the default setting. - - -<a id="test-sharding"></a> -## Test Sharding -<pre>--shard-count <#number of shards>, --shard-index <#shard index to run></pre> - -> [Introduced](https://github.com/catchorg/Catch2/pull/2257) in Catch2 3.0.1. - -When `--shard-count <#number of shards>` is used, the tests to execute -will be split evenly in to the given number of sets, identified by indices -starting at 0. The tests in the set given by -`--shard-index <#shard index to run>` will be executed. The default shard -count is `1`, and the default index to run is `0`. - -_Shard index must be less than number of shards. As the name suggests, -it is treated as an index of the shard to run._ - -Sharding is useful when you want to split test execution across multiple -processes, as is done with the [Bazel test sharding](https://docs.bazel.build/versions/main/test-encyclopedia.html#test-sharding). - - -<a id="no-tests-override"></a> -## Allow running the binary without tests -<pre>--allow-running-no-tests</pre> - -> Introduced in Catch2 3.0.1. - -By default, Catch2 test binaries return non-0 exit code if no tests were run, -e.g. if the binary was compiled with no tests, the provided test spec matched no -tests, or all tests [were skipped at runtime](skipping-passing-failing.md#top). This flag -overrides that, so a test run with no tests still returns 0. - -## Output verbosity -``` --v, --verbosity <quiet|normal|high> +[Test specs](#test-specs)<br> +[Selecting tests](#selecting-tests)<br> +[Running](#running)<br> +[How tests behave](#how-tests-behave)<br> +[History and analysis](#history-and-analysis)<br> +[Output](#output)<br> +[Information](#information)<br> +[Environment](#environment)<br> + +This is the command line of a test binary, which the +[Catch3 runner](progmasoft-runner.md#top) reads. `--help` prints it. + +Long options take their value after `=` or as the next argument: +`--jobs=4` and `--jobs 4` are the same. Short options take it attached or +after `=`: `-j4`, `-j=4`. An option the runner does not know is an error, +with a suggestion when it looks like a known one: + +```text +$ ./tests --jbos=4 +error: '--jbos=4' is not an option; did you mean --jobs? ``` -Changing verbosity might change how many details Catch2's reporters output. -However, you should consider changing the verbosity level as a _suggestion_. -Not all reporters support all verbosity levels, e.g. because the reporter's -format cannot meaningfully change. In that case, the verbosity level is -ignored. - -Verbosity defaults to _normal_. - - -## Create file to guard against silent early termination -<pre>--premature-exit-guard-file <path></pre> - -> Introduced in Catch2 3.11.0 - -Tells Catch2 to create an empty file at specified path before the tests -start, and delete it after the tests finish. If the file is present after -the process stops, it can be assumed that the testing binary exited -prematurely, e.g. due to the OOM killer. - -All directories in the path must already exist. If this option is used -and Catch2 cannot create the file (e.g. the location is not writable), -the test run will fail. - +`--` ends the options: what follows is test specs, even when it starts with +`-`. + + +## Test specs + +Arguments that are not options select tests by name or tag, as in Catch2: + +| Spec | Selects | +| --- | --- | +| `"parses lists"` | the test of that name | +| `"parses*"` | tests whose name starts with "parses"; `*` may stand at either end | +| `[parser]` | tests tagged `[parser]` | +| `[parser][fast]` | tests with both tags | +| `~[slow]` | every test but those tagged `[slow]` | +| `[parser],[lexer]` | tests with either tag | +| `"a b" [x]` | specs separated by spaces must all match | +| `[@alias]` | the tags of a [tag alias](test-cases-and-sections.md#tag-aliases) | + +Without a spec, every test runs except hidden ones, whose tags start with +`.`; a spec that names a hidden test runs it. A backslash makes `,`, `[`, +`]`, `*`, `~` and `"` literal: `"commas\, and more"`. + + +## Selecting tests + +| Option | | +| --- | --- | +| `-c`, `--section=NAME` | run only this section; repeat for nested sections | +| `-p`, `--path-filter=c:SECTION\|g:INDEX` | run one path through sections (`c:`) and generator values (`g:`, an index or `*`); repeat for each level. See [running a section](filtering-execution-path.md#top). | +| `-f`, `--input-file=FILE` | run the tests named in FILE, one exact name per line; lines starting with `#` are comments | +| `--select=WHICH` | `all` (default), `failed`, `changed`, `new` or `smart`; see [choosing tests](progmasoft-runner.md#choosing-and-ordering-tests) | +| `--changes-since=REVISION` | the Git revision `changed` and `smart` compare with (default `HEAD`) | +| `--incremental` | skip tests that passed last time and whose inputs did not change | +| `--shard=INDEX/COUNT` | run one of COUNT shards, counting from 0 | +| `--shard-durations=FILE` | balance shards by the durations in a JSON report or a history | +| `-#`, `--filenames-as-tags` | tag every test with `#` and the name of its file, as `[#parser.tests]` | + + +## Running + +| Option | | +| --- | --- | +| `-j`, `--jobs=N` | worker processes; 0, the default, uses one per hardware thread | +| `--in-process` | run the tests in this process, one after another | +| `--isolate` | start a fresh process for every test | +| `--timeout=SECONDS` | the limit for a test without history, and the cap of learned limits; 0 for none | +| `--no-adaptive-timeout` | use `--timeout` for every test | +| `--retries=N\|auto\|off` | `auto` (default) retries what may pass again; N retries every failure up to N times | +| `--repeat=N` | run every test N times, to find flaky tests | +| `-a`, `--fail-fast[=N]` | start no test after N failed tests (default 1) | +| `--order=ORDER` | `priority` (default), `declared`, `name` or `random` | +| `--seed=N\|random` | the random seed; a new one each run by default | +| `--no-bisect` | do not search for the test that leaves state behind | + + +## How tests behave + +| Option | | +| --- | --- | +| `-s`, `--success` | report passing assertions too, and show every assertion | +| `-b`, `--break` | break into the debugger on a failure; runs in process | +| `-e`, `--no-throw` | skip the assertions that expect exceptions | +| `-w`, `--warn=WHAT` | fail on `no-assertions`, `unmatched-specs` or `infinite-generators` | +| `--allow-no-tests` | succeed when no test runs | +| `--skip-benchmarks` | do not run `BENCHMARK` blocks | +| `--benchmark-samples=N` | samples per benchmark (default 100) | +| `--benchmark-resamples=N` | bootstrap resamples (default 100000) | +| `--benchmark-confidence-interval=X` | between 0 and 1 (default 0.95) | +| `--benchmark-no-analysis` | report the mean of the samples without statistics | +| `--benchmark-warmup-time=MILLISECONDS` | warm-up time per benchmark (default 100) | + + +## History and analysis + +| Option | | +| --- | --- | +| `--history=DIRECTORY` | where results are remembered (default `.catch3` beside the binary, or `CATCH3_HISTORY`) | +| `--no-history` | neither read nor write the history | +| `--quarantine=MODE` | `auto` (default), `report` or `off` | +| `--replay` | run last run's failures again, each in the order and with the seed it failed with | +| `--analytics[=json]` | print what the history says about the tests, and exit | + + +## Output + +| Option | | +| --- | --- | +| `-v`, `--verbose` | print every test as it ends, with what it printed | +| `-q`, `--quiet` | print only failures and the summary | +| `--color=WHEN` | `auto` (default), `always` or `never` | +| `--junit=FILE` | write a JUnit XML report | +| `--json=FILE` | write a JSON report with every attempt and its analysis | +| `--trace=FILE` | write a trace for `chrome://tracing` or Perfetto | +| `--slowest[=N]` | list the N slowest tests (default 10) | +| `--premature-exit-guard-file=FILE` | create FILE, private to the user, when the run starts and remove it when the run ends normally | + + +## Information + +| Option | | +| --- | --- | +| `-l`, `--list[=FORMAT]` | list the matching tests: `text` (default), `names` or `json` ([format](progmasoft-runner.md#formats-other-tools-read)) | +| `-t`, `--list-tags` | list the tags of the matching tests | +| `-h`, `--help` | print the options | +| `--version` | print the version of Catch3 | + + +## Environment + +| Variable | | +| --- | --- | +| `CATCH3_HISTORY` | the history directory | +| `CATCH3_PROPERTY_DATABASE` | the example database of [property tests](progmasoft-properties.md#top) | +| `CATCH3_WORKER` | set in each worker to its number | +| `TMPDIR`, `TMP`, `TEMP` | set while a test runs to its [own directory](progmasoft-runner.md#temporary-files) | +| Bazel's test variables | see [Bazel](progmasoft-runner.md#bazel-and-ctest) | --- diff --git a/docs/configuration.md b/docs/configuration.md index 080ab17d..3650c69c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -6,11 +6,8 @@ **Contents**<br> [Prefixing Catch macros](#prefixing-catch-macros)<br> -[Terminal colour](#terminal-colour)<br> -[Console width](#console-width)<br> [stdout](#stdout)<br> [Fallback stringifier](#fallback-stringifier)<br> -[Default reporter](#default-reporter)<br> [Bazel support](#bazel-support)<br> [C++11 toggles](#c11-toggles)<br> [C++17 toggles](#c17-toggles)<br> @@ -37,28 +34,6 @@ with the same name. To keep test code clean and uncluttered Catch uses short macro names (e.g. ```TEST_CASE``` and ```REQUIRE```). Occasionally these may conflict with identifiers from platform headers or the system under test. In this case the above identifier can be defined. This will cause all the Catch user macros to be prefixed with ```CATCH_``` (e.g. ```CATCH_TEST_CASE``` and ```CATCH_REQUIRE```). -## Terminal colour - - CATCH_CONFIG_COLOUR_WIN32 // Force enables compiling colouring impl based on Win32 console API - CATCH_CONFIG_NO_COLOUR_WIN32 // Force disables ... - -Yes, Catch2 uses the british spelling of colour. - -Catch2 attempts to autodetect whether the Win32 console colouring API, -`SetConsoleTextAttribute`, is available, and if it is available it compiles -in a console colouring implementation that uses it. - -This option can be used to override Catch2's autodetection and force the -compilation either ON or OFF. - - -## Console width - - CATCH_CONFIG_CONSOLE_WIDTH = x // where x is a number - -Catch formats output intended for the console to fit within a fixed number of characters. This is especially important as indentation is used extensively and uncontrolled line wraps break this. -By default a console width of 80 is assumed but this can be controlled by defining the above identifier to be a different value. - ## stdout CATCH_CONFIG_NOSTDOUT @@ -94,21 +69,11 @@ _Note that if the provided function does not handle a type and this type requires to be stringified, the compilation will fail._ -## Default reporter - -Catch's default reporter can be changed by defining macro -`CATCH_CONFIG_DEFAULT_REPORTER` to string literal naming the desired -default reporter. - -This means that defining `CATCH_CONFIG_DEFAULT_REPORTER` to `"console"` -is equivalent with the out-of-the-box experience. - - ## Bazel support -Compiling Catch2 with `CATCH_CONFIG_BAZEL_SUPPORT` force-enables Catch2's -support for Bazel's environment variables (normally Catch2 looks for -`BAZEL_TEST=1` env var first). +Compiling Catch2 with `CATCH_CONFIG_BAZEL_SUPPORT` makes the runner read +[Bazel's environment variables](progmasoft-runner.md#bazel-and-ctest) +always; otherwise it reads them when `BAZEL_TEST=1` is set. This can be useful if you are using older versions of Bazel, that do not yet have `BAZEL_TEST` env var support. diff --git a/docs/event-listeners.md b/docs/event-listeners.md index 6b9b6dc6..01d25365 100644 --- a/docs/event-listeners.md +++ b/docs/event-listeners.md @@ -4,14 +4,19 @@ # Event Listeners -An event listener is a bit like a reporter, in that it responds to various -reporter events in Catch2, but it is not expected to write any output. -Instead, an event listener performs actions within the test process, such -as performing global initialization (e.g. of a C library), or cleaning out -in-memory logs if they are not needed (the test case passed). - -Unlike reporters, each registered event listener is always active. Event -listeners are always notified before reporter(s). +An event listener responds to the events of a test run, but it is not +expected to write the run's output; that is the job of the +[reports](reporters.md#top). Instead, an event listener performs actions +within the test process, such as performing global initialization (e.g. of +a C library), or cleaning out in-memory logs if they are not needed (the +test case passed). + +Every registered event listener is always active. The +[Catch3 runner](progmasoft-runner.md#top) runs tests in worker processes, +and each worker is a test run of its own for its listeners: they are created +in every worker, and see `testRunStarting` when it starts and `testRunEnded` +when it ends. What a listener prints outside a test reaches the runner's +standard error. To write your own event listener, you should derive from `Catch::EventListenerBase`, as it provides empty stubs for all reporter events, allowing you to diff --git a/docs/filtering-execution-path.md b/docs/filtering-execution-path.md index b1d07433..17905049 100644 --- a/docs/filtering-execution-path.md +++ b/docs/filtering-execution-path.md @@ -11,9 +11,9 @@ sections and generator indices to run through. This is done by using one of the three commandline parameters, one or more times. ``` --c, --section <section name> --g, --generator-index <index in generator> --p, --path-filter <path filter spec> +-c, --section=NAME +-g, --generator-index=INDEX +-p, --path-filter=c:SECTION|g:INDEX ``` All the variants form a shared stack of filters, but if you use only @@ -125,8 +125,8 @@ TEST_CASE( "qux" ) { > The new behaviour was introduced in Catch2 3.13.0 ``` --g, --generator-index <index in generator> --p, --path-filter <path filter spec> +-g, --generator-index=INDEX +-p, --path-filter=c:SECTION|g:INDEX ``` The argument to `-g`/`--generator-index` must be either a non-negative diff --git a/docs/own-main.md b/docs/own-main.md index 3f8963b3..3acfd3c2 100644 --- a/docs/own-main.md +++ b/docs/own-main.md @@ -1,31 +1,25 @@ <a id="top"></a> - -> **Catch3 compatibility reference:** This page describes the bundled Catch2-compatible API. Names such as `Catch2`, `catch2/`, and `Catch2::Catch2WithMain` remain where they are actual compatibility interfaces. See [Catch3 additions](progmasoft-catch3.md) for Progmasoft APIs. - # Supplying main() yourself **Contents**<br> -[Let Catch2 take full control of args and config](#let-catch2-take-full-control-of-args-and-config)<br> -[Amending the Catch2 config](#amending-the-catch2-config)<br> -[Adding your own command line options](#adding-your-own-command-line-options)<br> +[Run the tests from your own main](#run-the-tests-from-your-own-main)<br> +[Options of your own](#options-of-your-own)<br> [Version detection](#version-detection)<br> -The easiest way to use Catch2 is to use its own `main` function, and let -it handle the command line arguments. This is done by linking against -Catch2Main library, e.g. through the [CMake target](cmake-integration.md#cmake-targets), -or pkg-config files. - -If you want to provide your own `main`, then you should link against -the static library (target) only, without the main part. You will then -have to write your own `main` and call into Catch2 test runner manually. +The easiest way to use Catch3 is its own `main`: link against +`Catch2::Catch2WithMain` (the [CMake target](cmake-integration.md#cmake-targets)) +or the `catch2-with-main` pkg-config file. To write your own `main`, link +against `Catch2::Catch2` and hand the command line to the +[Catch3 runner](progmasoft-runner.md#top). -Below are some basic recipes on what you can do supplying your own main. +The runner starts the same binary again for its worker processes, with the +same `main`. Whatever `main` does before and after `run`, it also does in +every worker. That is usually what global set-up needs; when only the +runner's own process should do something, check whether the environment +variable `CATCH3_WORKER` is set, which it is only in workers. -## Let Catch2 take full control of args and config - -This is useful if you just need to have code that executes before/after -Catch2 runs tests. +## Run the tests from your own main ```cpp #include <catch2/catch_session.hpp> @@ -41,81 +35,41 @@ int main( int argc, char* argv[] ) { } ``` -_Note that if you only want to run some set up before tests are run, it -might be simpler to use [event listeners](event-listeners.md#top) instead._ - - -## Amending the Catch2 config +`run` returns the [exit code](progmasoft-runner.md#exit-codes). A binary that +needs no `Catch::Session` may call `Progmasoft::Catch3::RunTests`, from +`<Progmasoft/Catch3/Runner.hpp>`, with the arguments as a +`std::vector<std::string>`. -If you want Catch2 to process command line arguments, but also want to -programmatically change the resulting configuration of Catch2 run, -you can do it in two ways: +_If you only want to run some set up before tests are run, it might be +simpler to use [event listeners](event-listeners.md#top) instead._ -```c++ -int main( int argc, char* argv[] ) { - Catch::Session session; // There must be exactly one instance - - // writing to session.configData() here sets defaults - // this is the preferred way to set them - - int returnCode = session.applyCommandLine( argc, argv ); - if( returnCode != 0 ) // Indicates a command line error - return returnCode; - // writing to session.configData() or session.Config() here - // overrides command line args - // only do this if you know you need to +## Options of your own - returnCode = session.run(); - - // returnCode encodes the type of error that occurred. See the - // integer constants in catch_session.hpp for more information - // on what each return code means. - - return numFailed; -} -``` - -If you want full control of the configuration, don't call `applyCommandLine`. - - -## Adding your own command line options - -You can add new command line options to Catch2, by composing the premade -CLI parser (called Clara), and add your own options. +Catch2's `Session::cli()` and `Session::configData()` went with Catch2's +runner. Take your own options out of the command line before passing the +rest on; the runner treats an option it does not know as an error. ```cpp int main( int argc, char* argv[] ) { - Catch::Session session; // There must be exactly one instance - - int height = 0; // Some user variable you want to be able to set - - // Build a new parser on top of Catch2's - using namespace Catch::Clara; - auto cli - = session.cli() // Get Catch2's command line parser - | Opt( height, "height" ) // bind variable to a new option, with a hint string - ["-g"]["--height"] // the option names it will respond to - ("how high?"); // description string for the help output - - // Now pass the new composite back to Catch2 so it uses that - session.cli( cli ); - - // Let Catch2 (using Clara) parse the command line - int returnCode = session.applyCommandLine( argc, argv ); - if( returnCode != 0 ) // Indicates a command line error - return returnCode; - - // if set on the command line then 'height' is now set at this point - if( height > 0 ) - std::cout << "height: " << height << std::endl; - - return session.run(); + int height = 0; + std::vector<char const*> arguments; + for ( int i = 0; i < argc; ++i ) { + std::string const argument = argv[i]; + if ( argument.rfind( "--height=", 0 ) == 0 ) { + height = std::atoi( argument.c_str() + 9 ); + } else { + arguments.push_back( argv[i] ); + } + } + + return Catch::Session().run( static_cast<int>( arguments.size() ), + arguments.data() ); } ``` -See the [Clara documentation](https://github.com/catchorg/Clara/blob/master/README.md) -for more details on how to use the Clara parser. +Workers are started with the runner's own options. A value the tests need +in every worker belongs in the environment, which workers inherit. ## Version detection @@ -128,7 +82,9 @@ Catch2 provides a triplet of macros providing the header's version, these macros expand into a single number, that corresponds to the appropriate part of the version. As an example, given single header version v2.3.4, -the macros would expand into `2`, `3`, and `4` respectively. +the macros would expand into `2`, `3`, and `4` respectively. The version of +the Catch3 runner is `Progmasoft::Catch3::kVersion` in +`<Progmasoft/Catch3/Version.hpp>`, and `--version` prints it. --- diff --git a/docs/progmasoft-catch3.md b/docs/progmasoft-catch3.md index 30ea7a15..28d54fd1 100644 --- a/docs/progmasoft-catch3.md +++ b/docs/progmasoft-catch3.md @@ -1,10 +1,13 @@ # Progmasoft Catch3 additions -Progmasoft Catch3 is a compatibility-preserving extension of Catch2, not a -replacement runner. The upstream `Catch2` API, command-line behavior, include -paths, and BSL-1.0 files remain the compatibility foundation. Progmasoft-owned -headers use the `Progmasoft/Catch3` include tree and `Progmasoft::Catch3` -namespace. +Progmasoft Catch3 keeps Catch2's API for writing tests and replaces Catch2's +runner with its own. Tests, `catch2/` include paths, `Catch2::` CMake targets +and the BSL-1.0 files of the engine stay as they are; the +[Catch3 runner](progmasoft-runner.md#top) runs them in parallel worker +processes and adds retries, flaky-test detection, quarantine, history, +predictive ordering, test selection, sharding, time limits and failure +analysis. Progmasoft-owned headers use the `Progmasoft/Catch3` include tree +and `Progmasoft::Catch3` namespace. ## Property testing @@ -146,24 +149,29 @@ CATCH3_CHECK_SNAPSHOT("rendered-report", RenderReport(), options); ## Structured results and XML -`<Progmasoft/Catch3/Results.hpp>` defines a runner-neutral hierarchy of -`TestRunResult`, `TestSuiteResult`, and `TestCaseResult`. `SummarizeTestRun` -validates finite, non-negative durations and computes pass/failure/error/skip -counts. `<Progmasoft/Catch3/XmlWriter.hpp>` serializes that model as deterministic -JUnit-style XML through `JUnitXmlWriter::Write` or `ToString`. It escapes XML -markup, replaces malformed UTF-8 and XML-forbidden control characters, and -omits timestamps so identical results produce identical output. Catch2's own -`--reporter xml` and `--reporter junit` remain the direct reporters for results -produced by its runner; this writer is for Progmasoft's explicit result model. - -The snapshot and result/XML APIs have compiled C++ implementations in the -`Progmasoft::Catch3` library target. Generator templates and convenience macros +`<Progmasoft/Catch3/Results.hpp>` defines the model the runner's reports are +written from: a `RunResult` holds a `TestResult` per test, with its verdict, +failure category, history classification, flakiness, quarantine, failure +signature, analysis and replay command, and every `Attempt` with its +assertions, sections, benchmarks, captured output and the tests that ran +before it in the same process. `Summarize` counts the verdicts, and +`DescribeFailure` writes a failed attempt as text. +`<Progmasoft/Catch3/XmlWriter.hpp>` writes a run as deterministic JUnit XML +through `JUnitXmlWriter::Write` or `ToString`, with `<rerunFailure>` and +`<flakyFailure>` for retries. It escapes XML markup, replaces malformed UTF-8 +and XML-forbidden control characters, and omits timestamps so identical +results produce identical output. + +The results, the JUnit writer and the runner are compiled into the +`Catch2::Catch2` library; the snapshot API is compiled into the +`Progmasoft::Catch3` target. Generator templates and convenience macros remain in headers where their types or call-site expansion require it. ## Build and integration The CMake build exposes `Progmasoft::Catch3`, a compiled C++20 library linked to -`Catch2::Catch2`. The existing Catch2 target retains its C++14 baseline. After +`Catch2::Catch2`. `Catch2::Catch2`, which holds the runner, is built as C++20; +code that includes only Catch2's headers still needs only C++14. After installation, consumers can use: ```cmake diff --git a/docs/progmasoft-roadmap.md b/docs/progmasoft-roadmap.md index 4b18a6c7..20b88dd4 100644 --- a/docs/progmasoft-roadmap.md +++ b/docs/progmasoft-roadmap.md @@ -6,9 +6,11 @@ unit testing, property testing, mocking, coverage, mutation testing, test generation and test selection, driven by a runner of its own. Two rules hold for every step: -- **Catch2 stays compatible.** The `Catch` namespace, the `catch2/` headers, - the macros, the command line and the reporters keep working unchanged. New - capabilities live under `Progmasoft::Catch3` and `Progmasoft/Catch3/`. +- **Catch2's API stays compatible.** The `Catch` namespace, the `catch2/` + headers, the macros, matchers, generators and listeners keep working + unchanged. The runner is Catch3's own: Catch2's session, command line and + reporters were replaced. New capabilities live under `Progmasoft::Catch3` + and `Progmasoft/Catch3/`. - **Nothing is claimed before it is tested.** A feature is listed as done when its tests run in CI on Windows, Linux and macOS. @@ -17,29 +19,35 @@ The steps are ordered by what each one builds on. | Step | Capability | State | | --- | --- | --- | | 1 | Native property testing: choice-sequence shrinking, combinators, state machines, statistics, targeted search, example database, differential and metamorphic oracles | done; see [Native property testing](progmasoft-properties.md#top) | -| 2 | The `catch3` runner | planned | +| 2 | The Catch3 runner | done; see [The Catch3 runner](progmasoft-runner.md#top) | | 3 | Mocking | planned | | 4 | Per-test code coverage | planned | | 5 | Change-impact test selection and test prioritization | planned | | 6 | Mutation testing | planned | | 7 | Coverage-, mutation- and history-guided test generation | planned | -## 2. The `catch3` runner +## 2. The Catch3 runner -Catch2's runner executes the tests of one binary in one process. The features -that follow need more: they run many binaries, each test in isolation, under +Catch2's runner executed the tests of one binary in one process. The +features that follow need more: they run each test in isolation, under coverage or mutation, and decide which tests to run and in which order. The -`catch3` runner is a separate program that drives Catch2 test binaries from -the outside, so existing binaries need no change: - -- it discovers test cases with `--list-tests --reporter json` and runs them - with `--reporter json`, both stable parts of Catch2's command line; -- it runs tests in parallel processes, shards them across machines, enforces - per-test timeouts, isolates crashes, and retries flaky tests with a record - of every attempt; -- it keeps a history of results and durations, the input for prioritization; -- it writes JUnit XML, JSON and a terminal report, and keeps Catch2's exit - codes; +Catch3 runner replaced Catch2's in every test binary: + +- tests run in parallel worker processes that take the next test when they + are free; a crash or hang ends one test, not the run; +- failures are retried alone, which tells deterministic, flaky and + order-dependent failures apart, and the test that leaves state behind is + found by bisection; +- a history of verdicts and durations drives quarantine, adaptive time + limits, predictive ordering, incremental runs, replay and analytics; +- tests are selected by change, by past failure or by prediction, and + sharded by a stable hash or by duration; +- tags declare dependencies, shared resources, serial tests, time limits and + retries; +- every failure gets a category, a signature and an analysis; +- reports are console, JUnit XML with flaky and rerun elements, JSON and a + trace; the discovery listing has a fixed, versioned format; +- every test gets a temporary directory of its own; - its command line is parsed by a hand-written, typed parser, like the other Progmasoft tools. diff --git a/docs/progmasoft-runner.md b/docs/progmasoft-runner.md new file mode 100644 index 00000000..1c5d0efe --- /dev/null +++ b/docs/progmasoft-runner.md @@ -0,0 +1,324 @@ +<a id="top"></a> +# The Catch3 runner + +**Contents**<br> +[How a run works](#how-a-run-works)<br> +[Retries, flaky tests and order dependence](#retries-flaky-tests-and-order-dependence)<br> +[History, quarantine and analytics](#history-quarantine-and-analytics)<br> +[Choosing and ordering tests](#choosing-and-ordering-tests)<br> +[Tags the runner reads](#tags-the-runner-reads)<br> +[Time limits](#time-limits)<br> +[Temporary files](#temporary-files)<br> +[What a failure report says](#what-a-failure-report-says)<br> +[Reports and traces](#reports-and-traces)<br> +[Formats other tools read](#formats-other-tools-read)<br> +[Bazel and CTest](#bazel-and-ctest)<br> +[Exit codes](#exit-codes)<br> +[Coming from Catch2's runner](#coming-from-catch2s-runner)<br> + +Every Catch3 test binary has the runner built in. It replaced Catch2's +session, command line and reporters; the API tests are written with stays +Catch2's: `TEST_CASE`, `SECTION`, the assertions, matchers, generators, +benchmarks, fixtures and listeners. The options are listed in the +[command line reference](command-line.md#top). + +```text +$ ./tests +Catch3: 446 test(s), up to 8 worker(s), seed 2153251184 + +446 test(s): 423 passed, 16 failed as expected, 7 skipped +84193 assertion(s), 446 attempt(s); 1.2 s (6.0 s of tests, 5.0x in parallel) +``` + + +## How a run works + +The runner reads the registered tests, decides which to run and in which +order, and hands them to worker processes: the same binary, started again +with an internal option. A free worker takes the next test, so the work +spreads by itself and a long test does not hold back a shard of short ones. +`--jobs=N` sets the number of workers; the default is one per hardware +thread. + +Running tests in their own processes gives the run what one process +cannot: + +* **A crash ends the test, not the run.** A worker that dies is replaced; + the test it was running is reported as crashed, with the signal or + structured exception, the last assertion it started and the section it + was in. +* **A hang is stopped.** A test that runs past its [time limit](#time-limits) + is ended together with every process it started, and reported as timed + out, with the section it was in. +* **Tests that need it run alone.** `--isolate`, or the tag `[!isolated]`, + gives a test a fresh process. + +The runner, its workers and every process a test starts form one group: a +job object on Windows, a process group elsewhere. Stopping a worker stops +all of it. + +What a test writes to `std::cout`, `std::cerr` and `std::clog` is captured +and belongs to the test. Output of a worker outside any test, such as what a +listener prints when the run starts, is passed on to the runner's standard +error. + +`--in-process` runs the tests in the runner's own process, one after +another, for a debugger. `--break` implies it. Crashes and hangs are then not +contained. + +Listeners registered with `CATCH_REGISTER_LISTENER` run in every worker: +each worker is one test run for them, from `testRunStarting` to +`testRunEnded`. + + +## Retries, flaky tests and order dependence + +With `--retries=auto`, the default, a failed test is tried once more, alone +in a fresh process. What happens then tells three kinds of failure apart: + +* It fails again: the failure is deterministic, and is reported. +* It passes, and it had failed alone too: the test is **flaky**. It counts as + passed, is marked flaky in every report, and its history remembers it. +* It passes alone, but had failed after other tests in a shared process: + the runner runs the same tests again in a fresh process, in the same + order. If the test fails again, it depends on state another test leaves + behind, and the runner bisects the tests that ran before it to find which + one. The report names it: *fails after 'X', which leaves state behind*. + Such a failure fails the run; `--no-bisect` turns the search off. + +Crashes, timeouts, tests in quarantine and tests with a flaky history get up +to two retries. `--retries=N` retries every failure up to N times, +`--retries=off` never retries, and `[!retries:N]` sets the limit of one +test. A run spends at most a tenth of its tests on retries, so a broken +build does not run everything three times. + +`--repeat=N` runs every test N times; a test whose verdicts differ is flaky. + + +## History, quarantine and analytics + +The runner remembers the results of the last 50 runs of each test: verdict, +duration, flakiness, failure signature and a fingerprint of the inputs. The +history of a binary lives in one JSON file in `.catch3` beside the binary, +or in the directory `--history` or the environment variable `CATCH3_HISTORY` +names. `--no-history` neither reads nor writes it. Runs that finish at the +same time all reach the file: it is updated under a lock, read again, and +replaced in one rename. + +The history drives several features: + +* **Adaptive time limits.** See [time limits](#time-limits). +* **Predictive ordering.** Each test gets a probability of failing: its + failure rate, with recent runs weighing more, and at least 70% after a + failure. Tests that are likely to fail run first, the most failures per + second of running time first, so a failing run fails early. The rest run + longest first, which keeps every worker busy until the end. +* **Quarantine.** A test with two flaky results in its last 20 runs goes + into quarantine. It still runs and is reported, but its failure does not + fail the run. Ten clean passes in a row release it. `[!quarantine]` + quarantines a test by hand; `--quarantine=report` reports quarantine + without excusing failures, and `--quarantine=off` ignores it. +* **Classification.** A failure is *new* (it passed last time), + *persistent* (it failed last time too) or *intermittent* (it flakes). +* **Replay.** `--replay` runs the last failures again, each with the seed it + failed with and after the same tests, in a fresh process. +* **Incremental runs.** `--incremental` skips tests that passed last time + with the same fingerprint; they are reported as up to date. +* **Analytics.** `--analytics` prints what the history says: flaky tests, + tests in quarantine, the tests that fail most often, the slowest tests and + the tests getting slower. `--analytics=json` prints it as JSON. + +Under `bazel test` there is no history unless `CATCH3_HISTORY` names a +directory that outlives the sandbox. + + +## Choosing and ordering tests + +Test specs work as in Catch2: names with `*` as a wildcard, `[tags]`, `~` +to exclude, a comma for "or". `--input-file` reads exact names from a file. +`--select` narrows the matching tests further: + +| `--select=` | Runs | +| --- | --- | +| `all` | every matching test (default) | +| `failed` | tests that failed, crashed, timed out or flaked last time | +| `changed` | tests defined in files Git reports as changed since `--changes-since` (default `HEAD`), staged or not, and untracked files | +| `new` | tests without a recorded result | +| `smart` | changed, failed and new tests, those in quarantine, and those at least 50% likely to fail | + +Each selected test carries the reason it was selected into the reports. + +`--order=priority` (default) orders by the prediction above; `declared`, +`name` and `random` are also available. Every test is seeded with the +run's seed, wherever it runs, so the order does not change its random +numbers. + +`--shard=INDEX/COUNT` runs one of COUNT shards. A test goes to the shard a +stable hash of its name chooses, so every machine agrees without talking to +the others; tests linked by `[!depends]` stay in one shard. With +`--shard-durations=FILE`, a history or a JSON report, shards are balanced by +duration instead, longest first into the least loaded shard. `--list` +honours `--shard`. + + +## Tags the runner reads + +Tags that start with `!` are reserved, as in Catch2. The runner adds these: + +| Tag | Effect | +| --- | --- | +| `[!serial]` | nothing else runs while this test runs | +| `[!isolated]` | the test runs in a process of its own | +| `[!resource:NAME]` | tests that share a resource never run at the same time | +| `[!depends:TEST]` | runs after TEST, and only if TEST passes; otherwise it is reported as blocked. TEST is selected along with it. Cycles are an error. | +| `[!timeout:SECONDS]` | the time limit of this test | +| `[!retries:N]` | the retry limit of this test | +| `[!quarantine]` | the test is in quarantine | + +```cpp +TEST_CASE("migrates the schema", "[db][!resource:database]") { ... } +TEST_CASE("reads after migration", "[db][!resource:database][!depends:migrates the schema]") { ... } +TEST_CASE("renders the full catalogue", "[!timeout:120]") { ... } +``` + + +## Time limits + +A test with history gets a limit learned from it: four times its 95th +percentile or ten times its median, whichever is larger, plus two seconds, +and at least five seconds. A test without history gets 300 seconds. +`--timeout=SECONDS` sets the limit for tests without history and caps the +learned ones; `--timeout=0` turns limits off; `--no-adaptive-timeout` uses +`--timeout` for every test. `[!timeout:SECONDS]` overrides all of these. +Benchmarks get no learned limit. + + +## Temporary files + +Every attempt of every test gets a temporary directory of its own. While the +test runs, `TMPDIR`, `TMP` and `TEMP` point at it, so +`std::filesystem::temp_directory_path()`, `tmpfile` and most libraries use +it, and tests that run at the same time never share a scratch file by +accident. `Progmasoft::Catch3::TestTemporaryDirectory()` in +`<Progmasoft/Catch3/Runner.hpp>` returns it. + +The directory is removed when the attempt passes and kept when it fails; +the report says where. + + +## What a failure report says + +```text +FAIL: parses nested lists +tests/parser.cpp:120 + tests/parser.cpp:131: FAILED: + REQUIRE( tokens.size() == 3 ) + with expansion: + 2 == 3 + in section with commas + analysis: new failure after 14 passing run(s); it last passed at commit 4c40a97b2f1e + analysis: fails the same way as 'parses flat lists': probably one cause + analysis: changed since HEAD: src/lexer.cpp + replay: ./tests "parses nested lists" --seed=2153251184 +``` + +Every failure has a **category**: assertion, unexpected exception, missing +exception, explicit failure, crash, timeout, dependency or infrastructure. +It has a **signature**: the category, the location of the first failure and +its message with numbers and addresses removed. Tests that share a +signature are named together, since they probably share a cause. + +The analysis adds what the runner knows: how the failure relates to the +history, the commit the test last passed at, the change it follows (the +test's file, or the file of the failing assertion, changed since +`--changes-since`), where a crash or a hang happened, the test that leaves +state behind, the quarantine, and where the temporary files were kept. The +replay line runs the failure again as it happened. + + +## Reports and traces + +* The **console report** shows progress and the failures with their + analysis, then a summary. `--verbose` shows every test and what passing + tests printed; `--success` also shows every assertion; `--quiet` shows + only failures. `--slowest=N` lists the slowest tests. +* `--junit=FILE` writes JUnit XML. Retries are `<rerunFailure>` elements, + and a flaky test is a success with `<flakyFailure>` elements, which + Jenkins, GitLab and most CI dashboards understand. +* `--json=FILE` writes everything the runner knows: every attempt, its + assertions, sections and benchmarks, the captured output, the analysis. +* `--trace=FILE` writes a trace for `chrome://tracing` or + [Perfetto](https://ui.perfetto.dev): one track per worker, one slice per + attempt and section. + + +## Formats other tools read + +These formats carry a `format` and a `version`. Within a version they only +gain members; a reader ignores members it does not know and refuses a newer +version. + +* **Discovery**, `--list=json`: `"format": "catch3-tests"`, version 1. One + object per test with `id`, `name`, `className`, `file`, `line`, `tags`, + `hidden`, and what the runner tags say: `serial`, `isolated`, + `quarantined`, `resources`, `dependsOn`, `timeoutSeconds` and `retries`. + CMake's `catch_discover_tests` reads it. +* **Run report**, `--json`: `"format": "catch3-run"`, version 1. +* **Analytics**, `--analytics=json`: `"format": "catch3-analytics"`, + version 1. +* **History**: `"format": "catch3-history"`, version 1. It is the runner's + own file; read it through `--analytics`. + +The messages between the runner and its workers are internal and change +with the binary. + + +## Bazel and CTest + +Under `bazel test` the runner reads Bazel's environment: `XML_OUTPUT_FILE` +for a JUnit report, `TESTBRIDGE_TEST_ONLY` for `--test_filter`, +`TEST_SHARD_INDEX`, `TEST_TOTAL_SHARDS` and `TEST_SHARD_STATUS_FILE` for +sharding, `TEST_RANDOM_SEED` for the seed, `TEST_PREMATURE_EXIT_FILE` for +the exit guard and `TEST_UNDECLARED_OUTPUTS_DIR` for a JSON report. + +[`catch_discover_tests`](cmake-integration.md#automatic-test-registration) +registers each test with CTest through the discovery format. The runner +also parallelises a whole binary under one CTest test, which usually starts +fewer processes. + + +## Exit codes + +| Code | Meaning | +| --- | --- | +| 0 | every test passed or failed as allowed by `[!mayfail]` or `[!shouldfail]`; flaky tests and failures in quarantine do not fail the run | +| 1 | the command line was wrong, the tests could not be registered, or workers could not start | +| 2 | no test ran (`--allow-no-tests` accepts this) | +| 3 | a test spec matched nothing and `--warn=unmatched-specs` was given | +| 4 | every test that ran was skipped | +| 5 | a test spec is invalid | +| 42 | a test failed, crashed, timed out or was blocked | + + +## Coming from Catch2's runner + +The test-writing API is unchanged. The runner is new: + +* Options are Catch3's own; `--help` lists them. Long options take their + value after `=` or as the next argument. `-s`, `-b`, `-e`, `-a`, `-c`, + `-p`, `-f`, `-#` and `-w` keep their meaning, under the names in the + [reference](command-line.md#top). +* Reporters and `CATCH_REGISTER_REPORTER` are gone. The console, JUnit and + JSON reports replace the built-in reporters; for anything else, read the + JSON report, or write a listener, which still runs in every worker. +* `Catch::Session` remains for a `main` of your own: `run(argc, argv)` hands + the command line to the runner. `Session::cli()` and `configData()` are + gone with Clara. See [supplying main yourself](own-main.md#top). +* `main` runs in every worker too, since workers are the same binary. +* Tests run in parallel by default. Tests that share files or other state + outside the process need `[!serial]` or `[!resource:NAME]`; per-test + temporary directories cover the common case. + +--- + +[Home](Readme.md#top) diff --git a/docs/reporter-events.md b/docs/reporter-events.md index 70548c79..45ecaaa0 100644 --- a/docs/reporter-events.md +++ b/docs/reporter-events.md @@ -7,20 +7,20 @@ **Contents**<br> [Test running events](#test-running-events)<br> [Benchmarking events](#benchmarking-events)<br> -[Listings events](#listings-events)<br> [Miscellaneous events](#miscellaneous-events)<br> Reporter events are one of the customization points for user code. They -are used by [reporters](reporters.md#top) to customize Catch2's output, -and by [event listeners](event-listeners.md#top) to perform in-process -actions under some conditions. +are used by [event listeners](event-listeners.md#top) to perform in-process +actions under some conditions. The reporters of Catch2 that also received +them were replaced by the [reports](reporters.md#top) of the Catch3 runner. -There are currently 21 reporter events in Catch2, split between 4 distinct -event groups: +There are 17 events, split between 3 event groups: * test running events (10 events) * benchmarking (4 events) -* listings (3 events) -* miscellaneous (4 events) +* miscellaneous (3 events) + +Each worker process of the [Catch3 runner](progmasoft-runner.md#top) is a +test run of its own: its listeners see the events of the tests it runs. ## Test running events @@ -41,9 +41,9 @@ void testRunStarting( TestRunInfo const& testRunInfo ); void testRunEnded( TestRunStats const& testRunStats ); ``` -The `testRun` events bookend the entire test run. `testRunStarting` is -emitted before the first test case is executed, and `testRunEnded` is -emitted after all the test cases have been executed. +The `testRun` events bookend the test run of one process. `testRunStarting` +is emitted before the first test case the process executes, and +`testRunEnded` after the last. ### `testCase` events @@ -136,23 +136,6 @@ and contains the benchmarking results. benchmarking itself fails. -## Listings events - -> Introduced in Catch2 3.0.1. - -Listings events are events that correspond to the test binary being -invoked with `--list-foo` flag. - -There are currently 3 listing events, one for reporters, one for tests, -and one for tags. Note that they are not exclusive to each other. - -```cpp -void listReporters( std::vector<ReporterDescription> const& descriptions ); -void listTests( std::vector<TestCaseHandle> const& tests ); -void listTags( std::vector<TagInfo> const& tagInfos ); -``` - - ## Miscellaneous events ```cpp @@ -163,15 +146,14 @@ void noMatchingTestCases( StringRef unmatchedSpec ); These are one-off events that do not neatly fit into other categories. -`reportInvalidTestSpec` is sent for each [test specification command line -argument](command-line.md#specifying-which-tests-to-run) that wasn't -parsed into a valid spec. +`reportInvalidTestSpec` and `noMatchingTestCases` remain for listeners +written against Catch2. The runner reports invalid and unmatched [test +specs](command-line.md#test-specs) itself, before any worker starts, so +listeners do not receive these two. `fatalErrorEncountered` is sent when Catch2's POSIX signal handling or Windows SE handler is called into with a fatal signal/exception. -`noMatchingTestCases` is sent for each user provided test specification -that did not match any registered tests. --- diff --git a/docs/reporters.md b/docs/reporters.md index 9ddba378..0be39804 100644 --- a/docs/reporters.md +++ b/docs/reporters.md @@ -1,220 +1,57 @@ <a id="top"></a> +# Reports -> **Catch3 compatibility reference:** This page describes the bundled Catch2-compatible API. Names such as `Catch2`, `catch2/`, and `Catch2::Catch2WithMain` remain where they are actual compatibility interfaces. See [Catch3 additions](progmasoft-catch3.md) for Progmasoft APIs. +**Contents**<br> +[The reports](#the-reports)<br> +[Custom output](#custom-output)<br> +[Where Catch2's reporters went](#where-catch2s-reporters-went)<br> -# Reporters +The [Catch3 runner](progmasoft-runner.md#top) writes its own reports. Catch2's +reporters, and `CATCH_REGISTER_REPORTER`, were removed with Catch2's runner: +a reporter sees one process, and a Catch3 run spreads over many. -Reporters are a customization point for most of Catch2's output, e.g. -formatting and writing out [assertions (whether passing or failing), -sections, test cases, benchmarks, and so on](reporter-events.md#top). -Catch2 comes with a bunch of reporters by default (currently 9), and -you can also write your own reporter. Because multiple reporters can -be active at the same time, your own reporters do not even have to handle -all reporter event, just the ones you are interested in, e.g. benchmarks. +## The reports +* **Console.** Progress while the run goes on, each failure with its + expansion, messages, sections and [analysis](progmasoft-runner.md#what-a-failure-report-says), + then a summary. `--verbose`, `--success`, `--quiet`, `--slowest` and + `--color` adjust it. +* **JUnit XML**, `--junit=FILE`. One `<testcase>` per test; a failure is + `<failure>`, a crash or timeout `<error>`, a skipped, blocked or not-run + test `<skipped>`. Earlier failed attempts are `<rerunFailure>` and, for a + test that passed in the end, `<flakyFailure>`. Under Bazel it is written + to `XML_OUTPUT_FILE`. +* **JSON**, `--json=FILE`. Everything the runner knows about the run, in the + `catch3-run` [format](progmasoft-runner.md#formats-other-tools-read). +* **Trace**, `--trace=FILE`, in the Trace Event Format. -## Using different reporters +`<Progmasoft/Catch3/Results.hpp>` holds the model the reports are written +from, and `<Progmasoft/Catch3/XmlWriter.hpp>` the JUnit writer, for tools +that build a `RunResult` themselves. -You can see which reporters are available by running the test binary -with `--list-reporters`. You can then pick one of them with the [`-r`, -`--reporter` option](command-line.md#choosing-a-reporter-to-use), followed -by the name of the desired reporter, like so: -``` ---reporter xml -``` +## Custom output -You can also select multiple reporters to be used at the same time. -In that case you should read the [section on using multiple -reporters](#multiple-reporters) to avoid any surprises from doing so. +For output of your own: +* Read the JSON report. It is the complete record of a run, and its format + is versioned. +* Write a [listener](event-listeners.md#top). Listeners receive Catch2's + events in every worker process; what one prints outside a test reaches + the runner's standard error. -<a id="multiple-reporters"></a> -## Using multiple reporters - -> Support for having multiple parallel reporters was [introduced](https://github.com/catchorg/Catch2/pull/2183) in Catch2 3.0.1 - -Catch2 supports using multiple reporters at the same time while having -them write into different destinations. The two main uses of this are - -* having both human-friendly and machine-parseable (e.g. in JUnit format) - output from one run of binary -* having "partial" reporters that are highly specialized, e.g. having one - reporter that writes out benchmark results as markdown tables and does - nothing else, while also having standard testing output separately - -Specifying multiple reporter looks like this: -``` ---reporter JUnit::out=result-junit.xml --reporter console::out=-::colour-mode=ansi::verbosity=quiet -``` - -This tells Catch2 to use two reporters, `JUnit` reporter that writes -its machine-readable XML output to file `result-junit.xml`, and the -`console` reporter that writes its user-friendly output to stdout, uses -ANSI colour codes for colouring the output and is set to "quiet" verbosity. - -Using multiple reporters (or one reporter and one-or-more [event -listeners](event-listeners.md#top)) can have surprisingly complex semantics -when using customization points provided to reporters by Catch2, namely -capturing stdout/stderr from test cases. - -As long as at least one reporter (or listener) asks Catch2 to capture -stdout/stderr, captured stdout and stderr will be available to all -reporters and listeners. - -Because this might be surprising to the users, if at least one active -_reporter_ is non-capturing, then Catch2 tries to roughly emulate -non-capturing behaviour by printing out the captured stdout/stderr -just before `testCasePartialEnded` event is sent out to the active -reporters and listeners. This means that stdout/stderr is no longer -printed out from tests as it is being written, but instead it is written -out in batch after each runthrough of a test case is finished. - - - -## Writing your own reporter - -You can also write your own custom reporter and tell Catch2 to use it. -When writing your reporter, you have two options: - -* Derive from `Catch::ReporterBase`. When doing this, you will have - to provide handling for all [reporter events](reporter-events.md#top). -* Derive from one of the provided [utility reporter bases in - Catch2](#utility-reporter-bases). - -Generally we recommend doing the latter, as it is less work. - -Apart from overriding handling of the individual reporter events, reporters -have access to some extra customization points, described below. - - -### Utility reporter bases - -Catch2 currently provides two utility reporter bases: - -* `Catch::StreamingReporterBase` -* `Catch::CumulativeReporterBase` - -`StreamingReporterBase` is useful for reporters that can format and write -out the events as they come in. It provides (usually empty) implementation -for all reporter events, and if you let it handle the relevant events, -it also handles storing information about active test run and test case. - -`CumulativeReporterBase` is a base for reporters that need to see the whole -test run, before they can start writing the output, such as the JUnit -and SonarQube reporters. This post-facto approach requires the assertions -to be stringified when it is finished, so that the assertion can be written -out later. Because the stringification can be expensive, and not all -cumulative reporters need the assertions, this base provides customization -point to change whether the assertions are saved or not, separate for -passing and failing assertions. - - -_Generally we recommend that if you override a member function from either -of the bases, you call into the base's implementation first. This is not -necessary in all cases, but it is safer and easier._ - - -Writing your own reporter then looks like this: - -```cpp -#include <catch2/reporters/catch_reporter_streaming_base.hpp> -#include <catch2/catch_test_case_info.hpp> -#include <catch2/reporters/catch_reporter_registrars.hpp> - -#include <iostream> - -class PartialReporter : public Catch::StreamingReporterBase { -public: - using StreamingReporterBase::StreamingReporterBase; - - static std::string getDescription() { - return "Reporter for testing TestCasePartialStarting/Ended events"; - } - - void testCasePartialStarting(Catch::TestCaseInfo const& testInfo, - uint64_t partNumber) override { - std::cout << "TestCaseStartingPartial: " << testInfo.name << '#' << partNumber << '\n'; - } - - void testCasePartialEnded(Catch::TestCaseStats const& testCaseStats, - uint64_t partNumber) override { - std::cout << "TestCasePartialEnded: " << testCaseStats.testInfo->name << '#' << partNumber << '\n'; - } -}; - - -CATCH_REGISTER_REPORTER("partial", PartialReporter) -``` - -This create a simple reporter that responds to `testCasePartial*` events, -and calls itself "partial" reporter, so it can be invoked with -`--reporter partial` command line flag. - - -### `ReporterPreferences` - -Each reporter instance contains instance of `ReporterPreferences`, a type -that holds flags for the behaviour of Catch2 when this reporter run. -Currently there are three customization options: - -* `shouldRedirectStdOut` - whether the reporter wants to handle - writes to stdout/stderr from user code, or not. This is useful for - reporters that output machine-parseable output, e.g. the JUnit - reporter, or the XML reporter. -* `shouldReportAllAssertions` - whether the reporter wants to handle - `assertionEnded` events for passing assertions as well as failing - assertions. Usually reporters do not report successful assertions - and don't need them for their output, but sometimes the desired output - format includes passing assertions even without the `-s` flag. -* `shouldReportAllAssertionStarts` - whether the reporter wants to handle - `assertionStarting` events. Most reporters do not, and opting out - explicitly enables a fast-path in Catch2's handling of assertions. - -> `shouldReportAllAssertionStarts` was introduced in Catch2 3.9.0 - - -### Per-reporter configuration - -> Per-reporter configuration was introduced in Catch2 3.0.1 - -Catch2 supports some configuration to happen per reporter. The configuration -options fall into one of two categories: - -* Catch2-recognized options -* Reporter-specific options - -The former is a small set of universal options that Catch2 handles for -the reporters, e.g. output file or console colour mode. The latter are -options that the reporters have to handle themselves, but the keys and -values can be arbitrary strings, as long as they don't contain `::`. This -allows writing reporters that can be significantly customized at runtime. - -Reporter-specific options always have to be prefixed with "X" (large -letter X). - - -### Other expected functionality of a reporter - -When writing a custom reporter, there are few more things that you should -keep in mind. These are not important for correctness, but they are -important for the reporter to work _nicely_. - -* Catch2 provides a simple verbosity option for users. There are three - verbosity levels, "quiet", "normal", and "high", and if it makes sense - for reporter's output format, it should respond to these by changing - what, and how much, it writes out. - -* Catch2 operates with an rng-seed. Knowing what seed a test run had - is important if you want to replicate it, so your reporter should - report the rng-seed, if at all possible given the target output format. - -* Catch2 also operates with test filters, or test specs. If a filter - is present, you should also report the filter, if at all possible given - the target output format. +## Where Catch2's reporters went +| Catch2 | Catch3 | +| --- | --- | +| `console`, `compact` | the console report | +| `junit` | `--junit=FILE` | +| `json`, `xml` | `--json=FILE` | +| `sonarqube`, `tap`, `teamcity`, `automake` | convert the JSON report, or a listener | +| `--reporter NAME::out=FILE` | `--junit=FILE`, `--json=FILE` | +| `CATCH_REGISTER_REPORTER` | a listener, or a tool reading the JSON report | --- diff --git a/docs/skipping-passing-failing.md b/docs/skipping-passing-failing.md index b35fc29d..013199d5 100644 --- a/docs/skipping-passing-failing.md +++ b/docs/skipping-passing-failing.md @@ -98,7 +98,7 @@ case the test is handled as _failed_ instead). Note that if all test cases in a run are skipped, Catch2 returns a non-zero exit code, same as it does if no test cases have run. This behaviour can -be overridden using the [--allow-running-no-tests](command-line.md#no-tests-override) +be overridden using the [--allow-no-tests](command-line.md#how-tests-behave) flag. ### `SKIP` inside generators diff --git a/docs/tutorial.md b/docs/tutorial.md index 03ca028b..fb316c10 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -48,7 +48,7 @@ TEST_CASE( "Factorials are computed", "[factorial]" ) { } ``` -This will compile to a complete executable which responds to [command line arguments](command-line.md#top). If you just run it with no arguments it will execute all test cases (in this case there is just one), report any failures, report a summary of how many tests passed and failed and return the number of failed tests (useful for if you just want a yes/ no answer to: "did it work"). +This will compile to a complete executable which responds to [command line arguments](command-line.md#top). If you just run it with no arguments it will execute all test cases (in this case there is just one) in parallel worker processes, report any failures and a summary, and exit with 0 when every test passed (useful for if you just want a yes/ no answer to: "did it work"). Anyway, as the tests above as written will pass, but there is a bug. The problem is that `Factorial(0)` should return 1 (due to [its diff --git a/docs/usage-tips.md b/docs/usage-tips.md index 0ed3d8d3..18e9cd11 100644 --- a/docs/usage-tips.md +++ b/docs/usage-tips.md @@ -9,77 +9,45 @@ Your tests should be run in a manner roughly equivalent with: ``` -./tests --order rand --warn NoAssertions +./tests --order=random --warn=no-assertions ``` -Notice that all the tests are run in a large batch, their relative order -is randomized, and that you ask Catch2 to fail test whose leaf-path -does not contain an assertion. +The [Catch3 runner](progmasoft-runner.md#top) runs them in parallel worker +processes, retries failures once alone to tell deterministic, flaky and +order-dependent failures apart, and fails tests whose leaf path contains no +assertion. Random order shakes out interference between tests that run in +the same worker; when a test fails only after another, the runner names the +other. -The reason I recommend running all your tests in the same process is that -this exposes your tests to interference from their runs. This can be both -positive interference, where the changes in global state from previous -test allow later tests to pass, but also negative interference, where -changes in global state from previous test causes later tests to fail. - -In my experience, interference, especially destructive interference, -usually comes from errors in the code under test, rather than the tests -themselves. This means that by allowing interference to happen, our tests -can find these issues. Obviously, to shake out interference coming from -different orderings of tests, the test order also need to be shuffled -between runs. - -However, running all tests in a single batch eventually becomes impractical -as they will take too long to run, and you will want to run your tests -in parallel. +Interference, especially destructive interference, usually comes from +errors in the code under test, rather than the tests themselves. Tests that +share state outside the process, such as a file at a fixed path or a +database, need `[!serial]` or `[!resource:NAME]`; the runner gives every +test its own temporary directory. <a id="parallel-tests"></a> ## Running tests in parallel -There are multiple ways of running tests in parallel, with various level -of structure. If you are using CMake and CTest, then we provide a helper -function [`catch_discover_tests`](cmake-integration.md#automatic-test-registration) -that registers each Catch2 `TEST_CASE` as a single CTest test, which -is then run in a separate process. This is an easy way to set up parallel -tests if you are already using CMake & CTest to run your tests, but you -will lose the advantage of running tests in batches. - - -Catch2 also supports [splitting tests in a binary into multiple -shards](command-line.md#test-sharding). This can be used by any test -runner to run batches of tests in parallel. Do note that when selecting -on the number of shards, you should have more shards than there are cores, -to avoid issues with long-running tests getting accidentally grouped in -the same shard, and causing long-tailed execution time. - -**Note that naively composing sharding and random ordering of tests will break.** - -Invoking Catch2 test executable like this +The runner already runs the tests of a binary in parallel, one worker per +hardware thread by default (`--jobs`). Across machines, split the run into +[shards](progmasoft-runner.md#choosing-and-ordering-tests): ```text -./tests --order rand --shard-index 0 --shard-count 3 -./tests --order rand --shard-index 1 --shard-count 3 -./tests --order rand --shard-index 2 --shard-count 3 +./tests --shard=0/3 +./tests --shard=1/3 +./tests --shard=2/3 ``` -does not guarantee covering all tests inside the executable, because -each invocation will have its own random seed, thus it will have its own -random order of tests and thus the partitioning of tests into shards will -be different as well. - -To do this properly, you need the individual shards to share the random -seed, e.g. -```text -./tests --order rand --shard-index 0 --shard-count 3 --rng-seed 0xBEEF -./tests --order rand --shard-index 1 --shard-count 3 --rng-seed 0xBEEF -./tests --order rand --shard-index 2 --shard-count 3 --rng-seed 0xBEEF -``` +A test goes to the shard a stable hash of its name chooses, so the shards +cover every test exactly once whatever the order or seed of each run. With +`--shard-durations=FILE`, a JSON report or history of an earlier run, the +shards are balanced by duration instead. -Catch2 actually provides a helper to automatically register multiple shards -as CTest tests, with shared random seed that changes each CTest invocation. -For details look at the documentation of -[`CatchShardTests.cmake` CMake script](cmake-integration.md#catchshardtestscmake). +If you use CMake and CTest, [`catch_discover_tests`](cmake-integration.md#automatic-test-registration) +registers each `TEST_CASE` as a CTest test, and +[`CatchShardTests.cmake`](cmake-integration.md#catchshardtestscmake) +registers shards. ## Organizing tests into binaries diff --git a/docs/why-catch3.md b/docs/why-catch3.md index 9be7a5f7..69b88a5e 100644 --- a/docs/why-catch3.md +++ b/docs/why-catch3.md @@ -8,31 +8,33 @@ namespace, build-target, and licensing boundary. The name does **not** mean that this is an official Catch2 v3 release, a new upstream version, or a project endorsed by Catch2's maintainers. The current compatibility engine is based on Catch2's 3.16.1 development line. The project -is still early-stage and does not yet replace or independently reimplement the -Catch2 runner. +is still early-stage. It keeps Catch2's API for writing tests and replaced +Catch2's runner with [its own](progmasoft-runner.md#top). ## What is different today? | Area | Catch2 compatibility engine | Progmasoft Catch3 additions | | --- | --- | --- | -| Test execution | Catch2's existing runner, reporters, command-line interface, and test registration | `CATCH3_CHECK_FOR_ALL` and `CATCH3_CHECK_PROPERTY` report property outcomes through Catch2's assertion path | +| Test execution | Catch2's test registration, `RunContext` and listeners | The Catch3 runner: parallel worker processes, retries, flaky-test detection, quarantine, history, selection, sharding and its own command line and reports; `CATCH3_CHECK_FOR_ALL` and `CATCH3_CHECK_PROPERTY` report property outcomes through Catch2's assertion path | | Public include path | Existing `catch2/...` headers | `Progmasoft/Catch3.hpp` and focused `Progmasoft/Catch3/...` headers | -| C++ baseline | Existing Catch2 target remains at C++14 | The added property API requires C++20 | +| C++ baseline | Code that includes Catch2's headers needs C++14; the library, which holds the runner, is built as C++20 | The added property API requires C++20 | | Randomness | Catch2's existing behavior is unchanged | A specified SplitMix64 sequence provides deterministic trial seeds and unbiased bounded integer sampling | | Generated values | No change to existing Catch2 generators | Strategies for numbers, text, collections and structures with `Map`, `Filter` and `FlatMap`; shrinking of recorded choices, so every combination shrinks; replay tokens and an example database; state machines, `Classify`/`Cover` statistics, targeted search, and differential and metamorphic oracles | | Build identity | Existing `Catch2` target and APIs remain available | Separate `Progmasoft::Catch3` CMake target and `//src/Progmasoft:catch3` Bazel target | | Source terms | Catch2-originated source retains its upstream BSL-1.0 terms | Progmasoft-authored additions state their own SPDX license in each file | Properties take any number of generated values. Shrinking is deterministic and -bounded, but local rather than globally minimal. Parallel trials and a -standalone Catch3 runner do not exist yet; [the roadmap](progmasoft-roadmap.md#top) +bounded, but local rather than globally minimal. Parallel trials do not +exist yet; [the roadmap](progmasoft-roadmap.md#top) orders them with mocking, per-test coverage, test selection, mutation testing and test generation. ## Compatibility policy -Catch2's `Catch` namespace, established headers, macros, command-line behavior, -reporters, and coding style remain the compatibility surface. Progmasoft +Catch2's `Catch` namespace, established headers, macros, matchers, +generators, listeners and coding style remain the compatibility surface; the +runner, its command line and its reports are Catch3's own and are not +compatible with Catch2's. Progmasoft features should be added under `Progmasoft::Catch3` and must not require users to rename existing Catch2 tests. The first compatibility goal is source-level continuity with the included Catch2 3.16.1 base; this is not a promise that every diff --git a/examples/210-Evt-EventListeners.cpp b/examples/210-Evt-EventListeners.cpp index d1cfdb4f..f0a9f0af 100644 --- a/examples/210-Evt-EventListeners.cpp +++ b/examples/210-Evt-EventListeners.cpp @@ -334,14 +334,6 @@ struct MyListener : Catch::EventListenerBase { print( std::cout, 1, "- testRunStats", testRunStats ); } - // A test is being skipped (because it is "hidden") - void skipTest( Catch::TestCaseInfo const& testInfo ) override { - std::cout - << dashed_line - << "\nEvent: skipTest:\n"; - print( std::cout, 1, "- testInfo", testInfo ); - } - // Test cases starting void testCaseStarting( Catch::TestCaseInfo const& testInfo ) override { std::cout diff --git a/examples/232-Cfg-CustomMain.cpp b/examples/232-Cfg-CustomMain.cpp index 27040993..6ef05fe7 100644 --- a/examples/232-Cfg-CustomMain.cpp +++ b/examples/232-Cfg-CustomMain.cpp @@ -7,35 +7,39 @@ // SPDX-License-Identifier: BSL-1.0 // 232-Cfg-CustomMain.cpp -// Show how to use custom main and add a custom option to the CLI parser +// Show how to use custom main and read an option of your own before the +// rest of the command line goes to the Catch3 runner. +// +// The runner starts this binary again for its worker processes, with the +// same `main`: what happens before `run` happens in every worker too, which +// is where global set-up belongs. #include <catch2/catch_session.hpp> +#include <cstdlib> #include <iostream> +#include <string> +#include <vector> int main(int argc, char** argv) { Catch::Session session; // There must be exactly one instance int height = 0; // Some user variable you want to be able to set - // Build a new parser on top of Catch2's - using namespace Catch::Clara; - auto cli - = session.cli() // Get Catch2's command line parser - | Opt( height, "height" ) // bind variable to a new option, with a hint string - ["--height"] // the option names it will respond to - ("how high?"); // description string for the help output + // Take our own option out of the command line; the runner reads the rest. + // Workers are started with the runner's options only, so a value they + // need too belongs in the environment, which they inherit. + std::vector<char const*> arguments; + for ( int i = 0; i < argc; ++i ) { + std::string const argument = argv[i]; + if ( argument.rfind( "--height=", 0 ) == 0 ) { + height = std::atoi( argument.c_str() + 9 ); + } else { + arguments.push_back( argv[i] ); + } + } - // Now pass the new composite back to Catch2 so it uses that - session.cli( cli ); - - // Let Catch2 (using Clara) parse the command line - int returnCode = session.applyCommandLine( argc, argv ); - if( returnCode != 0 ) // Indicates a command line error - return returnCode; - - // if set on the command line then 'height' is now set at this point std::cout << "height: " << height << '\n'; - return session.run(); + return session.run( static_cast<int>( arguments.size() ), arguments.data() ); } diff --git a/extras/Catch.cmake b/extras/Catch.cmake index 14ddf228..1b068de6 100644 --- a/extras/Catch.cmake +++ b/extras/Catch.cmake @@ -44,7 +44,8 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. ``catch_discover_tests`` sets up a post-build command on the test executable that generates the list of tests by parsing the output from running the test - with the ``--list-tests --reporter json`` argument. This ensures that the full + with the ``--list=json`` argument, the discovery protocol of the Catch3 runner + (``"format": "catch3-tests"``, version 1). This ensures that the full list of tests is obtained. Since test discovery occurs at build time, it is not necessary to re-run CMake when the list of tests changes. However, it requires that :prop_tgt:`CROSSCOMPILING_EMULATOR` is properly set @@ -103,25 +104,24 @@ same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. Note that this variable is only available in CTest. ``REPORTER reporter`` - Use the specified reporter when running the test case. The reporter will - be passed to the Catch executable as ``--reporter reporter``. + Write a report of each test: ``junit`` passes ``--junit=<file>`` and + ``json`` passes ``--json=<file>`` to the test executable; ``console`` + writes no file. ``junit`` and ``json`` need ``OUTPUT_DIR``. ``OUTPUT_DIR dir`` - If specified, the parameter is passed along as - ``--out dir/<test_name>`` to Catch executable. The actual file name is the - same as the test name. This should be used instead of - ``EXTRA_ARGS --out foo`` to avoid race conditions writing the result output - when using parallel test execution. + The directory the reports of ``REPORTER`` go to, one file per test, + named after the test. Use it instead of ``EXTRA_ARGS --junit=foo`` so that + tests that run at the same time do not write the same file. ``OUTPUT_PREFIX prefix`` May be used in conjunction with ``OUTPUT_DIR``. If specified, ``prefix`` is added to each output file name, like so - ``--out dir/prefix<test_name>``. + ``dir/prefix<test_name>``. ``OUTPUT_SUFFIX suffix`` May be used in conjunction with ``OUTPUT_DIR``. If specified, ``suffix`` is added to each output file name, like so - ``--out dir/<test_name>suffix``. This can be used to add a file extension to + ``dir/<test_name>suffix``. This can be used to add a file extension to the output e.g. ".xml". ``DL_PATHS path...`` diff --git a/extras/CatchAddTests.cmake b/extras/CatchAddTests.cmake index aba0d001..ef2fb439 100644 --- a/extras/CatchAddTests.cmake +++ b/extras/CatchAddTests.cmake @@ -74,7 +74,7 @@ function(magic_unescape_chars var) endfunction() -# Abuses knowledge of Catch2's JSON reporter output for listing tests to +# Abuses knowledge of the runner's JSON listing (`--list=json`) to # split JSON array of the test listings into a CMake list of strings, # with each element being the JSON string of one array entry. # @@ -286,19 +286,10 @@ function(catch_discover_tests_impl) set(ENV{DYLD_FRAMEWORK_PATH} "${paths}") endif() - make_temp_file_path(listing_output_path "${_TEST_WORKING_DIR}") - - set(_Verbosity "quiet") - if (add_tags) - set(_Verbosity "normal") - endif() - + # The discovery protocol of the Catch3 runner: `--list=json` prints one + # JSON document, "format": "catch3-tests", in registration order. execute_process( - COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} - --list-tests - --reporter "json::verbosity=${_Verbosity}" - --out "${listing_output_path}" - --order lex # Make sure the output order, and thus test registration order, is consistent across runs. + COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} --list=json OUTPUT_VARIABLE listing_output RESULT_VARIABLE result WORKING_DIRECTORY "${_TEST_WORKING_DIR}" @@ -310,34 +301,29 @@ function(catch_discover_tests_impl) " Output: ${listing_output}\n" ) endif() - - # Read the JSON output back from the output file and validate it. - file(READ ${listing_output_path} listing_output) - file(REMOVE ${listing_output_path}) + # Whatever the binary printed before the document is not part of it. + string(FIND "${listing_output}" "{" listing_start) + if(listing_start GREATER 0) + string(SUBSTRING "${listing_output}" ${listing_start} -1 listing_output) + endif() validate_input_noescapes(listing_output) - # Prepare reporter + # The runner writes reports itself: REPORTER chooses which, OUTPUT_DIR where. + set(report_option "") if(reporter) - set(reporter_arg "--reporter ${reporter}") - - # Run test executable to check whether reporter is available - # note that the use of --list-reporters is not the important part, - # we only want to check whether the execution succeeds with ${reporter_arg} - execute_process( - COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" ${spec} ${reporter_arg} --list-reporters - OUTPUT_VARIABLE reporter_check_output - RESULT_VARIABLE reporter_check_result - WORKING_DIRECTORY "${_TEST_WORKING_DIR}" - ) - if(${reporter_check_result} EQUAL 255) + string(TOLOWER "${reporter}" reporter_lower) + if(reporter_lower STREQUAL "junit") + set(report_option "--junit=") + elseif(reporter_lower STREQUAL "json") + set(report_option "--json=") + elseif(NOT reporter_lower STREQUAL "console") message(FATAL_ERROR - "\"${reporter}\" is not a valid reporter!\n" + "\"${reporter}\" is not a report of the Catch3 runner; use junit, json or console\n" ) - elseif(NOT ${reporter_check_result} EQUAL 0) + endif() + if(report_option AND NOT output_dir) message(FATAL_ERROR - "Error checking for reporter in test executable '${_TEST_EXECUTABLE}':\n" - " Result: ${reporter_check_result}\n" - " Output: ${reporter_check_output}\n" + "REPORTER ${reporter} writes a file and needs OUTPUT_DIR\n" ) endif() endif() @@ -365,14 +351,15 @@ function(catch_discover_tests_impl) endif() # Parse JSON output for list of tests/class names/tags + string(JSON format GET "${listing_output}" "format") string(JSON version GET "${listing_output}" "version") - if(NOT version STREQUAL "2") - message(FATAL_ERROR "Unsupported catch output version: '${version}'") + if(NOT format STREQUAL "catch3-tests" OR NOT version STREQUAL "1") + message(FATAL_ERROR "Unsupported test listing: '${format}' version '${version}'") endif() # Extract just the JSON array with tests and then split them into # individual objects. - string(JSON test_listing GET "${listing_output}" "listings" "tests") + string(JSON test_listing GET "${listing_output}" "tests") split_json_array(test_listing tests) # Exit early if no tests are detected @@ -405,8 +392,6 @@ function(catch_discover_tests_impl) ) prepare_command_fragment(_args_reporter_fragment "${extra_args}" - "${reporter_arg}" - "--order decl" ) # `set_tests_properties` calls are @@ -459,7 +444,9 @@ function(catch_discover_tests_impl) # ...add output dir if(output_dir) string(REGEX REPLACE "[^A-Za-z0-9_]" "_" escaped_name_clean "${escaped_name}") - set(output_dir_arg "--out ${output_dir}/${output_prefix}${escaped_name_clean}${output_suffix}") + if(report_option) + set(output_dir_arg "${report_option}${output_dir}/${output_prefix}${escaped_name_clean}${output_suffix}") + endif() endif() set(full_name "${prefix}${plain_name}${suffix}") diff --git a/extras/CatchShardTests.cmake b/extras/CatchShardTests.cmake index 89666d66..27f7c9f7 100644 --- a/extras/CatchShardTests.cmake +++ b/extras/CatchShardTests.cmake @@ -8,7 +8,7 @@ # Supported optional args: # * SHARD_COUNT - number of shards to split target's tests into -# * REPORTER - reporter spec to use for tests +# * REPORTER - report options for the shards, such as --junit=FILE # * TEST_SPEC - test spec used for filtering tests function(catch_add_sharded_tests TARGET) if(${CMAKE_VERSION} VERSION_LESS "3.10.0") diff --git a/extras/CatchShardTestsImpl.cmake b/extras/CatchShardTestsImpl.cmake index 83fac688..99fd900a 100644 --- a/extras/CatchShardTestsImpl.cmake +++ b/extras/CatchShardTestsImpl.cmake @@ -15,7 +15,8 @@ # * TARGET_NAME - name of the target to shard (used for test names) # * SHARD_COUNT - number of shards to split the binary into # Optional args: -# * REPORTER_SPEC - reporter specs to be passed down to the binary +# * REPORTER_SPEC - report options passed to the binary as they are, such +# as --junit=FILE # * TEST_SPEC - test spec to pass down to the test binary if(NOT EXISTS "${TEST_BINARY}") @@ -29,22 +30,24 @@ if(TEST_SPEC) set(other_args "${other_args} ${TEST_SPEC}") endif() if(REPORTER_SPEC) - set(other_args "${other_args} --reporter ${REPORTER_SPEC}") + set(other_args "${other_args} ${REPORTER_SPEC}") endif() # foreach RANGE in cmake is inclusive of the end, so we have to adjust it math(EXPR adjusted_shard_count "${SHARD_COUNT} - 1") +# The runner assigns tests to shards by a stable hash of their names, so the +# shards cover every test once whatever the seed; the shared seed keeps their +# random numbers the same. file(WRITE "${CTEST_FILE}" - "string(RANDOM LENGTH 8 ALPHABET \"0123456789abcdef\" rng_seed)\n" + "string(RANDOM LENGTH 9 ALPHABET \"0123456789\" rng_seed)\n" "\n" "foreach(shard_idx RANGE ${adjusted_shard_count})\n" " add_test(${TARGET_NAME}-shard-" [[${shard_idx}]] "/${adjusted_shard_count}\n" " ${TEST_BINARY}" - " --shard-index " [[${shard_idx}]] - " --shard-count ${SHARD_COUNT}" - " --rng-seed " [[0x${rng_seed}]] - " --order rand" + " --shard=" [[${shard_idx}]] "/${SHARD_COUNT}" + " --seed=" [[${rng_seed}]] + " --order=random" "${other_args}" "\n" " )\n" diff --git a/extras/catch_amalgamated.cpp b/extras/catch_amalgamated.cpp index 230849f1..506726dc 100644 --- a/extras/catch_amalgamated.cpp +++ b/extras/catch_amalgamated.cpp @@ -7,10 +7,14 @@ // SPDX-License-Identifier: BSL-1.0 // Catch v3.16.1 -// Generated: 2026-10-11 02:09:25.622664 +// Generated: 2026-10-23 23:12:01.455813 // ---------------------------------------------------------- // This file is an amalgamation of multiple different files. // You probably shouldn't edit it directly. +// +// The parts that come from src/Progmasoft, the Catch3 runner, keep +// their own copyright and license lines (MPL-2.0 WITH +// AdditionRef-Progmasoft-Exception-1.1). The runner needs C++20. // ---------------------------------------------------------- #include "catch_amalgamated.hpp" @@ -36,4048 +40,4414 @@ #endif // defined(CATCH_PLATFORM_WINDOWS) #endif // CATCH_WINDOWS_H_PROXY_HPP_INCLUDED +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#include <cassert> +#include <optional> +#include <string> +#include <utility> +#include <variant> -#include <algorithm> -#include <chrono> -#include <cmath> +namespace Progmasoft::Catch3::Runner +{ + /// A failure with a message written for the person who has to act on it. + class Error final + { + public: + explicit Error(std::string message) + : message_{std::move(message)} + { + } -namespace Catch { - namespace Benchmark { - namespace Detail { + [[nodiscard]] const std::string & + Message() const noexcept + { + return message_; + } - Environment measure_environment_default() { - return Detail::measure_environment<default_clock>(); - } + /// The same failure, with what was being done in front of it. + [[nodiscard]] Error + Context(const std::string &context) const + { + return Error{context + ": " + message_}; + } - ExecutionPlan prepare_default( const IConfig& cfg, - Environment env, - BenchmarkFunction&& fun ) { - // This mirrors Benchmark::prepare<default_clock>(), but with the - // clock fixed so it is instantiated once here in the library. - auto min_time = env.clock_resolution.mean * Detail::minimum_ticks; - auto run_time = std::max( - min_time, - std::chrono::duration_cast<decltype( min_time )>( - cfg.benchmarkWarmupTime() ) ); - auto&& test = Detail::run_for_at_least<default_clock>( - std::chrono::duration_cast<IDuration>( run_time ), 1, fun ); - int new_iters = static_cast<int>( - std::ceil( min_time * test.iterations / test.elapsed ) ); - return { new_iters, - test.elapsed / test.iterations * new_iters * - cfg.benchmarkSamples(), - CATCH_MOVE( fun ), - std::chrono::duration_cast<FDuration>( - cfg.benchmarkWarmupTime() ), - Detail::warmup_iterations }; - } + private: + std::string message_; + }; - std::vector<FDuration> run_plan_default( ExecutionPlan const& plan, - const IConfig& cfg, - Environment env ) { - return plan.run<default_clock>( cfg, env ); - } + /// A value, or the error that prevented it. + template<typename Value> + class [[nodiscard]] Expected final + { + public: + Expected(Value value) + : state_{std::in_place_index<0>, std::move(value)} + { + } - } // namespace Detail - } // namespace Benchmark -} // namespace Catch + Expected(Error error) + : state_{std::in_place_index<1>, std::move(error)} + { + } + + [[nodiscard]] explicit + operator bool() const noexcept + { + return state_.index() == 0; + } + + [[nodiscard]] Value & + operator*() & + { + assert(*this); + return std::get<0>(state_); + } + [[nodiscard]] const Value & + operator*() const & + { + assert(*this); + return std::get<0>(state_); + } + [[nodiscard]] Value && + operator*() && + { + assert(*this); + return std::get<0>(std::move(state_)); + } + [[nodiscard]] Value * + operator->() + { + assert(*this); + return &std::get<0>(state_); + } -namespace Catch { - namespace Benchmark { - namespace Detail { - ChronometerConcept::~ChronometerConcept() = default; - } // namespace Detail - } // namespace Benchmark -} // namespace Catch + [[nodiscard]] const Value * + operator->() const + { + assert(*this); + return &std::get<0>(state_); + } + [[nodiscard]] const Error & + GetError() const + { + assert(!*this); + return std::get<1>(state_); + } -// Adapted from donated nonius code. + private: + std::variant<Value, Error> state_; + }; + + /// Success, or the error that happened instead. + class [[nodiscard]] Status final + { + public: + Status() = default; + + Status(Error error) + : error_{std::move(error)} + { + } + + [[nodiscard]] static Status + Ok() + { + return Status{}; + } + + [[nodiscard]] explicit + operator bool() const noexcept + { + return !error_.has_value(); + } + + [[nodiscard]] const Error & + GetError() const + { + assert(error_.has_value()); + return *error_; + } + private: + std::optional<Error> error_; + }; +} // namespace Progmasoft::Catch3::Runner +#include <filesystem> +#include <string> #include <vector> -namespace Catch { - namespace Benchmark { - namespace Detail { - SampleAnalysis analyse(const IConfig &cfg, FDuration* first, FDuration* last) { - if (!cfg.benchmarkNoAnalysis()) { - std::vector<double> samples; - samples.reserve(static_cast<size_t>(last - first)); - for (auto current = first; current != last; ++current) { - samples.push_back( current->count() ); - } +namespace Progmasoft::Catch3::Runner +{ + /// What Git says about the checkout the tests come from. + struct Changes final + { + std::filesystem::path root; + std::string commit; + /// Changed since the revision, staged or not, and untracked files; + /// absolute paths. + std::vector<std::filesystem::path> files; + }; - auto analysis = Catch::Benchmark::Detail::analyse_samples( - cfg.benchmarkConfidenceInterval(), - cfg.benchmarkResamples(), - samples.data(), - samples.data() + samples.size() ); - auto outliers = Catch::Benchmark::Detail::classify_outliers( - samples.data(), samples.data() + samples.size() ); + /// The changes since `revision` (HEAD when empty) in the checkout that + /// holds `directory`. An error when Git or the checkout is missing. + [[nodiscard]] Expected<Changes> + ReadChanges(const std::filesystem::path &directory, + const std::string &revision); - auto wrap_estimate = [](Estimate<double> e) { - return Estimate<FDuration>{ - FDuration( e.point ), - FDuration( e.lower_bound ), - FDuration( e.upper_bound ), - e.confidence_interval, - }; - }; - std::vector<FDuration> samples2; - samples2.reserve(samples.size()); - for (auto s : samples) { - samples2.push_back( FDuration( s ) ); - } + /// The commit checked out in the checkout that holds `directory`, or + /// empty. + [[nodiscard]] std::string + CurrentCommit(const std::filesystem::path &directory); - return { - CATCH_MOVE(samples2), - wrap_estimate(analysis.mean), - wrap_estimate(analysis.standard_deviation), - outliers, - analysis.outlier_variance, - }; - } else { - std::vector<FDuration> samples; - samples.reserve(static_cast<size_t>(last - first)); + /// Whether two spellings name the same source file: equal after + /// normalisation, or one a relative path the other ends with. + [[nodiscard]] bool + SameSourceFile(const std::filesystem::path &left, + const std::filesystem::path &right); +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - FDuration mean = FDuration(0); - int i = 0; - for (auto it = first; it < last; ++it, ++i) { - samples.push_back(*it); - mean += *it; - } - mean /= i; +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - return SampleAnalysis{ - CATCH_MOVE(samples), - Estimate<FDuration>{ mean, mean, mean, 0.0 }, - Estimate<FDuration>{ FDuration( 0 ), - FDuration( 0 ), - FDuration( 0 ), - 0.0 }, - OutlierClassification{}, - 0.0 - }; - } - } - } // namespace Detail - } // namespace Benchmark -} // namespace Catch +#include <cstdint> +#include <initializer_list> +#include <memory> +#include <string> +#include <string_view> +#include <utility> +#include <vector> +namespace Progmasoft::Catch3::Runner +{ + class JsonValue; + /// The members of an object, in the order they were added or read. + using JsonMembers = std::vector<std::pair<std::string, JsonValue>>; -namespace Catch { - namespace Benchmark { - namespace Detail { - struct do_nothing { - void operator()() const {} - }; + /// One JSON value: the messages between the runner and its workers, the + /// reports, the history and the coverage exports of LLVM are all JSON. + /// + /// Numbers are kept as `double`, which represents every integer the + /// runner writes (counts, microseconds, 32-bit seeds) exactly. + class JsonValue final + { + public: + enum class Kind : std::uint8_t + { + kNull, + kBoolean, + kNumber, + kString, + kArray, + kObject, + }; - BenchmarkFunction::callable::~callable() = default; - BenchmarkFunction::BenchmarkFunction(): - f( new model<do_nothing>{ {} } ){} - } // namespace Detail - } // namespace Benchmark -} // namespace Catch + JsonValue() = default; + JsonValue(std::nullptr_t) {} + JsonValue(bool value); + JsonValue(double value); + JsonValue(int value); + JsonValue(unsigned value); + JsonValue(long value); + JsonValue(unsigned long value); + JsonValue(long long value); + JsonValue(unsigned long long value); + JsonValue(const char *value); + JsonValue(std::string value); + JsonValue(std::string_view value); + + JsonValue(const JsonValue &other); + JsonValue(JsonValue &&other) noexcept; + JsonValue & + operator=(const JsonValue &other); + JsonValue & + operator=(JsonValue &&other) noexcept; + ~JsonValue(); + + [[nodiscard]] static JsonValue + Array(std::vector<JsonValue> items = {}); + + [[nodiscard]] static JsonValue + Object(JsonMembers members = {}); + + [[nodiscard]] Kind + GetKind() const noexcept + { + return kind_; + } + [[nodiscard]] bool + IsNull() const noexcept + { + return kind_ == Kind::kNull; + } + [[nodiscard]] bool + IsObject() const noexcept + { + return kind_ == Kind::kObject; + } + [[nodiscard]] bool + IsArray() const noexcept + { + return kind_ == Kind::kArray; + } -#include <exception> + [[nodiscard]] bool + IsString() const noexcept + { + return kind_ == Kind::kString; + } -namespace Catch { - namespace Benchmark { - namespace Detail { - struct optimized_away_error final : std::exception { - const char* what() const noexcept override; - }; + [[nodiscard]] bool + IsNumber() const noexcept + { + return kind_ == Kind::kNumber; + } - const char* optimized_away_error::what() const noexcept { - return "could not measure benchmark, maybe it was optimized away"; - } + /// The value as a Boolean; anything else reads as `fallback`. + [[nodiscard]] bool + AsBoolean(bool fallback = false) const noexcept; - void throw_optimized_away_error() { - Catch::throw_exception(optimized_away_error{}); - } + [[nodiscard]] double + AsNumber(double fallback = 0.0) const noexcept; - } // namespace Detail - } // namespace Benchmark -} // namespace Catch + [[nodiscard]] std::int64_t + AsInteger(std::int64_t fallback = 0) const noexcept; + [[nodiscard]] std::uint64_t + AsUnsigned(std::uint64_t fallback = 0) const noexcept; -// Adapted from donated nonius code. + /// The text of a string; anything else reads as the empty string. + [[nodiscard]] const std::string & + AsString() const noexcept; + /// The items of an array; anything else has none. + [[nodiscard]] const std::vector<JsonValue> & + Items() const noexcept; + [[nodiscard]] std::vector<JsonValue> & + MutableItems(); -#include <algorithm> -#include <cassert> -#include <cmath> -#include <cstddef> -#include <numeric> -#include <random> + /// The members of an object; anything else has none. + [[nodiscard]] const JsonMembers & + Members() const noexcept; + [[nodiscard]] JsonMembers & + MutableMembers(); -#if defined(CATCH_CONFIG_USE_ASYNC) -#include <future> -#endif + /// The member named `key`, or null when there is none. + [[nodiscard]] const JsonValue & + operator[](std::string_view key) const noexcept; -namespace Catch { - namespace Benchmark { - namespace Detail { - namespace { + /// The member named `key` of an object, added when missing. + JsonValue & + Set(std::string_view key, JsonValue value); - template <typename URng, typename Estimator> - static sample - resample( URng& rng, - unsigned int resamples, - double const* first, - double const* last, - Estimator& estimator ) { - auto n = static_cast<size_t>( last - first ); - Catch::uniform_integer_distribution<size_t> dist( 0, n - 1 ); + /// Appends to an array. + void + Append(JsonValue value); - sample out; - out.reserve( resamples ); - std::vector<double> resampled; - resampled.reserve( n ); - for ( size_t i = 0; i < resamples; ++i ) { - resampled.clear(); - for ( size_t s = 0; s < n; ++s ) { - resampled.push_back( first[dist( rng )] ); - } - const auto estimate = - estimator( resampled.data(), resampled.data() + resampled.size() ); - out.push_back( estimate ); - } - std::sort( out.begin(), out.end() ); - return out; - } + /// The compact JSON text of the value, on one line. + [[nodiscard]] std::string + ToString() const; - static double outlier_variance( Estimate<double> mean, - Estimate<double> stddev, - int n ) { - double sb = stddev.point; - double mn = mean.point / n; - double mg_min = mn / 2.; - double sg = (std::min)( mg_min / 4., sb / std::sqrt( n ) ); - double sg2 = sg * sg; - double sb2 = sb * sb; + /// The JSON text with two spaces of indentation per level. + [[nodiscard]] std::string + ToPrettyString() const; - auto c_max = [n, mn, sb2, sg2]( double x ) -> double { - double k = mn - x; - double d = k * k; - double nd = n * d; - double k0 = -n * nd; - double k1 = sb2 - n * sg2 + nd; - double det = k1 * k1 - 4 * sg2 * k0; - return static_cast<int>( -2. * k0 / - ( k1 + std::sqrt( det ) ) ); - }; + private: + void + Write(std::string &output, int indent, int depth) const; - auto var_out = [n, sb2, sg2]( double c ) { - double nc = n - c; - return ( nc / n ) * ( sb2 - nc * sg2 ); - }; + void + Destroy() noexcept; - return (std::min)( var_out( 1 ), - var_out( - (std::min)( c_max( 0. ), - c_max( mg_min ) ) ) ) / - sb2; - } + void + CopyFrom(const JsonValue &other); - static double erf_inv( double x ) { - // Code accompanying the article "Approximating the erfinv - // function" in GPU Computing Gems, Volume 2 - double w, p; + void + MoveFrom(JsonValue &&other) noexcept; + + Kind kind_ = Kind::kNull; + bool boolean_ = false; + double number_ = 0.0; + std::string string_; + std::unique_ptr<std::vector<JsonValue>> items_; + std::unique_ptr<JsonMembers> members_; + }; - w = -log( ( 1.0 - x ) * ( 1.0 + x ) ); + /// Reads one JSON document; whitespace may surround it. + [[nodiscard]] Expected<JsonValue> + ParseJson(std::string_view text); - if ( w < 6.250000 ) { - w = w - 3.125000; - p = -3.6444120640178196996e-21; - p = -1.685059138182016589e-19 + p * w; - p = 1.2858480715256400167e-18 + p * w; - p = 1.115787767802518096e-17 + p * w; - p = -1.333171662854620906e-16 + p * w; - p = 2.0972767875968561637e-17 + p * w; - p = 6.6376381343583238325e-15 + p * w; - p = -4.0545662729752068639e-14 + p * w; - p = -8.1519341976054721522e-14 + p * w; - p = 2.6335093153082322977e-12 + p * w; - p = -1.2975133253453532498e-11 + p * w; - p = -5.4154120542946279317e-11 + p * w; - p = 1.051212273321532285e-09 + p * w; - p = -4.1126339803469836976e-09 + p * w; - p = -2.9070369957882005086e-08 + p * w; - p = 4.2347877827932403518e-07 + p * w; - p = -1.3654692000834678645e-06 + p * w; - p = -1.3882523362786468719e-05 + p * w; - p = 0.0001867342080340571352 + p * w; - p = -0.00074070253416626697512 + p * w; - p = -0.0060336708714301490533 + p * w; - p = 0.24015818242558961693 + p * w; - p = 1.6536545626831027356 + p * w; - } else if ( w < 16.000000 ) { - w = sqrt( w ) - 3.250000; - p = 2.2137376921775787049e-09; - p = 9.0756561938885390979e-08 + p * w; - p = -2.7517406297064545428e-07 + p * w; - p = 1.8239629214389227755e-08 + p * w; - p = 1.5027403968909827627e-06 + p * w; - p = -4.013867526981545969e-06 + p * w; - p = 2.9234449089955446044e-06 + p * w; - p = 1.2475304481671778723e-05 + p * w; - p = -4.7318229009055733981e-05 + p * w; - p = 6.8284851459573175448e-05 + p * w; - p = 2.4031110387097893999e-05 + p * w; - p = -0.0003550375203628474796 + p * w; - p = 0.00095328937973738049703 + p * w; - p = -0.0016882755560235047313 + p * w; - p = 0.0024914420961078508066 + p * w; - p = -0.0037512085075692412107 + p * w; - p = 0.005370914553590063617 + p * w; - p = 1.0052589676941592334 + p * w; - p = 3.0838856104922207635 + p * w; - } else { - w = sqrt( w ) - 5.000000; - p = -2.7109920616438573243e-11; - p = -2.5556418169965252055e-10 + p * w; - p = 1.5076572693500548083e-09 + p * w; - p = -3.7894654401267369937e-09 + p * w; - p = 7.6157012080783393804e-09 + p * w; - p = -1.4960026627149240478e-08 + p * w; - p = 2.9147953450901080826e-08 + p * w; - p = -6.7711997758452339498e-08 + p * w; - p = 2.2900482228026654717e-07 + p * w; - p = -9.9298272942317002539e-07 + p * w; - p = 4.5260625972231537039e-06 + p * w; - p = -1.9681778105531670567e-05 + p * w; - p = 7.5995277030017761139e-05 + p * w; - p = -0.00021503011930044477347 + p * w; - p = -0.00013871931833623122026 + p * w; - p = 1.0103004648645343977 + p * w; - p = 4.8499064014085844221 + p * w; - } - return p * x; - } + /// Appends `text` to `output` as a JSON string literal. + void + AppendJsonString(std::string &output, std::string_view text); +} // namespace Progmasoft::Catch3::Runner - static double - standard_deviation( double const* first, double const* last ) { - auto m = Catch::Benchmark::Detail::mean( first, last ); - double variance = - std::accumulate( first, - last, - 0., - [m]( double a, double b ) { - double diff = b - m; - return a + diff * diff; - } ) / - static_cast<double>( last - first ); - return std::sqrt( variance ); - } +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - static sample jackknife( double ( *estimator )( double const*, - double const* ), - double* first, - double* last ) { - const auto second = first + 1; - sample results; - results.reserve( static_cast<size_t>( last - first ) ); +#include <cstddef> +#include <cstdint> +#include <optional> +#include <string> +#include <vector> - for ( auto it = first; it != last; ++it ) { - std::iter_swap( it, first ); - results.push_back( estimator( second, last ) ); - } +namespace Progmasoft::Catch3 +{ + /// How one attempt at a test ended. + enum class Outcome : std::uint8_t + { + kPassed, + /// An assertion failed, or the test threw. + kFailed, + /// It failed, but `[!shouldfail]` or `[!mayfail]` allows that. + kFailedAsExpected, + /// `SKIP` was called. + kSkipped, + /// The process running it died: a signal, a structured exception or + /// an exit in the middle of the test. + kCrashed, + /// It ran past its time limit and was stopped. + kTimedOut, + /// A test it depends on did not pass, so it was not run. + kBlocked, + /// The run stopped before reaching it. + kNotRun, + /// It passed last time and nothing it depends on changed, so the + /// incremental run kept that result. + kUpToDate, + }; - return results; - } + /// Why a failure happened, as far as the runner can tell. + enum class FailureCategory : std::uint8_t + { + kNone, + /// A REQUIRE or CHECK expression was false. + kAssertion, + /// An exception escaped the test or an assertion. + kUnexpectedException, + /// REQUIRE_THROWS and its kind did not see the expected exception. + kMissingException, + /// FAIL or FAIL_CHECK. + kExplicitFailure, + /// A signal or structured exception inside the test. + kCrash, + kTimeout, + /// The worker could not run the test at all, for example because it + /// failed to start. + kInfrastructure, + /// A test it depends on failed. + kDependency, + }; + /// How a failure relates to earlier runs. + enum class FailureHistory : std::uint8_t + { + /// There is no history to compare with. + kUnknown, + /// The test passed in the previous run. + kNew, + /// The test failed in the previous run too. + kPersistent, + /// The test passes and fails with the same inputs. + kIntermittent, + }; - } // namespace - } // namespace Detail - } // namespace Benchmark -} // namespace Catch + /// One failed or reported assertion. + struct AssertionRecord final + { + std::string file; + std::size_t line = 0; + /// The macro, such as REQUIRE. + std::string macro; + /// The expression as written. + std::string expression; + /// The expression with its operands' values. + std::string expansion; + std::string message; + /// INFO, CAPTURE and UNSCOPED_INFO messages in force. + std::vector<std::string> context; + /// The sections the assertion was in, outermost first. + std::vector<std::string> sections; + FailureCategory category = FailureCategory::kAssertion; + bool passed = false; + /// The failure is allowed by `[!mayfail]` or `[!shouldfail]`. + bool allowed = false; + }; -namespace Catch { - namespace Benchmark { - namespace Detail { + /// The statistics of one BENCHMARK. + struct BenchmarkRecord final + { + std::string name; + std::size_t samples = 0; + std::size_t iterations = 0; + double meanNanoseconds = 0.0; + double lowMeanNanoseconds = 0.0; + double highMeanNanoseconds = 0.0; + double standardDeviationNanoseconds = 0.0; + double outlierVariance = 0.0; + }; - double weighted_average_quantile( int k, - int q, - double* first, - double* last ) { - auto count = last - first; - double idx = static_cast<double>((count - 1) * k) / static_cast<double>(q); - int j = static_cast<int>(idx); - double g = idx - j; - std::nth_element(first, first + j, last); - auto xj = first[j]; - if ( Catch::Detail::directCompare( g, 0 ) ) { - return xj; - } + /// One section entered while a test ran, with its timing. + struct SectionRecord final + { + std::string name; + std::string file; + std::size_t line = 0; + /// Microseconds since the start of the attempt. + std::int64_t startMicroseconds = 0; + /// -1 for a section that never ended: the attempt crashed or timed + /// out inside it. + std::int64_t durationMicroseconds = 0; + /// 0 for a section directly in the test case. + std::size_t depth = 0; + }; - auto xj1 = *std::min_element(first + (j + 1), last); - return xj + g * (xj1 - xj); - } + /// One run of a test in a worker. A test may be tried several times: + /// retries, repeats and the probes that look for an order dependency. + struct Attempt final + { + Outcome outcome = Outcome::kNotRun; + FailureCategory category = FailureCategory::kNone; + std::size_t number = 1; + /// The worker that ran it, and the tests it ran before, in order. + std::size_t worker = 0; + std::vector<std::string> precededBy; + bool freshProcess = false; + std::uint32_t seed = 0; + std::int64_t startMicroseconds = 0; + std::int64_t durationMicroseconds = 0; + std::size_t assertionsPassed = 0; + std::size_t assertionsFailed = 0; + std::vector<AssertionRecord> assertions; + std::vector<BenchmarkRecord> benchmarks; + std::vector<SectionRecord> sections; + /// What the test wrote to std::cout and std::cerr, captured. + std::string standardOutput; + std::string standardError; + /// What reached the worker's own streams while the test ran, such as + /// printf output or a sanitizer report. + std::string processOutput; + /// The fatal condition the worker reported before it died. + std::string fatalMessage; + /// How the worker ended, for a crash. + std::string exitDescription; + /// The last assertion started before a crash or a timeout. + std::string lastLocation; + /// The directory the test had for its temporary files; kept when the + /// attempt failed, so its contents can be inspected. + std::string temporaryDirectory; + }; - OutlierClassification - classify_outliers( double const* first, double const* last ) { - std::vector<double> copy( first, last ); + /// One test of the run and everything the runner learned about it. + struct TestResult final + { + /// Identifies the test across runs and builds: its name, with its + /// class and, when names repeat, its tags. History is kept by it. + std::string id; + std::string name; + std::string className; + std::string file; + std::size_t line = 0; + std::vector<std::string> tags; + /// The verdict over all attempts. + Outcome outcome = Outcome::kNotRun; + FailureCategory category = FailureCategory::kNone; + FailureHistory history = FailureHistory::kUnknown; + /// It failed and then passed with the same inputs. + bool flaky = false; + /// It is in quarantine, so its failure does not fail the run. + bool quarantined = false; + /// Why it was selected, when not every test runs: "failed last + /// run", "covers a changed file", "dependency of ...". + std::string selectionReason; + /// A stable identifier of the failure: its category, location and + /// message with numbers removed. Tests that share it probably share a + /// cause. + std::string signature; + /// What the runner found out about the failure, one finding per + /// entry, most useful first. + std::vector<std::string> analysis; + /// A command line that runs the failure again the way it happened. + std::string replayCommand; + std::vector<Attempt> attempts; + std::int64_t predictedDurationMicroseconds = 0; + double predictedFailureProbability = 0.0; + + /// The sum of the attempts' durations. + [[nodiscard]] std::int64_t + DurationMicroseconds() const noexcept; + + /// Whether the result fails the run. + [[nodiscard]] bool + FailsRun() const noexcept; + }; - auto q1 = weighted_average_quantile( 1, 4, copy.data(), copy.data() + copy.size() ); - auto q3 = weighted_average_quantile( 3, 4, copy.data(), copy.data() + copy.size() ); - auto iqr = q3 - q1; - auto los = q1 - ( iqr * 3. ); - auto lom = q1 - ( iqr * 1.5 ); - auto him = q3 + ( iqr * 1.5 ); - auto his = q3 + ( iqr * 3. ); + /// The totals of a run, by verdict. + struct RunSummary final + { + std::size_t tests = 0; + std::size_t passed = 0; + std::size_t failed = 0; + std::size_t failedAsExpected = 0; + std::size_t skipped = 0; + std::size_t crashed = 0; + std::size_t timedOut = 0; + std::size_t blocked = 0; + std::size_t notRun = 0; + std::size_t upToDate = 0; + std::size_t flaky = 0; + std::size_t quarantined = 0; + std::size_t attempts = 0; + std::size_t assertionsPassed = 0; + std::size_t assertionsFailed = 0; + /// Wall-clock time of the run, and the time its tests took added up; + /// their ratio is the speed-up of running in parallel. + std::int64_t wallMicroseconds = 0; + std::int64_t testMicroseconds = 0; + }; - OutlierClassification o; - for ( ; first != last; ++first ) { - const double t = *first; - if ( t < los ) { - ++o.low_severe; - } else if ( t < lom ) { - ++o.low_mild; - } else if ( t > his ) { - ++o.high_severe; - } else if ( t > him ) { - ++o.high_mild; - } - ++o.samples_seen; - } - return o; - } + /// A whole run of one test binary. + struct RunResult final + { + std::string binary; + std::uint32_t seed = 0; + std::size_t workers = 0; + /// The shard, when the run was one of several. + unsigned shardIndex = 0; + unsigned shardCount = 1; + /// The tests that matched before selection narrowed them. + std::size_t candidates = 0; + std::vector<TestResult> tests; + /// Test specs that matched nothing. + std::vector<std::string> unmatchedSpecs; + /// Messages about the run itself, such as a history that could not + /// be read. + std::vector<std::string> notes; + std::int64_t startUnixMilliseconds = 0; + std::int64_t wallMicroseconds = 0; + }; - double mean( double const* first, double const* last ) { - auto count = last - first; - double sum = 0.; - while (first != last) { - sum += *first; - ++first; - } - return sum / static_cast<double>(count); - } + [[nodiscard]] RunSummary + Summarize(const RunResult &run); - double normal_cdf( double x ) { - return std::erfc( -x / std::sqrt( 2.0 ) ) / 2.0; - } + /// The name an outcome has in reports: "passed", "timed-out", ... + [[nodiscard]] const char * + OutcomeName(Outcome outcome) noexcept; - double erfc_inv(double x) { - return erf_inv(1.0 - x); - } + [[nodiscard]] std::optional<Outcome> + OutcomeFromName(const std::string &name) noexcept; - double normal_quantile(double p) { - static const double ROOT_TWO = std::sqrt(2.0); + [[nodiscard]] const char * + CategoryName(FailureCategory category) noexcept; - double result = 0.0; - assert(p >= 0 && p <= 1); - if (p < 0 || p > 1) { - return result; - } + [[nodiscard]] std::optional<FailureCategory> + CategoryFromName(const std::string &name) noexcept; - result = -erfc_inv(2.0 * p); - // result *= normal distribution standard deviation (1.0) * sqrt(2) - result *= /*sd * */ ROOT_TWO; - // result += normal disttribution mean (0) - return result; - } + [[nodiscard]] const char * + HistoryName(FailureHistory history) noexcept; - Estimate<double> - bootstrap( double confidence_level, - double* first, - double* last, - sample const& resample, - double ( *estimator )( double const*, double const* ) ) { - auto n_samples = last - first; + /// The failure of an attempt as text: each failed assertion with its + /// expansion, messages and sections, or what is known about a crash or a + /// timeout. Empty for an attempt that did not fail. + [[nodiscard]] std::string + DescribeFailure(const Attempt &attempt); - double point = estimator( first, last ); - // Degenerate case with a single sample - if ( n_samples == 1 ) - return { point, point, point, confidence_level }; + /// The first line of `DescribeFailure`, for a summary. + [[nodiscard]] std::string + SummarizeFailure(const Attempt &attempt); - sample jack = jackknife( estimator, first, last ); - double jack_mean = - mean( jack.data(), jack.data() + jack.size() ); - double sum_squares = 0, sum_cubes = 0; - for ( double x : jack ) { - auto difference = jack_mean - x; - auto square = difference * difference; - auto cube = square * difference; - sum_squares += square; - sum_cubes += cube; - } + /// Whether an outcome counts as a failure of the attempt. + [[nodiscard]] bool + IsFailure(Outcome outcome) noexcept; +} // namespace Progmasoft::Catch3 - double accel = sum_cubes / ( 6 * std::pow( sum_squares, 1.5 ) ); - long n = static_cast<long>( resample.size() ); - double prob_n = static_cast<double>( - std::count_if( resample.begin(), - resample.end(), - [point]( double x ) { return x < point; } )) / - static_cast<double>( n ); - // degenerate case with uniform samples - if ( Catch::Detail::directCompare( prob_n, 0. ) ) { - return { point, point, point, confidence_level }; - } +#include <cstdint> +#include <filesystem> +#include <map> +#include <optional> +#include <string> +#include <vector> - double bias = normal_quantile( prob_n ); - double z1 = normal_quantile( ( 1. - confidence_level ) / 2. ); +namespace Progmasoft::Catch3::Runner +{ + /// One result of one test in one run. + struct TestRecord final + { + std::int64_t run = 0; + Outcome outcome = Outcome::kNotRun; + FailureCategory category = FailureCategory::kNone; + std::int64_t durationMicroseconds = 0; + std::size_t attempts = 0; + bool flaky = false; + std::string signature; + /// What the test's result depended on; equal fingerprints mean the + /// same inputs. + std::string fingerprint; + }; - auto cumn = [n]( double x ) -> long { - return std::lround( normal_cdf( x ) * - static_cast<double>( n ) ); - }; - auto a = [bias, accel]( double b ) { - return bias + b / ( 1. - accel * b ); - }; - double b1 = bias + z1; - double b2 = bias - z1; - double a1 = a( b1 ); - double a2 = a( b2 ); - auto lo = static_cast<size_t>( (std::max)( cumn( a1 ), 0l ) ); - auto hi = - static_cast<size_t>( (std::min)( cumn( a2 ), n - 1 ) ); + /// How to run a failure again the way it happened. + struct ReplayRecord final + { + std::int64_t run = 0; + std::uint32_t seed = 0; + /// The tests the worker ran before, in order. + std::vector<std::string> precededBy; + bool freshProcess = false; + }; - return { point, resample[lo], resample[hi], confidence_level }; - } + struct QuarantineRecord final + { + std::int64_t since = 0; + std::string reason; + }; - bootstrap_analysis analyse_samples(double confidence_level, - unsigned int n_resamples, - double* first, - double* last) { - auto mean = &Detail::mean; - auto stddev = &standard_deviation; + /// Everything remembered about one test. + struct TestHistory final + { + /// The most recent last. + std::vector<TestRecord> records; + std::optional<QuarantineRecord> quarantine; + std::optional<ReplayRecord> lastFailure; + + [[nodiscard]] const TestRecord * + Last() const noexcept; + + /// A percentile of the durations of recent runs that finished on + /// their own; nothing without such runs. + [[nodiscard]] std::optional<std::int64_t> + DurationPercentile(double fraction) const; + + /// The probability that the next run fails: a failure rate in which + /// each older run counts less, starting from a prior for tests with + /// little history. + [[nodiscard]] double + FailureProbability() const; + + /// Flaky results, and failures that alternate with passes on the + /// same fingerprint, among the last `window` runs. + [[nodiscard]] std::size_t + FlakyEvents(std::size_t window) const; + + /// Passes in a row, counting back from the latest run. + [[nodiscard]] std::size_t + PassStreak() const; + }; -#if defined(CATCH_CONFIG_USE_ASYNC) - auto Estimate = [=](double(*f)(double const*, double const*)) { - std::random_device rd; - auto seed = rd(); - return std::async(std::launch::async, [=] { - SimplePcg32 rng( seed ); - auto resampled = resample(rng, n_resamples, first, last, f); - return bootstrap(confidence_level, first, last, resampled, f); - }); - }; + /// One run, as the history remembers it. + struct RunRecord final + { + std::int64_t id = 0; + std::int64_t unixMilliseconds = 0; + std::string commit; + std::string binaryFingerprint; + std::uint32_t seed = 0; + std::size_t tests = 0; + std::size_t failed = 0; + std::int64_t wallMicroseconds = 0; + }; - auto mean_future = Estimate(mean); - auto stddev_future = Estimate(stddev); + /// When to put a test into quarantine and let it out. + struct QuarantinePolicy final + { + std::size_t window = 20; + std::size_t flakyEventsToEnter = 2; + std::size_t passesToLeave = 10; + }; - auto mean_estimate = mean_future.get(); - auto stddev_estimate = stddev_future.get(); -#else - auto Estimate = [=](double(*f)(double const* , double const*)) { - std::random_device rd; - auto seed = rd(); - SimplePcg32 rng( seed ); - auto resampled = resample(rng, n_resamples, first, last, f); - return bootstrap(confidence_level, first, last, resampled, f); - }; + /// The results of earlier runs of one test binary. + /// + /// It lives in one JSON file. Several runs of the same binary may finish + /// at the same time, so `Commit` changes the file under a lock: it reads + /// it again, adds its run and replaces it in one rename. + class History final + { + public: + History() = default; + + /// Reads the file; a missing file is an empty history. + [[nodiscard]] static Expected<History> + Load(const std::filesystem::path &file); + + /// Adds `run` to the history in `file`, updates quarantine and + /// returns the history as written. + [[nodiscard]] static Expected<History> + Commit(const std::filesystem::path &file, + const RunResult &run, + const RunRecord &record, + const std::map<std::string, std::string> &fingerprints, + const QuarantinePolicy &policy); + + [[nodiscard]] const TestHistory * + Find(const std::string &key) const; + + [[nodiscard]] const std::map<std::string, TestHistory> & + Tests() const noexcept + { + return tests_; + } - auto mean_estimate = Estimate(mean); - auto stddev_estimate = Estimate(stddev); -#endif // CATCH_USE_ASYNC + [[nodiscard]] const std::vector<RunRecord> & + Runs() const noexcept + { + return runs_; + } - auto n = static_cast<int>(last - first); // seriously, one can't use integral types without hell in C++ - double outlier_variance = Detail::outlier_variance(mean_estimate, stddev_estimate, n); + [[nodiscard]] JsonValue + ToJson() const; - return { mean_estimate, stddev_estimate, outlier_variance }; - } - } // namespace Detail - } // namespace Benchmark -} // namespace Catch + [[nodiscard]] static Expected<History> + FromJson(const JsonValue &value); + /// The key a test result is filed under. + [[nodiscard]] static std::string + KeyOf(const TestResult &test); + private: + void + Apply(const RunResult &run, + RunRecord record, + const std::map<std::string, std::string> &fingerprints, + const QuarantinePolicy &policy); + + std::int64_t nextRun_ = 1; + std::vector<RunRecord> runs_; + std::map<std::string, TestHistory> tests_; + }; -#include <cmath> -#include <limits> + /// The file the history of a binary is kept in under `directory`. + [[nodiscard]] std::filesystem::path + HistoryFile(const std::filesystem::path &directory, + const std::string &binaryName); +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 -namespace { +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 -// Performs equivalent check of std::fabs(lhs - rhs) <= margin -// But without the subtraction to allow for INFINITY in comparison -bool marginComparison(double lhs, double rhs, double margin) { - return (lhs + margin >= rhs) && (rhs + margin >= lhs); -} + +#include <chrono> +#include <cstddef> +#include <optional> +#include <string> +#include <vector> + +namespace Catch +{ + class IConfig; } -namespace Catch { +namespace Progmasoft::Catch3::Runner +{ + /// One registered test and what its tags tell the runner. + struct TestEntry final + { + /// The position in registration order, the same in every process + /// started from one binary. + std::size_t index = 0; + /// A name that identifies the test across runs and builds: the test + /// name, with its class when it has one, and its tags when another + /// test has the same name. + std::string key; + std::string name; + std::string className; + std::string file; + std::size_t line = 0; + std::vector<std::string> tags; + bool hidden = false; + bool benchmark = false; + bool mayFail = false; + bool shouldFail = false; + + // From the tags the runner reserves. + /// `[!serial]`: nothing else runs at the same time. + bool serial = false; + /// `[!isolated]`: runs in a process of its own. + bool isolated = false; + /// `[!quarantine]`: in quarantine whatever its history says. + bool quarantined = false; + /// `[!resource:NAME]`: tests sharing a resource never overlap. + std::vector<std::string> resources; + /// `[!depends:NAME]`: runs after these tests, and only if they pass. + std::vector<std::string> dependsOn; + /// `[!timeout:SECONDS]`. + std::optional<std::chrono::milliseconds> timeout; + /// `[!retries:N]`. + std::optional<unsigned> retries; + }; - Approx::Approx ( double value ) - : m_epsilon( static_cast<double>(std::numeric_limits<float>::epsilon())*100. ), - m_margin( 0.0 ), - m_scale( 0.0 ), - m_value( value ) - {} + /// The tests registered in this binary. + class Catalog final + { + public: + /// Reads the test registry. Problems with a reserved tag, such as a + /// timeout that is not a number, are errors. + [[nodiscard]] static Expected<Catalog> + Load(); - Approx Approx::custom() { - return Approx( 0 ); - } + [[nodiscard]] const std::vector<TestEntry> & + Tests() const noexcept + { + return tests_; + } - Approx Approx::operator-() const { - auto temp(*this); - temp.m_value = -temp.m_value; - return temp; - } + [[nodiscard]] const std::vector<Catch::TestCaseHandle> & + Handles() const noexcept + { + return handles_; + } + /// The test of a key or of a name; null when there is none. + [[nodiscard]] const TestEntry * + Find(const std::string &keyOrName) const; - std::string Approx::toString() const { - ReusableStringStream rss; - rss << "Approx( " << ::Catch::Detail::stringify( m_value ) << " )"; - return rss.str(); - } + /// A digest of every key in order. A worker reports it, so the runner + /// knows the worker sees the same tests. + [[nodiscard]] const std::string & + Identity() const noexcept + { + return identity_; + } - bool Approx::equalityComparisonImpl(const double other) const { - // First try with fixed margin, then compute margin based on epsilon, scale and Approx's value - // Thanks to Richard Harris for his help refining the scaled margin value - return marginComparison(m_value, other, m_margin) - || marginComparison(m_value, other, m_epsilon * (m_scale + std::fabs(std::isinf(m_value)? 0 : m_value))); - } + /// The tests the specs of `config` select, in registration order, as + /// Catch2 selects them: hidden tests only when a spec names them. + /// Specs that matched nothing are added to `unmatched`. + [[nodiscard]] std::vector<std::size_t> + Match(const Catch::IConfig &config, + std::vector<std::string> &unmatched) const; - void Approx::setMargin(double newMargin) { - CATCH_ENFORCE(newMargin >= 0, - "Invalid Approx::margin: " << newMargin << '.' - << " Approx::Margin has to be non-negative."); - m_margin = newMargin; - } + private: + std::vector<TestEntry> tests_; + std::vector<Catch::TestCaseHandle> handles_; + std::string identity_; + }; +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - void Approx::setEpsilon(double newEpsilon) { - CATCH_ENFORCE(newEpsilon >= 0 && newEpsilon <= 1.0, - "Invalid Approx::epsilon: " << newEpsilon << '.' - << " Approx::epsilon has to be in [0, 1]"); - m_epsilon = newEpsilon; - } -namespace literals { - Approx operator ""_a(long double val) { - return Approx(val); - } - Approx operator ""_a(unsigned long long val) { - return Approx(val); - } -} // end namespace literals +#include <chrono> +#include <cstdint> +#include <optional> +#include <string> +#include <vector> -std::string StringMaker<Catch::Approx>::convert(Catch::Approx const& value) { - return value.toString(); -} +namespace Progmasoft::Catch3::Runner +{ + /// What the command line asks for. + enum class Mode : std::uint8_t + { + kRun, + kListTests, + kListTags, + kAnalytics, + kHelp, + kVersion, + /// Internal: run tests on behalf of a runner, over the protocol. + kWorker, + }; -} // end namespace Catch + enum class ListFormat : std::uint8_t + { + kText, + kNames, + kJson, + }; + /// Which of the matching tests to run. + enum class Selection : std::uint8_t + { + /// Every matching test. + kAll, + /// Those that failed, crashed, timed out or were flaky last time. + kFailed, + /// Those defined in, or covering, files changed since a revision. + kChanged, + /// Those without a recorded result. + kNew, + /// Changed, failed and new tests, and those most likely to fail. + kSmart, + }; + enum class Order : std::uint8_t + { + /// Most likely to fail per second of running time first. + kPriority, + kDeclared, + kName, + kRandom, + }; -namespace Catch { + enum class RetryMode : std::uint8_t + { + kOff, + /// Retry only what may pass on a second try: crashes, timeouts, + /// tests with a flaky history and failures that do not repeat. + kAuto, + /// Retry every failure up to the limit. + kAlways, + }; - AssertionResultData::AssertionResultData(ResultWas::OfType _resultType, LazyExpression const& _lazyExpression): - lazyExpression(_lazyExpression), - resultType(_resultType) {} + enum class QuarantineMode : std::uint8_t + { + /// Tests that keep flaking go into quarantine on their own; a failure + /// in quarantine is reported but does not fail the run. + kAuto, + /// Quarantine is reported, but failures still fail the run. + kReport, + kOff, + }; - std::string AssertionResultData::reconstructExpression() const { + enum class ColorMode : std::uint8_t + { + kAuto, + kAlways, + kNever, + }; - if( reconstructedExpression.empty() ) { - if( lazyExpression ) { - ReusableStringStream rss; - rss << lazyExpression; - reconstructedExpression = rss.str(); - } - } - return reconstructedExpression; - } + enum class Verbosity : std::uint8_t + { + kQuiet, + kNormal, + kVerbose, + }; - AssertionResult::AssertionResult( AssertionInfo const& info, AssertionResultData&& data ) - : m_info( info ), - m_resultData( CATCH_MOVE(data) ) - {} + /// One shard of a run: `index` counts from zero. + struct Shard final + { + unsigned index = 0; + unsigned count = 1; + }; - // Result was a success - bool AssertionResult::succeeded() const { - return Catch::isOk( m_resultData.resultType ); - } + /// Everything the command line can say, already checked and typed. + struct Options final + { + Mode mode = Mode::kRun; + ListFormat listFormat = ListFormat::kText; + std::string programName; + + // Selecting tests. + std::vector<std::string> testSpecs; + std::vector<std::string> sections; + /// Path filters: "c:SECTION" or "g:INDEX" (a generator value, or *). + std::vector<std::string> paths; + Selection selection = Selection::kAll; + std::string changesSince; + bool incremental = false; + Shard shard; + std::string shardDurations; + bool filenamesAsTags = false; + + // Running. + unsigned jobs = 0; + bool inProcess = false; + bool isolate = false; + /// The per-test limit; zero disables it. + std::chrono::milliseconds timeout{0}; + bool timeoutGiven = false; + bool adaptiveTimeout = true; + RetryMode retryMode = RetryMode::kAuto; + unsigned retries = 2; + unsigned repeat = 1; + unsigned failFast = 0; + Order order = Order::kPriority; + std::uint32_t seed = 0; + bool seedGiven = false; + bool bisectOrder = true; + + // How tests behave. + bool includeSuccessful = false; + bool breakIntoDebugger = false; + bool noThrow = false; + bool warnNoAssertions = false; + bool warnUnmatchedSpecs = false; + bool warnInfiniteGenerators = false; + bool allowNoTests = false; + bool skipBenchmarks = false; + bool benchmarkNoAnalysis = false; + unsigned benchmarkSamples = 100; + unsigned benchmarkResamples = 100000; + double benchmarkConfidenceInterval = 0.95; + std::chrono::milliseconds benchmarkWarmupTime{100}; + + // History and analysis. + std::string historyDirectory; + bool historyEnabled = true; + QuarantineMode quarantine = QuarantineMode::kAuto; + bool replayFailures = false; + bool analyticsJson = false; + + // Output. + Verbosity verbosity = Verbosity::kNormal; + ColorMode color = ColorMode::kAuto; + std::string junitPath; + std::string jsonPath; + std::string tracePath; + unsigned slowest = 0; + /// Created when the run starts and removed when it ends normally, so + /// a file left behind shows that the run ended early. + std::string exitGuardPath; + }; - // Result was a success, or failure is suppressed - bool AssertionResult::isOk() const { - return Catch::isOk( m_resultData.resultType ) || shouldSuppressFailure( m_info.resultDisposition ); - } + /// Reads a command line. `arguments` does not include the program. + [[nodiscard]] Expected<Options> + ParseOptions(const std::vector<std::string> &arguments); - ResultWas::OfType AssertionResult::getResultType() const { - return m_resultData.resultType; - } + /// The help text, for `--help`. + [[nodiscard]] std::string + HelpText(const std::string &programName); - bool AssertionResult::hasExpression() const { - return !m_info.capturedExpression.empty(); - } + /// The arguments a worker is started with to behave like `options`. + [[nodiscard]] std::vector<std::string> + WorkerArguments(const Options &options); +} // namespace Progmasoft::Catch3::Runner - bool AssertionResult::hasMessage() const { - return !m_resultData.message.empty(); - } +#include <chrono> +#include <cstdint> +#include <map> +#include <optional> +#include <string> +#include <vector> - std::string AssertionResult::getExpression() const { - // Possibly overallocating by 3 characters should be basically free - std::string expr; expr.reserve(m_info.capturedExpression.size() + 3); - if (isFalseTest(m_info.resultDisposition)) { - expr += "!("; - } - expr += m_info.capturedExpression; - if (isFalseTest(m_info.resultDisposition)) { - expr += ')'; - } - return expr; - } +namespace Progmasoft::Catch3::Runner +{ + /// One test the run will start, with everything decided in advance. + struct PlannedTest final + { + const TestEntry *entry = nullptr; + std::string selectionReason; + double failureProbability = 0.0; + std::int64_t predictedMicroseconds = 0; + /// Zero: no limit. + std::chrono::milliseconds timeout{0}; + bool timeoutLearned = false; + unsigned retryLimit = 0; + bool quarantined = false; + std::string quarantineReason; + /// The history shows the test passing and failing with the same + /// inputs. + bool flakyHistory = false; + /// Positions in `Plan::tests` of the tests this one waits for. + std::vector<std::size_t> dependencies; + /// What the result depends on; equal for an unchanged test. + std::string fingerprint; + /// For --replay: the tests (catalog indices) to run before it in the + /// same fresh process, and the seed it failed with. + std::optional<std::vector<std::size_t>> replaySequence; + std::optional<std::uint32_t> replaySeed; + }; - std::string AssertionResult::getExpressionInMacro() const { - if ( m_info.macroName.empty() ) { - return static_cast<std::string>( m_info.capturedExpression ); - } - std::string expr; - expr.reserve( m_info.macroName.size() + m_info.capturedExpression.size() + 4 ); - expr += m_info.macroName; - expr += "( "; - expr += m_info.capturedExpression; - expr += " )"; - return expr; - } + /// What a run will do, decided before the first test starts. + struct Plan final + { + /// In the order the scheduler should prefer. + std::vector<PlannedTest> tests; + /// Tests whose earlier result stands: up to date or out of the + /// selection but needed by nothing. + std::vector<TestResult> kept; + std::vector<std::string> unmatchedSpecs; + std::vector<std::string> notes; + /// Tests that matched the specs, before selection and sharding. + std::size_t candidates = 0; + std::map<std::string, std::string> fingerprints; + }; - bool AssertionResult::hasExpandedExpression() const { - return hasExpression() && getExpandedExpression() != getExpression(); - } + /// What the planner reads besides the options. + struct PlanInputs final + { + const Catalog *catalog = nullptr; + /// The tests the specs matched, in registration order. + std::vector<std::size_t> matched; + std::vector<std::string> unmatchedSpecs; + const History *history = nullptr; + /// Present when Git answered. + const Changes *changes = nullptr; + /// A digest of the test binary. + std::string binaryFingerprint; + }; - std::string AssertionResult::getExpandedExpression() const { - std::string expr = m_resultData.reconstructExpression(); - return expr.empty() - ? getExpression() - : expr; - } + [[nodiscard]] Expected<Plan> + MakePlan(const Options &options, const PlanInputs &inputs); - StringRef AssertionResult::getMessage() const { - return m_resultData.message; - } - SourceLineInfo AssertionResult::getSourceInfo() const { - return m_info.lineInfo; - } + /// The shard of a group of tests that must stay together, by a hash of + /// its first key. Every machine computes the same answer. + [[nodiscard]] unsigned + StableShard(const std::string &key, unsigned count) noexcept; +} // namespace Progmasoft::Catch3::Runner - StringRef AssertionResult::getTestMacroName() const { - return m_info.macroName; - } -} // end namespace Catch +#include <string> +namespace Progmasoft::Catch3::Runner +{ + struct AnalysisInputs final + { + /// The history before this run. + const History *history = nullptr; + const Changes *changes = nullptr; + const Plan *plan = nullptr; + /// The binary, for the replay command. + std::string program; + std::string changesSince; + }; + /// Classifies every failure of the run and writes down what the runner + /// can tell about its cause: a signature shared with other failures, + /// the failure's history, the change it follows, where a crash or a + /// hang happened, and a command that replays it. + void + Analyze(RunResult &run, const AnalysisInputs &inputs); -#include <fstream> + /// The signature of a failed attempt: its category, the location of the + /// first failure and its message with numbers and addresses replaced. + [[nodiscard]] std::string + FailureSignature(const Attempt &attempt); -namespace Catch { + /// A test spec that matches exactly this name. + [[nodiscard]] std::string + QuoteTestSpec(const std::string &name); +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - namespace { - static bool enableBazelEnvSupport() { -#if defined( CATCH_CONFIG_BAZEL_SUPPORT ) - return true; -#else - return Detail::getEnv( "BAZEL_TEST" ) != nullptr; -#endif - } - struct bazelShardingOptions { - unsigned int shardIndex, shardCount; - std::string shardFilePath; - }; - static Optional<bazelShardingOptions> readBazelShardingOptions() { - const auto bazelShardIndex = Detail::getEnv( "TEST_SHARD_INDEX" ); - const auto bazelShardTotal = Detail::getEnv( "TEST_TOTAL_SHARDS" ); - const auto bazelShardInfoFile = Detail::getEnv( "TEST_SHARD_STATUS_FILE" ); - - - const bool has_all = - bazelShardIndex && bazelShardTotal && bazelShardInfoFile; - if ( !has_all ) { - // We provide nice warning message if the input is - // misconfigured. - auto warn = []( const char* env_var ) { - Catch::cerr() - << "Warning: Bazel shard configuration is missing '" - << env_var << "'. Shard configuration is skipped.\n"; - }; - if ( !bazelShardIndex ) { - warn( "TEST_SHARD_INDEX" ); - } - if ( !bazelShardTotal ) { - warn( "TEST_TOTAL_SHARDS" ); - } - if ( !bazelShardInfoFile ) { - warn( "TEST_SHARD_STATUS_FILE" ); - } - return {}; - } +#include <cstddef> +#include <cstdio> +#include <string> - auto shardIndex = parseUInt( bazelShardIndex ); - if ( !shardIndex ) { - Catch::cerr() - << "Warning: could not parse 'TEST_SHARD_INDEX' ('" << bazelShardIndex - << "') as unsigned int.\n"; - return {}; - } - auto shardTotal = parseUInt( bazelShardTotal ); - if ( !shardTotal ) { - Catch::cerr() - << "Warning: could not parse 'TEST_TOTAL_SHARD' ('" - << bazelShardTotal << "') as unsigned int.\n"; - return {}; - } +namespace Progmasoft::Catch3::Runner +{ + /// The report on the terminal: progress while the run goes on, then the + /// failures with their analysis and a summary. + class ConsoleReport final + { + public: + ConsoleReport(const Options &options, std::FILE *output); - return bazelShardingOptions{ - *shardIndex, *shardTotal, bazelShardInfoFile }; + void + RunStarting(std::size_t tests, + std::size_t workers, + std::uint32_t seed, + const std::vector<std::string> ¬es); - } - } // end namespace + void + TestFinished(const TestResult &test); + void + Retrying(const TestResult &test, const Attempt &attempt); - bool operator==( ProcessedReporterSpec const& lhs, - ProcessedReporterSpec const& rhs ) { - return lhs.name == rhs.name && - lhs.outputFilename == rhs.outputFilename && - lhs.colourMode == rhs.colourMode && - lhs.verbosity == rhs.verbosity && - lhs.customOptions == rhs.customOptions; - } + void + RunEnded(const RunResult &run); - bool operator==( PathFilter const& lhs, PathFilter const& rhs ) { - return lhs.type == rhs.type && lhs.filter == rhs.filter; - } + /// Text in a colour, when colours are on. + [[nodiscard]] std::string + Paint(std::string_view text, const char *colour) const; - Config::Config( ConfigData const& data ): - m_data( data ) { - // We need to trim filter specs to avoid trouble with superfluous - // whitespace (esp. important for bdd macros, as those are manually - // aligned with whitespace). + private: + void + ClearProgress(); - for (auto& elem : m_data.testsOrTags) { - elem = trim(elem); + void + PrintFailure(const TestResult &test); + + const Options &options_; + std::FILE *output_; + bool colour_ = false; + bool terminal_ = false; + std::size_t total_ = 0; + std::size_t finished_ = 0; + std::size_t failed_ = 0; + bool progressShown_ = false; + }; + + /// Writes a trace of the run in the Trace Event Format that + /// chrome://tracing and Perfetto open: one track per worker, one slice + /// per attempt and section. + [[nodiscard]] std::string + TraceJson(const RunResult &run); +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +#include <functional> +#include <memory> +#include <string> + + +namespace Catch +{ + struct ConfigData; +} // namespace Catch + +namespace Progmasoft::Catch3::Runner +{ + /// The Catch2 configuration a process that runs tests uses for + /// `options`. + [[nodiscard]] Catch::ConfigData + ToConfigData(const Options &options, const std::string &programName); + + /// Receives the events of the tests a `TestExecutor` runs, as the JSON + /// messages a worker sends; see `Protocol.hpp`. + using EventSink = std::function<void(const JsonValue &)>; + + /// Runs tests of this binary in this process with Catch2's engine: the + /// sections, generators, assertions and listeners are Catch2's. + /// + /// One executor lives for the whole process, because Catch2 allows one + /// run context per process; listeners see one test run, from the first + /// test to the destruction of the executor. + class TestExecutor final + { + public: + TestExecutor(const Options &options, + const Catalog &catalog, + const std::string &programName, + EventSink sink); + TestExecutor(const TestExecutor &) = delete; + TestExecutor & + operator=(const TestExecutor &) = delete; + ~TestExecutor(); + + /// Runs one test. `temporaryDirectory` becomes TMPDIR, TMP and TEMP + /// while it runs; empty leaves them alone. + void + Run(std::size_t index, + std::size_t attempt, + const std::string &temporaryDirectory); + + /// The executor's Catch2 configuration and run context. + struct State; + + private: + std::unique_ptr<State> state_; + }; + + /// Serves a runner over the standard streams until it says to stop. + [[nodiscard]] int + RunWorker(const Options &options, const std::string &programName); +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + + +#include <chrono> +#include <filesystem> +#include <string> +#include <string_view> + +namespace Progmasoft::Catch3::Runner +{ + /// A lock between processes: a file created exclusively beside the one + /// it protects. A lock file older than `stale` belongs to a process that + /// died and is broken. + class FileLock final + { + public: + FileLock(const std::filesystem::path &file, + std::chrono::milliseconds wait, + std::chrono::milliseconds stale = std::chrono::seconds(30)); + FileLock(const FileLock &) = delete; + FileLock & + operator=(const FileLock &) = delete; + ~FileLock(); + + /// Whether the lock was taken before `wait` ran out. + [[nodiscard]] bool + Held() const noexcept + { + return held_; } - // Insert the default reporter if user hasn't asked for a specific one - if ( m_data.reporterSpecifications.empty() ) { -#if defined( CATCH_CONFIG_DEFAULT_REPORTER ) - const auto default_spec = CATCH_CONFIG_DEFAULT_REPORTER; -#else - const auto default_spec = "console"; -#endif - auto parsed = parseReporterSpec(default_spec); - CATCH_ENFORCE( parsed, - "Cannot parse the provided default reporter spec: '" - << default_spec << '\'' ); - m_data.reporterSpecifications.push_back( std::move( *parsed ) ); + private: + std::filesystem::path path_; + bool held_ = false; + }; + + [[nodiscard]] Expected<std::string> + ReadFile(const std::filesystem::path &file); + + /// Replaces `file` with `contents` in one rename, so that a reader sees + /// the old or the new file, never half of one. Creates the directory. + Status + WriteFileAtomically(const std::filesystem::path &file, + std::string_view contents); + + /// Creates an empty file only the user can read and write, truncating a + /// regular file of that name. A symbolic link or anything else that is + /// not a regular file is refused, so the path cannot redirect the write. + Status + CreatePrivateFile(const std::filesystem::path &file); + + /// A name for a temporary file that no other process or thread uses. + [[nodiscard]] std::string + UniqueName(); +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + + +#include <array> +#include <cstdint> +#include <filesystem> +#include <string> +#include <string_view> + +namespace Progmasoft::Catch3::Runner +{ + /// SHA-256, for fingerprints that must not collide by accident: the + /// test binary, the source files a test depends on. + class Sha256 final + { + public: + Sha256(); + + void + Update(const void *data, std::size_t size); + + void + Update(std::string_view text) + { + Update(text.data(), text.size()); } - // Reading bazel env vars can change some parts of the config data, - // so we have to process the bazel env before acting on the config. - if ( enableBazelEnvSupport() ) { - readBazelEnvVars(); + /// The digest as 64 lowercase hexadecimal digits. The object cannot + /// be updated afterwards. + [[nodiscard]] std::string + HexDigest(); + + private: + void + Compress(const std::uint8_t *block); + + std::array<std::uint32_t, 8> state_{}; + std::array<std::uint8_t, 64> buffer_{}; + std::size_t bufferSize_ = 0; + std::uint64_t totalSize_ = 0; + }; + + [[nodiscard]] std::string + Sha256Hex(std::string_view text); + + /// The SHA-256 of a file's contents. + [[nodiscard]] Expected<std::string> + Sha256File(const std::filesystem::path &path); + + /// FNV-1a: a fast, stable 64-bit hash for sharding and signatures, + /// where a collision only costs balance or merges two groups. + [[nodiscard]] std::uint64_t + Fnv1a64(std::string_view text) noexcept; + + /// The value as 16 lowercase hexadecimal digits. + [[nodiscard]] std::string + ToHex(std::uint64_t value); +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +#include <chrono> +#include <cstdint> +#include <filesystem> +#include <memory> +#include <optional> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + + +namespace Progmasoft::Catch3::Runner +{ + /// How a process ended. + struct ExitStatus final + { + enum class Kind : std::uint8_t + { + /// It returned from `main` or called `exit` with `code`. + kExited, + /// A signal ended it (POSIX); `code` is the signal number. + kSignaled, + /// It ended with a structured exception (Windows); `code` is the + /// exception code, such as 0xC0000005 for an access violation. + kException, + }; + + Kind kind = Kind::kExited; + std::int64_t code = 0; + + [[nodiscard]] bool + Succeeded() const noexcept + { + return kind == Kind::kExited && code == 0; } - // Bazel support can modify the test specs, so parsing has to happen - // after reading Bazel env vars. - TestSpecParser parser( ITagAliasRegistry::get() ); - if ( !m_data.testsOrTags.empty() ) { - m_hasTestFilters = true; - for ( auto const& testOrTags : m_data.testsOrTags ) { - parser.parse( testOrTags ); - } + /// Whether the process died rather than exiting. + [[nodiscard]] bool + Crashed() const noexcept + { + return kind != Kind::kExited; } - m_testSpec = parser.testSpec(); + /// A description such as "exit code 3", "signal 11 (SIGSEGV)" or + /// "exception 0xC0000005 (access violation)". + [[nodiscard]] std::string + Describe() const; + }; - // We now fixup the reporter specs to handle default output spec, - // default colour spec, etc - bool defaultOutputUsed = false; - for ( auto const& reporterSpec : m_data.reporterSpecifications ) { - // We do the default-output check separately, while always - // using the default output below to make the code simpler - // and avoid superfluous copies. - if ( reporterSpec.outputFile().none() ) { - CATCH_ENFORCE( !defaultOutputUsed, - "Internal error: cannot use default output for " - "multiple reporters" ); - defaultOutputUsed = true; - } + /// The end of a pipe this process reads from. + class PipeReader final + { + public: + PipeReader() = default; + explicit PipeReader(std::intptr_t handle) noexcept + : handle_{ handle } + {} + PipeReader(const PipeReader &) = delete; + PipeReader & + operator=(const PipeReader &) = delete; + PipeReader(PipeReader &&other) noexcept; + PipeReader & + operator=(PipeReader &&other) noexcept; + ~PipeReader(); + + /// Reads at most `size` bytes. Returns the number read, 0 at the end + /// of the stream and -1 on an error. + [[nodiscard]] std::int64_t + Read(char *buffer, std::size_t size) noexcept; - m_processedReporterSpecs.push_back( ProcessedReporterSpec{ - reporterSpec.name(), - reporterSpec.outputFile() ? *reporterSpec.outputFile() - : data.defaultOutputFilename, - reporterSpec.colourMode().valueOr( data.defaultColourMode ), - reporterSpec.verbosity().valueOr( data.verbosity ), - reporterSpec.customOptions() } ); - } - } + void + Close() noexcept; - Config::~Config() = default; + [[nodiscard]] bool + IsOpen() const noexcept; + private: + std::intptr_t handle_ = -1; + }; - bool Config::listTests() const { return m_data.listTests; } - bool Config::listTags() const { return m_data.listTags; } - bool Config::listReporters() const { return m_data.listReporters; } - bool Config::listListeners() const { return m_data.listListeners; } + /// The end of a pipe this process writes to. + class PipeWriter final + { + public: + PipeWriter() = default; + explicit PipeWriter(std::intptr_t handle) noexcept + : handle_{ handle } + {} + PipeWriter(const PipeWriter &) = delete; + PipeWriter & + operator=(const PipeWriter &) = delete; + PipeWriter(PipeWriter &&other) noexcept; + PipeWriter & + operator=(PipeWriter &&other) noexcept; + ~PipeWriter(); + + /// Writes all of `data`, or reports why it could not. + Status + Write(std::string_view data) noexcept; - std::vector<std::string> const& Config::getTestsOrTags() const { return m_data.testsOrTags; } - std::vector<PathFilter> const& Config::getPathFilters() const { return m_data.pathFilters; } - bool Config::useNewFilterBehaviour() const { return m_data.useNewPathFilteringBehaviour; } + void + Close() noexcept; - std::vector<ReporterSpec> const& Config::getReporterSpecs() const { - return m_data.reporterSpecifications; - } + [[nodiscard]] bool + IsOpen() const noexcept; - std::vector<ProcessedReporterSpec> const& - Config::getProcessedReporterSpecs() const { - return m_processedReporterSpecs; - } + private: + std::intptr_t handle_ = -1; + }; - TestSpec const& Config::testSpec() const { return m_testSpec; } - bool Config::hasTestFilters() const { return m_hasTestFilters; } + /// What to start, and how. + struct ChildOptions final + { + /// The program and its arguments. The program is a path; it is not + /// looked up on `PATH` unless `searchPath` is set. + std::vector<std::string> arguments; + /// Variables added to, or replacing, those of this process. + std::vector<std::pair<std::string, std::string>> environment; + /// The working directory; empty keeps the current one. + std::filesystem::path workingDirectory; + bool searchPath = false; + }; - bool Config::showHelp() const { return m_data.showHelp; } + /// A child process with its standard streams connected to pipes. + /// + /// The child and every process it starts in turn form one group (a job + /// object on Windows, a process group elsewhere), so `Kill` ends a test + /// together with the helpers it spawned. Pipes are not inherited by + /// any other child, so the end of a stream is seen as soon as its + /// process is gone. + class ChildProcess final + { + public: + ChildProcess(const ChildProcess &) = delete; + ChildProcess & + operator=(const ChildProcess &) = delete; + ~ChildProcess(); - std::string const& Config::getExitGuardFilePath() const { return m_data.prematureExitGuardFilePath; } + [[nodiscard]] static Expected<std::unique_ptr<ChildProcess>> + Start(const ChildOptions &options); - // IConfig interface - bool Config::allowThrows() const { return !m_data.noThrow; } - StringRef Config::name() const { return m_data.name.empty() ? m_data.processName : m_data.name; } - bool Config::includeSuccessfulResults() const { return m_data.showSuccessfulTests; } - bool Config::warnAboutMissingAssertions() const { - return !!( m_data.warnings & WarnAbout::NoAssertions ); - } - bool Config::warnAboutUnmatchedTestSpecs() const { - return !!( m_data.warnings & WarnAbout::UnmatchedTestSpec ); - } - bool Config::warnAboutInfiniteGenerators() const { - return !!( m_data.warnings & WarnAbout::InfiniteGenerator ); - } - bool Config::zeroTestsCountAsSuccess() const { return m_data.allowZeroTests; } - ShowDurations Config::showDurations() const { return m_data.showDurations; } - double Config::minDuration() const { return m_data.minDuration; } - TestRunOrder Config::runOrder() const { return m_data.runOrder; } - uint32_t Config::rngSeed() const { return m_data.rngSeed; } - bool Config::rngSeedWasFixed() const { return m_data.rngSeedWasFixed; } - unsigned int Config::shardCount() const { return m_data.shardCount; } - unsigned int Config::shardIndex() const { return m_data.shardIndex; } - ColourMode Config::defaultColourMode() const { return m_data.defaultColourMode; } - bool Config::shouldDebugBreak() const { return m_data.shouldDebugBreak; } - int Config::abortAfter() const { return m_data.abortAfter; } - bool Config::showInvisibles() const { return m_data.showInvisibles; } - Verbosity Config::verbosity() const { return m_data.verbosity; } + /// The child's standard input. + [[nodiscard]] PipeWriter & + Input() noexcept + { + return input_; + } - bool Config::skipBenchmarks() const { return m_data.skipBenchmarks; } - bool Config::benchmarkNoAnalysis() const { return m_data.benchmarkNoAnalysis; } - unsigned int Config::benchmarkSamples() const { return m_data.benchmarkSamples; } - double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; } - unsigned int Config::benchmarkResamples() const { return m_data.benchmarkResamples; } - std::chrono::milliseconds Config::benchmarkWarmupTime() const { return std::chrono::milliseconds(m_data.benchmarkWarmupTime); } + /// The child's standard output. + [[nodiscard]] PipeReader & + Output() noexcept + { + return output_; + } - void Config::readBazelEnvVars() { - // Register a JUnit reporter for Bazel. Bazel sets an environment - // variable with the path to XML output. If this file is written to - // during test, Bazel will not generate a default XML output. - // This allows the XML output file to contain higher level of detail - // than what is possible otherwise. - const auto bazelOutputFile = Detail::getEnv( "XML_OUTPUT_FILE" ); - - if ( bazelOutputFile ) { - m_data.reporterSpecifications.push_back( - { "junit", std::string( bazelOutputFile ), {}, {}, {} } ); - } - - const auto bazelTestSpec = Detail::getEnv( "TESTBRIDGE_TEST_ONLY" ); - if ( bazelTestSpec ) { - // Presumably the test spec from environment should overwrite - // the one we got from CLI (if we got any) - m_data.testsOrTags.clear(); - m_data.testsOrTags.push_back( bazelTestSpec ); - } - - const auto bazelShardOptions = readBazelShardingOptions(); - if ( bazelShardOptions ) { - std::ofstream f( bazelShardOptions->shardFilePath, - std::ios_base::out | std::ios_base::trunc ); - if ( f.is_open() ) { - f << ""; - m_data.shardIndex = bazelShardOptions->shardIndex; - m_data.shardCount = bazelShardOptions->shardCount; - } - } - - const auto bazelExitGuardFile = Detail::getEnv( "TEST_PREMATURE_EXIT_FILE" ); - if (bazelExitGuardFile) { - m_data.prematureExitGuardFilePath = bazelExitGuardFile; - } - - const auto bazelRandomSeed = Detail::getEnv( "TEST_RANDOM_SEED" ); - if ( bazelRandomSeed ) { - auto parsedSeed = parseUInt( bazelRandomSeed, 0 ); - if ( !parsedSeed ) { - // Currently we handle issues with parsing other Bazel Env - // options by warning and ignoring the issue. So we do the - // same for random seed option. - Catch::cerr() - << "Warning: could not parse 'TEST_RANDOM_SEED' ('" - << bazelRandomSeed << "') as proper seed.\n"; - } else { - m_data.rngSeed = *parsedSeed; - m_data.rngSeedWasFixed = true; - } + /// The child's standard error. + [[nodiscard]] PipeReader & + Errors() noexcept + { + return errors_; } - } -} // end namespace Catch + /// Ends the child and every process it started. Safe to call more + /// than once and after the child ended. + void + Kill() noexcept; + /// Waits for the child to end. + [[nodiscard]] ExitStatus + Wait() noexcept; + /// Waits at most `timeout`; nothing when the child still runs. + [[nodiscard]] std::optional<ExitStatus> + WaitFor(std::chrono::milliseconds timeout) noexcept; + [[nodiscard]] std::int64_t + Id() const noexcept + { + return id_; + } + private: + ChildProcess() = default; + + PipeWriter input_; + PipeReader output_; + PipeReader errors_; + std::int64_t id_ = 0; + std::intptr_t process_ = -1; + std::intptr_t group_ = -1; + std::optional<ExitStatus> exit_; + }; -namespace Catch { - std::uint32_t getSeed() { - return getCurrentContext().getConfig()->rngSeed(); - } -} + /// The result of a command run to completion. + struct CapturedRun final + { + ExitStatus status; + std::string output; + std::string errors; + }; + /// Runs a command with no input and collects its output. + [[nodiscard]] Expected<CapturedRun> + RunAndCapture(const ChildOptions &options); + /// The two streams a worker talks to the runner over. + struct WorkerStreams final + { + PipeReader commands; + PipeWriter events; + }; + /// Takes standard input and output of this process for the runner + /// protocol. Afterwards standard input reads nothing and standard output + /// goes to standard error, so what a test prints cannot corrupt a + /// message. + [[nodiscard]] Expected<WorkerStreams> + TakeStandardStreams(); -#include <cassert> -#include <stack> + /// The path of the running executable. + [[nodiscard]] std::filesystem::path + CurrentExecutable(const char *argumentZero); -namespace Catch { + /// The number of hardware threads, at least 1. + [[nodiscard]] unsigned + HardwareThreads() noexcept; - //////////////////////////////////////////////////////////////////////////// + /// The value of an environment variable; empty when it is not set. + [[nodiscard]] std::string + ReadEnvironment(const char *name); + [[nodiscard]] bool + HasEnvironment(const char *name); - ScopedMessage::ScopedMessage( MessageBuilder&& builder ): - m_messageId( builder.m_info.sequence ) { - MessageInfo info( CATCH_MOVE( builder.m_info ) ); - info.message = builder.m_stream.str(); - Detail::pushScopedMessage( CATCH_MOVE( info ) ); - } + /// Whether standard output is a terminal. + [[nodiscard]] bool + IsStandardOutputTerminal() noexcept; - ScopedMessage::ScopedMessage( ScopedMessage&& old ) noexcept: - m_messageId( old.m_messageId ) { - old.m_moved = true; - } + /// Turns on ANSI colour sequences on a Windows console; true when the + /// terminal shows them. + [[nodiscard]] bool + EnableTerminalColours() noexcept; - ScopedMessage::~ScopedMessage() { - if ( !m_moved ) { Detail::popScopedMessage( m_messageId ); } - } + /// Keeps a crash of a worker on Windows from opening an error-reporting + /// dialog that would wait for a person. Elsewhere it does nothing. + void + QuietCrashReporting() noexcept; + /// The id of this process. + [[nodiscard]] std::int64_t + CurrentProcessId() noexcept; - Capturer::Capturer( StringRef macroName, - SourceLineInfo const& lineInfo, - ResultWas::OfType resultType, - StringRef names, - bool isScoped): - m_isScoped(isScoped) { - auto trimmed = [&] (size_t start, size_t end) { - while (names[start] == ',' || isspace(static_cast<unsigned char>(names[start]))) { - ++start; - } - while (names[end] == ',' || isspace(static_cast<unsigned char>(names[end]))) { - --end; - } - return names.substr(start, end - start + 1); - }; - auto skipq = [&] (size_t start, char quote) { - for (auto i = start + 1; i < names.size() ; ++i) { - if (names[i] == quote) - return i; - if (names[i] == '\\') - ++i; - } - CATCH_INTERNAL_ERROR("CAPTURE parsing encountered unmatched quote"); - }; + /// Writes `text` to descriptor 2 directly, past any stream buffer or + /// redirection, as far as it goes; for use while the process exits. + void + WriteStandardError(std::string_view text) noexcept; +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - size_t start = 0; - std::stack<char> openings; - for (size_t pos = 0; pos < names.size(); ++pos) { - char c = names[pos]; - switch (c) { - case '[': - case '{': - case '(': - // It is basically impossible to disambiguate between - // comparison and start of template args in this context -// case '<': - openings.push(c); - break; - case ']': - case '}': - case ')': -// case '>': - openings.pop(); - break; - case '"': - case '\'': - pos = skipq(pos, c); - break; - case ',': - if (start != pos && openings.empty()) { - m_messages.emplace_back(macroName, lineInfo, resultType); - m_messages.back().message += trimmed(start, pos); - m_messages.back().message += " := "_sr; - start = pos; - } - break; - default:; // noop - } - } - assert(openings.empty() && "Mismatched openings"); - m_messages.emplace_back(macroName, lineInfo, resultType); - m_messages.back().message += trimmed(start, names.size() - 1); - m_messages.back().message += " := "_sr; - } - Capturer::~Capturer() { - assert( m_captured == m_messages.size() ); - if ( m_isScoped ) { - for ( auto const& message : m_messages ) { - Detail::popScopedMessage( message.sequence ); - } - } - } - void Capturer::captureValue( size_t index, std::string const& value ) { - assert( index < m_messages.size() ); - m_messages[index].message += value; - if ( m_isScoped ) { - Detail::pushScopedMessage( CATCH_MOVE( m_messages[index] ) ); - } else { - Detail::addUnscopedMessage( CATCH_MOVE( m_messages[index] ) ); +#include <mutex> +#include <optional> +#include <string> + +namespace Progmasoft::Catch3::Runner +{ + /// The runner and its workers exchange one JSON object per line. Every + /// object has a member "t" naming its kind. + /// + /// From the runner to a worker: + /// run {index, attempt, temp} run one test + /// quit {} end the process + /// + /// From a worker to the runner: + /// hello {pid, identity, tests, protocol} + /// start {index, attempt} + /// section {begin: true, name, file, line, depth, at} + /// section {begin: false, depth, at, duration} + /// at {location} the last assertion started + /// assertion {an AssertionRecord} + /// benchmark {a BenchmarkRecord} + /// fatal {message, location} sent before the worker dies + /// end {index, attempt, outcome, category, passed, failed, + /// duration, stdout, stderr} + inline constexpr int kWorkerProtocolVersion = 1; + + /// Reads lines from a pipe. + class LineReader final + { + public: + explicit LineReader(PipeReader &pipe) + : pipe_{pipe} + { } - m_captured++; - } -} // end namespace Catch + /// The next line without its end, or nothing at the end of the + /// stream. + [[nodiscard]] std::optional<std::string> + Next(); + private: + PipeReader &pipe_; + std::string buffer_; + std::size_t scanned_ = 0; + bool ended_ = false; + }; + /// Writes messages to a pipe, one line each; safe from several + /// threads, since a test may assert from threads of its own. + class MessageWriter final + { + public: + explicit MessageWriter(PipeWriter &pipe) + : pipe_{pipe} + { + } + Status + Send(const JsonValue &message); -#include <exception> + private: + PipeWriter &pipe_; + std::mutex mutex_; + }; -namespace Catch { + /// A message of kind `kind` with no other members yet. + [[nodiscard]] JsonValue + Message(std::string_view kind); +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - namespace { - class RegistryHub : public IRegistryHub, - public IMutableRegistryHub, - private Detail::NonCopyable { - public: // IRegistryHub - RegistryHub() = default; - ReporterRegistry const& getReporterRegistry() const override { - return m_reporterRegistry; - } - ITestCaseRegistry const& getTestCaseRegistry() const override { - return m_testCaseRegistry; - } - IExceptionTranslatorRegistry const& getExceptionTranslatorRegistry() const override { - return m_exceptionTranslatorRegistry; - } - ITagAliasRegistry const& getTagAliasRegistry() const override { - return m_tagAliasRegistry; - } - StartupExceptionRegistry const& getStartupExceptionRegistry() const override { - return m_exceptionRegistry; - } +namespace Progmasoft::Catch3::Runner +{ + /// The version of the JSON shapes below. It is written into every + /// report and protocol stream; a reader refuses a newer major version. + inline constexpr int kResultsFormatVersion = 1; - public: // IMutableRegistryHub - void registerReporter( std::string const& name, IReporterFactoryPtr factory ) override { - m_reporterRegistry.registerReporter( name, CATCH_MOVE(factory) ); - } - void registerListener( Detail::unique_ptr<EventListenerFactory> factory ) override { - m_reporterRegistry.registerListener( CATCH_MOVE(factory) ); - } - void registerTest( Detail::unique_ptr<TestCaseInfo>&& testInfo, Detail::unique_ptr<ITestInvoker>&& invoker ) override { - m_testCaseRegistry.registerTest( CATCH_MOVE(testInfo), CATCH_MOVE(invoker) ); - } - void registerTranslator( Detail::unique_ptr<IExceptionTranslator>&& translator ) override { - m_exceptionTranslatorRegistry.registerTranslator( CATCH_MOVE(translator) ); - } - void registerTagAlias( std::string const& alias, std::string const& tag, SourceLineInfo const& lineInfo ) override { - m_tagAliasRegistry.add( alias, tag, lineInfo ); - } - void registerStartupException() noexcept override { -#if !defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) - m_exceptionRegistry.add(std::current_exception()); -#else - CATCH_INTERNAL_ERROR("Attempted to register active exception under CATCH_CONFIG_DISABLE_EXCEPTIONS!"); -#endif - } + [[nodiscard]] JsonValue + ToJson(const AssertionRecord &assertion); - ITestCaseRegistry& getMutableTestCaseRegistry() override { - return m_testCaseRegistry; - } + [[nodiscard]] AssertionRecord + AssertionFromJson(const JsonValue &value); - private: - TestRegistry m_testCaseRegistry; - ReporterRegistry m_reporterRegistry; - ExceptionTranslatorRegistry m_exceptionTranslatorRegistry; - TagAliasRegistry m_tagAliasRegistry; - StartupExceptionRegistry m_exceptionRegistry; - }; - } + [[nodiscard]] JsonValue + ToJson(const BenchmarkRecord &benchmark); - using RegistryHubSingleton = Singleton<RegistryHub, IRegistryHub, IMutableRegistryHub>; + [[nodiscard]] BenchmarkRecord + BenchmarkFromJson(const JsonValue &value); - IRegistryHub const& getRegistryHub() { - return RegistryHubSingleton::get(); - } - IMutableRegistryHub& getMutableRegistryHub() { - return RegistryHubSingleton::getMutable(); - } - void cleanUp() { - cleanupSingletons(); - } - std::string translateActiveException() { - return getRegistryHub().getExceptionTranslatorRegistry().translateActiveException(); - } + [[nodiscard]] JsonValue + ToJson(const SectionRecord §ion); + [[nodiscard]] SectionRecord + SectionFromJson(const JsonValue &value); -} // end namespace Catch + [[nodiscard]] JsonValue + ToJson(const Attempt &attempt); + [[nodiscard]] Attempt + AttemptFromJson(const JsonValue &value); + [[nodiscard]] JsonValue + ToJson(const TestResult &test); -#include <cassert> -#include <cerrno> -#include <cstdio> -#include <cstdlib> -#include <cstring> -#include <exception> -#include <iomanip> + [[nodiscard]] TestResult + TestResultFromJson(const JsonValue &value); -#if defined( _WIN32 ) -# include <fcntl.h> -# include <io.h> -# include <sys/stat.h> -#else -# include <fcntl.h> -# include <sys/stat.h> -# include <unistd.h> -#endif + /// The whole run, with a summary: the `--json` report. + [[nodiscard]] JsonValue + ToJson(const RunResult &run); -namespace Catch { + [[nodiscard]] Expected<RunResult> + RunResultFromJson(const JsonValue &value); +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - namespace { - IEventListenerPtr createReporter(std::string const& reporterName, ReporterConfig&& config) { - auto reporter = Catch::getRegistryHub().getReporterRegistry().create(reporterName, CATCH_MOVE(config)); - CATCH_ENFORCE(reporter, "No reporter registered with name: '" << reporterName << '\''); - return reporter; - } - IEventListenerPtr prepareReporters(Config const* config) { - if (Catch::getRegistryHub().getReporterRegistry().getListeners().empty() - && config->getProcessedReporterSpecs().size() == 1) { - auto const& spec = config->getProcessedReporterSpecs()[0]; - return createReporter( - spec.name, - ReporterConfig( config, - makeStream( spec.outputFilename ), - spec.colourMode, - spec.verbosity, - spec.customOptions ) ); - } +#include <functional> +#include <string> +#include <vector> - auto multi = Detail::make_unique<MultiReporter>(config); +namespace Progmasoft::Catch3::Runner +{ + /// Hooks the console report uses to show progress. + struct SchedulerEvents final + { + /// A test has its verdict. + std::function<void(const TestResult &)> testFinished; + /// An attempt ended and another will follow: a retry, a repeat. + std::function<void(const TestResult &, const Attempt &)> retrying; + }; - auto const& listeners = Catch::getRegistryHub().getReporterRegistry().getListeners(); - for (auto const& listener : listeners) { - multi->addListener(listener->create(config)); - } + struct ScheduleOutcome final + { + /// One result per planned test, in plan order. + std::vector<TestResult> tests; + std::size_t workers = 0; + std::vector<std::string> notes; + /// The workers could not be started at all. + bool infrastructureFailed = false; + }; - for ( auto const& reporterSpec : config->getProcessedReporterSpecs() ) { - multi->addReporter( createReporter( - reporterSpec.name, - ReporterConfig( config, - makeStream( reporterSpec.outputFilename ), - reporterSpec.colourMode, - reporterSpec.verbosity, - reporterSpec.customOptions ) ) ); - } + /// Runs a plan: in worker processes started from `program`, or in this + /// process with `--in-process`. + /// + /// Tests go to whichever worker is free, most important first. A worker + /// that crashes or hangs is replaced; the test it was running is + /// recorded as crashed or timed out. A failure is retried as the plan + /// allows, first in a fresh process: a test that then passes was either + /// flaky or broken by a test that ran before it in the same process, + /// and the scheduler finds that test by bisecting the ones that ran + /// before. + [[nodiscard]] ScheduleOutcome + RunPlan(const Options &options, + const Plan &plan, + const Catalog &catalog, + const std::string &program, + const std::string &programName, + const SchedulerEvents &events); +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +// The state the scheduler keeps of jobs, workers and tests. - return multi; - } +#include <chrono> +#include <cstddef> +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <string_view> +#include <thread> +#include <vector> - class TestGroup { - public: - explicit TestGroup(IEventListenerPtr&& reporter, Config const* config): - m_reporter(reporter.get()), - m_config{config}, - m_context{config, CATCH_MOVE(reporter)} { - - assert( m_config->testSpec().getInvalidSpecs().empty() && - "Invalid test specs should be handled before running tests" ); - - // Note that we are working with pointers into this vector - // of sorted test cases, so sorting/unique-ing the pointers - // keeps the user-specified order. - auto const& allTestCases = getAllTestCasesSorted(*m_config); - auto const& testSpec = m_config->testSpec(); - m_tests.reserve( allTestCases.size() ); - if ( !testSpec.hasFilters() ) { - for ( auto const& test : allTestCases ) { - if ( !test.getTestCaseInfo().isHidden() ) { - m_tests.push_back( &test ); - } - } - } else { - m_matches = testSpec.matchesByFilter( allTestCases, *m_config ); - for ( auto const& match : m_matches ) { - m_tests.insert( m_tests.end(), - match.tests.begin(), - match.tests.end() ); - } - std::sort( m_tests.begin(), m_tests.end() ); - m_tests.erase( - std::unique( m_tests.begin(), m_tests.end() ), - m_tests.end() ); - } - m_tests = createShard(CATCH_MOVE(m_tests), m_config->shardCount(), m_config->shardIndex()); - } +namespace Progmasoft::Catch3::Runner::Scheduling +{ + using Clock = std::chrono::steady_clock; - Totals execute() { - Totals totals; - for (auto const& testCase : m_tests) { - if (!m_context.aborting()) - totals += m_context.runTest(*testCase); - else - m_reporter->skipTest(testCase->getTestCaseInfo()); - } + inline constexpr std::size_t kMaximumRecordedAssertions = 200; + inline constexpr std::size_t kMaximumProcessOutput = 64 * 1024; + inline constexpr std::size_t kMaximumProbes = 12; + inline constexpr std::size_t kStartupFailuresToGiveUp = 3; + inline constexpr std::chrono::milliseconds kStartupTimeout{ 60000 }; - for (auto const& match : m_matches) { - if (match.tests.empty()) { - m_unmatchedTestSpecs = true; - m_reporter->noMatchingTestCases( match.name ); - } - } + inline std::int64_t + Microseconds(Clock::duration duration) + { + return std::chrono::duration_cast<std::chrono::microseconds>(duration) + .count(); + } - return totals; - } + inline void + AppendBounded(std::string &target, std::string_view text) + { + target.append(text); + if (target.size() > kMaximumProcessOutput) + { + // The end of the output says most about a crash. + target.erase(0, target.size() - kMaximumProcessOutput); + } + } - bool hadUnmatchedTestSpecs() const { - return m_unmatchedTestSpecs; - } + inline bool + Satisfies(Outcome outcome) + { + return outcome == Outcome::kPassed + || outcome == Outcome::kFailedAsExpected + || outcome == Outcome::kUpToDate; + } + inline bool + Passing(Outcome outcome) + { + return outcome == Outcome::kPassed + || outcome == Outcome::kFailedAsExpected + || outcome == Outcome::kSkipped; + } - private: - IEventListener* m_reporter; - Config const* m_config; - RunContext m_context; - std::vector<TestCaseHandle const*> m_tests; - TestSpec::Matches m_matches; - bool m_unmatchedTestSpecs = false; + /// What a worker is asked to do: run tests in order, the last one + /// being the one the job is about. + struct Job final + { + enum class Kind : std::uint8_t + { + /// An attempt of a planned test. + kAttempt, + /// A run that tells whether a failure depends on the tests + /// before it; its result is not an attempt. + kProbe, }; - void applyFilenamesAsTags() { - getMutableRegistryHub().getMutableTestCaseRegistry().enableFilenameTags(); - } - - std::FILE* openGuardFile( std::string const& guardFilePath, - int& error ) { -#if defined( _MSC_VER ) - std::FILE* file = nullptr; - error = fopen_s( &file, guardFilePath.c_str(), "w" ); - return error ? nullptr : file; -#elif defined( _WIN32 ) - const int descriptor = _open( guardFilePath.c_str(), - _O_WRONLY | _O_CREAT | _O_TRUNC | - _O_BINARY | _O_NOINHERIT, - _S_IREAD | _S_IWRITE ); - if ( descriptor == -1 ) { - error = errno; - return nullptr; - } - std::FILE* const file = _fdopen( descriptor, "w" ); - if ( file == nullptr ) { - error = errno; - _close( descriptor ); - } - return file; -#else - int flags = O_WRONLY | O_CREAT | O_TRUNC; -# if defined( O_NONBLOCK ) - flags |= O_NONBLOCK; -# endif -# if defined( O_CLOEXEC ) - flags |= O_CLOEXEC; -# endif -# if defined( O_NOFOLLOW ) - // Do not let a stale guard-file symlink redirect this write. - flags |= O_NOFOLLOW; -# endif - const int descriptor = - ::open( guardFilePath.c_str(), flags, S_IRUSR | S_IWUSR ); - if ( descriptor == -1 ) { - error = errno; - return nullptr; - } + Kind kind = Kind::kAttempt; + std::size_t test = 0; + /// Catalog indices to run first, in the same process. + std::vector<std::size_t> before; + bool fresh = false; + std::optional<std::uint32_t> seed; + /// For a probe: the tests that ran before it. + std::vector<std::size_t> probeSequence; + }; - struct stat fileStatus = {}; - if ( ::fstat( descriptor, &fileStatus ) != 0 ) { - error = errno; - ::close( descriptor ); - return nullptr; - } - if ( !S_ISREG( fileStatus.st_mode ) ) { - error = EINVAL; - ::close( descriptor ); - return nullptr; - } - // The creation mode is already owner-only; also tighten an - // existing regular guard file before it is reused. - if ( ::fchmod( descriptor, S_IRUSR | S_IWUSR ) != 0 ) { - error = errno; - ::close( descriptor ); - return nullptr; - } + /// One test being run in a worker. + struct Running final + { + std::size_t catalogIndex = 0; + bool target = false; + Attempt attempt; + Clock::time_point start; + Clock::time_point deadline = Clock::time_point::max(); + bool fatal = false; + }; - std::FILE* const file = ::fdopen( descriptor, "w" ); - if ( file == nullptr ) { - error = errno; - ::close( descriptor ); - } - return file; -#endif - } + struct Worker final + { + std::size_t id = 0; + std::unique_ptr<ChildProcess> process; + std::thread outputReader; + std::thread errorReader; + bool ready = false; + bool dedicated = false; + bool quitting = false; + bool outputClosed = false; + bool errorsClosed = false; + bool killedForTimeout = false; + std::optional<std::uint32_t> seed; + Clock::time_point started; + std::optional<Job> job; + /// Index into `job->before`, or its size for the target. + std::size_t step = 0; + std::optional<Running> running; + /// Catalog indices this process has run, in order. + std::vector<std::size_t> history; + std::string startupOutput; + }; - // Creates an owner-only guard file. The path must be writable; we do - // not create its parent directories. - void setUpGuardFile( std::string const& guardFilePath ) { - if ( !guardFilePath.empty() ) { - int err = 0; - std::FILE* const file = openGuardFile( guardFilePath, err ); - if ( file == nullptr ) { -#if defined( _MSC_VER ) - char msgBuffer[100] = {}; -#endif - std::string errMsg; -#if defined( _MSC_VER ) - if ( !strerror_s( msgBuffer, err ) ) { - errMsg = msgBuffer; - } else { - errMsg = "Could not translate errno to a string"; - } -#else - const char* const message = std::strerror( err ); - errMsg = message ? message : "Unknown error"; -#endif - CATCH_RUNTIME_ERROR( "Could not open the exit guard file '" - << guardFilePath << "' because '" - << errMsg << "' (" << err << ')' ); - } - const int ret = std::fclose( file ); - CATCH_ENFORCE( - ret == 0, - "Error when closing the exit guard file: " << ret ); - } - } + struct Event final + { + enum class Kind : std::uint8_t + { + kLine, + kErrors, + kOutputClosed, + kErrorsClosed, + }; + std::size_t worker = 0; + Kind kind = Kind::kLine; + std::string text; + }; - // Removes file at path. Assuming we created it in setUpGuardFile. - void tearDownGuardFile( std::string const& guardFilePath ) { - if ( !guardFilePath.empty() ) { - const int ret = std::remove( guardFilePath.c_str() ); - CATCH_ENFORCE( - ret == 0, - "Error when removing the exit guard file: " << ret ); - } - } + /// The search for the test that leaves state behind. + struct Bisection final + { + enum class Stage : std::uint8_t + { + kNone, + kConfirming, + kSearching, + kDone, + }; + Stage stage = Stage::kNone; + std::vector<std::size_t> candidates; + std::vector<std::size_t> probing; + std::size_t probes = 0; + std::optional<std::size_t> culprit; + bool reproduced = false; + }; - } // anon namespace - - Session::Session() { - static bool alreadyInstantiated = false; - if( alreadyInstantiated ) { - CATCH_TRY { CATCH_INTERNAL_ERROR( "Only one instance of Catch::Session can ever be used" ); } - CATCH_CATCH_ALL { getMutableRegistryHub().registerStartupException(); } - } - - // There cannot be exceptions at startup in no-exception mode. -#if !defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) - const auto& exceptions = getRegistryHub().getStartupExceptionRegistry().getExceptions(); - if ( !exceptions.empty() ) { - config(); - getCurrentMutableContext().setConfig(m_config.get()); - - m_startupExceptions = true; - auto errStream = makeStream( "%stderr" ); - auto colourImpl = makeColourImpl( - ColourMode::PlatformDefault, errStream.get() ); - auto guard = colourImpl->guardColour( Colour::Red ); - errStream->stream() << "Errors occurred during startup!" << '\n'; - // iterate over all exceptions and notify user - for ( const auto& ex_ptr : exceptions ) { - try { - std::rethrow_exception(ex_ptr); - } catch ( std::exception const& ex ) { - errStream->stream() << TextFlow::Column( ex.what() ).indent(2) << '\n'; - } - } - } -#endif - - alreadyInstantiated = true; - m_cli = makeCommandLineParser( m_configData ); - } - Session::~Session() { - Catch::cleanUp(); - } + struct TestState final + { + TestResult result; + bool queued = false; + bool finished = false; + std::size_t pendingJobs = 0; + std::size_t retriesUsed = 0; + std::size_t repeatsLeft = 0; + Bisection bisection; + }; +} // namespace Progmasoft::Catch3::Runner::Scheduling +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - void Session::showHelp() const { - Catch::cout() - << "\nCatch2 v" << libraryVersion() << '\n' - << m_cli << '\n' - << "For more detailed usage please see the project docs\n\n" << std::flush; - } - void Session::libIdentify() { - Catch::cout() - << std::left << std::setw(16) << "description: " << "A Catch2 test executable\n" - << std::left << std::setw(16) << "category: " << "testframework\n" - << std::left << std::setw(16) << "framework: " << "Catch2\n" - << std::left << std::setw(16) << "version: " << libraryVersion() << '\n' << std::flush; - } +#include <array> +#include <optional> +#include <string> - int Session::applyCommandLine( int argc, char const * const * argv ) { - if ( m_startupExceptions ) { return UnspecifiedErrorExitCode; } +namespace Progmasoft::Catch3::Runner +{ + /// Gives the test that runs while it lives a temporary directory of its + /// own: it creates the directory, points TMPDIR, TMP and TEMP at it and + /// makes `TestTemporaryDirectory` return it. The variables get their + /// previous values back afterwards; the directory stays, because the + /// runner decides whether to keep it. + class TemporaryDirectoryScope final + { + public: + explicit TemporaryDirectoryScope(const std::string &directory); + TemporaryDirectoryScope(const TemporaryDirectoryScope &) = delete; + TemporaryDirectoryScope & + operator=(const TemporaryDirectoryScope &) = delete; + ~TemporaryDirectoryScope(); - auto result = m_cli.parse( Clara::Args( argc, argv ) ); + private: + bool active_ = false; + std::array<std::optional<std::string>, 3> previous_; + }; +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - if( !result ) { - config(); - getCurrentMutableContext().setConfig(m_config.get()); - auto errStream = makeStream( "%stderr" ); - auto colour = makeColourImpl( ColourMode::PlatformDefault, errStream.get() ); +#include <filesystem> +#include <string> +#include <vector> - errStream->stream() - << colour->guardColour( Colour::Red ) - << "\nError(s) in input:\n" - << TextFlow::Column( result.errorMessage() ).indent( 2 ) - << "\n\n"; - errStream->stream() << "Run with -? for usage\n\n" << std::flush; - return UnspecifiedErrorExitCode; - } +namespace Progmasoft::Catch3 +{ + /// Runs the tests of this binary as `arguments` say and returns the exit + /// code. `arguments` starts with the program, as `argv` does. + /// + /// `Catch::Session::run` and the `main` of `Catch2::Catch2WithMain` call + /// this; a test binary with its own `main` may call it directly. + [[nodiscard]] int + RunTests(const std::vector<std::string> &arguments); + + /// Prints the options of the runner to standard output. + void + PrintHelp(const std::string &programName); + + /// The directory the running test may keep temporary files in. + /// + /// Every attempt of every test gets its own, so tests that run at the + /// same time in different workers never share a file by accident. The + /// runner also points TMPDIR, TMP and TEMP at it while the test runs, so + /// `std::filesystem::temp_directory_path()` and `tmpfile` use it too. + /// It is removed after a passing attempt and kept after a failing one, + /// for inspection. Outside a test it is the system temporary directory. + [[nodiscard]] std::filesystem::path + TestTemporaryDirectory(); +} // namespace Progmasoft::Catch3 +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +namespace Progmasoft::Catch3 +{ + /// The version of Catch3: of its runner, its tools and its formats. + inline constexpr const char *kVersion = "0.3.0"; +} // namespace Progmasoft::Catch3 +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - if( m_configData.showHelp ) - showHelp(); - if( m_configData.libIdentify ) - libIdentify(); +#include <iosfwd> +#include <string> - m_config.reset(); - return 0; - } +namespace Progmasoft::Catch3 +{ + struct RunResult; + + /// Writes a run as JUnit XML without timestamps or global state, so the + /// same run gives the same bytes. + /// + /// A failed test gets `<failure>`, a crash, timeout or infrastructure + /// problem `<error>`, and earlier failed attempts `<rerunFailure>`. A + /// test that failed and then passed is a success with `<flakyFailure>` + /// elements, the convention of Maven Surefire that Jenkins, GitLab and + /// most CI dashboards read. Skipped, blocked and not-run tests are + /// `<skipped>` with the reason as the message. + class JUnitXmlWriter final + { + public: + /// Returns false when the stream failed. + [[nodiscard]] static bool + Write(const RunResult &run, std::ostream &output); -#if defined(CATCH_CONFIG_WCHAR) && defined(_WIN32) && defined(UNICODE) - int Session::applyCommandLine( int argc, wchar_t const * const * argv ) { + [[nodiscard]] static std::string + ToString(const RunResult &run); + }; +} // namespace Progmasoft::Catch3 - char **utf8Argv = new char *[ argc ]; - for ( int i = 0; i < argc; ++i ) { - int bufSize = WideCharToMultiByte( CP_UTF8, 0, argv[i], -1, nullptr, 0, nullptr, nullptr ); - utf8Argv[ i ] = new char[ bufSize ]; - WideCharToMultiByte( CP_UTF8, 0, argv[i], -1, utf8Argv[i], bufSize, nullptr, nullptr ); - } +#include <algorithm> +#include <chrono> +#include <cmath> - int returnCode = applyCommandLine( argc, utf8Argv ); +namespace Catch { + namespace Benchmark { + namespace Detail { - for ( int i = 0; i < argc; ++i ) - delete [] utf8Argv[ i ]; + Environment measure_environment_default() { + return Detail::measure_environment<default_clock>(); + } - delete [] utf8Argv; + ExecutionPlan prepare_default( const IConfig& cfg, + Environment env, + BenchmarkFunction&& fun ) { + // This mirrors Benchmark::prepare<default_clock>(), but with the + // clock fixed so it is instantiated once here in the library. + auto min_time = env.clock_resolution.mean * Detail::minimum_ticks; + auto run_time = std::max( + min_time, + std::chrono::duration_cast<decltype( min_time )>( + cfg.benchmarkWarmupTime() ) ); + auto&& test = Detail::run_for_at_least<default_clock>( + std::chrono::duration_cast<IDuration>( run_time ), 1, fun ); + int new_iters = static_cast<int>( + std::ceil( min_time * test.iterations / test.elapsed ) ); + return { new_iters, + test.elapsed / test.iterations * new_iters * + cfg.benchmarkSamples(), + CATCH_MOVE( fun ), + std::chrono::duration_cast<FDuration>( + cfg.benchmarkWarmupTime() ), + Detail::warmup_iterations }; + } - return returnCode; - } -#endif + std::vector<FDuration> run_plan_default( ExecutionPlan const& plan, + const IConfig& cfg, + Environment env ) { + return plan.run<default_clock>( cfg, env ); + } - void Session::useConfigData( ConfigData const& configData ) { - m_configData = configData; - m_config.reset(); - } + } // namespace Detail + } // namespace Benchmark +} // namespace Catch - int Session::run() { - if( ( m_configData.waitForKeypress & WaitForKeypress::BeforeStart ) != 0 ) { - Catch::cout() << "...waiting for enter/ return before starting\n" << std::flush; - static_cast<void>(std::getchar()); - } - int exitCode = runInternal(); - if( ( m_configData.waitForKeypress & WaitForKeypress::BeforeExit ) != 0 ) { - Catch::cout() << "...waiting for enter/ return before exiting, with code: " << exitCode << '\n' << std::flush; - static_cast<void>(std::getchar()); - } - return exitCode; - } - Clara::Parser const& Session::cli() const { - return m_cli; - } - void Session::cli( Clara::Parser const& newParser ) { - m_cli = newParser; - } - ConfigData& Session::configData() { - return m_configData; - } - Config& Session::config() { - if( !m_config ) - m_config = Detail::make_unique<Config>( m_configData ); - return *m_config; - } - int Session::runInternal() { - if ( m_startupExceptions ) { return UnspecifiedErrorExitCode; } +namespace Catch { + namespace Benchmark { + namespace Detail { + ChronometerConcept::~ChronometerConcept() = default; + } // namespace Detail + } // namespace Benchmark +} // namespace Catch - if (m_configData.showHelp || m_configData.libIdentify) { - return 0; - } - if ( m_configData.shardIndex >= m_configData.shardCount ) { - Catch::cerr() << "The shard count (" << m_configData.shardCount - << ") must be greater than the shard index (" - << m_configData.shardIndex << ")\n" - << std::flush; - return UnspecifiedErrorExitCode; - } +// Adapted from donated nonius code. - CATCH_TRY { - config(); // Force config to be constructed - if ( m_config->shardCount() > 1 && - m_config->runOrder() == TestRunOrder::Randomized && - !m_config->rngSeedWasFixed() ) { - Catch::cerr() - << "Warning: using sharding (--shard-count) with random " - "order (--order rand, the default) and without a fixed " - "numeric --rng-seed does not guarantee disjoint coverage " - "between shard invocations. Pass the same numeric " - "--rng-seed to every shard, or use --order decl or " - "--order lex instead.\n" - << std::flush; - } +#include <vector> - // We need to retrieve potential Bazel config with the full Config - // constructor, so we have to create the guard file after it is created. - setUpGuardFile( m_config->getExitGuardFilePath() ); +namespace Catch { + namespace Benchmark { + namespace Detail { + SampleAnalysis analyse(const IConfig &cfg, FDuration* first, FDuration* last) { + if (!cfg.benchmarkNoAnalysis()) { + std::vector<double> samples; + samples.reserve(static_cast<size_t>(last - first)); + for (auto current = first; current != last; ++current) { + samples.push_back( current->count() ); + } - seedRng( *m_config ); + auto analysis = Catch::Benchmark::Detail::analyse_samples( + cfg.benchmarkConfidenceInterval(), + cfg.benchmarkResamples(), + samples.data(), + samples.data() + samples.size() ); + auto outliers = Catch::Benchmark::Detail::classify_outliers( + samples.data(), samples.data() + samples.size() ); - if (m_configData.filenamesAsTags) { - applyFilenamesAsTags(); - } + auto wrap_estimate = [](Estimate<double> e) { + return Estimate<FDuration>{ + FDuration( e.point ), + FDuration( e.lower_bound ), + FDuration( e.upper_bound ), + e.confidence_interval, + }; + }; + std::vector<FDuration> samples2; + samples2.reserve(samples.size()); + for (auto s : samples) { + samples2.push_back( FDuration( s ) ); + } - // Set up global config instance before we start calling into other functions - getCurrentMutableContext().setConfig(m_config.get()); + return { + CATCH_MOVE(samples2), + wrap_estimate(analysis.mean), + wrap_estimate(analysis.standard_deviation), + outliers, + analysis.outlier_variance, + }; + } else { + std::vector<FDuration> samples; + samples.reserve(static_cast<size_t>(last - first)); - // Create reporter(s) so we can route listings through them - IEventListenerPtr reporter; - CATCH_TRY { - auto reporter_ = prepareReporters( m_config.get() ); - swap( reporter, reporter_ ); - } -#if !defined( CATCH_CONFIG_DISABLE_EXCEPTIONS ) - catch ( std::exception& ex ) { - Catch::cerr() << "Error when creating reporter: " << ex.what() << '\n' << std::flush; - return ReporterConstructionErrorExitCode; - } -#endif + FDuration mean = FDuration(0); + int i = 0; + for (auto it = first; it < last; ++it, ++i) { + samples.push_back(*it); + mean += *it; + } + mean /= i; - auto const& invalidSpecs = m_config->testSpec().getInvalidSpecs(); - if ( !invalidSpecs.empty() ) { - for ( auto const& spec : invalidSpecs ) { - reporter->reportInvalidTestSpec( spec ); + return SampleAnalysis{ + CATCH_MOVE(samples), + Estimate<FDuration>{ mean, mean, mean, 0.0 }, + Estimate<FDuration>{ FDuration( 0 ), + FDuration( 0 ), + FDuration( 0 ), + 0.0 }, + OutlierClassification{}, + 0.0 + }; } - return InvalidTestSpecExitCode; } + } // namespace Detail + } // namespace Benchmark +} // namespace Catch - // Handle list request - if (list(*reporter, *m_config)) { - return 0; - } - - TestGroup tests { CATCH_MOVE(reporter), m_config.get() }; - auto const totals = tests.execute(); - // If we got here, running the tests finished normally-enough. - // They might've failed, but that would've been reported elsewhere. - tearDownGuardFile( m_config->getExitGuardFilePath() ); - if ( tests.hadUnmatchedTestSpecs() - && m_config->warnAboutUnmatchedTestSpecs() ) { - return UnmatchedTestSpecExitCode; - } +namespace Catch { + namespace Benchmark { + namespace Detail { + struct do_nothing { + void operator()() const {} + }; - if ( totals.testCases.total() == 0 - && !m_config->zeroTestsCountAsSuccess() ) { - return NoTestsRunExitCode; - } + BenchmarkFunction::callable::~callable() = default; + BenchmarkFunction::BenchmarkFunction(): + f( new model<do_nothing>{ {} } ){} + } // namespace Detail + } // namespace Benchmark +} // namespace Catch - if ( totals.testCases.total() > 0 && - totals.testCases.total() == totals.testCases.skipped - && !m_config->zeroTestsCountAsSuccess() ) { - return AllTestsSkippedExitCode; - } - if ( totals.assertions.failed ) { return TestFailureExitCode; } - return 0; - } -#if !defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) - catch( std::exception& ex ) { - Catch::cerr() << ex.what() << '\n' << std::flush; - return UnspecifiedErrorExitCode; - } -#endif - } -} // end namespace Catch +#include <exception> +namespace Catch { + namespace Benchmark { + namespace Detail { + struct optimized_away_error final : std::exception { + const char* what() const noexcept override; + }; + const char* optimized_away_error::what() const noexcept { + return "could not measure benchmark, maybe it was optimized away"; + } + void throw_optimized_away_error() { + Catch::throw_exception(optimized_away_error{}); + } -namespace Catch { + } // namespace Detail + } // namespace Benchmark +} // namespace Catch - RegistrarForTagAliases::RegistrarForTagAliases(char const* alias, char const* tag, SourceLineInfo const& lineInfo) { - CATCH_TRY { - getMutableRegistryHub().registerTagAlias(alias, tag, lineInfo); - } CATCH_CATCH_ALL { - // Do not throw when constructing global objects, instead register the exception to be processed later - getMutableRegistryHub().registerStartupException(); - } - } -} +// Adapted from donated nonius code. -#include <cassert> -#include <cctype> #include <algorithm> -#include <utility> +#include <cassert> +#include <cmath> +#include <cstddef> +#include <numeric> +#include <random> -namespace Catch { - namespace { - using TCP_underlying_type = uint8_t; - static_assert(sizeof(TestCaseProperties) == sizeof(TCP_underlying_type), - "The size of the TestCaseProperties is different from the assumed size"); +#if defined(CATCH_CONFIG_USE_ASYNC) +#include <future> +#endif - constexpr TestCaseProperties operator|(TestCaseProperties lhs, TestCaseProperties rhs) { - return static_cast<TestCaseProperties>( - static_cast<TCP_underlying_type>(lhs) | static_cast<TCP_underlying_type>(rhs) - ); - } +namespace Catch { + namespace Benchmark { + namespace Detail { + namespace { - constexpr TestCaseProperties& operator|=(TestCaseProperties& lhs, TestCaseProperties rhs) { - lhs = static_cast<TestCaseProperties>( - static_cast<TCP_underlying_type>(lhs) | static_cast<TCP_underlying_type>(rhs) - ); - return lhs; - } + template <typename URng, typename Estimator> + static sample + resample( URng& rng, + unsigned int resamples, + double const* first, + double const* last, + Estimator& estimator ) { + auto n = static_cast<size_t>( last - first ); + Catch::uniform_integer_distribution<size_t> dist( 0, n - 1 ); - constexpr TestCaseProperties operator&(TestCaseProperties lhs, TestCaseProperties rhs) { - return static_cast<TestCaseProperties>( - static_cast<TCP_underlying_type>(lhs) & static_cast<TCP_underlying_type>(rhs) - ); - } + sample out; + out.reserve( resamples ); + std::vector<double> resampled; + resampled.reserve( n ); + for ( size_t i = 0; i < resamples; ++i ) { + resampled.clear(); + for ( size_t s = 0; s < n; ++s ) { + resampled.push_back( first[dist( rng )] ); + } + const auto estimate = + estimator( resampled.data(), resampled.data() + resampled.size() ); + out.push_back( estimate ); + } + std::sort( out.begin(), out.end() ); + return out; + } - constexpr bool applies(TestCaseProperties tcp) { - static_assert(static_cast<TCP_underlying_type>(TestCaseProperties::None) == 0, - "TestCaseProperties::None must be equal to 0"); - return tcp != TestCaseProperties::None; - } + static double outlier_variance( Estimate<double> mean, + Estimate<double> stddev, + int n ) { + double sb = stddev.point; + double mn = mean.point / n; + double mg_min = mn / 2.; + double sg = (std::min)( mg_min / 4., sb / std::sqrt( n ) ); + double sg2 = sg * sg; + double sb2 = sb * sb; - TestCaseProperties propertiesFromTag( StringRef tag, SourceLineInfo const& _lineInfo ) { - if ( tag.empty() ) { return TestCaseProperties::None; } - // All tags starting with alphanumeric characters are normal. - if ( std::isalnum( tag[0] ) ) { return TestCaseProperties::None; } + auto c_max = [n, mn, sb2, sg2]( double x ) -> double { + double k = mn - x; + double d = k * k; + double nd = n * d; + double k0 = -n * nd; + double k1 = sb2 - n * sg2 + nd; + double det = k1 * k1 - 4 * sg2 * k0; + return static_cast<int>( -2. * k0 / + ( k1 + std::sqrt( det ) ) ); + }; - // Explicitly check known special tags, and error out if it is not one of them - if ( tag[0] == '.' ) { return TestCaseProperties::IsHidden; } - else if ( tag == "!throws"_sr ) { return TestCaseProperties::Throws; } - else if ( tag == "!shouldfail"_sr ) { return TestCaseProperties::ShouldFail; } - else if ( tag == "!mayfail"_sr ) { return TestCaseProperties::MayFail; } - else if ( tag == "!nonportable"_sr ) { return TestCaseProperties::NonPortable; } - else if ( tag == "!benchmark"_sr ) { - return TestCaseProperties::Benchmark | - TestCaseProperties::IsHidden; - } else { - CATCH_RUNTIME_ERROR( - "Tag name: [" << tag << "] is not allowed.\n" - << "Tag names starting with non-alphanumeric " - "characters are reserved.\n" - << _lineInfo ); + auto var_out = [n, sb2, sg2]( double c ) { + double nc = n - c; + return ( nc / n ) * ( sb2 - nc * sg2 ); + }; + + return (std::min)( var_out( 1 ), + var_out( + (std::min)( c_max( 0. ), + c_max( mg_min ) ) ) ) / + sb2; + } + + static double erf_inv( double x ) { + // Code accompanying the article "Approximating the erfinv + // function" in GPU Computing Gems, Volume 2 + double w, p; + + w = -log( ( 1.0 - x ) * ( 1.0 + x ) ); + + if ( w < 6.250000 ) { + w = w - 3.125000; + p = -3.6444120640178196996e-21; + p = -1.685059138182016589e-19 + p * w; + p = 1.2858480715256400167e-18 + p * w; + p = 1.115787767802518096e-17 + p * w; + p = -1.333171662854620906e-16 + p * w; + p = 2.0972767875968561637e-17 + p * w; + p = 6.6376381343583238325e-15 + p * w; + p = -4.0545662729752068639e-14 + p * w; + p = -8.1519341976054721522e-14 + p * w; + p = 2.6335093153082322977e-12 + p * w; + p = -1.2975133253453532498e-11 + p * w; + p = -5.4154120542946279317e-11 + p * w; + p = 1.051212273321532285e-09 + p * w; + p = -4.1126339803469836976e-09 + p * w; + p = -2.9070369957882005086e-08 + p * w; + p = 4.2347877827932403518e-07 + p * w; + p = -1.3654692000834678645e-06 + p * w; + p = -1.3882523362786468719e-05 + p * w; + p = 0.0001867342080340571352 + p * w; + p = -0.00074070253416626697512 + p * w; + p = -0.0060336708714301490533 + p * w; + p = 0.24015818242558961693 + p * w; + p = 1.6536545626831027356 + p * w; + } else if ( w < 16.000000 ) { + w = sqrt( w ) - 3.250000; + p = 2.2137376921775787049e-09; + p = 9.0756561938885390979e-08 + p * w; + p = -2.7517406297064545428e-07 + p * w; + p = 1.8239629214389227755e-08 + p * w; + p = 1.5027403968909827627e-06 + p * w; + p = -4.013867526981545969e-06 + p * w; + p = 2.9234449089955446044e-06 + p * w; + p = 1.2475304481671778723e-05 + p * w; + p = -4.7318229009055733981e-05 + p * w; + p = 6.8284851459573175448e-05 + p * w; + p = 2.4031110387097893999e-05 + p * w; + p = -0.0003550375203628474796 + p * w; + p = 0.00095328937973738049703 + p * w; + p = -0.0016882755560235047313 + p * w; + p = 0.0024914420961078508066 + p * w; + p = -0.0037512085075692412107 + p * w; + p = 0.005370914553590063617 + p * w; + p = 1.0052589676941592334 + p * w; + p = 3.0838856104922207635 + p * w; + } else { + w = sqrt( w ) - 5.000000; + p = -2.7109920616438573243e-11; + p = -2.5556418169965252055e-10 + p * w; + p = 1.5076572693500548083e-09 + p * w; + p = -3.7894654401267369937e-09 + p * w; + p = 7.6157012080783393804e-09 + p * w; + p = -1.4960026627149240478e-08 + p * w; + p = 2.9147953450901080826e-08 + p * w; + p = -6.7711997758452339498e-08 + p * w; + p = 2.2900482228026654717e-07 + p * w; + p = -9.9298272942317002539e-07 + p * w; + p = 4.5260625972231537039e-06 + p * w; + p = -1.9681778105531670567e-05 + p * w; + p = 7.5995277030017761139e-05 + p * w; + p = -0.00021503011930044477347 + p * w; + p = -0.00013871931833623122026 + p * w; + p = 1.0103004648645343977 + p * w; + p = 4.8499064014085844221 + p * w; + } + return p * x; + } + + static double + standard_deviation( double const* first, double const* last ) { + auto m = Catch::Benchmark::Detail::mean( first, last ); + double variance = + std::accumulate( first, + last, + 0., + [m]( double a, double b ) { + double diff = b - m; + return a + diff * diff; + } ) / + static_cast<double>( last - first ); + return std::sqrt( variance ); + } + + static sample jackknife( double ( *estimator )( double const*, + double const* ), + double* first, + double* last ) { + const auto second = first + 1; + sample results; + results.reserve( static_cast<size_t>( last - first ) ); + + for ( auto it = first; it != last; ++it ) { + std::iter_swap( it, first ); + results.push_back( estimator( second, last ) ); + } + + return results; + } + + + } // namespace + } // namespace Detail + } // namespace Benchmark +} // namespace Catch + +namespace Catch { + namespace Benchmark { + namespace Detail { + + double weighted_average_quantile( int k, + int q, + double* first, + double* last ) { + auto count = last - first; + double idx = static_cast<double>((count - 1) * k) / static_cast<double>(q); + int j = static_cast<int>(idx); + double g = idx - j; + std::nth_element(first, first + j, last); + auto xj = first[j]; + if ( Catch::Detail::directCompare( g, 0 ) ) { + return xj; + } + + auto xj1 = *std::min_element(first + (j + 1), last); + return xj + g * (xj1 - xj); + } + + OutlierClassification + classify_outliers( double const* first, double const* last ) { + std::vector<double> copy( first, last ); + + auto q1 = weighted_average_quantile( 1, 4, copy.data(), copy.data() + copy.size() ); + auto q3 = weighted_average_quantile( 3, 4, copy.data(), copy.data() + copy.size() ); + auto iqr = q3 - q1; + auto los = q1 - ( iqr * 3. ); + auto lom = q1 - ( iqr * 1.5 ); + auto him = q3 + ( iqr * 1.5 ); + auto his = q3 + ( iqr * 3. ); + + OutlierClassification o; + for ( ; first != last; ++first ) { + const double t = *first; + if ( t < los ) { + ++o.low_severe; + } else if ( t < lom ) { + ++o.low_mild; + } else if ( t > his ) { + ++o.high_severe; + } else if ( t > him ) { + ++o.high_mild; + } + ++o.samples_seen; + } + return o; + } + + double mean( double const* first, double const* last ) { + auto count = last - first; + double sum = 0.; + while (first != last) { + sum += *first; + ++first; + } + return sum / static_cast<double>(count); + } + + double normal_cdf( double x ) { + return std::erfc( -x / std::sqrt( 2.0 ) ) / 2.0; + } + + double erfc_inv(double x) { + return erf_inv(1.0 - x); + } + + double normal_quantile(double p) { + static const double ROOT_TWO = std::sqrt(2.0); + + double result = 0.0; + assert(p >= 0 && p <= 1); + if (p < 0 || p > 1) { + return result; + } + + result = -erfc_inv(2.0 * p); + // result *= normal distribution standard deviation (1.0) * sqrt(2) + result *= /*sd * */ ROOT_TWO; + // result += normal disttribution mean (0) + return result; + } + + Estimate<double> + bootstrap( double confidence_level, + double* first, + double* last, + sample const& resample, + double ( *estimator )( double const*, double const* ) ) { + auto n_samples = last - first; + + double point = estimator( first, last ); + // Degenerate case with a single sample + if ( n_samples == 1 ) + return { point, point, point, confidence_level }; + + sample jack = jackknife( estimator, first, last ); + double jack_mean = + mean( jack.data(), jack.data() + jack.size() ); + double sum_squares = 0, sum_cubes = 0; + for ( double x : jack ) { + auto difference = jack_mean - x; + auto square = difference * difference; + auto cube = square * difference; + sum_squares += square; + sum_cubes += cube; + } + + double accel = sum_cubes / ( 6 * std::pow( sum_squares, 1.5 ) ); + long n = static_cast<long>( resample.size() ); + double prob_n = static_cast<double>( + std::count_if( resample.begin(), + resample.end(), + [point]( double x ) { return x < point; } )) / + static_cast<double>( n ); + // degenerate case with uniform samples + if ( Catch::Detail::directCompare( prob_n, 0. ) ) { + return { point, point, point, confidence_level }; + } + + double bias = normal_quantile( prob_n ); + double z1 = normal_quantile( ( 1. - confidence_level ) / 2. ); + + auto cumn = [n]( double x ) -> long { + return std::lround( normal_cdf( x ) * + static_cast<double>( n ) ); + }; + auto a = [bias, accel]( double b ) { + return bias + b / ( 1. - accel * b ); + }; + double b1 = bias + z1; + double b2 = bias - z1; + double a1 = a( b1 ); + double a2 = a( b2 ); + auto lo = static_cast<size_t>( (std::max)( cumn( a1 ), 0l ) ); + auto hi = + static_cast<size_t>( (std::min)( cumn( a2 ), n - 1 ) ); + + return { point, resample[lo], resample[hi], confidence_level }; + } + + bootstrap_analysis analyse_samples(double confidence_level, + unsigned int n_resamples, + double* first, + double* last) { + auto mean = &Detail::mean; + auto stddev = &standard_deviation; + +#if defined(CATCH_CONFIG_USE_ASYNC) + auto Estimate = [=](double(*f)(double const*, double const*)) { + std::random_device rd; + auto seed = rd(); + return std::async(std::launch::async, [=] { + SimplePcg32 rng( seed ); + auto resampled = resample(rng, n_resamples, first, last, f); + return bootstrap(confidence_level, first, last, resampled, f); + }); + }; + + auto mean_future = Estimate(mean); + auto stddev_future = Estimate(stddev); + + auto mean_estimate = mean_future.get(); + auto stddev_estimate = stddev_future.get(); +#else + auto Estimate = [=](double(*f)(double const* , double const*)) { + std::random_device rd; + auto seed = rd(); + SimplePcg32 rng( seed ); + auto resampled = resample(rng, n_resamples, first, last, f); + return bootstrap(confidence_level, first, last, resampled, f); + }; + + auto mean_estimate = Estimate(mean); + auto stddev_estimate = Estimate(stddev); +#endif // CATCH_USE_ASYNC + + auto n = static_cast<int>(last - first); // seriously, one can't use integral types without hell in C++ + double outlier_variance = Detail::outlier_variance(mean_estimate, stddev_estimate, n); + + return { mean_estimate, stddev_estimate, outlier_variance }; + } + } // namespace Detail + } // namespace Benchmark +} // namespace Catch + + + +#include <cmath> +#include <limits> + +namespace { + +// Performs equivalent check of std::fabs(lhs - rhs) <= margin +// But without the subtraction to allow for INFINITY in comparison +bool marginComparison(double lhs, double rhs, double margin) { + return (lhs + margin >= rhs) && (rhs + margin >= lhs); +} + +} + +namespace Catch { + + Approx::Approx ( double value ) + : m_epsilon( static_cast<double>(std::numeric_limits<float>::epsilon())*100. ), + m_margin( 0.0 ), + m_scale( 0.0 ), + m_value( value ) + {} + + Approx Approx::custom() { + return Approx( 0 ); + } + + Approx Approx::operator-() const { + auto temp(*this); + temp.m_value = -temp.m_value; + return temp; + } + + + std::string Approx::toString() const { + ReusableStringStream rss; + rss << "Approx( " << ::Catch::Detail::stringify( m_value ) << " )"; + return rss.str(); + } + + bool Approx::equalityComparisonImpl(const double other) const { + // First try with fixed margin, then compute margin based on epsilon, scale and Approx's value + // Thanks to Richard Harris for his help refining the scaled margin value + return marginComparison(m_value, other, m_margin) + || marginComparison(m_value, other, m_epsilon * (m_scale + std::fabs(std::isinf(m_value)? 0 : m_value))); + } + + void Approx::setMargin(double newMargin) { + CATCH_ENFORCE(newMargin >= 0, + "Invalid Approx::margin: " << newMargin << '.' + << " Approx::Margin has to be non-negative."); + m_margin = newMargin; + } + + void Approx::setEpsilon(double newEpsilon) { + CATCH_ENFORCE(newEpsilon >= 0 && newEpsilon <= 1.0, + "Invalid Approx::epsilon: " << newEpsilon << '.' + << " Approx::epsilon has to be in [0, 1]"); + m_epsilon = newEpsilon; + } + +namespace literals { + Approx operator ""_a(long double val) { + return Approx(val); + } + Approx operator ""_a(unsigned long long val) { + return Approx(val); + } +} // end namespace literals + +std::string StringMaker<Catch::Approx>::convert(Catch::Approx const& value) { + return value.toString(); +} + +} // end namespace Catch + + + +namespace Catch { + + AssertionResultData::AssertionResultData(ResultWas::OfType _resultType, LazyExpression const& _lazyExpression): + lazyExpression(_lazyExpression), + resultType(_resultType) {} + + std::string AssertionResultData::reconstructExpression() const { + + if( reconstructedExpression.empty() ) { + if( lazyExpression ) { + ReusableStringStream rss; + rss << lazyExpression; + reconstructedExpression = rss.str(); } } + return reconstructedExpression; + } - std::string makeDefaultName() { - static size_t counter = 0; - return "Anonymous test case " + std::to_string(++counter); - } + AssertionResult::AssertionResult( AssertionInfo const& info, AssertionResultData&& data ) + : m_info( info ), + m_resultData( CATCH_MOVE(data) ) + {} - constexpr StringRef extractFilenamePart(StringRef filename) { - size_t lastDot = filename.size(); - while (lastDot > 0 && filename[lastDot - 1] != '.') { - --lastDot; - } - // In theory we could have filename without any extension in it - if ( lastDot == 0 ) { return StringRef(); } + // Result was a success + bool AssertionResult::succeeded() const { + return Catch::isOk( m_resultData.resultType ); + } - --lastDot; - size_t nameStart = lastDot; - while (nameStart > 0 && filename[nameStart - 1] != '/' && filename[nameStart - 1] != '\\') { - --nameStart; - } + // Result was a success, or failure is suppressed + bool AssertionResult::isOk() const { + return Catch::isOk( m_resultData.resultType ) || shouldSuppressFailure( m_info.resultDisposition ); + } - return filename.substr(nameStart, lastDot - nameStart); + ResultWas::OfType AssertionResult::getResultType() const { + return m_resultData.resultType; + } + + bool AssertionResult::hasExpression() const { + return !m_info.capturedExpression.empty(); + } + + bool AssertionResult::hasMessage() const { + return !m_resultData.message.empty(); + } + + std::string AssertionResult::getExpression() const { + // Possibly overallocating by 3 characters should be basically free + std::string expr; expr.reserve(m_info.capturedExpression.size() + 3); + if (isFalseTest(m_info.resultDisposition)) { + expr += "!("; } + expr += m_info.capturedExpression; + if (isFalseTest(m_info.resultDisposition)) { + expr += ')'; + } + return expr; + } - // Returns the upper bound on size of extra tags ([#file]+[.]) - constexpr size_t sizeOfExtraTags(StringRef filepath) { - // [.] is 3, [#] is another 3 - const size_t extras = 3 + 3; - return extractFilenamePart(filepath).size() + extras; + std::string AssertionResult::getExpressionInMacro() const { + if ( m_info.macroName.empty() ) { + return static_cast<std::string>( m_info.capturedExpression ); } - } // end unnamed namespace + std::string expr; + expr.reserve( m_info.macroName.size() + m_info.capturedExpression.size() + 4 ); + expr += m_info.macroName; + expr += "( "; + expr += m_info.capturedExpression; + expr += " )"; + return expr; + } - bool operator<( Tag const& lhs, Tag const& rhs ) { - Detail::CaseInsensitiveLess cmp; - return cmp( lhs.original, rhs.original ); + bool AssertionResult::hasExpandedExpression() const { + return hasExpression() && getExpandedExpression() != getExpression(); } - bool operator==( Tag const& lhs, Tag const& rhs ) { - Detail::CaseInsensitiveEqualTo cmp; - return cmp( lhs.original, rhs.original ); + + std::string AssertionResult::getExpandedExpression() const { + std::string expr = m_resultData.reconstructExpression(); + return expr.empty() + ? getExpression() + : expr; } - Detail::unique_ptr<TestCaseInfo> - makeTestCaseInfo(StringRef _className, - NameAndTags const& nameAndTags, - SourceLineInfo const& _lineInfo ) { - return Detail::make_unique<TestCaseInfo>(_className, nameAndTags, _lineInfo); + StringRef AssertionResult::getMessage() const { + return m_resultData.message; + } + SourceLineInfo AssertionResult::getSourceInfo() const { + return m_info.lineInfo; } - TestCaseInfo::TestCaseInfo(StringRef _className, - NameAndTags const& _nameAndTags, - SourceLineInfo const& _lineInfo): - name( _nameAndTags.name.empty() ? makeDefaultName() : _nameAndTags.name ), - className( _className ), - lineInfo( _lineInfo ) - { - StringRef originalTags = _nameAndTags.tags; - // We need to reserve enough space to store all of the tags - // (including optional hidden tag and filename tag) - auto requiredSize = originalTags.size() + sizeOfExtraTags(_lineInfo.file); - backingTags.reserve(requiredSize); + StringRef AssertionResult::getTestMacroName() const { + return m_info.macroName; + } - // Since there is usually only small number of tags on a test case, - // the geometric growth on vector does not get time to properly - // amortize the reallocations. But we can save a lot of time by - // pre-allocating space for all tags. - if ( !originalTags.empty() ) { - size_t tagCount = 1; // 1 for filename tag - for ( size_t i = 0; i < originalTags.size(); ++i ) { - if ( originalTags[i] == '[' ) { ++tagCount; } - } - tags.reserve( tagCount ); - } +} // end namespace Catch - // We cannot copy the tags directly, as we need to normalize - // some tags, so that [.foo] is copied as [.][foo]. - size_t tagStart = 0; - size_t tagEnd = 0; - bool inTag = false; - for (size_t idx = 0; idx < originalTags.size(); ++idx) { - auto c = originalTags[idx]; - if (c == '[') { - CATCH_ENFORCE( - !inTag, - "Found '[' inside a tag while registering test case '" - << _nameAndTags.name << "' at " << _lineInfo ); - inTag = true; - tagStart = idx; - } - if (c == ']') { - CATCH_ENFORCE( - inTag, - "Found unmatched ']' while registering test case '" - << _nameAndTags.name << "' at " << _lineInfo ); - inTag = false; - tagEnd = idx; - assert(tagStart < tagEnd); +namespace Catch { - // We need to check the tag for special meanings, copy - // it over to backing storage and actually reference the - // backing storage in the saved tags - StringRef tagStr = originalTags.substr(tagStart+1, tagEnd - tagStart - 1); - CATCH_ENFORCE( !tagStr.empty(), - "Found an empty tag while registering test case '" - << _nameAndTags.name << "' at " - << _lineInfo ); + bool operator==( PathFilter const& lhs, PathFilter const& rhs ) { + return lhs.type == rhs.type && lhs.filter == rhs.filter; + } - properties |= propertiesFromTag(tagStr, _lineInfo); - // When copying a tag to the backing storage, we need to - // check if it is a merged hide tag, such as [.foo], and - // if it is, we need to handle it as if it was [foo]. - if (tagStr.size() > 1 && tagStr[0] == '.') { - tagStr = tagStr.substr(1, tagStr.size() - 1); - } - // We skip over dealing with the [.] tag, as we will add - // it later unconditionally and then sort and unique all - // the tags. - internalAppendTag(tagStr); - } + Config::Config( ConfigData const& data ): + m_data( data ) { + // We need to trim filter specs to avoid trouble with superfluous + // whitespace (esp. important for bdd macros, as those are manually + // aligned with whitespace). + for (auto& elem : m_data.testsOrTags) { + elem = trim(elem); } - CATCH_ENFORCE( !inTag, - "Found an unclosed tag while registering test case '" - << _nameAndTags.name << "' at " << _lineInfo ); - - // Add [.] if relevant - if (isHidden()) { - internalAppendTag("."_sr); + TestSpecParser parser( ITagAliasRegistry::get() ); + if ( !m_data.testsOrTags.empty() ) { + m_hasTestFilters = true; + for ( auto const& testOrTags : m_data.testsOrTags ) { + parser.parse( testOrTags ); + } } + m_testSpec = parser.testSpec(); + } - // Tests usually have small number of tags, so we explicitly handle - // the 1 and 2 tags cases for better performance in low-opt builds. - switch ( tags.size() ) { - case 1: - break; - case 2: - if ( tags[0] == tags[1] ) { tags.pop_back(); } - else if ( tags[1] < tags[0] ) { std::swap( tags[0], tags[1] ); } - break; - default: - std::sort( begin( tags ), end( tags ) ); - tags.erase( std::unique( begin( tags ), end( tags ) ), - end( tags ) ); - } + Config::~Config() = default; - } + std::vector<std::string> const& Config::getTestsOrTags() const { return m_data.testsOrTags; } + std::vector<PathFilter> const& Config::getPathFilters() const { return m_data.pathFilters; } + bool Config::useNewFilterBehaviour() const { return m_data.useNewPathFilteringBehaviour; } - bool TestCaseInfo::isHidden() const { - return applies( properties & TestCaseProperties::IsHidden ); - } - bool TestCaseInfo::throws() const { - return applies( properties & TestCaseProperties::Throws ); + TestSpec const& Config::testSpec() const { return m_testSpec; } + bool Config::hasTestFilters() const { return m_hasTestFilters; } + + // IConfig interface + bool Config::allowThrows() const { return !m_data.noThrow; } + StringRef Config::name() const { return m_data.name.empty() ? m_data.processName : m_data.name; } + bool Config::includeSuccessfulResults() const { return m_data.showSuccessfulTests; } + bool Config::warnAboutMissingAssertions() const { + return !!( m_data.warnings & WarnAbout::NoAssertions ); } - bool TestCaseInfo::okToFail() const { - return applies( properties & (TestCaseProperties::ShouldFail | TestCaseProperties::MayFail ) ); + bool Config::warnAboutUnmatchedTestSpecs() const { + return !!( m_data.warnings & WarnAbout::UnmatchedTestSpec ); } - bool TestCaseInfo::expectedToFail() const { - return applies( properties & (TestCaseProperties::ShouldFail) ); + bool Config::warnAboutInfiniteGenerators() const { + return !!( m_data.warnings & WarnAbout::InfiniteGenerator ); } + bool Config::zeroTestsCountAsSuccess() const { return m_data.allowZeroTests; } + ShowDurations Config::showDurations() const { return m_data.showDurations; } + double Config::minDuration() const { return m_data.minDuration; } + TestRunOrder Config::runOrder() const { return m_data.runOrder; } + uint32_t Config::rngSeed() const { return m_data.rngSeed; } + bool Config::rngSeedWasFixed() const { return m_data.rngSeedWasFixed; } + unsigned int Config::shardCount() const { return m_data.shardCount; } + unsigned int Config::shardIndex() const { return m_data.shardIndex; } + ColourMode Config::defaultColourMode() const { return m_data.defaultColourMode; } + bool Config::shouldDebugBreak() const { return m_data.shouldDebugBreak; } + int Config::abortAfter() const { return m_data.abortAfter; } + bool Config::showInvisibles() const { return m_data.showInvisibles; } + Verbosity Config::verbosity() const { return m_data.verbosity; } + + bool Config::skipBenchmarks() const { return m_data.skipBenchmarks; } + bool Config::benchmarkNoAnalysis() const { return m_data.benchmarkNoAnalysis; } + unsigned int Config::benchmarkSamples() const { return m_data.benchmarkSamples; } + double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; } + unsigned int Config::benchmarkResamples() const { return m_data.benchmarkResamples; } + std::chrono::milliseconds Config::benchmarkWarmupTime() const { return std::chrono::milliseconds(m_data.benchmarkWarmupTime); } - void TestCaseInfo::addFilenameTag() { - std::string combined("#"); - combined += extractFilenamePart(lineInfo.file); - internalAppendTag(combined); - } +} // end namespace Catch - std::string TestCaseInfo::tagsAsString() const { - std::string ret; - // '[' and ']' per tag - std::size_t full_size = 2 * tags.size(); - for (const auto& tag : tags) { - full_size += tag.original.size(); - } - ret.reserve(full_size); - for (const auto& tag : tags) { - ret.push_back('['); - ret += tag.original; - ret.push_back(']'); - } - return ret; - } - void TestCaseInfo::internalAppendTag(StringRef tagStr) { - backingTags += '['; - const auto backingStart = backingTags.size(); - backingTags += tagStr; - const auto backingEnd = backingTags.size(); - backingTags += ']'; - tags.emplace_back(StringRef(backingTags.c_str() + backingStart, backingEnd - backingStart)); - } - bool operator<( TestCaseInfo const& lhs, TestCaseInfo const& rhs ) { - // We want to avoid redoing the string comparisons multiple times, - // so we store the result of a three-way comparison before using - // it in the actual comparison logic. - const auto cmpName = lhs.name.compare( rhs.name ); - if ( cmpName != 0 ) { - return cmpName < 0; - } - const auto cmpClassName = lhs.className.compare( rhs.className ); - if ( cmpClassName != 0 ) { - return cmpClassName < 0; - } - return lhs.tags < rhs.tags; - } - bool operator==( TestCaseInfo const& lhs, TestCaseInfo const& rhs ) { - return lhs.name == rhs.name - && lhs.className == rhs.className - && lhs.tags == rhs.tags; +namespace Catch { + std::uint32_t getSeed() { + return getCurrentContext().getConfig()->rngSeed(); } +} -} // end namespace Catch -#include <algorithm> -#include <string> -#include <vector> -#include <ostream> +#include <cassert> +#include <stack> namespace Catch { - TestSpec::Pattern::Pattern( std::string const& name ) - : m_name( name ) - {} + //////////////////////////////////////////////////////////////////////////// - TestSpec::Pattern::~Pattern() = default; - std::string const& TestSpec::Pattern::name() const { - return m_name; + ScopedMessage::ScopedMessage( MessageBuilder&& builder ): + m_messageId( builder.m_info.sequence ) { + MessageInfo info( CATCH_MOVE( builder.m_info ) ); + info.message = builder.m_stream.str(); + Detail::pushScopedMessage( CATCH_MOVE( info ) ); } - - TestSpec::NamePattern::NamePattern( std::string const& name, std::string const& filterString ) - : Pattern( filterString ) - , m_wildcardPattern( name, CaseSensitive::No ) - {} - - bool TestSpec::NamePattern::matches( TestCaseInfo const& testCase ) const { - return m_wildcardPattern.matches( testCase.name ); + ScopedMessage::ScopedMessage( ScopedMessage&& old ) noexcept: + m_messageId( old.m_messageId ) { + old.m_moved = true; } - void TestSpec::NamePattern::serializeTo( std::ostream& out ) const { - out << '"' << name() << '"'; + ScopedMessage::~ScopedMessage() { + if ( !m_moved ) { Detail::popScopedMessage( m_messageId ); } } - TestSpec::TagPattern::TagPattern( std::string const& tag, std::string const& filterString ) - : Pattern( filterString ) - , m_tag( tag ) - {} + Capturer::Capturer( StringRef macroName, + SourceLineInfo const& lineInfo, + ResultWas::OfType resultType, + StringRef names, + bool isScoped): + m_isScoped(isScoped) { + auto trimmed = [&] (size_t start, size_t end) { + while (names[start] == ',' || isspace(static_cast<unsigned char>(names[start]))) { + ++start; + } + while (names[end] == ',' || isspace(static_cast<unsigned char>(names[end]))) { + --end; + } + return names.substr(start, end - start + 1); + }; + auto skipq = [&] (size_t start, char quote) { + for (auto i = start + 1; i < names.size() ; ++i) { + if (names[i] == quote) + return i; + if (names[i] == '\\') + ++i; + } + CATCH_INTERNAL_ERROR("CAPTURE parsing encountered unmatched quote"); + }; - bool TestSpec::TagPattern::matches( TestCaseInfo const& testCase ) const { - return std::find( begin( testCase.tags ), - end( testCase.tags ), - Tag( m_tag ) ) != end( testCase.tags ); + size_t start = 0; + std::stack<char> openings; + for (size_t pos = 0; pos < names.size(); ++pos) { + char c = names[pos]; + switch (c) { + case '[': + case '{': + case '(': + // It is basically impossible to disambiguate between + // comparison and start of template args in this context +// case '<': + openings.push(c); + break; + case ']': + case '}': + case ')': +// case '>': + openings.pop(); + break; + case '"': + case '\'': + pos = skipq(pos, c); + break; + case ',': + if (start != pos && openings.empty()) { + m_messages.emplace_back(macroName, lineInfo, resultType); + m_messages.back().message += trimmed(start, pos); + m_messages.back().message += " := "_sr; + start = pos; + } + break; + default:; // noop + } + } + assert(openings.empty() && "Mismatched openings"); + m_messages.emplace_back(macroName, lineInfo, resultType); + m_messages.back().message += trimmed(start, names.size() - 1); + m_messages.back().message += " := "_sr; + } + Capturer::~Capturer() { + assert( m_captured == m_messages.size() ); + if ( m_isScoped ) { + for ( auto const& message : m_messages ) { + Detail::popScopedMessage( message.sequence ); + } + } } - void TestSpec::TagPattern::serializeTo( std::ostream& out ) const { - out << name(); + void Capturer::captureValue( size_t index, std::string const& value ) { + assert( index < m_messages.size() ); + m_messages[index].message += value; + if ( m_isScoped ) { + Detail::pushScopedMessage( CATCH_MOVE( m_messages[index] ) ); + } else { + Detail::addUnscopedMessage( CATCH_MOVE( m_messages[index] ) ); + } + m_captured++; } - bool TestSpec::Filter::matches( TestCaseInfo const& testCase ) const { - bool should_use = !testCase.isHidden(); - for (auto const& pattern : m_required) { - should_use = true; - if (!pattern->matches(testCase)) { - return false; +} // end namespace Catch + + + + +#include <exception> + +namespace Catch { + + namespace { + + class RegistryHub : public IRegistryHub, + public IMutableRegistryHub, + private Detail::NonCopyable { + + public: // IRegistryHub + RegistryHub() = default; + ReporterRegistry const& getReporterRegistry() const override { + return m_reporterRegistry; } - } - for (auto const& pattern : m_forbidden) { - if (pattern->matches(testCase)) { - return false; + ITestCaseRegistry const& getTestCaseRegistry() const override { + return m_testCaseRegistry; + } + IExceptionTranslatorRegistry const& getExceptionTranslatorRegistry() const override { + return m_exceptionTranslatorRegistry; + } + ITagAliasRegistry const& getTagAliasRegistry() const override { + return m_tagAliasRegistry; + } + StartupExceptionRegistry const& getStartupExceptionRegistry() const override { + return m_exceptionRegistry; } - } - return should_use; - } - void TestSpec::Filter::serializeTo( std::ostream& out ) const { - bool first = true; - for ( auto const& pattern : m_required ) { - if ( !first ) { - out << ' '; + public: // IMutableRegistryHub + void registerListener( Detail::unique_ptr<EventListenerFactory> factory ) override { + m_reporterRegistry.registerListener( CATCH_MOVE(factory) ); } - out << *pattern; - first = false; - } - for ( auto const& pattern : m_forbidden ) { - if ( !first ) { - out << ' '; + void registerTest( Detail::unique_ptr<TestCaseInfo>&& testInfo, Detail::unique_ptr<ITestInvoker>&& invoker ) override { + m_testCaseRegistry.registerTest( CATCH_MOVE(testInfo), CATCH_MOVE(invoker) ); + } + void registerTranslator( Detail::unique_ptr<IExceptionTranslator>&& translator ) override { + m_exceptionTranslatorRegistry.registerTranslator( CATCH_MOVE(translator) ); + } + void registerTagAlias( std::string const& alias, std::string const& tag, SourceLineInfo const& lineInfo ) override { + m_tagAliasRegistry.add( alias, tag, lineInfo ); + } + void registerStartupException() noexcept override { +#if !defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) + m_exceptionRegistry.add(std::current_exception()); +#else + CATCH_INTERNAL_ERROR("Attempted to register active exception under CATCH_CONFIG_DISABLE_EXCEPTIONS!"); +#endif } - out << *pattern; - first = false; - } - } + ITestCaseRegistry& getMutableTestCaseRegistry() override { + return m_testCaseRegistry; + } - std::string TestSpec::extractFilterName( Filter const& filter ) { - Catch::ReusableStringStream sstr; - sstr << filter; - return sstr.str(); + private: + TestRegistry m_testCaseRegistry; + ReporterRegistry m_reporterRegistry; + ExceptionTranslatorRegistry m_exceptionTranslatorRegistry; + TagAliasRegistry m_tagAliasRegistry; + StartupExceptionRegistry m_exceptionRegistry; + }; } - bool TestSpec::hasFilters() const { - return !m_filters.empty(); - } + using RegistryHubSingleton = Singleton<RegistryHub, IRegistryHub, IMutableRegistryHub>; - bool TestSpec::matches( TestCaseInfo const& testCase ) const { - return std::any_of( m_filters.begin(), m_filters.end(), [&]( Filter const& f ){ return f.matches( testCase ); } ); + IRegistryHub const& getRegistryHub() { + return RegistryHubSingleton::get(); } - - TestSpec::Matches TestSpec::matchesByFilter( std::vector<TestCaseHandle> const& testCases, IConfig const& config ) const { - Matches matches; - matches.reserve( m_filters.size() ); - for ( auto const& filter : m_filters ) { - std::vector<TestCaseHandle const*> currentMatches; - for ( auto const& test : testCases ) { - if ( isThrowSafe( test, config ) && - filter.matches( test.getTestCaseInfo() ) ) { - currentMatches.emplace_back( &test ); - } - } - matches.push_back( FilterMatch{ extractFilterName( filter ), - CATCH_MOVE( currentMatches ) } ); - } - return matches; + IMutableRegistryHub& getMutableRegistryHub() { + return RegistryHubSingleton::getMutable(); } - - const TestSpec::vectorStrings& TestSpec::getInvalidSpecs() const { - return m_invalidSpecs; + void cleanUp() { + cleanupSingletons(); } - - void TestSpec::serializeTo( std::ostream& out ) const { - bool first = true; - for ( auto const& filter : m_filters ) { - if ( !first ) { - out << ','; - } - out << filter; - first = false; - } + std::string translateActiveException() { + return getRegistryHub().getExceptionTranslatorRegistry().translateActiveException(); } -} + +} // end namespace Catch -#include <chrono> namespace Catch { - namespace { - static auto getCurrentNanosecondsSinceEpoch() -> uint64_t { - return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count(); + Session::Session() { + static bool alreadyInstantiated = false; + if( alreadyInstantiated ) { + CATCH_TRY { CATCH_INTERNAL_ERROR( "Only one instance of Catch::Session can ever be used" ); } + CATCH_CATCH_ALL { getMutableRegistryHub().registerStartupException(); } } - } // end unnamed namespace - - void Timer::start() { - m_nanoseconds = getCurrentNanosecondsSinceEpoch(); - } - auto Timer::getElapsedNanoseconds() const -> uint64_t { - return getCurrentNanosecondsSinceEpoch() - m_nanoseconds; + alreadyInstantiated = true; } - auto Timer::getElapsedMicroseconds() const -> uint64_t { - return getElapsedNanoseconds()/1000; + Session::~Session() { + Catch::cleanUp(); } - auto Timer::getElapsedMilliseconds() const -> unsigned int { - return static_cast<unsigned int>(getElapsedMicroseconds()/1000); + + void Session::showHelp() const { + Progmasoft::Catch3::PrintHelp( m_arguments.empty() ? std::string( "tests" ) + : m_arguments.front() ); } - auto Timer::getElapsedSeconds() const -> double { - return static_cast<double>(getElapsedMicroseconds())/1000000.0; + + int Session::applyCommandLine( int argc, char const * const * argv ) { + m_arguments.assign( argv, argv + argc ); + return 0; } +#if defined(CATCH_CONFIG_WCHAR) && defined(_WIN32) && defined(UNICODE) + int Session::applyCommandLine( int argc, wchar_t const * const * argv ) { + m_arguments.clear(); + for ( int i = 0; i < argc; ++i ) { + int bufSize = WideCharToMultiByte( CP_UTF8, 0, argv[i], -1, nullptr, 0, nullptr, nullptr ); + std::string utf8( static_cast<std::size_t>( bufSize ), '\0' ); + WideCharToMultiByte( CP_UTF8, 0, argv[i], -1, &utf8[0], bufSize, nullptr, nullptr ); + if ( !utf8.empty() ) { utf8.pop_back(); } // the terminator + m_arguments.push_back( utf8 ); + } + return 0; + } +#endif -} // namespace Catch + int Session::run() { + return Progmasoft::Catch3::RunTests( m_arguments ); + } +} // end namespace Catch -#include <iomanip> namespace Catch { -namespace Detail { - - namespace { - const int hexThreshold = 255; - - struct Endianness { - enum Arch : uint8_t { - Big, - Little - }; - - static Arch which() { - int one = 1; - // If the lowest byte we read is non-zero, we can assume - // that little endian format is used. - auto value = *reinterpret_cast<char*>(&one); - return value ? Little : Big; - } - }; - - template<typename T> - std::string fpToString(T value, int precision) { - if (Catch::isnan(value)) { - return "nan"; - } - - ReusableStringStream rss; - rss << std::setprecision(precision) - << std::fixed - << value; - std::string d = rss.str(); - std::size_t i = d.find_last_not_of('0'); - if (i != std::string::npos && i != d.size() - 1) { - if (d[i] == '.') - i++; - d = d.substr(0, i + 1); - } - return d; - } - } // end unnamed namespace - - std::size_t catch_strnlen( const char* str, std::size_t n ) { - auto ret = std::char_traits<char>::find( str, n, '\0' ); - if ( ret != nullptr ) { return static_cast<std::size_t>( ret - str ); } - return n; - } - - std::string formatTimeT(std::time_t time) { - std::tm timeInfo = {}; -#ifdef _MSC_VER - const auto err = gmtime_s( &timeInfo, &time ); - if ( err ) { - return "gmtime from provided timepoint has failed. This " - "happens e.g. with pre-1970 dates using Microsoft libc"; - } -#elif defined( __MINGW32__ ) - if ( gmtime_s( &timeInfo, &time ) != 0 ) { - return "gmtime from provided timepoint has failed"; - } -#elif defined( CATCH_PLATFORM_PLAYSTATION ) - gmtime_s( &time, &timeInfo ); -#elif defined( __IAR_SYSTEMS_ICC__ ) - // This toolchain does not provide gmtime_r. Copy the result immediately - // so the formatter does not retain the C library's static buffer. - const std::tm* const converted = std::gmtime( &time ); - if ( converted == nullptr ) { - return "gmtime from provided timepoint has failed"; - } - timeInfo = *converted; -#else - if ( gmtime_r( &time, &timeInfo ) == nullptr ) { - return "gmtime from provided timepoint has failed"; - } -#endif - - auto const timeStampSize = sizeof( "2017-01-16T17:06:45Z" ); - char timeStamp[timeStampSize] = {}; - const char* const fmt = "%Y-%m-%dT%H:%M:%SZ"; - - if ( std::strftime( timeStamp, timeStampSize, fmt, &timeInfo ) == 0 ) { - return "gmtime from provided timepoint has failed"; + RegistrarForTagAliases::RegistrarForTagAliases(char const* alias, char const* tag, SourceLineInfo const& lineInfo) { + CATCH_TRY { + getMutableRegistryHub().registerTagAlias(alias, tag, lineInfo); + } CATCH_CATCH_ALL { + // Do not throw when constructing global objects, instead register the exception to be processed later + getMutableRegistryHub().registerStartupException(); } - return std::string( timeStamp, timeStampSize - 1 ); } - std::string convertIntoString(StringRef string, bool escapeInvisibles) { - std::string ret; - // This is enough for the "don't escape invisibles" case, and a good - // lower bound on the "escape invisibles" case. - ret.reserve( string.size() + 2 ); +} - if ( !escapeInvisibles ) { - ret += '"'; - ret += string; - ret += '"'; - return ret; - } - size_t last_start = 0; - auto write_to = [&]( size_t idx ) { - if ( last_start < idx ) { - ret += string.substr( last_start, idx - last_start ); - } - last_start = idx + 1; - }; - ret += '"'; - for ( size_t i = 0; i < string.size(); ++i ) { - const char c = string[i]; - if ( c == '\r' || c == '\n' || c == '\t' || c == '\f' ) { - write_to( i ); - if ( c == '\r' ) { ret.append( "\\r" ); } - if ( c == '\n' ) { ret.append( "\\n" ); } - if ( c == '\t' ) { ret.append( "\\t" ); } - if ( c == '\f' ) { ret.append( "\\f" ); } - } - } - write_to( string.size() ); - ret += '"'; +#include <cassert> +#include <cctype> +#include <algorithm> +#include <utility> - return ret; - } +namespace Catch { - std::string convertIntoString(StringRef string) { - return convertIntoString(string, getCurrentContext().getConfig()->showInvisibles()); - } + namespace { + using TCP_underlying_type = uint8_t; + static_assert(sizeof(TestCaseProperties) == sizeof(TCP_underlying_type), + "The size of the TestCaseProperties is different from the assumed size"); - std::string rawMemoryToString( const void *object, std::size_t size ) { - // Reverse order for little endian architectures - int i = 0, end = static_cast<int>( size ), inc = 1; - if( Endianness::which() == Endianness::Little ) { - i = end-1; - end = inc = -1; + constexpr TestCaseProperties operator|(TestCaseProperties lhs, TestCaseProperties rhs) { + return static_cast<TestCaseProperties>( + static_cast<TCP_underlying_type>(lhs) | static_cast<TCP_underlying_type>(rhs) + ); } - unsigned char const *bytes = static_cast<unsigned char const *>(object); - ReusableStringStream rss; - rss << "0x" << std::setfill('0') << std::hex; - for( ; i != end; i += inc ) - rss << std::setw(2) << static_cast<unsigned>(bytes[i]); - return rss.str(); - } - - std::string makeExceptionHappenedString() { - return "{ stringification failed with an exception: \"" + - translateActiveException() + "\" }"; + constexpr TestCaseProperties& operator|=(TestCaseProperties& lhs, TestCaseProperties rhs) { + lhs = static_cast<TestCaseProperties>( + static_cast<TCP_underlying_type>(lhs) | static_cast<TCP_underlying_type>(rhs) + ); + return lhs; + } - } + constexpr TestCaseProperties operator&(TestCaseProperties lhs, TestCaseProperties rhs) { + return static_cast<TestCaseProperties>( + static_cast<TCP_underlying_type>(lhs) & static_cast<TCP_underlying_type>(rhs) + ); + } -} // end Detail namespace + constexpr bool applies(TestCaseProperties tcp) { + static_assert(static_cast<TCP_underlying_type>(TestCaseProperties::None) == 0, + "TestCaseProperties::None must be equal to 0"); + return tcp != TestCaseProperties::None; + } + bool isRunnerTagWithValue( StringRef tag ) { + for ( StringRef prefix : { "!resource:"_sr, "!depends:"_sr, + "!timeout:"_sr, "!retries:"_sr } ) { + if ( tag.size() >= prefix.size() && + tag.substr( 0, prefix.size() ) == prefix ) { + return true; + } + } + return false; + } + TestCaseProperties propertiesFromTag( StringRef tag, SourceLineInfo const& _lineInfo ) { + if ( tag.empty() ) { return TestCaseProperties::None; } + // All tags starting with alphanumeric characters are normal. + if ( std::isalnum( tag[0] ) ) { return TestCaseProperties::None; } -//// ======================================================= //// -// -// Out-of-line defs for full specialization of StringMaker -// -//// ======================================================= //// + // Explicitly check known special tags, and error out if it is not one of them + if ( tag[0] == '.' ) { return TestCaseProperties::IsHidden; } + else if ( tag == "!throws"_sr ) { return TestCaseProperties::Throws; } + else if ( tag == "!shouldfail"_sr ) { return TestCaseProperties::ShouldFail; } + else if ( tag == "!mayfail"_sr ) { return TestCaseProperties::MayFail; } + else if ( tag == "!nonportable"_sr ) { return TestCaseProperties::NonPortable; } + else if ( tag == "!benchmark"_sr ) { + return TestCaseProperties::Benchmark | + TestCaseProperties::IsHidden; + } + // Read by the Catch3 runner: scheduling, limits and quarantine. + else if ( tag == "!serial"_sr || tag == "!isolated"_sr || + tag == "!quarantine"_sr ) { + return TestCaseProperties::None; + } else if ( isRunnerTagWithValue( tag ) ) { + CATCH_ENFORCE( tag[tag.size() - 1] != ':', + "Tag name: [" << tag << "] needs a value after ':'\n" + << _lineInfo ); + return TestCaseProperties::None; + } else { + CATCH_RUNTIME_ERROR( + "Tag name: [" << tag << "] is not allowed.\n" + << "Tag names starting with non-alphanumeric " + "characters are reserved.\n" + << _lineInfo ); + } + } -std::string StringMaker<std::string>::convert(const std::string& str) { - return Detail::convertIntoString( str ); -} + std::string makeDefaultName() { + static size_t counter = 0; + return "Anonymous test case " + std::to_string(++counter); + } -#ifdef CATCH_CONFIG_CPP17_STRING_VIEW -std::string StringMaker<std::string_view>::convert(std::string_view str) { - return Detail::convertIntoString( StringRef( str.data(), str.size() ) ); -} -#endif + constexpr StringRef extractFilenamePart(StringRef filename) { + size_t lastDot = filename.size(); + while (lastDot > 0 && filename[lastDot - 1] != '.') { + --lastDot; + } + // In theory we could have filename without any extension in it + if ( lastDot == 0 ) { return StringRef(); } -std::string StringMaker<char const*>::convert(char const* str) { - if (str) { - return Detail::convertIntoString( str ); - } else { - return{ "{null string}" }; - } -} -std::string StringMaker<char*>::convert(char* str) { // NOLINT(readability-non-const-parameter) - if (str) { - return Detail::convertIntoString( str ); - } else { - return{ "{null string}" }; - } -} + --lastDot; + size_t nameStart = lastDot; + while (nameStart > 0 && filename[nameStart - 1] != '/' && filename[nameStart - 1] != '\\') { + --nameStart; + } -#ifdef CATCH_CONFIG_WCHAR -std::string StringMaker<std::wstring>::convert(const std::wstring& wstr) { - std::string s; - s.reserve(wstr.size()); - for (auto c : wstr) { - s += (c <= 0xff) ? static_cast<char>(c) : '?'; - } - return ::Catch::Detail::stringify(s); -} + return filename.substr(nameStart, lastDot - nameStart); + } -# ifdef CATCH_CONFIG_CPP17_STRING_VIEW -std::string StringMaker<std::wstring_view>::convert(std::wstring_view str) { - return StringMaker<std::wstring>::convert(std::wstring(str)); -} -# endif + // Returns the upper bound on size of extra tags ([#file]+[.]) + constexpr size_t sizeOfExtraTags(StringRef filepath) { + // [.] is 3, [#] is another 3 + const size_t extras = 3 + 3; + return extractFilenamePart(filepath).size() + extras; + } + } // end unnamed namespace -std::string StringMaker<wchar_t const*>::convert(wchar_t const * str) { - if (str) { - return ::Catch::Detail::stringify(std::wstring{ str }); - } else { - return{ "{null string}" }; + bool operator<( Tag const& lhs, Tag const& rhs ) { + Detail::CaseInsensitiveLess cmp; + return cmp( lhs.original, rhs.original ); } -} -std::string StringMaker<wchar_t *>::convert(wchar_t * str) { - if (str) { - return ::Catch::Detail::stringify(std::wstring{ str }); - } else { - return{ "{null string}" }; + bool operator==( Tag const& lhs, Tag const& rhs ) { + Detail::CaseInsensitiveEqualTo cmp; + return cmp( lhs.original, rhs.original ); } -} -#endif - -#if defined(CATCH_CONFIG_CPP17_BYTE) -#include <cstddef> -std::string StringMaker<std::byte>::convert(std::byte value) { - return ::Catch::Detail::stringify(std::to_integer<unsigned long long>(value)); -} -#endif // defined(CATCH_CONFIG_CPP17_BYTE) -std::string StringMaker<int>::convert(int value) { - return ::Catch::Detail::stringify(static_cast<long long>(value)); -} -std::string StringMaker<long>::convert(long value) { - return ::Catch::Detail::stringify(static_cast<long long>(value)); -} -std::string StringMaker<long long>::convert(long long value) { - ReusableStringStream rss; - rss << value; - if (value > Detail::hexThreshold) { - rss << " (0x" << std::hex << value << ')'; + Detail::unique_ptr<TestCaseInfo> + makeTestCaseInfo(StringRef _className, + NameAndTags const& nameAndTags, + SourceLineInfo const& _lineInfo ) { + return Detail::make_unique<TestCaseInfo>(_className, nameAndTags, _lineInfo); } - return rss.str(); -} -std::string StringMaker<unsigned int>::convert(unsigned int value) { - return ::Catch::Detail::stringify(static_cast<unsigned long long>(value)); -} -std::string StringMaker<unsigned long>::convert(unsigned long value) { - return ::Catch::Detail::stringify(static_cast<unsigned long long>(value)); -} -std::string StringMaker<unsigned long long>::convert(unsigned long long value) { - ReusableStringStream rss; - rss << value; - if (value > Detail::hexThreshold) { - rss << " (0x" << std::hex << value << ')'; - } - return rss.str(); -} + TestCaseInfo::TestCaseInfo(StringRef _className, + NameAndTags const& _nameAndTags, + SourceLineInfo const& _lineInfo): + name( _nameAndTags.name.empty() ? makeDefaultName() : _nameAndTags.name ), + className( _className ), + lineInfo( _lineInfo ) + { + StringRef originalTags = _nameAndTags.tags; + // We need to reserve enough space to store all of the tags + // (including optional hidden tag and filename tag) + auto requiredSize = originalTags.size() + sizeOfExtraTags(_lineInfo.file); + backingTags.reserve(requiredSize); -std::string StringMaker<signed char>::convert(signed char value) { - if (value == '\r') { - return "'\\r'"; - } else if (value == '\f') { - return "'\\f'"; - } else if (value == '\n') { - return "'\\n'"; - } else if (value == '\t') { - return "'\\t'"; - } else if ('\0' <= value && value < ' ') { - return ::Catch::Detail::stringify(static_cast<unsigned int>(value)); - } else { - char chstr[] = "' '"; - chstr[1] = value; - return chstr; - } -} -std::string StringMaker<char>::convert(char c) { - return ::Catch::Detail::stringify(static_cast<signed char>(c)); -} -std::string StringMaker<unsigned char>::convert(unsigned char value) { - return ::Catch::Detail::stringify(static_cast<char>(value)); -} + // Since there is usually only small number of tags on a test case, + // the geometric growth on vector does not get time to properly + // amortize the reallocations. But we can save a lot of time by + // pre-allocating space for all tags. + if ( !originalTags.empty() ) { + size_t tagCount = 1; // 1 for filename tag + for ( size_t i = 0; i < originalTags.size(); ++i ) { + if ( originalTags[i] == '[' ) { ++tagCount; } + } + tags.reserve( tagCount ); + } -int StringMaker<float>::precision = std::numeric_limits<float>::max_digits10; + // We cannot copy the tags directly, as we need to normalize + // some tags, so that [.foo] is copied as [.][foo]. + size_t tagStart = 0; + size_t tagEnd = 0; + bool inTag = false; + for (size_t idx = 0; idx < originalTags.size(); ++idx) { + auto c = originalTags[idx]; + if (c == '[') { + CATCH_ENFORCE( + !inTag, + "Found '[' inside a tag while registering test case '" + << _nameAndTags.name << "' at " << _lineInfo ); -std::string StringMaker<float>::convert(float value) { - return Detail::fpToString(value, precision) + 'f'; -} + inTag = true; + tagStart = idx; + } + if (c == ']') { + CATCH_ENFORCE( + inTag, + "Found unmatched ']' while registering test case '" + << _nameAndTags.name << "' at " << _lineInfo ); -int StringMaker<double>::precision = std::numeric_limits<double>::max_digits10; + inTag = false; + tagEnd = idx; + assert(tagStart < tagEnd); -std::string StringMaker<double>::convert(double value) { - return Detail::fpToString(value, precision); -} + // We need to check the tag for special meanings, copy + // it over to backing storage and actually reference the + // backing storage in the saved tags + StringRef tagStr = originalTags.substr(tagStart+1, tagEnd - tagStart - 1); + CATCH_ENFORCE( !tagStr.empty(), + "Found an empty tag while registering test case '" + << _nameAndTags.name << "' at " + << _lineInfo ); -} // end namespace Catch + properties |= propertiesFromTag(tagStr, _lineInfo); + // When copying a tag to the backing storage, we need to + // check if it is a merged hide tag, such as [.foo], and + // if it is, we need to handle it as if it was [foo]. + if (tagStr.size() > 1 && tagStr[0] == '.') { + tagStr = tagStr.substr(1, tagStr.size() - 1); + } + // We skip over dealing with the [.] tag, as we will add + // it later unconditionally and then sort and unique all + // the tags. + internalAppendTag(tagStr); + } + } + CATCH_ENFORCE( !inTag, + "Found an unclosed tag while registering test case '" + << _nameAndTags.name << "' at " << _lineInfo ); + // Add [.] if relevant + if (isHidden()) { + internalAppendTag("."_sr); + } -namespace Catch { + // Tests usually have small number of tags, so we explicitly handle + // the 1 and 2 tags cases for better performance in low-opt builds. + switch ( tags.size() ) { + case 1: + break; + case 2: + if ( tags[0] == tags[1] ) { tags.pop_back(); } + else if ( tags[1] < tags[0] ) { std::swap( tags[0], tags[1] ); } + break; + default: + std::sort( begin( tags ), end( tags ) ); + tags.erase( std::unique( begin( tags ), end( tags ) ), + end( tags ) ); + } - Counts Counts::operator - ( Counts const& other ) const { - Counts diff; - diff.passed = passed - other.passed; - diff.failed = failed - other.failed; - diff.failedButOk = failedButOk - other.failedButOk; - diff.skipped = skipped - other.skipped; - return diff; } - Counts& Counts::operator += ( Counts const& other ) { - passed += other.passed; - failed += other.failed; - failedButOk += other.failedButOk; - skipped += other.skipped; - return *this; + bool TestCaseInfo::isHidden() const { + return applies( properties & TestCaseProperties::IsHidden ); } - - std::uint64_t Counts::total() const { - return passed + failed + failedButOk + skipped; + bool TestCaseInfo::throws() const { + return applies( properties & TestCaseProperties::Throws ); } - bool Counts::allPassed() const { - return failed == 0 && failedButOk == 0 && skipped == 0; + bool TestCaseInfo::okToFail() const { + return applies( properties & (TestCaseProperties::ShouldFail | TestCaseProperties::MayFail ) ); } - bool Counts::allOk() const { - return failed == 0; + bool TestCaseInfo::expectedToFail() const { + return applies( properties & (TestCaseProperties::ShouldFail) ); } - Totals Totals::operator - ( Totals const& other ) const { - Totals diff; - diff.assertions = assertions - other.assertions; - diff.testCases = testCases - other.testCases; - return diff; + void TestCaseInfo::addFilenameTag() { + std::string combined("#"); + combined += extractFilenamePart(lineInfo.file); + internalAppendTag(combined); } - Totals& Totals::operator += ( Totals const& other ) { - assertions += other.assertions; - testCases += other.testCases; - return *this; - } + std::string TestCaseInfo::tagsAsString() const { + std::string ret; + // '[' and ']' per tag + std::size_t full_size = 2 * tags.size(); + for (const auto& tag : tags) { + full_size += tag.original.size(); + } + ret.reserve(full_size); + for (const auto& tag : tags) { + ret.push_back('['); + ret += tag.original; + ret.push_back(']'); + } - Totals Totals::delta( Totals const& prevTotals ) const { - Totals diff = *this - prevTotals; - if( diff.assertions.failed > 0 ) - ++diff.testCases.failed; - else if( diff.assertions.failedButOk > 0 ) - ++diff.testCases.failedButOk; - else if ( diff.assertions.skipped > 0 ) - ++ diff.testCases.skipped; - else - ++diff.testCases.passed; - return diff; + return ret; } -} - - - + void TestCaseInfo::internalAppendTag(StringRef tagStr) { + backingTags += '['; + const auto backingStart = backingTags.size(); + backingTags += tagStr; + const auto backingEnd = backingTags.size(); + backingTags += ']'; + tags.emplace_back(StringRef(backingTags.c_str() + backingStart, backingEnd - backingStart)); + } -namespace Catch { - namespace Detail { - void registerTranslatorImpl( - Detail::unique_ptr<IExceptionTranslator>&& translator ) { - getMutableRegistryHub().registerTranslator( - CATCH_MOVE( translator ) ); + bool operator<( TestCaseInfo const& lhs, TestCaseInfo const& rhs ) { + // We want to avoid redoing the string comparisons multiple times, + // so we store the result of a three-way comparison before using + // it in the actual comparison logic. + const auto cmpName = lhs.name.compare( rhs.name ); + if ( cmpName != 0 ) { + return cmpName < 0; } - } // namespace Detail -} // namespace Catch - - -#include <ostream> - -namespace Catch { - - Version::Version - ( unsigned int _majorVersion, - unsigned int _minorVersion, - unsigned int _patchNumber, - char const * const _branchName, - unsigned int _buildNumber ) - : majorVersion( _majorVersion ), - minorVersion( _minorVersion ), - patchNumber( _patchNumber ), - branchName( _branchName ), - buildNumber( _buildNumber ) - {} - - std::ostream& operator << ( std::ostream& os, Version const& version ) { - os << version.majorVersion << '.' - << version.minorVersion << '.' - << version.patchNumber; - // branchName is never null -> 0th char is \0 if it is empty - if (version.branchName[0]) { - os << '-' << version.branchName - << '.' << version.buildNumber; + const auto cmpClassName = lhs.className.compare( rhs.className ); + if ( cmpClassName != 0 ) { + return cmpClassName < 0; } - return os; - } - - Version const& libraryVersion() { - static Version version( 3, 16, 1, "", 0 ); - return version; + return lhs.tags < rhs.tags; } -} - - - - -namespace Catch { - - const char* GeneratorException::what() const noexcept { - return m_msg; + bool operator==( TestCaseInfo const& lhs, TestCaseInfo const& rhs ) { + return lhs.name == rhs.name + && lhs.className == rhs.className + && lhs.tags == rhs.tags; } } // end namespace Catch +#include <algorithm> +#include <string> +#include <vector> +#include <ostream> namespace Catch { - IGeneratorTracker::~IGeneratorTracker() = default; - -namespace Generators { - - GeneratorUntypedBase::~GeneratorUntypedBase() = default; + TestSpec::Pattern::Pattern( std::string const& name ) + : m_name( name ) + {} - IGeneratorTracker* acquireGeneratorTracker(StringRef generatorName, SourceLineInfo const& lineInfo ) { - return getResultCapture().acquireGeneratorTracker( generatorName, lineInfo ); - } + TestSpec::Pattern::~Pattern() = default; - IGeneratorTracker* createGeneratorTracker( StringRef generatorName, - SourceLineInfo lineInfo, - GeneratorBasePtr&& generator ) { - return getResultCapture().createGeneratorTracker( - generatorName, lineInfo, CATCH_MOVE( generator ) ); + std::string const& TestSpec::Pattern::name() const { + return m_name; } -} // namespace Generators -} // namespace Catch + TestSpec::NamePattern::NamePattern( std::string const& name, std::string const& filterString ) + : Pattern( filterString ) + , m_wildcardPattern( name, CaseSensitive::No ) + {} + bool TestSpec::NamePattern::matches( TestCaseInfo const& testCase ) const { + return m_wildcardPattern.matches( testCase.name ); + } + void TestSpec::NamePattern::serializeTo( std::ostream& out ) const { + out << '"' << name() << '"'; + } -#include <random> -namespace Catch { - namespace Generators { - namespace Detail { - std::uint32_t getSeed() { return sharedRng()(); } - } // namespace Detail + TestSpec::TagPattern::TagPattern( std::string const& tag, std::string const& filterString ) + : Pattern( filterString ) + , m_tag( tag ) + {} - struct RandomFloatingGenerator<long double>::PImpl { - PImpl( long double a, long double b, uint32_t seed ): - rng( seed ), dist( a, b ) {} + bool TestSpec::TagPattern::matches( TestCaseInfo const& testCase ) const { + return std::find( begin( testCase.tags ), + end( testCase.tags ), + Tag( m_tag ) ) != end( testCase.tags ); + } - Catch::SimplePcg32 rng; - std::uniform_real_distribution<long double> dist; - }; + void TestSpec::TagPattern::serializeTo( std::ostream& out ) const { + out << name(); + } - RandomFloatingGenerator<long double>::RandomFloatingGenerator( - long double a, long double b, std::uint32_t seed) : - m_pimpl(Catch::Detail::make_unique<PImpl>(a, b, seed)) { - static_cast<void>( next() ); + bool TestSpec::Filter::matches( TestCaseInfo const& testCase ) const { + bool should_use = !testCase.isHidden(); + for (auto const& pattern : m_required) { + should_use = true; + if (!pattern->matches(testCase)) { + return false; + } } - - RandomFloatingGenerator<long double>::~RandomFloatingGenerator() = - default; - bool RandomFloatingGenerator<long double>::next() { - m_current_number = m_pimpl->dist( m_pimpl->rng ); - return true; + for (auto const& pattern : m_forbidden) { + if (pattern->matches(testCase)) { + return false; + } } + return should_use; + } - bool RandomFloatingGenerator<long double>::isFinite() const { - return false; + void TestSpec::Filter::serializeTo( std::ostream& out ) const { + bool first = true; + for ( auto const& pattern : m_required ) { + if ( !first ) { + out << ' '; + } + out << *pattern; + first = false; + } + for ( auto const& pattern : m_forbidden ) { + if ( !first ) { + out << ' '; + } + out << *pattern; + first = false; } + } - } // namespace Generators -} // namespace Catch + std::string TestSpec::extractFilterName( Filter const& filter ) { + Catch::ReusableStringStream sstr; + sstr << filter; + return sstr.str(); + } + + bool TestSpec::hasFilters() const { + return !m_filters.empty(); + } + bool TestSpec::matches( TestCaseInfo const& testCase ) const { + return std::any_of( m_filters.begin(), m_filters.end(), [&]( Filter const& f ){ return f.matches( testCase ); } ); + } + TestSpec::Matches TestSpec::matchesByFilter( std::vector<TestCaseHandle> const& testCases, IConfig const& config ) const { + Matches matches; + matches.reserve( m_filters.size() ); + for ( auto const& filter : m_filters ) { + std::vector<TestCaseHandle const*> currentMatches; + for ( auto const& test : testCases ) { + if ( isThrowSafe( test, config ) && + filter.matches( test.getTestCaseInfo() ) ) { + currentMatches.emplace_back( &test ); + } + } + matches.push_back( FilterMatch{ extractFilterName( filter ), + CATCH_MOVE( currentMatches ) } ); + } + return matches; + } -namespace Catch { - namespace Generators { - namespace Detail { + const TestSpec::vectorStrings& TestSpec::getInvalidSpecs() const { + return m_invalidSpecs; + } - [[noreturn]] - void throw_generator_exception( char const* msg ) { - Catch::throw_exception( GeneratorException{ msg } ); + void TestSpec::serializeTo( std::ostream& out ) const { + bool first = true; + for ( auto const& filter : m_filters ) { + if ( !first ) { + out << ','; } + out << filter; + first = false; + } + } - } // namespace Detail - } // namespace Generators -} // namespace Catch +} +#include <chrono> namespace Catch { - namespace Detail { - void missingCaptureInstance() { - CATCH_INTERNAL_ERROR( "No result capture instance" ); - } - } // namespace Detail - IResultCapture::~IResultCapture() = default; -} // namespace Catch + namespace { + static auto getCurrentNanosecondsSinceEpoch() -> uint64_t { + return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::steady_clock::now().time_since_epoch()).count(); + } + } // end unnamed namespace + void Timer::start() { + m_nanoseconds = getCurrentNanosecondsSinceEpoch(); + } + auto Timer::getElapsedNanoseconds() const -> uint64_t { + return getCurrentNanosecondsSinceEpoch() - m_nanoseconds; + } + auto Timer::getElapsedMicroseconds() const -> uint64_t { + return getElapsedNanoseconds()/1000; + } + auto Timer::getElapsedMilliseconds() const -> unsigned int { + return static_cast<unsigned int>(getElapsedMicroseconds()/1000); + } + auto Timer::getElapsedSeconds() const -> double { + return static_cast<double>(getElapsedMicroseconds())/1000000.0; + } +} // namespace Catch -namespace Catch { - IConfig::~IConfig() = default; -} +#include <iomanip> namespace Catch { - IExceptionTranslator::~IExceptionTranslator() = default; - IExceptionTranslatorRegistry::~IExceptionTranslatorRegistry() = default; -} - +namespace Detail { + namespace { + const int hexThreshold = 255; -#include <string> + struct Endianness { + enum Arch : uint8_t { + Big, + Little + }; -namespace Catch { - namespace Generators { + static Arch which() { + int one = 1; + // If the lowest byte we read is non-zero, we can assume + // that little endian format is used. + auto value = *reinterpret_cast<char*>(&one); + return value ? Little : Big; + } + }; - bool GeneratorUntypedBase::countedNext() { - auto ret = next(); - if ( ret ) { - m_stringReprCache.clear(); - ++m_currentElementIndex; + template<typename T> + std::string fpToString(T value, int precision) { + if (Catch::isnan(value)) { + return "nan"; } - return ret; - } - void GeneratorUntypedBase::skipToNthElementImpl( std::size_t n ) { - for ( size_t i = m_currentElementIndex; i < n; ++i ) { - bool isValid = next(); - if ( !isValid ) { - Detail::throw_generator_exception( - "Could not jump to Nth element: not enough elements" ); - } + ReusableStringStream rss; + rss << std::setprecision(precision) + << std::fixed + << value; + std::string d = rss.str(); + std::size_t i = d.find_last_not_of('0'); + if (i != std::string::npos && i != d.size() - 1) { + if (d[i] == '.') + i++; + d = d.substr(0, i + 1); } + return d; } + } // end unnamed namespace - void GeneratorUntypedBase::skipToNthElement( std::size_t n ) { - if ( n < m_currentElementIndex ) { - Detail::throw_generator_exception( - "Tried to jump generator backwards" ); - } - if ( n == m_currentElementIndex ) { return; } + std::size_t catch_strnlen( const char* str, std::size_t n ) { + auto ret = std::char_traits<char>::find( str, n, '\0' ); + if ( ret != nullptr ) { return static_cast<std::size_t>( ret - str ); } + return n; + } - skipToNthElementImpl(n); - // Fixup tracking after moving the generator forward - // * Ensure that the correct element index is set after skipping - // * Invalidate cache - m_currentElementIndex = n; - m_stringReprCache.clear(); + std::string formatTimeT(std::time_t time) { + std::tm timeInfo = {}; +#ifdef _MSC_VER + const auto err = gmtime_s( &timeInfo, &time ); + if ( err ) { + return "gmtime from provided timepoint has failed. This " + "happens e.g. with pre-1970 dates using Microsoft libc"; } - - StringRef GeneratorUntypedBase::currentElementAsString() const { - if ( m_stringReprCache.empty() ) { - m_stringReprCache = stringifyImpl(); - } - return m_stringReprCache; +#elif defined( __MINGW32__ ) + if ( gmtime_s( &timeInfo, &time ) != 0 ) { + return "gmtime from provided timepoint has failed"; } +#elif defined( CATCH_PLATFORM_PLAYSTATION ) + gmtime_s( &time, &timeInfo ); +#elif defined( __IAR_SYSTEMS_ICC__ ) + // This toolchain does not provide gmtime_r. Copy the result immediately + // so the formatter does not retain the C library's static buffer. + const std::tm* const converted = std::gmtime( &time ); + if ( converted == nullptr ) { + return "gmtime from provided timepoint has failed"; + } + timeInfo = *converted; +#else + if ( gmtime_r( &time, &timeInfo ) == nullptr ) { + return "gmtime from provided timepoint has failed"; + } +#endif - bool GeneratorUntypedBase::isFinite() const { return true; } - - } // namespace Generators -} // namespace Catch - - - - -namespace Catch { - IRegistryHub::~IRegistryHub() = default; - IMutableRegistryHub::~IMutableRegistryHub() = default; -} + auto const timeStampSize = sizeof( "2017-01-16T17:06:45Z" ); + char timeStamp[timeStampSize] = {}; + const char* const fmt = "%Y-%m-%dT%H:%M:%SZ"; + if ( std::strftime( timeStamp, timeStampSize, fmt, &timeInfo ) == 0 ) { + return "gmtime from provided timepoint has failed"; + } + return std::string( timeStamp, timeStampSize - 1 ); + } + std::string convertIntoString(StringRef string, bool escapeInvisibles) { + std::string ret; + // This is enough for the "don't escape invisibles" case, and a good + // lower bound on the "escape invisibles" case. + ret.reserve( string.size() + 2 ); -#include <cassert> + if ( !escapeInvisibles ) { + ret += '"'; + ret += string; + ret += '"'; + return ret; + } -namespace Catch { + size_t last_start = 0; + auto write_to = [&]( size_t idx ) { + if ( last_start < idx ) { + ret += string.substr( last_start, idx - last_start ); + } + last_start = idx + 1; + }; - ReporterConfig::ReporterConfig( - IConfig const* _fullConfig, - Detail::unique_ptr<IStream> _stream, - ColourMode colourMode, - Verbosity verbosity, - std::map<std::string, std::string> customOptions ): - m_stream( CATCH_MOVE(_stream) ), - m_fullConfig( _fullConfig ), - m_colourMode( colourMode ), - m_verbosity( verbosity ), - m_customOptions( CATCH_MOVE( customOptions ) ) {} + ret += '"'; + for ( size_t i = 0; i < string.size(); ++i ) { + const char c = string[i]; + if ( c == '\r' || c == '\n' || c == '\t' || c == '\f' ) { + write_to( i ); + if ( c == '\r' ) { ret.append( "\\r" ); } + if ( c == '\n' ) { ret.append( "\\n" ); } + if ( c == '\t' ) { ret.append( "\\t" ); } + if ( c == '\f' ) { ret.append( "\\f" ); } + } + } + write_to( string.size() ); + ret += '"'; - Detail::unique_ptr<IStream> ReporterConfig::takeStream() && { - assert( m_stream ); - return CATCH_MOVE( m_stream ); + return ret; } - IConfig const * ReporterConfig::fullConfig() const { return m_fullConfig; } - ColourMode ReporterConfig::colourMode() const { return m_colourMode; } - Verbosity ReporterConfig::verbosity() const { return m_verbosity; } - std::map<std::string, std::string> const& - ReporterConfig::customOptions() const { - return m_customOptions; + std::string convertIntoString(StringRef string) { + return convertIntoString(string, getCurrentContext().getConfig()->showInvisibles()); } - ReporterConfig::~ReporterConfig() = default; - - AssertionStats::AssertionStats( AssertionResult const& _assertionResult, - std::vector<MessageInfo> const& _infoMessages, - Totals const& _totals ) - : assertionResult( _assertionResult ), - infoMessages( _infoMessages ), - totals( _totals ) - { - if( assertionResult.hasMessage() ) { - // Copy message into messages list. - // !TBD This should have been done earlier, somewhere - MessageBuilder builder( assertionResult.getTestMacroName(), assertionResult.getSourceInfo(), assertionResult.getResultType() ); - builder.m_info.message = static_cast<std::string>(assertionResult.getMessage()); - - infoMessages.push_back( CATCH_MOVE(builder.m_info) ); + std::string rawMemoryToString( const void *object, std::size_t size ) { + // Reverse order for little endian architectures + int i = 0, end = static_cast<int>( size ), inc = 1; + if( Endianness::which() == Endianness::Little ) { + i = end-1; + end = inc = -1; } + + unsigned char const *bytes = static_cast<unsigned char const *>(object); + ReusableStringStream rss; + rss << "0x" << std::setfill('0') << std::hex; + for( ; i != end; i += inc ) + rss << std::setw(2) << static_cast<unsigned>(bytes[i]); + return rss.str(); } - SectionStats::SectionStats( SectionInfo&& _sectionInfo, - Counts const& _assertions, - double _durationInSeconds, - bool _missingAssertions ) - : sectionInfo( CATCH_MOVE(_sectionInfo) ), - assertions( _assertions ), - durationInSeconds( _durationInSeconds ), - missingAssertions( _missingAssertions ) - {} + std::string makeExceptionHappenedString() { + return "{ stringification failed with an exception: \"" + + translateActiveException() + "\" }"; + } - TestCaseStats::TestCaseStats( TestCaseInfo const& _testInfo, - Totals const& _totals, - std::string&& _stdOut, - std::string&& _stdErr, - bool _aborting ) - : testInfo( &_testInfo ), - totals( _totals ), - stdOut( CATCH_MOVE(_stdOut) ), - stdErr( CATCH_MOVE(_stdErr) ), - aborting( _aborting ) - {} +} // end Detail namespace - TestRunStats::TestRunStats( TestRunInfo const& _runInfo, - Totals const& _totals, - bool _aborting ) - : runInfo( _runInfo ), - totals( _totals ), - aborting( _aborting ) - {} - IEventListener::~IEventListener() = default; +//// ======================================================= //// +// +// Out-of-line defs for full specialization of StringMaker +// +//// ======================================================= //// -} // end namespace Catch +std::string StringMaker<std::string>::convert(const std::string& str) { + return Detail::convertIntoString( str ); +} +#ifdef CATCH_CONFIG_CPP17_STRING_VIEW +std::string StringMaker<std::string_view>::convert(std::string_view str) { + return Detail::convertIntoString( StringRef( str.data(), str.size() ) ); +} +#endif +std::string StringMaker<char const*>::convert(char const* str) { + if (str) { + return Detail::convertIntoString( str ); + } else { + return{ "{null string}" }; + } +} +std::string StringMaker<char*>::convert(char* str) { // NOLINT(readability-non-const-parameter) + if (str) { + return Detail::convertIntoString( str ); + } else { + return{ "{null string}" }; + } +} +#ifdef CATCH_CONFIG_WCHAR +std::string StringMaker<std::wstring>::convert(const std::wstring& wstr) { + std::string s; + s.reserve(wstr.size()); + for (auto c : wstr) { + s += (c <= 0xff) ? static_cast<char>(c) : '?'; + } + return ::Catch::Detail::stringify(s); +} -namespace Catch { - IReporterFactory::~IReporterFactory() = default; - EventListenerFactory::~EventListenerFactory() = default; +# ifdef CATCH_CONFIG_CPP17_STRING_VIEW +std::string StringMaker<std::wstring_view>::convert(std::wstring_view str) { + return StringMaker<std::wstring>::convert(std::wstring(str)); } +# endif +std::string StringMaker<wchar_t const*>::convert(wchar_t const * str) { + if (str) { + return ::Catch::Detail::stringify(std::wstring{ str }); + } else { + return{ "{null string}" }; + } +} +std::string StringMaker<wchar_t *>::convert(wchar_t * str) { + if (str) { + return ::Catch::Detail::stringify(std::wstring{ str }); + } else { + return{ "{null string}" }; + } +} +#endif +#if defined(CATCH_CONFIG_CPP17_BYTE) +#include <cstddef> +std::string StringMaker<std::byte>::convert(std::byte value) { + return ::Catch::Detail::stringify(std::to_integer<unsigned long long>(value)); +} +#endif // defined(CATCH_CONFIG_CPP17_BYTE) +std::string StringMaker<int>::convert(int value) { + return ::Catch::Detail::stringify(static_cast<long long>(value)); +} +std::string StringMaker<long>::convert(long value) { + return ::Catch::Detail::stringify(static_cast<long long>(value)); +} +std::string StringMaker<long long>::convert(long long value) { + ReusableStringStream rss; + rss << value; + if (value > Detail::hexThreshold) { + rss << " (0x" << std::hex << value << ')'; + } + return rss.str(); +} -namespace Catch { - ITestCaseRegistry::~ITestCaseRegistry() = default; +std::string StringMaker<unsigned int>::convert(unsigned int value) { + return ::Catch::Detail::stringify(static_cast<unsigned long long>(value)); +} +std::string StringMaker<unsigned long>::convert(unsigned long value) { + return ::Catch::Detail::stringify(static_cast<unsigned long long>(value)); +} +std::string StringMaker<unsigned long long>::convert(unsigned long long value) { + ReusableStringStream rss; + rss << value; + if (value > Detail::hexThreshold) { + rss << " (0x" << std::hex << value << ')'; + } + return rss.str(); } +std::string StringMaker<signed char>::convert(signed char value) { + if (value == '\r') { + return "'\\r'"; + } else if (value == '\f') { + return "'\\f'"; + } else if (value == '\n') { + return "'\\n'"; + } else if (value == '\t') { + return "'\\t'"; + } else if ('\0' <= value && value < ' ') { + return ::Catch::Detail::stringify(static_cast<unsigned int>(value)); + } else { + char chstr[] = "' '"; + chstr[1] = value; + return chstr; + } +} +std::string StringMaker<char>::convert(char c) { + return ::Catch::Detail::stringify(static_cast<signed char>(c)); +} +std::string StringMaker<unsigned char>::convert(unsigned char value) { + return ::Catch::Detail::stringify(static_cast<char>(value)); +} +int StringMaker<float>::precision = std::numeric_limits<float>::max_digits10; -namespace Catch { +std::string StringMaker<float>::convert(float value) { + return Detail::fpToString(value, precision) + 'f'; +} - void AssertionHandler::finishIncomplete() { - m_resultCapture.handleIncomplete( m_assertionInfo ); - } +int StringMaker<double>::precision = std::numeric_limits<double>::max_digits10; - AssertionHandler::AssertionHandler - ( StringRef macroName, - SourceLineInfo const& lineInfo, - StringRef capturedExpression, - ResultDisposition::Flags resultDisposition ) - : m_assertionInfo{ macroName, lineInfo, capturedExpression, resultDisposition }, - m_resultCapture( static_cast<RunContext&>(getResultCapture()) ) - { - m_resultCapture.notifyAssertionStarted( m_assertionInfo ); - } +std::string StringMaker<double>::convert(double value) { + return Detail::fpToString(value, precision); +} - void AssertionHandler::handleExpr( ITransientExpression const& expr ) { - m_resultCapture.handleExpr( m_assertionInfo, expr, m_reaction ); - } - void AssertionHandler::handleMessage(ResultWas::OfType resultType, std::string&& message) { - m_resultCapture.handleMessage( m_assertionInfo, resultType, CATCH_MOVE(message), m_reaction ); - } +} // end namespace Catch - auto AssertionHandler::allowThrows() const -> bool { - return getCurrentContext().getConfig()->allowThrows(); - } - void AssertionHandler::complete() { - m_completed = true; - if( m_reaction.shouldDebugBreak ) { - // If you find your debugger stopping you here then go one level up on the - // call-stack for the code that caused it (typically a failed assertion) +namespace Catch { - // (To go back to the test and change execution, jump over the throw, next) - CATCH_BREAK_INTO_DEBUGGER(); - } - if (m_reaction.shouldThrow) { - throw_test_failure_exception(); - } - if ( m_reaction.shouldSkip ) { - throw_test_skip_exception(); - } + Counts Counts::operator - ( Counts const& other ) const { + Counts diff; + diff.passed = passed - other.passed; + diff.failed = failed - other.failed; + diff.failedButOk = failedButOk - other.failedButOk; + diff.skipped = skipped - other.skipped; + return diff; } - void AssertionHandler::handleUnexpectedInflightException() noexcept { - m_resultCapture.handleUnexpectedInflightException( m_assertionInfo, Catch::translateActiveException(), m_reaction ); + Counts& Counts::operator += ( Counts const& other ) { + passed += other.passed; + failed += other.failed; + failedButOk += other.failedButOk; + skipped += other.skipped; + return *this; } - void AssertionHandler::handleExceptionThrownAsExpected() { - m_resultCapture.handleNonExpr(m_assertionInfo, ResultWas::Ok, m_reaction); + std::uint64_t Counts::total() const { + return passed + failed + failedButOk + skipped; } - void AssertionHandler::handleExceptionNotThrownAsExpected() { - m_resultCapture.handleNonExpr(m_assertionInfo, ResultWas::Ok, m_reaction); + bool Counts::allPassed() const { + return failed == 0 && failedButOk == 0 && skipped == 0; + } + bool Counts::allOk() const { + return failed == 0; } - void AssertionHandler::handleUnexpectedExceptionNotThrown() { - m_resultCapture.handleUnexpectedExceptionNotThrown( m_assertionInfo, m_reaction ); + Totals Totals::operator - ( Totals const& other ) const { + Totals diff; + diff.assertions = assertions - other.assertions; + diff.testCases = testCases - other.testCases; + return diff; } - void AssertionHandler::handleThrowingCallSkipped() { - m_resultCapture.handleNonExpr(m_assertionInfo, ResultWas::Ok, m_reaction); + Totals& Totals::operator += ( Totals const& other ) { + assertions += other.assertions; + testCases += other.testCases; + return *this; } - // This is the overload that takes a string and infers the Equals matcher from it - // The more general overload, that takes any string matcher, is in catch_capture_matchers.cpp - void handleExceptionMatchExpr( AssertionHandler& handler, std::string const& str ) { - handleExceptionMatchExpr( handler, Matchers::Equals( str ) ); + Totals Totals::delta( Totals const& prevTotals ) const { + Totals diff = *this - prevTotals; + if( diff.assertions.failed > 0 ) + ++diff.testCases.failed; + else if( diff.assertions.failedButOk > 0 ) + ++diff.testCases.failedButOk; + else if ( diff.assertions.skipped > 0 ) + ++ diff.testCases.skipped; + else + ++diff.testCases.passed; + return diff; } -} // namespace Catch - +} -#include <algorithm> namespace Catch { namespace Detail { - - bool CaseInsensitiveLess::operator()( StringRef lhs, - StringRef rhs ) const { - return std::lexicographical_compare( - lhs.begin(), lhs.end(), - rhs.begin(), rhs.end(), - []( char l, char r ) { return toLower( l ) < toLower( r ); } ); - } - - bool - CaseInsensitiveEqualTo::operator()( StringRef lhs, - StringRef rhs ) const { - return std::equal( - lhs.begin(), lhs.end(), - rhs.begin(), rhs.end(), - []( char l, char r ) { return toLower( l ) == toLower( r ); } ); + void registerTranslatorImpl( + Detail::unique_ptr<IExceptionTranslator>&& translator ) { + getMutableRegistryHub().registerTranslator( + CATCH_MOVE( translator ) ); } - } // namespace Detail } // namespace Catch - - -#include <algorithm> #include <ostream> -namespace { - bool isOptPrefix( char c ) { - return c == '-' -#ifdef CATCH_PLATFORM_WINDOWS - || c == '/' -#endif - ; - } +namespace Catch { - Catch::StringRef normaliseOpt( Catch::StringRef optName ) { - if ( optName[0] == '-' -#if defined(CATCH_PLATFORM_WINDOWS) - || optName[0] == '/' -#endif - ) { - return optName.substr( 1, optName.size() ); + Version::Version + ( unsigned int _majorVersion, + unsigned int _minorVersion, + unsigned int _patchNumber, + char const * const _branchName, + unsigned int _buildNumber ) + : majorVersion( _majorVersion ), + minorVersion( _minorVersion ), + patchNumber( _patchNumber ), + branchName( _branchName ), + buildNumber( _buildNumber ) + {} + + std::ostream& operator << ( std::ostream& os, Version const& version ) { + os << version.majorVersion << '.' + << version.minorVersion << '.' + << version.patchNumber; + // branchName is never null -> 0th char is \0 if it is empty + if (version.branchName[0]) { + os << '-' << version.branchName + << '.' << version.buildNumber; } + return os; + } - return optName; + Version const& libraryVersion() { + static Version version( 3, 16, 1, "", 0 ); + return version; } - static size_t find_first_separator(Catch::StringRef sr) { - auto is_separator = []( char c ) { - return c == ' ' || c == ':' || c == '='; - }; - size_t pos = 0; - while (pos < sr.size()) { - if (is_separator(sr[pos])) { return pos; } - ++pos; - } +} + - return Catch::StringRef::npos; - } -} // namespace namespace Catch { - namespace Clara { - namespace Detail { - void TokenStream::loadBuffer() { - m_tokenBuffer.clear(); - - // Skip any empty strings - while ( it != itEnd && it->empty() ) { - ++it; - } - - if ( it != itEnd ) { - StringRef next = *it; - if ( isOptPrefix( next[0] ) ) { - auto delimiterPos = find_first_separator(next); - if ( delimiterPos != StringRef::npos ) { - m_tokenBuffer.push_back( - { TokenType::Option, - next.substr( 0, delimiterPos ) } ); - m_tokenBuffer.push_back( - { TokenType::Argument, - next.substr( delimiterPos + 1, next.size() ) } ); - } else { - if ( next.size() > 1 && next[1] != '-' && next.size() > 2 ) { - // Combined short args, e.g. "-ab" for "-a -b" - for ( size_t i = 1; i < next.size(); ++i ) { - m_tokenBuffer.push_back( - { TokenType::Option, - next.substr( i, 1 ) } ); - } - } else { - m_tokenBuffer.push_back( - { TokenType::Option, next } ); - } - } - } else { - m_tokenBuffer.push_back( - { TokenType::Argument, next } ); - } - } - } + const char* GeneratorException::what() const noexcept { + return m_msg; + } - TokenStream::TokenStream( Args const& args ): - TokenStream( args.m_args.begin(), args.m_args.end() ) {} +} // end namespace Catch - TokenStream::TokenStream( Iterator it_, Iterator itEnd_ ): - it( it_ ), itEnd( itEnd_ ) { - loadBuffer(); - } - TokenStream& TokenStream::operator++() { - if ( m_tokenBuffer.size() >= 2 ) { - m_tokenBuffer.erase( m_tokenBuffer.begin() ); - } else { - if ( it != itEnd ) - ++it; - loadBuffer(); - } - return *this; - } - ParserResult convertInto( std::string const& source, - std::string& target ) { - target = source; - return ParserResult::ok( ParseResultType::Matched ); - } - ParserResult convertInto( std::string const& source, - bool& target ) { - std::string srcLC = toLower( source ); +namespace Catch { - if ( srcLC == "y" || srcLC == "1" || srcLC == "true" || - srcLC == "yes" || srcLC == "on" ) { - target = true; - } else if ( srcLC == "n" || srcLC == "0" || srcLC == "false" || - srcLC == "no" || srcLC == "off" ) { - target = false; - } else { - return ParserResult::runtimeError( - "Expected a boolean value but did not recognise: '" + - source + '\'' ); - } - return ParserResult::ok( ParseResultType::Matched ); - } + IGeneratorTracker::~IGeneratorTracker() = default; - size_t ParserBase::cardinality() const { return 1; } +namespace Generators { - InternalParseResult ParserBase::parse( Args const& args ) const { - return parse( static_cast<std::string>(args.exeName()), TokenStream( args ) ); - } + GeneratorUntypedBase::~GeneratorUntypedBase() = default; - ParseState::ParseState( ParseResultType type, - TokenStream remainingTokens ): - m_type( type ), m_remainingTokens( CATCH_MOVE(remainingTokens) ) {} + IGeneratorTracker* acquireGeneratorTracker(StringRef generatorName, SourceLineInfo const& lineInfo ) { + return getResultCapture().acquireGeneratorTracker( generatorName, lineInfo ); + } - ParserResult BoundFlagRef::setFlag( bool flag ) { - m_ref = flag; - return ParserResult::ok( ParseResultType::Matched ); - } + IGeneratorTracker* createGeneratorTracker( StringRef generatorName, + SourceLineInfo lineInfo, + GeneratorBasePtr&& generator ) { + return getResultCapture().createGeneratorTracker( + generatorName, lineInfo, CATCH_MOVE( generator ) ); + } - ResultBase::~ResultBase() = default; +} // namespace Generators +} // namespace Catch - bool BoundRef::isContainer() const { return false; } - bool BoundRef::isFlag() const { return false; } - bool BoundFlagRefBase::isFlag() const { return true; } -} // namespace Detail +#include <random> - Detail::InternalParseResult Arg::parse(std::string const&, - Detail::TokenStream tokens) const { - auto validationResult = validate(); - if (!validationResult) - return Detail::InternalParseResult(validationResult); +namespace Catch { + namespace Generators { + namespace Detail { + std::uint32_t getSeed() { return sharedRng()(); } + } // namespace Detail - auto token = *tokens; - if (token.type != Detail::TokenType::Argument) - return Detail::InternalParseResult::ok(Detail::ParseState( - ParseResultType::NoMatch, CATCH_MOVE(tokens))); + struct RandomFloatingGenerator<long double>::PImpl { + PImpl( long double a, long double b, uint32_t seed ): + rng( seed ), dist( a, b ) {} - assert(!m_ref->isFlag()); - auto valueRef = - static_cast<Detail::BoundValueRefBase*>(m_ref.get()); + Catch::SimplePcg32 rng; + std::uniform_real_distribution<long double> dist; + }; - auto result = valueRef->setValue(static_cast<std::string>(token.token)); - if ( !result ) - return Detail::InternalParseResult( result ); - else - return Detail::InternalParseResult::ok( - Detail::ParseState( ParseResultType::Matched, - CATCH_MOVE( ++tokens ) ) ); + RandomFloatingGenerator<long double>::RandomFloatingGenerator( + long double a, long double b, std::uint32_t seed) : + m_pimpl(Catch::Detail::make_unique<PImpl>(a, b, seed)) { + static_cast<void>( next() ); } - Opt::Opt(bool& ref) : - ParserRefImpl(std::make_shared<Detail::BoundFlagRef>(ref)) {} - - Detail::HelpColumns Opt::getHelpColumns() const { - ReusableStringStream oss; - bool first = true; - for (auto const& opt : m_optNames) { - if (first) - first = false; - else - oss << ", "; - oss << opt; - } - if (!m_hint.empty()) - oss << " <" << m_hint << '>'; - return { oss.str(), m_description }; + RandomFloatingGenerator<long double>::~RandomFloatingGenerator() = + default; + bool RandomFloatingGenerator<long double>::next() { + m_current_number = m_pimpl->dist( m_pimpl->rng ); + return true; } - bool Opt::isMatch(StringRef optToken) const { - auto normalisedToken = normaliseOpt(optToken); - for (auto const& name : m_optNames) { - if (normaliseOpt(name) == normalisedToken) - return true; - } + bool RandomFloatingGenerator<long double>::isFinite() const { return false; } - Detail::InternalParseResult Opt::parse(std::string const&, - Detail::TokenStream tokens) const { - auto validationResult = validate(); - if (!validationResult) - return Detail::InternalParseResult(validationResult); - - if (tokens && - tokens->type == Detail::TokenType::Option) { - auto const& token = *tokens; - if (isMatch(token.token)) { - if (m_ref->isFlag()) { - auto flagRef = - static_cast<Detail::BoundFlagRefBase*>( - m_ref.get()); - auto result = flagRef->setFlag(true); - if (!result) - return Detail::InternalParseResult(result); - if (result.value() == - ParseResultType::ShortCircuitAll) - return Detail::InternalParseResult::ok(Detail::ParseState( - result.value(), CATCH_MOVE(tokens))); - } else { - auto valueRef = - static_cast<Detail::BoundValueRefBase*>( - m_ref.get()); - ++tokens; - if (!tokens) - return Detail::InternalParseResult::runtimeError( - "Expected argument following " + - token.token); - auto const& argToken = *tokens; - if (argToken.type != Detail::TokenType::Argument) - return Detail::InternalParseResult::runtimeError( - "Expected argument following " + - token.token); - const auto result = valueRef->setValue(static_cast<std::string>(argToken.token)); - if (!result) - return Detail::InternalParseResult(result); - if (result.value() == - ParseResultType::ShortCircuitAll) - return Detail::InternalParseResult::ok(Detail::ParseState( - result.value(), CATCH_MOVE(tokens))); - } - return Detail::InternalParseResult::ok(Detail::ParseState( - ParseResultType::Matched, CATCH_MOVE(++tokens))); - } - } - return Detail::InternalParseResult::ok( - Detail::ParseState(ParseResultType::NoMatch, CATCH_MOVE(tokens))); - } - - Detail::Result Opt::validate() const { - if (m_optNames.empty()) - return Detail::Result::logicError("No options supplied to Opt"); - for (auto const& name : m_optNames) { - if (name.empty()) - return Detail::Result::logicError( - "Option name cannot be empty"); -#ifdef CATCH_PLATFORM_WINDOWS - if (name[0] != '-' && name[0] != '/') - return Detail::Result::logicError( - "Option name must begin with '-' or '/'"); -#else - if (name[0] != '-') - return Detail::Result::logicError( - "Option name must begin with '-'"); -#endif + } // namespace Generators +} // namespace Catch + + + + +namespace Catch { + namespace Generators { + namespace Detail { + + [[noreturn]] + void throw_generator_exception( char const* msg ) { + Catch::throw_exception( GeneratorException{ msg } ); } - return ParserRefImpl::validate(); - } - ExeName::ExeName() : - m_name(std::make_shared<std::string>("<executable>")) {} + } // namespace Detail + } // namespace Generators +} // namespace Catch - ExeName::ExeName(std::string& ref) : ExeName() { - m_ref = std::make_shared<Detail::BoundValueRef<std::string>>(ref); - } - Detail::InternalParseResult - ExeName::parse(std::string const&, - Detail::TokenStream tokens) const { - return Detail::InternalParseResult::ok( - Detail::ParseState(ParseResultType::NoMatch, CATCH_MOVE(tokens))); - } - ParserResult ExeName::set(std::string const& newName) { - auto lastSlash = newName.find_last_of("\\/"); - auto filename = (lastSlash == std::string::npos) - ? newName - : newName.substr(lastSlash + 1); - *m_name = filename; - if (m_ref) - return m_ref->setValue(filename); - else - return ParserResult::ok(ParseResultType::Matched); +namespace Catch { + namespace Detail { + void missingCaptureInstance() { + CATCH_INTERNAL_ERROR( "No result capture instance" ); } + } // namespace Detail + + IResultCapture::~IResultCapture() = default; +} // namespace Catch - Parser& Parser::operator|=( Parser const& other ) { - m_options.insert( m_options.end(), - other.m_options.begin(), - other.m_options.end() ); - m_args.insert( - m_args.end(), other.m_args.begin(), other.m_args.end() ); - return *this; - } +namespace Catch { + IConfig::~IConfig() = default; +} + + + + +namespace Catch { + IExceptionTranslator::~IExceptionTranslator() = default; + IExceptionTranslatorRegistry::~IExceptionTranslatorRegistry() = default; +} - std::vector<Detail::HelpColumns> Parser::getHelpColumns() const { - std::vector<Detail::HelpColumns> cols; - cols.reserve( m_options.size() ); - for ( auto const& o : m_options ) { - cols.push_back(o.getHelpColumns()); - } - return cols; - } - void Parser::writeToStream( std::ostream& os ) const { - if ( !m_exeName.name().empty() ) { - os << "usage:\n" - << " " << m_exeName.name() << ' '; - bool required = true, first = true; - for ( auto const& arg : m_args ) { - if ( first ) - first = false; - else - os << ' '; - if ( arg.isOptional() && required ) { - os << '['; - required = false; - } - os << '<' << arg.hint() << '>'; - if ( arg.cardinality() == 0 ) - os << " ... "; - } - if ( !required ) - os << ']'; - if ( !m_options.empty() ) - os << " options"; - os << "\n\nwhere options are:\n"; - } - auto rows = getHelpColumns(); - size_t consoleWidth = CATCH_CONFIG_CONSOLE_WIDTH; - size_t optWidth = 0; - for ( auto const& cols : rows ) - optWidth = ( std::max )( optWidth, cols.left.size() + 2 ); - optWidth = ( std::min )( optWidth, consoleWidth / 2 ); +#include <string> + +namespace Catch { + namespace Generators { - for ( auto& cols : rows ) { - auto row = TextFlow::Column( CATCH_MOVE(cols.left) ) - .width( optWidth ) - .indent( 2 ) + - TextFlow::Spacer( 4 ) + - TextFlow::Column( static_cast<std::string>(cols.descriptions) ) - .width( consoleWidth - 7 - optWidth ); - os << row << '\n'; + bool GeneratorUntypedBase::countedNext() { + auto ret = next(); + if ( ret ) { + m_stringReprCache.clear(); + ++m_currentElementIndex; } + return ret; } - Detail::Result Parser::validate() const { - for ( auto const& opt : m_options ) { - auto result = opt.validate(); - if ( !result ) - return result; - } - for ( auto const& arg : m_args ) { - auto result = arg.validate(); - if ( !result ) - return result; + void GeneratorUntypedBase::skipToNthElementImpl( std::size_t n ) { + for ( size_t i = m_currentElementIndex; i < n; ++i ) { + bool isValid = next(); + if ( !isValid ) { + Detail::throw_generator_exception( + "Could not jump to Nth element: not enough elements" ); + } } - return Detail::Result::ok(); } - Detail::InternalParseResult - Parser::parse( std::string const& exeName, - Detail::TokenStream tokens ) const { + void GeneratorUntypedBase::skipToNthElement( std::size_t n ) { + if ( n < m_currentElementIndex ) { + Detail::throw_generator_exception( + "Tried to jump generator backwards" ); + } + if ( n == m_currentElementIndex ) { return; } - struct ParserInfo { - ParserBase const* parser = nullptr; - size_t count = 0; - }; - std::vector<ParserInfo> parseInfos; - parseInfos.reserve( m_options.size() + m_args.size() ); - for ( auto const& opt : m_options ) { - parseInfos.push_back( { &opt, 0 } ); - } - for ( auto const& arg : m_args ) { - parseInfos.push_back( { &arg, 0 } ); - } - - m_exeName.set( exeName ); - - auto result = Detail::InternalParseResult::ok( - Detail::ParseState( ParseResultType::NoMatch, CATCH_MOVE(tokens) ) ); - while ( result.value().remainingTokens() ) { - bool tokenParsed = false; - - for ( auto& parseInfo : parseInfos ) { - if ( parseInfo.parser->cardinality() == 0 || - parseInfo.count < parseInfo.parser->cardinality() ) { - result = parseInfo.parser->parse( - exeName, CATCH_MOVE(result).value().remainingTokens() ); - if ( !result ) - return result; - if ( result.value().type() != - ParseResultType::NoMatch ) { - tokenParsed = true; - ++parseInfo.count; - break; - } - } - } + skipToNthElementImpl(n); + // Fixup tracking after moving the generator forward + // * Ensure that the correct element index is set after skipping + // * Invalidate cache + m_currentElementIndex = n; + m_stringReprCache.clear(); + } - if ( result.value().type() == ParseResultType::ShortCircuitAll ) - return result; - if ( !tokenParsed ) - return Detail::InternalParseResult::runtimeError( - "Unrecognised token: " + - result.value().remainingTokens()->token ); + StringRef GeneratorUntypedBase::currentElementAsString() const { + if ( m_stringReprCache.empty() ) { + m_stringReprCache = stringifyImpl(); } - // !TBD Check missing required options - return result; + return m_stringReprCache; } - Args::Args(int argc, char const* const* argv) : - m_exeName(argv[0]), m_args(argv + 1, argv + argc) {} - - Args::Args(std::initializer_list<StringRef> args) : - m_exeName(*args.begin()), - m_args(args.begin() + 1, args.end()) {} - - - Help::Help( bool& showHelpFlag ): - Opt( [&]( bool flag ) { - showHelpFlag = flag; - return ParserResult::ok( ParseResultType::ShortCircuitAll ); - } ) { - static_cast<Opt&> ( *this )( - "display usage information" )["-?"]["-h"]["--help"] - .optional(); - } + bool GeneratorUntypedBase::isFinite() const { return true; } - } // namespace Clara + } // namespace Generators } // namespace Catch -#include <fstream> -#include <string> - namespace Catch { + IRegistryHub::~IRegistryHub() = default; + IMutableRegistryHub::~IMutableRegistryHub() = default; +} - Clara::Parser makeCommandLineParser( ConfigData& config ) { - - using namespace Clara; - auto const setWarning = [&]( std::string const& warning ) { - if ( warning == "NoAssertions" ) { - config.warnings = static_cast<WarnAbout::What>(config.warnings | WarnAbout::NoAssertions); - return ParserResult::ok( ParseResultType::Matched ); - } else if ( warning == "UnmatchedTestSpec" ) { - config.warnings = static_cast<WarnAbout::What>(config.warnings | WarnAbout::UnmatchedTestSpec); - return ParserResult::ok( ParseResultType::Matched ); - } else if ( warning == "InfiniteGenerators" ) { - config.warnings = static_cast<WarnAbout::What>(config.warnings | WarnAbout::InfiniteGenerator); - return ParserResult::ok( ParseResultType::Matched ); - } - return ParserResult ::runtimeError( - "Unrecognised warning option: '" + warning + '\'' ); - }; - auto const loadTestNamesFromFile = [&]( std::string const& filename ) { - std::ifstream f( filename.c_str() ); - if( !f.is_open() ) - return ParserResult::runtimeError( "Unable to load input file: '" + filename + '\'' ); - - std::string line; - while( std::getline( f, line ) ) { - line = trim(line); - if( !line.empty() && !startsWith( line, '#' ) ) { - if( !startsWith( line, '"' ) ) - line = '"' + CATCH_MOVE(line) + '"'; - config.testsOrTags.push_back( line ); - config.testsOrTags.emplace_back( "," ); - } - } - //Remove comma in the end - if(!config.testsOrTags.empty()) - config.testsOrTags.erase( config.testsOrTags.end()-1 ); - - return ParserResult::ok( ParseResultType::Matched ); - }; - auto const setTestOrder = [&]( std::string const& order ) { - if( startsWith( "declared", order ) ) - config.runOrder = TestRunOrder::Declared; - else if( startsWith( "lexical", order ) ) - config.runOrder = TestRunOrder::LexicographicallySorted; - else if( startsWith( "random", order ) ) - config.runOrder = TestRunOrder::Randomized; - else - return ParserResult::runtimeError( "Unrecognised ordering: '" + order + '\'' ); - return ParserResult::ok( ParseResultType::Matched ); - }; - auto const setRngSeed = [&]( std::string const& seed ) { - if( seed == "time" ) { - config.rngSeed = generateRandomSeed(GenerateFrom::Time); - config.rngSeedWasFixed = false; - return ParserResult::ok(ParseResultType::Matched); - } else if (seed == "random-device") { - config.rngSeed = generateRandomSeed(GenerateFrom::RandomDevice); - config.rngSeedWasFixed = false; - return ParserResult::ok(ParseResultType::Matched); - } - - // TODO: ideally we should be parsing uint32_t directly - // fix this later when we add new parse overload - auto parsedSeed = parseUInt( seed, 0 ); - if ( !parsedSeed ) { - return ParserResult::runtimeError( "Could not parse '" + seed + "' as seed" ); - } - config.rngSeed = *parsedSeed; - config.rngSeedWasFixed = true; - return ParserResult::ok( ParseResultType::Matched ); - }; - auto const setDefaultColourMode = [&]( std::string const& colourMode ) { - Optional<ColourMode> maybeMode = Catch::Detail::stringToColourMode(toLower( colourMode )); - if ( !maybeMode ) { - return ParserResult::runtimeError( - "colour mode must be one of: default, ansi, win32, " - "or none. '" + - colourMode + "' is not recognised" ); - } - auto mode = *maybeMode; - if ( !isColourImplAvailable( mode ) ) { - return ParserResult::runtimeError( - "colour mode '" + colourMode + - "' is not supported in this binary" ); - } - config.defaultColourMode = mode; - return ParserResult::ok( ParseResultType::Matched ); - }; - auto const setWaitForKeypress = [&]( std::string const& keypress ) { - auto keypressLc = toLower( keypress ); - if (keypressLc == "never") - config.waitForKeypress = WaitForKeypress::Never; - else if( keypressLc == "start" ) - config.waitForKeypress = WaitForKeypress::BeforeStart; - else if( keypressLc == "exit" ) - config.waitForKeypress = WaitForKeypress::BeforeExit; - else if( keypressLc == "both" ) - config.waitForKeypress = WaitForKeypress::BeforeStartAndExit; - else - return ParserResult::runtimeError( "keypress argument must be one of: never, start, exit or both. '" + keypress + "' not recognised" ); - return ParserResult::ok( ParseResultType::Matched ); - }; - auto const setVerbosity = [&]( std::string const& verbosity ) { - auto lcVerbosity = toLower( verbosity ); - if( lcVerbosity == "quiet" ) - config.verbosity = Verbosity::Quiet; - else if( lcVerbosity == "normal" ) - config.verbosity = Verbosity::Normal; - else if( lcVerbosity == "high" ) - config.verbosity = Verbosity::High; - else - return ParserResult::runtimeError( "Unrecognised verbosity, '" + verbosity + '\'' ); - return ParserResult::ok( ParseResultType::Matched ); - }; - auto const setReporter = [&]( std::string const& userReporterSpec ) { - if ( userReporterSpec.empty() ) { - return ParserResult::runtimeError( "Received empty reporter spec." ); - } +#include <cassert> - Optional<ReporterSpec> parsed = - parseReporterSpec( userReporterSpec ); - if ( !parsed ) { - return ParserResult::runtimeError( - "Could not parse reporter spec '" + userReporterSpec + - "'" ); - } +namespace Catch { - auto const& reporterSpec = *parsed; + AssertionStats::AssertionStats( AssertionResult const& _assertionResult, + std::vector<MessageInfo> const& _infoMessages, + Totals const& _totals ) + : assertionResult( _assertionResult ), + infoMessages( _infoMessages ), + totals( _totals ) + { + if( assertionResult.hasMessage() ) { + // Copy message into messages list. + // !TBD This should have been done earlier, somewhere + MessageBuilder builder( assertionResult.getTestMacroName(), assertionResult.getSourceInfo(), assertionResult.getResultType() ); + builder.m_info.message = static_cast<std::string>(assertionResult.getMessage()); - auto const& factories = - getRegistryHub().getReporterRegistry().getFactories(); - auto result = factories.find( reporterSpec.name() ); + infoMessages.push_back( CATCH_MOVE(builder.m_info) ); + } + } - if ( result == factories.end() ) { - return ParserResult::runtimeError( - "Unrecognized reporter, '" + reporterSpec.name() + - "'. Check available with --list-reporters" ); - } + SectionStats::SectionStats( SectionInfo&& _sectionInfo, + Counts const& _assertions, + double _durationInSeconds, + bool _missingAssertions ) + : sectionInfo( CATCH_MOVE(_sectionInfo) ), + assertions( _assertions ), + durationInSeconds( _durationInSeconds ), + missingAssertions( _missingAssertions ) + {} - const bool hadOutputFile = reporterSpec.outputFile().some(); - config.reporterSpecifications.push_back( CATCH_MOVE( *parsed ) ); - // It would be enough to check this only once at the very end, but - // there is not a place where we could call this check, so do it - // every time it could fail. For valid inputs, this is still called - // at most once. - if (!hadOutputFile) { - int n_reporters_without_file = 0; - for (auto const& spec : config.reporterSpecifications) { - if (spec.outputFile().none()) { - n_reporters_without_file++; - } - } - if (n_reporters_without_file > 1) { - return ParserResult::runtimeError( "Only one reporter may have unspecified output file." ); - } - } + TestCaseStats::TestCaseStats( TestCaseInfo const& _testInfo, + Totals const& _totals, + std::string&& _stdOut, + std::string&& _stdErr, + bool _aborting ) + : testInfo( &_testInfo ), + totals( _totals ), + stdOut( CATCH_MOVE(_stdOut) ), + stdErr( CATCH_MOVE(_stdErr) ), + aborting( _aborting ) + {} - return ParserResult::ok( ParseResultType::Matched ); - }; - auto const setShardCount = [&]( std::string const& shardCount ) { - auto parsedCount = parseUInt( shardCount ); - if ( !parsedCount ) { - return ParserResult::runtimeError( - "Could not parse '" + shardCount + "' as shard count" ); - } - if ( *parsedCount == 0 ) { - return ParserResult::runtimeError( - "Shard count must be positive" ); - } - config.shardCount = *parsedCount; - return ParserResult::ok( ParseResultType::Matched ); - }; - auto const setBenchmarkSamples = [&]( std::string const& samples ) { - auto parsedSamples = parseUInt( samples ); - if ( !parsedSamples ) { - return ParserResult::runtimeError( - "Could not parse '" + samples + "' as benchmark samples" ); - } - if ( *parsedSamples == 0 ) { - return ParserResult::runtimeError( - "Benchmark samples must be greater than 0" ); - } - config.benchmarkSamples = *parsedSamples; - return ParserResult::ok( ParseResultType::Matched ); - }; - auto const setShardIndex = [&](std::string const& shardIndex) { - auto parsedIndex = parseUInt( shardIndex ); - if ( !parsedIndex ) { - return ParserResult::runtimeError( - "Could not parse '" + shardIndex + "' as shard index" ); - } - config.shardIndex = *parsedIndex; - return ParserResult::ok( ParseResultType::Matched ); - }; + TestRunStats::TestRunStats( TestRunInfo const& _runInfo, + Totals const& _totals, + bool _aborting ) + : runInfo( _runInfo ), + totals( _totals ), + aborting( _aborting ) + {} - auto const setSectionFilter = [&]( std::string const& sectionFilter ) { - config.pathFilters.emplace_back( PathFilter::For::Section, trim(sectionFilter) ); - return ParserResult::ok( ParseResultType::Matched ); - }; - auto const setGeneratorFilter = [&]( std::string const& generatorFilter ) { - if (generatorFilter != "*") { - // TODO: avoid re-parsing the index? - auto parsedIndex = parseUInt( generatorFilter ); - if ( !parsedIndex ) { - return ParserResult::runtimeError( "Could not parse '" + - generatorFilter + - "' as generator index" ); - } - } - config.useNewPathFilteringBehaviour = true; - config.pathFilters.emplace_back( PathFilter::For::Generator, trim(generatorFilter) ); - return ParserResult::ok( ParseResultType::Matched ); - }; - // Copy-capturing other `setFoo` functions enables calling them later, - // as the config ref remains valid, but the local lambda vars won't. - auto const setPathFilter = [=, &config]( std::string const& pathFilter ) { - config.useNewPathFilteringBehaviour = true; - if ( pathFilter.size() < 3 ) { - return ParserResult::runtimeError( - "Path filter '" + pathFilter + "' is too short" ); - } - if ( startsWith( pathFilter, "g:" ) ) { - return setGeneratorFilter( pathFilter.substr( 2 ) ); - } - if ( startsWith( pathFilter, "c:" ) ) { - return setSectionFilter( pathFilter.substr( 2 ) ); - } - return ParserResult::runtimeError( "Path filter '" + pathFilter + - "' has unknown type prefix" ); - }; + IEventListener::~IEventListener() = default; +} // end namespace Catch - auto cli - = ExeName( config.processName ) - | Help( config.showHelp ) - | Opt( config.showSuccessfulTests ) - ["-s"]["--success"] - ( "include successful tests in output" ) - | Opt( config.shouldDebugBreak ) - ["-b"]["--break"] - ( "break into debugger on failure" ) - | Opt( config.noThrow ) - ["-e"]["--nothrow"] - ( "skip exception tests" ) - | Opt( config.showInvisibles ) - ["-i"]["--invisibles"] - ( "show invisibles (tabs, newlines)" ) - | Opt( config.defaultOutputFilename, "filename" ) - ["-o"]["--out"] - ( "default output filename" ) - | Opt( accept_many, setReporter, "name[::key=value]*" ) - ["-r"]["--reporter"] - ( "reporter to use (defaults to console)" ) - | Opt( config.name, "name" ) - ["-n"]["--name"] - ( "suite name" ) - | Opt( [&]( bool ){ config.abortAfter = 1; } ) - ["-a"]["--abort"] - ( "abort at first failure" ) - | Opt( [&]( int x ){ config.abortAfter = x; }, "no. failures" ) - ["-x"]["--abortx"] - ( "abort after x failures" ) - | Opt( accept_many, setWarning, "warning name" ) - ["-w"]["--warn"] - ( "enable warnings" ) - | Opt( [&]( bool flag ) { config.showDurations = flag ? ShowDurations::Always : ShowDurations::Never; }, "yes|no" ) - ["-d"]["--durations"] - ( "show test durations" ) - | Opt( config.minDuration, "seconds" ) - ["-D"]["--min-duration"] - ( "show test durations for tests taking at least the given number of seconds" ) - | Opt( loadTestNamesFromFile, "filename" ) - ["-f"]["--input-file"] - ( "load test names to run from a file" ) - | Opt( config.filenamesAsTags ) - ["-#"]["--filenames-as-tags"] - ( "adds a tag for the filename" ) - | Opt( accept_many, setSectionFilter, "section name" ) - ["-c"]["--section"] - ( "specify section to run" ) - | Opt( accept_many, setGeneratorFilter, "index spec" ) - ["-g"]["--generator-index"] - ( "specify generator elements to try" ) - | Opt( accept_many, setPathFilter, "path filter spec" ) - ["-p"]["--path-filter"] - ( "qualified path filter" ) - | Opt( setVerbosity, "quiet|normal|high" ) - ["-v"]["--verbosity"] - ( "set output verbosity" ) - | Opt( config.listTests ) - ["--list-tests"] - ( "list all/matching test cases" ) - | Opt( config.listTags ) - ["--list-tags"] - ( "list all/matching tags" ) - | Opt( config.listReporters ) - ["--list-reporters"] - ( "list all available reporters" ) - | Opt( config.listListeners ) - ["--list-listeners"] - ( "list all listeners" ) - | Opt( setTestOrder, "decl|lex|rand" ) - ["--order"] - ( "test case order (defaults to rand)" ) - | Opt( setRngSeed, "'time'|'random-device'|number" ) - ["--rng-seed"] - ( "set a specific seed for random numbers" ) - | Opt( setDefaultColourMode, "ansi|win32|none|default" ) - ["--colour-mode"] - ( "what color mode should be used as default" ) - | Opt( config.libIdentify ) - ["--libidentify"] - ( "report name and version according to libidentify standard" ) - | Opt( setWaitForKeypress, "never|start|exit|both" ) - ["--wait-for-keypress"] - ( "waits for a keypress before exiting" ) - | Opt( config.skipBenchmarks) - ["--skip-benchmarks"] - ( "disable running benchmarks") - | Opt( setBenchmarkSamples, "samples" ) - ["--benchmark-samples"] - ( "number of samples to collect (default: 100)" ) - | Opt( config.benchmarkResamples, "resamples" ) - ["--benchmark-resamples"] - ( "number of resamples for the bootstrap (default: 100000)" ) - | Opt( config.benchmarkConfidenceInterval, "confidence interval" ) - ["--benchmark-confidence-interval"] - ( "confidence interval for the bootstrap (between 0 and 1, default: 0.95)" ) - | Opt( config.benchmarkNoAnalysis ) - ["--benchmark-no-analysis"] - ( "perform only measurements; do not perform any analysis" ) - | Opt( config.benchmarkWarmupTime, "benchmarkWarmupTime" ) - ["--benchmark-warmup-time"] - ( "amount of time in milliseconds spent on warming up each test (default: 100)" ) - | Opt( setShardCount, "shard count" ) - ["--shard-count"] - ( "split the tests to execute into this many groups" ) - | Opt( setShardIndex, "shard index" ) - ["--shard-index"] - ( "index of the group of tests to execute (see --shard-count)" ) - | Opt( config.allowZeroTests ) - ["--allow-running-no-tests"] - ( "Treat 'No tests run' as a success" ) - | Opt( config.prematureExitGuardFilePath, "path" ) - ["--premature-exit-guard-file"] - ( "create a file before running tests and delete it during clean exit" ) - | Arg( config.testsOrTags, "test name|pattern|tags" ) - ( "which test or tests to use" ); - - return cli; - } -} // end namespace Catch -#if defined(__clang__) -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wexit-time-destructors" -#endif +namespace Catch { + EventListenerFactory::~EventListenerFactory() = default; +} -#include <cassert> -#include <ostream> -#include <utility> namespace Catch { + ITestCaseRegistry::~ITestCaseRegistry() = default; +} - ColourImpl::~ColourImpl() = default; - ColourImpl::ColourGuard ColourImpl::guardColour( Colour::Code colourCode ) { - return ColourGuard(colourCode, this ); - } - void ColourImpl::ColourGuard::engageImpl( std::ostream& stream ) { - assert( &stream == &m_colourImpl->m_stream->stream() && - "Engaging colour guard for different stream than used by the " - "parent colour implementation" ); - static_cast<void>( stream ); +namespace Catch { - m_engaged = true; - m_colourImpl->use( m_code ); + void AssertionHandler::finishIncomplete() { + m_resultCapture.handleIncomplete( m_assertionInfo ); } - ColourImpl::ColourGuard::ColourGuard( Colour::Code code, - ColourImpl const* colour ): - m_colourImpl( colour ), m_code( code ) { - } - ColourImpl::ColourGuard::ColourGuard( ColourGuard&& rhs ) noexcept: - m_colourImpl( rhs.m_colourImpl ), - m_code( rhs.m_code ), - m_engaged( rhs.m_engaged ) { - rhs.m_engaged = false; + AssertionHandler::AssertionHandler + ( StringRef macroName, + SourceLineInfo const& lineInfo, + StringRef capturedExpression, + ResultDisposition::Flags resultDisposition ) + : m_assertionInfo{ macroName, lineInfo, capturedExpression, resultDisposition }, + m_resultCapture( static_cast<RunContext&>(getResultCapture()) ) + { + m_resultCapture.notifyAssertionStarted( m_assertionInfo ); } - ColourImpl::ColourGuard& - ColourImpl::ColourGuard::operator=( ColourGuard&& rhs ) noexcept { - using std::swap; - swap( m_colourImpl, rhs.m_colourImpl ); - swap( m_code, rhs.m_code ); - swap( m_engaged, rhs.m_engaged ); - return *this; - } - ColourImpl::ColourGuard::~ColourGuard() { - if ( m_engaged ) { - m_colourImpl->use( Colour::None ); - } + void AssertionHandler::handleExpr( ITransientExpression const& expr ) { + m_resultCapture.handleExpr( m_assertionInfo, expr, m_reaction ); } - - ColourImpl::ColourGuard& - ColourImpl::ColourGuard::engage( std::ostream& stream ) & { - engageImpl( stream ); - return *this; + void AssertionHandler::handleMessage(ResultWas::OfType resultType, std::string&& message) { + m_resultCapture.handleMessage( m_assertionInfo, resultType, CATCH_MOVE(message), m_reaction ); } - ColourImpl::ColourGuard&& - ColourImpl::ColourGuard::engage( std::ostream& stream ) && { - engageImpl( stream ); - return CATCH_MOVE(*this); + auto AssertionHandler::allowThrows() const -> bool { + return getCurrentContext().getConfig()->allowThrows(); } - namespace { - //! A do-nothing implementation of colour, used as fallback for unknown - //! platforms, and when the user asks to deactivate all colours. - class NoColourImpl final : public ColourImpl { - public: - NoColourImpl( IStream* stream ): ColourImpl( stream ) {} - - private: - void use( Colour::Code ) const override {} - }; - } // namespace - - -} // namespace Catch - - -#if defined ( CATCH_CONFIG_COLOUR_WIN32 ) ///////////////////////////////////////// + void AssertionHandler::complete() { + m_completed = true; + if( m_reaction.shouldDebugBreak ) { -namespace Catch { -namespace { + // If you find your debugger stopping you here then go one level up on the + // call-stack for the code that caused it (typically a failed assertion) - class Win32ColourImpl final : public ColourImpl { - public: - Win32ColourImpl(IStream* stream): - ColourImpl(stream) { - CONSOLE_SCREEN_BUFFER_INFO csbiInfo; - GetConsoleScreenBufferInfo( GetStdHandle( STD_OUTPUT_HANDLE ), - &csbiInfo ); - originalForegroundAttributes = csbiInfo.wAttributes & ~( BACKGROUND_GREEN | BACKGROUND_RED | BACKGROUND_BLUE | BACKGROUND_INTENSITY ); - originalBackgroundAttributes = csbiInfo.wAttributes & ~( FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE | FOREGROUND_INTENSITY ); + // (To go back to the test and change execution, jump over the throw, next) + CATCH_BREAK_INTO_DEBUGGER(); } - - static bool useImplementationForStream(IStream const& stream) { - // Win32 text colour APIs can only be used on console streams - // We cannot check that the output hasn't been redirected, - // so we just check that the original stream is console stream. - return stream.isConsole(); + if (m_reaction.shouldThrow) { + throw_test_failure_exception(); } - - private: - void use( Colour::Code _colourCode ) const override { - switch( _colourCode ) { - case Colour::None: return setTextAttribute( originalForegroundAttributes ); - case Colour::White: return setTextAttribute( FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE ); - case Colour::Red: return setTextAttribute( FOREGROUND_RED ); - case Colour::Green: return setTextAttribute( FOREGROUND_GREEN ); - case Colour::Blue: return setTextAttribute( FOREGROUND_BLUE ); - case Colour::Cyan: return setTextAttribute( FOREGROUND_BLUE | FOREGROUND_GREEN ); - case Colour::Yellow: return setTextAttribute( FOREGROUND_RED | FOREGROUND_GREEN ); - case Colour::Grey: return setTextAttribute( 0 ); - - case Colour::LightGrey: return setTextAttribute( FOREGROUND_INTENSITY ); - case Colour::BrightRed: return setTextAttribute( FOREGROUND_INTENSITY | FOREGROUND_RED ); - case Colour::BrightGreen: return setTextAttribute( FOREGROUND_INTENSITY | FOREGROUND_GREEN ); - case Colour::BrightWhite: return setTextAttribute( FOREGROUND_INTENSITY | FOREGROUND_GREEN | FOREGROUND_RED | FOREGROUND_BLUE ); - case Colour::BrightYellow: return setTextAttribute( FOREGROUND_INTENSITY | FOREGROUND_RED | FOREGROUND_GREEN ); - - case Colour::Bright: CATCH_INTERNAL_ERROR( "not a colour" ); - - default: - CATCH_ERROR( "Unknown colour requested" ); - } + if ( m_reaction.shouldSkip ) { + throw_test_skip_exception(); } + } - void setTextAttribute( WORD _textAttribute ) const { - SetConsoleTextAttribute( GetStdHandle( STD_OUTPUT_HANDLE ), - _textAttribute | - originalBackgroundAttributes ); - } - WORD originalForegroundAttributes; - WORD originalBackgroundAttributes; - }; + void AssertionHandler::handleUnexpectedInflightException() noexcept { + m_resultCapture.handleUnexpectedInflightException( m_assertionInfo, Catch::translateActiveException(), m_reaction ); + } -} // end anon namespace -} // end namespace Catch + void AssertionHandler::handleExceptionThrownAsExpected() { + m_resultCapture.handleNonExpr(m_assertionInfo, ResultWas::Ok, m_reaction); + } + void AssertionHandler::handleExceptionNotThrownAsExpected() { + m_resultCapture.handleNonExpr(m_assertionInfo, ResultWas::Ok, m_reaction); + } -#endif // Windows/ ANSI/ None + void AssertionHandler::handleUnexpectedExceptionNotThrown() { + m_resultCapture.handleUnexpectedExceptionNotThrown( m_assertionInfo, m_reaction ); + } + void AssertionHandler::handleThrowingCallSkipped() { + m_resultCapture.handleNonExpr(m_assertionInfo, ResultWas::Ok, m_reaction); + } -#if defined( CATCH_PLATFORM_LINUX ) \ - || defined( CATCH_PLATFORM_MAC ) \ - || defined( __GLIBC__ ) \ - || (defined( __FreeBSD__ ) \ - /* PlayStation platform does not have `isatty()` */ \ - && !defined(CATCH_PLATFORM_PLAYSTATION)) \ - || defined( CATCH_PLATFORM_QNX ) -# define CATCH_INTERNAL_HAS_ISATTY -# include <unistd.h> -#endif + // This is the overload that takes a string and infers the Equals matcher from it + // The more general overload, that takes any string matcher, is in catch_capture_matchers.cpp + void handleExceptionMatchExpr( AssertionHandler& handler, std::string const& str ) { + handleExceptionMatchExpr( handler, Matchers::Equals( str ) ); + } -namespace Catch { -namespace { +} // namespace Catch - class ANSIColourImpl final : public ColourImpl { - public: - ANSIColourImpl( IStream* stream ): ColourImpl( stream ) {} - - static bool useImplementationForStream(IStream const& stream) { - // This is kinda messy due to trying to support a bunch of - // different platforms at once. - // The basic idea is that if we are asked to do autodetection (as - // opposed to being told to use posixy colours outright), then we - // only want to use the colours if we are writing to console. - // However, console might be redirected, so we make an attempt at - // checking for that on platforms where we know how to do that. - bool useColour = stream.isConsole(); -#if defined( CATCH_INTERNAL_HAS_ISATTY ) && \ - !( defined( __DJGPP__ ) && defined( __STRICT_ANSI__ ) ) - ErrnoGuard _; // for isatty - useColour = useColour && isatty( STDOUT_FILENO ); -# endif -# if defined( CATCH_PLATFORM_MAC ) || defined( CATCH_PLATFORM_IPHONE ) - useColour = useColour && !isDebuggerActive(); -# endif - return useColour; - } - private: - void use( Colour::Code _colourCode ) const override { - auto setColour = [&out = - m_stream->stream()]( char const* escapeCode ) { - // The escape sequence must be flushed to console, otherwise - // if stdin and stderr are intermixed, we'd get accidentally - // coloured output. - out << '\033' << escapeCode << std::flush; - }; - switch( _colourCode ) { - case Colour::None: - case Colour::White: return setColour( "[0m" ); - case Colour::Red: return setColour( "[0;31m" ); - case Colour::Green: return setColour( "[0;32m" ); - case Colour::Blue: return setColour( "[0;34m" ); - case Colour::Cyan: return setColour( "[0;36m" ); - case Colour::Yellow: return setColour( "[0;33m" ); - case Colour::Grey: return setColour( "[1;30m" ); - - case Colour::LightGrey: return setColour( "[0;37m" ); - case Colour::BrightRed: return setColour( "[1;31m" ); - case Colour::BrightGreen: return setColour( "[1;32m" ); - case Colour::BrightWhite: return setColour( "[1;37m" ); - case Colour::BrightYellow: return setColour( "[1;33m" ); - - case Colour::Bright: CATCH_INTERNAL_ERROR( "not a colour" ); - default: CATCH_INTERNAL_ERROR( "Unknown colour requested" ); - } - } - }; -} // end anon namespace -} // end namespace Catch +#include <algorithm> namespace Catch { + namespace Detail { - Detail::unique_ptr<ColourImpl> makeColourImpl( ColourMode colourSelection, - IStream* stream ) { -#if defined( CATCH_CONFIG_COLOUR_WIN32 ) - if ( colourSelection == ColourMode::Win32 ) { - return Detail::make_unique<Win32ColourImpl>( stream ); - } -#endif - if ( colourSelection == ColourMode::ANSI ) { - return Detail::make_unique<ANSIColourImpl>( stream ); - } - if ( colourSelection == ColourMode::None ) { - return Detail::make_unique<NoColourImpl>( stream ); - } - - if ( colourSelection == ColourMode::PlatformDefault) { -#if defined( CATCH_CONFIG_COLOUR_WIN32 ) - if ( Win32ColourImpl::useImplementationForStream( *stream ) ) { - return Detail::make_unique<Win32ColourImpl>( stream ); - } -#endif - if ( ANSIColourImpl::useImplementationForStream( *stream ) ) { - return Detail::make_unique<ANSIColourImpl>( stream ); - } - return Detail::make_unique<NoColourImpl>( stream ); + bool CaseInsensitiveLess::operator()( StringRef lhs, + StringRef rhs ) const { + return std::lexicographical_compare( + lhs.begin(), lhs.end(), + rhs.begin(), rhs.end(), + []( char l, char r ) { return toLower( l ) < toLower( r ); } ); } - CATCH_ERROR( "Could not create colour impl for selection " << static_cast<int>(colourSelection) ); - } - - bool isColourImplAvailable( ColourMode colourSelection ) { - switch ( colourSelection ) { -#if defined( CATCH_CONFIG_COLOUR_WIN32 ) - case ColourMode::Win32: -#endif - case ColourMode::ANSI: - case ColourMode::None: - case ColourMode::PlatformDefault: - return true; - default: - return false; + bool + CaseInsensitiveEqualTo::operator()( StringRef lhs, + StringRef rhs ) const { + return std::equal( + lhs.begin(), lhs.end(), + rhs.begin(), rhs.end(), + []( char l, char r ) { return toLower( l ) == toLower( r ); } ); } - } - - -} // end namespace Catch - -#if defined(__clang__) -# pragma clang diagnostic pop -#endif + } // namespace Detail +} // namespace Catch @@ -4085,50 +4455,16 @@ namespace Catch { Context Context::currentContext; - Context& getCurrentMutableContext() { - return Context::currentContext; - } - - SimplePcg32& sharedRng() { - static SimplePcg32 s_rng; - return s_rng; - } - -} - - - - - -#include <ostream> - -#if defined(CATCH_CONFIG_ANDROID_LOGWRITE) -#include <android/log.h> - - namespace Catch { - void writeToDebugConsole( std::string const& text ) { - __android_log_write( ANDROID_LOG_DEBUG, "Catch", text.c_str() ); - } - } - -#elif defined(CATCH_PLATFORM_WINDOWS) - - namespace Catch { - void writeToDebugConsole( std::string const& text ) { - ::OutputDebugStringA( text.c_str() ); - } + Context& getCurrentMutableContext() { + return Context::currentContext; } -#else - - namespace Catch { - void writeToDebugConsole( std::string const& text ) { - // !TBD: Need a version for Mac/ XCode and other IDEs - Catch::cout() << text; - } + SimplePcg32& sharedRng() { + static SimplePcg32 s_rng; + return s_rng; } -#endif // Platform +} @@ -4740,353 +5076,6 @@ namespace Catch { -#include <cstdio> -#include <fstream> - -namespace Catch { - - Catch::IStream::~IStream() = default; - -namespace Detail { - namespace { - template<typename WriterF, std::size_t bufferSize=256> - class StreamBufImpl final : public std::streambuf { - char data[bufferSize]; - WriterF m_writer; - - public: - StreamBufImpl() { - setp( data, data + sizeof(data) ); - } - - ~StreamBufImpl() noexcept override { - StreamBufImpl::sync(); - } - - private: - int overflow( int c ) override { - sync(); - - if( c != EOF ) { - if( pbase() == epptr() ) - m_writer( std::string( 1, static_cast<char>( c ) ) ); - else - sputc( static_cast<char>( c ) ); - } - return 0; - } - - int sync() override { - if( pbase() != pptr() ) { - m_writer( std::string( pbase(), static_cast<std::string::size_type>( pptr() - pbase() ) ) ); - setp( pbase(), epptr() ); - } - return 0; - } - }; - - /////////////////////////////////////////////////////////////////////////// - - struct OutputDebugWriter { - - void operator()( std::string const& str ) { - if ( !str.empty() ) { - writeToDebugConsole( str ); - } - } - }; - - /////////////////////////////////////////////////////////////////////////// - - class FileStream final : public IStream { - std::ofstream m_ofs; - public: - FileStream( std::string const& filename ) { - m_ofs.open( filename.c_str() ); - CATCH_ENFORCE( !m_ofs.fail(), "Unable to open file: '" << filename << '\'' ); - m_ofs << std::unitbuf; - } - public: // IStream - std::ostream& stream() override { - return m_ofs; - } - }; - - /////////////////////////////////////////////////////////////////////////// - - class CoutStream final : public IStream { - std::ostream m_os; - public: - // Store the streambuf from cout up-front because - // cout may get redirected when running tests - CoutStream() : m_os( Catch::cout().rdbuf() ) {} - - public: // IStream - std::ostream& stream() override { return m_os; } - bool isConsole() const override { return true; } - }; - - class CerrStream : public IStream { - std::ostream m_os; - - public: - // Store the streambuf from cerr up-front because - // cout may get redirected when running tests - CerrStream(): m_os( Catch::cerr().rdbuf() ) {} - - public: // IStream - std::ostream& stream() override { return m_os; } - bool isConsole() const override { return true; } - }; - - /////////////////////////////////////////////////////////////////////////// - - class DebugOutStream final : public IStream { - Detail::unique_ptr<StreamBufImpl<OutputDebugWriter>> m_streamBuf; - std::ostream m_os; - public: - DebugOutStream() - : m_streamBuf( Detail::make_unique<StreamBufImpl<OutputDebugWriter>>() ), - m_os( m_streamBuf.get() ) - {} - - public: // IStream - std::ostream& stream() override { return m_os; } - }; - - } // unnamed namespace -} // namespace Detail - - /////////////////////////////////////////////////////////////////////////// - - auto makeStream( std::string const& filename ) -> Detail::unique_ptr<IStream> { - if ( filename.empty() || filename == "-" ) { - return Detail::make_unique<Detail::CoutStream>(); - } - if( filename[0] == '%' ) { - if ( filename == "%debug" ) { - return Detail::make_unique<Detail::DebugOutStream>(); - } else if ( filename == "%stderr" ) { - return Detail::make_unique<Detail::CerrStream>(); - } else if ( filename == "%stdout" ) { - return Detail::make_unique<Detail::CoutStream>(); - } else { - CATCH_ERROR( "Unrecognised stream: '" << filename << '\'' ); - } - } - return Detail::make_unique<Detail::FileStream>( filename ); - } - -} - - - -#include <cmath> -#include <locale> - -namespace Catch { - - namespace { - struct EscapeLUT { - bool escape[256] = {}; - constexpr EscapeLUT() { - escape[static_cast<unsigned char>( '"' )] = true; - escape[static_cast<unsigned char>( '\\' )] = true; - escape[static_cast<unsigned char>( '\b' )] = true; - escape[static_cast<unsigned char>( '\f' )] = true; - escape[static_cast<unsigned char>( '\n' )] = true; - escape[static_cast<unsigned char>( '\r' )] = true; - escape[static_cast<unsigned char>( '\t' )] = true; - } - }; - static constexpr EscapeLUT escapeLUT{}; - - static constexpr bool needsEscape( char c ) { - return escapeLUT.escape[static_cast<unsigned char>( c )]; - } - - static Catch::StringRef makeEscapeStringRef( char c ) { - if ( c == '"' ) { - return "\\\""_sr; - } else if ( c == '\\' ) { - return "\\\\"_sr; - } else if ( c == '\b' ) { - return "\\b"_sr; - } else if ( c == '\f' ) { - return "\\f"_sr; - } else if ( c == '\n' ) { - return "\\n"_sr; - } else if ( c == '\r' ) { - return "\\r"_sr; - } else if ( c == '\t' ) { - return "\\t"_sr; - } - Catch::Detail::Unreachable(); - } - } // namespace - - void JsonUtils::indent( std::ostream& os, std::uint64_t level ) { - for ( std::uint64_t i = 0; i < level; ++i ) { - os << " "; - } - } - void JsonUtils::appendCommaNewline( std::ostream& os, - bool& should_comma, - std::uint64_t level ) { - if ( should_comma ) { os << ','; } - should_comma = true; - os << '\n'; - indent( os, level ); - } - - JsonObjectWriter::JsonObjectWriter( std::ostream& os ): - JsonObjectWriter{ os, 0 } {} - - JsonObjectWriter::JsonObjectWriter( std::ostream& os, - std::uint64_t indent_level ): - m_os{ os }, m_indent_level{ indent_level } { - m_os << '{'; - } - JsonObjectWriter::JsonObjectWriter( JsonObjectWriter&& source ) noexcept: - m_os{ source.m_os }, - m_indent_level{ source.m_indent_level }, - m_should_comma{ source.m_should_comma }, - m_active{ source.m_active } { - source.m_active = false; - } - - JsonObjectWriter::~JsonObjectWriter() { - if ( !m_active ) { return; } - - m_os << '\n'; - JsonUtils::indent( m_os, m_indent_level ); - m_os << '}'; - } - - JsonValueWriter JsonObjectWriter::write( StringRef key ) { - JsonUtils::appendCommaNewline( - m_os, m_should_comma, m_indent_level + 1 ); - - m_os << '"' << key << "\": "; - return JsonValueWriter{ m_os, m_indent_level + 1 }; - } - - JsonArrayWriter::JsonArrayWriter( std::ostream& os ): - JsonArrayWriter{ os, 0 } {} - JsonArrayWriter::JsonArrayWriter( std::ostream& os, - std::uint64_t indent_level ): - m_os{ os }, m_indent_level{ indent_level } { - m_os << '['; - } - JsonArrayWriter::JsonArrayWriter( JsonArrayWriter&& source ) noexcept: - m_os{ source.m_os }, - m_indent_level{ source.m_indent_level }, - m_should_comma{ source.m_should_comma }, - m_active{ source.m_active } { - source.m_active = false; - } - JsonArrayWriter::~JsonArrayWriter() { - if ( !m_active ) { return; } - - m_os << '\n'; - JsonUtils::indent( m_os, m_indent_level ); - m_os << ']'; - } - - JsonObjectWriter JsonArrayWriter::writeObject() { - JsonUtils::appendCommaNewline( - m_os, m_should_comma, m_indent_level + 1 ); - return JsonObjectWriter{ m_os, m_indent_level + 1 }; - } - - JsonArrayWriter JsonArrayWriter::writeArray() { - JsonUtils::appendCommaNewline( - m_os, m_should_comma, m_indent_level + 1 ); - return JsonArrayWriter{ m_os, m_indent_level + 1 }; - } - - JsonArrayWriter& JsonArrayWriter::write( bool value ) { - return writeImpl( value ); - } - - JsonValueWriter::JsonValueWriter( std::ostream& os ): - JsonValueWriter{ os, 0 } {} - - JsonValueWriter::JsonValueWriter( std::ostream& os, - std::uint64_t indent_level ): - m_os{ os }, m_indent_level{ indent_level } { - // We use C locale so that writing of numerical values is locale-independent. - m_sstream.imbue( std::locale::classic() ); - } - - JsonObjectWriter JsonValueWriter::writeObject() && { - return JsonObjectWriter{ m_os, m_indent_level }; - } - - JsonArrayWriter JsonValueWriter::writeArray() && { - return JsonArrayWriter{ m_os, m_indent_level }; - } - - void JsonValueWriter::write( Catch::StringRef value ) && { - writeImpl( value, true ); - } - - void JsonValueWriter::write( float value ) && { - writeFloatingPoint( value ); - } - - void JsonValueWriter::write( double value ) && { - writeFloatingPoint( value ); - } - - void JsonValueWriter::write( bool value ) && { - writeImpl( value ? "true"_sr : "false"_sr, false ); - } - - template <typename T> - void JsonValueWriter::writeFloatingPoint( T value ) { - if ( Catch::isnan( value ) ) { - writeImpl( "NaN"_sr, false ); - } else if ( std::isinf( value ) ) { - writeImpl( value < 0 ? "-Infinity"_sr : "Infinity"_sr, false ); - } else { - m_sstream << value; - writeImpl( m_sstream.str(), false ); - } - } - - void JsonValueWriter::writeImpl( Catch::StringRef value, bool quote ) { - // Escaping only makes sense for actual strings, which are passed - // with `quote == true`. Non-quoted values are things like bools - // and numbers, which cannot create inputs that need escaping. - if ( !quote ) { - m_os << value; - } else { - m_os << '"'; - size_t current_start = 0; - for ( size_t i = 0; i < value.size(); ++i ) { - if ( needsEscape( value[i] ) ) { - if ( current_start < i ) { - m_os - << value.substr( current_start, i - current_start ); - } - m_os << makeEscapeStringRef( value[i] ); - current_start = i + 1; - } - } - if ( current_start < value.size() ) { - m_os << value.substr( current_start, - value.size() - current_start ); - } - m_os << '"'; - } - } - -} // namespace Catch - - - - namespace Catch { auto operator << (std::ostream& os, LazyExpression const& lazyExpr) -> std::ostream& { @@ -5138,111 +5127,6 @@ Catch::LeakDetector::~LeakDetector() { - -namespace Catch { - namespace { - - void listTests(IEventListener& reporter, IConfig const& config) { - auto const& testSpec = config.testSpec(); - auto matchedTestCases = filterTests(getAllTestCasesSorted(config), testSpec, config); - reporter.listTests(matchedTestCases); - } - - void listTags(IEventListener& reporter, IConfig const& config) { - auto const& testSpec = config.testSpec(); - std::vector<TestCaseHandle> matchedTestCases = filterTests(getAllTestCasesSorted(config), testSpec, config); - - std::map<StringRef, TagInfo, Detail::CaseInsensitiveLess> tagCounts; - for (auto const& testCase : matchedTestCases) { - for (auto const& tagName : testCase.getTestCaseInfo().tags) { - auto it = tagCounts.find(tagName.original); - if (it == tagCounts.end()) - it = tagCounts.insert(std::make_pair(tagName.original, TagInfo())).first; - it->second.add(tagName.original); - } - } - - std::vector<TagInfo> infos; infos.reserve(tagCounts.size()); - for (auto& tagc : tagCounts) { - infos.push_back(CATCH_MOVE(tagc.second)); - } - - reporter.listTags(infos); - } - - void listReporters(IEventListener& reporter) { - std::vector<ReporterDescription> descriptions; - - auto const& factories = getRegistryHub().getReporterRegistry().getFactories(); - descriptions.reserve(factories.size()); - for (auto const& fac : factories) { - descriptions.push_back({ fac.first, fac.second->getDescription() }); - } - - reporter.listReporters(descriptions); - } - - void listListeners(IEventListener& reporter) { - std::vector<ListenerDescription> descriptions; - - auto const& factories = - getRegistryHub().getReporterRegistry().getListeners(); - descriptions.reserve( factories.size() ); - for ( auto const& fac : factories ) { - descriptions.push_back( { fac->getName(), fac->getDescription() } ); - } - - reporter.listListeners( descriptions ); - } - - } // end anonymous namespace - - void TagInfo::add( StringRef spelling ) { - ++count; - spellings.insert( spelling ); - } - - std::string TagInfo::all() const { - // 2 per tag for brackets '[' and ']' - size_t size = spellings.size() * 2; - for (auto const& spelling : spellings) { - size += spelling.size(); - } - - std::string out; out.reserve(size); - for (auto const& spelling : spellings) { - out += '['; - out += spelling; - out += ']'; - } - return out; - } - - bool list( IEventListener& reporter, Config const& config ) { - bool listed = false; - if (config.listTests()) { - listed = true; - listTests(reporter, config); - } - if (config.listTags()) { - listed = true; - listTags(reporter, config); - } - if (config.listReporters()) { - listed = true; - listReporters(reporter); - } - if ( config.listListeners() ) { - listed = true; - listListeners( reporter ); - } - return listed; - } - -} // end namespace Catch - - - namespace Catch { CATCH_INTERNAL_START_WARNINGS_SUPPRESSION CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS @@ -5804,258 +5688,22 @@ namespace Catch { namespace Catch { struct ReporterRegistry::ReporterRegistryImpl { std::vector<Detail::unique_ptr<EventListenerFactory>> listeners; - std::map<std::string, IReporterFactoryPtr, Detail::CaseInsensitiveLess> - factories; }; - ReporterRegistry::ReporterRegistry(): - m_impl( Detail::make_unique<ReporterRegistryImpl>() ) { - // Because it is impossible to move out of initializer list, - // we have to add the elements manually - m_impl->factories["Automake"] = - Detail::make_unique<ReporterFactory<AutomakeReporter>>(); - m_impl->factories["compact"] = - Detail::make_unique<ReporterFactory<CompactReporter>>(); - m_impl->factories["console"] = - Detail::make_unique<ReporterFactory<ConsoleReporter>>(); - m_impl->factories["JUnit"] = - Detail::make_unique<ReporterFactory<JunitReporter>>(); - m_impl->factories["SonarQube"] = - Detail::make_unique<ReporterFactory<SonarQubeReporter>>(); - m_impl->factories["TAP"] = - Detail::make_unique<ReporterFactory<TAPReporter>>(); - m_impl->factories["TeamCity"] = - Detail::make_unique<ReporterFactory<TeamCityReporter>>(); - m_impl->factories["XML"] = - Detail::make_unique<ReporterFactory<XmlReporter>>(); - m_impl->factories["JSON"] = - Detail::make_unique<ReporterFactory<JsonReporter>>(); - } - - ReporterRegistry::~ReporterRegistry() = default; - - IEventListenerPtr - ReporterRegistry::create( std::string const& name, - ReporterConfig&& config ) const { - auto it = m_impl->factories.find( name ); - if ( it == m_impl->factories.end() ) return nullptr; - return it->second->create( CATCH_MOVE( config ) ); - } - - void ReporterRegistry::registerReporter( std::string const& name, - IReporterFactoryPtr factory ) { - CATCH_ENFORCE( name.find( "::" ) == name.npos, - "'::' is not allowed in reporter name: '" + name + - '\'' ); - auto ret = m_impl->factories.emplace( name, CATCH_MOVE( factory ) ); - CATCH_ENFORCE( ret.second, - "reporter using '" + name + - "' as name was already registered" ); - } - void ReporterRegistry::registerListener( - Detail::unique_ptr<EventListenerFactory> factory ) { - m_impl->listeners.push_back( CATCH_MOVE( factory ) ); - } - - std::map<std::string, - IReporterFactoryPtr, - Detail::CaseInsensitiveLess> const& - ReporterRegistry::getFactories() const { - return m_impl->factories; - } - - std::vector<Detail::unique_ptr<EventListenerFactory>> const& - ReporterRegistry::getListeners() const { - return m_impl->listeners; - } -} // namespace Catch - - - - - -#include <algorithm> - -namespace Catch { - - namespace { - struct kvPair { - StringRef key, value; - }; - - kvPair splitKVPair(StringRef kvString) { - auto splitPos = static_cast<size_t>( - std::find( kvString.begin(), kvString.end(), '=' ) - - kvString.begin() ); - - return { kvString.substr( 0, splitPos ), - kvString.substr( splitPos + 1, kvString.size() ) }; - } - } - - namespace Detail { - std::vector<std::string> splitReporterSpec( StringRef reporterSpec ) { - static constexpr auto separator = "::"; - static constexpr size_t separatorSize = 2; - - size_t separatorPos = 0; - auto findNextSeparator = [&reporterSpec]( size_t startPos ) { - static_assert( - separatorSize == 2, - "The code below currently assumes 2 char separator" ); - - auto currentPos = startPos; - do { - while ( currentPos < reporterSpec.size() && - reporterSpec[currentPos] != separator[0] ) { - ++currentPos; - } - if ( currentPos + 1 < reporterSpec.size() && - reporterSpec[currentPos + 1] == separator[1] ) { - return currentPos; - } - ++currentPos; - } while ( currentPos < reporterSpec.size() ); - - return static_cast<size_t>( -1 ); - }; - - std::vector<std::string> parts; - - while ( separatorPos < reporterSpec.size() ) { - const auto nextSeparator = findNextSeparator( separatorPos ); - parts.push_back( static_cast<std::string>( reporterSpec.substr( - separatorPos, nextSeparator - separatorPos ) ) ); - - if ( nextSeparator == static_cast<size_t>( -1 ) ) { - break; - } - separatorPos = nextSeparator + separatorSize; - } - - // Handle a separator at the end. - // This is not a valid spec, but we want to do validation in a - // centralized place - if ( separatorPos == reporterSpec.size() ) { - parts.emplace_back(); - } - - return parts; - } - - Optional<ColourMode> stringToColourMode( StringRef colourMode ) { - if ( colourMode == "default" ) { - return ColourMode::PlatformDefault; - } else if ( colourMode == "ansi" ) { - return ColourMode::ANSI; - } else if ( colourMode == "win32" ) { - return ColourMode::Win32; - } else if ( colourMode == "none" ) { - return ColourMode::None; - } else { - return {}; - } - } - - Optional<Verbosity> stringToVerbosity( StringRef verbosity ) { - if (verbosity == "quiet") { return Verbosity::Quiet; - } else if ( verbosity == "normal" ) { - return Verbosity::Normal; - } else if ( verbosity == "high" ) { - return Verbosity::High; - } else { - return {}; - } - } - - } // namespace Detail - - - bool operator==( ReporterSpec const& lhs, ReporterSpec const& rhs ) { - return lhs.m_name == rhs.m_name && - lhs.m_outputFileName == rhs.m_outputFileName && - lhs.m_colourMode == rhs.m_colourMode && - lhs.m_verbosity == rhs.m_verbosity && - lhs.m_customOptions == rhs.m_customOptions; - } - - Optional<ReporterSpec> parseReporterSpec( StringRef reporterSpec ) { - auto parts = Detail::splitReporterSpec( reporterSpec ); - - assert( parts.size() > 0 && "Split should never return empty vector" ); - - std::map<std::string, std::string> kvPairs; - Optional<std::string> outputFileName; - Optional<ColourMode> colourMode; - Optional<Verbosity> verbosity; - - // First part is always reporter name, so we skip it - for ( size_t i = 1; i < parts.size(); ++i ) { - auto kv = splitKVPair( parts[i] ); - auto key = kv.key, value = kv.value; + ReporterRegistry::ReporterRegistry(): + m_impl( Detail::make_unique<ReporterRegistryImpl>() ) {} - if ( key.empty() || value.empty() ) { // NOLINT(bugprone-branch-clone) - return {}; - } else if ( key[0] == 'X' ) { - // This is a reporter-specific option, we don't check these - // apart from basic sanity checks - if ( key.size() == 1 ) { - return {}; - } - - auto ret = kvPairs.emplace( std::string(kv.key), std::string(kv.value) ); - if ( !ret.second ) { - // Duplicated key. We might want to handle this differently, - // e.g. by overwriting the existing value? - return {}; - } - } else if ( key == "out" ) { - // Duplicated key - if ( outputFileName ) { - return {}; - } - outputFileName = static_cast<std::string>( value ); - } else if ( key == "colour-mode" ) { - // Duplicated key - if ( colourMode ) { - return {}; - } - colourMode = Detail::stringToColourMode( value ); - // Parsing failed - if ( !colourMode ) { - return {}; - } - } else if ( key == "verbosity" ) { - // Duplicated key - if ( verbosity ) { return {}; } - verbosity = Detail::stringToVerbosity( value ); - // Parsing failed - if ( !verbosity ) { return {}; } - } else { - // Unrecognized option - return {}; - } - } + ReporterRegistry::~ReporterRegistry() = default; - return ReporterSpec{ CATCH_MOVE( parts[0] ), - CATCH_MOVE( outputFileName ), - CATCH_MOVE( colourMode ), - CATCH_MOVE( verbosity), - CATCH_MOVE( kvPairs ) }; + void ReporterRegistry::registerListener( + Detail::unique_ptr<EventListenerFactory> factory ) { + m_impl->listeners.push_back( CATCH_MOVE( factory ) ); } -ReporterSpec::ReporterSpec( - std::string name, - Optional<std::string> outputFileName, - Optional<ColourMode> colourMode, - Optional<Verbosity> verbosity, - std::map<std::string, std::string> customOptions ): - m_name( CATCH_MOVE( name ) ), - m_outputFileName( CATCH_MOVE( outputFileName ) ), - m_colourMode( CATCH_MOVE( colourMode ) ), - m_verbosity( CATCH_MOVE( verbosity ) ), - m_customOptions( CATCH_MOVE( customOptions ) ) {} - + std::vector<Detail::unique_ptr<EventListenerFactory>> const& + ReporterRegistry::getListeners() const { + return m_impl->listeners; + } } // namespace Catch @@ -6846,6 +6494,14 @@ namespace Catch { return m_atomicAssertionCount.failed >= m_abortAfterXFailedAssertions; } + std::string RunContext::capturedOutput() { + if ( !m_outputRedirect->isActive() ) { + return {}; + } + auto _ = scopedDeactivate( *m_outputRedirect ); + return m_outputRedirect->getStdout() + m_outputRedirect->getStderr(); + } + void RunContext::runCurrentTest() { auto const& testCaseInfo = m_activeTestCase->getTestCaseInfo(); SectionInfo testCaseSection(testCaseInfo.lineInfo, testCaseInfo.name); @@ -7573,7 +7229,8 @@ namespace Catch { filtered.push_back(testCase); } } - return createShard(CATCH_MOVE(filtered), config.shardCount(), config.shardIndex()); + // The Catch3 runner shards the run itself. + return filtered; } std::vector<TestCaseHandle> const& getAllTestCasesSorted( IConfig const& config ) { return getRegistryHub().getTestCaseRegistry().getAllTestsSorted( config ); @@ -8128,4737 +7785,11118 @@ namespace Catch { } } - void TestSpecParser::saveLastMode() { - lastMode = m_mode; + void TestSpecParser::saveLastMode() { + lastMode = m_mode; + } + + void TestSpecParser::revertBackToLastMode() { + m_mode = lastMode; + } + + bool TestSpecParser::separate() { + if( (m_mode==QuotedName) || (m_mode==Tag) ){ + //invalid argument, signal failure to previous scope. + m_mode = None; + m_pos = m_arg.size(); + m_substring.clear(); + m_patternName.clear(); + m_realPatternPos = 0; + return false; + } + endMode(); + addFilter(); + return true; //success + } + + std::string TestSpecParser::preprocessPattern() { + std::string token = m_patternName; + for (std::size_t i = 0; i < m_escapeChars.size(); ++i) + token = token.substr(0, m_escapeChars[i] - i) + token.substr(m_escapeChars[i] - i + 1); + m_escapeChars.clear(); + if (startsWith(token, "exclude:")) { + m_exclusion = true; + token = token.substr(8); + } + + m_patternName.clear(); + m_realPatternPos = 0; + + return token; + } + + void TestSpecParser::addNamePattern() { + auto token = preprocessPattern(); + + if (!token.empty()) { + if (m_exclusion) { + m_currentFilter.m_forbidden.emplace_back(Detail::make_unique<TestSpec::NamePattern>(token, m_substring)); + } else { + m_currentFilter.m_required.emplace_back(Detail::make_unique<TestSpec::NamePattern>(token, m_substring)); + } + } + m_substring.clear(); + m_exclusion = false; + m_mode = None; + } + + void TestSpecParser::addTagPattern() { + auto token = preprocessPattern(); + + if (!token.empty()) { + // If the tag pattern is the "hide and tag" shorthand (e.g. [.foo]) + // we have to create a separate hide tag and shorten the real one + if (token.size() > 1 && token[0] == '.') { + token.erase(token.begin()); + if (m_exclusion) { + m_currentFilter.m_forbidden.emplace_back(Detail::make_unique<TestSpec::TagPattern>(".", m_substring)); + } else { + m_currentFilter.m_required.emplace_back(Detail::make_unique<TestSpec::TagPattern>(".", m_substring)); + } + } + if (m_exclusion) { + m_currentFilter.m_forbidden.emplace_back(Detail::make_unique<TestSpec::TagPattern>(token, m_substring)); + } else { + m_currentFilter.m_required.emplace_back(Detail::make_unique<TestSpec::TagPattern>(token, m_substring)); + } + } + m_substring.clear(); + m_exclusion = false; + m_mode = None; + } + +} // namespace Catch + + + + +#include <exception> + +namespace Catch { + bool uncaught_exceptions() { +#if defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) + return false; +#elif defined(CATCH_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS) + return std::uncaught_exceptions() > 0; +#else + return std::uncaught_exception(); +#endif + } +} // end namespace Catch + + + +namespace Catch { + + WildcardPattern::WildcardPattern( std::string const& pattern, + CaseSensitive caseSensitivity ) + : m_caseSensitivity( caseSensitivity ), + m_pattern( trim(pattern) ) + { + if( startsWith( m_pattern, '*' ) ) { + m_pattern = m_pattern.erase( 0, 1 ); // effectively pop_front + m_wildcard = static_cast<WildcardPosition>( m_wildcard | WildcardAtStart ); + } + if( endsWith( m_pattern, '*' ) ) { + m_pattern.pop_back(); + m_wildcard = static_cast<WildcardPosition>( m_wildcard | WildcardAtEnd ); + } + } + + bool WildcardPattern::matches( StringRef input ) const { + input = trim( input ); + if ( input.size() < m_pattern.size() ) { return false; } + + auto casedMatch = [&]( StringRef lhs, StringRef rhs ) { + if ( m_caseSensitivity == CaseSensitive::Yes ) { + return lhs == rhs; + } else { + Detail::CaseInsensitiveEqualTo eq; + return eq( lhs, rhs ); + } + }; + + switch ( m_wildcard ) { + case NoWildcard: + return casedMatch(input, m_pattern); + case WildcardAtStart: + return casedMatch( + input.substr( input.size() - m_pattern.size(), input.size() ), + m_pattern ); + case WildcardAtEnd: + return casedMatch( input.substr( 0, m_pattern.size() ), m_pattern ); + case WildcardAtBothEnds: { + // TBD: This has terrible performance, but it is also the least + // common use case. Once the StringContains matcher gets + // efficient implementation, we should reuse it here. + const size_t lastOffset = input.size() - m_pattern.size(); + for ( size_t offset = 0; offset <= lastOffset; ++offset ) { + if ( casedMatch( input.substr(offset, m_pattern.size()), m_pattern ) ) { return true; } + } + return false; + } + default: + CATCH_INTERNAL_ERROR( "Unknown enum" ); + } + } +} + + + + + +namespace Catch { +namespace Matchers { + + std::string MatcherUntypedBase::toString() const { + return describe(); + } + + std::string MatcherUntypedBase::describe() const { + using namespace std::string_literals; + return "Undescribed matcher"s; + } + +} // namespace Matchers +} // namespace Catch + + + + +namespace Catch { +namespace Matchers { + + std::string IsEmptyMatcher::describe() const { + return "is empty"; + } + + std::string HasSizeMatcher::describe() const { + ReusableStringStream sstr; + sstr << "has size == " << m_target_size; + return sstr.str(); + } + +} // end namespace Matchers +} // end namespace Catch + + + +namespace Catch { +namespace Matchers { + +bool ExceptionMessageMatcher::match(std::exception const& ex) const { + return ex.what() == m_message; +} + +std::string ExceptionMessageMatcher::describe() const { + return "exception message matches \"" + m_message + '"'; +} + +ExceptionMessageMatcher Message(std::string const& message) { + return ExceptionMessageMatcher(message); +} + +} // namespace Matchers +} // namespace Catch + + + +#include <algorithm> +#include <cmath> +#include <cstdlib> +#include <cstdint> +#include <sstream> +#include <iomanip> +#include <limits> + + +namespace Catch { +namespace { + + template <typename FP> + bool almostEqualUlps(FP lhs, FP rhs, uint64_t maxUlpDiff) { + // Comparison with NaN should always be false. + // This way we can rule it out before getting into the ugly details + if (Catch::isnan(lhs) || Catch::isnan(rhs)) { + return false; + } + + // This should also handle positive and negative zeros, infinities + const auto ulpDist = ulpDistance(lhs, rhs); + + return ulpDist <= maxUlpDiff; } - void TestSpecParser::revertBackToLastMode() { - m_mode = lastMode; - } - bool TestSpecParser::separate() { - if( (m_mode==QuotedName) || (m_mode==Tag) ){ - //invalid argument, signal failure to previous scope. - m_mode = None; - m_pos = m_arg.size(); - m_substring.clear(); - m_patternName.clear(); - m_realPatternPos = 0; - return false; - } - endMode(); - addFilter(); - return true; //success +template <typename FP> +FP step(FP start, FP direction, uint64_t steps) { + for (uint64_t i = 0; i < steps; ++i) { + start = Catch::nextafter(start, direction); } + return start; +} - std::string TestSpecParser::preprocessPattern() { - std::string token = m_patternName; - for (std::size_t i = 0; i < m_escapeChars.size(); ++i) - token = token.substr(0, m_escapeChars[i] - i) + token.substr(m_escapeChars[i] - i + 1); - m_escapeChars.clear(); - if (startsWith(token, "exclude:")) { - m_exclusion = true; - token = token.substr(8); - } +// Performs equivalent check of std::fabs(lhs - rhs) <= margin +// But without the subtraction to allow for INFINITY in comparison +bool marginComparison(double lhs, double rhs, double margin) { + return (lhs + margin >= rhs) && (rhs + margin >= lhs); +} - m_patternName.clear(); - m_realPatternPos = 0; +template <typename FloatingPoint> +void write(std::ostream& out, FloatingPoint num) { + out << std::scientific + << std::setprecision(std::numeric_limits<FloatingPoint>::max_digits10 - 1) + << num; +} - return token; +} // end anonymous namespace + +namespace Matchers { +namespace Detail { + + enum class FloatingPointKind : uint8_t { + Float, + Double + }; + +} // end namespace Detail + + + WithinAbsMatcher::WithinAbsMatcher(double target, double margin) + :m_target{ target }, m_margin{ margin } { + CATCH_ENFORCE(margin >= 0, "Invalid margin: " << margin << '.' + << " Margin has to be non-negative."); } - void TestSpecParser::addNamePattern() { - auto token = preprocessPattern(); + // Performs equivalent check of std::fabs(lhs - rhs) <= margin + // But without the subtraction to allow for INFINITY in comparison + bool WithinAbsMatcher::match(double const& matchee) const { + return (matchee + m_margin >= m_target) && (m_target + m_margin >= matchee); + } - if (!token.empty()) { - if (m_exclusion) { - m_currentFilter.m_forbidden.emplace_back(Detail::make_unique<TestSpec::NamePattern>(token, m_substring)); - } else { - m_currentFilter.m_required.emplace_back(Detail::make_unique<TestSpec::NamePattern>(token, m_substring)); - } - } - m_substring.clear(); - m_exclusion = false; - m_mode = None; + std::string WithinAbsMatcher::describe() const { + return "is within " + ::Catch::Detail::stringify(m_margin) + " of " + ::Catch::Detail::stringify(m_target); } - void TestSpecParser::addTagPattern() { - auto token = preprocessPattern(); - if (!token.empty()) { - // If the tag pattern is the "hide and tag" shorthand (e.g. [.foo]) - // we have to create a separate hide tag and shorten the real one - if (token.size() > 1 && token[0] == '.') { - token.erase(token.begin()); - if (m_exclusion) { - m_currentFilter.m_forbidden.emplace_back(Detail::make_unique<TestSpec::TagPattern>(".", m_substring)); - } else { - m_currentFilter.m_required.emplace_back(Detail::make_unique<TestSpec::TagPattern>(".", m_substring)); - } - } - if (m_exclusion) { - m_currentFilter.m_forbidden.emplace_back(Detail::make_unique<TestSpec::TagPattern>(token, m_substring)); - } else { - m_currentFilter.m_required.emplace_back(Detail::make_unique<TestSpec::TagPattern>(token, m_substring)); - } + WithinUlpsMatcher::WithinUlpsMatcher(double target, uint64_t ulps, Detail::FloatingPointKind baseType) + :m_target{ target }, m_ulps{ ulps }, m_type{ baseType } { + CATCH_ENFORCE(m_type == Detail::FloatingPointKind::Double + || m_ulps < (std::numeric_limits<uint32_t>::max)(), + "Provided ULP is impossibly large for a float comparison."); + CATCH_ENFORCE( std::numeric_limits<double>::is_iec559, + "WithinUlp matcher only supports platforms with " + "IEEE-754 compatible floating point representation" ); + } + +#if defined(__clang__) +#pragma clang diagnostic push +// Clang <3.5 reports on the default branch in the switch below +#pragma clang diagnostic ignored "-Wunreachable-code" +#endif + + bool WithinUlpsMatcher::match(double const& matchee) const { + switch (m_type) { + case Detail::FloatingPointKind::Float: + return almostEqualUlps<float>(static_cast<float>(matchee), static_cast<float>(m_target), m_ulps); + case Detail::FloatingPointKind::Double: + return almostEqualUlps<double>(matchee, m_target, m_ulps); + default: + CATCH_INTERNAL_ERROR( "Unknown Detail::FloatingPointKind value" ); } - m_substring.clear(); - m_exclusion = false; - m_mode = None; } -} // namespace Catch +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + std::string WithinUlpsMatcher::describe() const { + std::stringstream ret; + ret << "is within " << m_ulps << " ULPs of "; -#include <algorithm> -#include <cstring> -#include <ostream> + if (m_type == Detail::FloatingPointKind::Float) { + write(ret, static_cast<float>(m_target)); + ret << 'f'; + } else { + write(ret, m_target); + } -namespace { - bool isWhitespace( char c ) { - return c == ' ' || c == '\t' || c == '\n' || c == '\r'; + ret << " (["; + if (m_type == Detail::FloatingPointKind::Double) { + write( ret, + step( m_target, + -std::numeric_limits<double>::infinity(), + m_ulps ) ); + ret << ", "; + write( ret, + step( m_target, + std::numeric_limits<double>::infinity(), + m_ulps ) ); + } else { + // We have to cast INFINITY to float because of MinGW, see #1782 + write( ret, + step( static_cast<float>( m_target ), + -std::numeric_limits<float>::infinity(), + m_ulps ) ); + ret << ", "; + write( ret, + step( static_cast<float>( m_target ), + std::numeric_limits<float>::infinity(), + m_ulps ) ); + } + ret << "])"; + + return ret.str(); } - bool isBreakableBefore( char c ) { - static const char chars[] = "[({<|"; - return std::memchr( chars, c, sizeof( chars ) - 1 ) != nullptr; + WithinRelMatcher::WithinRelMatcher(double target, double epsilon): + m_target(target), + m_epsilon(epsilon){ + CATCH_ENFORCE(m_epsilon >= 0., "Relative comparison with epsilon < 0 does not make sense."); + CATCH_ENFORCE(m_epsilon < 1., "Relative comparison with epsilon >= 1 does not make sense."); } - bool isBreakableAfter( char c ) { - static const char chars[] = "])}>.,:;*+-=&/\\"; - return std::memchr( chars, c, sizeof( chars ) - 1 ) != nullptr; + bool WithinRelMatcher::match(double const& matchee) const { + const auto relMargin = m_epsilon * (std::max)(std::fabs(matchee), std::fabs(m_target)); + return marginComparison(matchee, m_target, + std::isinf(relMargin)? 0 : relMargin); } - bool isUtf8ContinuationByte( char c ) { - return ( static_cast<unsigned char>( c ) & 0xC0 ) == 0x80; + std::string WithinRelMatcher::describe() const { + Catch::ReusableStringStream sstr; + sstr << "and " << ::Catch::Detail::stringify(m_target) << " are within " << m_epsilon * 100. << "% of each other"; + return sstr.str(); } -} // namespace -namespace Catch { - namespace TextFlow { - void AnsiSkippingString::preprocessString() { - // If there are no potential escapes, we can use fast path - if ( m_string.find( '\033' ) == std::string::npos ) { - size_t size = 0; - for ( const char c : m_string ) { - size += !isUtf8ContinuationByte( c ); - } - m_size = size; - return; - } - for ( auto it = m_string.begin(); it != m_string.end(); ) { - // try to read through an ansi sequence - while ( it != m_string.end() && *it == '\033' && - it + 1 != m_string.end() && *( it + 1 ) == '[' ) { - auto cursor = it + 2; - while ( cursor != m_string.end() && - ( isdigit( *cursor ) || *cursor == ';' ) ) { - ++cursor; - } - if ( cursor == m_string.end() || *cursor != 'm' ) { - break; - } - // record the escape sequence's byte range - m_escapes.push_back( - { std::distance( m_string.begin(), it ), - std::distance( m_string.begin(), cursor + 1 ) } ); - // if we've read an ansi sequence, set the iterator and - // return to the top of the loop - it = cursor + 1; - } - if ( it != m_string.end() ) { - ++m_size; - ++it; - // Skip UTF-8 continuation bytes - while ( it != m_string.end() && - isUtf8ContinuationByte( *it ) ) { - ++it; - } - } - } - } +WithinUlpsMatcher WithinULP(double target, uint64_t maxUlpDiff) { + return WithinUlpsMatcher(target, maxUlpDiff, Detail::FloatingPointKind::Double); +} - AnsiSkippingString::AnsiSkippingString( std::string const& text ): - m_string( text ) { - preprocessString(); - } +WithinUlpsMatcher WithinULP(float target, uint64_t maxUlpDiff) { + return WithinUlpsMatcher(target, maxUlpDiff, Detail::FloatingPointKind::Float); +} - AnsiSkippingString::AnsiSkippingString( std::string&& text ): - m_string( CATCH_MOVE( text ) ) { - preprocessString(); - } +WithinAbsMatcher WithinAbs(double target, double margin) { + return WithinAbsMatcher(target, margin); +} - AnsiSkippingString::const_iterator AnsiSkippingString::begin() const { - return const_iterator( *this ); - } +WithinRelMatcher WithinRel(double target, double eps) { + return WithinRelMatcher(target, eps); +} - AnsiSkippingString::const_iterator AnsiSkippingString::end() const { - return const_iterator( *this, const_iterator::EndTag{} ); - } +WithinRelMatcher WithinRel(double target) { + return WithinRelMatcher(target, std::numeric_limits<double>::epsilon() * 100); +} - std::string AnsiSkippingString::substring( const_iterator first, - const_iterator last ) const { - std::string ret; - appendSubstringTo( ret, first, last ); - return ret; - } +WithinRelMatcher WithinRel(float target, float eps) { + return WithinRelMatcher(target, eps); +} - void - AnsiSkippingString::appendSubstringTo( std::string& out, - const_iterator first, - const_iterator last ) const { - // There's one caveat here to an otherwise simple substring: when - // making a begin iterator we might have skipped ansi sequences at - // the start. If `first` here is a begin iterator, skipped over - // initial ansi sequences, we'll use the true beginning of the - // string. - if ( hasEscapes() && first == begin() ) { - out.append( m_string.begin(), last.m_it ); - } else { - out.append( first.m_it, last.m_it ); - } - } +WithinRelMatcher WithinRel(float target) { + return WithinRelMatcher(target, std::numeric_limits<float>::epsilon() * 100); +} - void AnsiSkippingString::const_iterator::jumpForwardOverEscapes() { - // Escapes can only start with '\033' - if ( m_it == m_string->m_string.end() || *m_it != '\033' ) { - return; - } - // If we are at '\033', check if we are at an actual escape - auto current_pos = std::distance( m_string->m_string.begin(), m_it ); - - auto const& escapes = m_string->m_escapes; - auto candidate = - std::lower_bound( escapes.begin(), - escapes.end(), - current_pos, - []( AnsiSkippingString::EscapeRange const& r, - std::ptrdiff_t o ) { return r.start < o; } ); - // Escapes can be consecutive, we need to jump over all of them - while ( candidate != escapes.end() && candidate->start == current_pos ) { - current_pos = candidate->end; - ++candidate; - } - m_it = m_string->m_string.begin() + current_pos; - } - - void AnsiSkippingString::const_iterator::jumpBackOverEscapes() { - auto strBegin = m_string->m_string.begin(); - // Escapes only end with 'm' - while ( *m_it == 'm' ) { - // If we are at 'm', check if we are at an actual escape - const auto current_pos = std::distance( strBegin, m_it ); - - auto const& escapes = m_string->m_escapes; - auto range = std::lower_bound( - escapes.begin(), - escapes.end(), - current_pos + 1, // to half-open range - []( AnsiSkippingString::EscapeRange const& r, - std::ptrdiff_t o ) { return r.end < o; } ); - if ( range == escapes.end() || range->end != current_pos + 1 ) { - break; - } - assert( range->start != 0 && - "Trying to go back with begin iterator" ); - m_it = strBegin + range->start - 1; - } - } - void AnsiSkippingString::const_iterator::advance() { - auto strEnd = m_string->m_string.end(); - assert( m_it != strEnd ); - m_it++; - // Skip UTF-8 continuation bytes - while ( m_it != strEnd && - isUtf8ContinuationByte( *m_it ) ) { - m_it++; - } - if ( m_string->hasEscapes() ) { jumpForwardOverEscapes(); } - } +bool IsNaNMatcher::match( double const& matchee ) const { + return std::isnan( matchee ); +} + +std::string IsNaNMatcher::describe() const { + using namespace std::string_literals; + return "is NaN"s; +} + +IsNaNMatcher IsNaN() { return IsNaNMatcher(); } + + } // namespace Matchers +} // namespace Catch + + + + +std::string Catch::Matchers::Detail::finalizeDescription(const std::string& desc) { + if (desc.empty()) { + return "matches undescribed predicate"; + } else { + return "matches predicate: \"" + desc + '"'; + } +} + - void AnsiSkippingString::const_iterator::unadvance() { - auto strBegin = m_string->m_string.begin(); - assert( m_it != strBegin ); - m_it--; - if ( m_string->hasEscapes() ) { jumpBackOverEscapes(); } - // Skip back over UTF-8 continuation bytes to the leading byte - while ( m_it != strBegin && - isUtf8ContinuationByte( *m_it ) ) { - m_it--; - } - } - static bool isBoundary( AnsiSkippingString const& line, - AnsiSkippingString::const_iterator it ) { - return it == line.end() || - ( isWhitespace( *it ) && - !isWhitespace( *it.oneBefore() ) ) || - isBreakableBefore( *it ) || - isBreakableAfter( *it.oneBefore() ); - } +namespace Catch { + namespace Matchers { - void Column::const_iterator::calcLength() { - m_addHyphen = false; - m_parsedTo = m_lineStart; - AnsiSkippingString const& current_line = m_column.m_string; + std::string AllTrueMatcher::describe() const { return "contains only true"; } - if ( m_parsedTo == current_line.end() ) { - m_lineEnd = m_parsedTo; - return; - } + std::string NoneTrueMatcher::describe() const { return "contains no true"; } - assert( m_lineStart != current_line.end() ); - if ( *m_lineStart == '\n' ) { ++m_parsedTo; } + std::string AnyTrueMatcher::describe() const { return "contains at least one true"; } - const auto maxLineLength = m_column.m_width - indentSize(); - std::size_t lineLength = 0; - while ( m_parsedTo != current_line.end() && - lineLength < maxLineLength && *m_parsedTo != '\n' ) { - ++m_parsedTo; - ++lineLength; - } + } // namespace Matchers +} // namespace Catch - // If we encountered a newline before the column is filled, - // then we linebreak at the newline and consider this line - // finished. - if ( lineLength < maxLineLength ) { - m_lineEnd = m_parsedTo; - } else { - // Look for a natural linebreak boundary in the column - // (We look from the end, so that the first found boundary is - // the right one) - m_lineEnd = m_parsedTo; - while ( lineLength > 0 && - !isBoundary( current_line, m_lineEnd ) ) { - --lineLength; - --m_lineEnd; - } - while ( lineLength > 0 && - isWhitespace( *m_lineEnd.oneBefore() ) ) { - --lineLength; - --m_lineEnd; - } - // If we found one, then that is where we linebreak, otherwise - // we have to split text with a hyphen - if ( lineLength == 0 ) { - m_addHyphen = true; - m_lineEnd = m_parsedTo.oneBefore(); - } - } - } - size_t Column::const_iterator::indentSize() const { - auto initial = m_lineStart == m_column.m_string.begin() - ? m_column.m_initialIndent - : std::string::npos; - return initial == std::string::npos ? m_column.m_indent : initial; - } +#include <regex> - std::string Column::const_iterator::addIndentAndSuffix( - AnsiSkippingString::const_iterator start, - AnsiSkippingString::const_iterator end ) const { - std::string ret; - const auto desired_indent = indentSize(); - ret.append( desired_indent, ' ' ); - m_column.m_string.appendSubstringTo( ret, start, end ); - if ( m_addHyphen ) { ret.push_back( '-' ); } +namespace Catch { - return ret; + namespace { + constexpr StringRef caseSensitivitySuffix( CaseSensitive caseSensitivity ) { + return caseSensitivity == CaseSensitive::Yes + ? StringRef{} + : " (case insensitive)"_sr; } + } // namespace - Column::const_iterator::const_iterator( Column const& column ): - m_column( column ), - m_lineStart( column.m_string.begin() ), - m_lineEnd( column.m_string.begin() ), - m_parsedTo( column.m_string.begin() ) { - assert( m_column.m_width > m_column.m_indent ); - assert( m_column.m_initialIndent == std::string::npos || - m_column.m_width > m_column.m_initialIndent ); - calcLength(); - if ( m_lineStart == m_lineEnd ) { - m_lineStart = m_column.m_string.end(); - } - } +namespace Matchers { - std::string Column::const_iterator::operator*() const { - assert( m_lineStart <= m_parsedTo ); - return addIndentAndSuffix( m_lineStart, m_lineEnd ); - } + StringMatcherBase::StringMatcherBase( std::string target, + StringRef operation, + CaseSensitive caseSensitivity ): + m_target( CATCH_MOVE( target ) ), + m_operation( operation ), + m_caseSensitivity( caseSensitivity ) {} - Column::const_iterator& Column::const_iterator::operator++() { - m_lineStart = m_lineEnd; - AnsiSkippingString const& current_line = m_column.m_string; - if ( m_lineStart != current_line.end() && *m_lineStart == '\n' ) { - m_lineStart++; - } else { - while ( m_lineStart != current_line.end() && - isWhitespace( *m_lineStart ) ) { - ++m_lineStart; - } - } - if ( m_lineStart != current_line.end() ) { calcLength(); } - return *this; - } + std::string StringMatcherBase::describe() const { + std::string description; + description.reserve(5 + m_operation.size() + m_target.size() + + caseSensitivitySuffix(m_caseSensitivity).size()); + description += m_operation; + description += ": \""; + description += m_target; + description += '"'; + description += caseSensitivitySuffix(m_caseSensitivity); + return description; + } - Column::const_iterator Column::const_iterator::operator++( int ) { - const_iterator prev( *this ); - operator++(); - return prev; - } - std::ostream& operator<<( std::ostream& os, Column const& col ) { - bool first = true; - for ( auto line : col ) { - if ( first ) { - first = false; - } else { - os << '\n'; - } - os << line; - } - return os; - } + StringEqualsMatcher::StringEqualsMatcher( std::string comparator, CaseSensitive caseSensitivity ): + StringMatcherBase( CATCH_MOVE( comparator ), "equals"_sr, caseSensitivity ) {} - Column Spacer( size_t spaceWidth ) { - Column ret{ "" }; - ret.width( spaceWidth ); - return ret; + bool StringEqualsMatcher::match( std::string const& source ) const { + if (m_caseSensitivity == CaseSensitive::Yes) { + return m_target == source; } + if (m_target.size() != source.size()) { return false; } + Catch::Detail::CaseInsensitiveEqualTo eq; + return eq( m_target, source ); + } - Columns::iterator::iterator( Columns const& columns, EndTag ): - m_columns( columns.m_columns ), m_activeIterators( 0 ) { - - m_iterators.reserve( m_columns.size() ); - for ( auto const& col : m_columns ) { - m_iterators.push_back( col.end() ); - } - } - Columns::iterator::iterator( Columns const& columns ): - m_columns( columns.m_columns ), - m_activeIterators( m_columns.size() ) { + StringContainsMatcher::StringContainsMatcher( + std::string comparator, CaseSensitive caseSensitivity ): + StringMatcherBase( CATCH_MOVE( comparator ), "contains"_sr, caseSensitivity ) {} - m_iterators.reserve( m_columns.size() ); - for ( auto const& col : m_columns ) { - m_iterators.push_back( col.begin() ); - } + bool StringContainsMatcher::match( std::string const& source ) const { + if ( m_caseSensitivity == CaseSensitive::Yes ) { + return contains( source, m_target ); + } + if ( source.size() < m_target.size() ) { return false; } + StringRef as_ref( source ); + // The worst case of this is O(m*n), which is terrible, BUT: + // * The average case is much better, the worst case only happens rarely + // * We can implement BMH/other better searchers later if it matters + Catch::Detail::CaseInsensitiveEqualTo eq; + for (size_t i = 0; i < source.size(); ++i) { + const auto substr = as_ref.substr( i, m_target.size() ); + bool found = eq( substr, m_target ); + if ( found ) { return true; } } + return false; + } - std::string Columns::iterator::operator*() const { - std::string row, padding; - for ( size_t i = 0; i < m_columns.size(); ++i ) { - const auto width = m_columns[i].width(); - if ( m_iterators[i] != m_columns[i].end() ) { - std::string col = *m_iterators[i]; - row += padding; - row += col; + StartsWithMatcher::StartsWithMatcher( std::string comparator, + CaseSensitive caseSensitivity ): + StringMatcherBase( CATCH_MOVE( comparator ), "starts with"_sr, caseSensitivity ) {} - padding.clear(); - if ( col.size() < width ) { - padding.append( width - col.size(), ' ' ); - } - } else { - padding.append( width, ' ' ); - } - } - return row; + bool StartsWithMatcher::match( std::string const& source ) const { + if ( m_caseSensitivity == CaseSensitive::Yes ) { + return startsWith( source, m_target ); } + if (source.size() < m_target.size()) { return false; } + Catch::Detail::CaseInsensitiveEqualTo eq; + return eq( + StringRef( source ).substr( 0, m_target.size() ), m_target ); + } - Columns::iterator& Columns::iterator::operator++() { - for ( size_t i = 0; i < m_columns.size(); ++i ) { - if ( m_iterators[i] != m_columns[i].end() ) { - ++m_iterators[i]; - } - } - return *this; - } - Columns::iterator Columns::iterator::operator++( int ) { - iterator prev( *this ); - operator++(); - return prev; - } + EndsWithMatcher::EndsWithMatcher( std::string comparator, + CaseSensitive caseSensitivity ): + StringMatcherBase( CATCH_MOVE( comparator ), "ends with"_sr, caseSensitivity ) {} - std::ostream& operator<<( std::ostream& os, Columns const& cols ) { - bool first = true; - for ( auto line : cols ) { - if ( first ) { - first = false; - } else { - os << '\n'; - } - os << line; - } - return os; + bool EndsWithMatcher::match( std::string const& source ) const { + if ( m_caseSensitivity == CaseSensitive::Yes ) { + return endsWith( source, m_target ); } + if ( source.size() < m_target.size() ) { return false; } + Catch::Detail::CaseInsensitiveEqualTo eq; + const size_t start_point = source.size() - m_target.size(); + return eq( StringRef( source ).substr( start_point, m_target.size() ), m_target ); + } - Columns operator+( Column const& lhs, Column const& rhs ) { - Columns cols; - cols += lhs; - cols += rhs; - return cols; - } - Columns operator+( Column&& lhs, Column&& rhs ) { - Columns cols; - cols += CATCH_MOVE( lhs ); - cols += CATCH_MOVE( rhs ); - return cols; - } - Columns& operator+=( Columns& lhs, Column const& rhs ) { - lhs.m_columns.push_back( rhs ); - return lhs; - } - Columns& operator+=( Columns& lhs, Column&& rhs ) { - lhs.m_columns.push_back( CATCH_MOVE( rhs ) ); - return lhs; - } - Columns operator+( Columns const& lhs, Column const& rhs ) { - auto combined( lhs ); - combined += rhs; - return combined; - } - Columns operator+( Columns&& lhs, Column&& rhs ) { - lhs += CATCH_MOVE( rhs ); - return CATCH_MOVE( lhs ); - } - } // namespace TextFlow -} // namespace Catch + RegexMatcher::RegexMatcher(std::string regex, CaseSensitive caseSensitivity): m_regex(CATCH_MOVE(regex)), m_caseSensitivity(caseSensitivity) {} + bool RegexMatcher::match(std::string const& matchee) const { + auto flags = std::regex::ECMAScript; // ECMAScript is the default syntax option anyway + if (m_caseSensitivity == CaseSensitive::No) { + flags |= std::regex::icase; + } + auto reg = std::regex(m_regex, flags); + return std::regex_match(matchee, reg); + } + std::string RegexMatcher::describe() const { + return "matches " + ::Catch::Detail::stringify(m_regex) + ((m_caseSensitivity == CaseSensitive::Yes)? " case sensitively" : " case insensitively"); + } -#include <exception> + StringEqualsMatcher Equals( std::string str, CaseSensitive caseSensitivity ) { + return StringEqualsMatcher( CATCH_MOVE( str ), caseSensitivity ); + } + StringContainsMatcher ContainsSubstring( std::string str, CaseSensitive caseSensitivity ) { + return StringContainsMatcher( CATCH_MOVE( str ), caseSensitivity ); + } + EndsWithMatcher EndsWith( std::string str, CaseSensitive caseSensitivity ) { + return EndsWithMatcher( CATCH_MOVE( str ), caseSensitivity ); + } + StartsWithMatcher StartsWith( std::string str, CaseSensitive caseSensitivity ) { + return StartsWithMatcher( CATCH_MOVE( str ), caseSensitivity ); + } -namespace Catch { - bool uncaught_exceptions() { -#if defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) - return false; -#elif defined(CATCH_CONFIG_CPP17_UNCAUGHT_EXCEPTIONS) - return std::uncaught_exceptions() > 0; -#else - return std::uncaught_exception(); -#endif - } -} // end namespace Catch + RegexMatcher Matches(std::string regex, CaseSensitive caseSensitivity) { + return RegexMatcher( CATCH_MOVE( regex ), caseSensitivity ); + } + +} // namespace Matchers +} // namespace Catch namespace Catch { +namespace Matchers { - WildcardPattern::WildcardPattern( std::string const& pattern, - CaseSensitive caseSensitivity ) - : m_caseS