Merge pull request #15 from Progmasoft/feature/catch3-runner

Replace Catch2's runner with the Catch3 runner

45 files changed, 12690 insertions, 311 deletions

diff --git a/src/Progmasoft/BUILD.bazel b/src/Progmasoft/BUILD.bazel
index b053c277..826d219e 100644
--- a/src/Progmasoft/BUILD.bazel
+++ b/src/Progmasoft/BUILD.bazel
@@ -4,15 +4,37 @@ load("@rules_cc//cc:defs.bzl", "cc_library")
 
 package(default_visibility = ["//visibility:public"])
 
+# The Catch3 runner, built into //:catch2. It is C++20; Catch2's headers
+# still need only C++14 in the code that includes them.
+cc_library(
+    name = "runner",
+    srcs = glob(["Runner/*.cpp"]) + [
+        "Results.cpp",
+        "XmlWriter.cpp",
+        "//:src/catch2/catch_session.cpp",
+    ],
+    hdrs = glob(["Runner/*.hpp"]) + [
+        "Catch3/Results.hpp",
+        "Catch3/Runner.hpp",
+        "Catch3/Version.hpp",
+        "Catch3/XmlWriter.hpp",
+    ],
+    copts = select({
+        "@platforms//os:windows": ["/std:c++20"],
+        "//conditions:default": ["-std=c++20"],
+    }),
+    linkstatic = True,
+    strip_include_prefix = "/src",
+    deps = ["//:catch2_core"],
+)
+
 cc_library(
     name = "catch3",
     srcs = [
         "ChoiceShrinker.cpp",
         "Context.cpp",
         "PropertyEngine.cpp",
-        "Results.cpp",
         "Snapshot.cpp",
-        "XmlWriter.cpp",
     ],
     hdrs = [
         "Catch3.hpp",
@@ -30,7 +52,6 @@ cc_library(
         "Catch3/Property.hpp",
         "Catch3/Random.hpp",
         "Catch3/Relations.hpp",
-        "Catch3/Results.hpp",
         "Catch3/Snapshot.hpp",
         "Catch3/Stateful.hpp",
         "Catch3/Strategy/ChoiceSource.hpp",
@@ -39,12 +60,13 @@ cc_library(
         "Catch3/Strategy/Engine.hpp",
         "Catch3/Strategy/Numeric.hpp",
         "Catch3/Strategy/Strategy.hpp",
-        "Catch3/XmlWriter.hpp",
     ],
     copts = select({
         "@platforms//os:windows": ["/std:c++20"],
         "//conditions:default": ["-std=c++20"],
     }),
     strip_include_prefix = "/src",
-    deps = ["//:catch2"],
+    deps = [
+        "//:catch2",
+    ],
 )
diff --git a/src/Progmasoft/Catch3/Results.hpp b/src/Progmasoft/Catch3/Results.hpp
index b0b69b55..5f0e0b7c 100644
--- a/src/Progmasoft/Catch3/Results.hpp
+++ b/src/Progmasoft/Catch3/Results.hpp
@@ -3,55 +3,277 @@
 #pragma once
 
 #include <cstddef>
+#include <cstdint>
+#include <optional>
 #include <string>
 #include <vector>
 
-namespace Progmasoft::Catch3 {
-
-    enum class TestCaseStatus : unsigned char {
+namespace Progmasoft::Catch3
+{
+    /// How one attempt at a test ended.
+    enum class Outcome : std::uint8_t
+    {
         kPassed,
+        /// An assertion failed, or the test threw.
         kFailed,
-        kErrored,
+        /// It failed, but `[!shouldfail]` or `[!mayfail]` allows that.
+        kFailedAsExpected,
+        /// `SKIP` was called.
         kSkipped,
+        /// The process running it died: a signal, a structured exception or
+        /// an exit in the middle of the test.
+        kCrashed,
+        /// It ran past its time limit and was stopped.
+        kTimedOut,
+        /// A test it depends on did not pass, so it was not run.
+        kBlocked,
+        /// The run stopped before reaching it.
+        kNotRun,
+        /// It passed last time and nothing it depends on changed, so the
+        /// incremental run kept that result.
+        kUpToDate,
+    };
+
+    /// Why a failure happened, as far as the runner can tell.
+    enum class FailureCategory : std::uint8_t
+    {
+        kNone,
+        /// A REQUIRE or CHECK expression was false.
+        kAssertion,
+        /// An exception escaped the test or an assertion.
+        kUnexpectedException,
+        /// REQUIRE_THROWS and its kind did not see the expected exception.
+        kMissingException,
+        /// FAIL or FAIL_CHECK.
+        kExplicitFailure,
+        /// A signal or structured exception inside the test.
+        kCrash,
+        kTimeout,
+        /// The worker could not run the test at all, for example because it
+        /// failed to start.
+        kInfrastructure,
+        /// A test it depends on failed.
+        kDependency,
+    };
+
+    /// How a failure relates to earlier runs.
+    enum class FailureHistory : std::uint8_t
+    {
+        /// There is no history to compare with.
+        kUnknown,
+        /// The test passed in the previous run.
+        kNew,
+        /// The test failed in the previous run too.
+        kPersistent,
+        /// The test passes and fails with the same inputs.
+        kIntermittent,
     };
 
-    /// Framework-neutral result for one completed test case.
-    struct TestCaseResult final {
-        std::string Name;
-        std::string ClassName;
-        std::string SourceFile;
-        TestCaseStatus Status = TestCaseStatus::kPassed;
-        double DurationSeconds = 0.0;
-        std::string Message;
-        std::string Details;
-        std::string StandardOutput;
-        std::string StandardError;
+    /// One failed or reported assertion.
+    struct AssertionRecord final
+    {
+        std::string file;
+        std::size_t line = 0;
+        /// The macro, such as REQUIRE.
+        std::string macro;
+        /// The expression as written.
+        std::string expression;
+        /// The expression with its operands' values.
+        std::string expansion;
+        std::string message;
+        /// INFO, CAPTURE and UNSCOPED_INFO messages in force.
+        std::vector<std::string> context;
+        /// The sections the assertion was in, outermost first.
+        std::vector<std::string> sections;
+        FailureCategory category = FailureCategory::kAssertion;
+        bool passed = false;
+        /// The failure is allowed by `[!mayfail]` or `[!shouldfail]`.
+        bool allowed = false;
     };
 
-    struct TestSuiteResult final {
-        std::string Name;
-        std::vector<TestCaseResult> TestCases;
+    /// The statistics of one BENCHMARK.
+    struct BenchmarkRecord final
+    {
+        std::string name;
+        std::size_t samples = 0;
+        std::size_t iterations = 0;
+        double meanNanoseconds = 0.0;
+        double lowMeanNanoseconds = 0.0;
+        double highMeanNanoseconds = 0.0;
+        double standardDeviationNanoseconds = 0.0;
+        double outlierVariance = 0.0;
     };
 
-    struct TestRunResult final {
-        std::string Name;
-        std::vector<TestSuiteResult> TestSuites;
+    /// One section entered while a test ran, with its timing.
+    struct SectionRecord final
+    {
+        std::string name;
+        std::string file;
+        std::size_t line = 0;
+        /// Microseconds since the start of the attempt.
+        std::int64_t startMicroseconds = 0;
+        /// -1 for a section that never ended: the attempt crashed or timed
+        /// out inside it.
+        std::int64_t durationMicroseconds = 0;
+        /// 0 for a section directly in the test case.
+        std::size_t depth = 0;
     };
 
-    struct TestRunSummary final {
-        std::size_t Tests = 0;
-        std::size_t Failures = 0;
-        std::size_t Errors = 0;
-        std::size_t Skipped = 0;
-        double DurationSeconds = 0.0;
+    /// One run of a test in a worker. A test may be tried several times:
+    /// retries, repeats and the probes that look for an order dependency.
+    struct Attempt final
+    {
+        Outcome outcome = Outcome::kNotRun;
+        FailureCategory category = FailureCategory::kNone;
+        std::size_t number = 1;
+        /// The worker that ran it, and the tests it ran before, in order.
+        std::size_t worker = 0;
+        std::vector<std::string> precededBy;
+        bool freshProcess = false;
+        std::uint32_t seed = 0;
+        std::int64_t startMicroseconds = 0;
+        std::int64_t durationMicroseconds = 0;
+        std::size_t assertionsPassed = 0;
+        std::size_t assertionsFailed = 0;
+        std::vector<AssertionRecord> assertions;
+        std::vector<BenchmarkRecord> benchmarks;
+        std::vector<SectionRecord> sections;
+        /// What the test wrote to std::cout and std::cerr, captured.
+        std::string standardOutput;
+        std::string standardError;
+        /// What reached the worker's own streams while the test ran, such as
+        /// printf output or a sanitizer report.
+        std::string processOutput;
+        /// The fatal condition the worker reported before it died.
+        std::string fatalMessage;
+        /// How the worker ended, for a crash.
+        std::string exitDescription;
+        /// The last assertion started before a crash or a timeout.
+        std::string lastLocation;
+        /// The directory the test had for its temporary files; kept when the
+        /// attempt failed, so its contents can be inspected.
+        std::string temporaryDirectory;
+    };
+
+    /// One test of the run and everything the runner learned about it.
+    struct TestResult final
+    {
+        /// Identifies the test across runs and builds: its name, with its
+        /// class and, when names repeat, its tags. History is kept by it.
+        std::string id;
+        std::string name;
+        std::string className;
+        std::string file;
+        std::size_t line = 0;
+        std::vector<std::string> tags;
+        /// The verdict over all attempts.
+        Outcome outcome = Outcome::kNotRun;
+        FailureCategory category = FailureCategory::kNone;
+        FailureHistory history = FailureHistory::kUnknown;
+        /// It failed and then passed with the same inputs.
+        bool flaky = false;
+        /// It is in quarantine, so its failure does not fail the run.
+        bool quarantined = false;
+        /// Why it was selected, when not every test runs: "failed last
+        /// run", "covers a changed file", "dependency of ...".
+        std::string selectionReason;
+        /// A stable identifier of the failure: its category, location and
+        /// message with numbers removed. Tests that share it probably share a
+        /// cause.
+        std::string signature;
+        /// What the runner found out about the failure, one finding per
+        /// entry, most useful first.
+        std::vector<std::string> analysis;
+        /// A command line that runs the failure again the way it happened.
+        std::string replayCommand;
+        std::vector<Attempt> attempts;
+        std::int64_t predictedDurationMicroseconds = 0;
+        double predictedFailureProbability = 0.0;
+
+        /// The sum of the attempts' durations.
+        [[nodiscard]] std::int64_t
+        DurationMicroseconds() const noexcept;
+
+        /// Whether the result fails the run.
+        [[nodiscard]] bool
+        FailsRun() const noexcept;
+    };
 
-        [[nodiscard]] std::size_t Passed() const noexcept;
+    /// The totals of a run, by verdict.
+    struct RunSummary final
+    {
+        std::size_t tests = 0;
+        std::size_t passed = 0;
+        std::size_t failed = 0;
+        std::size_t failedAsExpected = 0;
+        std::size_t skipped = 0;
+        std::size_t crashed = 0;
+        std::size_t timedOut = 0;
+        std::size_t blocked = 0;
+        std::size_t notRun = 0;
+        std::size_t upToDate = 0;
+        std::size_t flaky = 0;
+        std::size_t quarantined = 0;
+        std::size_t attempts = 0;
+        std::size_t assertionsPassed = 0;
+        std::size_t assertionsFailed = 0;
+        /// Wall-clock time of the run, and the time its tests took added up;
+        /// their ratio is the speed-up of running in parallel.
+        std::int64_t wallMicroseconds = 0;
+        std::int64_t testMicroseconds = 0;
     };
 
-    /// Validates durations and statuses while aggregating nested test results.
-    [[nodiscard]] TestRunSummary
-    SummarizeTestSuite( const TestSuiteResult& result );
-    [[nodiscard]] TestRunSummary
-    SummarizeTestRun( const TestRunResult& result );
+    /// A whole run of one test binary.
+    struct RunResult final
+    {
+        std::string binary;
+        std::uint32_t seed = 0;
+        std::size_t workers = 0;
+        /// The shard, when the run was one of several.
+        unsigned shardIndex = 0;
+        unsigned shardCount = 1;
+        /// The tests that matched before selection narrowed them.
+        std::size_t candidates = 0;
+        std::vector<TestResult> tests;
+        /// Test specs that matched nothing.
+        std::vector<std::string> unmatchedSpecs;
+        /// Messages about the run itself, such as a history that could not
+        /// be read.
+        std::vector<std::string> notes;
+        std::int64_t startUnixMilliseconds = 0;
+        std::int64_t wallMicroseconds = 0;
+    };
+
+    [[nodiscard]] RunSummary
+    Summarize(const RunResult &run);
+
+    /// The name an outcome has in reports: "passed", "timed-out", ...
+    [[nodiscard]] const char *
+    OutcomeName(Outcome outcome) noexcept;
+
+    [[nodiscard]] std::optional<Outcome>
+    OutcomeFromName(const std::string &name) noexcept;
+
+    [[nodiscard]] const char *
+    CategoryName(FailureCategory category) noexcept;
+
+    [[nodiscard]] std::optional<FailureCategory>
+    CategoryFromName(const std::string &name) noexcept;
+
+    [[nodiscard]] const char *
+    HistoryName(FailureHistory history) noexcept;
+
+    /// The failure of an attempt as text: each failed assertion with its
+    /// expansion, messages and sections, or what is known about a crash or a
+    /// timeout. Empty for an attempt that did not fail.
+    [[nodiscard]] std::string
+    DescribeFailure(const Attempt &attempt);
+
+    /// The first line of `DescribeFailure`, for a summary.
+    [[nodiscard]] std::string
+    SummarizeFailure(const Attempt &attempt);
 
+    /// Whether an outcome counts as a failure of the attempt.
+    [[nodiscard]] bool
+    IsFailure(Outcome outcome) noexcept;
 } // namespace Progmasoft::Catch3
diff --git a/src/Progmasoft/Catch3/Runner.hpp b/src/Progmasoft/Catch3/Runner.hpp
new file mode 100644
index 00000000..01d64807
--- /dev/null
+++ b/src/Progmasoft/Catch3/Runner.hpp
@@ -0,0 +1,33 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+#pragma once
+
+#include <filesystem>
+#include <string>
+#include <vector>
+
+namespace Progmasoft::Catch3
+{
+    /// Runs the tests of this binary as `arguments` say and returns the exit
+    /// code. `arguments` starts with the program, as `argv` does.
+    ///
+    /// `Catch::Session::run` and the `main` of `Catch2::Catch2WithMain` call
+    /// this; a test binary with its own `main` may call it directly.
+    [[nodiscard]] int
+    RunTests(const std::vector<std::string> &arguments);
+
+    /// Prints the options of the runner to standard output.
+    void
+    PrintHelp(const std::string &programName);
+
+    /// The directory the running test may keep temporary files in.
+    ///
+    /// Every attempt of every test gets its own, so tests that run at the
+    /// same time in different workers never share a file by accident. The
+    /// runner also points TMPDIR, TMP and TEMP at it while the test runs, so
+    /// `std::filesystem::temp_directory_path()` and `tmpfile` use it too.
+    /// It is removed after a passing attempt and kept after a failing one,
+    /// for inspection. Outside a test it is the system temporary directory.
+    [[nodiscard]] std::filesystem::path
+    TestTemporaryDirectory();
+} // namespace Progmasoft::Catch3
diff --git a/src/Progmasoft/Catch3/Version.hpp b/src/Progmasoft/Catch3/Version.hpp
new file mode 100644
index 00000000..982d583c
--- /dev/null
+++ b/src/Progmasoft/Catch3/Version.hpp
@@ -0,0 +1,9 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+#pragma once
+
+namespace Progmasoft::Catch3
+{
+    /// The version of Catch3: of its runner, its tools and its formats.
+    inline constexpr const char *kVersion = "0.3.0";
+} // namespace Progmasoft::Catch3
diff --git a/src/Progmasoft/Catch3/XmlWriter.hpp b/src/Progmasoft/Catch3/XmlWriter.hpp
index 3216b5ae..d8177eea 100644
--- a/src/Progmasoft/Catch3/XmlWriter.hpp
+++ b/src/Progmasoft/Catch3/XmlWriter.hpp
@@ -5,16 +5,27 @@
 #include <iosfwd>
 #include <string>
 
-namespace Progmasoft::Catch3 {
+namespace Progmasoft::Catch3
+{
+    struct RunResult;
 
-    struct TestRunResult;
-
-    /// Emits deterministic JUnit-style XML without timestamps or global state.
-    class JUnitXmlWriter final {
+    /// Writes a run as JUnit XML without timestamps or global state, so the
+    /// same run gives the same bytes.
+    ///
+    /// A failed test gets `<failure>`, a crash, timeout or infrastructure
+    /// problem `<error>`, and earlier failed attempts `<rerunFailure>`. A
+    /// test that failed and then passed is a success with `<flakyFailure>`
+    /// elements, the convention of Maven Surefire that Jenkins, GitLab and
+    /// most CI dashboards read. Skipped, blocked and not-run tests are
+    /// `<skipped>` with the reason as the message.
+    class JUnitXmlWriter final
+    {
     public:
-        static void Write( const TestRunResult& result, std::ostream& output );
+        /// Returns false when the stream failed.
+        [[nodiscard]] static bool
+        Write(const RunResult &run, std::ostream &output);
+
         [[nodiscard]] static std::string
-        ToString( const TestRunResult& result );
+        ToString(const RunResult &run);
     };
-
 } // namespace Progmasoft::Catch3
diff --git a/src/Progmasoft/PropertyEngine.cpp b/src/Progmasoft/PropertyEngine.cpp
index c42b235f..571a5ac3 100644
--- a/src/Progmasoft/PropertyEngine.cpp
+++ b/src/Progmasoft/PropertyEngine.cpp
@@ -2,7 +2,12 @@
 // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
 #include <Progmasoft/Catch3/Strategy/Engine.hpp>
 
+#include <Progmasoft/Runner/Files.hpp>
+
 #include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <cstdio>
 #include <cstdlib>
 #include <filesystem>
 #include <fstream>
@@ -10,6 +15,7 @@
 #include <sstream>
 #include <string>
 #include <system_error>
+#include <thread>
 #include <utility>
 
 namespace Progmasoft::Catch3::Detail
@@ -75,24 +81,53 @@ namespace Progmasoft::Catch3::Detail
                 return tokens;
             }
 
+            /// Changes the stored tokens under a lock: the file is read
+            /// again, `change` edits what it holds, and the result replaces
+            /// the file in one rename. Workers of a parallel run, retries and
+            /// repeats of one property may write at the same time; none of
+            /// them loses what another stored.
+            template<typename Change>
             void
-            Save(const std::vector<std::string> &tokens) const
+            Update(Change change) const
             {
                 if (file_.empty())
                 {
                     return;
                 }
                 std::error_code error;
+                std::filesystem::create_directories(file_.parent_path(), error);
+                const Runner::FileLock lock(file_, std::chrono::seconds(2),
+                                            std::chrono::seconds(10));
+                auto tokens = Load();
+                change(tokens);
                 if (tokens.empty())
                 {
                     std::filesystem::remove(file_, error);
                     return;
                 }
-                std::filesystem::create_directories(file_.parent_path(), error);
-                std::ofstream output(file_, std::ios::trunc);
-                for (const auto &token : tokens)
+                const auto temporary = std::filesystem::path(
+                    file_.string() + "." + Runner::UniqueName() + ".tmp");
+                {
+                    std::ofstream output(temporary,
+                                         std::ios::binary | std::ios::trunc);
+                    for (const auto &token : tokens)
+                    {
+                        output << token << '\n';
+                    }
+                    if (!output)
+                    {
+                        output.close();
+                        std::filesystem::remove(temporary, error);
+                        return;
+                    }
+                }
+                // rename replaces the target atomically on every platform
+                // the library supports, so a reader sees the old or the new
+                // file, never a half-written one.
+                std::filesystem::rename(temporary, file_, error);
+                if (error)
                 {
-                    output << token << '\n';
+                    std::filesystem::remove(temporary, error);
                 }
             }
 
@@ -243,8 +278,7 @@ namespace Progmasoft::Catch3::Detail
             std::optional<PropertyResult>
             ReplayDatabase()
             {
-                auto tokens = database_.Load();
-                std::vector<std::string> kept;
+                const auto tokens = database_.Load();
                 for (const auto &token : tokens)
                 {
                     const auto decoded = DecodeChoices(token);
@@ -265,7 +299,14 @@ namespace Progmasoft::Catch3::Detail
                 }
                 if (!tokens.empty())
                 {
-                    database_.Save(kept);
+                    // Only the examples checked here are dropped; another
+                    // process may have stored new ones meanwhile.