Merge pull request #15 from Progmasoft/feature/catch3-runner

Replace Catch2's runner with the Catch3 runner

36 files changed, 11677 insertions, 0 deletions

diff --git a/src/Progmasoft/Runner/.clang-format b/src/Progmasoft/Runner/.clang-format
new file mode 100644
index 00000000..1063b9ea
--- /dev/null
+++ b/src/Progmasoft/Runner/.clang-format
@@ -0,0 +1,112 @@
+---
+Language: Cpp
+BasedOnStyle: LLVM
+Standard: c++20
+
+UseTab: Never
+IndentWidth: 4
+ContinuationIndentWidth: 4
+ConstructorInitializerIndentWidth: 4
+AccessModifierOffset: -4
+IndentAccessModifiers: false
+ColumnLimit: 80
+
+BreakBeforeBraces: Custom
+BraceWrapping:
+  AfterCaseLabel: true
+  AfterClass: true
+  AfterControlStatement: Always
+  AfterEnum: true
+  AfterExternBlock: true
+  AfterFunction: true
+  AfterNamespace: true
+  AfterStruct: true
+  AfterUnion: true
+  BeforeCatch: true
+  BeforeElse: true
+  BeforeLambdaBody: false
+  BeforeWhile: false
+  IndentBraces: false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: false
+  SplitEmptyNamespace: false
+
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortEnumsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLambdasOnASingleLine: None
+AllowShortLoopsOnASingleLine: false
+AllowShortNamespacesOnASingleLine: false
+AllowShortRecordOnASingleLine: Never
+
+BreakAfterReturnType: All
+
+PackArguments:
+  BinPack: OnePerLine
+PackParameters:
+  BinPack: OnePerLine
+AllowAllArgumentsOnNextLine: false
+AllowAllParametersOfDeclarationOnNextLine: false
+
+BreakBeforeBinaryOperators: All
+BreakBeforeTernaryOperators: true
+AlignOperands: Align
+
+BreakConstructorInitializers: BeforeComma
+PackConstructorInitializers: Never
+BreakInheritanceList: BeforeComma
+
+PointerAlignment: Right
+DerivePointerAlignment: false
+QualifierAlignment: Leave
+
+SpaceBeforeParens: ControlStatements
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCaseColon: false
+SpaceBeforeCtorInitializerColon: false
+SpaceBeforeInheritanceColon: false
+SpaceBeforeCpp11BracedList: false
+SpacesInAngles: Never
+SpacesInSquareBrackets: false
+SpacesInParens: Never
+SpaceInEmptyBraces: Never
+
+Cpp11BracedListStyle: Block
+
+SpaceAfterTemplateKeyword: false
+BreakTemplateDeclarations: Yes
+
+IndentCaseLabels: true
+IndentCaseBlocks: false
+
+NamespaceIndentation: All
+FixNamespaceComments: true
+ShortNamespaceLines: 0
+CompactNamespaces: false
+
+IndentPPDirectives: AfterHash
+PPIndentWidth: 4
+
+IncludeBlocks: Regroup
+SortIncludes: CaseSensitive
+IncludeIsMainRegex: '$^'
+IncludeCategories:
+  - Regex: '^<.*>$'
+    Priority: 1
+    SortPriority: 1
+  - Regex: '^".*"$'
+    Priority: 2
+    SortPriority: 2
+
+MaxEmptyLinesToKeep: 1
+KeepEmptyLines:
+  AtStartOfBlock: false
+
+ReflowComments: true
+SortUsingDeclarations: Lexicographic
+
+LineEnding: DeriveLF
+InsertNewlineAtEOF: true
+...
diff --git a/src/Progmasoft/Runner/Analysis.cpp b/src/Progmasoft/Runner/Analysis.cpp
new file mode 100644
index 00000000..e1435cef
--- /dev/null
+++ b/src/Progmasoft/Runner/Analysis.cpp
@@ -0,0 +1,365 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+#include "Analysis.hpp"
+
+#include "Hash.hpp"
+
+#include <algorithm>
+#include <cctype>
+#include <map>
+
+namespace Progmasoft::Catch3::Runner
+{
+    namespace
+    {
+        /// Replaces what differs between two runs of the same failure:
+        /// numbers, hexadecimal addresses and the temporary directory.
+        std::string
+        Normalise(std::string_view text)
+        {
+            std::string result;
+            bool inNumber = false;
+            for (std::size_t index = 0; index < text.size(); ++index)
+            {
+                const char character = text[index];
+                const bool digit =
+                    std::isdigit(static_cast<unsigned char>(character)) != 0;
+                const bool hexLetter =
+                    inNumber &&
+                    std::isxdigit(static_cast<unsigned char>(character)) != 0;
+                if (digit || hexLetter ||
+                    (inNumber && (character == 'x' || character == '.')))
+                {
+                    if (!inNumber)
+                    {
+                        result.push_back('#');
+                    }
+                    inNumber = true;
+                    continue;
+                }
+                inNumber = false;
+                result.push_back(character);
+            }
+            return result;
+        }
+
+        const Attempt *
+        FirstFailure(const TestResult &test)
+        {
+            for (const auto &attempt : test.attempts)
+            {
+                if (IsFailure(attempt.outcome))
+                {
+                    return &attempt;
+                }
+            }
+            return nullptr;
+        }
+
+        std::string
+        Seconds(std::int64_t microseconds)
+        {
+            const auto tenths = (microseconds + 50000) / 100000;
+            return std::to_string(tenths / 10) + "." +
+                   std::to_string(tenths % 10) + " s";
+        }
+
+        std::string
+        OpenSections(const Attempt &attempt)
+        {
+            std::vector<std::string> open;
+            for (const auto &section : attempt.sections)
+            {
+                if (section.durationMicroseconds < 0)
+                {
+                    open.resize(section.depth);
+                    open.push_back(section.name);
+                }
+            }
+            std::string text;
+            for (const auto &name : open)
+            {
+                text += text.empty() ? "" : " / ";
+                text += name;
+            }
+            return text;
+        }
+
+        std::string
+        QuoteArgument(const std::string &argument)
+        {
+            if (argument.find_first_of(" \t\"'\\$`&|;<>()[]*?~!#") ==
+                    std::string::npos &&
+                !argument.empty())
+            {
+                return argument;
+            }
+            std::string quoted = "\"";
+            for (const char character : argument)
+            {
+                if (character == '"' || character == '\\' || character == '$' ||
+                    character == '`')
+                {
+                    quoted.push_back('\\');
+                }
+                quoted.push_back(character);
+            }
+            quoted.push_back('"');
+            return quoted;
+        }
+    } // namespace
+
+    std::string
+    QuoteTestSpec(const std::string &name)
+    {
+        // Catch2's spec syntax gives these characters a meaning; a backslash
+        // makes them literal.
+        std::string spec;
+        for (const char character : name)
+        {
+            if (character == ',' || character == '[' || character == ']' ||
+                character == '*' || character == '\\' || character == '~' ||
+                character == '"')
+            {
+                spec.push_back('\\');
+            }
+            spec.push_back(character);
+        }
+        return spec;
+    }
+
+    std::string
+    FailureSignature(const Attempt &attempt)
+    {
+        std::string text = CategoryName(attempt.category);
+        text += '|';
+        bool described = false;
+        for (const auto &assertion : attempt.assertions)
+        {
+            if (assertion.passed || assertion.allowed)
+            {
+                continue;
+            }
+            text += assertion.file + ":" + std::to_string(assertion.line) +
+                    "|" + Normalise(assertion.message) + "|" +
+                    assertion.expression;
+            described = true;
+            break;
+        }
+        if (!described)
+        {
+            text += Normalise(attempt.fatalMessage) + "|" +
+                    attempt.lastLocation + "|" + OpenSections(attempt);
+        }
+        return "sig-" + ToHex(Fnv1a64(text)).substr(0, 12);
+    }
+
+    void
+    Analyze(RunResult &run, const AnalysisInputs &inputs)
+    {
+        std::map<std::string, std::vector<std::size_t>> bySignature;
+        for (std::size_t index = 0; index < run.tests.size(); ++index)
+        {
+            TestResult &test = run.tests[index];
+            const Attempt *failure = FirstFailure(test);
+            if (failure == nullptr)
+            {
+                continue;
+            }
+            test.signature = FailureSignature(*failure);
+            bySignature[test.signature].push_back(index);
+        }
+
+        for (std::size_t index = 0; index < run.tests.size(); ++index)
+        {
+            TestResult &test = run.tests[index];
+            const Attempt *failure = FirstFailure(test);
+            if (failure == nullptr)
+            {
+                continue;
+            }
+            std::vector<std::string> findings;
+
+            // How it relates to earlier runs.
+            const TestHistory *past =
+                inputs.history == nullptr ? nullptr
+                                          : inputs.history->Find(test.id);
+            const TestRecord *last = past == nullptr ? nullptr : past->Last();
+            if (last == nullptr)
+            {
+                test.history = FailureHistory::kUnknown;
+            }
+            else if (test.flaky || past->FlakyEvents(20) > 0)
+            {
+                test.history = FailureHistory::kIntermittent;
+                const auto events = past->FlakyEvents(20) + (test.flaky ? 1 : 0);
+                findings.push_back("intermittent: " + std::to_string(events) +
+                                   " flaky result(s) in the last 20 runs");
+            }
+            else if (IsFailure(last->outcome))
+            {
+                test.history = FailureHistory::kPersistent;
+                std::size_t streak = 0;
+                for (auto record = past->records.rbegin();
+                     record != past->records.rend() &&
+                     IsFailure(record->outcome);
+                     ++record)
+                {
+                    ++streak;
+                }
+                std::string finding = "persistent: failed in the last " +
+                                      std::to_string(streak) + " run(s)";
+                if (!last->signature.empty() && last->signature != test.signature)
+                {
+                    finding += ", but differently last time";
+                }
+                findings.push_back(finding);
+            }
+            else
+            {
+                test.history = FailureHistory::kNew;
+                std::string finding = "new failure after " +
+                                      std::to_string(past->PassStreak()) +
+                                      " passing run(s)";
+                if (inputs.history != nullptr)
+                {
+                    for (const auto &runRecord : inputs.history->Runs())
+                    {
+                        if (runRecord.id == last->run && !runRecord.commit.empty())
+                        {
+                            finding += "; it last passed at commit " +
+                                       runRecord.commit.substr(0, 12);
+                        }
+                    }
+                }
+                findings.push_back(finding);
+            }
+
+            // Other tests that fail the same way.
+            const auto &group = bySignature[test.signature];
+            if (group.size() > 1)
+            {
+                std::string others;
+                std::size_t named = 0;
+                for (const std::size_t other : group)
+                {
+                    if (other == index)
+                    {
+                        continue;
+                    }
+                    if (named < 3)
+                    {
+                        others += (named > 0 ? ", '" : "'") +
+                                  run.tests[other].name + "'";
+                    }
+                    ++named;
+                }
+                if (named > 3)
+                {
+                    others += " and " + std::to_string(named - 3) + " more";
+                }
+                findings.push_back("fails the same way as " + others +
+                                   ": probably one cause");
+            }
+
+            // Where it stopped.
+            if (failure->outcome == Outcome::kCrashed)
+            {
+                std::string finding = "crashed";
+                if (!failure->exitDescription.empty())
+                {
+                    finding += " (" + failure->exitDescription + ")";
+                }
+                if (!failure->lastLocation.empty())
+                {
+                    finding += " after the assertion at " + failure->lastLocation;
+                }
+                const std::string sections = OpenSections(*failure);
+                if (!sections.empty())
+                {
+                    finding += ", in section " + sections;
+                }
+                findings.push_back(finding);
+            }
+            else if (failure->outcome == Outcome::kTimedOut)
+            {
+                std::string finding =
+                    "stopped after " + Seconds(failure->durationMicroseconds);
+                if (test.predictedDurationMicroseconds > 0 && last != nullptr)
+                {
+                    finding += "; it usually takes " +
+                               Seconds(test.predictedDurationMicroseconds);
+                }
+                const std::string sections = OpenSections(*failure);
+                if (!sections.empty())
+                {
+                    finding += "; it was in section " + sections;
+                }
+                if (!failure->lastLocation.empty())
+                {
+                    finding += "; last assertion started at " +
+                               failure->lastLocation;
+                }
+                findings.push_back(finding);
+            }
+
+            // The change it follows.
+            if (inputs.changes != nullptr)
+            {
+                std::vector<std::string> changedFiles;
+                const auto changed = [&](const std::string &file) {
+                    return std::any_of(inputs.changes->files.begin(),
+                                       inputs.changes->files.end(),
+                                       [&](const auto &path) {
+                                           return SameSourceFile(path, file);
+                                       });
+                };
+                if (changed(test.file))
+                {
+                    changedFiles.push_back(test.file);
+                }
+                for (const auto &assertion : failure->assertions)
+                {
+                    if (!assertion.passed && assertion.file != test.file &&
+                        changed(assertion.file) &&
+                        std::find(changedFiles.begin(),
+                                  changedFiles.end(),
+                                  assertion.file) == changedFiles.end())
+                    {
+                        changedFiles.push_back(assertion.file);
+                    }
+                }
+                for (const auto &file : changedFiles)
+                {
+                    findings.push_back(
+                        "changed since " +
+                        (inputs.changesSince.empty() ? std::string{"HEAD"}
+                                                     : inputs.changesSince) +
+                        ": " + file);
+                }
+            }
+
+            if (!failure->temporaryDirectory.empty())
+            {
+                findings.push_back("its temporary files are kept in " +
+                                   failure->temporaryDirectory);
+            }
+
+            // Findings of the scheduler (order dependence, flakiness,
+            // quarantine) come first.
+            test.analysis.insert(
+                test.analysis.end(), findings.begin(), findings.end());
+
+            std::string replay = QuoteArgument(inputs.program) + " " +
+                                 QuoteArgument(QuoteTestSpec(test.name)) +
+                                 " --seed=" + std::to_string(failure->seed);
+            if (!failure->precededBy.empty() && !test.flaky)
+            {
+                // The replay mode runs the same tests first, in the same
+                // process, as the failure did.
+                replay = QuoteArgument(inputs.program) + " --replay";
+            }
+            test.replayCommand = replay;
+        }
+    }
+} // namespace Progmasoft::Catch3::Runner
diff --git a/src/Progmasoft/Runner/Analysis.hpp b/src/Progmasoft/Runner/Analysis.hpp
new file mode 100644
index 00000000..a39ea8e6
--- /dev/null
+++ b/src/Progmasoft/Runner/Analysis.hpp
@@ -0,0 +1,41 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+#pragma once
+
+#include "Git.hpp"
+#include "History.hpp"
+#include "Planner.hpp"
+
+#include <Progmasoft/Catch3/Results.hpp>
+
+#include <string>
+
+namespace Progmasoft::Catch3::Runner
+{
+    struct AnalysisInputs final
+    {
+        /// The history before this run.
+        const History *history = nullptr;
+        const Changes *changes = nullptr;
+        const Plan *plan = nullptr;
+        /// The binary, for the replay command.
+        std::string program;
+        std::string changesSince;
+    };
+
+    /// Classifies every failure of the run and writes down what the runner
+    /// can tell about its cause: a signature shared with other failures,
+    /// the failure's history, the change it follows, where a crash or a
+    /// hang happened, and a command that replays it.
+    void
+    Analyze(RunResult &run, const AnalysisInputs &inputs);
+
+    /// The signature of a failed attempt: its category, the location of the
+    /// first failure and its message with numbers and addresses replaced.
+    [[nodiscard]] std::string
+    FailureSignature(const Attempt &attempt);
+
+    /// A test spec that matches exactly this name.
+    [[nodiscard]] std::string
+    QuoteTestSpec(const std::string &name);
+} // namespace Progmasoft::Catch3::Runner
diff --git a/src/Progmasoft/Runner/Catalog.cpp b/src/Progmasoft/Runner/Catalog.cpp
new file mode 100644
index 00000000..c9d838f4
--- /dev/null
+++ b/src/Progmasoft/Runner/Catalog.cpp
@@ -0,0 +1,293 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+#include <algorithm>
+#include <catch2/catch_test_case_info.hpp>
+#include <catch2/catch_test_spec.hpp>
+#include <catch2/interfaces/catch_interfaces_config.hpp>
+#include <catch2/interfaces/catch_interfaces_registry_hub.hpp>
+#include <catch2/interfaces/catch_interfaces_testcase.hpp>
+#include <catch2/internal/catch_test_case_registry_impl.hpp>
+#include <cmath>
+#include <iterator>
+#include <map>
+
+#include "Catalog.hpp"
+#include "Hash.hpp"
+
+namespace Progmasoft::Catch3::Runner
+{
+    namespace
+    {
+        std::optional<double>
+        ParseSeconds(std::string_view text)
+        {
+            if (text.empty())
+            {
+                return std::nullopt;
+            }
+            double value = 0.0;
+            double scale = 0.0;
+            for (const char character : text)
+            {
+                if (character == '.' && scale <= 0.0)
+                {
+                    scale = 1.0;
+                    continue;
+                }
+                if (character < '0' || character > '9')
+                {
+                    return std::nullopt;
+                }
+                if (scale > 0.0)
+                {
+                    scale /= 10.0;
+                    value += (character - '0') * scale;
+                }
+                else
+                {
+                    value = value * 10.0 + (character - '0');
+                }
+            }
+            return value;
+        }
+
+        bool
+        HasPrefix(std::string_view text, std::string_view prefix)
+        {
+            return text.substr(0, prefix.size()) == prefix;
+        }
+
+        Status
+        ApplyRunnerTag(TestEntry &entry, const std::string &tag)
+        {
+            const auto where = [&] {
+                return "test '" + entry.name + "' at " + entry.file + ":"
+                       + std::to_string(entry.line);
+            };
+            if (tag == "!serial")
+            {
+                entry.serial = true;
+            }
+            else if (tag == "!isolated")
+            {
+                entry.isolated = true;
+            }
+            else if (tag == "!quarantine")
+            {
+                entry.quarantined = true;
+            }
+            else if (HasPrefix(tag, "!resource:"))
+            {
+                entry.resources.push_back(tag.substr(10));
+            }
+            else if (HasPrefix(tag, "!depends:"))
+            {
+                entry.dependsOn.push_back(tag.substr(9));
+            }
+            else if (HasPrefix(tag, "!timeout:"))
+            {
+                const auto seconds = ParseSeconds(tag.substr(9));
+                if (!seconds || *seconds > 1e7)
+                {
+                    return Error{ "[" + tag + "] of " + where()
+                                  + " needs a number of seconds, such as "
+                                    "[!timeout:30]" };
+                }
+                entry.timeout = std::chrono::milliseconds{
+                    static_cast<long long>(std::llround(*seconds * 1000.0))
+                };
+            }
+            else if (HasPrefix(tag, "!retries:"))
+            {
+                const auto count = ParseSeconds(tag.substr(9));
+                if (!count || *count > 100 || std::floor(*count) < *count)
+                {
+                    return Error{ "[" + tag + "] of " + where()
+                                  + " needs a whole number up to 100, such as "
+                                    "[!retries:3]" };
+                }
+                entry.retries = static_cast<unsigned>(*count);
+            }
+            return Status::Ok();
+        }
+    } // namespace
+
+    Expected<Catalog>
+    Catalog::Load()
+    {
+        Catalog catalog;
+        const auto &handles
+            = Catch::getRegistryHub().getTestCaseRegistry().getAllTests();
+        catalog.handles_ = handles;
+        // Two tests with the same class, name and tags cannot be told apart
+        // by any filter, history record or report, so they are refused.
+        std::vector<const Catch::TestCaseInfo *> sorted;
+        sorted.reserve(handles.size());
+        for (const auto &handle : handles)
+        {
+            sorted.push_back(&handle.getTestCaseInfo());
+        }
+        std::stable_sort(sorted.begin(),
+                         sorted.end(),
+                         [](const auto *lhs, const auto *rhs) {
+                             return *lhs < *rhs;
+                         });
+        const auto duplicate
+            = std::adjacent_find(sorted.begin(),