Merge pull request #15 from Progmasoft/feature/catch3-runner

Replace Catch2's runner with the Catch3 runner

1 files changed, 451 insertions, 0 deletions

diff --git a/tests/Progmasoft/RunnerUnits.tests.cpp b/tests/Progmasoft/RunnerUnits.tests.cpp
new file mode 100644
index 00000000..29e832a5
--- /dev/null
+++ b/tests/Progmasoft/RunnerUnits.tests.cpp
@@ -0,0 +1,451 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+#include <Progmasoft/Runner/Analysis.hpp>
+#include <Progmasoft/Runner/Files.hpp>
+#include <Progmasoft/Runner/Git.hpp>
+#include <Progmasoft/Runner/Hash.hpp>
+#include <Progmasoft/Runner/History.hpp>
+#include <Progmasoft/Runner/Json.hpp>
+#include <Progmasoft/Runner/Options.hpp>
+#include <Progmasoft/Runner/Planner.hpp>
+#include <Progmasoft/Runner/ResultsJson.hpp>
+
+#include <Progmasoft/Catch3.hpp>
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/matchers/catch_matchers_string.hpp>
+
+#include <chrono>
+#include <filesystem>
+#include <set>
+#include <thread>
+#include <vector>
+
+namespace
+{
+    namespace Runner = Progmasoft::Catch3::Runner;
+    using namespace Progmasoft::Catch3;
+    using Catch::Matchers::ContainsSubstring;
+
+    Runner::Options
+    Parse(std::vector<std::string> arguments)
+    {
+        auto options = Runner::ParseOptions(arguments);
+        INFO((options ? std::string{} : options.GetError().Message()));
+        REQUIRE(options);
+        return *options;
+    }
+
+    std::string
+    ParseError(std::vector<std::string> arguments)
+    {
+        auto options = Runner::ParseOptions(arguments);
+        REQUIRE_FALSE(options);
+        return options.GetError().Message();
+    }
+
+    Runner::TestRecord
+    Record(std::int64_t run, Outcome outcome, bool flaky = false)
+    {
+        Runner::TestRecord record;
+        record.run = run;
+        record.outcome = outcome;
+        record.flaky = flaky;
+        record.durationMicroseconds = 1000 * run;
+        record.fingerprint = "same";
+        return record;
+    }
+} // namespace
+
+TEST_CASE("JSON reads back what it writes", "[progmasoft][runner][json]")
+{
+    Runner::JsonValue value = Runner::JsonValue::Object();
+    value.Set("text", "quote \" backslash \\ newline \n tab \t control \x01");
+    value.Set("number", 4294967295U);
+    value.Set("negative", -12);
+    value.Set("fraction", 0.25);
+    value.Set("flag", true);
+    value.Set("nothing", nullptr);
+    Runner::JsonValue list = Runner::JsonValue::Array();
+    list.Append("ü and 日本");
+    list.Append(Runner::JsonValue::Object());
+    value.Set("list", list);
+
+    for (const std::string &text : {value.ToString(), value.ToPrettyString()})
+    {
+        auto parsed = Runner::ParseJson(text);
+        REQUIRE(parsed);
+        CHECK((*parsed)["text"].AsString() == value["text"].AsString());
+        CHECK((*parsed)["number"].AsUnsigned() == 4294967295U);
+        CHECK((*parsed)["negative"].AsInteger() == -12);
+        CHECK((*parsed)["fraction"].AsNumber() == 0.25);
+        CHECK((*parsed)["flag"].AsBoolean());
+        CHECK((*parsed)["nothing"].IsNull());
+        CHECK((*parsed)["list"].Items().front().AsString() == "ü and 日本");
+    }
+    CHECK(value.ToString().find('\n') == std::string::npos);
+
+    auto escaped = Runner::ParseJson(R"("\u00fc\ud83d\ude00")");
+    REQUIRE(escaped);
+    CHECK(escaped->AsString() == "\xC3\xBC\xF0\x9F\x98\x80");
+}
+
+TEST_CASE("JSON refuses what is not JSON", "[progmasoft][runner][json]")
+{
+    for (const char *text : {"", "{", "[1,]", "{\"a\" 1}", "tru", "1 2",
+                             "\"unterminated", "\"\\x\"", "01e"})
+    {
+        CAPTURE(text);
+        CHECK_FALSE(Runner::ParseJson(text));
+    }
+    std::string deep(1000, '[');
+    deep += std::string(1000, ']');
+    CHECK_FALSE(Runner::ParseJson(deep));
+}
+
+TEST_CASE("Any string survives a trip through JSON", "[progmasoft][runner][json]")
+{
+    const auto result = ForAll(Gen::Strings(Gen::Integers<char>(),
+                                            SizeRange{0, 64}),
+                               [](const std::string &text) {
+                                   Runner::JsonValue value{text};
+                                   auto parsed =
+                                       Runner::ParseJson(value.ToString());
+                                   return parsed && parsed->AsString() == text;
+                               });
+    INFO(result.Describe());
+    CHECK(result.Status == PropertyStatus::kPassed);
+}
+
+TEST_CASE("The command line is typed and checked", "[progmasoft][runner][options]")
+{
+    const auto options = Parse({"--jobs=8",
+                                "-j", "3",
+                                "--timeout=1.5",
+                                "--retries=4",
+                                "--shard=2/5",
+                                "--order=declared",
+                                "--seed=17",
+                                "--select=smart",
+                                "-c", "outer",
+                                "--path-filter=g:2",
+                                "[fast]",
+                                "~[slow]",
+                                "--",
+                                "--looks-like-an-option"});
+    CHECK(options.jobs == 3);
+    CHECK(options.timeout == std::chrono::milliseconds{1500});
+    CHECK(options.timeoutGiven);
+    CHECK(options.retryMode == Runner::RetryMode::kAlways);
+    CHECK(options.retries == 4);
+    CHECK(options.shard.index == 2);
+    CHECK(options.shard.count == 5);
+    CHECK(options.order == Runner::Order::kDeclared);
+    CHECK(options.seed == 17);
+    CHECK(options.seedGiven);
+    CHECK(options.selection == Runner::Selection::kSmart);
+    CHECK(options.sections == std::vector<std::string>{"outer"});
+    CHECK(options.paths == std::vector<std::string>{"g:2"});
+    CHECK(options.testSpecs ==
+          std::vector<std::string>{"[fast]", "~[slow]", "--looks-like-an-option"});
+
+    CHECK(Parse({"--fail-fast"}).failFast == 1);
+    CHECK(Parse({"--fail-fast=3"}).failFast == 3);
+    CHECK(Parse({"--list=json"}).mode == Runner::Mode::kListTests);
+    CHECK(Parse({"--retries=off"}).retryMode == Runner::RetryMode::kOff);
+}
+
+TEST_CASE("Command line mistakes are explained", "[progmasoft][runner][options]")
+{
+    CHECK_THAT(ParseError({"--jbos=2"}), ContainsSubstring("did you mean --jobs?"));
+    CHECK_THAT(ParseError({"--jobs"}), ContainsSubstring("needs a value"));
+    CHECK_THAT(ParseError({"--jobs=many"}), ContainsSubstring("whole number"));
+    CHECK_THAT(ParseError({"--shard=4/4"}), ContainsSubstring("INDEX below"));
+    CHECK_THAT(ParseError({"--order=sideways"}),
+               ContainsSubstring("priority, declared, name, random"));
+    CHECK_THAT(ParseError({"--success=yes"}), ContainsSubstring("takes no value"));
+    CHECK_THAT(ParseError({"--path-filter=x:1"}), ContainsSubstring("c:SECTION"));
+}
+
+TEST_CASE("A worker is started with what changes how tests behave",
+          "[progmasoft][runner][options]")
+{
+    auto options = Parse({"--success", "--seed=99", "--section=s", "--warn=no-assertions"});
+    const auto arguments = Runner::WorkerArguments(options);
+    auto worker = Runner::ParseOptions(arguments);
+    REQUIRE(worker);
+    CHECK(worker->mode == Runner::Mode::kWorker);
+    CHECK(worker->includeSuccessful);
+    CHECK(worker->seed == 99);
+    CHECK(worker->sections == std::vector<std::string>{"s"});
+    CHECK(worker->warnNoAssertions);
+    CHECK(worker->benchmarkConfidenceInterval == options.benchmarkConfidenceInterval);
+}
+
+TEST_CASE("SHA-256 matches the published test vectors", "[progmasoft][runner][hash]")
+{
+    CHECK(Runner::Sha256Hex("") ==
+          "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
+    CHECK(Runner::Sha256Hex("abc") ==
+          "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
+    CHECK(Runner::Sha256Hex(
+              "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") ==
+          "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1");
+    CHECK(Runner::Sha256Hex(std::string(1000000, 'a')) ==
+          "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0");
+}
+
+TEST_CASE("History predicts failures and finds flakiness", "[progmasoft][runner][history]")
+{
+    Runner::TestHistory fresh;
+    CHECK(fresh.FailureProbability() == 0.3);
+    CHECK_FALSE(fresh.DurationPercentile(0.5).has_value());
+
+    Runner::TestHistory steady;
+    for (int run = 1; run <= 10; ++run)
+    {
+        steady.records.push_back(Record(run, Outcome::kPassed));
+    }
+    CHECK(steady.FailureProbability() < 0.1);
+    CHECK(steady.PassStreak() == 10);
+    CHECK(steady.FlakyEvents(20) == 0);
+    CHECK(steady.DurationPercentile(0.5) == 5000);
+    CHECK(steady.DurationPercentile(1.0) == 10000);
+
+    Runner::TestHistory broken = steady;
+    broken.records.push_back(Record(11, Outcome::kFailed));
+    CHECK(broken.FailureProbability() >= 0.7);
+    CHECK(broken.PassStreak() == 0);
+    // Pass, fail on the same fingerprint: the verdict flipped by itself.
+    CHECK(broken.FlakyEvents(20) == 1);
+
+    Runner::TestHistory flaky;
+    flaky.records.push_back(Record(1, Outcome::kPassed, true));
+    flaky.records.push_back(Record(2, Outcome::kPassed, true));
+    CHECK(flaky.FlakyEvents(20) == 2);
+}
+
+TEST_CASE("History quarantines flaky tests and releases them", "[progmasoft][runner][history]")
+{
+    std::error_code error;
+    const auto directory = std::filesystem::temp_directory_path(error) /
+                           ("catch3-history-" + Runner::UniqueName());
+    const auto file = Runner::HistoryFile(directory, "binary name");
+    CHECK(file.filename() == "binary_name.history.json");
+
+    const auto runWith = [&](bool flakyNow) {
+        RunResult run;
+        TestResult test;
+        test.id = "Suite::test";
+        test.name = "test";
+        test.outcome = Outcome::kPassed;
+        test.flaky = flakyNow;
+        Attempt attempt;
+        attempt.outcome = Outcome::kPassed;
+        test.attempts.push_back(attempt);
+        run.tests.push_back(test);
+        Runner::QuarantinePolicy policy;
+        policy.passesToLeave = 3;
+        auto history = Runner::History::Commit(file, run, {}, {}, policy);
+        REQUIRE(history);
+        return *history;
+    };
+    CHECK_FALSE(runWith(true).Find("Suite::test")->quarantine);
+    CHECK(runWith(true).Find("Suite::test")->quarantine);
+    CHECK(runWith(false).Find("Suite::test")->quarantine);
+    CHECK(runWith(false).Find("Suite::test")->quarantine);
+    const auto released = runWith(false);
+    CHECK_FALSE(released.Find("Suite::test")->quarantine);
+    CHECK(released.Runs().size() == 5);
+    CHECK(released.Runs().back().id == 5);
+
+    auto reloaded = Runner::History::Load(file);
+    REQUIRE(reloaded);
+    CHECK(reloaded->Find("Suite::test")->records.size() == 5);
+    std::filesystem::remove_all(directory, error);
+}
+
+TEST_CASE("A history held by another run is left alone",
+          "[progmasoft][runner][history]")
+{
+    std::error_code error;
+    const auto directory = std::filesystem::temp_directory_path(error) /
+                           ("catch3-history-" + Runner::UniqueName());
+    const auto file = Runner::HistoryFile(directory, "locked");
+    RunResult run;
+    TestResult test;
+    test.id = "test";
+    test.outcome = Outcome::kPassed;
+    run.tests.push_back(test);
+    {
+        const Runner::FileLock held{file, std::chrono::seconds(1)};
+        REQUIRE(held.Held());
+        auto blocked = Runner::History::Commit(
+            file, run, {}, {}, {}, std::chrono::milliseconds(50));
+        REQUIRE_FALSE(blocked);
+        CHECK_THAT(blocked.GetError().Message(),
+                   ContainsSubstring("holds the lock"));
+        CHECK_FALSE(std::filesystem::exists(file));
+    }
+    CHECK(Runner::History::Commit(file, run, {}, {}, {}));
+    std::filesystem::remove_all(directory, error);
+}
+
+TEST_CASE("Runs that finish together all reach the history",
+          "[progmasoft][runner][history]")
+{
+    std::error_code error;
+    const auto directory = std::filesystem::temp_directory_path(error) /
+                           ("catch3-history-" + Runner::UniqueName());
+    const auto file = Runner::HistoryFile(directory, "parallel");
+    std::vector<std::thread> threads;
+    for (int index = 0; index < 8; ++index)
+    {
+        threads.emplace_back([&, index] {
+            RunResult run;
+            TestResult test;
+            test.id = "test " + std::to_string(index);
+            test.outcome = Outcome::kPassed;
+            run.tests.push_back(test);
+            static_cast<void>(Runner::History::Commit(file, run, {}, {}, {}));
+        });
+    }
+    for (auto &thread : threads)
+    {
+        thread.join();
+    }
+    auto history = Runner::History::Load(file);
+    REQUIRE(history);
+    CHECK(history->Runs().size() == 8);
+    CHECK(history->Tests().size() == 8);
+    std::filesystem::remove_all(directory, error);
+}
+
+TEST_CASE("Stable shards do not depend on the order or the machine",
+          "[progmasoft][runner][planner]")
+{
+    CHECK(Runner::StableShard("anything", 1) == 0);
+    std::vector<std::size_t> sizes(4, 0);
+    for (int index = 0; index < 4000; ++index)
+    {
+        const auto shard = Runner::StableShard("test " + std::to_string(index), 4);
+        REQUIRE(shard < 4);
+        ++sizes[shard];
+    }
+    for (const auto size : sizes)
+    {
+        CHECK(size > 800);
+    }
+    CHECK(Runner::StableShard("parser parses", 7) ==
+          Runner::StableShard("parser parses", 7));
+}
+
+TEST_CASE("Failure signatures ignore numbers and addresses",
+          "[progmasoft][runner][analysis]")
+{
+    Attempt first;
+    first.outcome = Outcome::kFailed;
+    first.category = FailureCategory::kAssertion;
+    AssertionRecord assertion;
+    assertion.file = "pool.cpp";
+    assertion.line = 33;
+    assertion.expression = "pool.size() == limit";
+    assertion.message = "slot 17 at 0x7ffd0010 is taken";
+    first.assertions.push_back(assertion);
+    Attempt second = first;
+    second.assertions.front().message = "slot 4 at 0x51a0 is taken";
+    CHECK(Runner::FailureSignature(first) == Runner::FailureSignature(second));
+    Attempt elsewhere = first;
+    elsewhere.assertions.front().line = 34;
+    CHECK(Runner::FailureSignature(first) != Runner::FailureSignature(elsewhere));
+    CHECK_THAT(Runner::FailureSignature(first), ContainsSubstring("sig-"));
+
+    CHECK(Runner::QuoteTestSpec("a, [b] *c") == "a\\, \\[b\\] \\*c");
+}
+
+TEST_CASE("A run report reads back what it wrote", "[progmasoft][runner][json]")
+{
+    RunResult run;
+    run.binary = "tests";
+    run.seed = 7;
+    run.shardCount = 3;
+    run.notes = {"a note"};
+    TestResult test;
+    test.id = "Fixture::adds";
+    test.name = "adds";
+    test.className = "Fixture";
+    test.tags = {"math", "!timeout:3"};
+    test.outcome = Outcome::kTimedOut;
+    test.category = FailureCategory::kTimeout;
+    test.history = FailureHistory::kNew;
+    test.analysis = {"stopped after 3.0 s"};
+    Attempt attempt;
+    attempt.outcome = Outcome::kTimedOut;
+    attempt.category = FailureCategory::kTimeout;
+    attempt.precededBy = {"before"};
+    attempt.lastLocation = "math.cpp:9";
+    SectionRecord section;
+    section.name = "slow";
+    section.durationMicroseconds = -1;
+    attempt.sections.push_back(section);
+    BenchmarkRecord benchmark;
+    benchmark.name = "sum";
+    benchmark.meanNanoseconds = 12.5;
+    attempt.benchmarks.push_back(benchmark);
+    test.attempts.push_back(attempt);
+    run.tests.push_back(test);
+
+    auto json = Runner::ParseJson(Runner::ToJson(run).ToString());
+    REQUIRE(json);
+    auto read = Runner::RunResultFromJson(*json);
+    REQUIRE(read);
+    REQUIRE(read->tests.size() == 1);
+    const auto &back = read->tests.front();
+    CHECK(back.id == test.id);
+    CHECK(back.className == "Fixture");
+    CHECK(back.tags == test.tags);
+    CHECK(back.outcome == Outcome::kTimedOut);
+    CHECK(back.history == FailureHistory::kNew);
+    CHECK(back.analysis == test.analysis);
+    REQUIRE(back.attempts.size() == 1);
+    CHECK(back.attempts.front().precededBy == attempt.precededBy);
+    CHECK(back.attempts.front().sections.front().durationMicroseconds == -1);
+    CHECK(back.attempts.front().benchmarks.front().meanNanoseconds == 12.5);
+    CHECK(read->shardCount == 3);
+    CHECK((*json)["summary"]["timedOut"].AsInteger() == 1);
+
+    Runner::JsonValue future = *json;
+    future.Set("version", 99);
+    CHECK_FALSE(Runner::RunResultFromJson(future));
+}
+
+TEST_CASE("Source files match across spellings", "[progmasoft][runner][git]")
+{
+    CHECK(Runner::SameSourceFile("/work/repo/tests/a.cpp", "tests/a.cpp"));
+    CHECK(Runner::SameSourceFile("../tests/a.cpp", "/work/repo/tests/a.cpp"));
+    CHECK(Runner::SameSourceFile("/work/repo/./tests/../tests/a.cpp",
+                                 "/work/repo/tests/a.cpp"));
+    CHECK_FALSE(Runner::SameSourceFile("/work/repo/tests/a.cpp", "ests/a.cpp"));
+    CHECK_FALSE(Runner::SameSourceFile("/work/repo/tests/a.cpp",
+                                       "/work/repo/tests/b.cpp"));
+}
+
+TEST_CASE("A file lock lets one holder in at a time", "[progmasoft][runner][files]")
+{
+    std::error_code error;
+    const auto file = std::filesystem::temp_directory_path(error) /
+                      ("catch3-lock-" + Runner::UniqueName());
+    {
+        const Runner::FileLock first{file, std::chrono::milliseconds{100}};
+        CHECK(first.Held());
+        const Runner::FileLock second{file, std::chrono::milliseconds{50}};
+        CHECK_FALSE(second.Held());
+    }
+    const Runner::FileLock third{file, std::chrono::milliseconds{100}};
+    CHECK(third.Held());
+
+    REQUIRE(Runner::WriteFileAtomically(file, "first"));
+    REQUIRE(Runner::WriteFileAtomically(file, "second"));
+    CHECK(*Runner::ReadFile(file) == "second");
+    std::filesystem::remove(file, error);
+}