Merge pull request #15 from Progmasoft/feature/catch3-runner

Replace Catch2's runner with the Catch3 runner

10 files changed, 2902 insertions, 144 deletions

diff --git a/tests/Progmasoft/BUILD.bazel b/tests/Progmasoft/BUILD.bazel
index 853d99d4..b5fb21d8 100644
--- a/tests/Progmasoft/BUILD.bazel
+++ b/tests/Progmasoft/BUILD.bazel
@@ -1,6 +1,6 @@
 # Copyright (c) 2026 Progmasoft.
 # SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
-load("@rules_cc//cc:defs.bzl", "cc_test")
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_test")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -11,6 +11,13 @@ cc_test(
         "Generators.tests.cpp",
         "Property.tests.cpp",
         "Results.tests.cpp",
+        "Runner.tests.cpp",
+        "RunnerFixtureClient.hpp",
+        "RunnerUnits.tests.cpp",
+        "RunnerPlanner.tests.cpp",
+        "RunnerOptions.tests.cpp",
+        "RunnerAnalysis.tests.cpp",
+        "RunnerOutput.tests.cpp",
         "Snapshot.tests.cpp",
         "Stateful.tests.cpp",
         "Strategy.tests.cpp",
@@ -19,9 +26,26 @@ cc_test(
         "@platforms//os:windows": ["/std:c++20"],
         "//conditions:default": ["-std=c++20"],
     }),
+    args = ["--no-history"],
+    data = [":runner_fixture"],
+    env = {"CATCH3_RUNNER_FIXTURE": "$(rootpath :runner_fixture)"},
     deps = [
         "//:catch2",
         "//:catch2_main",
         "//src/Progmasoft:catch3",
     ],
 )
+
+# Tests that misbehave on purpose, for the runner's integration tests.
+cc_binary(
+    name = "runner_fixture",
+    srcs = ["RunnerFixture.cpp"],
+    copts = select({
+        "@platforms//os:windows": ["/std:c++20"],
+        "//conditions:default": ["-std=c++20"],
+    }),
+    deps = [
+        "//:catch2",
+        "//:catch2_main",
+    ],
+)
diff --git a/tests/Progmasoft/Results.tests.cpp b/tests/Progmasoft/Results.tests.cpp
index 91b06856..1216fdd2 100644
--- a/tests/Progmasoft/Results.tests.cpp
+++ b/tests/Progmasoft/Results.tests.cpp
@@ -1,165 +1,239 @@
 // Copyright (c) 2026 Progmasoft.
 // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
-#include <Progmasoft/Catch3.hpp>
+#include <Progmasoft/Catch3/Results.hpp>
+#include <Progmasoft/Catch3/XmlWriter.hpp>
+
+#include <catch2/catch_test_macros.hpp>
 
-#include <cmath>
-#include <ios>
-#include <limits>
 #include <sstream>
-#include <stdexcept>
 #include <string>
-#include <utility>
-
-namespace {
-
-    Progmasoft::Catch3::TestCaseResult
-    MakeCase( std::string name,
-              Progmasoft::Catch3::TestCaseStatus status,
-              double duration ) {
-        Progmasoft::Catch3::TestCaseResult testCase;
-        testCase.Name = std::move( name );
-        testCase.Status = status;
-        testCase.DurationSeconds = duration;
-        return testCase;
-    }
 
-} // namespace
-
-TEST_CASE( "Test result summaries count outcomes and duration",
-           "[progmasoft][results]" ) {
+namespace
+{
     using namespace Progmasoft::Catch3;
 
-    TestSuiteResult suite;
-    suite.Name = "Parser";
-    suite.TestCases = {
-        MakeCase( "accepts input", TestCaseStatus::kPassed, 0.125 ),
-        MakeCase( "rejects malformed input", TestCaseStatus::kFailed, 0.250 ),
-        MakeCase( "handles I/O", TestCaseStatus::kErrored, 0.500 ),
-        MakeCase( "optional backend", TestCaseStatus::kSkipped, 0.0 ),
-    };
-    const TestRunResult run{ "Compiler", { suite } };
-
-    const TestRunSummary summary = SummarizeTestRun( run );
-    CHECK( summary.Tests == 4 );
-    CHECK( summary.Passed() == 1 );
-    CHECK( summary.Failures == 1 );
-    CHECK( summary.Errors == 1 );
-    CHECK( summary.Skipped == 1 );
-    CHECK( summary.DurationSeconds == 0.875 );
-}
+    Attempt
+    MakeAttempt(Outcome outcome,
+                std::int64_t microseconds,
+                std::size_t number = 1)
+    {
+        Attempt attempt;
+        attempt.outcome = outcome;
+        attempt.number = number;
+        attempt.durationMicroseconds = microseconds;
+        if (IsFailure(outcome))
+        {
+            attempt.category = outcome == Outcome::kCrashed
+                                   ? FailureCategory::kCrash
+                               : outcome == Outcome::kTimedOut
+                                   ? FailureCategory::kTimeout
+                                   : FailureCategory::kAssertion;
+        }
+        return attempt;
+    }
 
-TEST_CASE( "JUnit XML writer serializes counts, details, and escaped text",
-           "[progmasoft][results][xml]" ) {
-    using namespace Progmasoft::Catch3;
+    TestResult
+    MakeTest(const std::string &name, Outcome outcome, std::int64_t microseconds)
+    {
+        TestResult test;
+        test.id = name;
+        test.name = name;
+        test.file = "tests/parser.cpp";
+        test.line = 12;
+        test.outcome = outcome;
+        test.attempts.push_back(MakeAttempt(outcome, microseconds));
+        test.category = test.attempts.back().category;
+        return test;
+    }
+} // namespace
 
-    TestSuiteResult suite;
-    suite.Name = "Parser & resolver";
-
-    auto failed =
-        MakeCase( "rejects <input>", TestCaseStatus::kFailed, 0.0125 );
-    failed.ClassName = "CompilerTests";
-    failed.SourceFile = "tests/parser.cpp";
-    failed.Message = "assertion: \"expected\" & actual";
-    failed.Details = "expected <node> but got &lt;none>";
-    failed.StandardOutput = "trace <start>&finish";
-    suite.TestCases.push_back( std::move( failed ) );
-
-    auto errored = MakeCase( "throws", TestCaseStatus::kErrored, 0.25 );
-    errored.Message = "runtime error";
-    errored.Details = "first line\nsecond line";
-    errored.StandardError = "failure detail";
-    suite.TestCases.push_back( std::move( errored ) );
-
-    auto skipped = MakeCase( "platform-only", TestCaseStatus::kSkipped, 0.0 );
-    skipped.Message = "not available";
-    suite.TestCases.push_back( std::move( skipped ) );
-
-    suite.TestCases.push_back(
-        MakeCase( "passes", TestCaseStatus::kPassed, 0.001 ) );
-    const TestRunResult run{ "Compiler run", { std::move( suite ) } };
-
-    const std::string xml = JUnitXmlWriter::ToString( run );
-    CHECK( xml.starts_with( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" ) );
-    CHECK( xml.find( "<testsuites name=\"Compiler run\" tests=\"4\" " ) !=
-           std::string::npos );
-    CHECK( xml.find( "failures=\"1\" errors=\"1\" skipped=\"1\"" ) !=
-           std::string::npos );
-    CHECK( xml.find( "name=\"Parser &amp; resolver\"" ) != std::string::npos );
-    CHECK( xml.find( "name=\"rejects &lt;input&gt;\"" ) != std::string::npos );
-    CHECK( xml.find(
-               "message=\"assertion: &quot;expected&quot; &amp; actual\"" ) !=
-           std::string::npos );
-    CHECK( xml.find( "expected &lt;node&gt; but got &amp;lt;none&gt;" ) !=
-           std::string::npos );
-    CHECK(
-        xml.find( "<system-out>trace &lt;start&gt;&amp;finish</system-out>" ) !=
-        std::string::npos );
-    CHECK( xml.find( "<error message=\"runtime error\" type=\"TestError\">" ) !=
-           std::string::npos );
-    CHECK( xml.find( "<skipped message=\"not available\"/>" ) !=
-           std::string::npos );
-    CHECK( xml.find( "classname=\"CompilerTests\"" ) != std::string::npos );
-    CHECK( xml.find( "file=\"tests/parser.cpp\"" ) != std::string::npos );
-    CHECK( xml.find( "time=\"0.263500\"" ) != std::string::npos );
-    CHECK( xml.ends_with( "</testsuites>\n" ) );
+TEST_CASE("A run summary counts every verdict once",
+          "[progmasoft][results]")
+{
+    RunResult run;
+    run.binary = "parser-tests";
+    run.wallMicroseconds = 2000000;
+    run.tests.push_back(MakeTest("parses", Outcome::kPassed, 125000));
+    run.tests.push_back(MakeTest("reports", Outcome::kFailed, 250000));
+    run.tests.push_back(MakeTest("loads", Outcome::kCrashed, 500000));
+    run.tests.push_back(MakeTest("waits", Outcome::kTimedOut, 1000000));
+    auto flaky = MakeTest("retries", Outcome::kPassed, 0);
+    flaky.flaky = true;
+    flaky.attempts.insert(flaky.attempts.begin(),
+                          MakeAttempt(Outcome::kFailed, 10000));
+    run.tests.push_back(flaky);
+    auto quarantined = MakeTest("quarantined", Outcome::kFailed, 1);
+    quarantined.quarantined = true;
+    run.tests.push_back(quarantined);
+
+    const RunSummary summary = Summarize(run);
+    CHECK(summary.tests == 6);
+    CHECK(summary.passed == 2);
+    CHECK(summary.failed == 2);
+    CHECK(summary.crashed == 1);
+    CHECK(summary.timedOut == 1);
+    CHECK(summary.flaky == 1);
+    CHECK(summary.quarantined == 1);
+    CHECK(summary.attempts == 7);
+    CHECK(summary.testMicroseconds == 1885001);
+
+    CHECK(run.tests[1].FailsRun());
+    CHECK_FALSE(run.tests[4].FailsRun());
+    CHECK_FALSE(run.tests[5].FailsRun()); // In quarantine.
 }
 
-TEST_CASE( "JUnit XML replaces malformed UTF-8 and forbidden XML controls",
-           "[progmasoft][results][xml]" ) {
-    using namespace Progmasoft::Catch3;
-
-    auto testCase = MakeCase( "valid", TestCaseStatus::kPassed, 0.0 );
-    testCase.Name = std::string( "bad\x01utf8\xF0\x28\x8C\x28", 12 );
-    const TestRunResult run{ std::string( "suite\0name", 10 ),
-                             { { "suite", { testCase } } } };
-
-    const std::string xml = JUnitXmlWriter::ToString( run );
-    CHECK( xml.find( "name=\"bad&#xFFFD;utf8&#xFFFD;(&#xFFFD;(\"" ) !=
-           std::string::npos );
-    CHECK( xml.find( "name=\"suite&#xFFFD;name\"" ) != std::string::npos );
+TEST_CASE("Outcome and category names read back", "[progmasoft][results]")
+{
+    for (const Outcome outcome : {Outcome::kPassed,
+                                  Outcome::kFailed,
+                                  Outcome::kFailedAsExpected,
+                                  Outcome::kSkipped,
+                                  Outcome::kCrashed,
+                                  Outcome::kTimedOut,
+                                  Outcome::kBlocked,
+                                  Outcome::kNotRun,
+                                  Outcome::kUpToDate})
+    {
+        CHECK(OutcomeFromName(OutcomeName(outcome)) == outcome);
+    }
+    CHECK(CategoryFromName(CategoryName(FailureCategory::kTimeout)) ==
+          FailureCategory::kTimeout);
+    CHECK_FALSE(OutcomeFromName("exploded").has_value());
 }
 
-TEST_CASE( "Result summaries reject invalid durations and statuses",
-           "[progmasoft][results]" ) {
-    using namespace Progmasoft::Catch3;
+TEST_CASE("A failure is described with its expansion, messages and sections",
+          "[progmasoft][results]")
+{
+    Attempt attempt = MakeAttempt(Outcome::kFailed, 1000);
+    AssertionRecord assertion;
+    assertion.file = "parser.cpp";
+    assertion.line = 40;
+    assertion.macro = "REQUIRE";
+    assertion.expression = "tokens.size() == 3";
+    assertion.expansion = "2 == 3";
+    assertion.message = "the comma was lost";
+    assertion.context = {"input := a,b"};
+    assertion.sections = {"lists", "with commas"};
+    attempt.assertions.push_back(assertion);
+
+    const std::string text = DescribeFailure(attempt);
+    CHECK(text.find("parser.cpp:40: FAILED:") != std::string::npos);
+    CHECK(text.find("REQUIRE( tokens.size() == 3 )") != std::string::npos);
+    CHECK(text.find("with expansion:\n  2 == 3") != std::string::npos);
+    CHECK(text.find("with message:\n  the comma was lost") != std::string::npos);
+    CHECK(text.find("with context:\n  input := a,b") != std::string::npos);
+    CHECK(text.find("in section lists / with commas") != std::string::npos);
+    CHECK(SummarizeFailure(attempt) == "parser.cpp:40: FAILED:");
+}
 
-    TestCaseResult invalidDuration =
-        MakeCase( "invalid", TestCaseStatus::kPassed, -1.0 );
-    CHECK_THROWS_AS(
-        SummarizeTestRun( { "run", { { "suite", { invalidDuration } } } } ),
-        std::invalid_argument );
-
-    invalidDuration.DurationSeconds = std::numeric_limits<double>::infinity();
-    CHECK_THROWS_AS(
-        SummarizeTestRun( { "run", { { "suite", { invalidDuration } } } } ),
-        std::invalid_argument );
-
-    invalidDuration.DurationSeconds = 0.0;
-    invalidDuration.Status = static_cast<TestCaseStatus>( 255 );
-    CHECK_THROWS_AS(
-        SummarizeTestRun( { "run", { { "suite", { invalidDuration } } } } ),
-        std::invalid_argument );
+TEST_CASE("A crash and a timeout say where they stopped",
+          "[progmasoft][results]")
+{
+    Attempt crash = MakeAttempt(Outcome::kCrashed, 1000);
+    crash.exitDescription = "signal 11 (SIGSEGV)";
+    crash.fatalMessage = "SIGSEGV - Segmentation violation signal";
+    crash.lastLocation = "parser.cpp:51";
+    SectionRecord open;
+    open.name = "deep input";
+    open.durationMicroseconds = -1;
+    crash.sections.push_back(open);
+    const std::string crashText = DescribeFailure(crash);
+    CHECK(crashText.find("crashed with signal 11 (SIGSEGV)") !=
+          std::string::npos);
+    CHECK(crashText.find("Segmentation violation") != std::string::npos);
+    CHECK(crashText.find("last assertion started at parser.cpp:51") !=
+          std::string::npos);
+    CHECK(crashText.find("in section deep input") != std::string::npos);
+
+    const Attempt timeout = MakeAttempt(Outcome::kTimedOut, 2500000);
+    CHECK(DescribeFailure(timeout).find("timed out after 2.5 s") !=
+          std::string::npos);
 }
 
-TEST_CASE( "JUnit XML writer validates results and output stream state",
-           "[progmasoft][results][xml]" ) {
-    using namespace Progmasoft::Catch3;
+TEST_CASE("JUnit XML keeps every attempt and marks flaky tests",
+          "[progmasoft][results][xml]")
+{
+    RunResult run;
+    run.binary = "parser-tests";
+    run.seed = 42;
+    run.workers = 4;
+    run.wallMicroseconds = 263500;
+
+    auto failed = MakeTest("reports <errors>", Outcome::kFailed, 263500);
+    failed.className = "ParserTests";
+    failed.attempts.back().standardOutput = "trace <start>&finish";
+    AssertionRecord assertion;
+    assertion.file = "parser.cpp";
+    assertion.line = 7;
+    assertion.expression = "ok";
+    failed.attempts.back().assertions.push_back(assertion);
+    failed.analysis.push_back("new failure after 3 passing run(s)");
+    failed.replayCommand = "parser-tests \"reports <errors>\" --seed=42";
+    run.tests.push_back(failed);
+
+    auto crashed = MakeTest("loads", Outcome::kCrashed, 1);
+    crashed.attempts.back().exitDescription = "signal 6 (SIGABRT)";
+    run.tests.push_back(crashed);
+
+    auto flaky = MakeTest("retries", Outcome::kPassed, 10);
+    flaky.flaky = true;
+    flaky.attempts.insert(flaky.attempts.begin(),
+                          MakeAttempt(Outcome::kTimedOut, 5000000));
+    run.tests.push_back(flaky);
+
+    auto blocked = MakeTest("needs the parser", Outcome::kBlocked, 0);
+    blocked.attempts.clear();
+    blocked.analysis.push_back("depends on 'reports <errors>', which failed");
+    run.tests.push_back(blocked);
+
+    const std::string xml = JUnitXmlWriter::ToString(run);
+    CHECK(xml.rfind("<?xml version=\"1.0\" encoding=\"UTF-8\"?>", 0) == 0);
+    CHECK(xml.find("tests=\"4\" failures=\"1\" errors=\"1\" skipped=\"1\"") !=
+          std::string::npos);
+    CHECK(xml.find("name=\"reports &lt;errors&gt;\"") != std::string::npos);
+    CHECK(xml.find("classname=\"ParserTests\"") != std::string::npos);
+    CHECK(xml.find("<failure message=\"parser.cpp:7: FAILED:\" "
+                   "type=\"assertion\">") != std::string::npos);
+    CHECK(xml.find("analysis: new failure after 3 passing run(s)") !=
+          std::string::npos);
+    CHECK(xml.find("<system-out>trace &lt;start&gt;&amp;finish</system-out>") !=
+          std::string::npos);
+    CHECK(xml.find("<error message=\"crashed with signal 6 (SIGABRT)\" "
+                   "type=\"crash\">") != std::string::npos);
+    CHECK(xml.find("<flakyFailure message=\"timed out after 5.0 s\" "
+                   "type=\"timeout\">") != std::string::npos);
+    CHECK(xml.find("<skipped message=\"depends on &apos;reports &lt;errors&gt;"
+                   "&apos;, which failed\"/>") != std::string::npos);
+    CHECK(xml.find("<property name=\"random-seed\" value=\"42\"/>") !=
+          std::string::npos);
+    CHECK(xml.find("time=\"0.263500\"") != std::string::npos);
+    CHECK(xml.ends_with("</testsuites>\n"));
+}
 
-    const TestRunResult emptyRun{ "empty", {} };
-    const std::string emptyXml = JUnitXmlWriter::ToString( emptyRun );
-    CHECK( emptyXml.find( "tests=\"0\" failures=\"0\" errors=\"0\" " ) !=
-           std::string::npos );
+TEST_CASE("JUnit XML replaces malformed UTF-8 and forbidden XML controls",
+          "[progmasoft][results][xml]")
+{
+    RunResult run;
+    run.binary = std::string("suite\0name", 10);
+    auto test = MakeTest("valid", Outcome::kPassed, 0);
+    test.name = std::string("bad\x01utf8\xF0\x28\x8C\x28", 12);
+    run.tests.push_back(test);
+
+    const std::string xml = JUnitXmlWriter::ToString(run);
+    CHECK(xml.find("name=\"bad&#xFFFD;utf8&#xFFFD;(&#xFFFD;(\"") !=
+          std::string::npos);
+    CHECK(xml.find("name=\"suite&#xFFFD;name\"") != std::string::npos);
+}
 
-    auto invalidCase = MakeCase( "bad", TestCaseStatus::kPassed, 0.0 );
-    invalidCase.DurationSeconds = std::numeric_limits<double>::quiet_NaN();
-    const TestRunResult invalidRun{ "invalid",
-                                    { { "suite", { invalidCase } } } };
+TEST_CASE("The JUnit writer reports a failed stream", "[progmasoft][results][xml]")
+{
+    const RunResult empty;
     std::ostringstream output;
-    CHECK_THROWS_AS( JUnitXmlWriter::Write( invalidRun, output ),
-                     std::invalid_argument );
+    CHECK(JUnitXmlWriter::Write(empty, output));
+    CHECK(output.str().find("tests=\"0\" failures=\"0\" errors=\"0\" ") !=
+          std::string::npos);
 
-    std::ostringstream failedOutput;
-    failedOutput.setstate( std::ios::badbit );
-    CHECK_THROWS_AS( JUnitXmlWriter::Write( emptyRun, failedOutput ),
-                     std::ios_base::failure );
+    std::ostringstream failed;
+    failed.setstate(std::ios::badbit);
+    CHECK_FALSE(JUnitXmlWriter::Write(empty, failed));
 }
diff --git a/tests/Progmasoft/Runner.tests.cpp b/tests/Progmasoft/Runner.tests.cpp
new file mode 100644
index 00000000..d458de10
--- /dev/null
+++ b/tests/Progmasoft/Runner.tests.cpp
@@ -0,0 +1,294 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+
+// Starts the runner fixture and checks what the runner made of its tests.
+
+#include "RunnerFixtureClient.hpp"
+
+#include <Progmasoft/Runner/Files.hpp>
+#include <Progmasoft/Runner/Json.hpp>
+#include <Progmasoft/Runner/Process.hpp>
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/matchers/catch_matchers_string.hpp>
+
+#include <chrono>
+#include <cstdlib>
+#include <filesystem>
+#include <set>
+#include <string>
+#include <vector>
+
+namespace
+{
+    namespace Runner = Progmasoft::Catch3::Runner;
+    using Catch::Matchers::ContainsSubstring;
+    using FixtureClient::ExitCode;
+    using FixtureClient::Fixture;
+    using FixtureClient::RunFixture;
+    using FixtureClient::Scratch;
+} // namespace
+
+TEST_CASE("Passing and failing tests get their verdicts", "[progmasoft][runner]")
+{
+    const Scratch scratch;
+    const auto run = RunFixture(scratch, {"--no-history", "[pass],[fail]"});
+    INFO(run.output);
+    CHECK(ExitCode(run) == 42);
+    CHECK(run.report["format"].AsString() == "catch3-run");
+    CHECK(run.Test("fixture passes")["outcome"].AsString() == "passed");
+
+    const auto &fails = run.Test("fixture fails");
+    CHECK(fails["outcome"].AsString() == "failed");
+    CHECK(fails["category"].AsString() == "assertion");
+    CHECK_FALSE(fails["flaky"].AsBoolean());
+    // A failure in a shared process is tried once more alone.
+    CHECK(fails["attempts"].Items().size() == 2);
+    const auto &assertion = fails["attempts"].Items().front()["assertions"].Items().front();
+    CHECK(assertion["expansion"].AsString() == "2 == 3");
+    CHECK(assertion["sections"].Items().front().AsString() == "in a section");
+
+    CHECK(run.Test("fixture throws")["category"].AsString() ==
+          "unexpected-exception");
+
+    const auto &prints = run.Test("fixture prints");
+    CHECK(prints["outcome"].AsString() == "passed");
+    CHECK_THAT(prints["attempts"].Items().front()["standardOutput"].AsString(),
+               ContainsSubstring("said through std::cout"));
+    CHECK_THAT(run.output, ContainsSubstring("FAIL fixture fails"));
+}
+
+TEST_CASE("A crashing test does not take the run with it", "[progmasoft][runner]")
+{
+    const Scratch scratch;
+    const auto run =
+        RunFixture(scratch, {"--no-history", "[crash],[pass]", "--jobs=1"});
+    INFO(run.output);
+    CHECK(ExitCode(run) == 42);
+    const auto &crash = run.Test("fixture crashes");
+    CHECK(crash["outcome"].AsString() == "crashed");
+    CHECK(crash["category"].AsString() == "crash");
+    CHECK_THAT(run.Analysis("fixture crashes"),
+               ContainsSubstring("in section deep down"));
+    // The worker that crashed is replaced; the other tests still ran.
+    CHECK(run.Test("fixture passes")["outcome"].AsString() == "passed");
+    CHECK(run.Test("fixture prints")["outcome"].AsString() == "passed");
+}
+
+TEST_CASE("A hanging test is stopped at its time limit", "[progmasoft][runner]")
+{
+    const Scratch scratch;
+    const auto run =
+        RunFixture(scratch, {"--no-history", "--retries=off", "[hang]"});
+    INFO(run.output);
+    CHECK(ExitCode(run) == 42);
+    const auto &hang = run.Test("fixture hangs");
+    CHECK(hang["outcome"].AsString() == "timed-out");
+    CHECK(hang["category"].AsString() == "timeout");
+    CHECK_THAT(hang["attempts"].Items().front()["lastLocation"].AsString(),
+               ContainsSubstring("RunnerFixture.cpp"));
+    // Well below the 60 s the test sleeps; a slow, busy disk can stretch
+    // the runner's own start-up and Git query to half a minute.
+    CHECK(run.wall < std::chrono::seconds(50));
+}
+
+TEST_CASE("A test that passes on a retry is flaky", "[progmasoft][runner]")
+{
+    const Scratch scratch;
+    const auto run = RunFixture(scratch, {"--no-history", "[flaky]"});
+    INFO(run.output);
+    CHECK(ExitCode(run) == 0);
+    const auto &flaky = run.Test("fixture is flaky");
+    CHECK(flaky["outcome"].AsString() == "passed");
+    CHECK(flaky["flaky"].AsBoolean());
+    CHECK(flaky["attempts"].Items().size() == 2);
+    CHECK_THAT(run.output, ContainsSubstring("FLAKY fixture is flaky"));
+}
+
+TEST_CASE("The test that leaves state behind is found", "[progmasoft][runner]")
+{
+    const Scratch scratch;
+    const auto run = RunFixture(
+        scratch, {"--no-history", "--jobs=1", "--order=declared", "[order]"});
+    INFO(run.output);
+    CHECK(ExitCode(run) == 42);
+    const auto &victim = run.Test("fixture victim");
+    CHECK(victim["outcome"].AsString() == "failed");
+    CHECK_FALSE(victim["flaky"].AsBoolean());
+    CHECK_THAT(run.Analysis("fixture victim"),
+               ContainsSubstring("it fails after 'fixture polluter'"));
+}
+
+TEST_CASE("A test whose dependency fails is blocked", "[progmasoft][runner]")
+{
+    const Scratch scratch;
+    const auto run = RunFixture(scratch, {"--no-history", "fixture dependent"});
+    INFO(run.output);
+    CHECK(ExitCode(run) == 42);
+    const auto &base = run.Test("fixture base");
+    CHECK(base["selectionReason"].AsString() ==
+          "needed by 'fixture dependent'");
+    const auto &dependent = run.Test("fixture dependent");
+    CHECK(dependent["outcome"].AsString() == "blocked");
+    CHECK(dependent["attempts"].Items().empty());
+    CHECK_THAT(run.Analysis("fixture dependent"),
+               ContainsSubstring("depends on 'fixture base', which failed"));
+}
+
+TEST_CASE("Every test gets a temporary directory of its own",
+          "[progmasoft][runner]")
+{
+    const Scratch scratch;
+    const auto passing = RunFixture(scratch, {"--no-history", "[temp]"});
+    INFO(passing.output);
+    CHECK(ExitCode(passing) == 0);
+
+    const auto failing = RunFixture(scratch,
+                                    {"--no-history", "--retries=off", "[temp]"},
+                                    {{"CATCH3_FIXTURE_FAIL_TEMP", "1"}});
+    INFO(failing.output);
+    CHECK(ExitCode(failing) == 42);
+    const std::string kept =
+        failing.Test("fixture temporary directory")["attempts"]
+            .Items()
+            .front()["temporaryDirectory"]
+            .AsString();
+    REQUIRE_FALSE(kept.empty());
+    CHECK(std::filesystem::exists(std::filesystem::path{kept} / "scratch.txt"));
+    std::error_code error;
+    std::filesystem::remove_all(kept, error);
+}
+
+TEST_CASE("Tests run in parallel workers", "[progmasoft][runner]")
+{
+    const Scratch scratch;
+    const auto run = RunFixture(scratch, {"--no-history", "--jobs=4", "[slow]"});
+    INFO(run.output);
+    CHECK(ExitCode(run) == 0);
+    std::set<std::int64_t> workers;
+    for (const auto &test : run.report["tests"].Items())
+    {
+        workers.insert(test["attempts"].Items().front()["worker"].AsInteger());
+    }
+    CHECK(workers.size() >= 2);
+    CHECK(run.report["summary"]["testMicroseconds"].AsInteger() >
+          run.report["wallMicroseconds"].AsInteger() / 2);
+}
+
+TEST_CASE("Shards split the tests without overlap", "[progmasoft][runner]")
+{
+    const Scratch scratch;
+    std::multiset<std::string> seen;
+    for (const char *shard : {"--shard=0/2", "--shard=1/2"})
+    {
+        const auto run = RunFixture(
+            scratch, {"--no-history", "--allow-no-tests", shard, "[slow],[pass]"});
+        INFO(run.output);
+        CHECK(ExitCode(run) == 0);
+        for (const auto &test : run.report["tests"].Items())
+        {
+            seen.insert(test["name"].AsString());
+        }
+    }
+    CHECK(seen.size() == 6);
+    CHECK(std::set<std::string>(seen.begin(), seen.end()).size() == 6);
+}
+
+TEST_CASE("History drives quarantine, incremental runs and replay",
+          "[progmasoft][runner]")
+{
+    const Scratch scratch;
+    const std::string history = "--history=" + (scratch.Path() / "history").string();
+
+    // Flaky twice in a row: into quarantine.
+    for (int run = 0; run < 2; ++run)
+    {
+        const auto result = RunFixture(scratch, {history, "[alternate]"});
+        INFO(result.output);
+        CHECK(result.Test("fixture alternates")["flaky"].AsBoolean());
+    }
+    const auto quarantined = RunFixture(scratch, {history, "[alternate]"});
+    INFO(quarantined.output);
+    CHECK(quarantined.Test("fixture alternates")["quarantined"].AsBoolean());
+
+    // Passed with the same binary: kept as up to date.
+    const auto first = RunFixture(scratch, {history, "--incremental", "[pass]"});
+    CHECK(ExitCode(first) == 0);
+    const auto second = RunFixture(scratch, {history, "--incremental", "[pass]"});
+    INFO(second.output);
+    CHECK(ExitCode(second) == 0);
+    CHECK(second.Test("fixture passes")["outcome"].AsString() == "up-to-date");
+
+    // Without the search for its cause, a failure that does not repeat
+    // alone counts as flaky; the replay runs it in the order it happened,
+    // where it fails again, and the search finds the cause.
+    const auto failed = RunFixture(
+        scratch, {history, "--jobs=1", "--order=declared", "--no-bisect", "[order]"});
+    CHECK(ExitCode(failed) == 0);
+    CHECK(failed.Test("fixture victim")["flaky"].AsBoolean());
+    const auto replay = RunFixture(scratch, {history, "--replay"});
+    INFO(replay.output);
+    CHECK(ExitCode(replay) == 42);
+    CHECK(replay.Test("fixture victim")["outcome"].AsString() == "failed");
+
+    // Analytics read the same history.
+    Runner::ChildOptions analytics;
+    analytics.arguments = {Fixture(), history, "--analytics=json"};
+    auto printed = Runner::RunAndCapture(analytics);
+    REQUIRE(printed);
+    auto json = Runner::ParseJson(printed->output);
+    REQUIRE(json);
+    CHECK((*json)["format"].AsString() == "catch3-analytics");
+    CHECK((*json)["runs"].AsInteger() >= 6);
+}
+
+TEST_CASE("The discovery protocol lists tests with their runner tags",
+          "[progmasoft][runner]")
+{
+    Runner::ChildOptions options;
+    options.arguments = {Fixture(), "--list=json", "[deps],[hang]"};
+    auto listed = Runner::RunAndCapture(options);
+    REQUIRE(listed);
+    INFO(listed->output);
+    auto json = Runner::ParseJson(listed->output);
+    REQUIRE(json);
+    CHECK((*json)["format"].AsString() == "catch3-tests");
+    CHECK((*json)["version"].AsInteger() == 1);
+    REQUIRE((*json)["tests"].Items().size() == 3);
+    bool sawDependency = false;
+    bool sawTimeout = false;
+    for (const auto &test : (*json)["tests"].Items())
+    {
+        CHECK_THAT(test["file"].AsString(), ContainsSubstring("RunnerFixture.cpp"));
+        CHECK(test["line"].AsInteger() > 0);
+        sawDependency = sawDependency || (!test["dependsOn"].Items().empty() &&
+                                          test["dependsOn"].Items().front().AsString() ==
+                                              "fixture base");
+        sawTimeout =
+            sawTimeout || test["timeoutSeconds"].AsNumber() > 0.5;
+    }
+    CHECK(sawDependency);
+    CHECK(sawTimeout);
+}
+
+TEST_CASE("JUnit and trace outputs are written", "[progmasoft][runner]")
+{
+    const Scratch scratch;
+    const auto junit = scratch.Path() / "junit.xml";
+    const auto trace = scratch.Path() / "trace.json";
+    const auto run = RunFixture(scratch,
+                                {"--no-history",
+                                 "[pass],[skip]",
+                                 "--junit=" + junit.string(),
+                                 "--trace=" + trace.string()});
+    INFO(run.output);
+    CHECK(ExitCode(run) == 0);
+    auto xml = Runner::ReadFile(junit);
+    REQUIRE(xml);
+    CHECK_THAT(*xml, ContainsSubstring("<skipped message=\"skipped\"/>"));
+    auto traceText = Runner::ReadFile(trace);
+    REQUIRE(traceText);
+    auto traceJson = Runner::ParseJson(*traceText);
+    REQUIRE(traceJson);
+    CHECK((*traceJson)["traceEvents"].Items().size() >= 3);
+}
diff --git a/tests/Progmasoft/RunnerAnalysis.tests.cpp b/tests/Progmasoft/RunnerAnalysis.tests.cpp
new file mode 100644
index 00000000..453fa671
--- /dev/null
+++ b/tests/Progmasoft/RunnerAnalysis.tests.cpp
@@ -0,0 +1,321 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+
+// What the runner writes down about a failure: how it relates to earlier
+// runs and to other failures, where it stopped, what changed and how to
+// run it again.
+
+#include <Progmasoft/Runner/Analysis.hpp>
+#include <Progmasoft/Runner/Files.hpp>
+#include <Progmasoft/Runner/History.hpp>
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/matchers/catch_matchers_string.hpp>
+#include <catch2/matchers/catch_matchers_vector.hpp>
+
+#include <filesystem>
+#include <string>
+#include <vector>
+
+namespace
+{
+    namespace Runner = Progmasoft::Catch3::Runner;
+    using Catch::Matchers::ContainsSubstring;
+    using Progmasoft::Catch3::AssertionRecord;
+    using Progmasoft::Catch3::Attempt;
+    using Progmasoft::Catch3::FailureCategory;
+    using Progmasoft::Catch3::FailureHistory;
+    using Progmasoft::Catch3::Outcome;
+    using Progmasoft::Catch3::RunResult;
+    using Progmasoft::Catch3::SectionRecord;
+    using Progmasoft::Catch3::TestResult;
+
+    /// A directory of its own, removed at the end of the test.
+    class Scratch final
+    {
+    public:
+        Scratch()
+        {
+            std::error_code error;
+            path_ = std::filesystem::temp_directory_path(error) /
+                    ("catch3-analysis-" + Runner::UniqueName());
+            std::filesystem::create_directories(path_, error);
+        }
+        Scratch(const Scratch &) = delete;
+        Scratch &
+        operator=(const Scratch &) = delete;
+        ~Scratch()
+        {
+            std::error_code error;
+            std::filesystem::remove_all(path_, error);
+        }
+
+        [[nodiscard]] const std::filesystem::path &
+        Path() const noexcept
+        {
+            return path_;
+        }
+
+    private:
+        std::filesystem::path path_;
+    };
+
+    /// A test that failed one assertion.
+    TestResult
+    Failed(const std::string &name,
+           const std::string &message = "wrong",
+           std::size_t line = 10)
+    {
+        TestResult test;<