Merge pull request #15 from Progmasoft/feature/catch3-runner

Replace Catch2's runner with the Catch3 runner

1 files changed, 1437 insertions, 0 deletions

diff --git a/src/Progmasoft/Runner/Scheduler.cpp b/src/Progmasoft/Runner/Scheduler.cpp
new file mode 100644
index 00000000..fce50f2c
--- /dev/null
+++ b/src/Progmasoft/Runner/Scheduler.cpp
@@ -0,0 +1,1437 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+#include <algorithm>
+#include <chrono>
+#include <condition_variable>
+#include <cstdio>
+#include <deque>
+#include <filesystem>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <set>
+#include <thread>
+
+#include "Execution.hpp"
+#include "Files.hpp"
+#include "Process.hpp"
+#include "Protocol.hpp"
+#include "ResultsJson.hpp"
+#include "Scheduler.hpp"
+#include "SchedulerModel.hpp"
+
+namespace Progmasoft::Catch3::Runner
+{
+    namespace
+    {
+        using Scheduling::AppendBounded;
+        using Scheduling::Bisection;
+        using Scheduling::Clock;
+        using Scheduling::Event;
+        using Scheduling::Job;
+        using Scheduling::kMaximumProbes;
+        using Scheduling::kMaximumProcessOutput;
+        using Scheduling::kMaximumRecordedAssertions;
+        using Scheduling::kStartupFailuresToGiveUp;
+        using Scheduling::kStartupTimeout;
+        using Scheduling::Microseconds;
+        using Scheduling::Passing;
+        using Scheduling::Running;
+        using Scheduling::Satisfies;
+        using Scheduling::TestState;
+        using Scheduling::Worker;
+        using std::chrono::milliseconds;
+
+        class Scheduler final
+        {
+        public:
+            Scheduler(const Options &options,
+                      const Plan &plan,
+                      const Catalog &catalog,
+                      std::string program,
+                      std::string programName,
+                      const SchedulerEvents &events)
+                : options_{ options }
+                , plan_{ plan }
+                , catalog_{ catalog }
+                , program_{ std::move(program) }
+                , programName_{ std::move(programName) }
+                , events_{ events }
+                , runStart_{ Clock::now() }
+            {
+                std::error_code error;
+                temporaryRoot_ = std::filesystem::temp_directory_path(error)
+                                 / ("catch3-" + UniqueName());
+                const unsigned hardware = HardwareThreads();
+                maximumWorkers_ = options.jobs == 0 ? hardware : options.jobs;
+                maximumWorkers_ = std::max<std::size_t>(
+                    1,
+                    std::min(maximumWorkers_, plan.tests.size()));
+                retryBudget_ = std::max<std::size_t>(5, plan.tests.size() / 10);
+                states_.resize(plan.tests.size());
+                for (std::size_t position = 0; position < plan.tests.size();
+                     ++position)
+                {
+                    const PlannedTest &planned = plan.tests[position];
+                    const TestEntry &entry = *planned.entry;
+                    TestResult &result = states_[position].result;
+                    result.id = entry.key;
+                    result.name = entry.name;
+                    result.className = entry.className;
+                    result.file = entry.file;
+                    result.line = entry.line;
+                    result.tags = entry.tags;
+                    result.selectionReason = planned.selectionReason;
+                    result.quarantined = planned.quarantined;
+                    result.predictedDurationMicroseconds
+                        = planned.predictedMicroseconds;
+                    result.predictedFailureProbability
+                        = planned.failureProbability;
+                    if (planned.quarantined)
+                    {
+                        result.analysis.push_back("in quarantine: "
+                                                  + planned.quarantineReason);
+                    }
+                    states_[position].repeatsLeft
+                        = options.repeat > 0 ? options.repeat - 1 : 0;
+                }
+            }
+
+            ScheduleOutcome
+            Run()
+            {
+                for (std::size_t position = 0; position < plan_.tests.size();
+                     ++position)
+                {
+                    Job job;
+                    job.test = position;
+                    const PlannedTest &planned = plan_.tests[position];
+                    if (planned.replaySequence)
+                    {
+                        job.before = *planned.replaySequence;
+                        job.fresh = true;
+                        job.seed = planned.replaySeed;
+                    }
+                    Enqueue(std::move(job));
+                }
+                if (options_.inProcess)
+                {
+                    RunInProcess();
+                }
+                else
+                {
+                    RunInWorkers();
+                }
+                ScheduleOutcome outcome;
+                for (auto &state : states_)
+                {
+                    if (!state.finished)
+                    {
+                        state.result.outcome = Outcome::kNotRun;
+                        if (infrastructureFailed_)
+                        {
+                            state.result.category
+                                = FailureCategory::kInfrastructure;
+                        }
+                    }
+                    outcome.tests.push_back(std::move(state.result));
+                }
+                outcome.workers = workersStarted_;
+                outcome.notes = std::move(notes_);
+                outcome.infrastructureFailed = infrastructureFailed_;
+                std::error_code error;
+                // Only an empty root goes; kept directories hold failures.
+                std::filesystem::remove(temporaryRoot_, error);
+                return outcome;
+            }
+
+        private:
+            // Queue and dispatch.
+
+            void
+            Enqueue(Job job)
+            {
+                ++states_[job.test].pendingJobs;
+                states_[job.test].queued = true;
+                if (job.kind == Job::Kind::kProbe || !job.before.empty()
+                    || states_[job.test].result.attempts.size() > 0)
+                {
+                    // Retries and probes go ahead of first attempts: they
+                    // finish a verdict that is already overdue.
+                    queue_.push_front(std::move(job));
+                }
+                else
+                {
+                    queue_.push_back(std::move(job));
+                }
+            }
+
+            /// Whether the dependencies of a test have their verdicts:
+            /// returns false while one is still open.
+            bool
+            DependenciesSettled(std::size_t position) const
+            {
+                for (const std::size_t dependency :
+                     plan_.tests[position].dependencies)
+                {
+                    if (!states_[dependency].finished)
+                    {
+                        return false;
+                    }
+                }
+                return true;
+            }
+
+            bool
+            ResourcesFree(std::size_t position) const
+            {
+                for (const auto &resource :
+                     plan_.tests[position].entry->resources)
+                {
+                    if (heldResources_.count(resource) > 0)
+                    {
+                        return false;
+                    }
+                }
+                return true;
+            }
+
+            std::size_t
+            RunningCount() const
+            {
+                std::size_t count = 0;
+                for (const auto &worker : workers_)
+                {
+                    count += worker->job.has_value() ? 1 : 0;
+                }
+                return count;
+            }
+
+            /// The next job that may start now, removed from the queue.
+            std::optional<Job>
+            TakeRunnable()
+            {
+                for (auto job = queue_.begin(); job != queue_.end(); ++job)
+                {
+                    const std::size_t position = job->test;
+                    if (job->kind == Job::Kind::kAttempt
+                        && !DependenciesSettled(position))
+                    {
+                        continue;
+                    }
+                    const bool serial = plan_.tests[position].entry->serial;
+                    if (serial && RunningCount() > 0)
+                    {
+                        // Nothing else starts until the serial test has run.
+                        return std::nullopt;
+                    }
+                    if (serialRunning_)
+                    {
+                        return std::nullopt;
+                    }
+                    if (!ResourcesFree(position))
+                    {
+                        continue;
+                    }
+                    Job taken = std::move(*job);
+                    queue_.erase(job);
+                    return taken;
+                }
+                return std::nullopt;
+            }
+
+            /// Settles tests whose dependencies failed, and those left when
+            /// the run stops early.
+            void
+            SettleBlocked()
+            {
+                bool changed = true;
+                while (changed)
+                {
+                    changed = false;
+                    for (auto job = queue_.begin(); job != queue_.end();)
+                    {
+                        const std::size_t position = job->test;
+                        std::string reason;
+                        if (job->kind == Job::Kind::kAttempt)
+                        {
+                            for (const std::size_t dependency :
+                                 plan_.tests[position].dependencies)
+                            {
+                                const auto &state = states_[dependency];
+                                if (state.finished
+                                    && !Satisfies(state.result.outcome))
+                                {
+                                    reason
+                                        = "depends on '" + state.result.name
+                                          + "', which "
+                                          + (state.result.outcome
+                                                     == Outcome::kBlocked
+                                                 ? std::string{ "was blocked" }
+                                                 : std::string{ OutcomeName(
+                                                       state.result.outcome) });
+                                    break;
+                                }
+                            }
+                        }
+                        if (reason.empty())
+                        {
+                            ++job;
+                            continue;
+                        }
+                        auto &state = states_[position];
+                        job = queue_.erase(job);
+                        --state.pendingJobs;
+                        if (state.pendingJobs == 0 && !state.finished
+                            && state.result.attempts.empty())
+                        {
+                            state.result.outcome = Outcome::kBlocked;
+                            state.result.category
+                                = FailureCategory::kDependency;
+                            state.result.analysis.insert(
+                                state.result.analysis.begin(),
+                                reason);
+                            Finish(position);
+                            changed = true;
+                        }
+                    }
+                }
+            }
+
+            std::string
+            TemporaryFor(std::size_t catalogIndex, std::size_t attempt)
+            {
+                return (temporaryRoot_
+                        / ("t" + std::to_string(catalogIndex) + "-a"
+                           + std::to_string(attempt) + "-"
+                           + std::to_string(++temporaryCounter_)))
+                    .string();
+            }
+
+            /// Starts the next step of a worker's job.
+            void
+            StartStep(Worker &worker)
+            {
+                const Job &job = *worker.job;
+                const bool target = worker.step >= job.before.size();
+                const std::size_t catalogIndex
+                    = target ? plan_.tests[job.test].entry->index
+                             : job.before[worker.step];
+                Running running;
+                running.catalogIndex = catalogIndex;
+                running.target = target;
+                running.start = Clock::now();
+                Attempt &attempt = running.attempt;
+                attempt.worker = worker.id;
+                attempt.freshProcess = worker.history.empty();
+                attempt.seed = worker.seed.value_or(options_.seed);
+                attempt.startMicroseconds
+                    = Microseconds(running.start - runStart_);
+                for (const std::size_t before : worker.history)
+                {
+                    attempt.precededBy.push_back(catalog_.Tests()[before].key);
+                }
+                if (target)
+                {
+                    const PlannedTest &planned = plan_.tests[job.test];
+                    attempt.number
+                        = job.kind == Job::Kind::kAttempt
+                              ? states_[job.test].result.attempts.size() + 1
+                              : 0;
+                    if (planned.timeout.count() > 0 && !options_.inProcess)
+                    {
+                        running.deadline = running.start + planned.timeout;
+                    }
+                }
+                attempt.temporaryDirectory
+                    = TemporaryFor(catalogIndex, attempt.number);
+                worker.running = std::move(running);
+                if (options_.inProcess)
+                {
+                    return;
+                }
+                JsonValue command = Message("run");
+                command.Set("index", catalogIndex);
+                command.Set("attempt", worker.running->attempt.number);
+                command.Set("temp", worker.running->attempt.temporaryDirectory);
+                if (!worker.process->Input().Write(command.ToString() + "\n"))
+                {
+                    // The worker died; its closed output reports it.
+                }
+            }
+
+            void
+            Assign(Worker &worker, Job job)
+            {
+                const PlannedTest &planned = plan_.tests[job.test];
+                for (const auto &resource : planned.entry->resources)
+                {
+                    heldResources_.insert(resource);
+                }
+                if (planned.entry->serial)
+                {
+                    serialRunning_ = true;
+                }
+                worker.job = std::move(job);
+                worker.step = 0;
+                StartStep(worker);
+            }
+
+            void
+            ReleaseJob(Worker &worker)
+            {
+                if (!worker.job)
+                {
+                    return;
+                }
+                const PlannedTest &planned = plan_.tests[worker.job->test];
+                for (const auto &resource : planned.entry->resources)
+                {
+                    heldResources_.erase(resource);
+                }
+                if (planned.entry->serial)
+                {
+                    serialRunning_ = false;
+                }
+                worker.job.reset();
+                worker.running.reset();
+            }
+
+            // Results.
+
+            /// An attempt of the job's target ended.
+            void
+            Complete(Worker &worker, Attempt attempt)
+            {
+                Job job = std::move(*worker.job);
+                ReleaseJob(worker);
+                auto &state = states_[job.test];
+                --state.pendingJobs;
+                std::error_code error;
+                if (Passing(attempt.outcome)
+                    && !attempt.temporaryDirectory.empty())
+                {
+                    std::filesystem::remove_all(attempt.temporaryDirectory,
+                                                error);
+                    attempt.temporaryDirectory.clear();
+                }
+                else if (!attempt.temporaryDirectory.empty()
+                         && std::filesystem::is_empty(
+                             attempt.temporaryDirectory,
+                             error))
+                {
+                    std::filesystem::remove(attempt.temporaryDirectory, error);
+                    attempt.temporaryDirectory.clear();
+                }
+                if (job.kind == Job::Kind::kProbe)
+                {
+                    ProbeEnded(job, IsFailure(attempt.outcome));
+                    return;
+                }
+                state.result.attempts.push_back(std::move(attempt));
+                Decide(job.test);
+            }
+
+            /// What follows an attempt: a repeat, a retry, a bisection, or
+            /// the verdict.
+            void
+            Decide(std::size_t position)
+            {
+                auto &state = states_[position];
+                const auto &attempts = state.result.attempts;
+                const Attempt &last = attempts.back();
+                const PlannedTest &planned = plan_.tests[position];
+                if (stopping_)
+                {
+                    Finish(position);
+                    return;
+                }
+                if (state.repeatsLeft > 0)
+                {
+                    --state.repeatsLeft;
+                    NotifyRetry(position);
+                    Job job;
+                    job.test = position;
+                    Enqueue(std::move(job));
+                    return;
+                }
+                const bool failed = IsFailure(last.outcome);
+                const bool everFailed
+                    = std::any_of(attempts.begin(),
+                                  attempts.end(),
+                                  [](const Attempt &attempt) {
+                                      return IsFailure(attempt.outcome);
+                                  });
+                if (failed && state.retriesUsed < planned.retryLimit
+                    && retryBudget_ > 0)
+                {
+                    bool retry = options_.retryMode == RetryMode::kAlways;
+                    if (options_.retryMode == RetryMode::kAuto)
+                    {
+                        // One retry, alone in a fresh process, tells a
+                        // deterministic failure from a flaky one and from
+                        // one caused by a test before it. Beyond that only
+                        // what is known to be unstable is worth more tries.
+                        const bool unstable
+                            = last.outcome == Outcome::kCrashed
+                              || last.outcome == Outcome::kTimedOut
+                              || planned.quarantined || planned.flakyHistory;
+                        retry = state.retriesUsed == 0 || unstable;
+                    }
+                    if (retry)
+                    {
+                        ++state.retriesUsed;
+                        --retryBudget_;
+                        NotifyRetry(position);
+                        Job job;
+                        job.test = position;
+                        job.fresh = true;
+                        Enqueue(std::move(job));
+                        return;
+                    }
+                }
+                // A pass alone after a failure in company: flaky, or broken
+                // by a test before it. A probe tells which.
+                if (!failed && everFailed && options_.bisectOrder
+                    && state.bisection.stage == Bisection::Stage::kNone)
+                {
+                    const Attempt *sharedFailure = nullptr;
+                    for (const auto &attempt : attempts)
+                    {
+                        if (IsFailure(attempt.outcome)
+                            && !attempt.precededBy.empty()
+                            && attempt.outcome != Outcome::kTimedOut)
+                        {
+                            sharedFailure = &attempt;
+                            break;
+                        }
+                    }
+                    if (sharedFailure != nullptr && last.freshProcess)
+                    {
+                        std::vector<std::size_t> sequence;
+                        for (const auto &key : sharedFailure->precededBy)
+                        {
+                            if (const TestEntry *entry = catalog_.Find(key))
+                            {
+                                sequence.push_back(entry->index);
+                            }
+                        }
+                        if (!sequence.empty())
+                        {
+                            state.bisection.stage
+                                = Bisection::Stage::kConfirming;
+                            state.bisection.candidates = sequence;
+                            StartProbe(position, sequence);
+                            return;
+                        }
+                    }
+                }
+                Finish(position);
+            }
+
+            void
+            StartProbe(std::size_t position, std::vector<std::size_t> sequence)
+            {
+                auto &bisection = states_[position].bisection;
+                ++bisection.probes;
+                bisection.probing = sequence;
+                Job job;
+                job.kind = Job::Kind::kProbe;
+                job.test = position;
+                job.before = sequence;
+                job.fresh = true;
+                job.probeSequence = std::move(sequence);
+                Enqueue(std::move(job));
+            }
+
+            void
+            ProbeEnded(const Job &job, bool failed)
+            {
+                const std::size_t position = job.test;
+                auto &bisection = states_[position].bisection;
+                if (bisection.stage == Bisection::Stage::kConfirming)
+                {
+                    if (!failed)
+                    {
+                        // It does not fail again after the same tests: flaky.
+                        bisection.stage = Bisection::Stage::kDone;
+                        Finish(position);
+                        return;
+                    }
+                    bisection.reproduced = true;
+                    bisection.stage = Bisection::Stage::kSearching;
+                }
+                else if (bisection.stage == Bisection::Stage::kSearching)
+                {
+                    if (failed)
+                    {
+                        bisection.candidates = bisection.probing;
+                    }
+                    else
+                    {
+                        std::vector<std::size_t> rest;
+                        for (const std::size_t candidate : bisection.candidates)
+                        {
+                            if (std::find(bisection.probing.begin(),
+                                          bisection.probing.end(),
+                                          candidate)
+                                == bisection.probing.end())
+                            {
+                                rest.push_back(candidate);
+                            }
+                        }
+                        bisection.candidates = std::move(rest);
+                    }
+                }
+                if (bisection.candidates.size() <= 1
+                    || bisection.probes >= kMaximumProbes)
+                {
+                    if (bisection.candidates.size() == 1)
+                    {
+                        bisection.culprit = bisection.candidates.front();
+                    }
+                    bisection.stage = Bisection::Stage::kDone;
+                    Finish(position);
+                    return;
+                }
+                const auto half = bisection.candidates.size() / 2;
+                StartProbe(position,
+                           std::vector<std::size_t>(
+                               bisection.candidates.begin(),
+                               bisection.candidates.begin()
+                                   + static_cast<std::ptrdiff_t>(half)));
+            }
+
+            void
+            NotifyRetry(std::size_t position)
+            {
+                const auto &result = states_[position].result;
+                if (events_.retrying && !result.attempts.empty())
+                {
+                    events_.retrying(result, result.attempts.back());
+                }
+            }
+
+            /// Gives a test its verdict.
+            void
+            Finish(std::size_t position)
+            {
+                auto &state = states_[position];
+                if (state.finished)
+                {
+                    return;
+                }
+                state.finished = true;
+                TestResult &result = state.result;
+                const auto &attempts = result.attempts;
+                if (!attempts.empty())
+                {
+                    const Attempt *firstFailure = nullptr;
+                    const Attempt *lastPass = nullptr;
+                    for (const auto &attempt : attempts)
+                    {
+                        if (IsFailure(attempt.outcome)
+                            && firstFailure == nullptr)
+                        {
+                            firstFailure = &attempt;
+                        }
+                        if (Passing(attempt.outcome))
+                        {
+                            lastPass = &attempt;
+                        }
+                    }
+                    const auto &bisection = state.bisection;
+                    if (bisection.reproduced && firstFailure != nullptr)
+                    {
+                        // Broken by state another test leaves behind: a real
+                        // defect that a retry must not hide.
+                        result.outcome = firstFailure->outcome;
+                        result.category = firstFailure->category;
+                        std::string finding
+                            = "fails only after other tests in the same "
+                              "process and passes alone";
+                        if (bisection.culprit)
+                        {
+                            finding
+                                += "; it fails after '"
+                                   + catalog_.Tests()[*bisection.culprit].name
+                                   + "', which leaves state behind";
+                        }
+                        result.analysis.insert(result.analysis.begin(),
+                                               finding);
+                    }
+                    else if (firstFailure != nullptr && lastPass != nullptr)
+                    {
+                        result.flaky = true;
+                        result.outcome = lastPass->outcome;
+                        result.category = FailureCategory::kNone;
+                        result.analysis.insert(
+                            result.analysis.begin(),
+                            "flaky: failed with "
+                                + std::string{ CategoryName(
+                                    firstFailure->category) }
+                                + " and passed on attempt "
+                                + std::to_string(lastPass->number));
+                    }
+                    else
+                    {
+                        result.outcome = attempts.back().outcome;
+                        result.category = attempts.back().category;
+                    }
+                }
+                if (result.FailsRun())
+                {
+                    ++failedTests_;
+                    if (options_.failFast > 0
+                        && failedTests_ >= options_.failFast && !stopping_)
+                    {
+                        stopping_ = true;
+                        notes_.push_back("stopped after "
+                                         + std::to_string(failedTests_)
+                                         + " failed test(s) (--fail-fast)");
+                    }
+                }
+                if (events_.testFinished)
+                {
+                    events_.testFinished(result);
+                }
+            }
+
+            // Messages.
+
+            void
+            HandleLine(Worker &worker, const std::string &line)
+            {
+                auto parsed = ParseJson(line);
+                if (!parsed)
+                {
+                    // Not a message: something wrote to the protocol stream.
+                    if (worker.running)
+                    {
+                        AppendBounded(worker.running->attempt.processOutput,
+                                      line + "\n");
+                    }
+                    return;
+                }
+                const JsonValue &message = *parsed;
+                const std::string &kind = message["t"].AsString();
+                if (kind == "hello")
+                {
+                    if (message["identity"].AsString() != catalog_.Identity())
+                    {
+                        notes_.push_back(
+                            "a worker saw different tests than the runner; "
+                            "the binary may have changed during the run");
+                        KillWorker(worker);
+                        return;
+                    }
+                    worker.ready = true;
+                    startupFailures_ = 0;
+                    // What the listeners printed when the worker started.
+                    const std::string &output = message["output"].AsString();
+                    if (!output.empty())
+                    {
+                        AppendBounded(worker.startupOutput, output);
+                        std::fwrite(output.data(), 1, output.size(), stderr);
+                        std::fflush(stderr);
+                    }
+                    return;
+                }
+                if (!worker.running)
+                {
+                    return;
+                }
+                Running &running = *worker.running;
+                Attempt &attempt = running.attempt;
+                if (kind == "section")
+                {
+                    if (message["begin"].AsBoolean())
+                    {
+                        SectionRecord section;
+                        section.name = message["name"].AsString();
+                        section.file = message["file"].AsString();
+                        section.line = static_cast<std::size_t>(
+                            message["line"].AsUnsigned());
+                        section.depth = static_cast<std::size_t>(
+                            message["depth"].AsUnsigned());
+                        section.startMicroseconds = message["at"].AsInteger();
+                        section.durationMicroseconds = -1;
+                        if (attempt.sections.size() < 1000)
+                        {
+                            attempt.sections.push_back(section);
+                        }
+                    }
+                    else if (!running.fatal)
+                    {
+                        // After a fatal signal Catch2 closes the open
+                        // sections itself; they stay open here, because the
+                        // crash happened inside them.
+                        const auto depth = message["depth"].AsUnsigned();
+                        for (auto section = attempt.sections.rbegin();
+                             section != attempt.sections.rend();
+                             ++section)
+                        {
+                            if (section->depth == depth
+                                && section->durationMicroseconds < 0)
+                            {
+                                section->durationMicroseconds
+                                    = message["duration"].AsInteger();
+                                break;
+                            }
+                        }
+                    }
+                }
+                else if (kind == "at")
+                {
+                    attempt.lastLocation = message["location"].AsString();
+                }
+                else if (kind == "assertion")
+                {
+                    // Newer than any location sent before it, which is sent
+                    // only now and then.
+                    attempt.lastLocation
+                        = message["file"].AsString() + ":"
+                          + std::to_string(message["line"].AsUnsigned());
+                    if (attempt.assertions.size() < kMaximumRecordedAssertions)
+                    {
+                        attempt.assertions.push_back(
+                            AssertionFromJson(message));
+                    }
+                }
+                else if (kind == "benchmark")
+                {
+                    attempt.benchmarks.push_back(BenchmarkFromJson(message));
+                }
+                else if (kind == "fatal")
+                {
+                    running.fatal = true;
+                    attempt.fatalMessage = message["message"].AsString();
+                    if (!message["location"].AsString().empty())
+                    {
+                        attempt.lastLocation = message["location"].AsString();
+                    }
+                }
+                else if (kind == "end")
+                {
+                    attempt.outcome
+                        = OutcomeFromName(message["outcome"].AsString())
+                              .value_or(Outcome::kFailed);
+                    attempt.category
+                        = CategoryFromName(message["category"].AsString())
+                              .value_or(FailureCategory::kNone);
+                    if (running.fatal)
+                    {
+                        attempt.outcome = Outcome::kCrashed;
+                        attempt.category = FailureCategory::kCrash;
+                    }
+                    attempt.assertionsPassed = static_cast<std::size_t>(
+                        message["passed"].AsUnsigned());
+                    attempt.assertionsFailed = static_cast<std::size_t>(
+                        m