Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

develroot / src / Progmasoft / Runner / History.cpp

17.0 KiB · Raw · Log

// Copyright (c) 2026 Progmasoft.
// SPDX-License-Identifier: LGPL-3.0-or-later
#include <algorithm>
#include <cmath>
#include <system_error>

#include "Files.hpp"
#include "History.hpp"

namespace Progmasoft::Catch3::Runner
{
    namespace
    {
        constexpr int kHistoryVersion = 1;
        constexpr std::size_t kRecordsPerTest = 50;
        constexpr std::size_t kRunsKept = 100;

        /// Runs that did not finish on their own say nothing about how long
        /// the test takes.
        bool
        HasDuration(Outcome outcome)
        {
            return outcome == Outcome::kPassed || outcome == Outcome::kFailed
                   || outcome == Outcome::kFailedAsExpected
                   || outcome == Outcome::kSkipped;
        }

        bool
        Counts(Outcome outcome)
        {
            return outcome != Outcome::kNotRun && outcome != Outcome::kBlocked
                   && outcome != Outcome::kUpToDate;
        }

        JsonValue
        RecordToJson(const TestRecord &record)
        {
            JsonValue value = JsonValue::Object();
            value.Set("run", record.run);
            value.Set("outcome", OutcomeName(record.outcome));
            if (record.category != FailureCategory::kNone)
            {
                value.Set("category", CategoryName(record.category));
            }
            value.Set("duration", record.durationMicroseconds);
            value.Set("attempts", record.attempts);
            if (record.flaky)
            {
                value.Set("flaky", true);
            }
            if (!record.signature.empty())
            {
                value.Set("signature", record.signature);
            }
            if (!record.fingerprint.empty())
            {
                value.Set("fingerprint", record.fingerprint);
            }
            return value;
        }

        TestRecord
        RecordFromJson(const JsonValue &value)
        {
            TestRecord record;
            record.run = value["run"].AsInteger();
            record.outcome = OutcomeFromName(value["outcome"].AsString())
                                 .value_or(Outcome::kNotRun);
            record.category = CategoryFromName(value["category"].AsString())
                                  .value_or(FailureCategory::kNone);
            record.durationMicroseconds = value["duration"].AsInteger();
            record.attempts
                = static_cast<std::size_t>(value["attempts"].AsUnsigned());
            record.flaky = value["flaky"].AsBoolean();
            record.signature = value["signature"].AsString();
            record.fingerprint = value["fingerprint"].AsString();
            return record;
        }
    } // namespace

    const TestRecord *
    TestHistory::Last() const noexcept
    {
        for (auto record = records.rbegin(); record != records.rend(); ++record)
        {
            if (Counts(record->outcome))
            {
                return &*record;
            }
        }
        return nullptr;
    }

    std::optional<std::int64_t>
    TestHistory::DurationPercentile(double fraction) const
    {
        std::vector<std::int64_t> durations;
        for (auto record = records.rbegin();
             record != records.rend() && durations.size() < 20;
             ++record)
        {
            if (HasDuration(record->outcome))
            {
                durations.push_back(record->durationMicroseconds);
            }
        }
        if (durations.empty())
        {
            return std::nullopt;
        }
        std::sort(durations.begin(), durations.end());
        const auto position = static_cast<std::size_t>(
            std::ceil(fraction * static_cast<double>(durations.size())));
        return durations[std::min(durations.size() - 1,
                                  position == 0 ? 0 : position - 1)];
    }

    double
    TestHistory::FailureProbability() const
    {
        // A beta prior worth one run: a test without history fails with
        // probability 0.3, a common share for new code.
        double failures = 0.3;
        double total = 1.0;
        double weight = 1.0;
        std::size_t seen = 0;
        for (auto record = records.rbegin();
             record != records.rend() && seen < 30;
             ++record)
        {
            if (!Counts(record->outcome))
            {
                continue;
            }
            ++seen;
            const bool failed = IsFailure(record->outcome) || record->flaky;
            failures += failed ? weight : 0.0;
            total += weight;
            weight *= 0.8;
        }
        double probability = failures / total;
        if (const TestRecord *last = Last();
            last != nullptr && IsFailure(last->outcome))
        {
            // What failed last time most often fails again.
            probability = std::max(probability, 0.7);
        }
        return probability;
    }

    std::size_t
    TestHistory::FlakyEvents(std::size_t window) const
    {
        std::size_t events = 0;
        std::size_t seen = 0;
        const TestRecord *newer = nullptr;
        for (auto record = records.rbegin();
             record != records.rend() && seen < window;
             ++record)
        {
            if (!Counts(record->outcome))
            {
                continue;
            }
            ++seen;
            if (record->flaky)
            {
                ++events;
            }
            else if (newer != nullptr && !newer->flaky
                     && !record->fingerprint.empty()
                     && record->fingerprint == newer->fingerprint
                     && IsFailure(record->outcome) != IsFailure(newer->outcome))
            {
                // The verdict changed while nothing the test depends on did.
                ++events;
            }
            newer = &*record;
        }
        return events;
    }

    std::size_t
    TestHistory::PassStreak() const
    {
        std::size_t streak = 0;
        for (auto record = records.rbegin(); record != records.rend(); ++record)
        {
            if (!Counts(record->outcome))
            {
                continue;
            }
            if (record->outcome != Outcome::kPassed || record->flaky)
            {
                break;
            }
            ++streak;
        }
        return streak;
    }

    std::filesystem::path
    HistoryFile(const std::filesystem::path &directory,
                const std::string &binaryName)
    {
        std::string name;
        for (const char character : binaryName)
        {
            const bool safe = (character >= 'a' && character <= 'z')
                              || (character >= 'A' && character <= 'Z')
                              || (character >= '0' && character <= '9')
                              || character == '-' || character == '_'
                              || character == '.';
            name.push_back(safe ? character : '_');
        }
        return directory / (name + ".history.json");
    }

    std::string
    History::KeyOf(const TestResult &test)
    {
        if (!test.id.empty())
        {
            return test.id;
        }
        return test.className.empty() ? test.name
                                      : test.className + "::" + test.name;
    }

    const TestHistory *
    History::Find(const std::string &key) const
    {
        const auto found = tests_.find(key);
        return found == tests_.end() ? nullptr : &found->second;
    }

    Expected<History>
    History::Load(const std::filesystem::path &file)
    {
        std::error_code error;
        if (!std::filesystem::exists(file, error))
        {
            return History{};
        }
        auto text = ReadFile(file);
        if (!text)
        {
            return text.GetError();
        }
        auto json = ParseJson(*text);
        if (!json)
        {
            return json.GetError().Context(file.string());
        }
        return FromJson(*json);
    }

    Expected<History>
    History::Commit(const std::filesystem::path &file,
                    const RunResult &run,
                    const RunRecord &record,
                    const std::map<std::string, std::string> &fingerprints,
                    const QuarantinePolicy &policy,
                    std::chrono::milliseconds lockWait)
    {
        const FileLock lock{ file, lockWait };
        if (!lock.Held())
        {
            // Writing without the lock could replace a run that another
            // process is adding at the same time.
            return Error{ "another run holds the lock of " + file.string() };
        }
        auto history = Load(file);
        if (!history)
        {
            // A damaged file is set aside rather than lost.
            std::error_code error;
            std::filesystem::rename(file,
                                    file.string() + ".damaged-" + UniqueName(),
                                    error);
            history = History{};
        }
        history->Apply(run, record, fingerprints, policy);
        if (auto status
            = WriteFileAtomically(file, history->ToJson().ToPrettyString());
            !status)
        {
            return status.GetError();
        }
        return history;
    }

    void
    History::Apply(const RunResult &run,
                   RunRecord record,
                   const std::map<std::string, std::string> &fingerprints,
                   const QuarantinePolicy &policy)
    {
        record.id = nextRun_++;
        runs_.push_back(record);
        if (runs_.size() > kRunsKept)
        {
            runs_.erase(
                runs_.begin(),
                runs_.begin()
                    + static_cast<std::ptrdiff_t>(runs_.size() - kRunsKept));
        }
        for (const auto &test : run.tests)
        {
            if (test.outcome == Outcome::kNotRun)
            {
                continue;
            }
            const std::string key = KeyOf(test);
            TestHistory &history = tests_[key];
            TestRecord result;
            result.run = record.id;
            result.outcome = test.outcome;
            result.category = test.category;
            result.durationMicroseconds
                = test.attempts.empty()
                      ? 0
                      : test.attempts.back().durationMicroseconds;
            result.attempts = test.attempts.size();
            result.flaky = test.flaky;
            result.signature = test.signature;
            if (const auto found = fingerprints.find(key);
                found != fingerprints.end())
            {
                result.fingerprint = found->second;
            }
            history.records.push_back(result);
            if (history.records.size() > kRecordsPerTest)
            {
                history.records.erase(history.records.begin());
            }
            const auto failed
                = std::find_if(test.attempts.begin(),
                               test.attempts.end(),
                               [](const Attempt &attempt) {
                                   return IsFailure(attempt.outcome);
                               });
            if ((IsFailure(test.outcome) || test.flaky)
                && failed != test.attempts.end())
            {
                // The first failure, as it happened: a flaky failure is
                // worth replaying too.
                ReplayRecord replay;
                replay.run = record.id;
                replay.seed = failed->seed;
                replay.precededBy = failed->precededBy;
                replay.freshProcess = failed->freshProcess;
                history.lastFailure = replay;
            }
            else if (test.outcome == Outcome::kPassed)
            {
                history.lastFailure.reset();
            }
            // Quarantine: in after repeated flakiness, out after a streak of
            // clean passes.
            if (!history.quarantine
                && history.FlakyEvents(policy.window)
                       >= policy.flakyEventsToEnter)
            {
                history.quarantine = QuarantineRecord{
                    record.id,
                    std::to_string(history.FlakyEvents(policy.window))
                        + " flaky results in the last "
                        + std::to_string(policy.window) + " runs"
                };
            }
            else if (history.quarantine
                     && history.PassStreak() >= policy.passesToLeave)
            {
                history.quarantine.reset();
            }
        }
    }

    JsonValue
    History::ToJson() const
    {
        JsonValue value = JsonValue::Object();
        value.Set("format", "catch3-history");
        value.Set("version", kHistoryVersion);
        value.Set("nextRun", nextRun_);
        JsonValue runs = JsonValue::Array();
        for (const auto &run : runs_)
        {
            JsonValue item = JsonValue::Object();
            item.Set("id", run.id);
            item.Set("time", run.unixMilliseconds);
            item.Set("commit", run.commit);
            item.Set("binary", run.binaryFingerprint);
            item.Set("seed", run.seed);
            item.Set("tests", run.tests);
            item.Set("failed", run.failed);
            item.Set("wall", run.wallMicroseconds);
            runs.Append(std::move(item));
        }
        value.Set("runs", std::move(runs));
        JsonValue tests = JsonValue::Object();
        for (const auto &[key, history] : tests_)
        {
            JsonValue item = JsonValue::Object();
            JsonValue records = JsonValue::Array();
            for (const auto &record : history.records)
            {
                records.Append(RecordToJson(record));
            }
            item.Set("results", std::move(records));
            if (history.quarantine)
            {
                JsonValue quarantine = JsonValue::Object();
                quarantine.Set("since", history.quarantine->since);
                quarantine.Set("reason", history.quarantine->reason);
                item.Set("quarantine", std::move(quarantine));
            }
            if (history.lastFailure)
            {
                JsonValue replay = JsonValue::Object();
                replay.Set("run", history.lastFailure->run);
                replay.Set("seed", history.lastFailure->seed);
                JsonValue preceding = JsonValue::Array();
                for (const auto &name : history.lastFailure->precededBy)
                {
                    preceding.Append(name);
                }
                replay.Set("precededBy", std::move(preceding));
                replay.Set("freshProcess", history.lastFailure->freshProcess);
                item.Set("lastFailure", std::move(replay));
            }
            tests.Set(key, std::move(item));
        }
        value.Set("tests", std::move(tests));
        return value;
    }

    Expected<History>
    History::FromJson(const JsonValue &value)
    {
        if (value["format"].AsString() != "catch3-history")
        {
            return Error{ "not a Catch3 history" };
        }
        if (value["version"].AsInteger() > kHistoryVersion)
        {
            return Error{ "the history was written by a newer Catch3" };
        }
        History history;
        history.nextRun_
            = std::max<std::int64_t>(1, value["nextRun"].AsInteger());
        for (const auto &item : value["runs"].Items())
        {
            RunRecord run;
            run.id = item["id"].AsInteger();
            run.unixMilliseconds = item["time"].AsInteger();
            run.commit = item["commit"].AsString();
            run.binaryFingerprint = item["binary"].AsString();
            run.seed = static_cast<std::uint32_t>(item["seed"].AsUnsigned());
            run.tests = static_cast<std::size_t>(item["tests"].AsUnsigned());
            run.failed = static_cast<std::size_t>(item["failed"].AsUnsigned());
            run.wallMicroseconds = item["wall"].AsInteger();
            history.runs_.push_back(run);
        }
        for (const auto &[key, item] : value["tests"].Members())
        {
            TestHistory test;
            for (const auto &record : item["results"].Items())
            {
                test.records.push_back(RecordFromJson(record));
            }
            if (item["quarantine"].IsObject())
            {
                test.quarantine = QuarantineRecord{
                    item["quarantine"]["since"].AsInteger(),
                    item["quarantine"]["reason"].AsString()
                };
            }
            if (item["lastFailure"].IsObject())
            {
                const JsonValue &failure = item["lastFailure"];
                ReplayRecord replay;
                replay.run = failure["run"].AsInteger();
                replay.seed
                    = static_cast<std::uint32_t>(failure["seed"].AsUnsigned());
                for (const auto &name : failure["precededBy"].Items())
                {
                    replay.precededBy.push_back(name.AsString());
                }
                replay.freshProcess = failure["freshProcess"].AsBoolean();
                test.lastFailure = replay;
            }
            history.tests_.emplace(key, std::move(test));
        }
        return history;
    }
} // namespace Progmasoft::Catch3::Runner