// Copyright (c) 2026 Progmasoft. // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 #pragma once #include #include #include #include #include namespace Progmasoft::Catch3 { /// How one attempt at a test ended. enum class Outcome : std::uint8_t { kPassed, /// An assertion failed, or the test threw. kFailed, /// It failed, but `[!shouldfail]` or `[!mayfail]` allows that. kFailedAsExpected, /// `SKIP` was called. kSkipped, /// The process running it died: a signal, a structured exception or /// an exit in the middle of the test. kCrashed, /// It ran past its time limit and was stopped. kTimedOut, /// A test it depends on did not pass, so it was not run. kBlocked, /// The run stopped before reaching it. kNotRun, /// It passed last time and nothing it depends on changed, so the /// incremental run kept that result. kUpToDate, }; /// Why a failure happened, as far as the runner can tell. enum class FailureCategory : std::uint8_t { kNone, /// A REQUIRE or CHECK expression was false. kAssertion, /// An exception escaped the test or an assertion. kUnexpectedException, /// REQUIRE_THROWS and its kind did not see the expected exception. kMissingException, /// FAIL or FAIL_CHECK. kExplicitFailure, /// A signal or structured exception inside the test. kCrash, kTimeout, /// The worker could not run the test at all, for example because it /// failed to start. kInfrastructure, /// A test it depends on failed. kDependency, }; /// How a failure relates to earlier runs. enum class FailureHistory : std::uint8_t { /// There is no history to compare with. kUnknown, /// The test passed in the previous run. kNew, /// The test failed in the previous run too. kPersistent, /// The test passes and fails with the same inputs. kIntermittent, }; /// One failed or reported assertion. struct AssertionRecord final { std::string file; std::size_t line = 0; /// The macro, such as REQUIRE. std::string macro; /// The expression as written. std::string expression; /// The expression with its operands' values. std::string expansion; std::string message; /// INFO, CAPTURE and UNSCOPED_INFO messages in force. std::vector context; /// The sections the assertion was in, outermost first. std::vector sections; FailureCategory category = FailureCategory::kAssertion; bool passed = false; /// The failure is allowed by `[!mayfail]` or `[!shouldfail]`. bool allowed = false; }; /// The statistics of one BENCHMARK. struct BenchmarkRecord final { std::string name; std::size_t samples = 0; std::size_t iterations = 0; double meanNanoseconds = 0.0; double lowMeanNanoseconds = 0.0; double highMeanNanoseconds = 0.0; double standardDeviationNanoseconds = 0.0; double outlierVariance = 0.0; }; /// One section entered while a test ran, with its timing. struct SectionRecord final { std::string name; std::string file; std::size_t line = 0; /// Microseconds since the start of the attempt. std::int64_t startMicroseconds = 0; /// -1 for a section that never ended: the attempt crashed or timed /// out inside it. std::int64_t durationMicroseconds = 0; /// 0 for a section directly in the test case. std::size_t depth = 0; }; /// One run of a test in a worker. A test may be tried several times: /// retries, repeats and the probes that look for an order dependency. struct Attempt final { Outcome outcome = Outcome::kNotRun; FailureCategory category = FailureCategory::kNone; std::size_t number = 1; /// The worker that ran it, and the tests it ran before, in order. std::size_t worker = 0; std::vector precededBy; bool freshProcess = false; std::uint32_t seed = 0; std::int64_t startMicroseconds = 0; std::int64_t durationMicroseconds = 0; std::size_t assertionsPassed = 0; std::size_t assertionsFailed = 0; std::vector assertions; std::vector benchmarks; std::vector sections; /// What the test wrote to std::cout and std::cerr, captured. std::string standardOutput; std::string standardError; /// What reached the worker's own streams while the test ran, such as /// printf output or a sanitizer report. std::string processOutput; /// The fatal condition the worker reported before it died. std::string fatalMessage; /// How the worker ended, for a crash. std::string exitDescription; /// The last assertion started before a crash or a timeout. std::string lastLocation; /// The directory the test had for its temporary files; kept when the /// attempt failed, so its contents can be inspected. std::string temporaryDirectory; }; /// One test of the run and everything the runner learned about it. struct TestResult final { /// Identifies the test across runs and builds: its name, with its /// class and, when names repeat, its tags. History is kept by it. std::string id; std::string name; std::string className; std::string file; std::size_t line = 0; std::vector tags; /// The verdict over all attempts. Outcome outcome = Outcome::kNotRun; FailureCategory category = FailureCategory::kNone; FailureHistory history = FailureHistory::kUnknown; /// It failed and then passed with the same inputs. bool flaky = false; /// It is in quarantine, so its failure does not fail the run. bool quarantined = false; /// Why it was selected, when not every test runs: "failed last /// run", "covers a changed file", "dependency of ...". std::string selectionReason; /// A stable identifier of the failure: its category, location and /// message with numbers removed. Tests that share it probably share a /// cause. std::string signature; /// What the runner found out about the failure, one finding per /// entry, most useful first. std::vector analysis; /// A command line that runs the failure again the way it happened. std::string replayCommand; std::vector attempts; std::int64_t predictedDurationMicroseconds = 0; double predictedFailureProbability = 0.0; /// The sum of the attempts' durations. [[nodiscard]] std::int64_t DurationMicroseconds() const noexcept; /// Whether the result fails the run. [[nodiscard]] bool FailsRun() const noexcept; }; /// The totals of a run, by verdict. struct RunSummary final { std::size_t tests = 0; std::size_t passed = 0; std::size_t failed = 0; std::size_t failedAsExpected = 0; std::size_t skipped = 0; std::size_t crashed = 0; std::size_t timedOut = 0; std::size_t blocked = 0; std::size_t notRun = 0; std::size_t upToDate = 0; std::size_t flaky = 0; std::size_t quarantined = 0; std::size_t attempts = 0; std::size_t assertionsPassed = 0; std::size_t assertionsFailed = 0; /// Wall-clock time of the run, and the time its tests took added up; /// their ratio is the speed-up of running in parallel. std::int64_t wallMicroseconds = 0; std::int64_t testMicroseconds = 0; }; /// A whole run of one test binary. struct RunResult final { std::string binary; std::uint32_t seed = 0; std::size_t workers = 0; /// The shard, when the run was one of several. unsigned shardIndex = 0; unsigned shardCount = 1; /// The tests that matched before selection narrowed them. std::size_t candidates = 0; std::vector tests; /// Test specs that matched nothing. std::vector unmatchedSpecs; /// Messages about the run itself, such as a history that could not /// be read. std::vector notes; std::int64_t startUnixMilliseconds = 0; std::int64_t wallMicroseconds = 0; }; [[nodiscard]] RunSummary Summarize(const RunResult &run); /// The name an outcome has in reports: "passed", "timed-out", ... [[nodiscard]] const char * OutcomeName(Outcome outcome) noexcept; [[nodiscard]] std::optional OutcomeFromName(const std::string &name) noexcept; [[nodiscard]] const char * CategoryName(FailureCategory category) noexcept; [[nodiscard]] std::optional CategoryFromName(const std::string &name) noexcept; [[nodiscard]] const char * HistoryName(FailureHistory history) noexcept; /// The failure of an attempt as text: each failed assertion with its /// expansion, messages and sections, or what is known about a crash or a /// timeout. Empty for an attempt that did not fail. [[nodiscard]] std::string DescribeFailure(const Attempt &attempt); /// The first line of `DescribeFailure`, for a summary. [[nodiscard]] std::string SummarizeFailure(const Attempt &attempt); /// Whether an outcome counts as a failure of the attempt. [[nodiscard]] bool IsFailure(Outcome outcome) noexcept; } // namespace Progmasoft::Catch3