// Copyright (c) 2026 Progmasoft. // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "Analysis.hpp" #include "Catalog.hpp" #include "Console.hpp" #include "Execution.hpp" #include "Files.hpp" #include "Git.hpp" #include "Hash.hpp" #include "History.hpp" #include "Json.hpp" #include "Options.hpp" #include "Planner.hpp" #include "Process.hpp" #include "ProjectConfig.hpp" #include "ResultsJson.hpp" #include "Scheduler.hpp" namespace Progmasoft::Catch3 { namespace { using Runner::JsonValue; void PrintError(const std::string &message) { std::fprintf(stderr, "error: %s\n", message.c_str()); } /// Problems found while the tests registered, such as two tests of /// the same name; reported before anything runs. bool ReportStartupErrors() { #if !defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) const auto &exceptions = Catch::getRegistryHub() .getStartupExceptionRegistry() .getExceptions(); if (exceptions.empty()) { return false; } std::fprintf(stderr, "error: the tests could not be registered:\n"); for (const auto &exception : exceptions) { try { std::rethrow_exception(exception); } catch (const std::exception &error) { std::fprintf(stderr, " %s\n", error.what()); } catch (...) { std::fprintf(stderr, " an unknown exception\n"); } } return true; #else return false; #endif } bool BazelTest() { #if defined(CATCH_CONFIG_BAZEL_SUPPORT) return true; #else return Runner::HasEnvironment("BAZEL_TEST"); #endif } /// Bazel passes what it wants through the environment; it wins over /// the command line, as `bazel test --test_filter` expects. void ApplyBazel(Runner::Options &options, std::vector ¬es) { const std::string xml = Runner::ReadEnvironment("XML_OUTPUT_FILE"); if (!xml.empty() && options.junitPath.empty()) { options.junitPath = xml; } const std::string filter = Runner::ReadEnvironment("TESTBRIDGE_TEST_ONLY"); if (!filter.empty()) { options.testSpecs = { filter }; } const std::string index = Runner::ReadEnvironment("TEST_SHARD_INDEX"); const std::string total = Runner::ReadEnvironment("TEST_TOTAL_SHARDS"); const std::string status = Runner::ReadEnvironment("TEST_SHARD_STATUS_FILE"); if (!index.empty() && !total.empty()) { auto parsed = Runner::ParseOptions( { "--shard=" + index + "/" + total }); if (parsed) { options.shard = parsed->shard; if (!status.empty()) { // Bazel needs to see that the shard was read. static_cast( Runner::WriteFileAtomically(status, "")); } } else { notes.push_back("ignoring Bazel's shard " + index + "/" + total + ": " + parsed.GetError().Message()); } } const std::string seed = Runner::ReadEnvironment("TEST_RANDOM_SEED"); if (!seed.empty()) { auto parsed = Runner::ParseOptions({ "--seed=" + seed }); if (parsed) { options.seed = parsed->seed; options.seedGiven = true; } } const std::string outputs = Runner::ReadEnvironment("TEST_UNDECLARED_OUTPUTS_DIR"); if (!outputs.empty() && options.jsonPath.empty()) { options.jsonPath = (std::filesystem::path{ outputs } / "catch3.json") .string(); } if (options.historyDirectory.empty() && !Runner::HasEnvironment("CATCH3_HISTORY")) { // A sandboxed test has nowhere lasting to keep history. options.historyEnabled = false; } } std::filesystem::path HistoryDirectory(const Runner::Options &options, const std::filesystem::path &program) { if (!options.historyDirectory.empty()) { return options.historyDirectory; } const std::string fromEnvironment = Runner::ReadEnvironment("CATCH3_HISTORY"); if (!fromEnvironment.empty()) { return fromEnvironment; } return program.parent_path() / ".catch3"; } JsonValue Strings(const std::vector &values) { JsonValue array = JsonValue::Array(); for (const auto &value : values) { array.Append(value); } return array; } /// The discovery protocol: what `--list=json` prints. Its shape is /// fixed for a given version; IDE adapters and CMake read it. std::string ListJson(const Runner::Catalog &catalog, const std::vector &matched, const std::string &programName) { JsonValue list = JsonValue::Object(); list.Set("format", "catch3-tests"); list.Set("version", 1); list.Set("binary", programName); list.Set("catch3", kVersion); JsonValue tests = JsonValue::Array(); for (const std::size_t index : matched) { const auto &entry = catalog.Tests()[index]; JsonValue test = JsonValue::Object(); test.Set("id", entry.key); test.Set("name", entry.name); test.Set("className", entry.className); test.Set("file", entry.file); test.Set("line", entry.line); test.Set("tags", Strings(entry.tags)); test.Set("hidden", entry.hidden); test.Set("serial", entry.serial); test.Set("isolated", entry.isolated); test.Set("quarantined", entry.quarantined); test.Set("resources", Strings(entry.resources)); test.Set("dependsOn", Strings(entry.dependsOn)); if (entry.timeout) { test.Set("timeoutSeconds", static_cast(entry.timeout->count()) / 1000.0); } if (entry.retries) { test.Set("retries", *entry.retries); } tests.Append(std::move(test)); } list.Set("tests", std::move(tests)); return list.ToPrettyString(); } int List(const Runner::Options &options, const Runner::Catalog &catalog, const std::vector &matched, const std::string &programName) { if (options.mode == Runner::Mode::kListTags) { std::map counts; for (const std::size_t index : matched) { for (const auto &tag : catalog.Tests()[index].tags) { ++counts[tag]; } } for (const auto &[tag, count] : counts) { std::printf("%6zu [%s]\n", count, tag.c_str()); } std::printf("%zu tag(s)\n", counts.size()); return 0; } switch (options.listFormat) { case Runner::ListFormat::kJson: std::fputs(ListJson(catalog, matched, programName).c_str(), stdout); break; case Runner::ListFormat::kNames: for (const std::size_t index : matched) { std::printf("%s\n", catalog.Tests()[index].name.c_str()); } break; case Runner::ListFormat::kText: for (const std::size_t index : matched) { const auto &entry = catalog.Tests()[index]; std::string tags; for (const auto &tag : entry.tags) { tags += "[" + tag + "]"; } std::printf("%s\n %s:%zu\n", entry.name.c_str(), entry.file.c_str(), entry.line); if (!tags.empty()) { std::printf(" %s\n", tags.c_str()); } } std::printf("%zu matching test(s)\n", matched.size()); break; } return 0; } std::string FormatMicroseconds(std::int64_t microseconds) { char buffer[32]; std::snprintf(buffer, sizeof(buffer), "%.3f s", static_cast(microseconds) / 1e6); return buffer; } /// What the history says about the tests: `--analytics`. int Analytics(const Runner::Options &options, const Runner::History &history, const std::filesystem::path &file) { struct Row { std::string key; double failureRate = 0.0; std::size_t runs = 0; std::size_t flakyEvents = 0; std::int64_t median = 0; std::int64_t recentMedian = 0; std::int64_t olderMedian = 0; std::string quarantine; }; std::vector rows; for (const auto &[key, test] : history.Tests()) { Row row; row.key = key; std::size_t failures = 0; std::vector recent; std::vector older; std::size_t position = 0; for (auto record = test.records.rbegin(); record != test.records.rend(); ++record, ++position) { ++row.runs; failures += IsFailure(record->outcome) ? 1 : 0; if (record->outcome == Outcome::kPassed) { (position < 5 ? recent : older) .push_back(record->durationMicroseconds); } } const auto median = [](std::vector values) { if (values.empty()) { return std::int64_t{ 0 }; } std::sort(values.begin(), values.end()); return values[values.size() / 2]; }; row.failureRate = row.runs == 0 ? 0.0 : static_cast(failures) / static_cast(row.runs); row.flakyEvents = test.FlakyEvents(20); row.median = test.DurationPercentile(0.5).value_or(0); row.recentMedian = median(recent); row.olderMedian = median(older); if (test.quarantine) { row.quarantine = test.quarantine->reason; } rows.push_back(std::move(row)); } const auto top = [&](auto key, std::size_t count) { std::vector ordered; for (const auto &row : rows) { ordered.push_back(&row); } std::stable_sort(ordered.begin(), ordered.end(), [&](const Row *left, const Row *right) { return key(*left) > key(*right); }); ordered.resize(std::min(count, ordered.size())); return ordered; }; const auto flaky = top( [](const Row &row) { return static_cast(row.flakyEvents); }, 10); const auto failing = top( [](const Row &row) { return row.failureRate; }, 10); const auto slowest = top( [](const Row &row) { return static_cast(row.median); }, 10); const auto slower = top( [](const Row &row) { return row.olderMedian > 0 ? static_cast(row.recentMedian) / static_cast(row.olderMedian) : 0.0; }, 10); if (options.analyticsJson) { JsonValue report = JsonValue::Object(); report.Set("format", "catch3-analytics"); report.Set("version", 1); report.Set("runs", history.Runs().size()); report.Set("tests", rows.size()); JsonValue tests = JsonValue::Array(); for (const auto &row : rows) { JsonValue item = JsonValue::Object(); item.Set("id", row.key); item.Set("runs", row.runs); item.Set("failureRate", row.failureRate); item.Set("flakyEvents", row.flakyEvents); item.Set("medianMicroseconds", row.median); item.Set("recentMedianMicroseconds", row.recentMedian); item.Set("olderMedianMicroseconds", row.olderMedian); if (!row.quarantine.empty()) { item.Set("quarantine", row.quarantine); } tests.Append(std::move(item)); } report.Set("tests", std::move(tests)); std::fputs(report.ToPrettyString().c_str(), stdout); return 0; } std::printf("history: %s\n%zu run(s), %zu test(s)\n", file.string().c_str(), history.Runs().size(), rows.size()); std::printf("\nflaky (last 20 runs):\n"); for (const Row *row : flaky) { if (row->flakyEvents > 0) { std::printf(" %3zu %s\n", row->flakyEvents, row->key.c_str()); } } std::printf("\nin quarantine:\n"); for (const auto &row : rows) { if (!row.quarantine.empty()) { std::printf(" %s: %s\n", row.key.c_str(), row.quarantine.c_str()); } } std::printf("\nfailing most often:\n"); for (const Row *row : failing) { if (row->failureRate > 0.0) { std::printf(" %3.0f%% %s\n", row->failureRate * 100.0, row->key.c_str()); } } std::printf("\nslowest (median):\n"); for (const Row *row : slowest) { std::printf(" %10s %s\n", FormatMicroseconds(row->median).c_str(), row->key.c_str()); } std::printf( "\ngetting slower (last 5 runs against the 15 before):\n"); for (const Row *row : slower) { if (row->olderMedian > 0 && row->recentMedian > row->olderMedian * 3 / 2) { std::printf(" %10s -> %-10s %s\n", FormatMicroseconds(row->olderMedian).c_str(), FormatMicroseconds(row->recentMedian).c_str(), row->key.c_str()); } } return 0; } std::filesystem::path SourceDirectory(const Runner::Catalog &catalog) { for (const auto &entry : catalog.Tests()) { const std::filesystem::path file{ entry.file }; if (file.is_absolute()) { return file.parent_path(); } } std::error_code error; return std::filesystem::current_path(error); } std::int64_t UnixMilliseconds() { return std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()) .count(); } int Run(const std::vector &arguments) { const std::filesystem::path program = Runner::CurrentExecutable( arguments.empty() ? nullptr : arguments.front().c_str()); const std::string programName = program.empty() ? "tests" : program.stem().string(); std::vector rest; if (arguments.size() > 1) { rest.assign(arguments.begin() + 1, arguments.end()); } std::error_code here; auto parsed = Runner::ParseCommandLine( rest, std::filesystem::current_path(here)); if (!parsed) { PrintError(parsed.GetError().Message()); return Catch::UnspecifiedErrorExitCode; } Runner::Options options = std::move(*parsed); options.programName = programName; if (options.mode == Runner::Mode::kHelp) { PrintHelp(programName); return 0; } if (options.mode == Runner::Mode::kVersion) { std::printf("Catch3 %s\n", kVersion); return 0; } if (ReportStartupErrors()) { return Catch::UnspecifiedErrorExitCode; } if (options.filenamesAsTags) { Catch::getMutableRegistryHub() .getMutableTestCaseRegistry() .enableFilenameTags(); } if (options.mode == Runner::Mode::kWorker) { return Runner::RunWorker(options, programName); } std::vector notes; if (!options.configPath.empty() && options.verbosity == Runner::Verbosity::kVerbose) { notes.push_back("configuration from " + options.configPath); } if (BazelTest()) { ApplyBazel(options, notes); } if (!options.seedGiven) { std::random_device device; options.seed = device(); } auto catalog = Runner::Catalog::Load(); if (!catalog) { PrintError(catalog.GetError().Message()); return Catch::UnspecifiedErrorExitCode; } const Catch::Config config{ Runner::ToConfigData(options, programName) }; const auto &invalid = config.testSpec().getInvalidSpecs(); if (!invalid.empty()) { for (const auto &spec : invalid) { PrintError("invalid test spec: " + spec); } return Catch::InvalidTestSpecExitCode; } std::vector unmatched; auto matched = catalog->Match(config, unmatched); if (options.mode == Runner::Mode::kListTests || options.mode == Runner::Mode::kListTags) { if (options.shard.count > 1) { // A shard lists the tests it would run, dependencies // included, in registration order. Runner::PlanInputs inputs; inputs.catalog = &*catalog; inputs.matched = matched; Runner::Options listing = options; listing.order = Runner::Order::kDeclared; auto plan = Runner::MakePlan(listing, inputs); if (!plan) { PrintError(plan.GetError().Message()); return Catch::UnspecifiedErrorExitCode; } matched.clear(); for (const auto &test : plan->tests) { matched.push_back(test.entry->index); } std::sort(matched.begin(), matched.end()); } return List(options, *catalog, matched, programName); } const std::filesystem::path historyFile = Runner::HistoryFile(HistoryDirectory(options, program), programName); Runner::History history; if (options.historyEnabled || options.mode == Runner::Mode::kAnalytics) { auto loaded = Runner::History::Load(historyFile); if (loaded) { history = std::move(*loaded); } else { notes.push_back("the history could not be read: " + loaded.GetError().Message()); } } if (options.mode == Runner::Mode::kAnalytics) { return Analytics(options, history, historyFile); } std::string binaryFingerprint; if (options.historyEnabled || options.incremental) { auto digest = Runner::Sha256File(program); binaryFingerprint = digest ? *digest : std::string{}; } std::optional changes; const auto readChanges = [&] { if (changes) { return; } auto read = Runner::ReadChanges(SourceDirectory(*catalog), options.changesSince); if (read) { changes = std::move(*read); } else if (options.selection == Runner::Selection::kChanged || options.selection == Runner::Selection::kSmart) { notes.push_back(read.GetError().Message()); } }; if (options.selection == Runner::Selection::kChanged || options.selection == Runner::Selection::kSmart) { readChanges(); } Runner::PlanInputs inputs; inputs.catalog = &*catalog; inputs.matched = matched; inputs.unmatchedSpecs = unmatched; inputs.history = options.historyEnabled ? &history : nullptr; inputs.changes = changes ? &*changes : nullptr; inputs.binaryFingerprint = binaryFingerprint; auto plan = Runner::MakePlan(options, inputs); if (!plan) { PrintError(plan.GetError().Message()); return Catch::UnspecifiedErrorExitCode; } notes.insert(notes.end(), plan->notes.begin(), plan->notes.end()); Runner::ConsoleReport console{ options, stdout }; const std::size_t workers = options.inProcess ? 1 : std::max( 1, std::min( options.jobs == 0 ? Runner::HardwareThreads() : options.jobs, plan->tests.size())); console.RunStarting(plan->tests.size(), workers, options.seed, notes); std::string guard = options.exitGuardPath; if (BazelTest() && guard.empty()) { guard = Runner::ReadEnvironment("TEST_PREMATURE_EXIT_FILE"); } if (!guard.empty()) { if (auto status = Runner::CreatePrivateFile(guard); !status) { PrintError("the exit guard file: " + status.GetError().Message()); return Catch::UnspecifiedErrorExitCode; } } const auto started = std::chrono::steady_clock::now(); RunResult run; run.binary = programName; run.seed = options.seed; run.shardIndex = options.shard.index; run.shardCount = options.shard.count; run.candidates = plan->candidates; run.unmatchedSpecs = plan->unmatchedSpecs; run.startUnixMilliseconds = UnixMilliseconds(); Runner::SchedulerEvents events; events.testFinished = [&](const TestResult &test) { console.TestFinished(test); }; events.retrying = [&](const TestResult &test, const Attempt &attempt) { console.Retrying(test, attempt); }; auto outcome = Runner::RunPlan(options, *plan, *catalog, program.string(), programName, events); run.wallMicroseconds = std::chrono::duration_cast( std::chrono::steady_clock::now() - started) .count(); run.workers = outcome.workers; run.tests = std::move(outcome.tests); for (auto &kept : plan->kept) { run.tests.push_back(std::move(kept)); } run.notes = std::move(outcome.notes); const bool anyFailure = std::any_of(run.tests.begin(), run.tests.end(), [](const TestResult &test) { return IsFailure(test.outcome) || test.flaky; }); if (anyFailure) { readChanges(); } Runner::AnalysisInputs analysis; analysis.history = options.historyEnabled ? &history : nullptr; analysis.changes = changes ? &*changes : nullptr; analysis.plan = &*plan; analysis.program = program.string(); analysis.changesSince = options.changesSince; Runner::Analyze(run, analysis); console.RunEnded(run); // A report that was asked for and is missing would let a build // server miss the run, so it fails the run. bool reportMissing = false; if (!options.junitPath.empty()) { if (auto status = Runner::WriteFileAtomically( options.junitPath, JUnitXmlWriter::ToString(run)); !status) { PrintError(status.GetError().Message()); reportMissing = true; } } if (!options.jsonPath.empty()) { if (auto status = Runner::WriteFileAtomically( options.jsonPath, Runner::ToJson(run).ToPrettyString()); !status) { PrintError(status.GetError().Message()); reportMissing = true; } } if (!options.tracePath.empty()) { if (auto status = Runner::WriteFileAtomically(options.tracePath, Runner::TraceJson(run)); !status) { PrintError(status.GetError().Message()); reportMissing = true; } } if (options.historyEnabled && !run.tests.empty()) { Runner::RunRecord record; record.unixMilliseconds = run.startUnixMilliseconds; record.commit = changes ? changes->commit : std::string{}; record.binaryFingerprint = binaryFingerprint; record.seed = run.seed; record.tests = run.tests.size(); record.wallMicroseconds = run.wallMicroseconds; for (const auto &test : run.tests) { record.failed += test.FailsRun() ? 1 : 0; } auto committed = Runner::History::Commit(historyFile, run, record, plan->fingerprints, {}); if (!committed) { std::fprintf(stderr, "warning: the history was not saved: %s\n", committed.GetError().Message().c_str()); } } if (!guard.empty()) { std::error_code error; std::filesystem::remove(guard, error); } if (outcome.infrastructureFailed) { return Catch::UnspecifiedErrorExitCode; } if (!run.unmatchedSpecs.empty() && options.warnUnmatchedSpecs) { return Catch::UnmatchedTestSpecExitCode; } const bool quarantineCounts = options.quarantine == Runner::QuarantineMode::kReport; bool failed = false; std::size_t ran = 0; std::size_t skipped = 0; for (const auto &test : run.tests) { if (test.outcome != Outcome::kUpToDate) { ++ran; } skipped += test.outcome == Outcome::kSkipped ? 1 : 0; failed = failed || test.FailsRun() || (quarantineCounts && test.quarantined && IsFailure(test.outcome)); } if (failed) { return Catch::TestFailureExitCode; } if (reportMissing) { return Catch::UnspecifiedErrorExitCode; } if (run.tests.empty() && !options.allowNoTests) { return Catch::NoTestsRunExitCode; } if (ran > 0 && skipped == ran && !options.allowNoTests) { return Catch::AllTestsSkippedExitCode; } return 0; } } // namespace int RunTests(const std::vector &arguments) { return Run(arguments); } void PrintHelp(const std::string &programName) { std::fputs(Runner::HelpText(programName).c_str(), stdout); } } // namespace Progmasoft::Catch3