catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
End the output capture when a test exits, fail on missing reports
- Commit
e18f19b7dd70c99fbcf2b3fe812ee18dabc92663- Author
- Leitwolf11 ·
- Parents
e104b6945139
Changes
extras/catch_amalgamated.cpp | 235 ++++++++++++++++----------- extras/catch_amalgamated.hpp | 11 +- premake5.lua | 1 + src/Progmasoft/Runner/Execution.cpp | 2 +- src/Progmasoft/Runner/Runner.cpp | 257 ++++++++++++++++++------------ src/catch2/internal/catch_run_context.cpp | 4 +- src/catch2/internal/catch_run_context.hpp | 9 +- tests/CMakeLists.txt | 1 + tests/Progmasoft/BUILD.bazel | 2 + tests/Progmasoft/Runner.tests.cpp | 178 ++------------------- tests/Progmasoft/RunnerFixtureClient.hpp | 213 +++++++++++++++++++++++++ tests/Progmasoft/RunnerOutput.tests.cpp | 223 ++++++++++++++++++++++++++ tests/meson.build | 1 + xmake.lua | 1 + 14 files changed, 760 insertions(+), 378 deletions(-) diff --git a/extras/catch_amalgamated.cpp b/extras/catch_amalgamated.cpp index 6521ef84..808e6d2f 100644 --- a/extras/catch_amalgamated.cpp +++ b/extras/catch_amalgamated.cpp @@ -7,7 +7,7 @@ // SPDX-License-Identifier: BSL-1.0 // Catch v3.16.1 -// Generated: 2026-10-24 10:02:49.316745 +// Generated: 2026-10-24 12:33:30.890575 // ---------------------------------------------------------- // This file is an amalgamation of multiple different files. // You probably shouldn't edit it directly. @@ -6499,11 +6499,11 @@ namespace Catch { return m_atomicAssertionCount.failed >= m_abortAfterXFailedAssertions; } - std::string RunContext::capturedOutput() { + std::string RunContext::endCapture() { if ( !m_outputRedirect->isActive() ) { return {}; } - auto _ = scopedDeactivate( *m_outputRedirect ); + m_outputRedirect->deactivate(); return m_outputRedirect->getStdout() + m_outputRedirect->getStderr(); } @@ -11003,7 +11003,7 @@ namespace Progmasoft::Catch3::Runner { return; } - const std::string output = state->context->capturedOutput(); + const std::string output = state->context->endCapture(); std::fflush(stdout); std::fflush(stderr); WriteStandardError(output); @@ -16581,9 +16581,6 @@ namespace Progmasoft::Catch3::Runner } // namespace Progmasoft::Catch3::Runner // Copyright (c) 2026 Progmasoft. // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - - - #include <algorithm> #include <chrono> #include <cstdio> @@ -16591,6 +16588,7 @@ namespace Progmasoft::Catch3::Runner #include <map> #include <random> + namespace Progmasoft::Catch3 { namespace @@ -16658,19 +16656,22 @@ namespace Progmasoft::Catch3 { options.junitPath = xml; } - const std::string filter = - Runner::ReadEnvironment("TESTBRIDGE_TEST_ONLY"); + const std::string filter + = Runner::ReadEnvironment("TESTBRIDGE_TEST_ONLY"); if (!filter.empty()) { - options.testSpecs = {filter}; + options.testSpecs = { filter }; } - const std::string index = Runner::ReadEnvironment("TEST_SHARD_INDEX"); - const std::string total = Runner::ReadEnvironment("TEST_TOTAL_SHARDS"); - const std::string status = - Runner::ReadEnvironment("TEST_SHARD_STATUS_FILE"); + const std::string index + = Runner::ReadEnvironment("TEST_SHARD_INDEX"); + const std::string total + = Runner::ReadEnvironment("TEST_TOTAL_SHARDS"); + const std::string status + = Runner::ReadEnvironment("TEST_SHARD_STATUS_FILE"); if (!index.empty() && !total.empty()) { - auto parsed = Runner::ParseOptions({"--shard=" + index + "/" + total}); + auto parsed = Runner::ParseOptions( + { "--shard=" + index + "/" + total }); if (parsed) { options.shard = parsed->shard; @@ -16683,29 +16684,32 @@ namespace Progmasoft::Catch3 } else { - notes.push_back("ignoring Bazel's shard " + index + "/" + - total + ": " + parsed.GetError().Message()); + notes.push_back("ignoring Bazel's shard " + index + "/" + + total + ": " + + parsed.GetError().Message()); } } - const std::string seed = Runner::ReadEnvironment("TEST_RANDOM_SEED"); + const std::string seed + = Runner::ReadEnvironment("TEST_RANDOM_SEED"); if (!seed.empty()) { - auto parsed = Runner::ParseOptions({"--seed=" + seed}); + auto parsed = Runner::ParseOptions({ "--seed=" + seed }); if (parsed) { options.seed = parsed->seed; options.seedGiven = true; } } - const std::string outputs = - Runner::ReadEnvironment("TEST_UNDECLARED_OUTPUTS_DIR"); + const std::string outputs + = Runner::ReadEnvironment("TEST_UNDECLARED_OUTPUTS_DIR"); if (!outputs.empty() && options.jsonPath.empty()) { - options.jsonPath = - (std::filesystem::path{outputs} / "catch3.json").string(); + options.jsonPath + = (std::filesystem::path{ outputs } / "catch3.json") + .string(); } - if (options.historyDirectory.empty() && - !Runner::HasEnvironment("CATCH3_HISTORY")) + if (options.historyDirectory.empty() + && !Runner::HasEnvironment("CATCH3_HISTORY")) { // A sandboxed test has nowhere lasting to keep history. options.historyEnabled = false; @@ -16720,8 +16724,8 @@ namespace Progmasoft::Catch3 { return options.historyDirectory; } - const std::string fromEnvironment = - Runner::ReadEnvironment("CATCH3_HISTORY"); + const std::string fromEnvironment + = Runner::ReadEnvironment("CATCH3_HISTORY"); if (!fromEnvironment.empty()) { return fromEnvironment; @@ -16772,7 +16776,8 @@ namespace Progmasoft::Catch3 if (entry.timeout) { test.Set("timeoutSeconds", - static_cast<double>(entry.timeout->count()) / 1000.0); + static_cast<double>(entry.timeout->count()) + / 1000.0); } if (entry.retries) { @@ -16816,7 +16821,8 @@ namespace Progmasoft::Catch3 case Runner::ListFormat::kNames: for (const std::size_t index : matched) { - std::printf("%s\n", catalog.Tests()[index].name.c_str()); + std::printf("%s\n", + catalog.Tests()[index].name.c_str()); } break; case Runner::ListFormat::kText: @@ -16895,15 +16901,15 @@ namespace Progmasoft::Catch3 const auto median = [](std::vector<std::int64_t> values) { if (values.empty()) { - return std::int64_t{0}; + return std::int64_t{ 0 }; } std::sort(values.begin(), values.end()); return values[values.size() / 2]; }; row.failureRate = row.runs == 0 ? 0.0 - : static_cast<double>(failures) / - static_cast<double>(row.runs); + : static_cast<double>(failures) + / static_cast<double>(row.runs); row.flakyEvents = test.FlakyEvents(20); row.median = test.DurationPercentile(0.5).value_or(0); row.recentMedian = median(recent); @@ -16928,17 +16934,26 @@ namespace Progmasoft::Catch3 ordered.resize(std::min(count, ordered.size())); return ordered; }; - const auto flaky = - top([](const Row &row) { return static_cast<double>(row.flakyEvents); }, 10); - const auto failing = - top([](const Row &row) { return row.failureRate; }, 10); - const auto slowest = - top([](const Row &row) { return static_cast<double>(row.median); }, 10); + const auto flaky = top( + [](const Row &row) { + return static_cast<double>(row.flakyEvents); + }, + 10); + const auto failing = top( + [](const Row &row) { + return row.failureRate; + }, + 10); + const auto slowest = top( + [](const Row &row) { + return static_cast<double>(row.median); + }, + 10); const auto slower = top( [](const Row &row) { return row.olderMedian > 0 - ? static_cast<double>(row.recentMedian) / - static_cast<double>(row.olderMedian) + ? static_cast<double>(row.recentMedian) + / static_cast<double>(row.olderMedian) : 0.0; }, 10); @@ -16980,7 +16995,9 @@ namespace Progmasoft::Catch3 { if (row->flakyEvents > 0) { - std::printf(" %3zu %s\n", row->flakyEvents, row->key.c_str()); + std::printf(" %3zu %s\n", + row->flakyEvents, + row->key.c_str()); } } std::printf("\nin quarantine:\n"); @@ -16988,7 +17005,9 @@ namespace Progmasoft::Catch3 { if (!row.quarantine.empty()) { - std::printf(" %s: %s\n", row.key.c_str(), row.quarantine.c_str()); + std::printf(" %s: %s\n", + row.key.c_str(), + row.quarantine.c_str()); } } std::printf("\nfailing most often:\n"); @@ -17008,11 +17027,12 @@ namespace Progmasoft::Catch3 FormatMicroseconds(row->median).c_str(), row->key.c_str()); } - std::printf("\ngetting slower (last 5 runs against the 15 before):\n"); + std::printf( + "\ngetting slower (last 5 runs against the 15 before):\n"); for (const Row *row : slower) { - if (row->olderMedian > 0 && - row->recentMedian > row->olderMedian * 3 / 2) + if (row->olderMedian > 0 + && row->recentMedian > row->olderMedian * 3 / 2) { std::printf(" %10s -> %-10s %s\n", FormatMicroseconds(row->olderMedian).c_str(), @@ -17028,7 +17048,7 @@ namespace Progmasoft::Catch3 { for (const auto &entry : catalog.Tests()) { - const std::filesystem::path file{entry.file}; + const std::filesystem::path file{ entry.file }; if (file.is_absolute()) { return file.parent_path(); @@ -17051,8 +17071,8 @@ namespace Progmasoft::Catch3 { const std::filesystem::path program = Runner::CurrentExecutable( arguments.empty() ? nullptr : arguments.front().c_str()); - const std::string programName = - program.empty() ? "tests" : program.stem().string(); + const std::string programName + = program.empty() ? "tests" : program.stem().string(); std::vector<std::string> rest; if (arguments.size() > 1) { @@ -17109,7 +17129,8 @@ namespace Progmasoft::Catch3 PrintError(catalog.GetError().Message()); return Catch::UnspecifiedErrorExitCode; } - const Catch::Config config{Runner::ToConfigData(options, programName)}; + const Catch::Config config{ Runner::ToConfigData(options, + programName) }; const auto &invalid = config.testSpec().getInvalidSpecs(); if (!invalid.empty()) { @@ -17121,8 +17142,8 @@ namespace Progmasoft::Catch3 } std::vector<std::string> unmatched; auto matched = catalog->Match(config, unmatched); - if (options.mode == Runner::Mode::kListTests || - options.mode == Runner::Mode::kListTags) + if (options.mode == Runner::Mode::kListTests + || options.mode == Runner::Mode::kListTags) { if (options.shard.count > 1) { @@ -17149,10 +17170,12 @@ namespace Progmasoft::Catch3 return List(options, *catalog, matched, programName); } - const std::filesystem::path historyFile = Runner::HistoryFile( - HistoryDirectory(options, program), programName); + const std::filesystem::path historyFile + = Runner::HistoryFile(HistoryDirectory(options, program), + programName); Runner::History history; - if (options.historyEnabled || options.mode == Runner::Mode::kAnalytics) + if (options.historyEnabled + || options.mode == Runner::Mode::kAnalytics) { auto loaded = Runner::History::Load(historyFile); if (loaded) @@ -17161,8 +17184,8 @@ namespace Progmasoft::Catch3 } else { - notes.push_back("the history could not be read: " + - loaded.GetError().Message()); + notes.push_back("the history could not be read: " + + loaded.GetError().Message()); } } if (options.mode == Runner::Mode::kAnalytics) @@ -17189,14 +17212,14 @@ namespace Progmasoft::Catch3 { changes = std::move(*read); } - else if (options.selection == Runner::Selection::kChanged || - options.selection == Runner::Selection::kSmart) + else if (options.selection == Runner::Selection::kChanged + || options.selection == Runner::Selection::kSmart) { notes.push_back(read.GetError().Message()); } }; - if (options.selection == Runner::Selection::kChanged || - options.selection == Runner::Selection::kSmart) + if (options.selection == Runner::Selection::kChanged + || options.selection == Runner::Selection::kSmart) { readChanges(); } @@ -17216,17 +17239,20 @@ namespace Progmasoft::Catch3 } notes.insert(notes.end(), plan->notes.begin(), plan->notes.end()); - Runner::ConsoleReport console{options, stdout}; - const std::size_t workers = - options.inProcess - ? 1 - : std::max<std::size_t>( - 1, - std::min<std::size_t>( - options.jobs == 0 ? Runner::HardwareThreads() - : options.jobs, - plan->tests.size())); - console.RunStarting(plan->tests.size(), workers, options.seed, notes); + Runner::ConsoleReport console{ options, stdout }; + const std::size_t workers + = options.inProcess + ? 1 + : std::max<std::size_t>( + 1, + std::min<std::size_t>( + options.jobs == 0 ? Runner::HardwareThreads() + : options.jobs, + plan->tests.size())); + console.RunStarting(plan->tests.size(), + workers, + options.seed, + notes); std::string guard = options.exitGuardPath; if (BazelTest() && guard.empty()) @@ -17237,8 +17263,8 @@ namespace Progmasoft::Catch3 { if (auto status = Runner::CreatePrivateFile(guard); !status) { - PrintError("the exit guard file: " + - status.GetError().Message()); + PrintError("the exit guard file: " + + status.GetError().Message()); return Catch::UnspecifiedErrorExitCode; } } @@ -17256,14 +17282,20 @@ namespace Progmasoft::Catch3 events.testFinished = [&](const TestResult &test) { console.TestFinished(test); }; - events.retrying = [&](const TestResult &test, const Attempt &attempt) { - console.Retrying(test, attempt); - }; - auto outcome = Runner::RunPlan( - options, *plan, *catalog, program.string(), programName, events); - run.wallMicroseconds = std::chrono::duration_cast<std::chrono::microseconds>( - std::chrono::steady_clock::now() - started) - .count(); + events.retrying + = [&](const TestResult &test, const Attempt &attempt) { + console.Retrying(test, attempt); + }; + auto outcome = Runner::RunPlan(options, + *plan, + *catalog, + program.string(), + programName, + events); + run.wallMicroseconds + = std::chrono::duration_cast<std::chrono::microseconds>( + std::chrono::steady_clock::now() - started) + .count(); run.workers = outcome.workers; run.tests = std::move(outcome.tests); for (auto &kept : plan->kept) @@ -17272,10 +17304,12 @@ namespace Progmasoft::Catch3 } run.notes = std::move(outcome.notes); - const bool anyFailure = std::any_of( - run.tests.begin(), run.tests.end(), [](const TestResult &test) { - return IsFailure(test.outcome) || test.flaky; - }); + const bool anyFailure + = std::any_of(run.tests.begin(), + run.tests.end(), + [](const TestResult &test) { + return IsFailure(test.outcome) || test.flaky; + }); if (anyFailure) { readChanges(); @@ -17289,31 +17323,40 @@ namespace Progmasoft::Catch3 Runner::Analyze(run, analysis); console.RunEnded(run); + // A report that was asked for and is missing would let a build + // server miss the run, so it fails the run. + bool reportMissing = false; if (!options.junitPath.empty()) { if (auto status = Runner::WriteFileAtomically( - options.junitPath, JUnitXmlWriter::ToString(run)); + options.junitPath, + JUnitXmlWriter::ToString(run)); !status) { PrintError(status.GetError().Message()); + reportMissing = true; } } if (!options.jsonPath.empty()) { if (auto status = Runner::WriteFileAtomically( - options.jsonPath, Runner::ToJson(run).ToPrettyString()); + options.jsonPath, + Runner::ToJson(run).ToPrettyString()); !status) { PrintError(status.GetError().Message()); + reportMissing = true; } } if (!options.tracePath.empty()) { - if (auto status = Runner::WriteFileAtomically( - options.tracePath, Runner::TraceJson(run)); + if (auto status + = Runner::WriteFileAtomically(options.tracePath, + Runner::TraceJson(run)); !status) { PrintError(status.GetError().Message()); + reportMissing = true; } } if (options.historyEnabled && !run.tests.empty()) @@ -17355,8 +17398,8 @@ namespace Progmasoft::Catch3 { return Catch::UnmatchedTestSpecExitCode; } - const bool quarantineCounts = - options.quarantine == Runner::QuarantineMode::kReport; + const bool quarantineCounts + = options.quarantine == Runner::QuarantineMode::kReport; bool failed = false; std::size_t ran = 0; std::size_t skipped = 0; @@ -17367,14 +17410,18 @@ namespace Progmasoft::Catch3 ++ran; } skipped += test.outcome == Outcome::kSkipped ? 1 : 0; - failed = failed || test.FailsRun() || - (quarantineCounts && test.quarantined && - IsFailure(test.outcome)); + failed = failed || test.FailsRun() + || (quarantineCounts && test.quarantined + && IsFailure(test.outcome)); } if (failed) { return Catch::TestFailureExitCode; } + if (reportMissing) + { + return Catch::UnspecifiedErrorExitCode; + } if (run.tests.empty() && !options.allowNoTests) { return Catch::NoTestsRunExitCode; diff --git a/extras/catch_amalgamated.hpp b/extras/catch_amalgamated.hpp index 8d730b62..7fcefe70 100644 --- a/extras/catch_amalgamated.hpp +++ b/extras/catch_amalgamated.hpp @@ -7,7 +7,7 @@ // SPDX-License-Identifier: BSL-1.0 // Catch v3.16.1 -// Generated: 2026-10-24 10:02:48.985091 +// Generated: 2026-10-24 12:33:30.737192 // ---------------------------------------------------------- // This file is an amalgamation of multiple different files. // You probably shouldn't edit it directly. @@ -9896,9 +9896,12 @@ namespace Catch { // !TBD We need to do this another way! bool aborting() const; - //! What the running test printed so far and was captured; for a - //! process that ends inside a test, whose end is never reported. - std::string capturedOutput(); + //! Ends the capture of the running test's output and returns what + //! it captured; for a process that ends inside a test, whose end is + //! never reported. The standard streams are their own again, so + //! flushing them at exit no longer reaches the capture buffers, + //! which may be gone by then. + std::string endCapture(); private: void assertionPassedFastPath( SourceLineInfo lineInfo ); diff --git a/premake5.lua b/premake5.lua index 9d191077..316b4009 100644 --- a/premake5.lua +++ b/premake5.lua @@ -94,6 +94,7 @@ project "ProgmasoftPropertyTests" "tests/Progmasoft/RunnerPlanner.tests.cpp", "tests/Progmasoft/RunnerOptions.tests.cpp", "tests/Progmasoft/RunnerAnalysis.tests.cpp", + "tests/Progmasoft/RunnerOutput.tests.cpp", "tests/Progmasoft/Snapshot.tests.cpp", "tests/Progmasoft/Stateful.tests.cpp", "tests/Progmasoft/Strategy.tests.cpp", diff --git a/src/Progmasoft/Runner/Execution.cpp b/src/Progmasoft/Runner/Execution.cpp index ab1e92c4..77a4771a 100644 --- a/src/Progmasoft/Runner/Execution.cpp +++ b/src/Progmasoft/Runner/Execution.cpp @@ -507,7 +507,7 @@ namespace Progmasoft::Catch3::Runner { return; } - const std::string output = state->context->capturedOutput(); + const std::string output = state->context->endCapture(); std::fflush(stdout); std::fflush(stderr); WriteStandardError(output); diff --git a/src/Progmasoft/Runner/Runner.cpp b/src/Progmasoft/Runner/Runner.cpp index f85af36b..c4a6feb9 100644 --- a/src/Progmasoft/Runner/Runner.cpp +++ b/src/Progmasoft/Runner/Runner.cpp @@ -1,24 +1,9 @@ // Copyright (c) 2026 Progmasoft. // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 -#include "Analysis.hpp" -#include "Catalog.hpp" -#include "Console.hpp" -#include "Execution.hpp" -#include "Files.hpp" -#include "Git.hpp" -#include "Hash.hpp" -#include "History.hpp" -#include "Json.hpp" -#include "Options.hpp" -#include "Planner.hpp" -#include "Process.hpp" -#include "ResultsJson.hpp" -#include "Scheduler.hpp" - #include <Progmasoft/Catch3/Runner.hpp> #include <Progmasoft/Catch3/Version.hpp> #include <Progmasoft/Catch3/XmlWriter.hpp> - +#include <algorithm> #include <catch2/catch_config.hpp> #include <catch2/catch_session.hpp> #include <catch2/catch_user_config.hpp> @@ -28,14 +13,27 @@ #include <catch2/internal/catch_context.hpp> #include <catch2/internal/catch_startup_exception_registry.hpp> #include <catch2/internal/catch_test_case_registry_impl.hpp> - -#include <algorithm> #include <chrono> #include <cstdio> #include <exception> #include <map> #include <random> +#include "Analysis.hpp" +#include "Catalog.hpp" +#include "Console.hpp" +#include "Execution.hpp" +#include "Files.hpp" +#include "Git.hpp" +#include "Hash.hpp" +#include "History.hpp" +#include "Json.hpp" +#include "Options.hpp" +#include "Planner.hpp" +#include "Process.hpp" +#include "ResultsJson.hpp" +#include "Scheduler.hpp" + namespace Progmasoft::Catch3 { namespace @@ -103,19 +101,22 @@ namespace Progmasoft::Catch3 { options.junitPath = xml; } - const std::string filter = - Runner::ReadEnvironment("TESTBRIDGE_TEST_ONLY"); + const std::string filter + = Runner::ReadEnvironment("TESTBRIDGE_TEST_ONLY"); if (!filter.empty()) { - options.testSpecs = {filter}; + options.testSpecs = { filter }; } - const std::string index = Runner::ReadEnvironment("TEST_SHARD_INDEX"); - const std::string total = Runner::ReadEnvironment("TEST_TOTAL_SHARDS"); - const std::string status = - Runner::ReadEnvironment("TEST_SHARD_STATUS_FILE"); + const std::string index + = Runner::ReadEnvironment("TEST_SHARD_INDEX"); + const std::string total + = Runner::ReadEnvironment("TEST_TOTAL_SHARDS"); + const std::string status + = Runner::ReadEnvironment("TEST_SHARD_STATUS_FILE"); if (!index.empty() && !total.empty()) { - auto parsed = Runner::ParseOptions({"--shard=" + index + "/" + total}); + auto parsed = Runner::ParseOptions( + { "--shard=" + index + "/" + total }); if (parsed) { options.shard = parsed->shard; @@ -128,29 +129,32 @@ namespace Progmasoft::Catch3 } else { - notes.push_back("ignoring Bazel's shard " + index + "/" + - total + ": " + parsed.GetError().Message()); + notes.push_back("ignoring Bazel's shard " + index + "/" + + total + ": " + + parsed.GetError().Message()); } } - const std::string seed = Runner::ReadEnvironment("TEST_RANDOM_SEED"); + const std::string seed + = Runner::ReadEnvironment("TEST_RANDOM_SEED"); if (!seed.empty()) { - auto parsed = Runner::ParseOptions({"--seed=" + seed}); + auto parsed = Runner::ParseOptions({ "--seed=" + seed }); if (parsed) { options.seed = parsed->seed; options.seedGiven = true; } } - const std::string outputs = - Runner::ReadEnvironment("TEST_UNDECLARED_OUTPUTS_DIR"); + const std::string outputs + = Runner::ReadEnvironment("TEST_UNDECLARED_OUTPUTS_DIR"); if (!outputs.empty() && options.jsonPath.empty()) { - options.jsonPath = - (std::filesystem::path{outputs} / "catch3.json").string(); + options.jsonPath + = (std::filesystem::path{ outputs } / "catch3.json") + .string(); } - if (options.historyDirectory.empty() && - !Runner::HasEnvironment("CATCH3_HISTORY")) + if (options.historyDirectory.empty() + && !Runner::HasEnvironment("CATCH3_HISTORY")) { // A sandboxed test has nowhere lasting to keep history. options.historyEnabled = false; @@ -165,8 +169,8 @@ namespace Progmasoft::Catch3 { return options.historyDirectory; } - const std::string fromEnvironment = - Runner::ReadEnvironment("CATCH3_HISTORY"); + const std::string fromEnvironment + = Runner::ReadEnvironment("CATCH3_HISTORY"); if (!fromEnvironment.empty()) { return fromEnvironment; @@ -217,7 +221,8 @@ namespace Progmasoft::Catch3 if (entry.timeout) { test.Set("timeoutSeconds", - static_cast<double>(entry.timeout->count()) / 1000.0); + static_cast<double>(entry.timeout->count()) + / 1000.0); } if (entry.retries) { @@ -261,7 +266,8 @@ namespace Progmasoft::Catch3 case Runner::ListFormat::kNames: for (const std::size_t index : matched) { - std::printf("%s\n", catalog.Tests()[index].name.c_str()); + std::printf("%s\n", + catalog.Tests()[index].name.c_str()); } break; case Runner::ListFormat::kText: @@ -340,15 +346,15 @@ namespace Progmasoft::Catch3 const auto median = [](std::vector<std::int64_t> values) { if (values.empty()) { - return std::int64_t{0}; + return std::int64_t{ 0 }; } std::sort(values.begin(), values.end()); return values[values.size() / 2]; }; row.failureRate = row.runs == 0 ? 0.0 - : static_cast<double>(failures) / - static_cast<double>(row.runs); + : static_cast<double>(failures) + / static_cast<double>(row.runs); row.flakyEvents = test.FlakyEvents(20); row.median = test.DurationPercentile(0.5).value_or(0); row.recentMedian = median(recent); @@ -373,17 +379,26 @@ namespace Progmasoft::Catch3 ordered.resize(std::min(count, ordered.size())); return ordered; }; - const auto flaky = - top([](const Row &row) { return static_cast<double>(row.flakyEvents); }, 10); - const auto failing = - top([](const Row &row) { return row.failureRate; }, 10); - const auto slowest = - top([](const Row &row) { return static_cast<double>(row.median); }, 10); + const auto flaky = top( + [](const Row &row) { + return static_cast<double>(row.flakyEvents); + }, + 10); + const auto failing = top( + [](const Row &row) { + return row.failureRate; + }, + 10); + const auto slowest = top( + [](const Row &row) { + return static_cast<double>(row.median); + }, + 10); const auto slower = top( [](const Row &row) { return row.olderMedian > 0 - ? static_cast<double>(row.recentMedian) / - static_cast<double>(row.olderMedian) + ? static_cast<double>(row.recentMedian) + / static_cast<double>(row.olderMedian) : 0.0; }, 10); @@ -425,7 +440,9 @@ namespace Progmasoft::Catch3 { if (row->flakyEvents > 0) { - std::printf(" %3zu %s\n", row->flakyEvents, row->key.c_str()); + std::printf(" %3zu %s\n", + row->flakyEvents, + row->key.c_str()); } } std::printf("\nin quarantine:\n"); @@ -433,7 +450,9 @@ namespace Progmasoft::Catch3 { if (!row.quarantine.empty()) { - std::printf(" %s: %s\n", row.key.c_str(), row.quarantine.c_str()); + std::printf(" %s: %s\n", + row.key.c_str(), + row.quarantine.c_str()); } } std::printf("\nfailing most often:\n"); @@ -453,11 +472,12 @@ namespace Progmasoft::Catch3 FormatMicroseconds(row->median).c_str(), row->key.c_str()); } - std::printf("\ngetting slower (last 5 runs against the 15 before):\n"); + std::printf( + "\ngetting slower (last 5 runs against the 15 before):\n"); for (const Row *row : slower) { - if (row->olderMedian > 0 && - row->recentMedian > row->olderMedian * 3 / 2) + if (row->olderMedian > 0 + && row->recentMedian > row->olderMedian * 3 / 2) { std::printf(" %10s -> %-10s %s\n", FormatMicroseconds(row->olderMedian).c_str(), @@ -473,7 +493,7 @@ namespace Progmasoft::Catch3 { for (const auto &entry : catalog.Tests()) { - const std::filesystem::path file{entry.file}; + const std::filesystem::path file{ entry.file }; if (file.is_absolute()) { return file.parent_path(); @@ -496,8 +516,8 @@ namespace Progmasoft::Catch3 { const std::filesystem::path program = Runner::CurrentExecutable( arguments.empty() ? nullptr : arguments.front().c_str()); - const std::string programName = - program.empty() ? "tests" : program.stem().string(); + const std::string programName + = program.empty() ? "tests" : program.stem().string(); std::vector<std::string> rest; if (arguments.size() > 1) { @@ -554,7 +574,8 @@ namespace Progmasoft::Catch3 PrintError(catalog.GetError().Message()); return Catch::UnspecifiedErrorExitCode; } - const Catch::Config config{Runner::ToConfigData(options, programName)}; + const Catch::Config config{ Runner::ToConfigData(options, + programName) }; const auto &invalid = config.testSpec().getInvalidSpecs(); if (!invalid.empty()) { @@ -566,8 +587,8 @@ namespace Progmasoft::Catch3 } std::vector<std::string> unmatched; auto matched = catalog->Match(config, unmatched); - if (options.mode == Runner::Mode::kListTests || - options.mode == Runner::Mode::kListTags) + if (options.mode == Runner::Mode::kListTests + || options.mode == Runner::Mode::kListTags) { if (options.shard.count > 1) { @@ -594,10 +615,12 @@ namespace Progmasoft::Catch3 return List(options, *catalog, matched, programName); } - const std::filesystem::path historyFile = Runner::HistoryFile( - HistoryDirectory(options, program), programName); + const std::filesystem::path historyFile + = Runner::HistoryFile(HistoryDirectory(options, program), + programName); Runner::History history; - if (options.historyEnabled || options.mode == Runner::Mode::kAnalytics) + if (options.historyEnabled + || options.mode == Runner::Mode::kAnalytics) { auto loaded = Runner::History::Load(historyFile); if (loaded) @@ -606,8 +629,8 @@ namespace Progmasoft::Catch3 } else { - notes.push_back("the history could not be read: " + - loaded.GetError().Message()); + notes.push_back("the history could not be read: " + + loaded.GetError().Message()); } } if (options.mode == Runner::Mode::kAnalytics) @@ -634,14 +657,14 @@ namespace Progmasoft::Catch3 { changes = std::move(*read); } - else if (options.selection == Runner::Selection::kChanged || - options.selection == Runner::Selection::kSmart) + else if (options.selection == Runner::Selection::kChanged + || options.selection == Runner::Selection::kSmart) { notes.push_back(read.GetError().Message()); } }; - if (options.selection == Runner::Selection::kChanged || - options.selection == Runner::Selection::kSmart) + if (options.selection == Runner::Selection::kChanged + || options.selection == Runner::Selection::kSmart) { readChanges(); } @@ -661,17 +684,20 @@ namespace Progmasoft::Catch3 } notes.insert(notes.end(), plan->notes.begin(), plan->notes.end()); - Runner::ConsoleReport console{options, stdout}; - const std::size_t workers = - options.inProcess - ? 1 - : std::max<std::size_t>( - 1, - std::min<std::size_t>( - options.jobs == 0 ? Runner::HardwareThreads() - : options.jobs, - plan->tests.size())); - console.RunStarting(plan->tests.size(), workers, options.seed, notes); + Runner::ConsoleReport console{ options, stdout }; + const std::size_t workers + = options.inProcess + ? 1 + : std::max<std::size_t>( + 1, + std::min<std::size_t>( + options.jobs == 0 ? Runner::HardwareThreads() + : options.jobs, + plan->tests.size())); + console.RunStarting(plan->tests.size(), + workers, + options.seed, + notes); std::string guard = options.exitGuardPath; if (BazelTest() && guard.empty()) @@ -682,8 +708,8 @@ namespace Progmasoft::Catch3 { if (auto status = Runner::CreatePrivateFile(guard); !status) { - PrintError("the exit guard file: " + - status.GetError().Message()); + PrintError("the exit guard file: " + + status.GetError().Message()); return Catch::UnspecifiedErrorExitCode; } } @@ -701,14 +727,20 @@ namespace Progmasoft::Catch3 events.testFinished = [&](const TestResult &test) { console.TestFinished(test); }; - events.retrying = [&](const TestResult &test, const Attempt &attempt) { - console.Retrying(test, attempt); - }; - auto outcome = Runner::RunPlan( - options, *plan, *catalog, program.string(), programName, events); - run.wallMicroseconds = std::chrono::duration_cast<std::chrono::microseconds>( - std::chrono::steady_clock::now() - started) - .count(); + events.retrying + = [&](const TestResult &test, const Attempt &attempt) { + console.Retrying(test, attempt); + }; + auto outcome = Runner::RunPlan(options, + *plan, + *catalog, + program.string(), + programName, + events); + run.wallMicroseconds + = std::chrono::duration_cast<std::chrono::microseconds>( + std::chrono::steady_clock::now() - started) + .count(); run.workers = outcome.workers; run.tests = std::move(outcome.tests); for (auto &kept : plan->kept) @@ -717,10 +749,12 @@ namespace Progmasoft::Catch3 } run.notes = std::move(outcome.notes); - const bool anyFailure = std::any_of( - run.tests.begin(), run.tests.end(), [](const TestResult &test) { - return IsFailure(test.outcome) || test.flaky; - }); + const bool anyFailure + = std::any_of(run.tests.begin(), + run.tests.end(), + [](const TestResult &test) { + return IsFailure(test.outcome) || test.flaky; + }); if (anyFailure) { readChanges(); @@ -734,31 +768,40 @@ namespace Progmasoft::Catch3 Runner::Analyze(run, analysis); console.RunEnded(run); + // A report that was asked for and is missing would let a build + // server miss the run, so it fails the run. + bool reportMissing = false; if (!options.junitPath.empty()) { if (auto status = Runner::WriteFileAtomically( - options.junitPath, JUnitXmlWriter::ToString(run)); + options.junitPath, + JUnitXmlWriter::ToString(run)); !status) { PrintError(status.GetError().Message()); + reportMissing = true; } } if (!options.jsonPath.empty()) { if (auto status = Runner::WriteFileAtomically( - options.jsonPath, Runner::ToJson(run).ToPrettyString()); + options.jsonPath, + Runner::ToJson(run).ToPrettyString()); !status) { PrintError(status.GetError().Message()); + reportMissing = true; } } if (!options.tracePath.empty()) { - if (auto status = Runner::WriteFileAtomically( - options.tracePath, Runner::TraceJson(run)); + if (auto status + = Runner::WriteFileAtomically(options.tracePath, + Runner::TraceJson(run)); !status) { PrintError(status.GetError().Message()); + reportMissing = true; } } if (options.historyEnabled && !run.tests.empty()) @@ -800,8 +843,8 @@ namespace Progmasoft::Catch3 { return Catch::UnmatchedTestSpecExitCode; } - const bool quarantineCounts = - options.quarantine == Runner::QuarantineMode::kReport; + const bool quarantineCounts + = options.quarantine == Runner::QuarantineMode::kReport; bool failed = false; std::size_t ran = 0; std::size_t skipped = 0; @@ -812,14 +855,18 @@ namespace Progmasoft::Catch3 ++ran; } skipped += test.outcome == Outcome::kSkipped ? 1 : 0; - failed = failed || test.FailsRun() || - (quarantineCounts && test.quarantined && - IsFailure(test.outcome)); + failed = failed || test.FailsRun() + || (quarantineCounts && test.quarantined + && IsFailure(test.outcome)); } if (failed) { return Catch::TestFailureExitCode; } + if (reportMissing) + { + return Catch::UnspecifiedErrorExitCode; + } if (run.tests.empty() && !options.allowNoTests) { return Catch::NoTestsRunExitCode; diff --git a/src/catch2/internal/catch_run_context.cpp b/src/catch2/internal/catch_run_context.cpp index 5c57ef18..30fa3edd 100644 --- a/src/catch2/internal/catch_run_context.cpp +++ b/src/catch2/internal/catch_run_context.cpp @@ -748,11 +748,11 @@ namespace Catch { return m_atomicAssertionCount.failed >= m_abortAfterXFailedAssertions; } - std::string RunContext::capturedOutput() { + std::string RunContext::endCapture() { if ( !m_outputRedirect->isActive() ) { return {}; } - auto _ = scopedDeactivate( *m_outputRedirect ); + m_outputRedirect->deactivate(); return m_outputRedirect->getStdout() + m_outputRedirect->getStderr(); } diff --git a/src/catch2/internal/catch_run_context.hpp b/src/catch2/internal/catch_run_context.hpp index c7bee285..5feb8890 100644 --- a/src/catch2/internal/catch_run_context.hpp +++ b/src/catch2/internal/catch_run_context.hpp @@ -106,9 +106,12 @@ namespace Catch { // !TBD We need to do this another way! bool aborting() const; - //! What the running test printed so far and was captured; for a - //! process that ends inside a test, whose end is never reported. - std::string capturedOutput(); + //! Ends the capture of the running test's output and returns what + //! it captured; for a process that ends inside a test, whose end is + //! never reported. The standard streams are their own again, so + //! flushing them at exit no longer reaches the capture buffers, + //! which may be gone by then. + std::string endCapture(); private: void assertionPassedFastPath( SourceLineInfo lineInfo ); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e57ad94a..71411b53 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -160,6 +160,7 @@ add_executable(ProgmasoftSelfTest Progmasoft/RunnerPlanner.tests.cpp Progmasoft/RunnerOptions.tests.cpp Progmasoft/RunnerAnalysis.tests.cpp + Progmasoft/RunnerOutput.tests.cpp Progmasoft/Snapshot.tests.cpp Progmasoft/Stateful.tests.cpp Progmasoft/Strategy.tests.cpp diff --git a/tests/Progmasoft/BUILD.bazel b/tests/Progmasoft/BUILD.bazel index 06df86d6..b5fb21d8 100644 --- a/tests/Progmasoft/BUILD.bazel +++ b/tests/Progmasoft/BUILD.bazel @@ -12,10 +12,12 @@ cc_test( "Property.tests.cpp", "Results.tests.cpp", "Runner.tests.cpp", + "RunnerFixtureClient.hpp", "RunnerUnits.tests.cpp", "RunnerPlanner.tests.cpp", "RunnerOptions.tests.cpp", "RunnerAnalysis.tests.cpp", + "RunnerOutput.tests.cpp", "Snapshot.tests.cpp", "Stateful.tests.cpp", "Strategy.tests.cpp", diff --git a/tests/Progmasoft/Runner.tests.cpp b/tests/Progmasoft/Runner.tests.cpp index 3196b2a5..d458de10 100644 --- a/tests/Progmasoft/Runner.tests.cpp +++ b/tests/Progmasoft/Runner.tests.cpp @@ -3,6 +3,8 @@ // Starts the runner fixture and checks what the runner made of its tests. +#include "RunnerFixtureClient.hpp" + #include <Progmasoft/Runner/Files.hpp> #include <Progmasoft/Runner/Json.hpp> #include <Progmasoft/Runner/Process.hpp> @@ -21,174 +23,10 @@ namespace { namespace Runner = Progmasoft::Catch3::Runner; using Catch::Matchers::ContainsSubstring; - - struct FixtureRun final - { - Runner::ExitStatus status; - std::string output; - Runner::JsonValue report; - std::chrono::milliseconds wall{0}; - - /// The report entry of a test. - [[nodiscard]] const Runner::JsonValue & - Test(const std::string &name) const - { - for (const auto &test : report["tests"].Items()) - { - if (test["name"].AsString() == name) - { - return test; - } - } - FAIL("no result for '" << name << "' in:\n" << output); - return report; - } - - [[nodiscard]] std::string - Analysis(const std::string &name) const - { - std::string text; - for (const auto &finding : Test(name)["analysis"].Items()) - { - text += finding.AsString() + "\n"; - } - return text; - } - }; - - /// Finds a file Bazel provides as runfiles: beside the test, in the - /// runfiles tree, or through the manifest Bazel writes on Windows. - std::filesystem::path - FromRunfiles(const std::string &path) - { - std::error_code error; - if (std::filesystem::exists(path, error)) - { - return std::filesystem::absolute(path, error); - } - for (const char *variable : {"RUNFILES_DIR", "TEST_SRCDIR"}) - { - const auto directory = Runner::ReadEnvironment(variable); - if (!directory.empty()) - { - const auto candidate = - std::filesystem::path{directory} / "_main" / path; - if (std::filesystem::exists(candidate, error)) - { - return candidate; - } - } - } - const auto manifest = Runner::ReadEnvironment("RUNFILES_MANIFEST_FILE"); - if (!manifest.empty()) - { - auto text = Runner::ReadFile(manifest); - const std::string key = "_main/" + path + " "; - if (text) - { - const auto found = text->find(key); - if (found != std::string::npos) - { - const auto start = found + key.size(); - auto end = text->find('\n', start); - std::string target = text->substr( - start, end == std::string::npos ? std::string::npos : end - start); - if (!target.empty() && target.back() == '\r') - { - target.pop_back(); - } - return target; - } - } - } - return path; - } - - std::string - Fixture() - { - const auto path = Runner::ReadEnvironment("CATCH3_RUNNER_FIXTURE"); - if (path.empty()) - { - SKIP("CATCH3_RUNNER_FIXTURE names no fixture binary"); - } - return FromRunfiles(path).string(); - } - - /// A fresh directory for one test. - class Scratch final - { - public: - Scratch() - { - std::error_code error; - path_ = std::filesystem::temp_directory_path(error) / - ("catch3-runner-test-" + Runner::UniqueName()); - std::filesystem::create_directories(path_, error); - } - Scratch(const Scratch &) = delete; - Scratch & - operator=(const Scratch &) = delete; - ~Scratch() - { - std::error_code error; - std::filesystem::remove_all(path_, error); - } - - [[nodiscard]] const std::filesystem::path & - Path() const noexcept - { - return path_; - } - - private: - std::filesystem::path path_; - }; - - FixtureRun - RunFixture(const Scratch &scratch, - std::vector<std::string> arguments, - std::vector<std::pair<std::string, std::string>> environment = {}) - { - Runner::ChildOptions options; - options.arguments.push_back(Fixture()); - const auto report = scratch.Path() / ("report-" + Runner::UniqueName() + ".json"); - for (auto &argument : arguments) - { - options.arguments.push_back(std::move(argument)); - } - options.arguments.push_back("--json=" + report.string()); - options.arguments.emplace_back("--color=never"); - options.environment = std::move(environment); - options.environment.emplace_back("CATCH3_FIXTURE_STATE", - scratch.Path().string()); - const auto started = std::chrono::steady_clock::now(); - auto run = Runner::RunAndCapture(options); - INFO((run ? std::string{} : run.GetError().Message())); - REQUIRE(run); - FixtureRun result; - result.wall = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now() - started); - result.status = run->status; - result.output = run->output + run->errors; - if (std::filesystem::exists(report)) - { - auto text = Runner::ReadFile(report); - REQUIRE(text); - auto json = Runner::ParseJson(*text); - REQUIRE(json); - result.report = std::move(*json); - } - return result; - } - - std::int64_t - ExitCode(const FixtureRun &run) - { - INFO(run.output); - REQUIRE(run.status.kind == Runner::ExitStatus::Kind::kExited); - return run.status.code; - } + using FixtureClient::ExitCode; + using FixtureClient::Fixture; + using FixtureClient::RunFixture; + using FixtureClient::Scratch; } // namespace TEST_CASE("Passing and failing tests get their verdicts", "[progmasoft][runner]") @@ -249,7 +87,9 @@ TEST_CASE("A hanging test is stopped at its time limit", "[progmasoft][runner]") CHECK(hang["category"].AsString() == "timeout"); CHECK_THAT(hang["attempts"].Items().front()["lastLocation"].AsString(), ContainsSubstring("RunnerFixture.cpp")); - CHECK(run.wall < std::chrono::seconds(30)); + // Well below the 60 s the test sleeps; a slow, busy disk can stretch + // the runner's own start-up and Git query to half a minute. + CHECK(run.wall < std::chrono::seconds(50)); } TEST_CASE("A test that passes on a retry is flaky", "[progmasoft][runner]") diff --git a/tests/Progmasoft/RunnerFixtureClient.hpp b/tests/Progmasoft/RunnerFixtureClient.hpp new file mode 100644 index 00000000..1352e1eb --- /dev/null +++ b/tests/Progmasoft/RunnerFixtureClient.hpp @@ -0,0 +1,213 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#pragma once + +// What the runner's integration tests share: finding the fixture binary, +// running it in a directory of its own and reading its JSON report. + +#include <Progmasoft/Runner/Files.hpp> +#include <Progmasoft/Runner/Json.hpp> +#include <Progmasoft/Runner/Process.hpp> +#include <catch2/catch_test_macros.hpp> +#include <chrono> +#include <cstdint> +#include <filesystem> +#include <string> +#include <utility> +#include <vector> + +namespace FixtureClient +{ + namespace Runner = Progmasoft::Catch3::Runner; + + struct FixtureRun final + { + Runner::ExitStatus status; + std::string output; + Runner::JsonValue report; + std::chrono::milliseconds wall{ 0 }; + + /// The report entry of a test. + [[nodiscard]] const Runner::JsonValue & + Test(const std::string &name) const + { + for (const auto &test : report["tests"].Items()) + { + if (test["name"].AsString() == name) + { + return test; + } + } + FAIL("no result for '" << name << "' in:\n" << output); + return report; + } + + [[nodiscard]] std::string + Analysis(const std::string &name) const + { + std::string text; + for (const auto &finding : Test(name)["analysis"].Items()) + { + text += finding.AsString() + "\n"; + } + return text; + } + }; + + /// Finds a file Bazel provides as runfiles: beside the test, in the + /// runfiles tree, or through the manifest Bazel writes on Windows. + inline std::filesystem::path + FromRunfiles(const std::string &path) + { + std::error_code error; + if (std::filesystem::exists(path, error)) + { + return std::filesystem::absolute(path, error); + } + for (const char *variable : { "RUNFILES_DIR", "TEST_SRCDIR" }) + { + const auto directory = Runner::ReadEnvironment(variable); + if (!directory.empty()) + { + const auto candidate + = std::filesystem::path{ directory } / "_main" / path; + if (std::filesystem::exists(candidate, error)) + { + return candidate; + } + } + } + const auto manifest = Runner::ReadEnvironment("RUNFILES_MANIFEST_FILE"); + if (!manifest.empty()) + { + auto text = Runner::ReadFile(manifest); + const std::string key = "_main/" + path + " "; + if (text) + { + const auto found = text->find(key); + if (found != std::string::npos) + { + const auto start = found + key.size(); + auto end = text->find('\n', start); + std::string target = text->substr(start, + end == std::string::npos + ? std::string::npos + : end - start); + if (!target.empty() && target.back() == '\r') + { + target.pop_back(); + } + return target; + } + } + } + return path; + } + + inline std::string + Fixture() + { + const auto path = Runner::ReadEnvironment("CATCH3_RUNNER_FIXTURE"); + if (path.empty()) + { + SKIP("CATCH3_RUNNER_FIXTURE names no fixture binary"); + } + return FromRunfiles(path).string(); + } + + /// A fresh directory for one test. + class Scratch final + { + public: + Scratch() + { + std::error_code error; + path_ = std::filesystem::temp_directory_path(error) + / ("catch3-runner-test-" + Runner::UniqueName()); + std::filesystem::create_directories(path_, error); + } + Scratch(const Scratch &) = delete; + Scratch & + operator=(const Scratch &) = delete; + ~Scratch() + { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + [[nodiscard]] const std::filesystem::path & + Path() const noexcept + { + return path_; + } + + private: + std::filesystem::path path_; + }; + + inline FixtureRun + RunFixture(const Scratch &scratch, + std::vector<std::string> arguments, + std::vector<std::pair<std::string, std::string>> environment + = {}) + { + Runner::ChildOptions options; + options.arguments.push_back(Fixture()); + const auto report + = scratch.Path() / ("report-" + Runner::UniqueName() + ".json"); + for (auto &argument : arguments) + { + options.arguments.push_back(std::move(argument)); + } + options.arguments.push_back("--json=" + report.string()); + options.arguments.emplace_back("--color=never"); + options.environment = std::move(environment); + options.environment.emplace_back("CATCH3_FIXTURE_STATE", + scratch.Path().string()); + const auto started = std::chrono::steady_clock::now(); + auto run = Runner::RunAndCapture(options); + INFO((run ? std::string{} : run.GetError().Message())); + REQUIRE(run); + FixtureRun result; + result.wall = std::chrono::duration_cast<std::chrono::milliseconds>( + std::chrono::steady_clock::now() - started); + result.status = run->status; + result.output = run->output + run->errors; + if (std::filesystem::exists(report)) + { + auto text = Runner::ReadFile(report); + REQUIRE(text); + auto json = Runner::ParseJson(*text); + REQUIRE(json); + result.report = std::move(*json); + } + return result; + } + + inline std::int64_t + ExitCode(const FixtureRun &run) + { + INFO(run.output); + REQUIRE(run.status.kind == Runner::ExitStatus::Kind::kExited); + return run.status.code; + } + + /// Runs the fixture with `arguments` as they are, for the modes that + /// write no report. + inline Runner::CapturedRun + RunPlain(std::vector<std::string> arguments, + std::vector<std::pair<std::string, std::string>> environment = {}) + { + Runner::ChildOptions options; + options.arguments.push_back(Fixture()); + for (auto &argument : arguments) + { + options.arguments.push_back(std::move(argument)); + } + options.environment = std::move(environment); + auto run = Runner::RunAndCapture(options); + INFO((run ? std::string{} : run.GetError().Message())); + REQUIRE(run); + return std::move(*run); + } +} // namespace FixtureClient diff --git a/tests/Progmasoft/RunnerOutput.tests.cpp b/tests/Progmasoft/RunnerOutput.tests.cpp new file mode 100644 index 00000000..572adf78 --- /dev/null +++ b/tests/Progmasoft/RunnerOutput.tests.cpp @@ -0,0 +1,223 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +// What the runner prints and returns: its modes, the console report, the +// exit codes and Bazel's test environment. + +#include <Progmasoft/Catch3/Version.hpp> +#include <Progmasoft/Runner/Files.hpp> +#include <Progmasoft/Runner/Json.hpp> +#include <catch2/catch_test_macros.hpp> +#include <catch2/matchers/catch_matchers_string.hpp> +#include <filesystem> +#include <fstream> +#include <string> +#include <vector> + +#include "RunnerFixtureClient.hpp" + +namespace +{ + namespace Runner = Progmasoft::Catch3::Runner; + using Catch::Matchers::ContainsSubstring; + using FixtureClient::ExitCode; + using FixtureClient::RunFixture; + using FixtureClient::RunPlain; + using FixtureClient::Scratch; + + std::int64_t + CodeOf(const Runner::CapturedRun &run) + { + INFO(run.output << run.errors); + REQUIRE(run.status.kind == Runner::ExitStatus::Kind::kExited); + return run.status.code; + } +} // namespace + +TEST_CASE("The runner tells its version and how to use it", + "[progmasoft][runner][output]") +{ + const auto version = RunPlain({ "--version" }); + CHECK(CodeOf(version) == 0); + CHECK_THAT(version.output, + ContainsSubstring(std::string{ "Catch3 " } + + Progmasoft::Catch3::kVersion)); + + const auto help = RunPlain({ "--help" }); + CHECK(CodeOf(help) == 0); + CHECK_THAT(help.output, ContainsSubstring("usage: ")); + CHECK_THAT(help.output, ContainsSubstring("--shard=INDEX/COUNT")); +} + +TEST_CASE("Tests and tags are listed", "[progmasoft][runner][output]") +{ + const auto text = RunPlain({ "--list", "[deps]" }); + CHECK(CodeOf(text) == 0); + CHECK_THAT(text.output, ContainsSubstring("fixture dependent")); + CHECK_THAT(text.output, ContainsSubstring("RunnerFixture.cpp:")); + CHECK_THAT(text.output, ContainsSubstring("[!depends:fixture base]")); + CHECK_THAT(text.output, ContainsSubstring("2 matching test(s)")); + + const auto names = RunPlain({ "--list=names", "[deps]" }); + CHECK_THAT(names.output, ContainsSubstring("fixture base")); + CHECK(names.output.find("matching") == std::string::npos); + + const auto tags = RunPlain({ "--list-tags" }); + CHECK(CodeOf(tags) == 0); + CHECK_THAT(tags.output, ContainsSubstring("[fixture]")); + CHECK_THAT(tags.output, ContainsSubstring("tag(s)")); +} + +TEST_CASE("Mistakes on the command line stop the run", + "[progmasoft][runner][output]") +{ + const auto unknown = RunPlain({ "--jbos=2" }); + CHECK(CodeOf(unknown) == 1); + CHECK_THAT(unknown.errors, ContainsSubstring("error: ")); + CHECK_THAT(unknown.errors, ContainsSubstring("did you mean --jobs?")); +} + +TEST_CASE("Exit codes say what kind of run it was", + "[progmasoft][runner][output]") +{ + const Scratch scratch; + SECTION("nothing matched") + { + const auto run + = RunFixture(scratch, { "--no-history", "no such test" }); + CHECK(ExitCode(run) == 2); + // Catch2 quotes a spec that has spaces. + CHECK_THAT( + run.output, + ContainsSubstring("warning: no test matches '\"no such test\"'")); + } + SECTION("nothing matched, and that was a warning") + { + const auto run = RunFixture(scratch, + { "--no-history", + "--warn=unmatched-specs", + "[pass]", + "no such test" }); + CHECK(ExitCode(run) == 3); + } + SECTION("nothing matched, and that was allowed") + { + const auto run = RunFixture( + scratch, + { "--no-history", "--allow-no-tests", "no such test" }); + CHECK(ExitCode(run) == 0); + } + SECTION("everything skipped") + { + const auto run = RunFixture(scratch, { "--no-history", "[skip]" }); + CHECK(ExitCode(run) == 4); + } + SECTION("a report that cannot be written") + { + // A directory where the report file should go. + const auto blocked = scratch.Path() / "blocked.xml"; + std::filesystem::create_directories(blocked / "inside"); + const auto run = RunFixture( + scratch, + { "--no-history", "[pass]", "--junit=" + blocked.string() }); + CHECK(ExitCode(run) == 1); + CHECK_THAT(run.output, ContainsSubstring("error: ")); + } +} + +TEST_CASE("The console report follows the verbosity", + "[progmasoft][runner][output]") +{ + const Scratch scratch; + // Passing tests get a line, and their output, only when asked for. + const auto normal = RunFixture(scratch, { "--no-history", "[pass]" }); + CHECK_THAT(normal.output, ContainsSubstring("Catch3: 2 test(s)")); + CHECK(normal.output.find("PASS fixture prints") == std::string::npos); + CHECK(normal.output.find("said through std::cout") == std::string::npos); + + const auto verbose + = RunFixture(scratch, { "--no-history", "-v", "[pass]" }); + CHECK_THAT(verbose.output, ContainsSubstring("PASS fixture prints")); + CHECK_THAT(verbose.output, ContainsSubstring("said through std::cout")); + + const auto quiet = RunFixture(scratch, { "--no-history", "-q", "[pass]" }); + CHECK(ExitCode(quiet) == 0); + CHECK(quiet.output.find("Catch3: ") == std::string::npos); + + const auto slowest + = RunFixture(scratch, { "--no-history", "--slowest=2", "[slow]" }); + CHECK_THAT(slowest.output, ContainsSubstring("slowest tests:")); + + const auto coloured + = RunPlain({ "--no-history", "--color=always", "[pass]" }); + CHECK_THAT(coloured.output, ContainsSubstring("\x1b[")); + const auto plain = RunPlain({ "--no-history", "--color=never", "[pass]" }); + CHECK(plain.output.find("\x1b[") == std::string::npos); +} + +TEST_CASE("The history is summarised on request", + "[progmasoft][runner][output]") +{ + const Scratch scratch; + const auto history = "--history=" + scratch.Path().string(); + for (int run = 0; run < 3; ++run) + { + static_cast<void>( + RunFixture(scratch, + { history, "--retries=off", "[pass],[alternate]" })); + } + + const auto text = RunPlain({ history, "--analytics" }); + CHECK(CodeOf(text) == 0); + CHECK_THAT(text.output, ContainsSubstring("3 run(s)")); + CHECK_THAT(text.output, ContainsSubstring("flaky (last 20 runs):")); + CHECK_THAT(text.output, ContainsSubstring("fixture alternates")); + CHECK_THAT(text.output, ContainsSubstring("slowest (median):")); + + const auto json = RunPlain({ history, "--analytics=json" }); + CHECK(CodeOf(json) == 0); + auto parsed = Runner::ParseJson(json.output); + REQUIRE(parsed); + CHECK((*parsed)["format"].AsString() == "catch3-analytics"); + CHECK((*parsed)["runs"].AsInteger() == 3); + CHECK_FALSE((*parsed)["tests"].Items().empty()); +} + +TEST_CASE("Bazel's test environment is honoured", + "[progmasoft][runner][output]") +{ + const Scratch scratch; + const auto xml = scratch.Path() / "test.xml"; + const auto outputs = scratch.Path() / "outputs"; + const auto status = scratch.Path() / "shard.status"; + std::filesystem::create_directories(outputs); + const auto run + = RunPlain({ "--color=never" }, + { { "BAZEL_TEST", "1" }, + { "XML_OUTPUT_FILE", xml.string() }, + { "TESTBRIDGE_TEST_ONLY", "fixture passes" }, + { "TEST_SHARD_INDEX", "0" }, + { "TEST_TOTAL_SHARDS", "1" }, + { "TEST_SHARD_STATUS_FILE", status.string() }, + { "TEST_RANDOM_SEED", "1234" }, + { "TEST_UNDECLARED_OUTPUTS_DIR", outputs.string() }, + { "CATCH3_FIXTURE_STATE", scratch.Path().string() } }); + CHECK(CodeOf(run) == 0); + CHECK_THAT(run.output, ContainsSubstring("1 test(s): 1 passed")); + CHECK(std::filesystem::exists(status)); + + auto junit = Runner::ReadFile(xml); + REQUIRE(junit); + CHECK_THAT(*junit, ContainsSubstring("fixture passes")); + + auto report = Runner::ReadFile(outputs / "catch3.json"); + REQUIRE(report); + auto parsed = Runner::ParseJson(*report); + REQUIRE(parsed); + CHECK((*parsed)["seed"].AsInteger() == 1234); + // A sandboxed test keeps no history beside the binary. + const auto beside + = std::filesystem::path{ FixtureClient::Fixture() }.parent_path() + / ".catch3"; + CHECK_FALSE(std::filesystem::exists(beside)); +} diff --git a/tests/meson.build b/tests/meson.build index dc1a8edf..4e42ee12 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -82,6 +82,7 @@ progmasoft_property_tests = executable( 'Progmasoft/RunnerPlanner.tests.cpp', 'Progmasoft/RunnerOptions.tests.cpp', 'Progmasoft/RunnerAnalysis.tests.cpp', + 'Progmasoft/RunnerOutput.tests.cpp', 'Progmasoft/Snapshot.tests.cpp', 'Progmasoft/Stateful.tests.cpp', 'Progmasoft/Strategy.tests.cpp', diff --git a/xmake.lua b/xmake.lua index bcb486ca..aec3b1d4 100644 --- a/xmake.lua +++ b/xmake.lua @@ -67,6 +67,7 @@ target("ProgmasoftPropertyTests") "tests/Progmasoft/RunnerPlanner.tests.cpp", "tests/Progmasoft/RunnerOptions.tests.cpp", "tests/Progmasoft/RunnerAnalysis.tests.cpp", + "tests/Progmasoft/RunnerOutput.tests.cpp", "tests/Progmasoft/Snapshot.tests.cpp", "tests/Progmasoft/Stateful.tests.cpp", "tests/Progmasoft/Strategy.tests.cpp",