diff options
| author | Leitwolf11 | 2026-10-24 12:41:08 +0300 |
|---|---|---|
| committer | Leitwolf11 | 2026-10-24 12:41:08 +0300 |
| commit | e18f19b7dd70c99fbcf2b3fe812ee18dabc92663 (patch) | |
| tree | 4cfc1a3c51ade9b8281925d1c122c12c87f8e381 /tests | |
| parent | e104b694513964d5a84e77f751203cda970d741b (diff) | |
| download | catch3-e18f19b7dd70c99fbcf2b3fe812ee18dabc92663.tar.gz | |
End the output capture when a test exits, fail on missing reports
A test that called exit() crashed its worker with SIGSEGV on Linux and
macOS. The exit hook read Catch2's captured output through a scoped
deactivation, which switched the capture back on; libstdc++ then flushed
std::cout at exit into a capture buffer that static destruction had
already freed. RunContext::endCapture() now ends the capture for good and
returns what it held, so the standard streams are their own again.
A JUnit, JSON or trace report that could not be written was only
reported on stderr, and a run whose tests passed still exited with 0, so a
build server could miss it. Such a run now exits with 1; failed tests
still exit with 42.
Integration tests of the runner's output move their helpers into
RunnerFixtureClient.hpp and cover --version, --help, the listing modes, a
bad option, exit codes 1 to 4, -v, -q, --slowest, colours, --analytics
in text and JSON, and Bazel's test environment.
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | tests/Progmasoft/BUILD.bazel | 2 | ||||
| -rw-r--r-- | tests/Progmasoft/Runner.tests.cpp | 178 | ||||
| -rw-r--r-- | tests/Progmasoft/RunnerFixtureClient.hpp | 213 | ||||
| -rw-r--r-- | tests/Progmasoft/RunnerOutput.tests.cpp | 223 | ||||
| -rw-r--r-- | tests/meson.build | 1 |
6 files changed, 449 insertions, 169 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e57ad94a..71411b53 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -160,6 +160,7 @@ add_executable(ProgmasoftSelfTest Progmasoft/RunnerPlanner.tests.cpp Progmasoft/RunnerOptions.tests.cpp Progmasoft/RunnerAnalysis.tests.cpp + Progmasoft/RunnerOutput.tests.cpp Progmasoft/Snapshot.tests.cpp Progmasoft/Stateful.tests.cpp Progmasoft/Strategy.tests.cpp diff --git a/tests/Progmasoft/BUILD.bazel b/tests/Progmasoft/BUILD.bazel index 06df86d6..b5fb21d8 100644 --- a/tests/Progmasoft/BUILD.bazel +++ b/tests/Progmasoft/BUILD.bazel @@ -12,10 +12,12 @@ cc_test( "Property.tests.cpp", "Results.tests.cpp", "Runner.tests.cpp", + "RunnerFixtureClient.hpp", "RunnerUnits.tests.cpp", "RunnerPlanner.tests.cpp", "RunnerOptions.tests.cpp", "RunnerAnalysis.tests.cpp", + "RunnerOutput.tests.cpp", "Snapshot.tests.cpp", "Stateful.tests.cpp", "Strategy.tests.cpp", diff --git a/tests/Progmasoft/Runner.tests.cpp b/tests/Progmasoft/Runner.tests.cpp index 3196b2a5..d458de10 100644 --- a/tests/Progmasoft/Runner.tests.cpp +++ b/tests/Progmasoft/Runner.tests.cpp @@ -3,6 +3,8 @@ // Starts the runner fixture and checks what the runner made of its tests. +#include "RunnerFixtureClient.hpp" + #include <Progmasoft/Runner/Files.hpp> #include <Progmasoft/Runner/Json.hpp> #include <Progmasoft/Runner/Process.hpp> @@ -21,174 +23,10 @@ namespace { namespace Runner = Progmasoft::Catch3::Runner; using Catch::Matchers::ContainsSubstring; - - struct FixtureRun final - { - Runner::ExitStatus status; - std::string output; - Runner::JsonValue report; - std::chrono::milliseconds wall{0}; - - /// The report entry of a test. - [[nodiscard]] const Runner::JsonValue & - Test(const std::string &name) const - { - for (const auto &test : report["tests"].Items()) - { - if (test["name"].AsString() == name) - { - return test; - } - } - FAIL("no result for '" << name << "' in:\n" << output); - return report; - } - - [[nodiscard]] std::string - Analysis(const std::string &name) const - { - std::string text; - for (const auto &finding : Test(name)["analysis"].Items()) - { - text += finding.AsString() + "\n"; - } - return text; - } - }; - - /// Finds a file Bazel provides as runfiles: beside the test, in the - /// runfiles tree, or through the manifest Bazel writes on Windows. - std::filesystem::path - FromRunfiles(const std::string &path) - { - std::error_code error; - if (std::filesystem::exists(path, error)) - { - return std::filesystem::absolute(path, error); - } - for (const char *variable : {"RUNFILES_DIR", "TEST_SRCDIR"}) - { - const auto directory = Runner::ReadEnvironment(variable); - if (!directory.empty()) - { - const auto candidate = - std::filesystem::path{directory} / "_main" / path; - if (std::filesystem::exists(candidate, error)) - { - return candidate; - } - } - } - const auto manifest = Runner::ReadEnvironment("RUNFILES_MANIFEST_FILE"); - if (!manifest.empty()) - { - auto text = Runner::ReadFile(manifest); - const std::string key = "_main/" + path + " "; - if (text) - { - const auto found = text->find(key); - if (found != std::string::npos) - { - const auto start = found + key.size(); - auto end = text->find('\n', start); - std::string target = text->substr( - start, end == std::string::npos ? std::string::npos : end - start); - if (!target.empty() && target.back() == '\r') - { - target.pop_back(); - } - return target; - } - } - } - return path; - } - - std::string - Fixture() - { - const auto path = Runner::ReadEnvironment("CATCH3_RUNNER_FIXTURE"); - if (path.empty()) - { - SKIP("CATCH3_RUNNER_FIXTURE names no fixture binary"); - } - return FromRunfiles(path).string(); - } - - /// A fresh directory for one test. - class Scratch final - { - public: - Scratch() - { - std::error_code error; - path_ = std::filesystem::temp_directory_path(error) / - ("catch3-runner-test-" + Runner::UniqueName()); - std::filesystem::create_directories(path_, error); - } - Scratch(const Scratch &) = delete; - Scratch & - operator=(const Scratch &) = delete; - ~Scratch() - { - std::error_code error; - std::filesystem::remove_all(path_, error); - } - - [[nodiscard]] const std::filesystem::path & - Path() const noexcept - { - return path_; - } - - private: - std::filesystem::path path_; - }; - - FixtureRun - RunFixture(const Scratch &scratch, - std::vector<std::string> arguments, - std::vector<std::pair<std::string, std::string>> environment = {}) - { - Runner::ChildOptions options; - options.arguments.push_back(Fixture()); - const auto report = scratch.Path() / ("report-" + Runner::UniqueName() + ".json"); - for (auto &argument : arguments) - { - options.arguments.push_back(std::move(argument)); - } - options.arguments.push_back("--json=" + report.string()); - options.arguments.emplace_back("--color=never"); - options.environment = std::move(environment); - options.environment.emplace_back("CATCH3_FIXTURE_STATE", - scratch.Path().string()); - const auto started = std::chrono::steady_clock::now(); - auto run = Runner::RunAndCapture(options); - INFO((run ? std::string{} : run.GetError().Message())); - REQUIRE(run); - FixtureRun result; - result.wall = std::chrono::duration_cast<std::chrono::milliseconds>( - std::chrono::steady_clock::now() - started); - result.status = run->status; - result.output = run->output + run->errors; - if (std::filesystem::exists(report)) - { - auto text = Runner::ReadFile(report); - REQUIRE(text); - auto json = Runner::ParseJson(*text); - REQUIRE(json); - result.report = std::move(*json); - } - return result; - } - - std::int64_t - ExitCode(const FixtureRun &run) - { - INFO(run.output); - REQUIRE(run.status.kind == Runner::ExitStatus::Kind::kExited); - return run.status.code; - } + using FixtureClient::ExitCode; + using FixtureClient::Fixture; + using FixtureClient::RunFixture; + using FixtureClient::Scratch; } // namespace TEST_CASE("Passing and failing tests get their verdicts", "[progmasoft][runner]") @@ -249,7 +87,9 @@ TEST_CASE("A hanging test is stopped at its time limit", "[progmasoft][runner]") CHECK(hang["category"].AsString() == "timeout"); CHECK_THAT(hang["attempts"].Items().front()["lastLocation"].AsString(), ContainsSubstring("RunnerFixture.cpp")); - CHECK(run.wall < std::chrono::seconds(30)); + // Well below the 60 s the test sleeps; a slow, busy disk can stretch + // the runner's own start-up and Git query to half a minute. + CHECK(run.wall < std::chrono::seconds(50)); } TEST_CASE("A test that passes on a retry is flaky", "[progmasoft][runner]") diff --git a/tests/Progmasoft/RunnerFixtureClient.hpp b/tests/Progmasoft/RunnerFixtureClient.hpp new file mode 100644 index 00000000..1352e1eb --- /dev/null +++ b/tests/Progmasoft/RunnerFixtureClient.hpp @@ -0,0 +1,213 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#pragma once + +// What the runner's integration tests share: finding the fixture binary, +// running it in a directory of its own and reading its JSON report. + +#include <Progmasoft/Runner/Files.hpp> +#include <Progmasoft/Runner/Json.hpp> +#include <Progmasoft/Runner/Process.hpp> +#include <catch2/catch_test_macros.hpp> +#include <chrono> +#include <cstdint> +#include <filesystem> +#include <string> +#include <utility> +#include <vector> + +namespace FixtureClient +{ + namespace Runner = Progmasoft::Catch3::Runner; + + struct FixtureRun final + { + Runner::ExitStatus status; + std::string output; + Runner::JsonValue report; + std::chrono::milliseconds wall{ 0 }; + + /// The report entry of a test. + [[nodiscard]] const Runner::JsonValue & + Test(const std::string &name) const + { + for (const auto &test : report["tests"].Items()) + { + if (test["name"].AsString() == name) + { + return test; + } + } + FAIL("no result for '" << name << "' in:\n" << output); + return report; + } + + [[nodiscard]] std::string + Analysis(const std::string &name) const + { + std::string text; + for (const auto &finding : Test(name)["analysis"].Items()) + { + text += finding.AsString() + "\n"; + } + return text; + } + }; + + /// Finds a file Bazel provides as runfiles: beside the test, in the + /// runfiles tree, or through the manifest Bazel writes on Windows. + inline std::filesystem::path + FromRunfiles(const std::string &path) + { + std::error_code error; + if (std::filesystem::exists(path, error)) + { + return std::filesystem::absolute(path, error); + } + for (const char *variable : { "RUNFILES_DIR", "TEST_SRCDIR" }) + { + const auto directory = Runner::ReadEnvironment(variable); + if (!directory.empty()) + { + const auto candidate + = std::filesystem::path{ directory } / "_main" / path; + if (std::filesystem::exists(candidate, error)) + { + return candidate; + } + } + } + const auto manifest = Runner::ReadEnvironment("RUNFILES_MANIFEST_FILE"); + if (!manifest.empty()) + { + auto text = Runner::ReadFile(manifest); + const std::string key = "_main/" + path + " "; + if (text) + { + const auto found = text->find(key); + if (found != std::string::npos) + { + const auto start = found + key.size(); + auto end = text->find('\n', start); + std::string target = text->substr(start, + end == std::string::npos + ? std::string::npos + : end - start); + if (!target.empty() && target.back() == '\r') + { + target.pop_back(); + } + return target; + } + } + } + return path; + } + + inline std::string + Fixture() + { + const auto path = Runner::ReadEnvironment("CATCH3_RUNNER_FIXTURE"); + if (path.empty()) + { + SKIP("CATCH3_RUNNER_FIXTURE names no fixture binary"); + } + return FromRunfiles(path).string(); + } + + /// A fresh directory for one test. + class Scratch final + { + public: + Scratch() + { + std::error_code error; + path_ = std::filesystem::temp_directory_path(error) + / ("catch3-runner-test-" + Runner::UniqueName()); + std::filesystem::create_directories(path_, error); + } + Scratch(const Scratch &) = delete; + Scratch & + operator=(const Scratch &) = delete; + ~Scratch() + { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + [[nodiscard]] const std::filesystem::path & + Path() const noexcept + { + return path_; + } + + private: + std::filesystem::path path_; + }; + + inline FixtureRun + RunFixture(const Scratch &scratch, + std::vector<std::string> arguments, + std::vector<std::pair<std::string, std::string>> environment + = {}) + { + Runner::ChildOptions options; + options.arguments.push_back(Fixture()); + const auto report + = scratch.Path() / ("report-" + Runner::UniqueName() + ".json"); + for (auto &argument : arguments) + { + options.arguments.push_back(std::move(argument)); + } + options.arguments.push_back("--json=" + report.string()); + options.arguments.emplace_back("--color=never"); + options.environment = std::move(environment); + options.environment.emplace_back("CATCH3_FIXTURE_STATE", + scratch.Path().string()); + const auto started = std::chrono::steady_clock::now(); + auto run = Runner::RunAndCapture(options); + INFO((run ? std::string{} : run.GetError().Message())); + REQUIRE(run); + FixtureRun result; + result.wall = std::chrono::duration_cast<std::chrono::milliseconds>( + std::chrono::steady_clock::now() - started); + result.status = run->status; + result.output = run->output + run->errors; + if (std::filesystem::exists(report)) + { + auto text = Runner::ReadFile(report); + REQUIRE(text); + auto json = Runner::ParseJson(*text); + REQUIRE(json); + result.report = std::move(*json); + } + return result; + } + + inline std::int64_t + ExitCode(const FixtureRun &run) + { + INFO(run.output); + REQUIRE(run.status.kind == Runner::ExitStatus::Kind::kExited); + return run.status.code; + } + + /// Runs the fixture with `arguments` as they are, for the modes that + /// write no report. + inline Runner::CapturedRun + RunPlain(std::vector<std::string> arguments, + std::vector<std::pair<std::string, std::string>> environment = {}) + { + Runner::ChildOptions options; + options.arguments.push_back(Fixture()); + for (auto &argument : arguments) + { + options.arguments.push_back(std::move(argument)); + } + options.environment = std::move(environment); + auto run = Runner::RunAndCapture(options); + INFO((run ? std::string{} : run.GetError().Message())); + REQUIRE(run); + return std::move(*run); + } +} // namespace FixtureClient diff --git a/tests/Progmasoft/RunnerOutput.tests.cpp b/tests/Progmasoft/RunnerOutput.tests.cpp new file mode 100644 index 00000000..572adf78 --- /dev/null +++ b/tests/Progmasoft/RunnerOutput.tests.cpp @@ -0,0 +1,223 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +// What the runner prints and returns: its modes, the console report, the +// exit codes and Bazel's test environment. + +#include <Progmasoft/Catch3/Version.hpp> +#include <Progmasoft/Runner/Files.hpp> +#include <Progmasoft/Runner/Json.hpp> +#include <catch2/catch_test_macros.hpp> +#include <catch2/matchers/catch_matchers_string.hpp> +#include <filesystem> +#include <fstream> +#include <string> +#include <vector> + +#include "RunnerFixtureClient.hpp" + +namespace +{ + namespace Runner = Progmasoft::Catch3::Runner; + using Catch::Matchers::ContainsSubstring; + using FixtureClient::ExitCode; + using FixtureClient::RunFixture; + using FixtureClient::RunPlain; + using FixtureClient::Scratch; + + std::int64_t + CodeOf(const Runner::CapturedRun &run) + { + INFO(run.output << run.errors); + REQUIRE(run.status.kind == Runner::ExitStatus::Kind::kExited); + return run.status.code; + } +} // namespace + +TEST_CASE("The runner tells its version and how to use it", + "[progmasoft][runner][output]") +{ + const auto version = RunPlain({ "--version" }); + CHECK(CodeOf(version) == 0); + CHECK_THAT(version.output, + ContainsSubstring(std::string{ "Catch3 " } + + Progmasoft::Catch3::kVersion)); + + const auto help = RunPlain({ "--help" }); + CHECK(CodeOf(help) == 0); + CHECK_THAT(help.output, ContainsSubstring("usage: ")); + CHECK_THAT(help.output, ContainsSubstring("--shard=INDEX/COUNT")); +} + +TEST_CASE("Tests and tags are listed", "[progmasoft][runner][output]") +{ + const auto text = RunPlain({ "--list", "[deps]" }); + CHECK(CodeOf(text) == 0); + CHECK_THAT(text.output, ContainsSubstring("fixture dependent")); + CHECK_THAT(text.output, ContainsSubstring("RunnerFixture.cpp:")); + CHECK_THAT(text.output, ContainsSubstring("[!depends:fixture base]")); + CHECK_THAT(text.output, ContainsSubstring("2 matching test(s)")); + + const auto names = RunPlain({ "--list=names", "[deps]" }); + CHECK_THAT(names.output, ContainsSubstring("fixture base")); + CHECK(names.output.find("matching") == std::string::npos); + + const auto tags = RunPlain({ "--list-tags" }); + CHECK(CodeOf(tags) == 0); + CHECK_THAT(tags.output, ContainsSubstring("[fixture]")); + CHECK_THAT(tags.output, ContainsSubstring("tag(s)")); +} + +TEST_CASE("Mistakes on the command line stop the run", + "[progmasoft][runner][output]") +{ + const auto unknown = RunPlain({ "--jbos=2" }); + CHECK(CodeOf(unknown) == 1); + CHECK_THAT(unknown.errors, ContainsSubstring("error: ")); + CHECK_THAT(unknown.errors, ContainsSubstring("did you mean --jobs?")); +} + +TEST_CASE("Exit codes say what kind of run it was", + "[progmasoft][runner][output]") +{ + const Scratch scratch; + SECTION("nothing matched") + { + const auto run + = RunFixture(scratch, { "--no-history", "no such test" }); + CHECK(ExitCode(run) == 2); + // Catch2 quotes a spec that has spaces. + CHECK_THAT( + run.output, + ContainsSubstring("warning: no test matches '\"no such test\"'")); + } + SECTION("nothing matched, and that was a warning") + { + const auto run = RunFixture(scratch, + { "--no-history", + "--warn=unmatched-specs", + "[pass]", + "no such test" }); + CHECK(ExitCode(run) == 3); + } + SECTION("nothing matched, and that was allowed") + { + const auto run = RunFixture( + scratch, + { "--no-history", "--allow-no-tests", "no such test" }); + CHECK(ExitCode(run) == 0); + } + SECTION("everything skipped") + { + const auto run = RunFixture(scratch, { "--no-history", "[skip]" }); + CHECK(ExitCode(run) == 4); + } + SECTION("a report that cannot be written") + { + // A directory where the report file should go. + const auto blocked = scratch.Path() / "blocked.xml"; + std::filesystem::create_directories(blocked / "inside"); + const auto run = RunFixture( + scratch, + { "--no-history", "[pass]", "--junit=" + blocked.string() }); + CHECK(ExitCode(run) == 1); + CHECK_THAT(run.output, ContainsSubstring("error: ")); + } +} + +TEST_CASE("The console report follows the verbosity", + "[progmasoft][runner][output]") +{ + const Scratch scratch; + // Passing tests get a line, and their output, only when asked for. + const auto normal = RunFixture(scratch, { "--no-history", "[pass]" }); + CHECK_THAT(normal.output, ContainsSubstring("Catch3: 2 test(s)")); + CHECK(normal.output.find("PASS fixture prints") == std::string::npos); + CHECK(normal.output.find("said through std::cout") == std::string::npos); + + const auto verbose + = RunFixture(scratch, { "--no-history", "-v", "[pass]" }); + CHECK_THAT(verbose.output, ContainsSubstring("PASS fixture prints")); + CHECK_THAT(verbose.output, ContainsSubstring("said through std::cout")); + + const auto quiet = RunFixture(scratch, { "--no-history", "-q", "[pass]" }); + CHECK(ExitCode(quiet) == 0); + CHECK(quiet.output.find("Catch3: ") == std::string::npos); + + const auto slowest + = RunFixture(scratch, { "--no-history", "--slowest=2", "[slow]" }); + CHECK_THAT(slowest.output, ContainsSubstring("slowest tests:")); + + const auto coloured + = RunPlain({ "--no-history", "--color=always", "[pass]" }); + CHECK_THAT(coloured.output, ContainsSubstring("\x1b[")); + const auto plain = RunPlain({ "--no-history", "--color=never", "[pass]" }); + CHECK(plain.output.find("\x1b[") == std::string::npos); +} + +TEST_CASE("The history is summarised on request", + "[progmasoft][runner][output]") +{ + const Scratch scratch; + const auto history = "--history=" + scratch.Path().string(); + for (int run = 0; run < 3; ++run) + { + static_cast<void>( + RunFixture(scratch, + { history, "--retries=off", "[pass],[alternate]" })); + } + + const auto text = RunPlain({ history, "--analytics" }); + CHECK(CodeOf(text) == 0); + CHECK_THAT(text.output, ContainsSubstring("3 run(s)")); + CHECK_THAT(text.output, ContainsSubstring("flaky (last 20 runs):")); + CHECK_THAT(text.output, ContainsSubstring("fixture alternates")); + CHECK_THAT(text.output, ContainsSubstring("slowest (median):")); + + const auto json = RunPlain({ history, "--analytics=json" }); + CHECK(CodeOf(json) == 0); + auto parsed = Runner::ParseJson(json.output); + REQUIRE(parsed); + CHECK((*parsed)["format"].AsString() == "catch3-analytics"); + CHECK((*parsed)["runs"].AsInteger() == 3); + CHECK_FALSE((*parsed)["tests"].Items().empty()); +} + +TEST_CASE("Bazel's test environment is honoured", + "[progmasoft][runner][output]") +{ + const Scratch scratch; + const auto xml = scratch.Path() / "test.xml"; + const auto outputs = scratch.Path() / "outputs"; + const auto status = scratch.Path() / "shard.status"; + std::filesystem::create_directories(outputs); + const auto run + = RunPlain({ "--color=never" }, + { { "BAZEL_TEST", "1" }, + { "XML_OUTPUT_FILE", xml.string() }, + { "TESTBRIDGE_TEST_ONLY", "fixture passes" }, + { "TEST_SHARD_INDEX", "0" }, + { "TEST_TOTAL_SHARDS", "1" }, + { "TEST_SHARD_STATUS_FILE", status.string() }, + { "TEST_RANDOM_SEED", "1234" }, + { "TEST_UNDECLARED_OUTPUTS_DIR", outputs.string() }, + { "CATCH3_FIXTURE_STATE", scratch.Path().string() } }); + CHECK(CodeOf(run) == 0); + CHECK_THAT(run.output, ContainsSubstring("1 test(s): 1 passed")); + CHECK(std::filesystem::exists(status)); + + auto junit = Runner::ReadFile(xml); + REQUIRE(junit); + CHECK_THAT(*junit, ContainsSubstring("fixture passes")); + + auto report = Runner::ReadFile(outputs / "catch3.json"); + REQUIRE(report); + auto parsed = Runner::ParseJson(*report); + REQUIRE(parsed); + CHECK((*parsed)["seed"].AsInteger() == 1234); + // A sandboxed test keeps no history beside the binary. + const auto beside + = std::filesystem::path{ FixtureClient::Fixture() }.parent_path() + / ".catch3"; + CHECK_FALSE(std::filesystem::exists(beside)); +} diff --git a/tests/meson.build b/tests/meson.build index dc1a8edf..4e42ee12 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -82,6 +82,7 @@ progmasoft_property_tests = executable( 'Progmasoft/RunnerPlanner.tests.cpp', 'Progmasoft/RunnerOptions.tests.cpp', 'Progmasoft/RunnerAnalysis.tests.cpp', + 'Progmasoft/RunnerOutput.tests.cpp', 'Progmasoft/Snapshot.tests.cpp', 'Progmasoft/Stateful.tests.cpp', 'Progmasoft/Strategy.tests.cpp', |