Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/tests
diff options
context:
space:
mode:
authorLeitwolf112026-10-24 12:41:08 +0300
committerLeitwolf112026-10-24 12:41:08 +0300
commite18f19b7dd70c99fbcf2b3fe812ee18dabc92663 (patch)
tree4cfc1a3c51ade9b8281925d1c122c12c87f8e381 /tests
parente104b694513964d5a84e77f751203cda970d741b (diff)
downloadcatch3-e18f19b7dd70c99fbcf2b3fe812ee18dabc92663.tar.gz
End the output capture when a test exits, fail on missing reports
A test that called exit() crashed its worker with SIGSEGV on Linux and macOS. The exit hook read Catch2's captured output through a scoped deactivation, which switched the capture back on; libstdc++ then flushed std::cout at exit into a capture buffer that static destruction had already freed. RunContext::endCapture() now ends the capture for good and returns what it held, so the standard streams are their own again. A JUnit, JSON or trace report that could not be written was only reported on stderr, and a run whose tests passed still exited with 0, so a build server could miss it. Such a run now exits with 1; failed tests still exit with 42. Integration tests of the runner's output move their helpers into RunnerFixtureClient.hpp and cover --version, --help, the listing modes, a bad option, exit codes 1 to 4, -v, -q, --slowest, colours, --analytics in text and JSON, and Bazel's test environment.
Diffstat (limited to 'tests')
-rw-r--r--tests/CMakeLists.txt1
-rw-r--r--tests/Progmasoft/BUILD.bazel2
-rw-r--r--tests/Progmasoft/Runner.tests.cpp178
-rw-r--r--tests/Progmasoft/RunnerFixtureClient.hpp213
-rw-r--r--tests/Progmasoft/RunnerOutput.tests.cpp223
-rw-r--r--tests/meson.build1
6 files changed, 449 insertions, 169 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index e57ad94a..71411b53 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -160,6 +160,7 @@ add_executable(ProgmasoftSelfTest
Progmasoft/RunnerPlanner.tests.cpp
Progmasoft/RunnerOptions.tests.cpp
Progmasoft/RunnerAnalysis.tests.cpp
+ Progmasoft/RunnerOutput.tests.cpp
Progmasoft/Snapshot.tests.cpp
Progmasoft/Stateful.tests.cpp
Progmasoft/Strategy.tests.cpp
diff --git a/tests/Progmasoft/BUILD.bazel b/tests/Progmasoft/BUILD.bazel
index 06df86d6..b5fb21d8 100644
--- a/tests/Progmasoft/BUILD.bazel
+++ b/tests/Progmasoft/BUILD.bazel
@@ -12,10 +12,12 @@ cc_test(
"Property.tests.cpp",
"Results.tests.cpp",
"Runner.tests.cpp",
+ "RunnerFixtureClient.hpp",
"RunnerUnits.tests.cpp",
"RunnerPlanner.tests.cpp",
"RunnerOptions.tests.cpp",
"RunnerAnalysis.tests.cpp",
+ "RunnerOutput.tests.cpp",
"Snapshot.tests.cpp",
"Stateful.tests.cpp",
"Strategy.tests.cpp",
diff --git a/tests/Progmasoft/Runner.tests.cpp b/tests/Progmasoft/Runner.tests.cpp
index 3196b2a5..d458de10 100644
--- a/tests/Progmasoft/Runner.tests.cpp
+++ b/tests/Progmasoft/Runner.tests.cpp
@@ -3,6 +3,8 @@
// Starts the runner fixture and checks what the runner made of its tests.
+#include "RunnerFixtureClient.hpp"
+
#include <Progmasoft/Runner/Files.hpp>
#include <Progmasoft/Runner/Json.hpp>
#include <Progmasoft/Runner/Process.hpp>
@@ -21,174 +23,10 @@ namespace
{
namespace Runner = Progmasoft::Catch3::Runner;
using Catch::Matchers::ContainsSubstring;
-
- struct FixtureRun final
- {
- Runner::ExitStatus status;
- std::string output;
- Runner::JsonValue report;
- std::chrono::milliseconds wall{0};
-
- /// The report entry of a test.
- [[nodiscard]] const Runner::JsonValue &
- Test(const std::string &name) const
- {
- for (const auto &test : report["tests"].Items())
- {
- if (test["name"].AsString() == name)
- {
- return test;
- }
- }
- FAIL("no result for '" << name << "' in:\n" << output);
- return report;
- }
-
- [[nodiscard]] std::string
- Analysis(const std::string &name) const
- {
- std::string text;
- for (const auto &finding : Test(name)["analysis"].Items())
- {
- text += finding.AsString() + "\n";
- }
- return text;
- }
- };
-
- /// Finds a file Bazel provides as runfiles: beside the test, in the
- /// runfiles tree, or through the manifest Bazel writes on Windows.
- std::filesystem::path
- FromRunfiles(const std::string &path)
- {
- std::error_code error;
- if (std::filesystem::exists(path, error))
- {
- return std::filesystem::absolute(path, error);
- }
- for (const char *variable : {"RUNFILES_DIR", "TEST_SRCDIR"})
- {
- const auto directory = Runner::ReadEnvironment(variable);
- if (!directory.empty())
- {
- const auto candidate =
- std::filesystem::path{directory} / "_main" / path;
- if (std::filesystem::exists(candidate, error))
- {
- return candidate;
- }
- }
- }
- const auto manifest = Runner::ReadEnvironment("RUNFILES_MANIFEST_FILE");
- if (!manifest.empty())
- {
- auto text = Runner::ReadFile(manifest);
- const std::string key = "_main/" + path + " ";
- if (text)
- {
- const auto found = text->find(key);
- if (found != std::string::npos)
- {
- const auto start = found + key.size();
- auto end = text->find('\n', start);
- std::string target = text->substr(
- start, end == std::string::npos ? std::string::npos : end - start);
- if (!target.empty() && target.back() == '\r')
- {
- target.pop_back();
- }
- return target;
- }
- }
- }
- return path;
- }
-
- std::string
- Fixture()
- {
- const auto path = Runner::ReadEnvironment("CATCH3_RUNNER_FIXTURE");
- if (path.empty())
- {
- SKIP("CATCH3_RUNNER_FIXTURE names no fixture binary");
- }
- return FromRunfiles(path).string();
- }
-
- /// A fresh directory for one test.
- class Scratch final
- {
- public:
- Scratch()
- {
- std::error_code error;
- path_ = std::filesystem::temp_directory_path(error) /
- ("catch3-runner-test-" + Runner::UniqueName());
- std::filesystem::create_directories(path_, error);
- }
- Scratch(const Scratch &) = delete;
- Scratch &
- operator=(const Scratch &) = delete;
- ~Scratch()
- {
- std::error_code error;
- std::filesystem::remove_all(path_, error);
- }
-
- [[nodiscard]] const std::filesystem::path &
- Path() const noexcept
- {
- return path_;
- }
-
- private:
- std::filesystem::path path_;
- };
-
- FixtureRun
- RunFixture(const Scratch &scratch,
- std::vector<std::string> arguments,
- std::vector<std::pair<std::string, std::string>> environment = {})
- {
- Runner::ChildOptions options;
- options.arguments.push_back(Fixture());
- const auto report = scratch.Path() / ("report-" + Runner::UniqueName() + ".json");
- for (auto &argument : arguments)
- {
- options.arguments.push_back(std::move(argument));
- }
- options.arguments.push_back("--json=" + report.string());
- options.arguments.emplace_back("--color=never");
- options.environment = std::move(environment);
- options.environment.emplace_back("CATCH3_FIXTURE_STATE",
- scratch.Path().string());
- const auto started = std::chrono::steady_clock::now();
- auto run = Runner::RunAndCapture(options);
- INFO((run ? std::string{} : run.GetError().Message()));
- REQUIRE(run);
- FixtureRun result;
- result.wall = std::chrono::duration_cast<std::chrono::milliseconds>(
- std::chrono::steady_clock::now() - started);
- result.status = run->status;
- result.output = run->output + run->errors;
- if (std::filesystem::exists(report))
- {
- auto text = Runner::ReadFile(report);
- REQUIRE(text);
- auto json = Runner::ParseJson(*text);
- REQUIRE(json);
- result.report = std::move(*json);
- }
- return result;
- }
-
- std::int64_t
- ExitCode(const FixtureRun &run)
- {
- INFO(run.output);
- REQUIRE(run.status.kind == Runner::ExitStatus::Kind::kExited);
- return run.status.code;
- }
+ using FixtureClient::ExitCode;
+ using FixtureClient::Fixture;
+ using FixtureClient::RunFixture;
+ using FixtureClient::Scratch;
} // namespace
TEST_CASE("Passing and failing tests get their verdicts", "[progmasoft][runner]")
@@ -249,7 +87,9 @@ TEST_CASE("A hanging test is stopped at its time limit", "[progmasoft][runner]")
CHECK(hang["category"].AsString() == "timeout");
CHECK_THAT(hang["attempts"].Items().front()["lastLocation"].AsString(),
ContainsSubstring("RunnerFixture.cpp"));
- CHECK(run.wall < std::chrono::seconds(30));
+ // Well below the 60 s the test sleeps; a slow, busy disk can stretch
+ // the runner's own start-up and Git query to half a minute.
+ CHECK(run.wall < std::chrono::seconds(50));
}
TEST_CASE("A test that passes on a retry is flaky", "[progmasoft][runner]")
diff --git a/tests/Progmasoft/RunnerFixtureClient.hpp b/tests/Progmasoft/RunnerFixtureClient.hpp
new file mode 100644
index 00000000..1352e1eb
--- /dev/null
+++ b/tests/Progmasoft/RunnerFixtureClient.hpp
@@ -0,0 +1,213 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+#pragma once
+
+// What the runner's integration tests share: finding the fixture binary,
+// running it in a directory of its own and reading its JSON report.
+
+#include <Progmasoft/Runner/Files.hpp>
+#include <Progmasoft/Runner/Json.hpp>
+#include <Progmasoft/Runner/Process.hpp>
+#include <catch2/catch_test_macros.hpp>
+#include <chrono>
+#include <cstdint>
+#include <filesystem>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace FixtureClient
+{
+ namespace Runner = Progmasoft::Catch3::Runner;
+
+ struct FixtureRun final
+ {
+ Runner::ExitStatus status;
+ std::string output;
+ Runner::JsonValue report;
+ std::chrono::milliseconds wall{ 0 };
+
+ /// The report entry of a test.
+ [[nodiscard]] const Runner::JsonValue &
+ Test(const std::string &name) const
+ {
+ for (const auto &test : report["tests"].Items())
+ {
+ if (test["name"].AsString() == name)
+ {
+ return test;
+ }
+ }
+ FAIL("no result for '" << name << "' in:\n" << output);
+ return report;
+ }
+
+ [[nodiscard]] std::string
+ Analysis(const std::string &name) const
+ {
+ std::string text;
+ for (const auto &finding : Test(name)["analysis"].Items())
+ {
+ text += finding.AsString() + "\n";
+ }
+ return text;
+ }
+ };
+
+ /// Finds a file Bazel provides as runfiles: beside the test, in the
+ /// runfiles tree, or through the manifest Bazel writes on Windows.
+ inline std::filesystem::path
+ FromRunfiles(const std::string &path)
+ {
+ std::error_code error;
+ if (std::filesystem::exists(path, error))
+ {
+ return std::filesystem::absolute(path, error);
+ }
+ for (const char *variable : { "RUNFILES_DIR", "TEST_SRCDIR" })
+ {
+ const auto directory = Runner::ReadEnvironment(variable);
+ if (!directory.empty())
+ {
+ const auto candidate
+ = std::filesystem::path{ directory } / "_main" / path;
+ if (std::filesystem::exists(candidate, error))
+ {
+ return candidate;
+ }
+ }
+ }
+ const auto manifest = Runner::ReadEnvironment("RUNFILES_MANIFEST_FILE");
+ if (!manifest.empty())
+ {
+ auto text = Runner::ReadFile(manifest);
+ const std::string key = "_main/" + path + " ";
+ if (text)
+ {
+ const auto found = text->find(key);
+ if (found != std::string::npos)
+ {
+ const auto start = found + key.size();
+ auto end = text->find('\n', start);
+ std::string target = text->substr(start,
+ end == std::string::npos
+ ? std::string::npos
+ : end - start);
+ if (!target.empty() && target.back() == '\r')
+ {
+ target.pop_back();
+ }
+ return target;
+ }
+ }
+ }
+ return path;
+ }
+
+ inline std::string
+ Fixture()
+ {
+ const auto path = Runner::ReadEnvironment("CATCH3_RUNNER_FIXTURE");
+ if (path.empty())
+ {
+ SKIP("CATCH3_RUNNER_FIXTURE names no fixture binary");
+ }
+ return FromRunfiles(path).string();
+ }
+
+ /// A fresh directory for one test.
+ class Scratch final
+ {
+ public:
+ Scratch()
+ {
+ std::error_code error;
+ path_ = std::filesystem::temp_directory_path(error)
+ / ("catch3-runner-test-" + Runner::UniqueName());
+ std::filesystem::create_directories(path_, error);
+ }
+ Scratch(const Scratch &) = delete;
+ Scratch &
+ operator=(const Scratch &) = delete;
+ ~Scratch()
+ {
+ std::error_code error;
+ std::filesystem::remove_all(path_, error);
+ }
+
+ [[nodiscard]] const std::filesystem::path &
+ Path() const noexcept
+ {
+ return path_;
+ }
+
+ private:
+ std::filesystem::path path_;
+ };
+
+ inline FixtureRun
+ RunFixture(const Scratch &scratch,
+ std::vector<std::string> arguments,
+ std::vector<std::pair<std::string, std::string>> environment
+ = {})
+ {
+ Runner::ChildOptions options;
+ options.arguments.push_back(Fixture());
+ const auto report
+ = scratch.Path() / ("report-" + Runner::UniqueName() + ".json");
+ for (auto &argument : arguments)
+ {
+ options.arguments.push_back(std::move(argument));
+ }
+ options.arguments.push_back("--json=" + report.string());
+ options.arguments.emplace_back("--color=never");
+ options.environment = std::move(environment);
+ options.environment.emplace_back("CATCH3_FIXTURE_STATE",
+ scratch.Path().string());
+ const auto started = std::chrono::steady_clock::now();
+ auto run = Runner::RunAndCapture(options);
+ INFO((run ? std::string{} : run.GetError().Message()));
+ REQUIRE(run);
+ FixtureRun result;
+ result.wall = std::chrono::duration_cast<std::chrono::milliseconds>(
+ std::chrono::steady_clock::now() - started);
+ result.status = run->status;
+ result.output = run->output + run->errors;
+ if (std::filesystem::exists(report))
+ {
+ auto text = Runner::ReadFile(report);
+ REQUIRE(text);
+ auto json = Runner::ParseJson(*text);
+ REQUIRE(json);
+ result.report = std::move(*json);
+ }
+ return result;
+ }
+
+ inline std::int64_t
+ ExitCode(const FixtureRun &run)
+ {
+ INFO(run.output);
+ REQUIRE(run.status.kind == Runner::ExitStatus::Kind::kExited);
+ return run.status.code;
+ }
+
+ /// Runs the fixture with `arguments` as they are, for the modes that
+ /// write no report.
+ inline Runner::CapturedRun
+ RunPlain(std::vector<std::string> arguments,
+ std::vector<std::pair<std::string, std::string>> environment = {})
+ {
+ Runner::ChildOptions options;
+ options.arguments.push_back(Fixture());
+ for (auto &argument : arguments)
+ {
+ options.arguments.push_back(std::move(argument));
+ }
+ options.environment = std::move(environment);
+ auto run = Runner::RunAndCapture(options);
+ INFO((run ? std::string{} : run.GetError().Message()));
+ REQUIRE(run);
+ return std::move(*run);
+ }
+} // namespace FixtureClient
diff --git a/tests/Progmasoft/RunnerOutput.tests.cpp b/tests/Progmasoft/RunnerOutput.tests.cpp
new file mode 100644
index 00000000..572adf78
--- /dev/null
+++ b/tests/Progmasoft/RunnerOutput.tests.cpp
@@ -0,0 +1,223 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+
+// What the runner prints and returns: its modes, the console report, the
+// exit codes and Bazel's test environment.
+
+#include <Progmasoft/Catch3/Version.hpp>
+#include <Progmasoft/Runner/Files.hpp>
+#include <Progmasoft/Runner/Json.hpp>
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/matchers/catch_matchers_string.hpp>
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <vector>
+
+#include "RunnerFixtureClient.hpp"
+
+namespace
+{
+ namespace Runner = Progmasoft::Catch3::Runner;
+ using Catch::Matchers::ContainsSubstring;
+ using FixtureClient::ExitCode;
+ using FixtureClient::RunFixture;
+ using FixtureClient::RunPlain;
+ using FixtureClient::Scratch;
+
+ std::int64_t
+ CodeOf(const Runner::CapturedRun &run)
+ {
+ INFO(run.output << run.errors);
+ REQUIRE(run.status.kind == Runner::ExitStatus::Kind::kExited);
+ return run.status.code;
+ }
+} // namespace
+
+TEST_CASE("The runner tells its version and how to use it",
+ "[progmasoft][runner][output]")
+{
+ const auto version = RunPlain({ "--version" });
+ CHECK(CodeOf(version) == 0);
+ CHECK_THAT(version.output,
+ ContainsSubstring(std::string{ "Catch3 " }
+ + Progmasoft::Catch3::kVersion));
+
+ const auto help = RunPlain({ "--help" });
+ CHECK(CodeOf(help) == 0);
+ CHECK_THAT(help.output, ContainsSubstring("usage: "));
+ CHECK_THAT(help.output, ContainsSubstring("--shard=INDEX/COUNT"));
+}
+
+TEST_CASE("Tests and tags are listed", "[progmasoft][runner][output]")
+{
+ const auto text = RunPlain({ "--list", "[deps]" });
+ CHECK(CodeOf(text) == 0);
+ CHECK_THAT(text.output, ContainsSubstring("fixture dependent"));
+ CHECK_THAT(text.output, ContainsSubstring("RunnerFixture.cpp:"));
+ CHECK_THAT(text.output, ContainsSubstring("[!depends:fixture base]"));
+ CHECK_THAT(text.output, ContainsSubstring("2 matching test(s)"));
+
+ const auto names = RunPlain({ "--list=names", "[deps]" });
+ CHECK_THAT(names.output, ContainsSubstring("fixture base"));
+ CHECK(names.output.find("matching") == std::string::npos);
+
+ const auto tags = RunPlain({ "--list-tags" });
+ CHECK(CodeOf(tags) == 0);
+ CHECK_THAT(tags.output, ContainsSubstring("[fixture]"));
+ CHECK_THAT(tags.output, ContainsSubstring("tag(s)"));
+}
+
+TEST_CASE("Mistakes on the command line stop the run",
+ "[progmasoft][runner][output]")
+{
+ const auto unknown = RunPlain({ "--jbos=2" });
+ CHECK(CodeOf(unknown) == 1);
+ CHECK_THAT(unknown.errors, ContainsSubstring("error: "));
+ CHECK_THAT(unknown.errors, ContainsSubstring("did you mean --jobs?"));
+}
+
+TEST_CASE("Exit codes say what kind of run it was",
+ "[progmasoft][runner][output]")
+{
+ const Scratch scratch;
+ SECTION("nothing matched")
+ {
+ const auto run
+ = RunFixture(scratch, { "--no-history", "no such test" });
+ CHECK(ExitCode(run) == 2);
+ // Catch2 quotes a spec that has spaces.
+ CHECK_THAT(
+ run.output,
+ ContainsSubstring("warning: no test matches '\"no such test\"'"));
+ }
+ SECTION("nothing matched, and that was a warning")
+ {
+ const auto run = RunFixture(scratch,
+ { "--no-history",
+ "--warn=unmatched-specs",
+ "[pass]",
+ "no such test" });
+ CHECK(ExitCode(run) == 3);
+ }
+ SECTION("nothing matched, and that was allowed")
+ {
+ const auto run = RunFixture(
+ scratch,
+ { "--no-history", "--allow-no-tests", "no such test" });
+ CHECK(ExitCode(run) == 0);
+ }
+ SECTION("everything skipped")
+ {
+ const auto run = RunFixture(scratch, { "--no-history", "[skip]" });
+ CHECK(ExitCode(run) == 4);
+ }
+ SECTION("a report that cannot be written")
+ {
+ // A directory where the report file should go.
+ const auto blocked = scratch.Path() / "blocked.xml";
+ std::filesystem::create_directories(blocked / "inside");
+ const auto run = RunFixture(
+ scratch,
+ { "--no-history", "[pass]", "--junit=" + blocked.string() });
+ CHECK(ExitCode(run) == 1);
+ CHECK_THAT(run.output, ContainsSubstring("error: "));
+ }
+}
+
+TEST_CASE("The console report follows the verbosity",
+ "[progmasoft][runner][output]")
+{
+ const Scratch scratch;
+ // Passing tests get a line, and their output, only when asked for.
+ const auto normal = RunFixture(scratch, { "--no-history", "[pass]" });
+ CHECK_THAT(normal.output, ContainsSubstring("Catch3: 2 test(s)"));
+ CHECK(normal.output.find("PASS fixture prints") == std::string::npos);
+ CHECK(normal.output.find("said through std::cout") == std::string::npos);
+
+ const auto verbose
+ = RunFixture(scratch, { "--no-history", "-v", "[pass]" });
+ CHECK_THAT(verbose.output, ContainsSubstring("PASS fixture prints"));
+ CHECK_THAT(verbose.output, ContainsSubstring("said through std::cout"));
+
+ const auto quiet = RunFixture(scratch, { "--no-history", "-q", "[pass]" });
+ CHECK(ExitCode(quiet) == 0);
+ CHECK(quiet.output.find("Catch3: ") == std::string::npos);
+
+ const auto slowest
+ = RunFixture(scratch, { "--no-history", "--slowest=2", "[slow]" });
+ CHECK_THAT(slowest.output, ContainsSubstring("slowest tests:"));
+
+ const auto coloured
+ = RunPlain({ "--no-history", "--color=always", "[pass]" });
+ CHECK_THAT(coloured.output, ContainsSubstring("\x1b["));
+ const auto plain = RunPlain({ "--no-history", "--color=never", "[pass]" });
+ CHECK(plain.output.find("\x1b[") == std::string::npos);
+}
+
+TEST_CASE("The history is summarised on request",
+ "[progmasoft][runner][output]")
+{
+ const Scratch scratch;
+ const auto history = "--history=" + scratch.Path().string();
+ for (int run = 0; run < 3; ++run)
+ {
+ static_cast<void>(
+ RunFixture(scratch,
+ { history, "--retries=off", "[pass],[alternate]" }));
+ }
+
+ const auto text = RunPlain({ history, "--analytics" });
+ CHECK(CodeOf(text) == 0);
+ CHECK_THAT(text.output, ContainsSubstring("3 run(s)"));
+ CHECK_THAT(text.output, ContainsSubstring("flaky (last 20 runs):"));
+ CHECK_THAT(text.output, ContainsSubstring("fixture alternates"));
+ CHECK_THAT(text.output, ContainsSubstring("slowest (median):"));
+
+ const auto json = RunPlain({ history, "--analytics=json" });
+ CHECK(CodeOf(json) == 0);
+ auto parsed = Runner::ParseJson(json.output);
+ REQUIRE(parsed);
+ CHECK((*parsed)["format"].AsString() == "catch3-analytics");
+ CHECK((*parsed)["runs"].AsInteger() == 3);
+ CHECK_FALSE((*parsed)["tests"].Items().empty());
+}
+
+TEST_CASE("Bazel's test environment is honoured",
+ "[progmasoft][runner][output]")
+{
+ const Scratch scratch;
+ const auto xml = scratch.Path() / "test.xml";
+ const auto outputs = scratch.Path() / "outputs";
+ const auto status = scratch.Path() / "shard.status";
+ std::filesystem::create_directories(outputs);
+ const auto run
+ = RunPlain({ "--color=never" },
+ { { "BAZEL_TEST", "1" },
+ { "XML_OUTPUT_FILE", xml.string() },
+ { "TESTBRIDGE_TEST_ONLY", "fixture passes" },
+ { "TEST_SHARD_INDEX", "0" },
+ { "TEST_TOTAL_SHARDS", "1" },
+ { "TEST_SHARD_STATUS_FILE", status.string() },
+ { "TEST_RANDOM_SEED", "1234" },
+ { "TEST_UNDECLARED_OUTPUTS_DIR", outputs.string() },
+ { "CATCH3_FIXTURE_STATE", scratch.Path().string() } });
+ CHECK(CodeOf(run) == 0);
+ CHECK_THAT(run.output, ContainsSubstring("1 test(s): 1 passed"));
+ CHECK(std::filesystem::exists(status));
+
+ auto junit = Runner::ReadFile(xml);
+ REQUIRE(junit);
+ CHECK_THAT(*junit, ContainsSubstring("fixture passes"));
+
+ auto report = Runner::ReadFile(outputs / "catch3.json");
+ REQUIRE(report);
+ auto parsed = Runner::ParseJson(*report);
+ REQUIRE(parsed);
+ CHECK((*parsed)["seed"].AsInteger() == 1234);
+ // A sandboxed test keeps no history beside the binary.
+ const auto beside
+ = std::filesystem::path{ FixtureClient::Fixture() }.parent_path()
+ / ".catch3";
+ CHECK_FALSE(std::filesystem::exists(beside));
+}
diff --git a/tests/meson.build b/tests/meson.build
index dc1a8edf..4e42ee12 100644
--- a/tests/meson.build
+++ b/tests/meson.build
@@ -82,6 +82,7 @@ progmasoft_property_tests = executable(
'Progmasoft/RunnerPlanner.tests.cpp',
'Progmasoft/RunnerOptions.tests.cpp',
'Progmasoft/RunnerAnalysis.tests.cpp',
+ 'Progmasoft/RunnerOutput.tests.cpp',
'Progmasoft/Snapshot.tests.cpp',
'Progmasoft/Stateful.tests.cpp',
'Progmasoft/Strategy.tests.cpp',