End the output capture when a test exits, fail on missing reports

A test that called exit() crashed its worker with SIGSEGV on Linux and
macOS. The exit hook read Catch2's captured output through a scoped
deactivation, which switched the capture back on; libstdc++ then flushed
std::cout at exit into a capture buffer that static destruction had
already freed. RunContext::endCapture() now ends the capture for good and
returns what it held, so the standard streams are their own again.

A JUnit, JSON or trace report that could not be written was only
reported on stderr, and a run whose tests passed still exited with 0, so a
build server could miss it. Such a run now exits with 1; failed tests
still exit with 42.

Integration tests of the runner's output move their helpers into
RunnerFixtureClient.hpp and cover --version, --help, the listing modes, a
bad option, exit codes 1 to 4, -v, -q, --slowest, colours, --analytics
in text and JSON, and Bazel's test environment.

1 files changed, 9 insertions, 169 deletions

diff --git a/tests/Progmasoft/Runner.tests.cpp b/tests/Progmasoft/Runner.tests.cpp
index 3196b2a5..d458de10 100644
--- a/tests/Progmasoft/Runner.tests.cpp
+++ b/tests/Progmasoft/Runner.tests.cpp
@@ -3,6 +3,8 @@
 
 // Starts the runner fixture and checks what the runner made of its tests.
 
+#include "RunnerFixtureClient.hpp"
+
 #include <Progmasoft/Runner/Files.hpp>
 #include <Progmasoft/Runner/Json.hpp>
 #include <Progmasoft/Runner/Process.hpp>
@@ -21,174 +23,10 @@ namespace
 {
     namespace Runner = Progmasoft::Catch3::Runner;
     using Catch::Matchers::ContainsSubstring;
-
-    struct FixtureRun final
-    {
-        Runner::ExitStatus status;
-        std::string output;
-        Runner::JsonValue report;
-        std::chrono::milliseconds wall{0};
-
-        /// The report entry of a test.
-        [[nodiscard]] const Runner::JsonValue &
-        Test(const std::string &name) const
-        {
-            for (const auto &test : report["tests"].Items())
-            {
-                if (test["name"].AsString() == name)
-                {
-                    return test;
-                }
-            }
-            FAIL("no result for '" << name << "' in:\n" << output);
-            return report;
-        }
-
-        [[nodiscard]] std::string
-        Analysis(const std::string &name) const
-        {
-            std::string text;
-            for (const auto &finding : Test(name)["analysis"].Items())
-            {
-                text += finding.AsString() + "\n";
-            }
-            return text;
-        }
-    };
-
-    /// Finds a file Bazel provides as runfiles: beside the test, in the
-    /// runfiles tree, or through the manifest Bazel writes on Windows.
-    std::filesystem::path
-    FromRunfiles(const std::string &path)
-    {
-        std::error_code error;
-        if (std::filesystem::exists(path, error))
-        {
-            return std::filesystem::absolute(path, error);
-        }
-        for (const char *variable : {"RUNFILES_DIR", "TEST_SRCDIR"})
-        {
-            const auto directory = Runner::ReadEnvironment(variable);
-            if (!directory.empty())
-            {
-                const auto candidate =
-                    std::filesystem::path{directory} / "_main" / path;
-                if (std::filesystem::exists(candidate, error))
-                {
-                    return candidate;
-                }
-            }
-        }
-        const auto manifest = Runner::ReadEnvironment("RUNFILES_MANIFEST_FILE");
-        if (!manifest.empty())
-        {
-            auto text = Runner::ReadFile(manifest);
-            const std::string key = "_main/" + path + " ";
-            if (text)
-            {
-                const auto found = text->find(key);
-                if (found != std::string::npos)
-                {
-                    const auto start = found + key.size();
-                    auto end = text->find('\n', start);
-                    std::string target = text->substr(
-                        start, end == std::string::npos ? std::string::npos : end - start);
-                    if (!target.empty() && target.back() == '\r')
-                    {
-                        target.pop_back();
-                    }
-                    return target;
-                }
-            }
-        }
-        return path;
-    }
-
-    std::string
-    Fixture()
-    {
-        const auto path = Runner::ReadEnvironment("CATCH3_RUNNER_FIXTURE");
-        if (path.empty())
-        {
-            SKIP("CATCH3_RUNNER_FIXTURE names no fixture binary");
-        }
-        return FromRunfiles(path).string();
-    }
-
-    /// A fresh directory for one test.
-    class Scratch final
-    {
-    public:
-        Scratch()
-        {
-            std::error_code error;
-            path_ = std::filesystem::temp_directory_path(error) /
-                    ("catch3-runner-test-" + Runner::UniqueName());
-            std::filesystem::create_directories(path_, error);
-        }
-        Scratch(const Scratch &) = delete;
-        Scratch &
-        operator=(const Scratch &) = delete;
-        ~Scratch()
-        {
-            std::error_code error;
-            std::filesystem::remove_all(path_, error);
-        }
-
-        [[nodiscard]] const std::filesystem::path &
-        Path() const noexcept
-        {
-            return path_;
-        }
-
-    private:
-        std::filesystem::path path_;
-    };
-
-    FixtureRun
-    RunFixture(const Scratch &scratch,
-               std::vector<std::string> arguments,
-               std::vector<std::pair<std::string, std::string>> environment = {})
-    {
-        Runner::ChildOptions options;
-        options.arguments.push_back(Fixture());
-        const auto report = scratch.Path() / ("report-" + Runner::UniqueName() + ".json");
-        for (auto &argument : arguments)
-        {
-            options.arguments.push_back(std::move(argument));
-        }
-        options.arguments.push_back("--json=" + report.string());
-        options.arguments.emplace_back("--color=never");
-        options.environment = std::move(environment);
-        options.environment.emplace_back("CATCH3_FIXTURE_STATE",
-                                         scratch.Path().string());
-        const auto started = std::chrono::steady_clock::now();
-        auto run = Runner::RunAndCapture(options);
-        INFO((run ? std::string{} : run.GetError().Message()));
-        REQUIRE(run);
-        FixtureRun result;
-        result.wall = std::chrono::duration_cast<std::chrono::milliseconds>(
-            std::chrono::steady_clock::now() - started);
-        result.status = run->status;
-        result.output = run->output + run->errors;
-        if (std::filesystem::exists(report))
-        {
-            auto text = Runner::ReadFile(report);
-            REQUIRE(text);
-            auto json = Runner::ParseJson(*text);
-            REQUIRE(json);
-            result.report = std::move(*json);
-        }
-        return result;
-    }
-
-    std::int64_t
-    ExitCode(const FixtureRun &run)
-    {
-        INFO(run.output);
-        REQUIRE(run.status.kind == Runner::ExitStatus::Kind::kExited);
-        return run.status.code;
-    }
+    using FixtureClient::ExitCode;
+    using FixtureClient::Fixture;
+    using FixtureClient::RunFixture;
+    using FixtureClient::Scratch;
 } // namespace
 
 TEST_CASE("Passing and failing tests get their verdicts", "[progmasoft][runner]")
@@ -249,7 +87,9 @@ TEST_CASE("A hanging test is stopped at its time limit", "[progmasoft][runner]")
     CHECK(hang["category"].AsString() == "timeout");
     CHECK_THAT(hang["attempts"].Items().front()["lastLocation"].AsString(),
                ContainsSubstring("RunnerFixture.cpp"));
-    CHECK(run.wall < std::chrono::seconds(30));
+    // Well below the 60 s the test sleeps; a slow, busy disk can stretch
+    // the runner's own start-up and Git query to half a minute.
+    CHECK(run.wall < std::chrono::seconds(50));
 }
 
 TEST_CASE("A test that passes on a retry is flaky", "[progmasoft][runner]")