Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/tests/Progmasoft/RunnerFixture.cpp
diff options
context:
space:
mode:
authorLeitwolf112026-10-23 23:25:41 +0300
committerLeitwolf112026-10-23 23:25:41 +0300
commit93d4e7dbebab867e0a7f59fc25308460a7a76280 (patch)
tree54c568a3063d49e21f039b6d38f40f51be7991f0 /tests/Progmasoft/RunnerFixture.cpp
parent83c6fc3ac13abe83fa39f7b5f8559ccaf4412233 (diff)
downloadcatch3-93d4e7dbebab867e0a7f59fc25308460a7a76280.tar.gz
Replace Catch2's runner with the Catch3 runner
Catch3 keeps Catch2's API for writing tests (test cases, sections, assertions, matchers, generators, listeners, benchmarks) and replaces its session, command line and reporters with a runner of its own. The runner starts the test binary again as worker processes that take the next test when they are free. A crash, an exit or a hang ends one test, not the run. On top of that it adds: - retries that tell deterministic, flaky and order-dependent failures apart, and a bisection that names the test leaving state behind; - a history of verdicts and durations, kept under a file lock, that drives automatic quarantine, adaptive time limits, predictive ordering, incremental runs, failure replay and analytics; - selection by change, past failure or prediction, dependency closure, and sharding by a stable hash or by recorded durations; - tags for serial and isolated tests, shared resources, dependencies, time limits and retries; - a failure category, a signature, an analysis and a replay command for every failure; - console, JUnit XML (with flakyFailure and rerunFailure), JSON and Chrome trace reports, and a versioned discovery listing (--list=json); - a temporary directory per test (TMPDIR, TMP, TEMP and TestTemporaryDirectory()); - Bazel's test environment: XML output file, test filter, sharding, seed, premature exit file and undeclared outputs. The command line is parsed by a hand-written, typed parser. Catch::Session remains as a thin facade over the runner for custom main functions. The result model in Progmasoft/Catch3/Results.hpp was rewritten around tests and attempts, and the JUnit writer now writes it. The property engine's example database is updated under a file lock with an atomic rename, so parallel workers can write it at the same time. Removed: Clara, the command-line and reporter-spec parsers, every reporter and CATCH_REGISTER_REPORTER, listing, text flow, console colour and width, the JSON and XML writers, sharding in the test registry, approval tests and their baselines, Conan packaging, and the unused third_party/clara.hpp. Listeners keep working; reporters' list and skip events are gone. The Catch2 library is now compiled as C++20; code that includes only Catch2's headers still needs C++14. Duplicate test cases are still refused at start-up. Version 0.3.0. ExtraTests now turn exceptions off for clang-cl with MSVC-style options, which -fno-exceptions did not do.
Diffstat (limited to 'tests/Progmasoft/RunnerFixture.cpp')
-rw-r--r--tests/Progmasoft/RunnerFixture.cpp190
1 files changed, 190 insertions, 0 deletions
diff --git a/tests/Progmasoft/RunnerFixture.cpp b/tests/Progmasoft/RunnerFixture.cpp
new file mode 100644
index 00000000..fe9fb150
--- /dev/null
+++ b/tests/Progmasoft/RunnerFixture.cpp
@@ -0,0 +1,190 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+
+// A test binary whose tests misbehave on purpose. The runner's integration
+// tests start it and check what the runner made of each misbehaviour.
+
+#include <Progmasoft/Catch3/Runner.hpp>
+#include <catch2/catch_test_macros.hpp>
+#include <chrono>
+#include <cstdlib>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <optional>
+#include <string>
+#include <thread>
+
+namespace
+{
+ /// The value of the environment variable `name`, if it is set.
+ std::optional<std::string>
+ Environment(const char *name)
+ {
+#if defined(_WIN32)
+ char *value = nullptr;
+ std::size_t length = 0;
+ if (_dupenv_s(&value, &length, name) != 0 || value == nullptr)
+ {
+ return std::nullopt;
+ }
+ std::string result{ value };
+ std::free(value);
+ return result;
+#else
+ const char *value = std::getenv(name);
+ return value == nullptr ? std::nullopt
+ : std::optional<std::string>{ value };
+#endif
+ }
+
+ /// A directory the integration test gives every run of the fixture, to
+ /// count attempts across worker processes.
+ std::filesystem::path
+ StateDirectory()
+ {
+ const auto directory = Environment("CATCH3_FIXTURE_STATE");
+ return directory ? std::filesystem::path{ *directory }
+ : std::filesystem::path{};
+ }
+
+ /// How often `name` ran before, counted in a file; increments it.
+ int
+ CountRun(const std::string &name)
+ {
+ const auto file = StateDirectory() / (name + ".count");
+ int count = 0;
+ {
+ std::ifstream input(file);
+ input >> count;
+ }
+ std::ofstream output(file, std::ios::trunc);
+ output << count + 1;
+ return count;
+ }
+
+ bool polluted = false;
+} // namespace
+
+TEST_CASE("fixture passes", "[fixture][pass]")
+{
+ REQUIRE(1 + 1 == 2);
+}
+
+TEST_CASE("fixture prints", "[fixture][pass]")
+{
+ std::cout << "said through std::cout\n";
+ std::cerr << "said through std::cerr\n";
+ CHECK(true);
+}
+
+TEST_CASE("fixture fails", "[fixture][fail]")
+{
+ const int expected = 3;
+ SECTION("in a section")
+ {
+ CHECK(1 + 1 == expected);
+ }
+}
+
+TEST_CASE("fixture throws", "[fixture][fail]")
+{
+ throw std::runtime_error("broken on purpose");
+}
+
+TEST_CASE("fixture crashes", "[fixture][crash]")
+{
+ SECTION("deep down")
+ {
+ CHECK(true);
+ std::abort();
+ }
+}
+
+TEST_CASE("fixture hangs", "[fixture][hang][!timeout:1]")
+{
+ CHECK(true);
+ std::this_thread::sleep_for(std::chrono::seconds(60));
+}
+
+TEST_CASE("fixture is flaky", "[fixture][flaky]")
+{
+ // Fails the first time it runs, passes afterwards.
+ REQUIRE(CountRun("flaky") > 0);
+}
+
+TEST_CASE("fixture alternates", "[fixture][alternate]")
+{
+ // Fails on every even run: flaky on every run of the runner.
+ REQUIRE(CountRun("alternates") % 2 == 1);
+}
+
+TEST_CASE("fixture innocent one", "[fixture][order]")
+{
+ CHECK(true);
+}
+
+TEST_CASE("fixture polluter", "[fixture][order]")
+{
+ polluted = true;
+ CHECK(true);
+}
+
+TEST_CASE("fixture innocent two", "[fixture][order]")
+{
+ CHECK(true);
+}
+
+TEST_CASE("fixture victim", "[fixture][order]")
+{
+ REQUIRE_FALSE(polluted);
+}
+
+TEST_CASE("fixture base", "[fixture][deps]")
+{
+ FAIL("the base is broken");
+}
+
+TEST_CASE("fixture dependent", "[fixture][deps][!depends:fixture base]")
+{
+ CHECK(true);
+}
+
+TEST_CASE("fixture temporary directory", "[fixture][temp]")
+{
+ const auto directory = Progmasoft::Catch3::TestTemporaryDirectory();
+ REQUIRE(std::filesystem::exists(directory));
+ REQUIRE(std::filesystem::equivalent(std::filesystem::temp_directory_path(),
+ directory));
+ std::ofstream(directory / "scratch.txt") << "kept only on failure";
+ CHECK_FALSE(Environment("CATCH3_FIXTURE_FAIL_TEMP").has_value());
+}
+
+TEST_CASE("fixture slow one", "[fixture][slow]")
+{
+ std::this_thread::sleep_for(std::chrono::milliseconds(300));
+ CHECK(true);
+}
+
+TEST_CASE("fixture slow two", "[fixture][slow]")
+{
+ std::this_thread::sleep_for(std::chrono::milliseconds(300));
+ CHECK(true);
+}
+
+TEST_CASE("fixture slow three", "[fixture][slow]")
+{
+ std::this_thread::sleep_for(std::chrono::milliseconds(300));
+ CHECK(true);
+}
+
+TEST_CASE("fixture slow four", "[fixture][slow]")
+{
+ std::this_thread::sleep_for(std::chrono::milliseconds(300));
+ CHECK(true);
+}
+
+TEST_CASE("fixture skips", "[fixture][skip]")
+{
+ SKIP("not on this machine");
+}