diff options
| author | Leitwolf11 | 2026-10-23 23:25:41 +0300 |
|---|---|---|
| committer | Leitwolf11 | 2026-10-23 23:25:41 +0300 |
| commit | 93d4e7dbebab867e0a7f59fc25308460a7a76280 (patch) | |
| tree | 54c568a3063d49e21f039b6d38f40f51be7991f0 /tests/Progmasoft/RunnerUnits.tests.cpp | |
| parent | 83c6fc3ac13abe83fa39f7b5f8559ccaf4412233 (diff) | |
| download | catch3-93d4e7dbebab867e0a7f59fc25308460a7a76280.tar.gz | |
Replace Catch2's runner with the Catch3 runner
Catch3 keeps Catch2's API for writing tests (test cases, sections,
assertions, matchers, generators, listeners, benchmarks) and replaces its
session, command line and reporters with a runner of its own.
The runner starts the test binary again as worker processes that take the
next test when they are free. A crash, an exit or a hang ends one test, not
the run. On top of that it adds:
- retries that tell deterministic, flaky and order-dependent failures
apart, and a bisection that names the test leaving state behind;
- a history of verdicts and durations, kept under a file lock, that drives
automatic quarantine, adaptive time limits, predictive ordering,
incremental runs, failure replay and analytics;
- selection by change, past failure or prediction, dependency closure,
and sharding by a stable hash or by recorded durations;
- tags for serial and isolated tests, shared resources, dependencies,
time limits and retries;
- a failure category, a signature, an analysis and a replay command for
every failure;
- console, JUnit XML (with flakyFailure and rerunFailure), JSON and Chrome
trace reports, and a versioned discovery listing (--list=json);
- a temporary directory per test (TMPDIR, TMP, TEMP and
TestTemporaryDirectory());
- Bazel's test environment: XML output file, test filter, sharding, seed,
premature exit file and undeclared outputs.
The command line is parsed by a hand-written, typed parser. Catch::Session
remains as a thin facade over the runner for custom main functions.
The result model in Progmasoft/Catch3/Results.hpp was rewritten around
tests and attempts, and the JUnit writer now writes it. The property
engine's example database is updated under a file lock with an atomic
rename, so parallel workers can write it at the same time.
Removed: Clara, the command-line and reporter-spec parsers, every reporter
and CATCH_REGISTER_REPORTER, listing, text flow, console colour and width,
the JSON and XML writers, sharding in the test registry, approval tests and
their baselines, Conan packaging, and the unused third_party/clara.hpp.
Listeners keep working; reporters' list and skip events are gone.
The Catch2 library is now compiled as C++20; code that includes only
Catch2's headers still needs C++14. Duplicate test cases are still refused
at start-up. Version 0.3.0.
ExtraTests now turn exceptions off for clang-cl with MSVC-style options,
which -fno-exceptions did not do.
Diffstat (limited to 'tests/Progmasoft/RunnerUnits.tests.cpp')
| -rw-r--r-- | tests/Progmasoft/RunnerUnits.tests.cpp | 424 |
1 files changed, 424 insertions, 0 deletions
diff --git a/tests/Progmasoft/RunnerUnits.tests.cpp b/tests/Progmasoft/RunnerUnits.tests.cpp new file mode 100644 index 00000000..9f71a08e --- /dev/null +++ b/tests/Progmasoft/RunnerUnits.tests.cpp @@ -0,0 +1,424 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#include <Progmasoft/Runner/Analysis.hpp> +#include <Progmasoft/Runner/Files.hpp> +#include <Progmasoft/Runner/Git.hpp> +#include <Progmasoft/Runner/Hash.hpp> +#include <Progmasoft/Runner/History.hpp> +#include <Progmasoft/Runner/Json.hpp> +#include <Progmasoft/Runner/Options.hpp> +#include <Progmasoft/Runner/Planner.hpp> +#include <Progmasoft/Runner/ResultsJson.hpp> + +#include <Progmasoft/Catch3.hpp> + +#include <catch2/catch_test_macros.hpp> +#include <catch2/matchers/catch_matchers_string.hpp> + +#include <filesystem> +#include <set> +#include <thread> +#include <vector> + +namespace +{ + namespace Runner = Progmasoft::Catch3::Runner; + using namespace Progmasoft::Catch3; + using Catch::Matchers::ContainsSubstring; + + Runner::Options + Parse(std::vector<std::string> arguments) + { + auto options = Runner::ParseOptions(arguments); + INFO((options ? std::string{} : options.GetError().Message())); + REQUIRE(options); + return *options; + } + + std::string + ParseError(std::vector<std::string> arguments) + { + auto options = Runner::ParseOptions(arguments); + REQUIRE_FALSE(options); + return options.GetError().Message(); + } + + Runner::TestRecord + Record(std::int64_t run, Outcome outcome, bool flaky = false) + { + Runner::TestRecord record; + record.run = run; + record.outcome = outcome; + record.flaky = flaky; + record.durationMicroseconds = 1000 * run; + record.fingerprint = "same"; + return record; + } +} // namespace + +TEST_CASE("JSON reads back what it writes", "[progmasoft][runner][json]") +{ + Runner::JsonValue value = Runner::JsonValue::Object(); + value.Set("text", "quote \" backslash \\ newline \n tab \t control \x01"); + value.Set("number", 4294967295U); + value.Set("negative", -12); + value.Set("fraction", 0.25); + value.Set("flag", true); + value.Set("nothing", nullptr); + Runner::JsonValue list = Runner::JsonValue::Array(); + list.Append("ü and 日本"); + list.Append(Runner::JsonValue::Object()); + value.Set("list", list); + + for (const std::string &text : {value.ToString(), value.ToPrettyString()}) + { + auto parsed = Runner::ParseJson(text); + REQUIRE(parsed); + CHECK((*parsed)["text"].AsString() == value["text"].AsString()); + CHECK((*parsed)["number"].AsUnsigned() == 4294967295U); + CHECK((*parsed)["negative"].AsInteger() == -12); + CHECK((*parsed)["fraction"].AsNumber() == 0.25); + CHECK((*parsed)["flag"].AsBoolean()); + CHECK((*parsed)["nothing"].IsNull()); + CHECK((*parsed)["list"].Items().front().AsString() == "ü and 日本"); + } + CHECK(value.ToString().find('\n') == std::string::npos); + + auto escaped = Runner::ParseJson(R"("\u00fc\ud83d\ude00")"); + REQUIRE(escaped); + CHECK(escaped->AsString() == "\xC3\xBC\xF0\x9F\x98\x80"); +} + +TEST_CASE("JSON refuses what is not JSON", "[progmasoft][runner][json]") +{ + for (const char *text : {"", "{", "[1,]", "{\"a\" 1}", "tru", "1 2", + "\"unterminated", "\"\\x\"", "01e"}) + { + CAPTURE(text); + CHECK_FALSE(Runner::ParseJson(text)); + } + std::string deep(1000, '['); + deep += std::string(1000, ']'); + CHECK_FALSE(Runner::ParseJson(deep)); +} + +TEST_CASE("Any string survives a trip through JSON", "[progmasoft][runner][json]") +{ + const auto result = ForAll(Gen::Strings(Gen::Integers<char>(), + SizeRange{0, 64}), + [](const std::string &text) { + Runner::JsonValue value{text}; + auto parsed = + Runner::ParseJson(value.ToString()); + return parsed && parsed->AsString() == text; + }); + INFO(result.Describe()); + CHECK(result.Status == PropertyStatus::kPassed); +} + +TEST_CASE("The command line is typed and checked", "[progmasoft][runner][options]") +{ + const auto options = Parse({"--jobs=8", + "-j", "3", + "--timeout=1.5", + "--retries=4", + "--shard=2/5", + "--order=declared", + "--seed=17", + "--select=smart", + "-c", "outer", + "--path-filter=g:2", + "[fast]", + "~[slow]", + "--", + "--looks-like-an-option"}); + CHECK(options.jobs == 3); + CHECK(options.timeout == std::chrono::milliseconds{1500}); + CHECK(options.timeoutGiven); + CHECK(options.retryMode == Runner::RetryMode::kAlways); + CHECK(options.retries == 4); + CHECK(options.shard.index == 2); + CHECK(options.shard.count == 5); + CHECK(options.order == Runner::Order::kDeclared); + CHECK(options.seed == 17); + CHECK(options.seedGiven); + CHECK(options.selection == Runner::Selection::kSmart); + CHECK(options.sections == std::vector<std::string>{"outer"}); + CHECK(options.paths == std::vector<std::string>{"g:2"}); + CHECK(options.testSpecs == + std::vector<std::string>{"[fast]", "~[slow]", "--looks-like-an-option"}); + + CHECK(Parse({"--fail-fast"}).failFast == 1); + CHECK(Parse({"--fail-fast=3"}).failFast == 3); + CHECK(Parse({"--list=json"}).mode == Runner::Mode::kListTests); + CHECK(Parse({"--retries=off"}).retryMode == Runner::RetryMode::kOff); +} + +TEST_CASE("Command line mistakes are explained", "[progmasoft][runner][options]") +{ + CHECK_THAT(ParseError({"--jbos=2"}), ContainsSubstring("did you mean --jobs?")); + CHECK_THAT(ParseError({"--jobs"}), ContainsSubstring("needs a value")); + CHECK_THAT(ParseError({"--jobs=many"}), ContainsSubstring("whole number")); + CHECK_THAT(ParseError({"--shard=4/4"}), ContainsSubstring("INDEX below")); + CHECK_THAT(ParseError({"--order=sideways"}), + ContainsSubstring("priority, declared, name, random")); + CHECK_THAT(ParseError({"--success=yes"}), ContainsSubstring("takes no value")); + CHECK_THAT(ParseError({"--path-filter=x:1"}), ContainsSubstring("c:SECTION")); +} + +TEST_CASE("A worker is started with what changes how tests behave", + "[progmasoft][runner][options]") +{ + auto options = Parse({"--success", "--seed=99", "--section=s", "--warn=no-assertions"}); + const auto arguments = Runner::WorkerArguments(options); + auto worker = Runner::ParseOptions(arguments); + REQUIRE(worker); + CHECK(worker->mode == Runner::Mode::kWorker); + CHECK(worker->includeSuccessful); + CHECK(worker->seed == 99); + CHECK(worker->sections == std::vector<std::string>{"s"}); + CHECK(worker->warnNoAssertions); + CHECK(worker->benchmarkConfidenceInterval == options.benchmarkConfidenceInterval); +} + +TEST_CASE("SHA-256 matches the published test vectors", "[progmasoft][runner][hash]") +{ + CHECK(Runner::Sha256Hex("") == + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + CHECK(Runner::Sha256Hex("abc") == + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + CHECK(Runner::Sha256Hex( + "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq") == + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); + CHECK(Runner::Sha256Hex(std::string(1000000, 'a')) == + "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0"); +} + +TEST_CASE("History predicts failures and finds flakiness", "[progmasoft][runner][history]") +{ + Runner::TestHistory fresh; + CHECK(fresh.FailureProbability() == 0.3); + CHECK_FALSE(fresh.DurationPercentile(0.5).has_value()); + + Runner::TestHistory steady; + for (int run = 1; run <= 10; ++run) + { + steady.records.push_back(Record(run, Outcome::kPassed)); + } + CHECK(steady.FailureProbability() < 0.1); + CHECK(steady.PassStreak() == 10); + CHECK(steady.FlakyEvents(20) == 0); + CHECK(steady.DurationPercentile(0.5) == 5000); + CHECK(steady.DurationPercentile(1.0) == 10000); + + Runner::TestHistory broken = steady; + broken.records.push_back(Record(11, Outcome::kFailed)); + CHECK(broken.FailureProbability() >= 0.7); + CHECK(broken.PassStreak() == 0); + // Pass, fail on the same fingerprint: the verdict flipped by itself. + CHECK(broken.FlakyEvents(20) == 1); + + Runner::TestHistory flaky; + flaky.records.push_back(Record(1, Outcome::kPassed, true)); + flaky.records.push_back(Record(2, Outcome::kPassed, true)); + CHECK(flaky.FlakyEvents(20) == 2); +} + +TEST_CASE("History quarantines flaky tests and releases them", "[progmasoft][runner][history]") +{ + std::error_code error; + const auto directory = std::filesystem::temp_directory_path(error) / + ("catch3-history-" + Runner::UniqueName()); + const auto file = Runner::HistoryFile(directory, "binary name"); + CHECK(file.filename() == "binary_name.history.json"); + + const auto runWith = [&](bool flakyNow) { + RunResult run; + TestResult test; + test.id = "Suite::test"; + test.name = "test"; + test.outcome = Outcome::kPassed; + test.flaky = flakyNow; + Attempt attempt; + attempt.outcome = Outcome::kPassed; + test.attempts.push_back(attempt); + run.tests.push_back(test); + Runner::QuarantinePolicy policy; + policy.passesToLeave = 3; + auto history = Runner::History::Commit(file, run, {}, {}, policy); + REQUIRE(history); + return *history; + }; + CHECK_FALSE(runWith(true).Find("Suite::test")->quarantine); + CHECK(runWith(true).Find("Suite::test")->quarantine); + CHECK(runWith(false).Find("Suite::test")->quarantine); + CHECK(runWith(false).Find("Suite::test")->quarantine); + const auto released = runWith(false); + CHECK_FALSE(released.Find("Suite::test")->quarantine); + CHECK(released.Runs().size() == 5); + CHECK(released.Runs().back().id == 5); + + auto reloaded = Runner::History::Load(file); + REQUIRE(reloaded); + CHECK(reloaded->Find("Suite::test")->records.size() == 5); + std::filesystem::remove_all(directory, error); +} + +TEST_CASE("Runs that finish together all reach the history", + "[progmasoft][runner][history]") +{ + std::error_code error; + const auto directory = std::filesystem::temp_directory_path(error) / + ("catch3-history-" + Runner::UniqueName()); + const auto file = Runner::HistoryFile(directory, "parallel"); + std::vector<std::thread> threads; + for (int index = 0; index < 8; ++index) + { + threads.emplace_back([&, index] { + RunResult run; + TestResult test; + test.id = "test " + std::to_string(index); + test.outcome = Outcome::kPassed; + run.tests.push_back(test); + static_cast<void>(Runner::History::Commit(file, run, {}, {}, {})); + }); + } + for (auto &thread : threads) + { + thread.join(); + } + auto history = Runner::History::Load(file); + REQUIRE(history); + CHECK(history->Runs().size() == 8); + CHECK(history->Tests().size() == 8); + std::filesystem::remove_all(directory, error); +} + +TEST_CASE("Stable shards do not depend on the order or the machine", + "[progmasoft][runner][planner]") +{ + CHECK(Runner::StableShard("anything", 1) == 0); + std::vector<std::size_t> sizes(4, 0); + for (int index = 0; index < 4000; ++index) + { + const auto shard = Runner::StableShard("test " + std::to_string(index), 4); + REQUIRE(shard < 4); + ++sizes[shard]; + } + for (const auto size : sizes) + { + CHECK(size > 800); + } + CHECK(Runner::StableShard("parser parses", 7) == + Runner::StableShard("parser parses", 7)); +} + +TEST_CASE("Failure signatures ignore numbers and addresses", + "[progmasoft][runner][analysis]") +{ + Attempt first; + first.outcome = Outcome::kFailed; + first.category = FailureCategory::kAssertion; + AssertionRecord assertion; + assertion.file = "pool.cpp"; + assertion.line = 33; + assertion.expression = "pool.size() == limit"; + assertion.message = "slot 17 at 0x7ffd0010 is taken"; + first.assertions.push_back(assertion); + Attempt second = first; + second.assertions.front().message = "slot 4 at 0x51a0 is taken"; + CHECK(Runner::FailureSignature(first) == Runner::FailureSignature(second)); + Attempt elsewhere = first; + elsewhere.assertions.front().line = 34; + CHECK(Runner::FailureSignature(first) != Runner::FailureSignature(elsewhere)); + CHECK_THAT(Runner::FailureSignature(first), ContainsSubstring("sig-")); + + CHECK(Runner::QuoteTestSpec("a, [b] *c") == "a\\, \\[b\\] \\*c"); +} + +TEST_CASE("A run report reads back what it wrote", "[progmasoft][runner][json]") +{ + RunResult run; + run.binary = "tests"; + run.seed = 7; + run.shardCount = 3; + run.notes = {"a note"}; + TestResult test; + test.id = "Fixture::adds"; + test.name = "adds"; + test.className = "Fixture"; + test.tags = {"math", "!timeout:3"}; + test.outcome = Outcome::kTimedOut; + test.category = FailureCategory::kTimeout; + test.history = FailureHistory::kNew; + test.analysis = {"stopped after 3.0 s"}; + Attempt attempt; + attempt.outcome = Outcome::kTimedOut; + attempt.category = FailureCategory::kTimeout; + attempt.precededBy = {"before"}; + attempt.lastLocation = "math.cpp:9"; + SectionRecord section; + section.name = "slow"; + section.durationMicroseconds = -1; + attempt.sections.push_back(section); + BenchmarkRecord benchmark; + benchmark.name = "sum"; + benchmark.meanNanoseconds = 12.5; + attempt.benchmarks.push_back(benchmark); + test.attempts.push_back(attempt); + run.tests.push_back(test); + + auto json = Runner::ParseJson(Runner::ToJson(run).ToString()); + REQUIRE(json); + auto read = Runner::RunResultFromJson(*json); + REQUIRE(read); + REQUIRE(read->tests.size() == 1); + const auto &back = read->tests.front(); + CHECK(back.id == test.id); + CHECK(back.className == "Fixture"); + CHECK(back.tags == test.tags); + CHECK(back.outcome == Outcome::kTimedOut); + CHECK(back.history == FailureHistory::kNew); + CHECK(back.analysis == test.analysis); + REQUIRE(back.attempts.size() == 1); + CHECK(back.attempts.front().precededBy == attempt.precededBy); + CHECK(back.attempts.front().sections.front().durationMicroseconds == -1); + CHECK(back.attempts.front().benchmarks.front().meanNanoseconds == 12.5); + CHECK(read->shardCount == 3); + CHECK((*json)["summary"]["timedOut"].AsInteger() == 1); + + Runner::JsonValue future = *json; + future.Set("version", 99); + CHECK_FALSE(Runner::RunResultFromJson(future)); +} + +TEST_CASE("Source files match across spellings", "[progmasoft][runner][git]") +{ + CHECK(Runner::SameSourceFile("/work/repo/tests/a.cpp", "tests/a.cpp")); + CHECK(Runner::SameSourceFile("../tests/a.cpp", "/work/repo/tests/a.cpp")); + CHECK(Runner::SameSourceFile("/work/repo/./tests/../tests/a.cpp", + "/work/repo/tests/a.cpp")); + CHECK_FALSE(Runner::SameSourceFile("/work/repo/tests/a.cpp", "ests/a.cpp")); + CHECK_FALSE(Runner::SameSourceFile("/work/repo/tests/a.cpp", + "/work/repo/tests/b.cpp")); +} + +TEST_CASE("A file lock lets one holder in at a time", "[progmasoft][runner][files]") +{ + std::error_code error; + const auto file = std::filesystem::temp_directory_path(error) / + ("catch3-lock-" + Runner::UniqueName()); + { + const Runner::FileLock first{file, std::chrono::milliseconds{100}}; + CHECK(first.Held()); + const Runner::FileLock second{file, std::chrono::milliseconds{50}}; + CHECK_FALSE(second.Held()); + } + const Runner::FileLock third{file, std::chrono::milliseconds{100}}; + CHECK(third.Held()); + + REQUIRE(Runner::WriteFileAtomically(file, "first")); + REQUIRE(Runner::WriteFileAtomically(file, "second")); + CHECK(*Runner::ReadFile(file) == "second"); + std::filesystem::remove(file, error); +} |