Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/tests
diff options
context:
space:
mode:
authorLeitwolf112026-10-24 10:06:25 +0300
committerLeitwolf112026-10-24 10:06:25 +0300
commite104b694513964d5a84e77f751203cda970d741b (patch)
treea7d6e6d5cca1e98702549f02489e3570599ca3b4 /tests
parent15eb534ee493a3388c168709eb79a954e9cb5fcf (diff)
downloadcatch3-e104b694513964d5a84e77f751203cda970d741b.tar.gz
Fix the CI failures of the runner and test its planner, options and analysis
ExtraTests X21 and X27 defined their listeners at namespace scope, which clang rejects under -Wweak-vtables with -Werror on Linux and macOS; they are now in anonymous namespaces. After a fatal signal Catch2 closes the open sections itself, so on POSIX a crash was reported outside the section it happened in. The scheduler now ignores section ends that follow a fatal condition. New unit tests cover the planner (every selection, dependency closure and cycles, hash and duration shards, incremental runs, learned time limits, ordering, retries, quarantine and replay), every command-line option and its errors, and the failure analysis (history classes, signature groups, crash and hang locations, changed files, replay commands). Catalog::FromEntries builds a catalog without a test registry so the planner can be tested on its own.
Diffstat (limited to 'tests')
-rw-r--r--tests/CMakeLists.txt3
-rw-r--r--tests/ExtraTests/X21-PartialTestCaseEvents.cpp2
-rw-r--r--tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp2
-rw-r--r--tests/Progmasoft/BUILD.bazel3
-rw-r--r--tests/Progmasoft/RunnerAnalysis.tests.cpp321
-rw-r--r--tests/Progmasoft/RunnerOptions.tests.cpp369
-rw-r--r--tests/Progmasoft/RunnerPlanner.tests.cpp599
-rw-r--r--tests/meson.build3
8 files changed, 1302 insertions, 0 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 14f2ed56..e57ad94a 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -157,6 +157,9 @@ add_executable(ProgmasoftSelfTest
Progmasoft/Results.tests.cpp
Progmasoft/Runner.tests.cpp
Progmasoft/RunnerUnits.tests.cpp
+ Progmasoft/RunnerPlanner.tests.cpp
+ Progmasoft/RunnerOptions.tests.cpp
+ Progmasoft/RunnerAnalysis.tests.cpp
Progmasoft/Snapshot.tests.cpp
Progmasoft/Stateful.tests.cpp
Progmasoft/Strategy.tests.cpp
diff --git a/tests/ExtraTests/X21-PartialTestCaseEvents.cpp b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp
index c55f6c2b..99002822 100644
--- a/tests/ExtraTests/X21-PartialTestCaseEvents.cpp
+++ b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp
@@ -28,6 +28,7 @@
using Catch::TestCaseInfo;
using Catch::TestCaseStats;
+namespace {
class PartialListener : public Catch::EventListenerBase {
public:
using EventListenerBase::EventListenerBase;
@@ -58,6 +59,7 @@ public:
private:
std::string m_events;
};
+} // namespace
CATCH_REGISTER_LISTENER(PartialListener)
diff --git a/tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp b/tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp
index 7ceebcee..20b39d78 100644
--- a/tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp
+++ b/tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp
@@ -27,6 +27,7 @@
#include <iostream>
#include <string>
+namespace {
class TestListener : public Catch::EventListenerBase {
std::string stdOutString( uint64_t iter ){
return "stdout " + std::to_string( iter ) + '\n';
@@ -66,6 +67,7 @@ public:
std::cerr << "X27 - listener saw " << m_errors << " error(s)\n";
}
};
+} // namespace
CATCH_REGISTER_LISTENER( TestListener )
diff --git a/tests/Progmasoft/BUILD.bazel b/tests/Progmasoft/BUILD.bazel
index 296fd5f1..06df86d6 100644
--- a/tests/Progmasoft/BUILD.bazel
+++ b/tests/Progmasoft/BUILD.bazel
@@ -13,6 +13,9 @@ cc_test(
"Results.tests.cpp",
"Runner.tests.cpp",
"RunnerUnits.tests.cpp",
+ "RunnerPlanner.tests.cpp",
+ "RunnerOptions.tests.cpp",
+ "RunnerAnalysis.tests.cpp",
"Snapshot.tests.cpp",
"Stateful.tests.cpp",
"Strategy.tests.cpp",
diff --git a/tests/Progmasoft/RunnerAnalysis.tests.cpp b/tests/Progmasoft/RunnerAnalysis.tests.cpp
new file mode 100644
index 00000000..453fa671
--- /dev/null
+++ b/tests/Progmasoft/RunnerAnalysis.tests.cpp
@@ -0,0 +1,321 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+
+// What the runner writes down about a failure: how it relates to earlier
+// runs and to other failures, where it stopped, what changed and how to
+// run it again.
+
+#include <Progmasoft/Runner/Analysis.hpp>
+#include <Progmasoft/Runner/Files.hpp>
+#include <Progmasoft/Runner/History.hpp>
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/matchers/catch_matchers_string.hpp>
+#include <catch2/matchers/catch_matchers_vector.hpp>
+
+#include <filesystem>
+#include <string>
+#include <vector>
+
+namespace
+{
+ namespace Runner = Progmasoft::Catch3::Runner;
+ using Catch::Matchers::ContainsSubstring;
+ using Progmasoft::Catch3::AssertionRecord;
+ using Progmasoft::Catch3::Attempt;
+ using Progmasoft::Catch3::FailureCategory;
+ using Progmasoft::Catch3::FailureHistory;
+ using Progmasoft::Catch3::Outcome;
+ using Progmasoft::Catch3::RunResult;
+ using Progmasoft::Catch3::SectionRecord;
+ using Progmasoft::Catch3::TestResult;
+
+ /// A directory of its own, removed at the end of the test.
+ class Scratch final
+ {
+ public:
+ Scratch()
+ {
+ std::error_code error;
+ path_ = std::filesystem::temp_directory_path(error) /
+ ("catch3-analysis-" + Runner::UniqueName());
+ std::filesystem::create_directories(path_, error);
+ }
+ Scratch(const Scratch &) = delete;
+ Scratch &
+ operator=(const Scratch &) = delete;
+ ~Scratch()
+ {
+ std::error_code error;
+ std::filesystem::remove_all(path_, error);
+ }
+
+ [[nodiscard]] const std::filesystem::path &
+ Path() const noexcept
+ {
+ return path_;
+ }
+
+ private:
+ std::filesystem::path path_;
+ };
+
+ /// A test that failed one assertion.
+ TestResult
+ Failed(const std::string &name,
+ const std::string &message = "wrong",
+ std::size_t line = 10)
+ {
+ TestResult test;
+ test.id = name;
+ test.name = name;
+ test.file = "src/test.cpp";
+ test.line = 1;
+ test.outcome = Outcome::kFailed;
+ test.category = FailureCategory::kAssertion;
+ Attempt attempt;
+ attempt.outcome = Outcome::kFailed;
+ attempt.category = FailureCategory::kAssertion;
+ attempt.seed = 7;
+ AssertionRecord assertion;
+ assertion.file = "src/test.cpp";
+ assertion.line = line;
+ assertion.expression = "a == b";
+ assertion.message = message;
+ attempt.assertions.push_back(assertion);
+ test.attempts.push_back(attempt);
+ return test;
+ }
+
+ TestResult
+ Passed(const std::string &name)
+ {
+ TestResult test;
+ test.id = name;
+ test.name = name;
+ test.outcome = Outcome::kPassed;
+ Attempt attempt;
+ attempt.outcome = Outcome::kPassed;
+ test.attempts.push_back(attempt);
+ return test;
+ }
+
+ /// Commits runs of `tests` to a history, the last commit last.
+ Runner::History
+ HistoryOf(const std::filesystem::path &file,
+ const std::vector<std::vector<TestResult>> &runs)
+ {
+ Runner::History history;
+ int number = 0;
+ for (const auto &tests : runs)
+ {
+ RunResult run;
+ run.tests = tests;
+ for (auto &test : run.tests)
+ {
+ // As the runner does before it commits a run.
+ for (const auto &attempt : test.attempts)
+ {
+ if (Progmasoft::Catch3::IsFailure(attempt.outcome))
+ {
+ test.signature = Runner::FailureSignature(attempt);
+ break;
+ }
+ }
+ }
+ Runner::RunRecord record;
+ record.commit = "commit" + std::to_string(++number) + "0123456789";
+ auto committed =
+ Runner::History::Commit(file, run, record, {}, {});
+ REQUIRE(committed);
+ history = *committed;
+ }
+ return history;
+ }
+
+ std::string
+ Findings(const TestResult &test)
+ {
+ std::string text;
+ for (const auto &finding : test.analysis)
+ {
+ text += finding + "\n";
+ }
+ return text;
+ }
+} // namespace
+
+TEST_CASE("A failure is placed in the test's history",
+ "[progmasoft][runner][analysis]")
+{
+ const Scratch scratch;
+ const auto file = Runner::HistoryFile(scratch.Path(), "classes");
+ auto flaky = Passed("flaky");
+ flaky.flaky = true;
+ flaky.attempts.front().outcome = Outcome::kFailed;
+ const auto history = HistoryOf(
+ file,
+ {{Passed("fresh"), Failed("stuck", "first"), Passed("wobbly")},
+ {Passed("fresh"), Failed("stuck", "first"), flaky}});
+
+ RunResult run;
+ run.tests = {Failed("unknown"),
+ Failed("fresh", "one"),
+ Failed("stuck", "second", 20),
+ Failed("flaky", "three", 30)};
+ Runner::AnalysisInputs inputs;
+ inputs.history = &history;
+ inputs.program = "tests";
+ Runner::Analyze(run, inputs);
+
+ CHECK(run.tests[0].history == FailureHistory::kUnknown);
+
+ CHECK(run.tests[1].history == FailureHistory::kNew);
+ CHECK_THAT(Findings(run.tests[1]),
+ ContainsSubstring("new failure after 2 passing run(s); it "
+ "last passed at commit commit201234"));
+
+ CHECK(run.tests[2].history == FailureHistory::kPersistent);
+ CHECK_THAT(Findings(run.tests[2]),
+ ContainsSubstring("persistent: failed in the last 2 run(s), "
+ "but differently last time"));
+
+ CHECK(run.tests[3].history == FailureHistory::kIntermittent);
+ CHECK_THAT(Findings(run.tests[3]),
+ ContainsSubstring("intermittent: 1 flaky result(s)"));
+}
+
+TEST_CASE("Failures with one signature are named together",
+ "[progmasoft][runner][analysis]")
+{
+ RunResult run;
+ for (const char *name : {"a", "b", "c", "d", "e"})
+ {
+ // Numbers in the message do not split the group.
+ run.tests.push_back(Failed(
+ name,
+ "connection refused after " +
+ std::to_string((run.tests.size() + 1) * 100) + " ms"));
+ }
+ run.tests.push_back(Failed("elsewhere", "other", 99));
+ Runner::AnalysisInputs inputs;
+ inputs.program = "tests";
+ Runner::Analyze(run, inputs);
+
+ CHECK(run.tests[0].signature == run.tests[4].signature);
+ CHECK(run.tests[0].signature != run.tests[5].signature);
+ CHECK_THAT(Findings(run.tests[0]),
+ ContainsSubstring("fails the same way as 'b', 'c', 'd' and 1 "
+ "more: probably one cause"));
+ CHECK_THAT(Findings(run.tests[5]),
+ !ContainsSubstring("fails the same way"));
+}
+
+TEST_CASE("A crash and a hang say where they stopped",
+ "[progmasoft][runner][analysis]")
+{
+ SectionRecord outer;
+ outer.name = "outer";
+ outer.depth = 0;
+ outer.durationMicroseconds = -1;
+ SectionRecord done;
+ done.name = "done";
+ done.depth = 1;
+ done.durationMicroseconds = 5;
+ SectionRecord inner;
+ inner.name = "inner";
+ inner.depth = 1;
+ inner.durationMicroseconds = -1;
+
+ TestResult crash = Failed("crash");
+ crash.outcome = Outcome::kCrashed;
+ Attempt &crashed = crash.attempts.front();
+ crashed.outcome = Outcome::kCrashed;
+ crashed.category = FailureCategory::kCrash;
+ crashed.assertions.clear();
+ crashed.exitDescription = "signal 11 (SIGSEGV)";
+ crashed.lastLocation = "src/test.cpp:44";
+ crashed.sections = {outer, done, inner};
+
+ TestResult hang = Failed("hang");
+ hang.outcome = Outcome::kTimedOut;
+ hang.predictedDurationMicroseconds = 200000;
+ Attempt &hung = hang.attempts.front();
+ hung.outcome = Outcome::kTimedOut;
+ hung.category = FailureCategory::kTimeout;
+ hung.assertions.clear();
+ hung.durationMicroseconds = 5000000;
+ hung.lastLocation = "src/test.cpp:50";
+ hung.sections = {outer};
+
+ RunResult run;
+ run.tests = {crash, hang};
+ Runner::AnalysisInputs inputs;
+ inputs.program = "tests";
+ Runner::Analyze(run, inputs);
+
+ CHECK_THAT(Findings(run.tests[0]),
+ ContainsSubstring("crashed (signal 11 (SIGSEGV)) after the "
+ "assertion at src/test.cpp:44, in section "
+ "outer / inner"));
+ CHECK_THAT(Findings(run.tests[1]),
+ ContainsSubstring("stopped after 5.0 s"));
+ CHECK_THAT(Findings(run.tests[1]),
+ ContainsSubstring("it was in section outer"));
+ CHECK_THAT(Findings(run.tests[1]),
+ ContainsSubstring("last assertion started at src/test.cpp:50"));
+ // Two crashes in different places are different failures.
+ CHECK(run.tests[0].signature != run.tests[1].signature);
+}
+
+TEST_CASE("A failure names the changed files it touches",
+ "[progmasoft][runner][analysis]")
+{
+ const Scratch scratch;
+ Runner::Changes changes;
+ changes.root = scratch.Path();
+ changes.files = {scratch.Path() / "src" / "test.cpp",
+ scratch.Path() / "src" / "helper.hpp"};
+ TestResult test = Failed("touched");
+ AssertionRecord inHelper;
+ inHelper.file = "src/helper.hpp";
+ inHelper.line = 3;
+ inHelper.passed = false;
+ test.attempts.front().assertions.push_back(inHelper);
+ test.attempts.front().temporaryDirectory = "/tmp/kept";
+
+ RunResult run;
+ run.tests = {test};
+ Runner::AnalysisInputs inputs;
+ inputs.changes = &changes;
+ inputs.changesSince = "main";
+ inputs.program = "tests";
+ Runner::Analyze(run, inputs);
+
+ const auto findings = Findings(run.tests[0]);
+ CHECK_THAT(findings, ContainsSubstring("changed since main: src/test.cpp"));
+ CHECK_THAT(findings, ContainsSubstring("changed since main: src/helper.hpp"));
+ CHECK_THAT(findings,
+ ContainsSubstring("its temporary files are kept in /tmp/kept"));
+}
+
+TEST_CASE("Every failure gets a command that runs it again",
+ "[progmasoft][runner][analysis]")
+{
+ RunResult run;
+ run.tests = {Failed("plain, with [specials]"), Failed("after another"),
+ Passed("fine")};
+ run.tests[1].attempts.front().precededBy = {"polluter"};
+ Runner::AnalysisInputs inputs;
+ inputs.program = "build dir/tests";
+ Runner::Analyze(run, inputs);
+
+ CHECK_THAT(run.tests[0].replayCommand,
+ ContainsSubstring("plain\\\\, with \\\\[specials\\\\]"));
+ CHECK_THAT(run.tests[0].replayCommand, ContainsSubstring("--seed=7"));
+ CHECK_THAT(run.tests[0].replayCommand, ContainsSubstring("build dir/tests"));
+ // A failure that needed other tests first is replayed with them.
+ CHECK_THAT(run.tests[1].replayCommand, ContainsSubstring("--replay"));
+ CHECK(run.tests[2].replayCommand.empty());
+ CHECK(run.tests[2].analysis.empty());
+}
diff --git a/tests/Progmasoft/RunnerOptions.tests.cpp b/tests/Progmasoft/RunnerOptions.tests.cpp
new file mode 100644
index 00000000..2ad8c0b7
--- /dev/null
+++ b/tests/Progmasoft/RunnerOptions.tests.cpp
@@ -0,0 +1,369 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+
+// Every option of the runner's command line: how it is spelled, what it
+// sets and how a wrong value is explained.
+
+#include <Progmasoft/Runner/Files.hpp>
+#include <Progmasoft/Runner/Options.hpp>
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/generators/catch_generators.hpp>
+#include <catch2/matchers/catch_matchers_string.hpp>
+
+#include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <vector>
+
+namespace
+{
+ namespace Runner = Progmasoft::Catch3::Runner;
+ using Catch::Matchers::ContainsSubstring;
+ using std::chrono::milliseconds;
+
+ Runner::Options
+ Parse(const std::vector<std::string> &arguments)
+ {
+ auto options = Runner::ParseOptions(arguments);
+ INFO((options ? std::string{} : options.GetError().Message()));
+ REQUIRE(options);
+ return *options;
+ }
+
+ std::string
+ ParseError(const std::vector<std::string> &arguments)
+ {
+ auto options = Runner::ParseOptions(arguments);
+ REQUIRE_FALSE(options);
+ return options.GetError().Message();
+ }
+} // namespace
+
+TEST_CASE("Options have their documented defaults",
+ "[progmasoft][runner][options]")
+{
+ const auto options = Parse({});
+ CHECK(options.mode == Runner::Mode::kRun);
+ CHECK(options.selection == Runner::Selection::kAll);
+ CHECK(options.order == Runner::Order::kPriority);
+ CHECK(options.retryMode == Runner::RetryMode::kAuto);
+ CHECK(options.retries == 2);
+ CHECK(options.repeat == 1);
+ CHECK(options.failFast == 0);
+ CHECK(options.jobs == 0);
+ CHECK(options.shard.index == 0);
+ CHECK(options.shard.count == 1);
+ CHECK_FALSE(options.timeoutGiven);
+ CHECK(options.adaptiveTimeout);
+ CHECK(options.bisectOrder);
+ CHECK(options.historyEnabled);
+ CHECK(options.quarantine == Runner::QuarantineMode::kAuto);
+ CHECK(options.verbosity == Runner::Verbosity::kNormal);
+ CHECK(options.color == Runner::ColorMode::kAuto);
+ CHECK(options.benchmarkSamples == 100);
+ CHECK(options.benchmarkResamples == 100000);
+ CHECK(options.benchmarkConfidenceInterval == 0.95);
+ CHECK(options.benchmarkWarmupTime == milliseconds{100});
+ CHECK(options.testSpecs.empty());
+}
+
+TEST_CASE("A value can follow an option in every accepted spelling",
+ "[progmasoft][runner][options]")
+{
+ const std::vector<std::string> arguments = GENERATE(
+ std::vector<std::string>{"--jobs=4"},
+ std::vector<std::string>{"--jobs", "4"},
+ std::vector<std::string>{"-j4"},
+ std::vector<std::string>{"-j=4"},
+ std::vector<std::string>{"-j", "4"});
+ CAPTURE(arguments);
+ CHECK(Parse(arguments).jobs == 4);
+}
+
+TEST_CASE("Switches set what they say", "[progmasoft][runner][options]")
+{
+ CHECK(Parse({"--incremental"}).incremental);
+ CHECK(Parse({"--filenames-as-tags"}).filenamesAsTags);
+ CHECK(Parse({"-#"}).filenamesAsTags);
+ CHECK(Parse({"--in-process"}).inProcess);
+ CHECK(Parse({"--isolate"}).isolate);
+ CHECK_FALSE(Parse({"--no-adaptive-timeout"}).adaptiveTimeout);
+ CHECK_FALSE(Parse({"--no-bisect"}).bisectOrder);
+ CHECK(Parse({"--success"}).includeSuccessful);
+ CHECK(Parse({"-s"}).includeSuccessful);
+ CHECK(Parse({"--no-throw"}).noThrow);
+ CHECK(Parse({"-e"}).noThrow);
+ CHECK(Parse({"--allow-no-tests"}).allowNoTests);
+ CHECK(Parse({"--skip-benchmarks"}).skipBenchmarks);
+ CHECK(Parse({"--benchmark-no-analysis"}).benchmarkNoAnalysis);
+ CHECK_FALSE(Parse({"--no-history"}).historyEnabled);
+ CHECK(Parse({"--replay"}).replayFailures);
+ CHECK(Parse({"--verbose"}).verbosity == Runner::Verbosity::kVerbose);
+ CHECK(Parse({"-q"}).verbosity == Runner::Verbosity::kQuiet);
+
+ // A debugger needs the test in this process.
+ const auto debugging = Parse({"-b"});
+ CHECK(debugging.breakIntoDebugger);
+ CHECK(debugging.inProcess);
+}
+
+TEST_CASE("Modes are chosen by their options", "[progmasoft][runner][options]")
+{
+ CHECK(Parse({"--help"}).mode == Runner::Mode::kHelp);
+ CHECK(Parse({"-h"}).mode == Runner::Mode::kHelp);
+ CHECK(Parse({"--version"}).mode == Runner::Mode::kVersion);
+ CHECK(Parse({"--list-tags"}).mode == Runner::Mode::kListTags);
+ CHECK(Parse({"-t"}).mode == Runner::Mode::kListTags);
+ CHECK(Parse({"--catch3-worker"}).mode == Runner::Mode::kWorker);
+
+ const auto text = Parse({"-l"});
+ CHECK(text.mode == Runner::Mode::kListTests);
+ CHECK(text.listFormat == Runner::ListFormat::kText);
+ CHECK(Parse({"--list=names"}).listFormat == Runner::ListFormat::kNames);
+ CHECK(Parse({"--list=json"}).listFormat == Runner::ListFormat::kJson);
+
+ const auto analytics = Parse({"--analytics"});
+ CHECK(analytics.mode == Runner::Mode::kAnalytics);
+ CHECK_FALSE(analytics.analyticsJson);
+ CHECK(Parse({"--analytics=json"}).analyticsJson);
+}
+
+TEST_CASE("Choices accept their names and nothing else",
+ "[progmasoft][runner][options]")
+{
+ CHECK(Parse({"--select=all"}).selection == Runner::Selection::kAll);
+ CHECK(Parse({"--select=failed"}).selection == Runner::Selection::kFailed);
+ CHECK(Parse({"--select=changed"}).selection == Runner::Selection::kChanged);
+ CHECK(Parse({"--select=new"}).selection == Runner::Selection::kNew);
+ CHECK(Parse({"--select=smart"}).selection == Runner::Selection::kSmart);
+ CHECK(Parse({"--order=name"}).order == Runner::Order::kName);
+ CHECK(Parse({"--order=random"}).order == Runner::Order::kRandom);
+ CHECK(Parse({"--order=priority"}).order == Runner::Order::kPriority);
+ CHECK(Parse({"--quarantine=report"}).quarantine ==
+ Runner::QuarantineMode::kReport);
+ CHECK(Parse({"--quarantine=off"}).quarantine ==
+ Runner::QuarantineMode::kOff);
+ CHECK(Parse({"--color=always"}).color == Runner::ColorMode::kAlways);
+ CHECK(Parse({"--color=never"}).color == Runner::ColorMode::kNever);
+
+ CHECK_THAT(ParseError({"--select=some"}),
+ ContainsSubstring("all, failed, changed, new, smart"));
+ CHECK_THAT(ParseError({"--quarantine=maybe"}),
+ ContainsSubstring("auto, report, off"));
+ CHECK_THAT(ParseError({"--color=blue"}),
+ ContainsSubstring("auto, always, never"));
+ CHECK_THAT(ParseError({"--list=xml"}),
+ ContainsSubstring("text, names, json"));
+ CHECK_THAT(ParseError({"--analytics=xml"}), ContainsSubstring("json"));
+}
+
+TEST_CASE("Numbers are checked", "[progmasoft][runner][options]")
+{
+ CHECK(Parse({"--repeat=3"}).repeat == 3);
+ CHECK(Parse({"-a"}).failFast == 1);
+ CHECK(Parse({"--slowest"}).slowest == 10);
+ CHECK(Parse({"--slowest=3"}).slowest == 3);
+ CHECK(Parse({"--benchmark-samples=7"}).benchmarkSamples == 7);
+ CHECK(Parse({"--benchmark-resamples=9"}).benchmarkResamples == 9);
+ CHECK(Parse({"--benchmark-confidence-interval=0.5"})
+ .benchmarkConfidenceInterval == 0.5);
+ CHECK(Parse({"--benchmark-warmup-time=250"}).benchmarkWarmupTime ==
+ milliseconds{250});
+
+ const auto unlimited = Parse({"--timeout=0"});
+ CHECK(unlimited.timeoutGiven);
+ CHECK(unlimited.timeout == milliseconds{0});
+
+ const auto retried = Parse({"--retries=auto"});
+ CHECK(retried.retryMode == Runner::RetryMode::kAuto);
+ const auto off = Parse({"--retries=off"});
+ CHECK(off.retryMode == Runner::RetryMode::kOff);
+ CHECK(off.retries == 0);
+
+ const auto random = Parse({"--seed=5", "--seed=random"});
+ CHECK_FALSE(random.seedGiven);
+
+ CHECK_THAT(ParseError({"--repeat=0"}), ContainsSubstring("at least 1"));
+ CHECK_THAT(ParseError({"--fail-fast=0"}), ContainsSubstring("at least 1"));
+ CHECK_THAT(ParseError({"--slowest=0"}), ContainsSubstring("at least 1"));
+ CHECK_THAT(ParseError({"--retries=-1"}), ContainsSubstring("whole number"));
+ CHECK_THAT(ParseError({"--timeout=soon"}), ContainsSubstring("seconds"));
+ CHECK_THAT(ParseError({"--timeout=-2"}), ContainsSubstring("seconds"));
+ CHECK_THAT(ParseError({"--benchmark-confidence-interval=1"}),
+ ContainsSubstring("--benchmark-confidence-interval"));
+ CHECK_THAT(ParseError({"--benchmark-samples=0"}),
+ ContainsSubstring("at least 1"));
+ CHECK_THAT(ParseError({"--seed=-1"}), ContainsSubstring("--seed"));
+ CHECK_THAT(ParseError({"--generator-index=x"}),
+ ContainsSubstring("index or *"));
+ CHECK_THAT(ParseError({"--path-filter=g:x"}),
+ ContainsSubstring("generator index or *"));
+}
+
+TEST_CASE("Shards are an index below a count", "[progmasoft][runner][options]")
+{
+ const auto last = Parse({"--shard=3/4"});
+ CHECK(last.shard.index == 3);
+ CHECK(last.shard.count == 4);
+ const std::string wrong = GENERATE("4/4", "1", "/2", "1/", "0/0", "a/b");
+ CAPTURE(wrong);
+ CHECK_THAT(ParseError({"--shard=" + wrong}), ContainsSubstring("INDEX/COUNT"));
+}
+
+TEST_CASE("Warnings are named", "[progmasoft][runner][options]")
+{
+ CHECK(Parse({"--warn=no-assertions"}).warnNoAssertions);
+ CHECK(Parse({"-w", "NoAssertions"}).warnNoAssertions);
+ CHECK(Parse({"--warn=unmatched-specs"}).warnUnmatchedSpecs);
+ CHECK(Parse({"--warn=infinite-generators"}).warnInfiniteGenerators);
+ CHECK_THAT(ParseError({"--warn=everything"}),
+ ContainsSubstring("no-assertions"));
+}
+
+TEST_CASE("Files and directories are taken as written",
+ "[progmasoft][runner][options]")
+{
+ const auto options = Parse({"--junit=out/junit.xml",
+ "--json",
+ "out/run.json",
+ "--trace=out/trace.json",
+ "--history=cache",
+ "--shard-durations=last.json",
+ "--changes-since=main",
+ "--premature-exit-guard-file=guard"});
+ CHECK(options.junitPath == "out/junit.xml");
+ CHECK(options.jsonPath == "out/run.json");
+ CHECK(options.tracePath == "out/trace.json");
+ CHECK(options.historyDirectory == "cache");
+ CHECK(options.historyEnabled);
+ CHECK(options.shardDurations == "last.json");
+ CHECK(options.changesSince == "main");
+ CHECK(options.exitGuardPath == "guard");
+}
+
+TEST_CASE("Test specs are kept apart from options",
+ "[progmasoft][runner][options]")
+{
+ const auto options =
+ Parse({"name", "[tag]", "~[slow]", "-", "--", "-j", "--x"});
+ CHECK(options.jobs == 0);
+ CHECK(options.testSpecs ==
+ std::vector<std::string>{"name", "[tag]", "~[slow]", "-", "-j", "--x"});
+}
+
+TEST_CASE("An input file names tests exactly", "[progmasoft][runner][options]")
+{
+ std::error_code error;
+ const auto file = std::filesystem::temp_directory_path(error) /
+ ("catch3-input-" + Runner::UniqueName());
+ std::ofstream(file) << "# a comment\n"
+ "first test\r\n"
+ "\n"
+ " \"quoted, with a comma\" \n"
+ "last*\n";
+ const auto options = Parse({"--input-file=" + file.string()});
+ CHECK(options.testSpecs == std::vector<std::string>{"\"first test\"",
+ ",",
+ "\"quoted, with a comma\"",
+ ",",
+ "\"last*\""});
+ std::filesystem::remove(file, error);
+ CHECK_THAT(ParseError({"-f", file.string()}), ContainsSubstring("cannot read"));
+}
+
+TEST_CASE("Unknown options are refused with a suggestion",
+ "[progmasoft][runner][options]")
+{
+ CHECK_THAT(ParseError({"--sucess"}), ContainsSubstring("did you mean --success?"));
+ CHECK_THAT(ParseError({"--histroy=x"}), ContainsSubstring("did you mean --history?"));
+ CHECK_THAT(ParseError({"-z"}), ContainsSubstring("'-z' is not an option"));
+ // Catch2's reporter options are gone with its reporters.
+ CHECK_THAT(ParseError({"--reporter=junit"}), ContainsSubstring("is not an option"));
+ CHECK_THAT(ParseError({"--isolate=yes"}), ContainsSubstring("takes no value"));
+ CHECK_THAT(ParseError({"--junit"}), ContainsSubstring("--junit=FILE"));
+}
+
+TEST_CASE("The help lists every option", "[progmasoft][runner][options]")
+{
+ const std::string help = Runner::HelpText("tests");
+ CHECK_THAT(help, ContainsSubstring("usage: tests [options] [test specs...]"));
+ for (const char *name : {"--section", "--generator-index",
+ "--path-filter", "--input-file",
+ "--select", "--changes-since",
+ "--incremental", "--shard",
+ "--shard-durations", "--filenames-as-tags",
+ "--jobs", "--in-process",
+ "--isolate", "--timeout",
+ "--no-adaptive-timeout", "--retries",
+ "--repeat", "--fail-fast",
+ "--order", "--seed",
+ "--no-bisect", "--success",
+ "--break", "--no-throw",
+ "--warn", "--allow-no-tests",
+ "--skip-benchmarks", "--benchmark-samples",
+ "--benchmark-resamples",
+ "--benchmark-confidence-interval",
+ "--benchmark-no-analysis",
+ "--benchmark-warmup-time",
+ "--history", "--no-history",
+ "--quarantine", "--replay",
+ "--analytics", "--verbose",
+ "--quiet", "--color",
+ "--junit", "--json",
+ "--trace", "--premature-exit-guard-file",
+ "--slowest", "--list",
+ "--list-tags", "--help",
+ "--version"})
+ {
+ CAPTURE(name);
+ CHECK_THAT(help, ContainsSubstring(name));
+ }
+ // The worker option is internal.
+ CHECK_THAT(help, !ContainsSubstring("--catch3-worker"));
+}
+
+TEST_CASE("A worker gets every option that changes how tests behave",
+ "[progmasoft][runner][options]")
+{
+ const auto options = Parse({"--section=outer",
+ "--section=inner",
+ "-p",
+ "g:1",
+ "-#",
+ "-s",
+ "-e",
+ "--warn=no-assertions",
+ "--warn=infinite-generators",
+ "--skip-benchmarks",
+ "--benchmark-no-analysis",
+ "--benchmark-samples=3",
+ "--benchmark-resamples=4",
+ "--benchmark-warmup-time=5",
+ "--benchmark-confidence-interval=0.9",
+ "--seed=6",
+ // Runner-side only:
+ "--jobs=9",
+ "--junit=x.xml",
+ "[tag]"});
+ const auto worker = Parse(Runner::WorkerArguments(options));
+ CHECK(worker.mode == Runner::Mode::kWorker);
+ CHECK(worker.sections == options.sections);
+ CHECK(worker.paths == options.paths);
+ CHECK(worker.filenamesAsTags);
+ CHECK(worker.includeSuccessful);
+ CHECK(worker.noThrow);
+ CHECK(worker.warnNoAssertions);
+ CHECK(worker.warnInfiniteGenerators);
+ CHECK(worker.skipBenchmarks);
+ CHECK(worker.benchmarkNoAnalysis);
+ CHECK(worker.benchmarkSamples == 3);
+ CHECK(worker.benchmarkResamples == 4);
+ CHECK(worker.benchmarkWarmupTime == milliseconds{5});
+ CHECK(worker.benchmarkConfidenceInterval == 0.9);
+ CHECK(worker.seed == 6);
+ CHECK(worker.jobs == 0);
+ CHECK(worker.junitPath.empty());
+ CHECK(worker.testSpecs.empty());
+}
diff --git a/tests/Progmasoft/RunnerPlanner.tests.cpp b/tests/Progmasoft/RunnerPlanner.tests.cpp
new file mode 100644
index 00000000..4c930632
--- /dev/null
+++ b/tests/Progmasoft/RunnerPlanner.tests.cpp
@@ -0,0 +1,599 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+
+// The planner decides which tests run, where, in which order and under
+// which limits, from the options, the history and the changed files. These
+// tests give it a catalog of their own, so no test of this binary is
+// involved.
+
+#include <Progmasoft/Runner/Catalog.hpp>
+#include <Progmasoft/Runner/Files.hpp>
+#include <Progmasoft/Runner/History.hpp>
+#include <Progmasoft/Runner/Planner.hpp>
+
+#include <catch2/catch_test_macros.hpp>
+#include <catch2/matchers/catch_matchers_string.hpp>
+
+#include <algorithm>
+#include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+namespace
+{
+ namespace Runner = Progmasoft::Catch3::Runner;
+ using Catch::Matchers::ContainsSubstring;
+ using Progmasoft::Catch3::Attempt;
+ using Progmasoft::Catch3::Outcome;
+ using Progmasoft::Catch3::RunResult;
+ using Progmasoft::Catch3::TestResult;
+ using std::chrono::milliseconds;
+
+ /// A directory of its own, removed at the end of the test.
+ class Scratch final
+ {
+ public:
+ Scratch()
+ {
+ std::error_code error;
+ path_ = std::filesystem::temp_directory_path(error) /
+ ("catch3-planner-" + Runner::UniqueName());
+ std::filesystem::create_directories(path_, error);
+ }
+ Scratch(const Scratch &) = delete;
+ Scratch &
+ operator=(const Scratch &) = delete;
+ ~Scratch()
+ {
+ std::error_code error;
+ std::filesystem::remove_all(path_, error);
+ }
+
+ [[nodiscard]] const std::filesystem::path &
+ Path() const noexcept