diff options
| author | Leitwolf11 | 2026-10-24 10:06:25 +0300 |
|---|---|---|
| committer | Leitwolf11 | 2026-10-24 10:06:25 +0300 |
| commit | e104b694513964d5a84e77f751203cda970d741b (patch) | |
| tree | a7d6e6d5cca1e98702549f02489e3570599ca3b4 /tests | |
| parent | 15eb534ee493a3388c168709eb79a954e9cb5fcf (diff) | |
| download | catch3-e104b694513964d5a84e77f751203cda970d741b.tar.gz | |
Fix the CI failures of the runner and test its planner, options and analysis
ExtraTests X21 and X27 defined their listeners at namespace scope, which
clang rejects under -Wweak-vtables with -Werror on Linux and macOS; they
are now in anonymous namespaces.
After a fatal signal Catch2 closes the open sections itself, so on POSIX a
crash was reported outside the section it happened in. The scheduler now
ignores section ends that follow a fatal condition.
New unit tests cover the planner (every selection, dependency closure and
cycles, hash and duration shards, incremental runs, learned time limits,
ordering, retries, quarantine and replay), every command-line option and
its errors, and the failure analysis (history classes, signature groups,
crash and hang locations, changed files, replay commands).
Catalog::FromEntries builds a catalog without a test registry so the
planner can be tested on its own.
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/CMakeLists.txt | 3 | ||||
| -rw-r--r-- | tests/ExtraTests/X21-PartialTestCaseEvents.cpp | 2 | ||||
| -rw-r--r-- | tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp | 2 | ||||
| -rw-r--r-- | tests/Progmasoft/BUILD.bazel | 3 | ||||
| -rw-r--r-- | tests/Progmasoft/RunnerAnalysis.tests.cpp | 321 | ||||
| -rw-r--r-- | tests/Progmasoft/RunnerOptions.tests.cpp | 369 | ||||
| -rw-r--r-- | tests/Progmasoft/RunnerPlanner.tests.cpp | 599 | ||||
| -rw-r--r-- | tests/meson.build | 3 |
8 files changed, 1302 insertions, 0 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 14f2ed56..e57ad94a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -157,6 +157,9 @@ add_executable(ProgmasoftSelfTest Progmasoft/Results.tests.cpp Progmasoft/Runner.tests.cpp Progmasoft/RunnerUnits.tests.cpp + Progmasoft/RunnerPlanner.tests.cpp + Progmasoft/RunnerOptions.tests.cpp + Progmasoft/RunnerAnalysis.tests.cpp Progmasoft/Snapshot.tests.cpp Progmasoft/Stateful.tests.cpp Progmasoft/Strategy.tests.cpp diff --git a/tests/ExtraTests/X21-PartialTestCaseEvents.cpp b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp index c55f6c2b..99002822 100644 --- a/tests/ExtraTests/X21-PartialTestCaseEvents.cpp +++ b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp @@ -28,6 +28,7 @@ using Catch::TestCaseInfo; using Catch::TestCaseStats; +namespace { class PartialListener : public Catch::EventListenerBase { public: using EventListenerBase::EventListenerBase; @@ -58,6 +59,7 @@ public: private: std::string m_events; }; +} // namespace CATCH_REGISTER_LISTENER(PartialListener) diff --git a/tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp b/tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp index 7ceebcee..20b39d78 100644 --- a/tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp +++ b/tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp @@ -27,6 +27,7 @@ #include <iostream> #include <string> +namespace { class TestListener : public Catch::EventListenerBase { std::string stdOutString( uint64_t iter ){ return "stdout " + std::to_string( iter ) + '\n'; @@ -66,6 +67,7 @@ public: std::cerr << "X27 - listener saw " << m_errors << " error(s)\n"; } }; +} // namespace CATCH_REGISTER_LISTENER( TestListener ) diff --git a/tests/Progmasoft/BUILD.bazel b/tests/Progmasoft/BUILD.bazel index 296fd5f1..06df86d6 100644 --- a/tests/Progmasoft/BUILD.bazel +++ b/tests/Progmasoft/BUILD.bazel @@ -13,6 +13,9 @@ cc_test( "Results.tests.cpp", "Runner.tests.cpp", "RunnerUnits.tests.cpp", + "RunnerPlanner.tests.cpp", + "RunnerOptions.tests.cpp", + "RunnerAnalysis.tests.cpp", "Snapshot.tests.cpp", "Stateful.tests.cpp", "Strategy.tests.cpp", diff --git a/tests/Progmasoft/RunnerAnalysis.tests.cpp b/tests/Progmasoft/RunnerAnalysis.tests.cpp new file mode 100644 index 00000000..453fa671 --- /dev/null +++ b/tests/Progmasoft/RunnerAnalysis.tests.cpp @@ -0,0 +1,321 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +// What the runner writes down about a failure: how it relates to earlier +// runs and to other failures, where it stopped, what changed and how to +// run it again. + +#include <Progmasoft/Runner/Analysis.hpp> +#include <Progmasoft/Runner/Files.hpp> +#include <Progmasoft/Runner/History.hpp> + +#include <catch2/catch_test_macros.hpp> +#include <catch2/matchers/catch_matchers_string.hpp> +#include <catch2/matchers/catch_matchers_vector.hpp> + +#include <filesystem> +#include <string> +#include <vector> + +namespace +{ + namespace Runner = Progmasoft::Catch3::Runner; + using Catch::Matchers::ContainsSubstring; + using Progmasoft::Catch3::AssertionRecord; + using Progmasoft::Catch3::Attempt; + using Progmasoft::Catch3::FailureCategory; + using Progmasoft::Catch3::FailureHistory; + using Progmasoft::Catch3::Outcome; + using Progmasoft::Catch3::RunResult; + using Progmasoft::Catch3::SectionRecord; + using Progmasoft::Catch3::TestResult; + + /// A directory of its own, removed at the end of the test. + class Scratch final + { + public: + Scratch() + { + std::error_code error; + path_ = std::filesystem::temp_directory_path(error) / + ("catch3-analysis-" + Runner::UniqueName()); + std::filesystem::create_directories(path_, error); + } + Scratch(const Scratch &) = delete; + Scratch & + operator=(const Scratch &) = delete; + ~Scratch() + { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + [[nodiscard]] const std::filesystem::path & + Path() const noexcept + { + return path_; + } + + private: + std::filesystem::path path_; + }; + + /// A test that failed one assertion. + TestResult + Failed(const std::string &name, + const std::string &message = "wrong", + std::size_t line = 10) + { + TestResult test; + test.id = name; + test.name = name; + test.file = "src/test.cpp"; + test.line = 1; + test.outcome = Outcome::kFailed; + test.category = FailureCategory::kAssertion; + Attempt attempt; + attempt.outcome = Outcome::kFailed; + attempt.category = FailureCategory::kAssertion; + attempt.seed = 7; + AssertionRecord assertion; + assertion.file = "src/test.cpp"; + assertion.line = line; + assertion.expression = "a == b"; + assertion.message = message; + attempt.assertions.push_back(assertion); + test.attempts.push_back(attempt); + return test; + } + + TestResult + Passed(const std::string &name) + { + TestResult test; + test.id = name; + test.name = name; + test.outcome = Outcome::kPassed; + Attempt attempt; + attempt.outcome = Outcome::kPassed; + test.attempts.push_back(attempt); + return test; + } + + /// Commits runs of `tests` to a history, the last commit last. + Runner::History + HistoryOf(const std::filesystem::path &file, + const std::vector<std::vector<TestResult>> &runs) + { + Runner::History history; + int number = 0; + for (const auto &tests : runs) + { + RunResult run; + run.tests = tests; + for (auto &test : run.tests) + { + // As the runner does before it commits a run. + for (const auto &attempt : test.attempts) + { + if (Progmasoft::Catch3::IsFailure(attempt.outcome)) + { + test.signature = Runner::FailureSignature(attempt); + break; + } + } + } + Runner::RunRecord record; + record.commit = "commit" + std::to_string(++number) + "0123456789"; + auto committed = + Runner::History::Commit(file, run, record, {}, {}); + REQUIRE(committed); + history = *committed; + } + return history; + } + + std::string + Findings(const TestResult &test) + { + std::string text; + for (const auto &finding : test.analysis) + { + text += finding + "\n"; + } + return text; + } +} // namespace + +TEST_CASE("A failure is placed in the test's history", + "[progmasoft][runner][analysis]") +{ + const Scratch scratch; + const auto file = Runner::HistoryFile(scratch.Path(), "classes"); + auto flaky = Passed("flaky"); + flaky.flaky = true; + flaky.attempts.front().outcome = Outcome::kFailed; + const auto history = HistoryOf( + file, + {{Passed("fresh"), Failed("stuck", "first"), Passed("wobbly")}, + {Passed("fresh"), Failed("stuck", "first"), flaky}}); + + RunResult run; + run.tests = {Failed("unknown"), + Failed("fresh", "one"), + Failed("stuck", "second", 20), + Failed("flaky", "three", 30)}; + Runner::AnalysisInputs inputs; + inputs.history = &history; + inputs.program = "tests"; + Runner::Analyze(run, inputs); + + CHECK(run.tests[0].history == FailureHistory::kUnknown); + + CHECK(run.tests[1].history == FailureHistory::kNew); + CHECK_THAT(Findings(run.tests[1]), + ContainsSubstring("new failure after 2 passing run(s); it " + "last passed at commit commit201234")); + + CHECK(run.tests[2].history == FailureHistory::kPersistent); + CHECK_THAT(Findings(run.tests[2]), + ContainsSubstring("persistent: failed in the last 2 run(s), " + "but differently last time")); + + CHECK(run.tests[3].history == FailureHistory::kIntermittent); + CHECK_THAT(Findings(run.tests[3]), + ContainsSubstring("intermittent: 1 flaky result(s)")); +} + +TEST_CASE("Failures with one signature are named together", + "[progmasoft][runner][analysis]") +{ + RunResult run; + for (const char *name : {"a", "b", "c", "d", "e"}) + { + // Numbers in the message do not split the group. + run.tests.push_back(Failed( + name, + "connection refused after " + + std::to_string((run.tests.size() + 1) * 100) + " ms")); + } + run.tests.push_back(Failed("elsewhere", "other", 99)); + Runner::AnalysisInputs inputs; + inputs.program = "tests"; + Runner::Analyze(run, inputs); + + CHECK(run.tests[0].signature == run.tests[4].signature); + CHECK(run.tests[0].signature != run.tests[5].signature); + CHECK_THAT(Findings(run.tests[0]), + ContainsSubstring("fails the same way as 'b', 'c', 'd' and 1 " + "more: probably one cause")); + CHECK_THAT(Findings(run.tests[5]), + !ContainsSubstring("fails the same way")); +} + +TEST_CASE("A crash and a hang say where they stopped", + "[progmasoft][runner][analysis]") +{ + SectionRecord outer; + outer.name = "outer"; + outer.depth = 0; + outer.durationMicroseconds = -1; + SectionRecord done; + done.name = "done"; + done.depth = 1; + done.durationMicroseconds = 5; + SectionRecord inner; + inner.name = "inner"; + inner.depth = 1; + inner.durationMicroseconds = -1; + + TestResult crash = Failed("crash"); + crash.outcome = Outcome::kCrashed; + Attempt &crashed = crash.attempts.front(); + crashed.outcome = Outcome::kCrashed; + crashed.category = FailureCategory::kCrash; + crashed.assertions.clear(); + crashed.exitDescription = "signal 11 (SIGSEGV)"; + crashed.lastLocation = "src/test.cpp:44"; + crashed.sections = {outer, done, inner}; + + TestResult hang = Failed("hang"); + hang.outcome = Outcome::kTimedOut; + hang.predictedDurationMicroseconds = 200000; + Attempt &hung = hang.attempts.front(); + hung.outcome = Outcome::kTimedOut; + hung.category = FailureCategory::kTimeout; + hung.assertions.clear(); + hung.durationMicroseconds = 5000000; + hung.lastLocation = "src/test.cpp:50"; + hung.sections = {outer}; + + RunResult run; + run.tests = {crash, hang}; + Runner::AnalysisInputs inputs; + inputs.program = "tests"; + Runner::Analyze(run, inputs); + + CHECK_THAT(Findings(run.tests[0]), + ContainsSubstring("crashed (signal 11 (SIGSEGV)) after the " + "assertion at src/test.cpp:44, in section " + "outer / inner")); + CHECK_THAT(Findings(run.tests[1]), + ContainsSubstring("stopped after 5.0 s")); + CHECK_THAT(Findings(run.tests[1]), + ContainsSubstring("it was in section outer")); + CHECK_THAT(Findings(run.tests[1]), + ContainsSubstring("last assertion started at src/test.cpp:50")); + // Two crashes in different places are different failures. + CHECK(run.tests[0].signature != run.tests[1].signature); +} + +TEST_CASE("A failure names the changed files it touches", + "[progmasoft][runner][analysis]") +{ + const Scratch scratch; + Runner::Changes changes; + changes.root = scratch.Path(); + changes.files = {scratch.Path() / "src" / "test.cpp", + scratch.Path() / "src" / "helper.hpp"}; + TestResult test = Failed("touched"); + AssertionRecord inHelper; + inHelper.file = "src/helper.hpp"; + inHelper.line = 3; + inHelper.passed = false; + test.attempts.front().assertions.push_back(inHelper); + test.attempts.front().temporaryDirectory = "/tmp/kept"; + + RunResult run; + run.tests = {test}; + Runner::AnalysisInputs inputs; + inputs.changes = &changes; + inputs.changesSince = "main"; + inputs.program = "tests"; + Runner::Analyze(run, inputs); + + const auto findings = Findings(run.tests[0]); + CHECK_THAT(findings, ContainsSubstring("changed since main: src/test.cpp")); + CHECK_THAT(findings, ContainsSubstring("changed since main: src/helper.hpp")); + CHECK_THAT(findings, + ContainsSubstring("its temporary files are kept in /tmp/kept")); +} + +TEST_CASE("Every failure gets a command that runs it again", + "[progmasoft][runner][analysis]") +{ + RunResult run; + run.tests = {Failed("plain, with [specials]"), Failed("after another"), + Passed("fine")}; + run.tests[1].attempts.front().precededBy = {"polluter"}; + Runner::AnalysisInputs inputs; + inputs.program = "build dir/tests"; + Runner::Analyze(run, inputs); + + CHECK_THAT(run.tests[0].replayCommand, + ContainsSubstring("plain\\\\, with \\\\[specials\\\\]")); + CHECK_THAT(run.tests[0].replayCommand, ContainsSubstring("--seed=7")); + CHECK_THAT(run.tests[0].replayCommand, ContainsSubstring("build dir/tests")); + // A failure that needed other tests first is replayed with them. + CHECK_THAT(run.tests[1].replayCommand, ContainsSubstring("--replay")); + CHECK(run.tests[2].replayCommand.empty()); + CHECK(run.tests[2].analysis.empty()); +} diff --git a/tests/Progmasoft/RunnerOptions.tests.cpp b/tests/Progmasoft/RunnerOptions.tests.cpp new file mode 100644 index 00000000..2ad8c0b7 --- /dev/null +++ b/tests/Progmasoft/RunnerOptions.tests.cpp @@ -0,0 +1,369 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +// Every option of the runner's command line: how it is spelled, what it +// sets and how a wrong value is explained. + +#include <Progmasoft/Runner/Files.hpp> +#include <Progmasoft/Runner/Options.hpp> + +#include <catch2/catch_test_macros.hpp> +#include <catch2/generators/catch_generators.hpp> +#include <catch2/matchers/catch_matchers_string.hpp> + +#include <chrono> +#include <filesystem> +#include <fstream> +#include <string> +#include <vector> + +namespace +{ + namespace Runner = Progmasoft::Catch3::Runner; + using Catch::Matchers::ContainsSubstring; + using std::chrono::milliseconds; + + Runner::Options + Parse(const std::vector<std::string> &arguments) + { + auto options = Runner::ParseOptions(arguments); + INFO((options ? std::string{} : options.GetError().Message())); + REQUIRE(options); + return *options; + } + + std::string + ParseError(const std::vector<std::string> &arguments) + { + auto options = Runner::ParseOptions(arguments); + REQUIRE_FALSE(options); + return options.GetError().Message(); + } +} // namespace + +TEST_CASE("Options have their documented defaults", + "[progmasoft][runner][options]") +{ + const auto options = Parse({}); + CHECK(options.mode == Runner::Mode::kRun); + CHECK(options.selection == Runner::Selection::kAll); + CHECK(options.order == Runner::Order::kPriority); + CHECK(options.retryMode == Runner::RetryMode::kAuto); + CHECK(options.retries == 2); + CHECK(options.repeat == 1); + CHECK(options.failFast == 0); + CHECK(options.jobs == 0); + CHECK(options.shard.index == 0); + CHECK(options.shard.count == 1); + CHECK_FALSE(options.timeoutGiven); + CHECK(options.adaptiveTimeout); + CHECK(options.bisectOrder); + CHECK(options.historyEnabled); + CHECK(options.quarantine == Runner::QuarantineMode::kAuto); + CHECK(options.verbosity == Runner::Verbosity::kNormal); + CHECK(options.color == Runner::ColorMode::kAuto); + CHECK(options.benchmarkSamples == 100); + CHECK(options.benchmarkResamples == 100000); + CHECK(options.benchmarkConfidenceInterval == 0.95); + CHECK(options.benchmarkWarmupTime == milliseconds{100}); + CHECK(options.testSpecs.empty()); +} + +TEST_CASE("A value can follow an option in every accepted spelling", + "[progmasoft][runner][options]") +{ + const std::vector<std::string> arguments = GENERATE( + std::vector<std::string>{"--jobs=4"}, + std::vector<std::string>{"--jobs", "4"}, + std::vector<std::string>{"-j4"}, + std::vector<std::string>{"-j=4"}, + std::vector<std::string>{"-j", "4"}); + CAPTURE(arguments); + CHECK(Parse(arguments).jobs == 4); +} + +TEST_CASE("Switches set what they say", "[progmasoft][runner][options]") +{ + CHECK(Parse({"--incremental"}).incremental); + CHECK(Parse({"--filenames-as-tags"}).filenamesAsTags); + CHECK(Parse({"-#"}).filenamesAsTags); + CHECK(Parse({"--in-process"}).inProcess); + CHECK(Parse({"--isolate"}).isolate); + CHECK_FALSE(Parse({"--no-adaptive-timeout"}).adaptiveTimeout); + CHECK_FALSE(Parse({"--no-bisect"}).bisectOrder); + CHECK(Parse({"--success"}).includeSuccessful); + CHECK(Parse({"-s"}).includeSuccessful); + CHECK(Parse({"--no-throw"}).noThrow); + CHECK(Parse({"-e"}).noThrow); + CHECK(Parse({"--allow-no-tests"}).allowNoTests); + CHECK(Parse({"--skip-benchmarks"}).skipBenchmarks); + CHECK(Parse({"--benchmark-no-analysis"}).benchmarkNoAnalysis); + CHECK_FALSE(Parse({"--no-history"}).historyEnabled); + CHECK(Parse({"--replay"}).replayFailures); + CHECK(Parse({"--verbose"}).verbosity == Runner::Verbosity::kVerbose); + CHECK(Parse({"-q"}).verbosity == Runner::Verbosity::kQuiet); + + // A debugger needs the test in this process. + const auto debugging = Parse({"-b"}); + CHECK(debugging.breakIntoDebugger); + CHECK(debugging.inProcess); +} + +TEST_CASE("Modes are chosen by their options", "[progmasoft][runner][options]") +{ + CHECK(Parse({"--help"}).mode == Runner::Mode::kHelp); + CHECK(Parse({"-h"}).mode == Runner::Mode::kHelp); + CHECK(Parse({"--version"}).mode == Runner::Mode::kVersion); + CHECK(Parse({"--list-tags"}).mode == Runner::Mode::kListTags); + CHECK(Parse({"-t"}).mode == Runner::Mode::kListTags); + CHECK(Parse({"--catch3-worker"}).mode == Runner::Mode::kWorker); + + const auto text = Parse({"-l"}); + CHECK(text.mode == Runner::Mode::kListTests); + CHECK(text.listFormat == Runner::ListFormat::kText); + CHECK(Parse({"--list=names"}).listFormat == Runner::ListFormat::kNames); + CHECK(Parse({"--list=json"}).listFormat == Runner::ListFormat::kJson); + + const auto analytics = Parse({"--analytics"}); + CHECK(analytics.mode == Runner::Mode::kAnalytics); + CHECK_FALSE(analytics.analyticsJson); + CHECK(Parse({"--analytics=json"}).analyticsJson); +} + +TEST_CASE("Choices accept their names and nothing else", + "[progmasoft][runner][options]") +{ + CHECK(Parse({"--select=all"}).selection == Runner::Selection::kAll); + CHECK(Parse({"--select=failed"}).selection == Runner::Selection::kFailed); + CHECK(Parse({"--select=changed"}).selection == Runner::Selection::kChanged); + CHECK(Parse({"--select=new"}).selection == Runner::Selection::kNew); + CHECK(Parse({"--select=smart"}).selection == Runner::Selection::kSmart); + CHECK(Parse({"--order=name"}).order == Runner::Order::kName); + CHECK(Parse({"--order=random"}).order == Runner::Order::kRandom); + CHECK(Parse({"--order=priority"}).order == Runner::Order::kPriority); + CHECK(Parse({"--quarantine=report"}).quarantine == + Runner::QuarantineMode::kReport); + CHECK(Parse({"--quarantine=off"}).quarantine == + Runner::QuarantineMode::kOff); + CHECK(Parse({"--color=always"}).color == Runner::ColorMode::kAlways); + CHECK(Parse({"--color=never"}).color == Runner::ColorMode::kNever); + + CHECK_THAT(ParseError({"--select=some"}), + ContainsSubstring("all, failed, changed, new, smart")); + CHECK_THAT(ParseError({"--quarantine=maybe"}), + ContainsSubstring("auto, report, off")); + CHECK_THAT(ParseError({"--color=blue"}), + ContainsSubstring("auto, always, never")); + CHECK_THAT(ParseError({"--list=xml"}), + ContainsSubstring("text, names, json")); + CHECK_THAT(ParseError({"--analytics=xml"}), ContainsSubstring("json")); +} + +TEST_CASE("Numbers are checked", "[progmasoft][runner][options]") +{ + CHECK(Parse({"--repeat=3"}).repeat == 3); + CHECK(Parse({"-a"}).failFast == 1); + CHECK(Parse({"--slowest"}).slowest == 10); + CHECK(Parse({"--slowest=3"}).slowest == 3); + CHECK(Parse({"--benchmark-samples=7"}).benchmarkSamples == 7); + CHECK(Parse({"--benchmark-resamples=9"}).benchmarkResamples == 9); + CHECK(Parse({"--benchmark-confidence-interval=0.5"}) + .benchmarkConfidenceInterval == 0.5); + CHECK(Parse({"--benchmark-warmup-time=250"}).benchmarkWarmupTime == + milliseconds{250}); + + const auto unlimited = Parse({"--timeout=0"}); + CHECK(unlimited.timeoutGiven); + CHECK(unlimited.timeout == milliseconds{0}); + + const auto retried = Parse({"--retries=auto"}); + CHECK(retried.retryMode == Runner::RetryMode::kAuto); + const auto off = Parse({"--retries=off"}); + CHECK(off.retryMode == Runner::RetryMode::kOff); + CHECK(off.retries == 0); + + const auto random = Parse({"--seed=5", "--seed=random"}); + CHECK_FALSE(random.seedGiven); + + CHECK_THAT(ParseError({"--repeat=0"}), ContainsSubstring("at least 1")); + CHECK_THAT(ParseError({"--fail-fast=0"}), ContainsSubstring("at least 1")); + CHECK_THAT(ParseError({"--slowest=0"}), ContainsSubstring("at least 1")); + CHECK_THAT(ParseError({"--retries=-1"}), ContainsSubstring("whole number")); + CHECK_THAT(ParseError({"--timeout=soon"}), ContainsSubstring("seconds")); + CHECK_THAT(ParseError({"--timeout=-2"}), ContainsSubstring("seconds")); + CHECK_THAT(ParseError({"--benchmark-confidence-interval=1"}), + ContainsSubstring("--benchmark-confidence-interval")); + CHECK_THAT(ParseError({"--benchmark-samples=0"}), + ContainsSubstring("at least 1")); + CHECK_THAT(ParseError({"--seed=-1"}), ContainsSubstring("--seed")); + CHECK_THAT(ParseError({"--generator-index=x"}), + ContainsSubstring("index or *")); + CHECK_THAT(ParseError({"--path-filter=g:x"}), + ContainsSubstring("generator index or *")); +} + +TEST_CASE("Shards are an index below a count", "[progmasoft][runner][options]") +{ + const auto last = Parse({"--shard=3/4"}); + CHECK(last.shard.index == 3); + CHECK(last.shard.count == 4); + const std::string wrong = GENERATE("4/4", "1", "/2", "1/", "0/0", "a/b"); + CAPTURE(wrong); + CHECK_THAT(ParseError({"--shard=" + wrong}), ContainsSubstring("INDEX/COUNT")); +} + +TEST_CASE("Warnings are named", "[progmasoft][runner][options]") +{ + CHECK(Parse({"--warn=no-assertions"}).warnNoAssertions); + CHECK(Parse({"-w", "NoAssertions"}).warnNoAssertions); + CHECK(Parse({"--warn=unmatched-specs"}).warnUnmatchedSpecs); + CHECK(Parse({"--warn=infinite-generators"}).warnInfiniteGenerators); + CHECK_THAT(ParseError({"--warn=everything"}), + ContainsSubstring("no-assertions")); +} + +TEST_CASE("Files and directories are taken as written", + "[progmasoft][runner][options]") +{ + const auto options = Parse({"--junit=out/junit.xml", + "--json", + "out/run.json", + "--trace=out/trace.json", + "--history=cache", + "--shard-durations=last.json", + "--changes-since=main", + "--premature-exit-guard-file=guard"}); + CHECK(options.junitPath == "out/junit.xml"); + CHECK(options.jsonPath == "out/run.json"); + CHECK(options.tracePath == "out/trace.json"); + CHECK(options.historyDirectory == "cache"); + CHECK(options.historyEnabled); + CHECK(options.shardDurations == "last.json"); + CHECK(options.changesSince == "main"); + CHECK(options.exitGuardPath == "guard"); +} + +TEST_CASE("Test specs are kept apart from options", + "[progmasoft][runner][options]") +{ + const auto options = + Parse({"name", "[tag]", "~[slow]", "-", "--", "-j", "--x"}); + CHECK(options.jobs == 0); + CHECK(options.testSpecs == + std::vector<std::string>{"name", "[tag]", "~[slow]", "-", "-j", "--x"}); +} + +TEST_CASE("An input file names tests exactly", "[progmasoft][runner][options]") +{ + std::error_code error; + const auto file = std::filesystem::temp_directory_path(error) / + ("catch3-input-" + Runner::UniqueName()); + std::ofstream(file) << "# a comment\n" + "first test\r\n" + "\n" + " \"quoted, with a comma\" \n" + "last*\n"; + const auto options = Parse({"--input-file=" + file.string()}); + CHECK(options.testSpecs == std::vector<std::string>{"\"first test\"", + ",", + "\"quoted, with a comma\"", + ",", + "\"last*\""}); + std::filesystem::remove(file, error); + CHECK_THAT(ParseError({"-f", file.string()}), ContainsSubstring("cannot read")); +} + +TEST_CASE("Unknown options are refused with a suggestion", + "[progmasoft][runner][options]") +{ + CHECK_THAT(ParseError({"--sucess"}), ContainsSubstring("did you mean --success?")); + CHECK_THAT(ParseError({"--histroy=x"}), ContainsSubstring("did you mean --history?")); + CHECK_THAT(ParseError({"-z"}), ContainsSubstring("'-z' is not an option")); + // Catch2's reporter options are gone with its reporters. + CHECK_THAT(ParseError({"--reporter=junit"}), ContainsSubstring("is not an option")); + CHECK_THAT(ParseError({"--isolate=yes"}), ContainsSubstring("takes no value")); + CHECK_THAT(ParseError({"--junit"}), ContainsSubstring("--junit=FILE")); +} + +TEST_CASE("The help lists every option", "[progmasoft][runner][options]") +{ + const std::string help = Runner::HelpText("tests"); + CHECK_THAT(help, ContainsSubstring("usage: tests [options] [test specs...]")); + for (const char *name : {"--section", "--generator-index", + "--path-filter", "--input-file", + "--select", "--changes-since", + "--incremental", "--shard", + "--shard-durations", "--filenames-as-tags", + "--jobs", "--in-process", + "--isolate", "--timeout", + "--no-adaptive-timeout", "--retries", + "--repeat", "--fail-fast", + "--order", "--seed", + "--no-bisect", "--success", + "--break", "--no-throw", + "--warn", "--allow-no-tests", + "--skip-benchmarks", "--benchmark-samples", + "--benchmark-resamples", + "--benchmark-confidence-interval", + "--benchmark-no-analysis", + "--benchmark-warmup-time", + "--history", "--no-history", + "--quarantine", "--replay", + "--analytics", "--verbose", + "--quiet", "--color", + "--junit", "--json", + "--trace", "--premature-exit-guard-file", + "--slowest", "--list", + "--list-tags", "--help", + "--version"}) + { + CAPTURE(name); + CHECK_THAT(help, ContainsSubstring(name)); + } + // The worker option is internal. + CHECK_THAT(help, !ContainsSubstring("--catch3-worker")); +} + +TEST_CASE("A worker gets every option that changes how tests behave", + "[progmasoft][runner][options]") +{ + const auto options = Parse({"--section=outer", + "--section=inner", + "-p", + "g:1", + "-#", + "-s", + "-e", + "--warn=no-assertions", + "--warn=infinite-generators", + "--skip-benchmarks", + "--benchmark-no-analysis", + "--benchmark-samples=3", + "--benchmark-resamples=4", + "--benchmark-warmup-time=5", + "--benchmark-confidence-interval=0.9", + "--seed=6", + // Runner-side only: + "--jobs=9", + "--junit=x.xml", + "[tag]"}); + const auto worker = Parse(Runner::WorkerArguments(options)); + CHECK(worker.mode == Runner::Mode::kWorker); + CHECK(worker.sections == options.sections); + CHECK(worker.paths == options.paths); + CHECK(worker.filenamesAsTags); + CHECK(worker.includeSuccessful); + CHECK(worker.noThrow); + CHECK(worker.warnNoAssertions); + CHECK(worker.warnInfiniteGenerators); + CHECK(worker.skipBenchmarks); + CHECK(worker.benchmarkNoAnalysis); + CHECK(worker.benchmarkSamples == 3); + CHECK(worker.benchmarkResamples == 4); + CHECK(worker.benchmarkWarmupTime == milliseconds{5}); + CHECK(worker.benchmarkConfidenceInterval == 0.9); + CHECK(worker.seed == 6); + CHECK(worker.jobs == 0); + CHECK(worker.junitPath.empty()); + CHECK(worker.testSpecs.empty()); +} diff --git a/tests/Progmasoft/RunnerPlanner.tests.cpp b/tests/Progmasoft/RunnerPlanner.tests.cpp new file mode 100644 index 00000000..4c930632 --- /dev/null +++ b/tests/Progmasoft/RunnerPlanner.tests.cpp @@ -0,0 +1,599 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +// The planner decides which tests run, where, in which order and under +// which limits, from the options, the history and the changed files. These +// tests give it a catalog of their own, so no test of this binary is +// involved. + +#include <Progmasoft/Runner/Catalog.hpp> +#include <Progmasoft/Runner/Files.hpp> +#include <Progmasoft/Runner/History.hpp> +#include <Progmasoft/Runner/Planner.hpp> + +#include <catch2/catch_test_macros.hpp> +#include <catch2/matchers/catch_matchers_string.hpp> + +#include <algorithm> +#include <chrono> +#include <filesystem> +#include <fstream> +#include <map> +#include <set> +#include <string> +#include <vector> + +namespace +{ + namespace Runner = Progmasoft::Catch3::Runner; + using Catch::Matchers::ContainsSubstring; + using Progmasoft::Catch3::Attempt; + using Progmasoft::Catch3::Outcome; + using Progmasoft::Catch3::RunResult; + using Progmasoft::Catch3::TestResult; + using std::chrono::milliseconds; + + /// A directory of its own, removed at the end of the test. + class Scratch final + { + public: + Scratch() + { |