catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
Fix the CI failures of the runner and test its planner, options and analysis
- Commit
e104b694513964d5a84e77f751203cda970d741b- Author
- Leitwolf11 ·
- Parents
15eb534ee493
Changes
extras/catch_amalgamated.cpp | 39 +- extras/catch_amalgamated.hpp | 2 +- premake5.lua | 3 + src/Progmasoft/Runner/Catalog.cpp | 21 + src/Progmasoft/Runner/Catalog.hpp | 13 +- src/Progmasoft/Runner/Scheduler.cpp | 5 +- tests/CMakeLists.txt | 3 + tests/ExtraTests/X21-PartialTestCaseEvents.cpp | 2 + .../X27-CapturedStdoutInTestCaseEvents.cpp | 2 + tests/Progmasoft/BUILD.bazel | 3 + tests/Progmasoft/RunnerAnalysis.tests.cpp | 321 +++++++++++ tests/Progmasoft/RunnerOptions.tests.cpp | 369 +++++++++++++ tests/Progmasoft/RunnerPlanner.tests.cpp | 599 +++++++++++++++++++++ tests/meson.build | 3 + xmake.lua | 3 + 15 files changed, 1377 insertions(+), 11 deletions(-) diff --git a/extras/catch_amalgamated.cpp b/extras/catch_amalgamated.cpp index 506726dc..6521ef84 100644 --- a/extras/catch_amalgamated.cpp +++ b/extras/catch_amalgamated.cpp @@ -7,7 +7,7 @@ // SPDX-License-Identifier: BSL-1.0 // Catch v3.16.1 -// Generated: 2026-10-23 23:12:01.455813 +// Generated: 2026-10-24 10:02:49.316745 // ---------------------------------------------------------- // This file is an amalgamation of multiple different files. // You probably shouldn't edit it directly. @@ -846,18 +846,17 @@ namespace Progmasoft::Catch3::Runner // Copyright (c) 2026 Progmasoft. // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 - - #include <chrono> #include <cstddef> #include <optional> #include <string> #include <vector> + namespace Catch { class IConfig; -} +} // namespace Catch namespace Progmasoft::Catch3::Runner { @@ -907,6 +906,12 @@ namespace Progmasoft::Catch3::Runner [[nodiscard]] static Expected<Catalog> Load(); + /// A catalog of the given tests, without a registry behind it: the + /// tests cannot run, but they can be planned. Each entry's index + /// becomes its position, and an empty key becomes its name. + [[nodiscard]] static Catalog + FromEntries(std::vector<TestEntry> entries); + [[nodiscard]] const std::vector<TestEntry> & Tests() const noexcept { @@ -9907,6 +9912,27 @@ namespace Progmasoft::Catch3::Runner return catalog; } + Catalog + Catalog::FromEntries(std::vector<TestEntry> entries) + { + Catalog catalog; + Sha256 identity; + for (std::size_t index = 0; index < entries.size(); ++index) + { + TestEntry &entry = entries[index]; + entry.index = index; + if (entry.key.empty()) + { + entry.key = entry.name; + } + identity.Update(entry.key); + identity.Update("\n"); + } + catalog.tests_ = std::move(entries); + catalog.identity_ = identity.HexDigest(); + return catalog; + } + const TestEntry * Catalog::Find(const std::string &keyOrName) const { @@ -18122,8 +18148,11 @@ namespace Progmasoft::Catch3::Runner attempt.sections.push_back(section); } } - else + else if (!running.fatal) { + // After a fatal signal Catch2 closes the open + // sections itself; they stay open here, because the + // crash happened inside them. const auto depth = message["depth"].AsUnsigned(); for (auto section = attempt.sections.rbegin(); section != attempt.sections.rend(); diff --git a/extras/catch_amalgamated.hpp b/extras/catch_amalgamated.hpp index 63c2045b..8d730b62 100644 --- a/extras/catch_amalgamated.hpp +++ b/extras/catch_amalgamated.hpp @@ -7,7 +7,7 @@ // SPDX-License-Identifier: BSL-1.0 // Catch v3.16.1 -// Generated: 2026-10-23 23:12:01.222722 +// Generated: 2026-10-24 10:02:48.985091 // ---------------------------------------------------------- // This file is an amalgamation of multiple different files. // You probably shouldn't edit it directly. diff --git a/premake5.lua b/premake5.lua index d9227084..9d191077 100644 --- a/premake5.lua +++ b/premake5.lua @@ -91,6 +91,9 @@ project "ProgmasoftPropertyTests" "tests/Progmasoft/Results.tests.cpp", "tests/Progmasoft/Runner.tests.cpp", "tests/Progmasoft/RunnerUnits.tests.cpp", + "tests/Progmasoft/RunnerPlanner.tests.cpp", + "tests/Progmasoft/RunnerOptions.tests.cpp", + "tests/Progmasoft/RunnerAnalysis.tests.cpp", "tests/Progmasoft/Snapshot.tests.cpp", "tests/Progmasoft/Stateful.tests.cpp", "tests/Progmasoft/Strategy.tests.cpp", diff --git a/src/Progmasoft/Runner/Catalog.cpp b/src/Progmasoft/Runner/Catalog.cpp index c34e81e0..c9d838f4 100644 --- a/src/Progmasoft/Runner/Catalog.cpp +++ b/src/Progmasoft/Runner/Catalog.cpp @@ -209,6 +209,27 @@ namespace Progmasoft::Catch3::Runner return catalog; } + Catalog + Catalog::FromEntries(std::vector<TestEntry> entries) + { + Catalog catalog; + Sha256 identity; + for (std::size_t index = 0; index < entries.size(); ++index) + { + TestEntry &entry = entries[index]; + entry.index = index; + if (entry.key.empty()) + { + entry.key = entry.name; + } + identity.Update(entry.key); + identity.Update("\n"); + } + catalog.tests_ = std::move(entries); + catalog.identity_ = identity.HexDigest(); + return catalog; + } + const TestEntry * Catalog::Find(const std::string &keyOrName) const { diff --git a/src/Progmasoft/Runner/Catalog.hpp b/src/Progmasoft/Runner/Catalog.hpp index 49c7a9f5..5cc9d0c6 100644 --- a/src/Progmasoft/Runner/Catalog.hpp +++ b/src/Progmasoft/Runner/Catalog.hpp @@ -2,20 +2,19 @@ // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 #pragma once -#include "Expected.hpp" - #include <catch2/catch_test_case_info.hpp> - #include <chrono> #include <cstddef> #include <optional> #include <string> #include <vector> +#include "Expected.hpp" + namespace Catch { class IConfig; -} +} // namespace Catch namespace Progmasoft::Catch3::Runner { @@ -65,6 +64,12 @@ namespace Progmasoft::Catch3::Runner [[nodiscard]] static Expected<Catalog> Load(); + /// A catalog of the given tests, without a registry behind it: the + /// tests cannot run, but they can be planned. Each entry's index + /// becomes its position, and an empty key becomes its name. + [[nodiscard]] static Catalog + FromEntries(std::vector<TestEntry> entries); + [[nodiscard]] const std::vector<TestEntry> & Tests() const noexcept { diff --git a/src/Progmasoft/Runner/Scheduler.cpp b/src/Progmasoft/Runner/Scheduler.cpp index e6c8eec5..6a65f289 100644 --- a/src/Progmasoft/Runner/Scheduler.cpp +++ b/src/Progmasoft/Runner/Scheduler.cpp @@ -754,8 +754,11 @@ namespace Progmasoft::Catch3::Runner attempt.sections.push_back(section); } } - else + else if (!running.fatal) { + // After a fatal signal Catch2 closes the open + // sections itself; they stay open here, because the + // crash happened inside them. const auto depth = message["depth"].AsUnsigned(); for (auto section = attempt.sections.rbegin(); section != attempt.sections.rend(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 14f2ed56..e57ad94a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -157,6 +157,9 @@ add_executable(ProgmasoftSelfTest Progmasoft/Results.tests.cpp Progmasoft/Runner.tests.cpp Progmasoft/RunnerUnits.tests.cpp + Progmasoft/RunnerPlanner.tests.cpp + Progmasoft/RunnerOptions.tests.cpp + Progmasoft/RunnerAnalysis.tests.cpp Progmasoft/Snapshot.tests.cpp Progmasoft/Stateful.tests.cpp Progmasoft/Strategy.tests.cpp diff --git a/tests/ExtraTests/X21-PartialTestCaseEvents.cpp b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp index c55f6c2b..99002822 100644 --- a/tests/ExtraTests/X21-PartialTestCaseEvents.cpp +++ b/tests/ExtraTests/X21-PartialTestCaseEvents.cpp @@ -28,6 +28,7 @@ using Catch::TestCaseInfo; using Catch::TestCaseStats; +namespace { class PartialListener : public Catch::EventListenerBase { public: using EventListenerBase::EventListenerBase; @@ -58,6 +59,7 @@ public: private: std::string m_events; }; +} // namespace CATCH_REGISTER_LISTENER(PartialListener) diff --git a/tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp b/tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp index 7ceebcee..20b39d78 100644 --- a/tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp +++ b/tests/ExtraTests/X27-CapturedStdoutInTestCaseEvents.cpp @@ -27,6 +27,7 @@ #include <iostream> #include <string> +namespace { class TestListener : public Catch::EventListenerBase { std::string stdOutString( uint64_t iter ){ return "stdout " + std::to_string( iter ) + '\n'; @@ -66,6 +67,7 @@ public: std::cerr << "X27 - listener saw " << m_errors << " error(s)\n"; } }; +} // namespace CATCH_REGISTER_LISTENER( TestListener ) diff --git a/tests/Progmasoft/BUILD.bazel b/tests/Progmasoft/BUILD.bazel index 296fd5f1..06df86d6 100644 --- a/tests/Progmasoft/BUILD.bazel +++ b/tests/Progmasoft/BUILD.bazel @@ -13,6 +13,9 @@ cc_test( "Results.tests.cpp", "Runner.tests.cpp", "RunnerUnits.tests.cpp", + "RunnerPlanner.tests.cpp", + "RunnerOptions.tests.cpp", + "RunnerAnalysis.tests.cpp", "Snapshot.tests.cpp", "Stateful.tests.cpp", "Strategy.tests.cpp", diff --git a/tests/Progmasoft/RunnerAnalysis.tests.cpp b/tests/Progmasoft/RunnerAnalysis.tests.cpp new file mode 100644 index 00000000..453fa671 --- /dev/null +++ b/tests/Progmasoft/RunnerAnalysis.tests.cpp @@ -0,0 +1,321 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +// What the runner writes down about a failure: how it relates to earlier +// runs and to other failures, where it stopped, what changed and how to +// run it again. + +#include <Progmasoft/Runner/Analysis.hpp> +#include <Progmasoft/Runner/Files.hpp> +#include <Progmasoft/Runner/History.hpp> + +#include <catch2/catch_test_macros.hpp> +#include <catch2/matchers/catch_matchers_string.hpp> +#include <catch2/matchers/catch_matchers_vector.hpp> + +#include <filesystem> +#include <string> +#include <vector> + +namespace +{ + namespace Runner = Progmasoft::Catch3::Runner; + using Catch::Matchers::ContainsSubstring; + using Progmasoft::Catch3::AssertionRecord; + using Progmasoft::Catch3::Attempt; + using Progmasoft::Catch3::FailureCategory; + using Progmasoft::Catch3::FailureHistory; + using Progmasoft::Catch3::Outcome; + using Progmasoft::Catch3::RunResult; + using Progmasoft::Catch3::SectionRecord; + using Progmasoft::Catch3::TestResult; + + /// A directory of its own, removed at the end of the test. + class Scratch final + { + public: + Scratch() + { + std::error_code error; + path_ = std::filesystem::temp_directory_path(error) / + ("catch3-analysis-" + Runner::UniqueName()); + std::filesystem::create_directories(path_, error); + } + Scratch(const Scratch &) = delete; + Scratch & + operator=(const Scratch &) = delete; + ~Scratch() + { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + [[nodiscard]] const std::filesystem::path & + Path() const noexcept + { + return path_; + } + + private: + std::filesystem::path path_; + }; + + /// A test that failed one assertion. + TestResult + Failed(const std::string &name, + const std::string &message = "wrong", + std::size_t line = 10) + { + TestResult test; + test.id = name; + test.name = name; + test.file = "src/test.cpp"; + test.line = 1; + test.outcome = Outcome::kFailed; + test.category = FailureCategory::kAssertion; + Attempt attempt; + attempt.outcome = Outcome::kFailed; + attempt.category = FailureCategory::kAssertion; + attempt.seed = 7; + AssertionRecord assertion; + assertion.file = "src/test.cpp"; + assertion.line = line; + assertion.expression = "a == b"; + assertion.message = message; + attempt.assertions.push_back(assertion); + test.attempts.push_back(attempt); + return test; + } + + TestResult + Passed(const std::string &name) + { + TestResult test; + test.id = name; + test.name = name; + test.outcome = Outcome::kPassed; + Attempt attempt; + attempt.outcome = Outcome::kPassed; + test.attempts.push_back(attempt); + return test; + } + + /// Commits runs of `tests` to a history, the last commit last. + Runner::History + HistoryOf(const std::filesystem::path &file, + const std::vector<std::vector<TestResult>> &runs) + { + Runner::History history; + int number = 0; + for (const auto &tests : runs) + { + RunResult run; + run.tests = tests; + for (auto &test : run.tests) + { + // As the runner does before it commits a run. + for (const auto &attempt : test.attempts) + { + if (Progmasoft::Catch3::IsFailure(attempt.outcome)) + { + test.signature = Runner::FailureSignature(attempt); + break; + } + } + } + Runner::RunRecord record; + record.commit = "commit" + std::to_string(++number) + "0123456789"; + auto committed = + Runner::History::Commit(file, run, record, {}, {}); + REQUIRE(committed); + history = *committed; + } + return history; + } + + std::string + Findings(const TestResult &test) + { + std::string text; + for (const auto &finding : test.analysis) + { + text += finding + "\n"; + } + return text; + } +} // namespace + +TEST_CASE("A failure is placed in the test's history", + "[progmasoft][runner][analysis]") +{ + const Scratch scratch; + const auto file = Runner::HistoryFile(scratch.Path(), "classes"); + auto flaky = Passed("flaky"); + flaky.flaky = true; + flaky.attempts.front().outcome = Outcome::kFailed; + const auto history = HistoryOf( + file, + {{Passed("fresh"), Failed("stuck", "first"), Passed("wobbly")}, + {Passed("fresh"), Failed("stuck", "first"), flaky}}); + + RunResult run; + run.tests = {Failed("unknown"), + Failed("fresh", "one"), + Failed("stuck", "second", 20), + Failed("flaky", "three", 30)}; + Runner::AnalysisInputs inputs; + inputs.history = &history; + inputs.program = "tests"; + Runner::Analyze(run, inputs); + + CHECK(run.tests[0].history == FailureHistory::kUnknown); + + CHECK(run.tests[1].history == FailureHistory::kNew); + CHECK_THAT(Findings(run.tests[1]), + ContainsSubstring("new failure after 2 passing run(s); it " + "last passed at commit commit201234")); + + CHECK(run.tests[2].history == FailureHistory::kPersistent); + CHECK_THAT(Findings(run.tests[2]), + ContainsSubstring("persistent: failed in the last 2 run(s), " + "but differently last time")); + + CHECK(run.tests[3].history == FailureHistory::kIntermittent); + CHECK_THAT(Findings(run.tests[3]), + ContainsSubstring("intermittent: 1 flaky result(s)")); +} + +TEST_CASE("Failures with one signature are named together", + "[progmasoft][runner][analysis]") +{ + RunResult run; + for (const char *name : {"a", "b", "c", "d", "e"}) + { + // Numbers in the message do not split the group. + run.tests.push_back(Failed( + name, + "connection refused after " + + std::to_string((run.tests.size() + 1) * 100) + " ms")); + } + run.tests.push_back(Failed("elsewhere", "other", 99)); + Runner::AnalysisInputs inputs; + inputs.program = "tests"; + Runner::Analyze(run, inputs); + + CHECK(run.tests[0].signature == run.tests[4].signature); + CHECK(run.tests[0].signature != run.tests[5].signature); + CHECK_THAT(Findings(run.tests[0]), + ContainsSubstring("fails the same way as 'b', 'c', 'd' and 1 " + "more: probably one cause")); + CHECK_THAT(Findings(run.tests[5]), + !ContainsSubstring("fails the same way")); +} + +TEST_CASE("A crash and a hang say where they stopped", + "[progmasoft][runner][analysis]") +{ + SectionRecord outer; + outer.name = "outer"; + outer.depth = 0; + outer.durationMicroseconds = -1; + SectionRecord done; + done.name = "done"; + done.depth = 1; + done.durationMicroseconds = 5; + SectionRecord inner; + inner.name = "inner"; + inner.depth = 1; + inner.durationMicroseconds = -1; + + TestResult crash = Failed("crash"); + crash.outcome = Outcome::kCrashed; + Attempt &crashed = crash.attempts.front(); + crashed.outcome = Outcome::kCrashed; + crashed.category = FailureCategory::kCrash; + crashed.assertions.clear(); + crashed.exitDescription = "signal 11 (SIGSEGV)"; + crashed.lastLocation = "src/test.cpp:44"; + crashed.sections = {outer, done, inner}; + + TestResult hang = Failed("hang"); + hang.outcome = Outcome::kTimedOut; + hang.predictedDurationMicroseconds = 200000; + Attempt &hung = hang.attempts.front(); + hung.outcome = Outcome::kTimedOut; + hung.category = FailureCategory::kTimeout; + hung.assertions.clear(); + hung.durationMicroseconds = 5000000; + hung.lastLocation = "src/test.cpp:50"; + hung.sections = {outer}; + + RunResult run; + run.tests = {crash, hang}; + Runner::AnalysisInputs inputs; + inputs.program = "tests"; + Runner::Analyze(run, inputs); + + CHECK_THAT(Findings(run.tests[0]), + ContainsSubstring("crashed (signal 11 (SIGSEGV)) after the " + "assertion at src/test.cpp:44, in section " + "outer / inner")); + CHECK_THAT(Findings(run.tests[1]), + ContainsSubstring("stopped after 5.0 s")); + CHECK_THAT(Findings(run.tests[1]), + ContainsSubstring("it was in section outer")); + CHECK_THAT(Findings(run.tests[1]), + ContainsSubstring("last assertion started at src/test.cpp:50")); + // Two crashes in different places are different failures. + CHECK(run.tests[0].signature != run.tests[1].signature); +} + +TEST_CASE("A failure names the changed files it touches", + "[progmasoft][runner][analysis]") +{ + const Scratch scratch; + Runner::Changes changes; + changes.root = scratch.Path(); + changes.files = {scratch.Path() / "src" / "test.cpp", + scratch.Path() / "src" / "helper.hpp"}; + TestResult test = Failed("touched"); + AssertionRecord inHelper; + inHelper.file = "src/helper.hpp"; + inHelper.line = 3; + inHelper.passed = false; + test.attempts.front().assertions.push_back(inHelper); + test.attempts.front().temporaryDirectory = "/tmp/kept"; + + RunResult run; + run.tests = {test}; + Runner::AnalysisInputs inputs; + inputs.changes = &changes; + inputs.changesSince = "main"; + inputs.program = "tests"; + Runner::Analyze(run, inputs); + + const auto findings = Findings(run.tests[0]); + CHECK_THAT(findings, ContainsSubstring("changed since main: src/test.cpp")); + CHECK_THAT(findings, ContainsSubstring("changed since main: src/helper.hpp")); + CHECK_THAT(findings, + ContainsSubstring("its temporary files are kept in /tmp/kept")); +} + +TEST_CASE("Every failure gets a command that runs it again", + "[progmasoft][runner][analysis]") +{ + RunResult run; + run.tests = {Failed("plain, with [specials]"), Failed("after another"), + Passed("fine")}; + run.tests[1].attempts.front().precededBy = {"polluter"}; + Runner::AnalysisInputs inputs; + inputs.program = "build dir/tests"; + Runner::Analyze(run, inputs); + + CHECK_THAT(run.tests[0].replayCommand, + ContainsSubstring("plain\\\\, with \\\\[specials\\\\]")); + CHECK_THAT(run.tests[0].replayCommand, ContainsSubstring("--seed=7")); + CHECK_THAT(run.tests[0].replayCommand, ContainsSubstring("build dir/tests")); + // A failure that needed other tests first is replayed with them. + CHECK_THAT(run.tests[1].replayCommand, ContainsSubstring("--replay")); + CHECK(run.tests[2].replayCommand.empty()); + CHECK(run.tests[2].analysis.empty()); +} diff --git a/tests/Progmasoft/RunnerOptions.tests.cpp b/tests/Progmasoft/RunnerOptions.tests.cpp new file mode 100644 index 00000000..2ad8c0b7 --- /dev/null +++ b/tests/Progmasoft/RunnerOptions.tests.cpp @@ -0,0 +1,369 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +// Every option of the runner's command line: how it is spelled, what it +// sets and how a wrong value is explained. + +#include <Progmasoft/Runner/Files.hpp> +#include <Progmasoft/Runner/Options.hpp> + +#include <catch2/catch_test_macros.hpp> +#include <catch2/generators/catch_generators.hpp> +#include <catch2/matchers/catch_matchers_string.hpp> + +#include <chrono> +#include <filesystem> +#include <fstream> +#include <string> +#include <vector> + +namespace +{ + namespace Runner = Progmasoft::Catch3::Runner; + using Catch::Matchers::ContainsSubstring; + using std::chrono::milliseconds; + + Runner::Options + Parse(const std::vector<std::string> &arguments) + { + auto options = Runner::ParseOptions(arguments); + INFO((options ? std::string{} : options.GetError().Message())); + REQUIRE(options); + return *options; + } + + std::string + ParseError(const std::vector<std::string> &arguments) + { + auto options = Runner::ParseOptions(arguments); + REQUIRE_FALSE(options); + return options.GetError().Message(); + } +} // namespace + +TEST_CASE("Options have their documented defaults", + "[progmasoft][runner][options]") +{ + const auto options = Parse({}); + CHECK(options.mode == Runner::Mode::kRun); + CHECK(options.selection == Runner::Selection::kAll); + CHECK(options.order == Runner::Order::kPriority); + CHECK(options.retryMode == Runner::RetryMode::kAuto); + CHECK(options.retries == 2); + CHECK(options.repeat == 1); + CHECK(options.failFast == 0); + CHECK(options.jobs == 0); + CHECK(options.shard.index == 0); + CHECK(options.shard.count == 1); + CHECK_FALSE(options.timeoutGiven); + CHECK(options.adaptiveTimeout); + CHECK(options.bisectOrder); + CHECK(options.historyEnabled); + CHECK(options.quarantine == Runner::QuarantineMode::kAuto); + CHECK(options.verbosity == Runner::Verbosity::kNormal); + CHECK(options.color == Runner::ColorMode::kAuto); + CHECK(options.benchmarkSamples == 100); + CHECK(options.benchmarkResamples == 100000); + CHECK(options.benchmarkConfidenceInterval == 0.95); + CHECK(options.benchmarkWarmupTime == milliseconds{100}); + CHECK(options.testSpecs.empty()); +} + +TEST_CASE("A value can follow an option in every accepted spelling", + "[progmasoft][runner][options]") +{ + const std::vector<std::string> arguments = GENERATE( + std::vector<std::string>{"--jobs=4"}, + std::vector<std::string>{"--jobs", "4"}, + std::vector<std::string>{"-j4"}, + std::vector<std::string>{"-j=4"}, + std::vector<std::string>{"-j", "4"}); + CAPTURE(arguments); + CHECK(Parse(arguments).jobs == 4); +} + +TEST_CASE("Switches set what they say", "[progmasoft][runner][options]") +{ + CHECK(Parse({"--incremental"}).incremental); + CHECK(Parse({"--filenames-as-tags"}).filenamesAsTags); + CHECK(Parse({"-#"}).filenamesAsTags); + CHECK(Parse({"--in-process"}).inProcess); + CHECK(Parse({"--isolate"}).isolate); + CHECK_FALSE(Parse({"--no-adaptive-timeout"}).adaptiveTimeout); + CHECK_FALSE(Parse({"--no-bisect"}).bisectOrder); + CHECK(Parse({"--success"}).includeSuccessful); + CHECK(Parse({"-s"}).includeSuccessful); + CHECK(Parse({"--no-throw"}).noThrow); + CHECK(Parse({"-e"}).noThrow); + CHECK(Parse({"--allow-no-tests"}).allowNoTests); + CHECK(Parse({"--skip-benchmarks"}).skipBenchmarks); + CHECK(Parse({"--benchmark-no-analysis"}).benchmarkNoAnalysis); + CHECK_FALSE(Parse({"--no-history"}).historyEnabled); + CHECK(Parse({"--replay"}).replayFailures); + CHECK(Parse({"--verbose"}).verbosity == Runner::Verbosity::kVerbose); + CHECK(Parse({"-q"}).verbosity == Runner::Verbosity::kQuiet); + + // A debugger needs the test in this process. + const auto debugging = Parse({"-b"}); + CHECK(debugging.breakIntoDebugger); + CHECK(debugging.inProcess); +} + +TEST_CASE("Modes are chosen by their options", "[progmasoft][runner][options]") +{ + CHECK(Parse({"--help"}).mode == Runner::Mode::kHelp); + CHECK(Parse({"-h"}).mode == Runner::Mode::kHelp); + CHECK(Parse({"--version"}).mode == Runner::Mode::kVersion); + CHECK(Parse({"--list-tags"}).mode == Runner::Mode::kListTags); + CHECK(Parse({"-t"}).mode == Runner::Mode::kListTags); + CHECK(Parse({"--catch3-worker"}).mode == Runner::Mode::kWorker); + + const auto text = Parse({"-l"}); + CHECK(text.mode == Runner::Mode::kListTests); + CHECK(text.listFormat == Runner::ListFormat::kText); + CHECK(Parse({"--list=names"}).listFormat == Runner::ListFormat::kNames); + CHECK(Parse({"--list=json"}).listFormat == Runner::ListFormat::kJson); + + const auto analytics = Parse({"--analytics"}); + CHECK(analytics.mode == Runner::Mode::kAnalytics); + CHECK_FALSE(analytics.analyticsJson); + CHECK(Parse({"--analytics=json"}).analyticsJson); +} + +TEST_CASE("Choices accept their names and nothing else", + "[progmasoft][runner][options]") +{ + CHECK(Parse({"--select=all"}).selection == Runner::Selection::kAll); + CHECK(Parse({"--select=failed"}).selection == Runner::Selection::kFailed); + CHECK(Parse({"--select=changed"}).selection == Runner::Selection::kChanged); + CHECK(Parse({"--select=new"}).selection == Runner::Selection::kNew); + CHECK(Parse({"--select=smart"}).selection == Runner::Selection::kSmart); + CHECK(Parse({"--order=name"}).order == Runner::Order::kName); + CHECK(Parse({"--order=random"}).order == Runner::Order::kRandom); + CHECK(Parse({"--order=priority"}).order == Runner::Order::kPriority); + CHECK(Parse({"--quarantine=report"}).quarantine == + Runner::QuarantineMode::kReport); + CHECK(Parse({"--quarantine=off"}).quarantine == + Runner::QuarantineMode::kOff); + CHECK(Parse({"--color=always"}).color == Runner::ColorMode::kAlways); + CHECK(Parse({"--color=never"}).color == Runner::ColorMode::kNever); + + CHECK_THAT(ParseError({"--select=some"}), + ContainsSubstring("all, failed, changed, new, smart")); + CHECK_THAT(ParseError({"--quarantine=maybe"}), + ContainsSubstring("auto, report, off")); + CHECK_THAT(ParseError({"--color=blue"}), + ContainsSubstring("auto, always, never")); + CHECK_THAT(ParseError({"--list=xml"}), + ContainsSubstring("text, names, json")); + CHECK_THAT(ParseError({"--analytics=xml"}), ContainsSubstring("json")); +} + +TEST_CASE("Numbers are checked", "[progmasoft][runner][options]") +{ + CHECK(Parse({"--repeat=3"}).repeat == 3); + CHECK(Parse({"-a"}).failFast == 1); + CHECK(Parse({"--slowest"}).slowest == 10); + CHECK(Parse({"--slowest=3"}).slowest == 3); + CHECK(Parse({"--benchmark-samples=7"}).benchmarkSamples == 7); + CHECK(Parse({"--benchmark-resamples=9"}).benchmarkResamples == 9); + CHECK(Parse({"--benchmark-confidence-interval=0.5"}) + .benchmarkConfidenceInterval == 0.5); + CHECK(Parse({"--benchmark-warmup-time=250"}).benchmarkWarmupTime == + milliseconds{250}); + + const auto unlimited = Parse({"--timeout=0"}); + CHECK(unlimited.timeoutGiven); + CHECK(unlimited.timeout == milliseconds{0}); + + const auto retried = Parse({"--retries=auto"}); + CHECK(retried.retryMode == Runner::RetryMode::kAuto); + const auto off = Parse({"--retries=off"}); + CHECK(off.retryMode == Runner::RetryMode::kOff); + CHECK(off.retries == 0); + + const auto random = Parse({"--seed=5", "--seed=random"}); + CHECK_FALSE(random.seedGiven); + + CHECK_THAT(ParseError({"--repeat=0"}), ContainsSubstring("at least 1")); + CHECK_THAT(ParseError({"--fail-fast=0"}), ContainsSubstring("at least 1")); + CHECK_THAT(ParseError({"--slowest=0"}), ContainsSubstring("at least 1")); + CHECK_THAT(ParseError({"--retries=-1"}), ContainsSubstring("whole number")); + CHECK_THAT(ParseError({"--timeout=soon"}), ContainsSubstring("seconds")); + CHECK_THAT(ParseError({"--timeout=-2"}), ContainsSubstring("seconds")); + CHECK_THAT(ParseError({"--benchmark-confidence-interval=1"}), + ContainsSubstring("--benchmark-confidence-interval")); + CHECK_THAT(ParseError({"--benchmark-samples=0"}), + ContainsSubstring("at least 1")); + CHECK_THAT(ParseError({"--seed=-1"}), ContainsSubstring("--seed")); + CHECK_THAT(ParseError({"--generator-index=x"}), + ContainsSubstring("index or *")); + CHECK_THAT(ParseError({"--path-filter=g:x"}), + ContainsSubstring("generator index or *")); +} + +TEST_CASE("Shards are an index below a count", "[progmasoft][runner][options]") +{ + const auto last = Parse({"--shard=3/4"}); + CHECK(last.shard.index == 3); + CHECK(last.shard.count == 4); + const std::string wrong = GENERATE("4/4", "1", "/2", "1/", "0/0", "a/b"); + CAPTURE(wrong); + CHECK_THAT(ParseError({"--shard=" + wrong}), ContainsSubstring("INDEX/COUNT")); +} + +TEST_CASE("Warnings are named", "[progmasoft][runner][options]") +{ + CHECK(Parse({"--warn=no-assertions"}).warnNoAssertions); + CHECK(Parse({"-w", "NoAssertions"}).warnNoAssertions); + CHECK(Parse({"--warn=unmatched-specs"}).warnUnmatchedSpecs); + CHECK(Parse({"--warn=infinite-generators"}).warnInfiniteGenerators); + CHECK_THAT(ParseError({"--warn=everything"}), + ContainsSubstring("no-assertions")); +} + +TEST_CASE("Files and directories are taken as written", + "[progmasoft][runner][options]") +{ + const auto options = Parse({"--junit=out/junit.xml", + "--json", + "out/run.json", + "--trace=out/trace.json", + "--history=cache", + "--shard-durations=last.json", + "--changes-since=main", + "--premature-exit-guard-file=guard"}); + CHECK(options.junitPath == "out/junit.xml"); + CHECK(options.jsonPath == "out/run.json"); + CHECK(options.tracePath == "out/trace.json"); + CHECK(options.historyDirectory == "cache"); + CHECK(options.historyEnabled); + CHECK(options.shardDurations == "last.json"); + CHECK(options.changesSince == "main"); + CHECK(options.exitGuardPath == "guard"); +} + +TEST_CASE("Test specs are kept apart from options", + "[progmasoft][runner][options]") +{ + const auto options = + Parse({"name", "[tag]", "~[slow]", "-", "--", "-j", "--x"}); + CHECK(options.jobs == 0); + CHECK(options.testSpecs == + std::vector<std::string>{"name", "[tag]", "~[slow]", "-", "-j", "--x"}); +} + +TEST_CASE("An input file names tests exactly", "[progmasoft][runner][options]") +{ + std::error_code error; + const auto file = std::filesystem::temp_directory_path(error) / + ("catch3-input-" + Runner::UniqueName()); + std::ofstream(file) << "# a comment\n" + "first test\r\n" + "\n" + " \"quoted, with a comma\" \n" + "last*\n"; + const auto options = Parse({"--input-file=" + file.string()}); + CHECK(options.testSpecs == std::vector<std::string>{"\"first test\"", + ",", + "\"quoted, with a comma\"", + ",", + "\"last*\""}); + std::filesystem::remove(file, error); + CHECK_THAT(ParseError({"-f", file.string()}), ContainsSubstring("cannot read")); +} + +TEST_CASE("Unknown options are refused with a suggestion", + "[progmasoft][runner][options]") +{ + CHECK_THAT(ParseError({"--sucess"}), ContainsSubstring("did you mean --success?")); + CHECK_THAT(ParseError({"--histroy=x"}), ContainsSubstring("did you mean --history?")); + CHECK_THAT(ParseError({"-z"}), ContainsSubstring("'-z' is not an option")); + // Catch2's reporter options are gone with its reporters. + CHECK_THAT(ParseError({"--reporter=junit"}), ContainsSubstring("is not an option")); + CHECK_THAT(ParseError({"--isolate=yes"}), ContainsSubstring("takes no value")); + CHECK_THAT(ParseError({"--junit"}), ContainsSubstring("--junit=FILE")); +} + +TEST_CASE("The help lists every option", "[progmasoft][runner][options]") +{ + const std::string help = Runner::HelpText("tests"); + CHECK_THAT(help, ContainsSubstring("usage: tests [options] [test specs...]")); + for (const char *name : {"--section", "--generator-index", + "--path-filter", "--input-file", + "--select", "--changes-since", + "--incremental", "--shard", + "--shard-durations", "--filenames-as-tags", + "--jobs", "--in-process", + "--isolate", "--timeout", + "--no-adaptive-timeout", "--retries", + "--repeat", "--fail-fast", + "--order", "--seed", + "--no-bisect", "--success", + "--break", "--no-throw", + "--warn", "--allow-no-tests", + "--skip-benchmarks", "--benchmark-samples", + "--benchmark-resamples", + "--benchmark-confidence-interval", + "--benchmark-no-analysis", + "--benchmark-warmup-time", + "--history", "--no-history", + "--quarantine", "--replay", + "--analytics", "--verbose", + "--quiet", "--color", + "--junit", "--json", + "--trace", "--premature-exit-guard-file", + "--slowest", "--list", + "--list-tags", "--help", + "--version"}) + { + CAPTURE(name); + CHECK_THAT(help, ContainsSubstring(name)); + } + // The worker option is internal. + CHECK_THAT(help, !ContainsSubstring("--catch3-worker")); +} + +TEST_CASE("A worker gets every option that changes how tests behave", + "[progmasoft][runner][options]") +{ + const auto options = Parse({"--section=outer", + "--section=inner", + "-p", + "g:1", + "-#", + "-s", + "-e", + "--warn=no-assertions", + "--warn=infinite-generators", + "--skip-benchmarks", + "--benchmark-no-analysis", + "--benchmark-samples=3", + "--benchmark-resamples=4", + "--benchmark-warmup-time=5", + "--benchmark-confidence-interval=0.9", + "--seed=6", + // Runner-side only: + "--jobs=9", + "--junit=x.xml", + "[tag]"}); + const auto worker = Parse(Runner::WorkerArguments(options)); + CHECK(worker.mode == Runner::Mode::kWorker); + CHECK(worker.sections == options.sections); + CHECK(worker.paths == options.paths); + CHECK(worker.filenamesAsTags); + CHECK(worker.includeSuccessful); + CHECK(worker.noThrow); + CHECK(worker.warnNoAssertions); + CHECK(worker.warnInfiniteGenerators); + CHECK(worker.skipBenchmarks); + CHECK(worker.benchmarkNoAnalysis); + CHECK(worker.benchmarkSamples == 3); + CHECK(worker.benchmarkResamples == 4); + CHECK(worker.benchmarkWarmupTime == milliseconds{5}); + CHECK(worker.benchmarkConfidenceInterval == 0.9); + CHECK(worker.seed == 6); + CHECK(worker.jobs == 0); + CHECK(worker.junitPath.empty()); + CHECK(worker.testSpecs.empty()); +} diff --git a/tests/Progmasoft/RunnerPlanner.tests.cpp b/tests/Progmasoft/RunnerPlanner.tests.cpp new file mode 100644 index 00000000..4c930632 --- /dev/null +++ b/tests/Progmasoft/RunnerPlanner.tests.cpp @@ -0,0 +1,599 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + +// The planner decides which tests run, where, in which order and under +// which limits, from the options, the history and the changed files. These +// tests give it a catalog of their own, so no test of this binary is +// involved. + +#include <Progmasoft/Runner/Catalog.hpp> +#include <Progmasoft/Runner/Files.hpp> +#include <Progmasoft/Runner/History.hpp> +#include <Progmasoft/Runner/Planner.hpp> + +#include <catch2/catch_test_macros.hpp> +#include <catch2/matchers/catch_matchers_string.hpp> + +#include <algorithm> +#include <chrono> +#include <filesystem> +#include <fstream> +#include <map> +#include <set> +#include <string> +#include <vector> + +namespace +{ + namespace Runner = Progmasoft::Catch3::Runner; + using Catch::Matchers::ContainsSubstring; + using Progmasoft::Catch3::Attempt; + using Progmasoft::Catch3::Outcome; + using Progmasoft::Catch3::RunResult; + using Progmasoft::Catch3::TestResult; + using std::chrono::milliseconds; + + /// A directory of its own, removed at the end of the test. + class Scratch final + { + public: + Scratch() + { + std::error_code error; + path_ = std::filesystem::temp_directory_path(error) / + ("catch3-planner-" + Runner::UniqueName()); + std::filesystem::create_directories(path_, error); + } + Scratch(const Scratch &) = delete; + Scratch & + operator=(const Scratch &) = delete; + ~Scratch() + { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + [[nodiscard]] const std::filesystem::path & + Path() const noexcept + { + return path_; + } + + private: + std::filesystem::path path_; + }; + + Runner::TestEntry + Entry(const std::string &name, const std::string &file = "src/a.cpp") + { + Runner::TestEntry entry; + entry.name = name; + entry.file = file; + entry.line = 1; + return entry; + } + + std::vector<std::size_t> + All(const Runner::Catalog &catalog) + { + std::vector<std::size_t> indices(catalog.Tests().size()); + for (std::size_t index = 0; index < indices.size(); ++index) + { + indices[index] = index; + } + return indices; + } + + Runner::Plan + PlanOf(const Runner::Options &options, const Runner::PlanInputs &inputs) + { + auto plan = Runner::MakePlan(options, inputs); + INFO((plan ? std::string{} : plan.GetError().Message())); + REQUIRE(plan); + return *plan; + } + + std::string + PlanError(const Runner::Options &options, const Runner::PlanInputs &inputs) + { + auto plan = Runner::MakePlan(options, inputs); + REQUIRE_FALSE(plan); + return plan.GetError().Message(); + } + + std::vector<std::string> + Names(const Runner::Plan &plan) + { + std::vector<std::string> names; + for (const auto &test : plan.tests) + { + names.push_back(test.entry->name); + } + return names; + } + + const Runner::PlannedTest & + Planned(const Runner::Plan &plan, const std::string &name) + { + const auto found = + std::find_if(plan.tests.begin(), + plan.tests.end(), + [&](const Runner::PlannedTest &test) + { return test.entry->name == name; }); + REQUIRE(found != plan.tests.end()); + return *found; + } + + /// One test's result in a run committed to the history. + struct Result final + { + std::string name; + Outcome outcome = Outcome::kPassed; + std::int64_t microseconds = 1000; + bool flaky = false; + std::vector<std::string> precededBy; + std::uint32_t seed = 0; + }; + + /// Adds one run to the history in `file` and returns the history. + Runner::History + Commit(const std::filesystem::path &file, + const std::vector<Result> &results, + const std::map<std::string, std::string> &fingerprints = {}) + { + RunResult run; + for (const auto &result : results) + { + TestResult test; + test.id = result.name; + test.name = result.name; + test.outcome = result.outcome; + test.flaky = result.flaky; + Attempt attempt; + attempt.outcome = result.flaky ? Outcome::kFailed : result.outcome; + attempt.durationMicroseconds = result.microseconds; + attempt.precededBy = result.precededBy; + attempt.seed = result.seed; + test.attempts.push_back(attempt); + run.tests.push_back(test); + } + auto history = + Runner::History::Commit(file, run, {}, fingerprints, {}); + INFO((history ? std::string{} : history.GetError().Message())); + REQUIRE(history); + return *history; + } +} // namespace + +TEST_CASE("The planner selects tests by their history and by change", + "[progmasoft][runner][planner]") +{ + const Scratch scratch; + const auto file = Runner::HistoryFile(scratch.Path(), "selection"); + const auto catalog = Runner::Catalog::FromEntries( + {Entry("steady"), + Entry("broken"), + Entry("shaky"), + Entry("edited", "src/edited.cpp"), + Entry("unseen")}); + const auto history = Commit(file, + {{"steady"}, + {"broken", Outcome::kFailed}, + {"shaky", Outcome::kPassed, 1000, true}, + {"edited"}}); + Runner::Changes changes; + changes.root = scratch.Path(); + changes.files.push_back(scratch.Path() / "src" / "edited.cpp"); + + Runner::PlanInputs inputs; + inputs.catalog = &catalog; + inputs.matched = All(catalog); + inputs.history = &history; + inputs.changes = &changes; + Runner::Options options; + options.order = Runner::Order::kDeclared; + + SECTION("all") + { + CHECK(PlanOf(options, inputs).tests.size() == 5); + } + SECTION("failed: failures and flaky passes of the last run") + { + options.selection = Runner::Selection::kFailed; + const auto plan = PlanOf(options, inputs); + CHECK(Names(plan) == std::vector<std::string>{"broken", "shaky"}); + CHECK(Planned(plan, "broken").selectionReason == "failed last run"); + CHECK(Planned(plan, "shaky").selectionReason == "flaky last run"); + } + SECTION("new: tests without a result") + { + options.selection = Runner::Selection::kNew; + const auto plan = PlanOf(options, inputs); + CHECK(Names(plan) == std::vector<std::string>{"unseen"}); + CHECK(Planned(plan, "unseen").selectionReason == "no earlier result"); + } + SECTION("changed: tests in a changed file") + { + options.selection = Runner::Selection::kChanged; + const auto plan = PlanOf(options, inputs); + CHECK(Names(plan) == std::vector<std::string>{"edited"}); + CHECK(Planned(plan, "edited").selectionReason == + "its source file changed"); + } + SECTION("smart: everything but the steady test") + { + options.selection = Runner::Selection::kSmart; + const auto plan = PlanOf(options, inputs); + CHECK(Names(plan) == std::vector<std::string>{ + "broken", "shaky", "edited", "unseen"}); + } + SECTION("changed, when Git did not answer, selects nothing and says so") + { + options.selection = Runner::Selection::kChanged; + inputs.changes = nullptr; + const auto plan = PlanOf(options, inputs); + CHECK(plan.tests.empty()); + REQUIRE(plan.notes.size() == 1); + CHECK_THAT(plan.notes.front(), ContainsSubstring("Git")); + } + SECTION("only the matched tests are candidates") + { + inputs.matched = {0, 4}; + const auto plan = PlanOf(options, inputs); + CHECK(plan.candidates == 2); + CHECK(Names(plan) == std::vector<std::string>{"steady", "unseen"}); + } +} + +TEST_CASE("Dependencies come along and run first", + "[progmasoft][runner][planner]") +{ + auto user = Entry("user"); + user.dependsOn = {"setup"}; + auto setup = Entry("setup"); + setup.dependsOn = {"base"}; + const auto catalog = Runner::Catalog::FromEntries( + {Entry("base"), setup, user, Entry("other")}); + Runner::PlanInputs inputs; + inputs.catalog = &catalog; + inputs.matched = {2}; + Runner::Options options; + options.order = Runner::Order::kName; + + const auto plan = PlanOf(options, inputs); + CHECK(Names(plan) == std::vector<std::string>{"base", "setup", "user"}); + CHECK(Planned(plan, "setup").selectionReason == "needed by 'user'"); + CHECK(Planned(plan, "base").selectionReason == "needed by 'setup'"); + // Dependencies are positions in the plan, wherever the order put them. + REQUIRE(Planned(plan, "user").dependencies.size() == 1); + CHECK(plan.tests[Planned(plan, "user").dependencies.front()] + .entry->name == "setup"); + CHECK(Planned(plan, "base").dependencies.empty()); +} + +TEST_CASE("Broken dependencies stop the plan with an explanation", + "[progmasoft][runner][planner]") +{ + Runner::Options options; + Runner::PlanInputs inputs; + + SECTION("a dependency that is not a test") + { + auto lonely = Entry("lonely"); + lonely.dependsOn = {"nowhere"}; + const auto catalog = Runner::Catalog::FromEntries({lonely}); + inputs.catalog = &catalog; + inputs.matched = All(catalog); + CHECK_THAT(PlanError(options, inputs), + ContainsSubstring("'lonely' depends on 'nowhere'")); + } + SECTION("a cycle") + { + auto first = Entry("first"); + first.dependsOn = {"second"}; + auto second = Entry("second"); + second.dependsOn = {"third"}; + auto third = Entry("third"); + third.dependsOn = {"first"}; + const auto catalog = + Runner::Catalog::FromEntries({first, second, third}); + inputs.catalog = &catalog; + inputs.matched = {0}; + const auto message = PlanError(options, inputs); + CHECK_THAT(message, ContainsSubstring("cycle")); + CHECK_THAT(message, + ContainsSubstring("'first' -> 'second' -> 'third' -> " + "'first'")); + } +} + +TEST_CASE("Hash shards cover every test once and keep dependencies together", + "[progmasoft][runner][planner]") +{ + std::vector<Runner::TestEntry> entries; + for (int index = 0; index < 40; ++index) + { + entries.push_back(Entry("test " + std::to_string(index))); + } + // A chain of three tests must land in one shard. + entries[10].dependsOn = {"test 20"}; + entries[20].dependsOn = {"test 30"}; + const auto catalog = Runner::Catalog::FromEntries(entries); + Runner::PlanInputs inputs; + inputs.catalog = &catalog; + inputs.matched = All(catalog); + Runner::Options options; + options.shard.count = 3; + + std::multiset<std::string> seen; + std::set<unsigned> shardsOfChain; + for (unsigned shard = 0; shard < 3; ++shard) + { + options.shard.index = shard; + const auto plan = PlanOf(options, inputs); + CHECK_FALSE(plan.tests.empty()); + for (const auto &name : Names(plan)) + { + seen.insert(name); + if (name == "test 10" || name == "test 20" || name == "test 30") + { + shardsOfChain.insert(shard); + } + } + } + CHECK(seen.size() == 40); + CHECK(std::set<std::string>(seen.begin(), seen.end()).size() == 40); + CHECK(shardsOfChain.size() == 1); +} + +TEST_CASE("Duration shards balance the recorded time", + "[progmasoft][runner][planner]") +{ + const Scratch scratch; + const auto report = scratch.Path() / "report.json"; + std::ofstream(report) + << R"({"format": "catch3-run", "version": 1, "tests": [)" + << R"({"id": "long", "durationMicroseconds": 9000000},)" + << R"({"id": "medium", "durationMicroseconds": 5000000},)" + << R"({"id": "short", "durationMicroseconds": 4000000}]})"; + const auto catalog = Runner::Catalog::FromEntries( + {Entry("short"), Entry("medium"), Entry("long")}); + Runner::PlanInputs inputs; + inputs.catalog = &catalog; + inputs.matched = All(catalog); + Runner::Options options; + options.order = Runner::Order::kName; + options.shard.count = 2; + options.shardDurations = report.string(); + + // Longest first into the emptier shard: 9 s alone, 5 s + 4 s together. + options.shard.index = 0; + CHECK(Names(PlanOf(options, inputs)) == std::vector<std::string>{"long"}); + options.shard.index = 1; + CHECK(Names(PlanOf(options, inputs)) == + std::vector<std::string>{"medium", "short"}); + + SECTION("a file that is no report is an error") + { + std::ofstream(report, std::ios::trunc) << R"({"format": "other"})"; + CHECK_THAT(PlanError(options, inputs), + ContainsSubstring("neither a Catch3 history")); + } +} + +TEST_CASE("An incremental run keeps passes with the same inputs", + "[progmasoft][runner][planner]") +{ + const Scratch scratch; + const auto file = Runner::HistoryFile(scratch.Path(), "incremental"); + const auto catalog = Runner::Catalog::FromEntries( + {Entry("passes"), Entry("fails"), Entry("flakes")}); + Runner::PlanInputs inputs; + inputs.catalog = &catalog; + inputs.matched = All(catalog); + inputs.binaryFingerprint = "build 1"; + Runner::Options options; + options.order = Runner::Order::kDeclared; + + const auto first = PlanOf(options, inputs); + CHECK(first.fingerprints.size() == 3); + const auto history = + Commit(file, + {{"passes"}, + {"fails", Outcome::kFailed}, + {"flakes", Outcome::kPassed, 1000, true}}, + first.fingerprints); + inputs.history = &history; + options.incremental = true; + + const auto again = PlanOf(options, inputs); + CHECK(Names(again) == std::vector<std::string>{"fails", "flakes"}); + REQUIRE(again.kept.size() == 1); + CHECK(again.kept.front().name == "passes"); + CHECK(again.kept.front().outcome == Outcome::kUpToDate); + + // Another build changes every fingerprint. + inputs.binaryFingerprint = "build 2"; + CHECK(PlanOf(options, inputs).tests.size() == 3); +} + +TEST_CASE("Time limits are learned from the durations", + "[progmasoft][runner][planner]") +{ + const Scratch scratch; + const auto file = Runner::HistoryFile(scratch.Path(), "timeouts"); + auto tagged = Entry("tagged"); + tagged.timeout = milliseconds{1500}; + auto benchmark = Entry("benchmark"); + benchmark.benchmark = true; + const auto catalog = Runner::Catalog::FromEntries( + {Entry("quick"), Entry("slow"), tagged, benchmark, Entry("unseen")}); + Runner::History history; + for (int run = 0; run < 5; ++run) + { + history = Commit(file, + {{"quick", Outcome::kPassed, 100000}, + {"slow", Outcome::kPassed, 2000000}, + {"tagged", Outcome::kPassed, 100000}, + {"benchmark", Outcome::kPassed, 2000000}}); + } + Runner::PlanInputs inputs; + inputs.catalog = &catalog; + inputs.matched = All(catalog); + inputs.history = &history; + Runner::Options options; + + auto plan = PlanOf(options, inputs); + // max(p95 x 4, median x 10) + 2 s, at least 5 s. + CHECK(Planned(plan, "quick").timeout == milliseconds{5000}); + CHECK(Planned(plan, "quick").timeoutLearned); + CHECK(Planned(plan, "slow").timeout == milliseconds{22000}); + CHECK(Planned(plan, "slow").predictedMicroseconds == 2000000); + // A tag wins; benchmarks and unknown tests get the global limit. + CHECK(Planned(plan, "tagged").timeout == milliseconds{1500}); + CHECK_FALSE(Planned(plan, "tagged").timeoutLearned); + CHECK(Planned(plan, "benchmark").timeout == milliseconds{300000}); + CHECK(Planned(plan, "unseen").timeout == milliseconds{300000}); + CHECK(Planned(plan, "unseen").predictedMicroseconds == 500000); + + SECTION("--timeout caps what was learned") + { + options.timeout = milliseconds{10000}; + options.timeoutGiven = true; + plan = PlanOf(options, inputs); + CHECK(Planned(plan, "slow").timeout == milliseconds{10000}); + CHECK(Planned(plan, "unseen").timeout == milliseconds{10000}); + } + SECTION("--no-adaptive-timeout uses the global limit") + { + options.adaptiveTimeout = false; + plan = PlanOf(options, inputs); + CHECK(Planned(plan, "quick").timeout == milliseconds{300000}); + CHECK_FALSE(Planned(plan, "quick").timeoutLearned); + } +} + +TEST_CASE("Likely failures go first, then the longest tests", + "[progmasoft][runner][planner]") +{ + const Scratch scratch; + const auto file = Runner::HistoryFile(scratch.Path(), "priority"); + const auto catalog = Runner::Catalog::FromEntries( + {Entry("long pass"), + Entry("short pass"), + Entry("slow failure"), + Entry("quick failure"), + Entry("unseen")}); + const auto history = Commit(file, + {{"long pass", Outcome::kPassed, 3000000}, + {"short pass", Outcome::kPassed, 10000}, + {"slow failure", Outcome::kFailed, 2000000}, + {"quick failure", Outcome::kFailed, 10000}}); + Runner::PlanInputs inputs; + inputs.catalog = &catalog; + inputs.matched = All(catalog); + inputs.history = &history; + Runner::Options options; + + SECTION("priority") + { + const auto plan = PlanOf(options, inputs); + CHECK(Planned(plan, "quick failure").failureProbability >= 0.5); + CHECK(Names(plan) == std::vector<std::string>{"quick failure", + "slow failure", + "long pass", + "unseen", + "short pass"}); + } + SECTION("declared") + { + options.order = Runner::Order::kDeclared; + CHECK(Names(PlanOf(options, inputs)) == + std::vector<std::string>{"long pass", + "short pass", + "slow failure", + "quick failure", + "unseen"}); + } + SECTION("random, the same for the same seed") + { + options.order = Runner::Order::kRandom; + options.seed = 7; + const auto once = Names(PlanOf(options, inputs)); + CHECK(Names(PlanOf(options, inputs)) == once); + auto sorted = once; + std::sort(sorted.begin(), sorted.end()); + CHECK(sorted.size() == 5); + } +} + +TEST_CASE("Retries and quarantine come from tags, options and history", + "[progmasoft][runner][planner]") +{ + auto retried = Entry("retried"); + retried.retries = 5U; + auto quarantined = Entry("quarantined"); + quarantined.quarantined = true; + const auto catalog = + Runner::Catalog::FromEntries({Entry("plain"), retried, quarantined}); + Runner::PlanInputs inputs; + inputs.catalog = &catalog; + inputs.matched = All(catalog); + Runner::Options options; + + auto plan = PlanOf(options, inputs); + CHECK(Planned(plan, "plain").retryLimit == 2); + CHECK(Planned(plan, "retried").retryLimit == 5); + CHECK_FALSE(Planned(plan, "plain").quarantined); + CHECK(Planned(plan, "quarantined").quarantined); + CHECK(Planned(plan, "quarantined").quarantineReason == + "tagged [!quarantine]"); + + options.retryMode = Runner::RetryMode::kOff; + options.quarantine = Runner::QuarantineMode::kOff; + plan = PlanOf(options, inputs); + CHECK(Planned(plan, "plain").retryLimit == 0); + // A tag still asks for its retries. + CHECK(Planned(plan, "retried").retryLimit == 5); + CHECK_FALSE(Planned(plan, "quarantined").quarantined); +} + +TEST_CASE("Replay runs the last failures the way they happened", + "[progmasoft][runner][planner]") +{ + const Scratch scratch; + const auto file = Runner::HistoryFile(scratch.Path(), "replay"); + const auto catalog = Runner::Catalog::FromEntries( + {Entry("polluter"), Entry("victim"), Entry("bystander")}); + Runner::PlanInputs inputs; + inputs.catalog = &catalog; + inputs.matched = All(catalog); + Runner::Options options; + options.replayFailures = true; + + SECTION("with a failure") + { + Result victim{"victim", Outcome::kFailed}; + victim.precededBy = {"polluter"}; + victim.seed = 42; + const auto history = + Commit(file, {{"polluter"}, victim, {"bystander"}}); + inputs.history = &history; + const auto plan = PlanOf(options, inputs); + REQUIRE(Names(plan) == std::vector<std::string>{"victim"}); + const auto &test = plan.tests.front(); + CHECK(test.replaySeed == 42U); + REQUIRE(test.replaySequence.has_value()); + CHECK(*test.replaySequence == std::vector<std::size_t>{0}); + CHECK_THAT(test.selectionReason, ContainsSubstring("failed in run")); + } + SECTION("without one") + { + const auto history = Commit(file, {{"polluter"}, {"victim"}}); + inputs.history = &history; + const auto plan = PlanOf(options, inputs); + CHECK(plan.tests.empty()); + REQUIRE(plan.notes.size() == 1); + CHECK_THAT(plan.notes.front(), ContainsSubstring("no failure")); + } +} diff --git a/tests/meson.build b/tests/meson.build index bb036e04..dc1a8edf 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -79,6 +79,9 @@ progmasoft_property_tests = executable( 'Progmasoft/Results.tests.cpp', 'Progmasoft/Runner.tests.cpp', 'Progmasoft/RunnerUnits.tests.cpp', + 'Progmasoft/RunnerPlanner.tests.cpp', + 'Progmasoft/RunnerOptions.tests.cpp', + 'Progmasoft/RunnerAnalysis.tests.cpp', 'Progmasoft/Snapshot.tests.cpp', 'Progmasoft/Stateful.tests.cpp', 'Progmasoft/Strategy.tests.cpp', diff --git a/xmake.lua b/xmake.lua index 7730be4e..bcb486ca 100644 --- a/xmake.lua +++ b/xmake.lua @@ -64,6 +64,9 @@ target("ProgmasoftPropertyTests") "tests/Progmasoft/Results.tests.cpp", "tests/Progmasoft/Runner.tests.cpp", "tests/Progmasoft/RunnerUnits.tests.cpp", + "tests/Progmasoft/RunnerPlanner.tests.cpp", + "tests/Progmasoft/RunnerOptions.tests.cpp", + "tests/Progmasoft/RunnerAnalysis.tests.cpp", "tests/Progmasoft/Snapshot.tests.cpp", "tests/Progmasoft/Stateful.tests.cpp", "tests/Progmasoft/Strategy.tests.cpp",