catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
develroot / tests / Progmasoft / RunnerPlanner.tests.cpp
// Copyright (c) 2026 Progmasoft.
// SPDX-License-Identifier: LGPL-3.0-or-later
// The planner decides which tests run, where, in which order and under
// which limits, from the options, the history and the changed files. These
// tests give it a catalog of their own, so no test of this binary is
// involved.
#include <Progmasoft/Runner/Catalog.hpp>
#include <Progmasoft/Runner/Files.hpp>
#include <Progmasoft/Runner/History.hpp>
#include <Progmasoft/Runner/Planner.hpp>
#include <Progmasoft/Catch3/TestMacros.hpp>
#include <Progmasoft/Catch3/Matchers/String.hpp>
#include <algorithm>
#include <chrono>
#include <filesystem>
#include <fstream>
#include <map>
#include <set>
#include <string>
#include <vector>
namespace
{
namespace Runner = Progmasoft::Catch3::Runner;
using Catch::Matchers::ContainsSubstring;
using Progmasoft::Catch3::Attempt;
using Progmasoft::Catch3::Outcome;
using Progmasoft::Catch3::RunResult;
using Progmasoft::Catch3::TestResult;
using std::chrono::milliseconds;
/// A directory of its own, removed at the end of the test.
class Scratch final
{
public:
Scratch()
{
std::error_code error;
path_ = std::filesystem::temp_directory_path(error) /
("catch3-planner-" + Runner::UniqueName());
std::filesystem::create_directories(path_, error);
}
Scratch(const Scratch &) = delete;
Scratch &
operator=(const Scratch &) = delete;
~Scratch()
{
std::error_code error;
std::filesystem::remove_all(path_, error);
}
[[nodiscard]] const std::filesystem::path &
Path() const noexcept
{
return path_;
}
private:
std::filesystem::path path_;
};
Runner::TestEntry
Entry(const std::string &name, const std::string &file = "src/a.cpp")
{
Runner::TestEntry entry;
entry.name = name;
entry.file = file;
entry.line = 1;
return entry;
}
std::vector<std::size_t>
All(const Runner::Catalog &catalog)
{
std::vector<std::size_t> indices(catalog.Tests().size());
for (std::size_t index = 0; index < indices.size(); ++index)
{
indices[index] = index;
}
return indices;
}
Runner::Plan
PlanOf(const Runner::Options &options, const Runner::PlanInputs &inputs)
{
auto plan = Runner::MakePlan(options, inputs);
INFO((plan ? std::string{} : plan.GetError().Message()));
REQUIRE(plan);
return *plan;
}
std::string
PlanError(const Runner::Options &options, const Runner::PlanInputs &inputs)
{
auto plan = Runner::MakePlan(options, inputs);
REQUIRE_FALSE(plan);
return plan.GetError().Message();
}
std::vector<std::string>
Names(const Runner::Plan &plan)
{
std::vector<std::string> names;
for (const auto &test : plan.tests)
{
names.push_back(test.entry->name);
}
return names;
}
const Runner::PlannedTest &
Planned(const Runner::Plan &plan, const std::string &name)
{
const auto found =
std::find_if(plan.tests.begin(),
plan.tests.end(),
[&](const Runner::PlannedTest &test)
{ return test.entry->name == name; });
REQUIRE(found != plan.tests.end());
return *found;
}
/// One test's result in a run committed to the history.
struct Result final
{
std::string name;
Outcome outcome = Outcome::kPassed;
std::int64_t microseconds = 1000;
bool flaky = false;
std::vector<std::string> precededBy;
std::uint32_t seed = 0;
};
/// Adds one run to the history in `file` and returns the history.
Runner::History
Commit(const std::filesystem::path &file,
const std::vector<Result> &results,
const std::map<std::string, std::string> &fingerprints = {})
{
RunResult run;
for (const auto &result : results)
{
TestResult test;
test.id = result.name;
test.name = result.name;
test.outcome = result.outcome;
test.flaky = result.flaky;
Attempt attempt;
attempt.outcome = result.flaky ? Outcome::kFailed : result.outcome;
attempt.durationMicroseconds = result.microseconds;
attempt.precededBy = result.precededBy;
attempt.seed = result.seed;
test.attempts.push_back(attempt);
run.tests.push_back(test);
}
auto history =
Runner::History::Commit(file, run, {}, fingerprints, {});
INFO((history ? std::string{} : history.GetError().Message()));
REQUIRE(history);
return *history;
}
} // namespace
TEST_CASE("The planner selects tests by their history and by change",
"[progmasoft][runner][planner]")
{
const Scratch scratch;
const auto file = Runner::HistoryFile(scratch.Path(), "selection");
const auto catalog = Runner::Catalog::FromEntries(
{Entry("steady"),
Entry("broken"),
Entry("shaky"),
Entry("edited", "src/edited.cpp"),
Entry("unseen")});
const auto history = Commit(file,
{{"steady"},
{"broken", Outcome::kFailed},
{"shaky", Outcome::kPassed, 1000, true},
{"edited"}});
Runner::Changes changes;
changes.root = scratch.Path();
changes.files.push_back(scratch.Path() / "src" / "edited.cpp");
Runner::PlanInputs inputs;
inputs.catalog = &catalog;
inputs.matched = All(catalog);
inputs.history = &history;
inputs.changes = &changes;
Runner::Options options;
options.order = Runner::Order::kDeclared;
SECTION("all")
{
CHECK(PlanOf(options, inputs).tests.size() == 5);
}
SECTION("failed: failures and flaky passes of the last run")
{
options.selection = Runner::Selection::kFailed;
const auto plan = PlanOf(options, inputs);
CHECK(Names(plan) == std::vector<std::string>{"broken", "shaky"});
CHECK(Planned(plan, "broken").selectionReason == "failed last run");
CHECK(Planned(plan, "shaky").selectionReason == "flaky last run");
}
SECTION("new: tests without a result")
{
options.selection = Runner::Selection::kNew;
const auto plan = PlanOf(options, inputs);
CHECK(Names(plan) == std::vector<std::string>{"unseen"});
CHECK(Planned(plan, "unseen").selectionReason == "no earlier result");
}
SECTION("changed: tests in a changed file")
{
options.selection = Runner::Selection::kChanged;
const auto plan = PlanOf(options, inputs);
CHECK(Names(plan) == std::vector<std::string>{"edited"});
CHECK(Planned(plan, "edited").selectionReason ==
"its source file changed");
}
SECTION("smart: everything but the steady test")
{
options.selection = Runner::Selection::kSmart;
const auto plan = PlanOf(options, inputs);
CHECK(Names(plan) == std::vector<std::string>{
"broken", "shaky", "edited", "unseen"});
}
SECTION("changed, when Git did not answer, selects nothing and says so")
{
options.selection = Runner::Selection::kChanged;
inputs.changes = nullptr;
const auto plan = PlanOf(options, inputs);
CHECK(plan.tests.empty());
REQUIRE(plan.notes.size() == 1);
CHECK_THAT(plan.notes.front(), ContainsSubstring("Git"));
}
SECTION("only the matched tests are candidates")
{
inputs.matched = {0, 4};
const auto plan = PlanOf(options, inputs);
CHECK(plan.candidates == 2);
CHECK(Names(plan) == std::vector<std::string>{"steady", "unseen"});
}
}
TEST_CASE("Dependencies come along and run first",
"[progmasoft][runner][planner]")
{
auto user = Entry("user");
user.dependsOn = {"setup"};
auto setup = Entry("setup");
setup.dependsOn = {"base"};
const auto catalog = Runner::Catalog::FromEntries(
{Entry("base"), setup, user, Entry("other")});
Runner::PlanInputs inputs;
inputs.catalog = &catalog;
inputs.matched = {2};
Runner::Options options;
options.order = Runner::Order::kName;
const auto plan = PlanOf(options, inputs);
CHECK(Names(plan) == std::vector<std::string>{"base", "setup", "user"});
CHECK(Planned(plan, "setup").selectionReason == "needed by 'user'");
CHECK(Planned(plan, "base").selectionReason == "needed by 'setup'");
// Dependencies are positions in the plan, wherever the order put them.
REQUIRE(Planned(plan, "user").dependencies.size() == 1);
CHECK(plan.tests[Planned(plan, "user").dependencies.front()]
.entry->name == "setup");
CHECK(Planned(plan, "base").dependencies.empty());
}
TEST_CASE("Broken dependencies stop the plan with an explanation",
"[progmasoft][runner][planner]")
{
Runner::Options options;
Runner::PlanInputs inputs;
SECTION("a dependency that is not a test")
{
auto lonely = Entry("lonely");
lonely.dependsOn = {"nowhere"};
const auto catalog = Runner::Catalog::FromEntries({lonely});
inputs.catalog = &catalog;
inputs.matched = All(catalog);
CHECK_THAT(PlanError(options, inputs),
ContainsSubstring("'lonely' depends on 'nowhere'"));
}
SECTION("a cycle")
{
auto first = Entry("first");
first.dependsOn = {"second"};
auto second = Entry("second");
second.dependsOn = {"third"};
auto third = Entry("third");
third.dependsOn = {"first"};
const auto catalog =
Runner::Catalog::FromEntries({first, second, third});
inputs.catalog = &catalog;
inputs.matched = {0};
const auto message = PlanError(options, inputs);
CHECK_THAT(message, ContainsSubstring("cycle"));
CHECK_THAT(message,
ContainsSubstring("'first' -> 'second' -> 'third' -> "
"'first'"));
}
}
TEST_CASE("Hash shards cover every test once and keep dependencies together",
"[progmasoft][runner][planner]")
{
std::vector<Runner::TestEntry> entries;
for (int index = 0; index < 40; ++index)
{
entries.push_back(Entry("test " + std::to_string(index)));
}
// A chain of three tests must land in one shard.
entries[10].dependsOn = {"test 20"};
entries[20].dependsOn = {"test 30"};
const auto catalog = Runner::Catalog::FromEntries(entries);
Runner::PlanInputs inputs;
inputs.catalog = &catalog;
inputs.matched = All(catalog);
Runner::Options options;
options.shard.count = 3;
std::multiset<std::string> seen;
std::set<unsigned> shardsOfChain;
for (unsigned shard = 0; shard < 3; ++shard)
{
options.shard.index = shard;
const auto plan = PlanOf(options, inputs);
CHECK_FALSE(plan.tests.empty());
for (const auto &name : Names(plan))
{
seen.insert(name);
if (name == "test 10" || name == "test 20" || name == "test 30")
{
shardsOfChain.insert(shard);
}
}
}
CHECK(seen.size() == 40);
CHECK(std::set<std::string>(seen.begin(), seen.end()).size() == 40);
CHECK(shardsOfChain.size() == 1);
}
TEST_CASE("Duration shards balance the recorded time",
"[progmasoft][runner][planner]")
{
const Scratch scratch;
const auto report = scratch.Path() / "report.json";
std::ofstream(report)
<< R"({"format": "catch3-run", "version": 1, "tests": [)"
<< R"({"id": "long", "durationMicroseconds": 9000000},)"
<< R"({"id": "medium", "durationMicroseconds": 5000000},)"
<< R"({"id": "short", "durationMicroseconds": 4000000}]})";
const auto catalog = Runner::Catalog::FromEntries(
{Entry("short"), Entry("medium"), Entry("long")});
Runner::PlanInputs inputs;
inputs.catalog = &catalog;
inputs.matched = All(catalog);
Runner::Options options;
options.order = Runner::Order::kName;
options.shard.count = 2;
options.shardDurations = report.string();
// Longest first into the emptier shard: 9 s alone, 5 s + 4 s together.
options.shard.index = 0;
CHECK(Names(PlanOf(options, inputs)) == std::vector<std::string>{"long"});
options.shard.index = 1;
CHECK(Names(PlanOf(options, inputs)) ==
std::vector<std::string>{"medium", "short"});
SECTION("a file that is no report is an error")
{
std::ofstream(report, std::ios::trunc) << R"({"format": "other"})";
CHECK_THAT(PlanError(options, inputs),
ContainsSubstring("neither a Catch3 history"));
}
}
TEST_CASE("An incremental run keeps passes with the same inputs",
"[progmasoft][runner][planner]")
{
const Scratch scratch;
const auto file = Runner::HistoryFile(scratch.Path(), "incremental");
const auto catalog = Runner::Catalog::FromEntries(
{Entry("passes"), Entry("fails"), Entry("flakes")});
Runner::PlanInputs inputs;
inputs.catalog = &catalog;
inputs.matched = All(catalog);
inputs.binaryFingerprint = "build 1";
Runner::Options options;
options.order = Runner::Order::kDeclared;
const auto first = PlanOf(options, inputs);
CHECK(first.fingerprints.size() == 3);
const auto history =
Commit(file,
{{"passes"},
{"fails", Outcome::kFailed},
{"flakes", Outcome::kPassed, 1000, true}},
first.fingerprints);
inputs.history = &history;
options.incremental = true;
const auto again = PlanOf(options, inputs);
CHECK(Names(again) == std::vector<std::string>{"fails", "flakes"});
REQUIRE(again.kept.size() == 1);
CHECK(again.kept.front().name == "passes");
CHECK(again.kept.front().outcome == Outcome::kUpToDate);
// Another build changes every fingerprint.
inputs.binaryFingerprint = "build 2";
CHECK(PlanOf(options, inputs).tests.size() == 3);
}
TEST_CASE("Time limits are learned from the durations",
"[progmasoft][runner][planner]")
{
const Scratch scratch;
const auto file = Runner::HistoryFile(scratch.Path(), "timeouts");
auto tagged = Entry("tagged");
tagged.timeout = milliseconds{1500};
auto benchmark = Entry("benchmark");
benchmark.benchmark = true;
const auto catalog = Runner::Catalog::FromEntries(
{Entry("quick"), Entry("slow"), tagged, benchmark, Entry("unseen")});
Runner::History history;
for (int run = 0; run < 5; ++run)
{
history = Commit(file,
{{"quick", Outcome::kPassed, 100000},
{"slow", Outcome::kPassed, 2000000},
{"tagged", Outcome::kPassed, 100000},
{"benchmark", Outcome::kPassed, 2000000}});
}
Runner::PlanInputs inputs;
inputs.catalog = &catalog;
inputs.matched = All(catalog);
inputs.history = &history;
Runner::Options options;
auto plan = PlanOf(options, inputs);
// max(p95 x 4, median x 10) + 2 s, at least 5 s.
CHECK(Planned(plan, "quick").timeout == milliseconds{5000});
CHECK(Planned(plan, "quick").timeoutLearned);
CHECK(Planned(plan, "slow").timeout == milliseconds{22000});
CHECK(Planned(plan, "slow").predictedMicroseconds == 2000000);
// A tag wins; benchmarks and unknown tests get the global limit.
CHECK(Planned(plan, "tagged").timeout == milliseconds{1500});
CHECK_FALSE(Planned(plan, "tagged").timeoutLearned);
CHECK(Planned(plan, "benchmark").timeout == milliseconds{300000});
CHECK(Planned(plan, "unseen").timeout == milliseconds{300000});
CHECK(Planned(plan, "unseen").predictedMicroseconds == 500000);
SECTION("--timeout caps what was learned")
{
options.timeout = milliseconds{10000};
options.timeoutGiven = true;
plan = PlanOf(options, inputs);
CHECK(Planned(plan, "slow").timeout == milliseconds{10000});
CHECK(Planned(plan, "unseen").timeout == milliseconds{10000});
}
SECTION("--no-adaptive-timeout uses the global limit")
{
options.adaptiveTimeout = false;
plan = PlanOf(options, inputs);
CHECK(Planned(plan, "quick").timeout == milliseconds{300000});
CHECK_FALSE(Planned(plan, "quick").timeoutLearned);
}
}
TEST_CASE("Likely failures go first, then the longest tests",
"[progmasoft][runner][planner]")
{
const Scratch scratch;
const auto file = Runner::HistoryFile(scratch.Path(), "priority");
const auto catalog = Runner::Catalog::FromEntries(
{Entry("long pass"),
Entry("short pass"),
Entry("slow failure"),
Entry("quick failure"),
Entry("unseen")});
const auto history = Commit(file,
{{"long pass", Outcome::kPassed, 3000000},
{"short pass", Outcome::kPassed, 10000},
{"slow failure", Outcome::kFailed, 2000000},
{"quick failure", Outcome::kFailed, 10000}});
Runner::PlanInputs inputs;
inputs.catalog = &catalog;
inputs.matched = All(catalog);
inputs.history = &history;
Runner::Options options;
SECTION("priority")
{
const auto plan = PlanOf(options, inputs);
CHECK(Planned(plan, "quick failure").failureProbability >= 0.5);
CHECK(Names(plan) == std::vector<std::string>{"quick failure",
"slow failure",
"long pass",
"unseen",
"short pass"});
}
SECTION("declared")
{
options.order = Runner::Order::kDeclared;
CHECK(Names(PlanOf(options, inputs)) ==
std::vector<std::string>{"long pass",
"short pass",
"slow failure",
"quick failure",
"unseen"});
}
SECTION("random, the same for the same seed")
{
options.order = Runner::Order::kRandom;
options.seed = 7;
const auto once = Names(PlanOf(options, inputs));
CHECK(Names(PlanOf(options, inputs)) == once);
auto sorted = once;
std::sort(sorted.begin(), sorted.end());
CHECK(sorted.size() == 5);
}
}
TEST_CASE("Retries and quarantine come from tags, options and history",
"[progmasoft][runner][planner]")
{
auto retried = Entry("retried");
retried.retries = 5U;
auto quarantined = Entry("quarantined");
quarantined.quarantined = true;
const auto catalog =
Runner::Catalog::FromEntries({Entry("plain"), retried, quarantined});
Runner::PlanInputs inputs;
inputs.catalog = &catalog;
inputs.matched = All(catalog);
Runner::Options options;
auto plan = PlanOf(options, inputs);
CHECK(Planned(plan, "plain").retryLimit == 2);
CHECK(Planned(plan, "retried").retryLimit == 5);
CHECK_FALSE(Planned(plan, "plain").quarantined);
CHECK(Planned(plan, "quarantined").quarantined);
CHECK(Planned(plan, "quarantined").quarantineReason ==
"tagged [!quarantine]");
options.retryMode = Runner::RetryMode::kOff;
options.quarantine = Runner::QuarantineMode::kOff;
plan = PlanOf(options, inputs);
CHECK(Planned(plan, "plain").retryLimit == 0);
// A tag still asks for its retries.
CHECK(Planned(plan, "retried").retryLimit == 5);
CHECK_FALSE(Planned(plan, "quarantined").quarantined);
}
TEST_CASE("Replay runs the last failures the way they happened",
"[progmasoft][runner][planner]")
{
const Scratch scratch;
const auto file = Runner::HistoryFile(scratch.Path(), "replay");
const auto catalog = Runner::Catalog::FromEntries(
{Entry("polluter"), Entry("victim"), Entry("bystander")});
Runner::PlanInputs inputs;
inputs.catalog = &catalog;
inputs.matched = All(catalog);
Runner::Options options;
options.replayFailures = true;
SECTION("with a failure")
{
Result victim{"victim", Outcome::kFailed};
victim.precededBy = {"polluter"};
victim.seed = 42;
const auto history =
Commit(file, {{"polluter"}, victim, {"bystander"}});
inputs.history = &history;
const auto plan = PlanOf(options, inputs);
REQUIRE(Names(plan) == std::vector<std::string>{"victim"});
const auto &test = plan.tests.front();
CHECK(test.replaySeed == 42U);
REQUIRE(test.replaySequence.has_value());
CHECK(*test.replaySequence == std::vector<std::size_t>{0});
CHECK_THAT(test.selectionReason, ContainsSubstring("failed in run"));
}
SECTION("without one")
{
const auto history = Commit(file, {{"polluter"}, {"victim"}});
inputs.history = &history;
const auto plan = PlanOf(options, inputs);
CHECK(plan.tests.empty());
REQUIRE(plan.notes.size() == 1);
CHECK_THAT(plan.notes.front(), ContainsSubstring("no failure"));
}
}