catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
develroot / tests / Progmasoft / RunnerOptions.tests.cpp
// Copyright (c) 2026 Progmasoft.
// SPDX-License-Identifier: LGPL-3.0-or-later
// Every option of the runner's command line: how it is spelled, what it
// sets and how a wrong value is explained.
#include <Progmasoft/Runner/Files.hpp>
#include <Progmasoft/Runner/Options.hpp>
#include <Progmasoft/Catch3/TestMacros.hpp>
#include <Progmasoft/Catch3/Generators/Generators.hpp>
#include <Progmasoft/Catch3/Matchers/String.hpp>
#include <chrono>
#include <filesystem>
#include <fstream>
#include <string>
#include <vector>
namespace
{
namespace Runner = Progmasoft::Catch3::Runner;
using Catch::Matchers::ContainsSubstring;
using std::chrono::milliseconds;
Runner::Options
Parse(const std::vector<std::string> &arguments)
{
auto options = Runner::ParseOptions(arguments);
INFO((options ? std::string{} : options.GetError().Message()));
REQUIRE(options);
return *options;
}
std::string
ParseError(const std::vector<std::string> &arguments)
{
auto options = Runner::ParseOptions(arguments);
REQUIRE_FALSE(options);
return options.GetError().Message();
}
} // namespace
TEST_CASE("Options have their documented defaults",
"[progmasoft][runner][options]")
{
const auto options = Parse({});
CHECK(options.mode == Runner::Mode::kRun);
CHECK(options.selection == Runner::Selection::kAll);
CHECK(options.order == Runner::Order::kPriority);
CHECK(options.retryMode == Runner::RetryMode::kAuto);
CHECK(options.retries == 2);
CHECK(options.repeat == 1);
CHECK(options.failFast == 0);
CHECK(options.jobs == 0);
CHECK(options.shard.index == 0);
CHECK(options.shard.count == 1);
CHECK_FALSE(options.timeoutGiven);
CHECK(options.adaptiveTimeout);
CHECK(options.bisectOrder);
CHECK(options.historyEnabled);
CHECK(options.quarantine == Runner::QuarantineMode::kAuto);
CHECK(options.verbosity == Runner::Verbosity::kNormal);
CHECK(options.color == Runner::ColorMode::kAuto);
CHECK(options.benchmarkSamples == 100);
CHECK(options.benchmarkResamples == 100000);
CHECK(options.benchmarkConfidenceInterval == 0.95);
CHECK(options.benchmarkWarmupTime == milliseconds{100});
CHECK(options.testSpecs.empty());
}
TEST_CASE("A value can follow an option in every accepted spelling",
"[progmasoft][runner][options]")
{
const std::vector<std::string> arguments = GENERATE(
std::vector<std::string>{"--jobs=4"},
std::vector<std::string>{"--jobs", "4"},
std::vector<std::string>{"-j4"},
std::vector<std::string>{"-j=4"},
std::vector<std::string>{"-j", "4"});
CAPTURE(arguments);
CHECK(Parse(arguments).jobs == 4);
}
TEST_CASE("Switches set what they say", "[progmasoft][runner][options]")
{
CHECK(Parse({"--incremental"}).incremental);
CHECK(Parse({"--filenames-as-tags"}).filenamesAsTags);
CHECK(Parse({"-#"}).filenamesAsTags);
CHECK(Parse({"--in-process"}).inProcess);
CHECK(Parse({"--isolate"}).isolate);
CHECK_FALSE(Parse({"--no-adaptive-timeout"}).adaptiveTimeout);
CHECK_FALSE(Parse({"--no-bisect"}).bisectOrder);
CHECK(Parse({"--success"}).includeSuccessful);
CHECK(Parse({"-s"}).includeSuccessful);
CHECK(Parse({"--no-throw"}).noThrow);
CHECK(Parse({"-e"}).noThrow);
CHECK(Parse({"--allow-no-tests"}).allowNoTests);
CHECK(Parse({"--skip-benchmarks"}).skipBenchmarks);
CHECK(Parse({"--benchmark-no-analysis"}).benchmarkNoAnalysis);
CHECK_FALSE(Parse({"--no-history"}).historyEnabled);
CHECK(Parse({"--replay"}).replayFailures);
CHECK(Parse({"--verbose"}).verbosity == Runner::Verbosity::kVerbose);
CHECK(Parse({"-q"}).verbosity == Runner::Verbosity::kQuiet);
// A debugger needs the test in this process.
const auto debugging = Parse({"-b"});
CHECK(debugging.breakIntoDebugger);
CHECK(debugging.inProcess);
}
TEST_CASE("Modes are chosen by their options", "[progmasoft][runner][options]")
{
CHECK(Parse({"--help"}).mode == Runner::Mode::kHelp);
CHECK(Parse({"-h"}).mode == Runner::Mode::kHelp);
CHECK(Parse({"--version"}).mode == Runner::Mode::kVersion);
CHECK(Parse({"--list-tags"}).mode == Runner::Mode::kListTags);
CHECK(Parse({"-t"}).mode == Runner::Mode::kListTags);
CHECK(Parse({"--catch3-worker"}).mode == Runner::Mode::kWorker);
const auto text = Parse({"-l"});
CHECK(text.mode == Runner::Mode::kListTests);
CHECK(text.listFormat == Runner::ListFormat::kText);
CHECK(Parse({"--list=names"}).listFormat == Runner::ListFormat::kNames);
CHECK(Parse({"--list=json"}).listFormat == Runner::ListFormat::kJson);
const auto analytics = Parse({"--analytics"});
CHECK(analytics.mode == Runner::Mode::kAnalytics);
CHECK_FALSE(analytics.analyticsJson);
CHECK(Parse({"--analytics=json"}).analyticsJson);
}
TEST_CASE("Choices accept their names and nothing else",
"[progmasoft][runner][options]")
{
CHECK(Parse({"--select=all"}).selection == Runner::Selection::kAll);
CHECK(Parse({"--select=failed"}).selection == Runner::Selection::kFailed);
CHECK(Parse({"--select=changed"}).selection == Runner::Selection::kChanged);
CHECK(Parse({"--select=new"}).selection == Runner::Selection::kNew);
CHECK(Parse({"--select=smart"}).selection == Runner::Selection::kSmart);
CHECK(Parse({"--order=name"}).order == Runner::Order::kName);
CHECK(Parse({"--order=random"}).order == Runner::Order::kRandom);
CHECK(Parse({"--order=priority"}).order == Runner::Order::kPriority);
CHECK(Parse({"--quarantine=report"}).quarantine ==
Runner::QuarantineMode::kReport);
CHECK(Parse({"--quarantine=off"}).quarantine ==
Runner::QuarantineMode::kOff);
CHECK(Parse({"--color=always"}).color == Runner::ColorMode::kAlways);
CHECK(Parse({"--color=never"}).color == Runner::ColorMode::kNever);
CHECK_THAT(ParseError({"--select=some"}),
ContainsSubstring("all, failed, changed, new, smart"));
CHECK_THAT(ParseError({"--quarantine=maybe"}),
ContainsSubstring("auto, report, off"));
CHECK_THAT(ParseError({"--color=blue"}),
ContainsSubstring("auto, always, never"));
CHECK_THAT(ParseError({"--list=xml"}),
ContainsSubstring("text, names, json"));
CHECK_THAT(ParseError({"--analytics=xml"}), ContainsSubstring("json"));
}
TEST_CASE("Numbers are checked", "[progmasoft][runner][options]")
{
CHECK(Parse({"--repeat=3"}).repeat == 3);
CHECK(Parse({"-a"}).failFast == 1);
CHECK(Parse({"--slowest"}).slowest == 10);
CHECK(Parse({"--slowest=3"}).slowest == 3);
CHECK(Parse({"--benchmark-samples=7"}).benchmarkSamples == 7);
CHECK(Parse({"--benchmark-resamples=9"}).benchmarkResamples == 9);
CHECK(Parse({"--benchmark-confidence-interval=0.5"})
.benchmarkConfidenceInterval == 0.5);
CHECK(Parse({"--benchmark-warmup-time=250"}).benchmarkWarmupTime ==
milliseconds{250});
const auto unlimited = Parse({"--timeout=0"});
CHECK(unlimited.timeoutGiven);
CHECK(unlimited.timeout == milliseconds{0});
const auto retried = Parse({"--retries=auto"});
CHECK(retried.retryMode == Runner::RetryMode::kAuto);
const auto off = Parse({"--retries=off"});
CHECK(off.retryMode == Runner::RetryMode::kOff);
CHECK(off.retries == 0);
const auto random = Parse({"--seed=5", "--seed=random"});
CHECK_FALSE(random.seedGiven);
CHECK_THAT(ParseError({"--repeat=0"}), ContainsSubstring("at least 1"));
CHECK_THAT(ParseError({"--fail-fast=0"}), ContainsSubstring("at least 1"));
CHECK_THAT(ParseError({"--slowest=0"}), ContainsSubstring("at least 1"));
CHECK_THAT(ParseError({"--retries=-1"}), ContainsSubstring("whole number"));
CHECK_THAT(ParseError({"--timeout=soon"}), ContainsSubstring("seconds"));
CHECK_THAT(ParseError({"--timeout=-2"}), ContainsSubstring("seconds"));
CHECK_THAT(ParseError({"--benchmark-confidence-interval=1"}),
ContainsSubstring("--benchmark-confidence-interval"));
CHECK_THAT(ParseError({"--benchmark-samples=0"}),
ContainsSubstring("at least 1"));
CHECK_THAT(ParseError({"--seed=-1"}), ContainsSubstring("--seed"));
CHECK_THAT(ParseError({"--generator-index=x"}),
ContainsSubstring("index or *"));
CHECK_THAT(ParseError({"--path-filter=g:x"}),
ContainsSubstring("generator index or *"));
}
TEST_CASE("Shards are an index below a count", "[progmasoft][runner][options]")
{
const auto last = Parse({"--shard=3/4"});
CHECK(last.shard.index == 3);
CHECK(last.shard.count == 4);
const std::string wrong = GENERATE("4/4", "1", "/2", "1/", "0/0", "a/b");
CAPTURE(wrong);
CHECK_THAT(ParseError({"--shard=" + wrong}), ContainsSubstring("INDEX/COUNT"));
}
TEST_CASE("Warnings are named", "[progmasoft][runner][options]")
{
CHECK(Parse({"--warn=no-assertions"}).warnNoAssertions);
CHECK(Parse({"-w", "NoAssertions"}).warnNoAssertions);
CHECK(Parse({"--warn=unmatched-specs"}).warnUnmatchedSpecs);
CHECK(Parse({"--warn=infinite-generators"}).warnInfiniteGenerators);
CHECK_THAT(ParseError({"--warn=everything"}),
ContainsSubstring("no-assertions"));
}
TEST_CASE("Files and directories are taken as written",
"[progmasoft][runner][options]")
{
const auto options = Parse({"--junit=out/junit.xml",
"--json",
"out/run.json",
"--trace=out/trace.json",
"--history=cache",
"--shard-durations=last.json",
"--changes-since=main",
"--premature-exit-guard-file=guard"});
CHECK(options.junitPath == "out/junit.xml");
CHECK(options.jsonPath == "out/run.json");
CHECK(options.tracePath == "out/trace.json");
CHECK(options.historyDirectory == "cache");
CHECK(options.historyEnabled);
CHECK(options.shardDurations == "last.json");
CHECK(options.changesSince == "main");
CHECK(options.exitGuardPath == "guard");
}
TEST_CASE("Test specs are kept apart from options",
"[progmasoft][runner][options]")
{
const auto options =
Parse({"name", "[tag]", "~[slow]", "-", "--", "-j", "--x"});
CHECK(options.jobs == 0);
CHECK(options.testSpecs ==
std::vector<std::string>{"name", "[tag]", "~[slow]", "-", "-j", "--x"});
}
TEST_CASE("An input file names tests exactly", "[progmasoft][runner][options]")
{
std::error_code error;
const auto file = std::filesystem::temp_directory_path(error) /
("catch3-input-" + Runner::UniqueName());
std::ofstream(file) << "# a comment\n"
"first test\r\n"
"\n"
" \"quoted, with a comma\" \n"
"last*\n";
const auto options = Parse({"--input-file=" + file.string()});
CHECK(options.testSpecs == std::vector<std::string>{"\"first test\"",
",",
"\"quoted, with a comma\"",
",",
"\"last*\""});
std::filesystem::remove(file, error);
CHECK_THAT(ParseError({"-f", file.string()}), ContainsSubstring("cannot read"));
}
TEST_CASE("Unknown options are refused with a suggestion",
"[progmasoft][runner][options]")
{
CHECK_THAT(ParseError({"--sucess"}), ContainsSubstring("did you mean --success?"));
CHECK_THAT(ParseError({"--histroy=x"}), ContainsSubstring("did you mean --history?"));
CHECK_THAT(ParseError({"-z"}), ContainsSubstring("'-z' is not an option"));
// Catch2's reporter options are gone with its reporters.
CHECK_THAT(ParseError({"--reporter=junit"}), ContainsSubstring("is not an option"));
CHECK_THAT(ParseError({"--isolate=yes"}), ContainsSubstring("takes no value"));
CHECK_THAT(ParseError({"--junit"}), ContainsSubstring("--junit=FILE"));
}
TEST_CASE("The help lists every option", "[progmasoft][runner][options]")
{
const std::string help = Runner::HelpText("tests");
CHECK_THAT(help, ContainsSubstring("usage: tests [options] [test specs...]"));
for (const char *name : {"--section", "--generator-index",
"--path-filter", "--input-file",
"--select", "--changes-since",
"--incremental", "--shard",
"--shard-durations", "--filenames-as-tags",
"--jobs", "--in-process",
"--isolate", "--timeout",
"--no-adaptive-timeout", "--retries",
"--repeat", "--fail-fast",
"--order", "--seed",
"--no-bisect", "--success",
"--break", "--no-throw",
"--warn", "--allow-no-tests",
"--skip-benchmarks", "--benchmark-samples",
"--benchmark-resamples",
"--benchmark-confidence-interval",
"--benchmark-no-analysis",
"--benchmark-warmup-time",
"--history", "--no-history",
"--quarantine", "--replay",
"--analytics", "--import",
"--import-format", "--import-name",
"--import-revision", "--verbose",
"--quiet", "--color",
"--junit", "--json",
"--trace", "--premature-exit-guard-file",
"--slowest", "--list",
"--list-tags", "--help",
"--version"})
{
CAPTURE(name);
CHECK_THAT(help, ContainsSubstring(name));
}
// The worker option is internal.
CHECK_THAT(help, !ContainsSubstring("--catch3-worker"));
}
TEST_CASE("A worker gets every option that changes how tests behave",
"[progmasoft][runner][options]")
{
const auto options = Parse({"--section=outer",
"--section=inner",
"-p",
"g:1",
"-#",
"-s",
"-e",
"--warn=no-assertions",
"--warn=infinite-generators",
"--skip-benchmarks",
"--benchmark-no-analysis",
"--benchmark-samples=3",
"--benchmark-resamples=4",
"--benchmark-warmup-time=5",
"--benchmark-confidence-interval=0.9",
"--seed=6",
// Runner-side only:
"--jobs=9",
"--junit=x.xml",
"[tag]"});
const auto worker = Parse(Runner::WorkerArguments(options));
CHECK(worker.mode == Runner::Mode::kWorker);
CHECK(worker.sections == options.sections);
CHECK(worker.paths == options.paths);
CHECK(worker.filenamesAsTags);
CHECK(worker.includeSuccessful);
CHECK(worker.noThrow);
CHECK(worker.warnNoAssertions);
CHECK(worker.warnInfiniteGenerators);
CHECK(worker.skipBenchmarks);
CHECK(worker.benchmarkNoAnalysis);
CHECK(worker.benchmarkSamples == 3);
CHECK(worker.benchmarkResamples == 4);
CHECK(worker.benchmarkWarmupTime == milliseconds{5});
CHECK(worker.benchmarkConfidenceInterval == 0.9);
CHECK(worker.seed == 6);
CHECK(worker.jobs == 0);
CHECK(worker.junitPath.empty());
CHECK(worker.testSpecs.empty());
}