diff options
| author | Leitwolf | 2026-10-24 19:03:40 +0300 |
|---|---|---|
| committer | GitHub | 2026-10-24 19:03:40 +0300 |
| commit | 4f1c759cffcb14bd6511e1d7c5fbace682e24f2d (patch) | |
| tree | 415fe5d48a5b48650bba3747beaf245d727c4f61 /src/Progmasoft/Runner/Options.cpp | |
| parent | 83c6fc3ac13abe83fa39f7b5f8559ccaf4412233 (diff) | |
| parent | 622188f9c08e4af35941c146979883ed4882410e (diff) | |
| download | catch3-4f1c759cffcb14bd6511e1d7c5fbace682e24f2d.tar.gz | |
Merge pull request #15 from Progmasoft/feature/catch3-runner
Replace Catch2's runner with the Catch3 runner
Diffstat (limited to 'src/Progmasoft/Runner/Options.cpp')
| -rw-r--r-- | src/Progmasoft/Runner/Options.cpp | 1143 |
1 files changed, 1143 insertions, 0 deletions
diff --git a/src/Progmasoft/Runner/Options.cpp b/src/Progmasoft/Runner/Options.cpp new file mode 100644 index 00000000..bb36ab36 --- /dev/null +++ b/src/Progmasoft/Runner/Options.cpp @@ -0,0 +1,1143 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#include "Options.hpp" + +#include <algorithm> +#include <clocale> +#include <cmath> +#include <cstdio> +#include <cstdlib> +#include <fstream> +#include <functional> +#include <limits> +#include <string_view> + +namespace Progmasoft::Catch3::Runner +{ + namespace + { + enum class Takes : std::uint8_t + { + kNothing, + kValue, + kOptionalValue, + }; + + using Apply = + std::function<Status(Options &, const std::optional<std::string> &)>; + + /// One option: its spellings, what it takes and what it does. + struct OptionSpec final + { + std::string_view name; + char shortName = '\0'; + Takes takes = Takes::kNothing; + std::string_view valueName; + std::string_view group; + std::string_view help; + Apply apply; + }; + + std::optional<unsigned long long> + ParseUnsigned(std::string_view text) + { + if (text.empty() || text.size() > 19) + { + return std::nullopt; + } + unsigned long long value = 0; + for (const char character : text) + { + if (character < '0' || character > '9') + { + return std::nullopt; + } + value = value * 10 + static_cast<unsigned>(character - '0'); + } + return value; + } + + std::optional<double> + ParseDecimal(std::string_view text) + { + // Digits with at most one point: the format the runner writes. + bool point = false; + bool digit = false; + for (const char character : text) + { + if (character == '.' && !point) + { + point = true; + continue; + } + if (character < '0' || character > '9') + { + return std::nullopt; + } + digit = true; + } + if (!digit || text.size() > 64) + { + return std::nullopt; + } + // strtod rounds correctly, in the decimal separator of the + // current locale, which a global constructor may have changed. + std::string copy{text}; + const auto at = copy.find('.'); + if (at != std::string::npos) + { + const char *separator = std::localeconv()->decimal_point; + copy.replace(at, + 1, + separator == nullptr || *separator == '\0' + ? "." + : separator); + } + return std::strtod(copy.c_str(), nullptr); + } + + Status + ReadCount(const std::optional<std::string> &value, + std::string_view option, + unsigned &target, + unsigned minimum = 0) + { + const auto parsed = ParseUnsigned(value.value_or("")); + if (!parsed || *parsed > std::numeric_limits<unsigned>::max() || + *parsed < minimum) + { + return Error{"--" + std::string{option} + + " takes a whole number" + + (minimum > 0 ? " of at least " + + std::to_string(minimum) + : std::string{}) + + ", not '" + value.value_or("") + "'"}; + } + target = static_cast<unsigned>(*parsed); + return Status::Ok(); + } + + template<typename Value> + Status + ReadChoice(const std::optional<std::string> &value, + std::string_view option, + std::initializer_list<std::pair<std::string_view, Value>> + choices, + Value &target) + { + std::string names; + for (const auto &[name, choice] : choices) + { + if (value && *value == name) + { + target = choice; + return Status::Ok(); + } + names += names.empty() ? "" : ", "; + names += name; + } + return Error{"--" + std::string{option} + " takes one of " + + names + ", not '" + value.value_or("") + "'"}; + } + + std::size_t + EditDistance(std::string_view left, std::string_view right) + { + std::vector<std::size_t> previous(right.size() + 1); + std::vector<std::size_t> current(right.size() + 1); + for (std::size_t index = 0; index <= right.size(); ++index) + { + previous[index] = index; + } + for (std::size_t row = 1; row <= left.size(); ++row) + { + current[0] = row; + for (std::size_t column = 1; column <= right.size(); ++column) + { + const std::size_t substitution = + previous[column - 1] + + (left[row - 1] == right[column - 1] ? 0 : 1); + current[column] = std::min({previous[column] + 1, + current[column - 1] + 1, + substitution}); + } + std::swap(previous, current); + } + return previous[right.size()]; + } + + const std::vector<OptionSpec> & + Specs() + { + static const auto *specs = new std::vector<OptionSpec>{ + // Selecting tests. + {"section", + 'c', + Takes::kValue, + "NAME", + "Selecting tests", + "run only this section; repeat for nested sections", + [](Options &options, const std::optional<std::string> &value) { + options.sections.push_back(*value); + return Status::Ok(); + }}, + {"generator-index", + 'g', + Takes::kValue, + "INDEX", + "Selecting tests", + "run one value of the next generator, or * for all; the same " + "as --path-filter=g:INDEX", + [](Options &options, const std::optional<std::string> &value) { + if (*value != "*" && !ParseUnsigned(*value)) + { + return Status{Error{"--generator-index takes an index " + "or *, not '" + + *value + "'"}}; + } + options.paths.push_back("g:" + *value); + return Status::Ok(); + }}, + {"path-filter", + 'p', + Takes::kValue, + "c:SECTION|g:INDEX", + "Selecting tests", + "run one path through sections and generators; repeat for " + "each level", + [](Options &options, const std::optional<std::string> &value) { + const bool section = value->rfind("c:", 0) == 0; + const bool generator = value->rfind("g:", 0) == 0; + if (value->size() < 3 || (!section && !generator)) + { + return Status{Error{ + "--path-filter takes c:SECTION or g:INDEX, not '" + + *value + "'"}}; + } + if (generator && value->substr(2) != "*" && + !ParseUnsigned(value->substr(2))) + { + return Status{Error{"--path-filter g: takes a generator " + "index or *, not '" + + value->substr(2) + "'"}}; + } + options.paths.push_back(*value); + return Status::Ok(); + }}, + {"input-file", + 'f', + Takes::kValue, + "FILE", + "Selecting tests", + "run the tests named in FILE, one exact name per line; " + "lines starting with # are comments", + [](Options &options, const std::optional<std::string> &value) { + std::ifstream file(*value); + if (!file) + { + return Status{Error{"cannot read '" + *value + "'"}}; + } + std::string line; + bool first = true; + while (std::getline(file, line)) + { + while (!line.empty() && + (line.back() == '\r' || line.back() == ' ' || + line.back() == '\t')) + { + line.pop_back(); + } + const auto start = line.find_first_not_of(" \t"); + line.erase(0, start == std::string::npos ? line.size() + : start); + if (line.empty() || line.front() == '#') + { + continue; + } + // Quoted, a line is one exact test name; a comma + // between specs means "or". + if (line.front() != '"') + { + line = '"' + line + '"'; + } + if (!first) + { + options.testSpecs.emplace_back(","); + } + options.testSpecs.push_back(line); + first = false; + } + return Status::Ok(); + }}, + {"select", + '\0', + Takes::kValue, + "WHICH", + "Selecting tests", + "all (default), failed, changed, new or smart", + [](Options &options, const std::optional<std::string> &value) { + return ReadChoice<Selection>( + value, + "select", + {{"all", Selection::kAll}, + {"failed", Selection::kFailed}, + {"changed", Selection::kChanged}, + {"new", Selection::kNew}, + {"smart", Selection::kSmart}}, + options.selection); + }}, + {"changes-since", + '\0', + Takes::kValue, + "REVISION", + "Selecting tests", + "the Git revision changes are counted from (default HEAD)", + [](Options &options, const std::optional<std::string> &value) { + options.changesSince = *value; + return Status::Ok(); + }}, + {"incremental", + '\0', + Takes::kNothing, + "", + "Selecting tests", + "skip tests that passed last time and whose inputs are " + "unchanged", + [](Options &options, const std::optional<std::string> &) { + options.incremental = true; + return Status::Ok(); + }}, + {"shard", + '\0', + Takes::kValue, + "INDEX/COUNT", + "Selecting tests", + "run one of COUNT shards, counting from 0", + [](Options &options, const std::optional<std::string> &value) { + const auto slash = value->find('/'); + const auto index = + ParseUnsigned(std::string_view{*value}.substr( + 0, + slash == std::string::npos ? 0 : slash)); + const auto count = slash == std::string::npos + ? std::nullopt + : ParseUnsigned( + std::string_view{*value} + .substr(slash + 1)); + if (!index || !count || *count == 0 || *index >= *count || + *count > std::numeric_limits<unsigned>::max()) + { + return Status{Error{ + "--shard takes INDEX/COUNT with INDEX below " + "COUNT, such as 0/4, not '" + + *value + "'"}}; + } + options.shard.index = static_cast<unsigned>(*index); + options.shard.count = static_cast<unsigned>(*count); + return Status::Ok(); + }}, + {"shard-durations", + '\0', + Takes::kValue, + "FILE", + "Selecting tests", + "balance shards by the durations in a JSON report or history", + [](Options &options, const std::optional<std::string> &value) { + options.shardDurations = *value; + return Status::Ok(); + }}, + {"filenames-as-tags", + '#', + Takes::kNothing, + "", + "Selecting tests", + "tag every test with the name of its file", + [](Options &options, const std::optional<std::string> &) { + options.filenamesAsTags = true; + return Status::Ok(); + }}, + // Running. + {"jobs", + 'j', + Takes::kValue, + "N", + "Running", + "worker processes; 0, the default, uses one per hardware " + "thread", + [](Options &options, const std::optional<std::string> &value) { + return ReadCount(value, "jobs", options.jobs); + }}, + {"in-process", + '\0', + Takes::kNothing, + "", + "Running", + "run the tests in this process, one after another", + [](Options &options, const std::optional<std::string> &) { + options.inProcess = true; + return Status::Ok(); + }}, + {"isolate", + '\0', + Takes::kNothing, + "", + "Running", + "start a fresh process for every test", + [](Options &options, const std::optional<std::string> &) { + options.isolate = true; + return Status::Ok(); + }}, + {"timeout", + '\0', + Takes::kValue, + "SECONDS", + "Running", + "the time limit of one test; 0 for none (default: learned " + "from history, 300 for an unknown test)", + [](Options &options, const std::optional<std::string> &value) { + const auto seconds = ParseDecimal(*value); + if (!seconds || *seconds > 1e7) + { + return Status{Error{"--timeout takes seconds, such " + "as 30 or 0.5, not '" + + *value + "'"}}; + } + options.timeout = std::chrono::milliseconds{ + static_cast<long long>(std::llround(*seconds * 1000))}; + options.timeoutGiven = true; + return Status::Ok(); + }}, + {"no-adaptive-timeout", + '\0', + Takes::kNothing, + "", + "Running", + "use --timeout for every test instead of learning limits", + [](Options &options, const std::optional<std::string> &) { + options.adaptiveTimeout = false; + return Status::Ok(); + }}, + {"retries", + '\0', + Takes::kValue, + "N|auto|off", + "Running", + "retry failures: auto (default) retries up to 2 times what " + "may pass again, N retries every failure", + [](Options &options, const std::optional<std::string> &value) { + if (*value == "auto") + { + options.retryMode = RetryMode::kAuto; + return Status::Ok(); + } + if (*value == "off") + { + options.retryMode = RetryMode::kOff; + options.retries = 0; + return Status::Ok(); + } + options.retryMode = RetryMode::kAlways; + return ReadCount(value, "retries", options.retries); + }}, + {"repeat", + '\0', + Takes::kValue, + "N", + "Running", + "run every test N times, to find flaky tests", + [](Options &options, const std::optional<std::string> &value) { + return ReadCount(value, "repeat", options.repeat, 1); + }}, + {"fail-fast", + 'a', + Takes::kOptionalValue, + "N", + "Running", + "stop starting tests after N failed tests (default 1)", + [](Options &options, const std::optional<std::string> &value) { + if (!value) + { + options.failFast = 1; + return Status::Ok(); + } + return ReadCount(value, "fail-fast", options.failFast, 1); + }}, + {"order", + '\0', + Takes::kValue, + "ORDER", + "Running", + "priority (default), declared, name or random", + [](Options &options, const std::optional<std::string> &value) { + return ReadChoice<Order>( + value, + "order", + {{"priority", Order::kPriority}, + {"declared", Order::kDeclared}, + {"name", Order::kName}, + {"random", Order::kRandom}}, + options.order); + }}, + {"seed", + '\0', + Takes::kValue, + "N", + "Running", + "the random seed of the run (default: a new one each run)", + [](Options &options, const std::optional<std::string> &value) { + if (*value == "random") + { + options.seedGiven = false; + return Status::Ok(); + } + const auto seed = ParseUnsigned(*value); + if (!seed || *seed > 0xFFFFFFFFULL) + { + return Status{Error{ + "--seed takes a number below 2^32 or 'random', " + "not '" + + *value + "'"}}; + } + options.seed = static_cast<std::uint32_t>(*seed); + options.seedGiven = true; + return Status::Ok(); + }}, + {"no-bisect", + '\0', + Takes::kNothing, + "", + "Running", + "do not search for the test that leaves state behind when a " + "failure depends on the order", + [](Options &options, const std::optional<std::string> &) { + options.bisectOrder = false; + return Status::Ok(); + }}, + // How tests behave. + {"success", + 's', + Takes::kNothing, + "", + "Test behaviour", + "report passing assertions too", + [](Options &options, const std::optional<std::string> &) { + options.includeSuccessful = true; + return Status::Ok(); + }}, + {"break", + 'b', + Takes::kNothing, + "", + "Test behaviour", + "break into the debugger on a failure; runs in process", + [](Options &options, const std::optional<std::string> &) { + options.breakIntoDebugger = true; + options.inProcess = true; + return Status::Ok(); + }}, + {"no-throw", + 'e', + Takes::kNothing, + "", + "Test behaviour", + "skip the assertions that expect exceptions", + [](Options &options, const std::optional<std::string> &) { + options.noThrow = true; + return Status::Ok(); + }}, + {"warn", + 'w', + Takes::kValue, + "WHAT", + "Test behaviour", + "fail on: no-assertions, unmatched-specs or " + "infinite-generators", + [](Options &options, const std::optional<std::string> &value) { + if (*value == "no-assertions" || *value == "NoAssertions") + { + options.warnNoAssertions = true; + } + else if (*value == "unmatched-specs" || + *value == "UnmatchedTestSpec") + { + options.warnUnmatchedSpecs = true; + } + else if (*value == "infinite-generators" || + *value == "InfiniteGenerators") + { + options.warnInfiniteGenerators = true; + } + else + { + return Status{Error{ + "--warn takes no-assertions, unmatched-specs or " + "infinite-generators, not '" + + *value + "'"}}; + } + return Status::Ok(); + }}, + {"allow-no-tests", + '\0', + Takes::kNothing, + "", + "Test behaviour", + "succeed when no test runs", + [](Options &options, const std::optional<std::string> &) { + options.allowNoTests = true; + return Status::Ok(); + }}, + {"skip-benchmarks", + '\0', + Takes::kNothing, + "", + "Test behaviour", + "do not run BENCHMARK blocks", + [](Options &options, const std::optional<std::string> &) { + options.skipBenchmarks = true; + return Status::Ok(); + }}, + {"benchmark-samples", + '\0', + Takes::kValue, + "N", + "Test behaviour", + "samples per benchmark (default 100)", + [](Options &options, const std::optional<std::string> &value) { + return ReadCount( + value, "benchmark-samples", options.benchmarkSamples, 1); + }}, + {"benchmark-resamples", + '\0', + Takes::kValue, + "N", + "Test behaviour", + "bootstrap resamples (default 100000)", + [](Options &options, const std::optional<std::string> &value) { + return ReadCount(value, + "benchmark-resamples", + options.benchmarkResamples, + 1); + }}, + {"benchmark-confidence-interval", + '\0', + Takes::kValue, + "X", + "Test behaviour", + "between 0 and 1 (default 0.95)", + [](Options &options, const std::optional<std::string> &value) { + const auto interval = ParseDecimal(*value); + if (!interval || *interval <= 0.0 || *interval >= 1.0) + { + return Status{ + Error{"--benchmark-confidence-interval takes a " + "number between 0 and 1, not '" + + *value + "'"}}; + } + options.benchmarkConfidenceInterval = *interval; + return Status::Ok(); + }}, + {"benchmark-no-analysis", + '\0', + Takes::kNothing, + "", + "Test behaviour", + "report the mean of the samples without statistics", + [](Options &options, const std::optional<std::string> &) { + options.benchmarkNoAnalysis = true; + return Status::Ok(); + }}, + {"benchmark-warmup-time", + '\0', + Takes::kValue, + "MILLISECONDS", + "Test behaviour", + "warm-up time per benchmark (default 100)", + [](Options &options, const std::optional<std::string> &value) { + unsigned milliseconds = 0; + auto status = + ReadCount(value, "benchmark-warmup-time", milliseconds); + options.benchmarkWarmupTime = + std::chrono::milliseconds{milliseconds}; + return status; + }}, + // History and analysis. + {"history", + '\0', + Takes::kValue, + "DIRECTORY", + "History and analysis", + "where results are remembered (default: .catch3 beside the " + "binary, or CATCH3_HISTORY)", + [](Options &options, const std::optional<std::string> &value) { + options.historyDirectory = *value; + options.historyEnabled = true; + return Status::Ok(); + }}, + {"no-history", + '\0', + Takes::kNothing, + "", + "History and analysis", + "neither read nor write the history", + [](Options &options, const std::optional<std::string> &) { + options.historyEnabled = false; + return Status::Ok(); + }}, + {"quarantine", + '\0', + Takes::kValue, + "MODE", + "History and analysis", + "auto (default), report or off", + [](Options &options, const std::optional<std::string> &value) { + return ReadChoice<QuarantineMode>( + value, + "quarantine", + {{"auto", QuarantineMode::kAuto}, + {"report", QuarantineMode::kReport}, + {"off", QuarantineMode::kOff}}, + options.quarantine); + }}, + {"replay", + '\0', + Takes::kNothing, + "", + "History and analysis", + "run last run's failures again, each in the order and with " + "the seed it failed with", + [](Options &options, const std::optional<std::string> &) { + options.replayFailures = true; + return Status::Ok(); + }}, + {"analytics", + '\0', + Takes::kOptionalValue, + "json", + "History and analysis", + "print what the history says about the tests and exit", + [](Options &options, const std::optional<std::string> &value) { + if (value && *value != "json") + { + return Status{Error{ + "--analytics takes no value or 'json', not '" + + *value + "'"}}; + } + options.mode = Mode::kAnalytics; + options.analyticsJson = value.has_value(); + return Status::Ok(); + }}, + // Output. + {"verbose", + 'v', + Takes::kNothing, + "", + "Output", + "print every test as it ends", + [](Options &options, const std::optional<std::string> &) { + options.verbosity = Verbosity::kVerbose; + return Status::Ok(); + }}, + {"quiet", + 'q', + Takes::kNothing, + "", + "Output", + "print only failures and the summary", + [](Options &options, const std::optional<std::string> &) { + options.verbosity = Verbosity::kQuiet; + return Status::Ok(); + }}, + {"color", + '\0', + Takes::kValue, + "WHEN", + "Output", + "auto (default), always or never", + [](Options &options, const std::optional<std::string> &value) { + return ReadChoice<ColorMode>( + value, + "color", + {{"auto", ColorMode::kAuto}, + {"always", ColorMode::kAlways}, + {"never", ColorMode::kNever}}, + options.color); + }}, + {"junit", + '\0', + Takes::kValue, + "FILE", + "Output", + "write a JUnit XML report", + [](Options &options, const std::optional<std::string> &value) { + options.junitPath = *value; + return Status::Ok(); + }}, + {"json", + '\0', + Takes::kValue, + "FILE", + "Output", + "write a JSON report with every attempt and its analysis", + [](Options &options, const std::optional<std::string> &value) { + options.jsonPath = *value; + return Status::Ok(); + }}, + {"trace", + '\0', + Takes::kValue, + "FILE", + "Output", + "write a trace of the run for chrome://tracing or Perfetto", + [](Options &options, const std::optional<std::string> &value) { + options.tracePath = *value; + return Status::Ok(); + }}, + {"premature-exit-guard-file", + '\0', + Takes::kValue, + "FILE", + "Output", + "create FILE when the run starts and remove it when the run " + "ends normally", + [](Options &options, const std::optional<std::string> &value) { + options.exitGuardPath = *value; + return Status::Ok(); + }}, + {"slowest", + '\0', + Takes::kOptionalValue, + "N", + "Output", + "list the N slowest tests (default 10)", + [](Options &options, const std::optional<std::string> &value) { + if (!value) + { + options.slowest = 10; + return Status::Ok(); + } + return ReadCount(value, "slowest", options.slowest, 1); + }}, + // Information. + {"list", + 'l', + Takes::kOptionalValue, + "FORMAT", + "Information", + "list the matching tests: text (default), names or json", + [](Options &options, const std::optional<std::string> &value) { + options.mode = Mode::kListTests; + if (!value) + { + options.listFormat = ListFormat::kText; + return Status::Ok(); + } + return ReadChoice<ListFormat>( + value, + |