diff options
| author | Leitwolf | 2026-10-30 00:07:08 +0300 |
|---|---|---|
| committer | GitHub | 2026-10-30 00:07:08 +0300 |
| commit | 5d713685c54bb491ad1bbb0fd8f80318bfaf7103 (patch) | |
| tree | fed3945312bd75d050c9c3163584b1ae6fcb1b93 | |
| parent | db9f14bb7f38425e6005ae2212d83f01f9f2f8be (diff) | |
| parent | 471756b77331fa1ab77bc94e6aae228baef903e0 (diff) | |
| download | catch3-ci/narrow-macos.tar.gz | |
Merge branch 'devel' into ci/narrow-macosci/narrow-macos
| -rw-r--r-- | docs/command-line.md | 12 | ||||
| -rw-r--r-- | docs/progmasoft-runner.md | 49 | ||||
| -rw-r--r-- | docs/release-notes.md | 4 | ||||
| -rw-r--r-- | extras/catch_amalgamated.cpp | 696 | ||||
| -rw-r--r-- | extras/catch_amalgamated.hpp | 2 | ||||
| -rw-r--r-- | extras/catch_amalgamated_third_party.hpp | 2 | ||||
| -rw-r--r-- | fuzzing/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | fuzzing/Progmasoft/fuzz_ProjectConfig.cpp | 30 | ||||
| -rw-r--r-- | premake5.lua | 1 | ||||
| -rw-r--r-- | src/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | src/Progmasoft/Catch3/meson.build | 1 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/Options.cpp | 78 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/Options.hpp | 19 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/ProjectConfig.cpp | 548 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/ProjectConfig.hpp | 44 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/Runner.cpp | 12 | ||||
| -rw-r--r-- | tests/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | tests/Progmasoft/BUILD.bazel | 1 | ||||
| -rw-r--r-- | tests/Progmasoft/RunnerConfig.tests.cpp | 425 | ||||
| -rw-r--r-- | tests/meson.build | 1 | ||||
| -rw-r--r-- | xmake.lua | 1 |
21 files changed, 1865 insertions, 66 deletions
diff --git a/docs/command-line.md b/docs/command-line.md index 29abc11a..4bd46f3b 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -8,6 +8,7 @@ [How tests behave](#how-tests-behave)<br> [History and analysis](#history-and-analysis)<br> [Output](#output)<br> +[Configuration](#configuration)<br> [Information](#information)<br> [Environment](#environment)<br> @@ -122,6 +123,17 @@ Without a spec, every test runs except hidden ones, whose tags start with | `--premature-exit-guard-file=FILE` | create FILE, private to the user, when the run starts and remove it when the run ends normally | +## Configuration + +A project can keep its options in a [`catch3.json`](progmasoft-runner.md#project-configuration); +options on the command line win over it. + +| Option | | +| --- | --- | +| `--config=FILE` | read this `catch3.json` instead of searching for one | +| `--no-config` | read no `catch3.json` | + + ## Information | Option | | diff --git a/docs/progmasoft-runner.md b/docs/progmasoft-runner.md index baf5521a..300c541e 100644 --- a/docs/progmasoft-runner.md +++ b/docs/progmasoft-runner.md @@ -12,6 +12,7 @@ [What a failure report says](#what-a-failure-report-says)<br> [Reports and traces](#reports-and-traces)<br> [Formats other tools read](#formats-other-tools-read)<br> +[Project configuration](#project-configuration)<br> [Bazel and CTest](#bazel-and-ctest)<br> [Exit codes](#exit-codes)<br> [Coming from Catch2's runner](#coming-from-catch2s-runner)<br> @@ -273,6 +274,54 @@ The messages between the runner and its workers are internal and change with the binary. +## Project configuration + +Options a project always wants go in a `catch3.json`: + +```json +{ + "version": 1, + "tests": ["~[slow]"], + "jobs": 8, + "timeout": 60, + "warn": ["no-assertions"], + "junit": "build/test-results/junit.xml" +} +``` + +The runner looks for the file in the working directory and in each +directory above it, up to the root of the Git repository, and reads the +nearest one. Outside a repository it looks only in the working directory. +`--config=FILE` names a file instead, and `--no-config` reads none. + +The keys are the long options, and their values go through the same checks +as on the command line, with the key named in the message: + +| Value | Keys | +| --- | --- | +| `true` turns the switch on | `incremental`, `filenames-as-tags`, `in-process`, `isolate`, `success`, `no-throw`, `allow-no-tests`, `skip-benchmarks`, `benchmark-no-analysis` | +| `false` turns it off | `adaptive-timeout`, `bisect` | +| a string | `select`, `changes-since`, `order`, `quarantine`, `color` | +| a whole number | `jobs`, `repeat`, `fail-fast`, `slowest`, `benchmark-samples`, `benchmark-resamples`, `benchmark-warmup-time` | +| a whole number or a string | `retries` (`"auto"`, `"off"`), `seed` (`"random"`) | +| a number | `timeout` (seconds), `benchmark-confidence-interval` | +| a path, relative to the file | `junit`, `json`, `trace`, `shard-durations`; `history`, or `false` for none | +| an array of strings | `tests` (test specs), `warn` | +| `"quiet"`, `"normal"` or `"verbose"` | `verbosity` | + +`"version": 1` is required; `"$schema"` may name a schema for editors. +The file is strict JSON: no comments and no trailing commas. A key it does +not know is an error, with a suggestion when it looks like a known one. +Options that describe one run, such as `--shard`, `--section`, `--list` or +`--replay`, belong on the command line only. + +Options on the command line win over the file, and test specs on the +command line replace the file's `tests` rather than adding to them. A +switch the file turns on stays on; `--no-config` leaves the file out. +`--verbose` names the file that was read. Workers, `--help` and +`--version` read no file. + + ## Bazel and CTest Under `bazel test` the runner reads Bazel's environment: `XML_OUTPUT_FILE` diff --git a/docs/release-notes.md b/docs/release-notes.md index aef49cf0..b18ee5be 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -33,6 +33,10 @@ line and reporters with [a runner of its own](progmasoft-runner.md#top): * Reports are console, JUnit XML, JSON and a Chrome trace; `--list=json` is a versioned discovery format. * Every test gets a temporary directory of its own. +* A project keeps its options in a + [`catch3.json`](progmasoft-runner.md#project-configuration), found up to + the root of the repository; the command line wins over it, and + `--config=FILE` and `--no-config` choose another file or none. * JSON is read and written with [yyjson](https://github.com/ibireme/yyjson) and JUnit XML is written with [pugixml](https://github.com/zeux/pugixml), both MIT and vendored in `third_party/`. They are compiled into Catch3's diff --git a/extras/catch_amalgamated.cpp b/extras/catch_amalgamated.cpp index a9cdb04b..fc87bf40 100644 --- a/extras/catch_amalgamated.cpp +++ b/extras/catch_amalgamated.cpp @@ -7,7 +7,7 @@ // SPDX-License-Identifier: BSL-1.0 // Catch3 v0.3.0, on the Catch2 v3.16.1 engine -// Generated: 2026-10-25 13:52:54.556111 +// Generated: 2026-10-29 19:30:41.305371 // ---------------------------------------------------------- // This file is an amalgamation of multiple different files. // You probably shouldn't edit it directly. @@ -950,8 +950,10 @@ namespace Progmasoft::Catch3::Runner #include <chrono> #include <cstdint> +#include <cstddef> #include <optional> #include <string> +#include <string_view> #include <vector> namespace Progmasoft::Catch3::Runner @@ -1109,11 +1111,24 @@ namespace Progmasoft::Catch3::Runner /// Created when the run starts and removed when it ends normally, so /// a file left behind shows that the run ended early. std::string exitGuardPath; + + // Configuration. + /// The `catch3.json` named by --config; after ParseCommandLine, the + /// one that was read, if any. + std::string configPath; + bool noConfig = false; }; - /// Reads a command line. `arguments` does not include the program. + /// Reads a command line on top of `options`. `arguments` does not include + /// the program. [[nodiscard]] Expected<Options> - ParseOptions(const std::vector<std::string> &arguments); + ParseOptions(const std::vector<std::string> &arguments, + Options options = {}); + + /// How many single-character edits turn `left` into `right`, for + /// suggesting the name that was meant. + [[nodiscard]] std::size_t + EditDistance(std::string_view left, std::string_view right); /// The help text, for `--help`. [[nodiscard]] std::string @@ -1727,6 +1742,47 @@ namespace Progmasoft::Catch3::Runner // Copyright (c) 2026 Progmasoft. // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#include <filesystem> +#include <optional> +#include <string> +#include <string_view> +#include <vector> + + +namespace Progmasoft::Catch3::Runner +{ + /// The name of the project configuration file. + inline constexpr std::string_view kProjectConfigName = "catch3.json"; + + /// The `catch3.json` that applies in `start`: the nearest one in `start` + /// or a directory above it, up to the root of the Git repository that + /// holds `start`. Outside a repository only `start` itself is searched, + /// so a stray file in a home directory never applies. + [[nodiscard]] std::optional<std::filesystem::path> + FindProjectConfig(const std::filesystem::path &start); + + /// Applies the text of a `catch3.json` to `options`. Its keys are the long + /// options of the command line, and its values go through the same checks; + /// relative paths are read from `directory`, the file's own. `file` names + /// the file in messages. + [[nodiscard]] Expected<Options> + ApplyProjectConfig(std::string_view text, + const std::filesystem::path &directory, + std::string_view file, + Options options = {}); + + /// Reads the command line together with the `catch3.json` it selects: + /// --config names one, --no-config reads none, and otherwise + /// FindProjectConfig searches from `workingDirectory`. Options on the + /// command line win over the file, and test specs on the command line + /// replace those of the file. Workers, --help and --version read no file. + [[nodiscard]] Expected<Options> + ParseCommandLine(const std::vector<std::string> &arguments, + const std::filesystem::path &workingDirectory); +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 + #include <mutex> #include <optional> @@ -12776,32 +12832,6 @@ namespace Progmasoft::Catch3::Runner names + ", not '" + value.value_or("") + "'"}; } - std::size_t - EditDistance(std::string_view left, std::string_view right) - { - std::vector<std::size_t> previous(right.size() + 1); - std::vector<std::size_t> current(right.size() + 1); - for (std::size_t index = 0; index <= right.size(); ++index) - { - previous[index] = index; - } - for (std::size_t row = 1; row <= left.size(); ++row) - { - current[0] = row; - for (std::size_t column = 1; column <= right.size(); ++column) - { - const std::size_t substitution = - previous[column - 1] + - (left[row - 1] == right[column - 1] ? 0 : 1); - current[column] = std::min({previous[column] + 1, - current[column - 1] + 1, - substitution}); - } - std::swap(previous, current); - } - return previous[right.size()]; - } - const std::vector<OptionSpec> & Specs() { @@ -13453,6 +13483,29 @@ namespace Progmasoft::Catch3::Runner } return ReadCount(value, "slowest", options.slowest, 1); }}, + // Configuration. + {"config", + '\0', + Takes::kValue, + "FILE", + "Configuration", + "read this catch3.json instead of searching for one", + [](Options &options, const std::optional<std::string> &value) { + options.configPath = *value; + options.noConfig = false; + return Status::Ok(); + }}, + {"no-config", + '\0', + Takes::kNothing, + "", + "Configuration", + "read no catch3.json", + [](Options &options, const std::optional<std::string> &) { + options.noConfig = true; + options.configPath.clear(); + return Status::Ok(); + }}, // Information. {"list", 'l', @@ -13585,10 +13638,35 @@ namespace Progmasoft::Catch3::Runner } } // namespace + std::size_t + EditDistance(std::string_view left, std::string_view right) + { + std::vector<std::size_t> previous(right.size() + 1); + std::vector<std::size_t> current(right.size() + 1); + for (std::size_t index = 0; index <= right.size(); ++index) + { + previous[index] = index; + } + for (std::size_t row = 1; row <= left.size(); ++row) + { + current[0] = row; + for (std::size_t column = 1; column <= right.size(); ++column) + { + const std::size_t substitution = + previous[column - 1] + + (left[row - 1] == right[column - 1] ? 0 : 1); + current[column] = std::min({previous[column] + 1, + current[column - 1] + 1, + substitution}); + } + std::swap(previous, current); + } + return previous[right.size()]; + } + Expected<Options> - ParseOptions(const std::vector<std::string> &arguments) + ParseOptions(const std::vector<std::string> &arguments, Options options) { - Options options; bool onlySpecs = false; for (std::size_t index = 0; index < arguments.size(); ++index) { @@ -15683,6 +15761,551 @@ namespace Progmasoft::Catch3::Runner } // namespace Progmasoft::Catch3::Runner // Copyright (c) 2026 Progmasoft. // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#include <cmath> +#include <cstdint> +#include <cstdio> +#include <set> +#include <system_error> +#include <utility> + + +namespace Progmasoft::Catch3::Runner +{ + namespace + { + /// The version of the file format this runner reads. + constexpr std::uint64_t kConfigVersion = 1; + + /// What the value of a key is, and how it becomes arguments. + enum class ValueKind : std::uint8_t + { + /// true gives the option; false, the default, gives nothing. + kFlag, + /// false gives the negating option; true is the default. + kNegated, + /// A string, given as the value of the option. + kWord, + /// A file or directory, relative to the file's directory. + kPath, + /// A whole number. + kCount, + /// A whole number or a word, such as "auto". + kCountOrWord, + /// A number with a fraction, such as seconds. + kNumber, + /// An array of strings, each given to the option. + kWords, + /// A directory, or false for none. + kPathOrFalse, + /// "quiet", "normal" or "verbose". + kVerbosity, + /// An array of test specs. + kTests, + }; + + struct ConfigKey final + { + std::string_view name; + ValueKind kind; + /// The option it stands for, when it is not spelled like the key. + std::string_view option = {}; + }; + + const std::vector<ConfigKey> & + Keys() + { + static const auto *keys = new std::vector<ConfigKey>{ + { "tests", ValueKind::kTests }, + // Selecting tests. + { "select", ValueKind::kWord }, + { "changes-since", ValueKind::kWord }, + { "incremental", ValueKind::kFlag }, + { "shard-durations", ValueKind::kPath }, + { "filenames-as-tags", ValueKind::kFlag }, + // Running. + { "jobs", ValueKind::kCount }, + { "in-process", ValueKind::kFlag }, + { "isolate", ValueKind::kFlag }, + { "timeout", ValueKind::kNumber }, + { "adaptive-timeout", + ValueKind::kNegated, + "no-adaptive-timeout" }, + { "retries", ValueKind::kCountOrWord }, + { "repeat", ValueKind::kCount }, + { "fail-fast", ValueKind::kCount }, + { "order", ValueKind::kWord }, + { "seed", ValueKind::kCountOrWord }, + { "bisect", ValueKind::kNegated, "no-bisect" }, + // How tests behave. + { "success", ValueKind::kFlag }, + { "no-throw", ValueKind::kFlag }, + { "warn", ValueKind::kWords }, + { "allow-no-tests", ValueKind::kFlag }, + { "skip-benchmarks", ValueKind::kFlag }, + { "benchmark-samples", ValueKind::kCount }, + { "benchmark-resamples", ValueKind::kCount }, + { "benchmark-confidence-interval", ValueKind::kNumber }, + { "benchmark-no-analysis", ValueKind::kFlag }, + { "benchmark-warmup-time", ValueKind::kCount }, + // History and analysis. + { "history", ValueKind::kPathOrFalse }, + { "quarantine", ValueKind::kWord }, + // Output. + { "verbosity", ValueKind::kVerbosity }, + { "color", ValueKind::kWord }, + { "junit", ValueKind::kPath }, + { "json", ValueKind::kPath }, + { "trace", ValueKind::kPath }, + { "slowest", ValueKind::kCount }, + }; + return *keys; + } + + /// Options that describe one run rather than a project. + constexpr std::string_view kCommandLineOnly[] = { + "section", + "generator-index", + "path-filter", + "input-file", + "shard", + "break", + "replay", + "analytics", + "verbose", + "quiet", + "premature-exit-guard-file", + "list", + "list-tags", + "help", + "config", + "no-config", + "catch3-worker", + }; + + std::string + Quoted(std::string_view key) + { + return "\"" + std::string{ key } + "\""; + } + + std::string + UnknownKey(std::string_view key) + { + for (const std::string_view option : kCommandLineOnly) + { + if (key == option) + { + return Quoted(key) + + " belongs on the command line, not in a " + "catch3.json"; + } + } + std::string message = Quoted(key) + " is not a key of catch3.json"; + std::string_view closest; + std::size_t best = 3; + const auto consider = [&](std::string_view name) { + const std::size_t distance = EditDistance(key, name); + if (distance < best) + { + best = distance; + closest = name; + } + }; + consider("version"); + for (const auto &entry : Keys()) + { + consider(entry.name); + } + if (!closest.empty()) + { + message += "; did you mean " + Quoted(closest) + "?"; + } + return message; + } + + /// A whole number JSON can say exactly, or nothing. + std::optional<std::uint64_t> + WholeNumber(const JsonValue &value) + { + if (!value.IsNumber()) + { + return std::nullopt; + } + const double number = value.AsNumber(); + if (!std::isfinite(number) || number < 0 + || number > std::floor(number) || number > 9007199254740992.0) + { + return std::nullopt; + } + return static_cast<std::uint64_t>(number); + } + + /// A number in the form the command line reads: digits and a point. + std::string + DecimalText(double number) + { + char text[400]; + std::snprintf(text, sizeof(text), "%.17f", number); + std::string result{ text }; + for (char &character : result) + { + // The decimal separator of the locale a test may have set. + if (character == ',') + { + character = '.'; + } + } + return result; + } + + std::string + PathText(const std::string &value, + const std::filesystem::path &directory) + { + const std::filesystem::path path{ value }; + if (path.is_absolute() || directory.empty()) + { + return path.lexically_normal().string(); + } + return (directory / path).lexically_normal().string(); + } + + /// The arguments that one key and its value stand for. + Expected<std::vector<std::string>> + KeyArguments(const ConfigKey &key, + const JsonValue &value, + const std::filesystem::path &directory) + { + const std::string option + = "--" + + std::string{ key.option.empty() ? key.name : key.option }; + const auto wrong = [&](std::string_view expected) { + return Error{ Quoted(key.name) + " takes " + + std::string{ expected } + ", not " + + value.ToString() }; + }; + switch (key.kind) + { + case ValueKind::kFlag: + if (value.GetKind() != JsonValue::Kind::kBoolean) + { + return wrong("true or false"); + } + if (value.AsBoolean()) + { + return std::vector<std::string>{ option }; + } + return std::vector<std::string>{}; + case ValueKind::kNegated: + if (value.GetKind() != JsonValue::Kind::kBoolean) + { + return wrong("true or false"); + } + if (!value.AsBoolean()) + { + return std::vector<std::string>{ option }; + } + return std::vector<std::string>{}; + case ValueKind::kWord: + if (!value.IsString()) + { + return wrong("a string"); + } + return std::vector<std::string>{ option + "=" + + value.AsString() }; + case ValueKind::kPath: + if (!value.IsString() || value.AsString().empty()) + { + return wrong("a path"); + } + return std::vector<std::string>{ + option + "=" + PathText(value.AsString(), directory) + }; + case ValueKind::kCount: + if (const auto number = WholeNumber(value)) + { + return std::vector<std::string>{ + option + "=" + std::to_string(*number) + }; + } + return wrong("a whole number"); + case ValueKind::kCountOrWord: + if (value.IsString()) + { + return std::vector<std::string>{ option + "=" + + value.AsString() }; + } + if (const auto number = WholeNumber(value)) + { + return std::vector<std::string>{ + option + "=" + std::to_string(*number) + }; + } + return wrong("a whole number or a string"); + case ValueKind::kNumber: + if (!value.IsNumber() || !std::isfinite(value.AsNumber()) + || value.AsNumber() < 0) + { + return wrong("a number of at least 0"); + } + return std::vector<std::string>{ + option + "=" + DecimalText(value.AsNumber()) + }; + case ValueKind::kWords: + { + if (!value.IsArray()) + { + return wrong("an array of strings"); + } + std::vector<std::string> arguments; + for (const auto &item : value.Items()) + { + if (!item.IsString()) + { + return wrong("an array of strings"); + } + arguments.push_back(option + "=" + item.AsString()); + } + return arguments; + } + case ValueKind::kPathOrFalse: + if (value.GetKind() == JsonValue::Kind::kBoolean + && !value.AsBoolean()) + { + return std::vector<std::string>{ + "--no-" + std::string{ key.name } + }; + } + if (!value.IsString() || value.AsString().empty()) + { + return wrong("a path or false"); + } + return std::vector<std::string>{ + option + "=" + PathText(value.AsString(), directory) + }; + case ValueKind::kVerbosity: + if (value.IsString()) + { + if (value.AsString() == "quiet") + { + return std::vector<std::string>{ "--quiet" }; + } + if (value.AsString() == "verbose") + { + return std::vector<std::string>{ "--verbose" }; + } + if (value.AsString() == "normal") + { + return std::vector<std::string>{}; + } + } + return wrong("\"quiet\", \"normal\" or \"verbose\""); + case ValueKind::kTests: + { + if (!value.IsArray()) + { + return wrong("an array of test specs"); + } + // After --, every argument is a test spec, even one that + // starts with a dash. + std::vector<std::string> arguments{ "--" }; + for (const auto &item : value.Items()) + { + if (!item.IsString() || item.AsString().empty()) + { + return wrong("an array of test specs"); + } + arguments.push_back(item.AsString()); + } + return arguments; + } + } + return wrong("a known kind of value"); + } + } // namespace + + std::optional<std::filesystem::path> + FindProjectConfig(const std::filesystem::path &start) + { + std::error_code error; + std::filesystem::path directory + = std::filesystem::absolute(start, error).lexically_normal(); + if (error) + { + return std::nullopt; + } + if (directory.filename().empty() && directory.has_relative_path()) + { + directory = directory.parent_path(); + } + std::vector<std::filesystem::path> candidates; + for (std::filesystem::path current = directory;; + current = current.parent_path()) + { + candidates.push_back(current); + if (std::filesystem::exists(current / ".git", error)) + { + break; + } + if (!current.has_relative_path()) + { + // No repository holds the directory. + candidates = { directory }; + break; + } + } + for (const auto &candidate : candidates) + { + const auto file = candidate / kProjectConfigName; + if (std::filesystem::is_regular_file(file, error)) + { + return file; + } + } + return std::nullopt; + } + + Expected<Options> + ApplyProjectConfig(std::string_view text, + const std::filesystem::path &directory, + std::string_view file, + Options options) + { + const std::string prefix = std::string{ file } + ": "; + auto document = ParseJson(text); + if (!document) + { + return Error{ prefix + document.GetError().Message() }; + } + if (!document->IsObject()) + { + return Error{ prefix + "the file holds an object, not " + + document->ToString() }; + } + std::set<std::string_view> seen; + bool versioned = false; + for (const auto &[name, value] : document->Members()) + { + if (!seen.insert(name).second) + { + return Error{ prefix + Quoted(name) + " appears twice" }; + } + if (name == "version") + { + if (WholeNumber(value) != kConfigVersion) + { + return Error{ + prefix + "version " + value.ToString() + + " is not known; this Catch3 reads version 1" + }; + } + versioned = true; + continue; + } + if (name == "$schema") + { + // For editors; the runner knows its own format. + continue; + } + const ConfigKey *key = nullptr; + for (const auto &entry : Keys()) + { + if (entry.name == name) + { + key = &entry; + break; + } + } + if (key == nullptr) + { + return Error{ prefix + UnknownKey(name) }; |