Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

develroot / src / Progmasoft / Runner / Options.hpp

6.8 KiB · Raw · Log

// Copyright (c) 2026 Progmasoft.
// SPDX-License-Identifier: LGPL-3.0-or-later
#pragma once

#include "Expected.hpp"

#include <chrono>
#include <cstdint>
#include <cstddef>
#include <optional>
#include <string>
#include <string_view>
#include <vector>

namespace Progmasoft::Catch3::Runner
{
    /// What the command line asks for.
    enum class Mode : std::uint8_t
    {
        kRun,
        kListTests,
        kListTags,
        kAnalytics,
        /// Read the reports of other test frameworks.
        kImport,
        kHelp,
        kVersion,
        /// Internal: run tests on behalf of a runner, over the protocol.
        kWorker,
    };

    enum class ListFormat : std::uint8_t
    {
        kText,
        kNames,
        kJson,
    };

    /// Which of the matching tests to run.
    enum class Selection : std::uint8_t
    {
        /// Every matching test.
        kAll,
        /// Those that failed, crashed, timed out or were flaky last time.
        kFailed,
        /// Those defined in, or covering, files changed since a revision.
        kChanged,
        /// Those without a recorded result.
        kNew,
        /// Changed, failed and new tests, and those most likely to fail.
        kSmart,
    };

    enum class Order : std::uint8_t
    {
        /// Most likely to fail per second of running time first.
        kPriority,
        kDeclared,
        kName,
        kRandom,
    };

    enum class RetryMode : std::uint8_t
    {
        kOff,
        /// Retry only what may pass on a second try: crashes, timeouts,
        /// tests with a flaky history and failures that do not repeat.
        kAuto,
        /// Retry every failure up to the limit.
        kAlways,
    };

    enum class QuarantineMode : std::uint8_t
    {
        /// Tests that keep flaking go into quarantine on their own; a failure
        /// in quarantine is reported but does not fail the run.
        kAuto,
        /// Quarantine is reported, but failures still fail the run.
        kReport,
        kOff,
    };

    /// The reports `--import` reads.
    enum class ImportFormat : std::uint8_t
    {
        /// Whichever the contents show.
        kAuto,
        /// Catch3's own: `--json` reports and catch3-results JSON Lines.
        kCatch3,
        kJUnit,
        kXUnit,
        kTrx,
        kTap,
        kCtrf,
    };

    enum class ColorMode : std::uint8_t
    {
        kAuto,
        kAlways,
        kNever,
    };

    enum class Verbosity : std::uint8_t
    {
        kQuiet,
        kNormal,
        kVerbose,
    };

    /// One shard of a run: `index` counts from zero.
    struct Shard final
    {
        unsigned index = 0;
        unsigned count = 1;
    };

    /// Everything the command line can say, already checked and typed.
    struct Options final
    {
        Mode mode = Mode::kRun;
        ListFormat listFormat = ListFormat::kText;
        std::string programName;

        // Selecting tests.
        std::vector<std::string> testSpecs;
        std::vector<std::string> sections;
        /// Path filters: "c:SECTION" or "g:INDEX" (a generator value, or *).
        std::vector<std::string> paths;
        Selection selection = Selection::kAll;
        std::string changesSince;
        bool incremental = false;
        Shard shard;
        std::string shardDurations;
        bool filenamesAsTags = false;

        // Running.
        unsigned jobs = 0;
        bool inProcess = false;
        bool isolate = false;
        /// Variables the tests keep although their names look like secrets.
        std::vector<std::string> passedEnvironment;
        /// The per-test limit; zero disables it.
        std::chrono::milliseconds timeout{0};
        bool timeoutGiven = false;
        bool adaptiveTimeout = true;
        RetryMode retryMode = RetryMode::kAuto;
        unsigned retries = 2;
        unsigned repeat = 1;
        unsigned failFast = 0;
        Order order = Order::kPriority;
        std::uint32_t seed = 0;
        bool seedGiven = false;
        bool bisectOrder = true;

        // How tests behave.
        bool includeSuccessful = false;
        bool breakIntoDebugger = false;
        bool noThrow = false;
        bool warnNoAssertions = false;
        bool warnUnmatchedSpecs = false;
        bool warnInfiniteGenerators = false;
        bool allowNoTests = false;
        bool skipBenchmarks = false;
        bool benchmarkNoAnalysis = false;
        unsigned benchmarkSamples = 100;
        unsigned benchmarkResamples = 100000;
        double benchmarkConfidenceInterval = 0.95;
        std::chrono::milliseconds benchmarkWarmupTime{100};

        // History and analysis.
        std::string historyDirectory;
        bool historyEnabled = true;
        QuarantineMode quarantine = QuarantineMode::kAuto;
        bool replayFailures = false;
        bool analyticsJson = false;

        // Importing the reports of other frameworks.
        std::vector<std::string> importFiles;
        ImportFormat importFormat = ImportFormat::kAuto;
        /// The history they go to; "imported" when empty.
        std::string importName;
        /// What was tested: a verdict that changes on one revision counts
        /// as flaky. Without it, only retries in the reports do.
        std::string importRevision;

        // Output.
        Verbosity verbosity = Verbosity::kNormal;
        ColorMode color = ColorMode::kAuto;
        std::string junitPath;
        std::string jsonPath;
        std::string tracePath;
        /// Per-test coverage: every attempt runs in a process of its own and
        /// leaves its profile below this directory, with the map of what
        /// each test reached and the combined report; see CoverageReport.
        std::string coverageDirectory;
        unsigned slowest = 0;
        /// Created when the run starts and removed when it ends normally, so
        /// a file left behind shows that the run ended early.
        std::string exitGuardPath;

        // Configuration.
        /// The `catch3.json` named by --config; after ParseCommandLine, the
        /// one that was read, if any.
        std::string configPath;
        bool noConfig = false;
    };

    /// Reads a command line on top of `options`. `arguments` does not include
    /// the program.
    [[nodiscard]] Expected<Options>
    ParseOptions(const std::vector<std::string> &arguments,
                 Options options = {});

    /// How many single-character edits turn `left` into `right`, for
    /// suggesting the name that was meant.
    [[nodiscard]] std::size_t
    EditDistance(std::string_view left, std::string_view right);

    /// The help text, for `--help`.
    [[nodiscard]] std::string
    HelpText(const std::string &programName);

    /// The arguments a worker is started with to behave like `options`.
    [[nodiscard]] std::vector<std::string>
    WorkerArguments(const Options &options);
} // namespace Progmasoft::Catch3::Runner