diff options
| author | Leitwolf | 2026-11-14 16:46:04 +0300 |
|---|---|---|
| committer | GitHub | 2026-11-14 16:46:04 +0300 |
| commit | 94a996a3ceafa9f0004ffc28a7de5c543cdb51c2 (patch) | |
| tree | e8db06877788de36d3fd4166472c4511f0c0412a | |
| parent | 586a42e35788d269871dcadbed2ec59b6b8a9203 (diff) | |
| parent | 5ad02bccfcb925df02d1273901b6e248d477c5fa (diff) | |
| download | catch3-94a996a3ceafa9f0004ffc28a7de5c543cdb51c2.tar.gz | |
Merge pull request #39 from Progmasoft/feature/per-test-coverage
Keep per-test coverage with --coverage (roadmap step 4)
25 files changed, 2298 insertions, 18 deletions
diff --git a/.github/workflows/linux-builds.yml b/.github/workflows/linux-builds.yml index a9c38c66..02ee8d67 100644 --- a/.github/workflows/linux-builds.yml +++ b/.github/workflows/linux-builds.yml @@ -32,10 +32,10 @@ jobs: with: persist-credentials: false - - name: Install Clang, CMake and Ninja + - name: Install Clang, LLVM's coverage tools, CMake and Ninja run: | sudo apt-get update - sudo apt-get install -y clang cmake ninja-build python3 + sudo apt-get install -y clang llvm cmake ninja-build python3 - name: Configure run: | @@ -51,6 +51,8 @@ jobs: - name: Test run: ctest --test-dir build -j + env: + CATCH3_COVERAGE_REQUIRE_MAP: 1 - name: Install, and find the installed package run: | @@ -75,10 +77,10 @@ jobs: with: persist-credentials: false - - name: Install Clang, CMake and Ninja + - name: Install Clang, LLVM's coverage tools, CMake and Ninja run: | sudo apt-get update - sudo apt-get install -y clang cmake ninja-build python3 + sudo apt-get install -y clang llvm cmake ninja-build python3 - name: Configure with the sanitizers env: @@ -111,8 +113,8 @@ jobs: steps: # Git comes first: without it the checkout falls back to an archive of # the repository, which is not a checkout the tests can ask questions of. - - name: Install Git, Clang, CMake and Ninja - run: dnf install -y git clang gcc-c++ cmake ninja-build python3 + - name: Install Git, Clang, LLVM's coverage tools, CMake and Ninja + run: dnf install -y git clang llvm gcc-c++ cmake ninja-build python3 - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: @@ -132,3 +134,41 @@ jobs: - name: Test run: ctest --test-dir build -j + env: + CATCH3_COVERAGE_REQUIRE_MAP: 1 + + # Catch3 is built with Clang everywhere; GCC's stricter warnings are not + # errors here. The job checks per-test coverage in gcov's format, which + # only GCC writes. + gcc-coverage: + name: Ubuntu 26.04, GCC, per-test coverage with gcov + runs-on: ubuntu-26.04 + timeout-minutes: 30 + + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + + - name: Install GCC, CMake and Ninja + run: | + sudo apt-get update + sudo apt-get install -y g++ cmake ninja-build python3 + + - name: Configure + run: | + cmake --preset basic-tests -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_CXX_STANDARD=17 \ + -DCATCH3_ENABLE_WERROR=OFF + + - name: Build + run: cmake --build build --target ProgmasoftSelfTest + + - name: Test per-test coverage + env: + CATCH3_RUNNER_FIXTURE: build/tests/RunnerFixture + CATCH3_COVERAGE_FIXTURE: build/tests/CoverageFixture + CATCH3_COVERAGE_REQUIRE_MAP: 1 + run: build/tests/ProgmasoftSelfTest --no-history "[coverage]" diff --git a/.github/workflows/mac-builds.yml b/.github/workflows/mac-builds.yml index c115bf00..4044db9b 100644 --- a/.github/workflows/mac-builds.yml +++ b/.github/workflows/mac-builds.yml @@ -44,3 +44,5 @@ jobs: - name: Test run: ctest --test-dir build -j + env: + CATCH3_COVERAGE_REQUIRE_MAP: 1 diff --git a/docs/command-line.md b/docs/command-line.md index 236128e1..ccca35ff 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -120,6 +120,7 @@ Without a spec, every test runs except hidden ones, whose tags start with | `--junit=FILE` | write a JUnit XML report | | `--json=FILE` | write a JSON report with every attempt and its analysis | | `--trace=FILE` | write a trace for `chrome://tracing` or Perfetto | +| `--coverage=DIR` | run every test in a process of its own and keep its coverage in DIR, with a map of what each test reached; see [Per-test coverage](progmasoft-runner.md#per-test-coverage) | | `--slowest[=N]` | list the N slowest tests (default 10) | | `--premature-exit-guard-file=FILE` | create FILE, private to the user, when the run starts and remove it when the run ends normally | diff --git a/docs/progmasoft-roadmap.md b/docs/progmasoft-roadmap.md index 81a8956f..1f61be79 100644 --- a/docs/progmasoft-roadmap.md +++ b/docs/progmasoft-roadmap.md @@ -22,7 +22,7 @@ The steps are ordered by what each one builds on. | 1 | Native property testing: choice-sequence shrinking, combinators, state machines, statistics, targeted search, example database, differential and metamorphic oracles | done; see [Native property testing](progmasoft-properties.md#top) | | 2 | The Catch3 runner | done; see [The Catch3 runner](progmasoft-runner.md#top) | | 3 | Mocking | done; see [Mocking](progmasoft-mocking.md#top) | -| 4 | Per-test code coverage | planned | +| 4 | Per-test code coverage | in progress: Linux and macOS, with Clang and GCC; see [Per-test coverage](progmasoft-runner.md#per-test-coverage). Windows with clang-cl is next | | 5 | Change-impact test selection and test prioritization | planned | | 6 | Mutation testing | planned | | 7 | Coverage-, mutation- and history-guided test generation | planned | diff --git a/docs/progmasoft-runner.md b/docs/progmasoft-runner.md index 19d1e06d..d61ebadd 100644 --- a/docs/progmasoft-runner.md +++ b/docs/progmasoft-runner.md @@ -12,6 +12,7 @@ [Temporary files](#temporary-files)<br> [What a failure report says](#what-a-failure-report-says)<br> [Reports and traces](#reports-and-traces)<br> +[Per-test coverage](#per-test-coverage)<br> [Formats other tools read](#formats-other-tools-read)<br> [Project configuration](#project-configuration)<br> [Bazel and CTest](#bazel-and-ctest)<br> @@ -332,6 +333,70 @@ replay line runs the failure again as it happened. attempt and section. +## Per-test coverage + +`--coverage=DIR` runs every test in a process of its own and keeps what each +one reached. Build the tests with coverage instrumentation: + +* Clang: `-fprofile-instr-generate -fcoverage-mapping`, LLVM source-based + coverage; +* GCC: `--coverage`, gcov's format. + +```text +$ ./tests --coverage=coverage +Catch3: 120 test(s), up to 8 worker(s), seed 2153251184 +note: --coverage: 120 test profile(s); what each test reached is in coverage/coverage.json, the run combined in coverage/coverage.lcov + +120 test(s): 120 passed +``` + +Each attempt gets a process of its own, so the profile its process writes at +exit is the coverage of that one test: `DIR/profiles/t<index>-a<attempt>` +with `.profraw` for LLVM, or a `.gcov` directory below which gcov writes its +`.gcda` files under their absolute paths. The runner sets `LLVM_PROFILE_FILE` +and `GCOV_PREFIX` for the process; each runtime reads its own. A process +that runs other tests first, to find a test that leaves state behind, writes +beside the profiles, never into one. The profiles and reports of an earlier +run in `DIR` are removed when the run starts: gcov adds to the data it finds, +and a run that writes no reports leaves none to be read as its own. + +After the run, the runner reads, on as many threads as the run had workers, +the profile of the last attempt of every test that left one; when a retry or +repeat crashed before writing its profile, the attempt before it counts: + +* `DIR/coverage.json` holds, for each test, its outcome, its profile and the + files it reached, with the lines it ran and, per line, how many of its + branches it took: + + ```json + { "version": 1, "format": "llvm", "binary": "./tests", + "tests": [ { "name": "Adds one", "outcome": "passed", + "profile": "coverage/profiles/t0-a1.profraw", + "files": [ { "file": "/src/calc.cpp", "lines": [ 12, 13 ], + "branches": [ { "line": 12, "taken": 1, "total": 2 } ] } ] } ] } + ``` + +* `DIR/coverage.lcov` is the coverage of the whole run as an LCOV tracefile, + which `genhtml`, Codecov and editors read. + +LLVM profiles are read with `llvm-profdata` and `llvm-cov`, gcov data with +`gcov`. They are looked up on `PATH`, through `xcrun` on macOS, or named by +`CATCH3_LLVM_PROFDATA`, `CATCH3_LLVM_COV` and `CATCH3_GCOV`. Without them the +profiles stay in `DIR/profiles` and the run says so, as it does when the +tests wrote no profiles; neither fails the run. A profile the tools cannot +read, or a report that cannot be written, fails a run whose tests passed, +like any report the run was asked for. A test that crashes or is stopped at +its time limit leaves no profile. gcov is run with `--branch-probabilities`, +without which its JSON has no branches. The runner's own process is +instrumented too and writes its profile where any instrumented program does, +`default.profraw` in the working directory unless `LLVM_PROFILE_FILE` says +otherwise; it holds no test. + +`--coverage` cannot be used with `--in-process` or `--break`, which run the +tests in one process. Per-test coverage is tested on Linux and macOS; Windows +builds with clang-cl are next. + + ## Formats other tools read These formats carry a `format` and a `version`. Within a version they only diff --git a/extras/catch_amalgamated.cpp b/extras/catch_amalgamated.cpp index 737af11a..4b928849 100644 --- a/extras/catch_amalgamated.cpp +++ b/extras/catch_amalgamated.cpp @@ -371,6 +371,9 @@ namespace Progmasoft::Catch3 /// The directory the test had for its temporary files; kept when the /// attempt failed, so its contents can be inspected. std::string temporaryDirectory; + /// In a coverage run, where the profile of this attempt went, + /// without its extension (`.profraw` or `.gcov`). + std::string coverageProfile; }; /// One test of the run and everything the runner learned about it. @@ -1133,6 +1136,10 @@ namespace Progmasoft::Catch3::Runner std::string junitPath; std::string jsonPath; std::string tracePath; + /// Per-test coverage: every attempt runs in a process of its own and + /// leaves its profile below this directory, with the map of what + /// each test reached and the combined report; see CoverageReport. + std::string coverageDirectory; unsigned slowest = 0; /// Created when the run starts and removed when it ends normally, so /// a file left behind shows that the run ended early. @@ -1337,6 +1344,82 @@ namespace Progmasoft::Catch3::Runner // Copyright (c) 2026 Progmasoft. // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 +#include <cstdint> +#include <filesystem> +#include <map> +#include <string> +#include <utility> +#include <vector> + + +namespace Progmasoft::Catch3::Runner +{ + /// What one test, or the whole run, reached in one source file. + struct FileCoverage final + { + /// Execution count of each instrumented line. + std::map<std::size_t, std::uint64_t> lines; + /// Count of each branch, by line and by its position on the line. + std::map<std::pair<std::size_t, std::size_t>, std::uint64_t> branches; + }; + + /// Coverage by source file. + using CoverageMap = std::map<std::string, FileCoverage>; + + /// Reads an LCOV tracefile: `SF`, `DA` and `BRDA` records. + [[nodiscard]] CoverageMap + ParseLcov(const std::string &text); + + /// Reads the JSON that `gcov --json-format` writes, adding to `map`. + void + AddGcovJson(const std::string &text, CoverageMap &map); + + /// Writes `map` as an LCOV tracefile. + [[nodiscard]] std::string + ToLcov(const CoverageMap &map); + + /// Prepares the directory of a coverage run: the profiles and reports of + /// an earlier run below it are removed, because gcov adds to the .gcda + /// files it finds instead of replacing them, and because a run that + /// writes no reports must not leave the earlier ones to be read as its + /// own. + Status + PrepareCoverageDirectory(const std::filesystem::path &directory); + + /// The last attempt of `test` whose profile, its base name followed by + /// `suffix` (`.profraw` or `.gcov`), exists: an attempt that crashed or + /// timed out before writing one is passed over for an earlier one. + [[nodiscard]] const Attempt * + ProfiledAttempt(const TestResult &test, const std::string &suffix); + + /// What writing the coverage reports gave. + struct CoverageReports final + { + /// What the run should be told: how many profiles there were and + /// where the reports are, or why there are none. + std::vector<std::string> notes; + /// What failed: a profile that could not be read, a report that + /// could not be written. A run with failures is missing a report. + std::vector<std::string> failures; + }; + + /// Turns the profiles a coverage run left in `directory` into + /// `directory/coverage.json`, the lines and branches each test reached, + /// and `directory/coverage.lcov`, the run combined. LLVM profiles are + /// read with `llvm-profdata` and `llvm-cov`, gcov data with `gcov`, + /// found on PATH or named by CATCH3_LLVM_PROFDATA, CATCH3_LLVM_COV and + /// CATCH3_GCOV; the profiles of the tests are read up to `jobs` at a + /// time. A run whose tests wrote no profiles, or whose tools are + /// missing, gets a note and no reports, not a failure. + [[nodiscard]] CoverageReports + WriteCoverageReports(const std::filesystem::path &directory, + const RunResult &run, + const std::string &program, + std::size_t jobs); +} // namespace Progmasoft::Catch3::Runner +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 + #include <functional> #include <memory> #include <string> @@ -2193,6 +2276,10 @@ namespace Progmasoft::Catch3::Runner::Scheduling /// Catalog indices this process has run, in order. std::vector<std::size_t> history; std::string startupOutput; + /// In a coverage run, where the profile of the attempt this process + /// was started for goes, without its extension; empty for a process + /// that runs other tests first or only probes. + std::string coverageProfile; }; struct Event final @@ -10864,6 +10951,688 @@ namespace Progmasoft::Catch3::Runner } // namespace Progmasoft::Catch3::Runner // Copyright (c) 2026 Progmasoft. // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 +#include <algorithm> +#include <atomic> +#include <cstdlib> +#include <initializer_list> +#include <optional> +#include <string_view> +#include <system_error> +#include <thread> + + +namespace Progmasoft::Catch3::Runner +{ + namespace + { + /// Reads a non-negative decimal number; nothing for anything else, + /// such as the `-` LCOV writes for a branch never reached. + std::optional<std::uint64_t> + Number(std::string_view text) + { + if (text.empty() || text.size() > 19) + { + return std::nullopt; + } + std::uint64_t value = 0; + for (const char character : text) + { + if (character < '0' || character > '9') + { + return std::nullopt; + } + value + = value * 10 + static_cast<std::uint64_t>(character - '0'); + } + return value; + } + + std::vector<std::string_view> + Split(std::string_view text, char separator) + { + std::vector<std::string_view> parts; + std::size_t start = 0; + while (true) + { + const auto end = text.find(separator, start); + parts.push_back(text.substr(start, end - start)); + if (end == std::string_view::npos) + { + return parts; + } + start = end + 1; + } + } + + /// A tool of a coverage format: the program, or the command that + /// runs it, such as `xcrun llvm-cov` on macOS. + using Tool = std::vector<std::string>; + + std::optional<Tool> + FindTool(const char *variable, const std::string &name) + { + std::vector<Tool> candidates; + if (auto configured = ReadEnvironment(variable); + !configured.empty()) + { + candidates.push_back({ std::move(configured) }); + } + candidates.push_back({ name }); +#if defined(__APPLE__) + candidates.push_back({ "xcrun", name }); +#endif + for (auto &candidate : candidates) + { + ChildOptions options; + options.arguments = candidate; + options.arguments.emplace_back("--version"); + options.searchPath = true; + if (auto run = RunAndCapture(options); + run && run->status.Succeeded()) + { + return std::move(candidate); + } + } + return std::nullopt; + } + + /// Runs `tool` with `arguments`; its output, or why it failed. + Expected<std::string> + RunTool(const Tool &tool, + const std::vector<std::string> &arguments, + const std::filesystem::path &directory = {}) + { + ChildOptions options; + options.arguments = tool; + options.arguments.insert(options.arguments.end(), + arguments.begin(), + arguments.end()); + options.searchPath = true; + options.workingDirectory = directory; + auto run = RunAndCapture(options); + if (!run) + { + return run.GetError(); + } + if (!run->status.Succeeded()) + { + std::string message = run->errors; + while (!message.empty() + && (message.back() == '\n' || message.back() == '\r')) + { + message.pop_back(); + } + return Error{ tool.back() + " failed: " + message }; + } + return std::move(run->output); + } + + void + Add(CoverageMap &total, const CoverageMap &part) + { + for (const auto &[file, coverage] : part) + { + FileCoverage &into = total[file]; + for (const auto &[line, count] : coverage.lines) + { + into.lines[line] += count; + } + for (const auto &[branch, count] : coverage.branches) + { + into.branches[branch] += count; + } + } + } + + /// What one test reached: the lines it ran and, per line, how many + /// of its branches it took. + JsonValue + ReachedJson(const CoverageMap &map) + { + JsonValue files = JsonValue::Array(); + for (const auto &[file, coverage] : map) + { + JsonValue lines = JsonValue::Array(); + for (const auto &[line, count] : coverage.lines) + { + if (count > 0) + { + lines.Append(line); + } + } + JsonValue branches = JsonValue::Array(); + std::map<std::size_t, std::pair<std::size_t, std::size_t>> + perLine; + for (const auto &[branch, count] : coverage.branches) + { + auto &[taken, total] = perLine[branch.first]; + ++total; + if (count > 0) + { + ++taken; + } + } + for (const auto &[line, counts] : perLine) + { + if (counts.first == 0) + { + continue; + } + branches.Append( + JsonValue::Object({ { "line", line }, + { "taken", counts.first }, + { "total", counts.second } })); + } + if (lines.Items().empty()) + { + continue; + } + files.Append( + JsonValue::Object({ { "file", file }, + { "lines", std::move(lines) }, + { "branches", std::move(branches) } })); + } + return files; + } + + /// The .gcda files gcov wrote below `prefix`, each with the path the + /// object was built at, where its .gcno is. + std::vector<std::pair<std::filesystem::path, std::filesystem::path>> + GcdaFiles(const std::filesystem::path &prefix) + { + std::vector<std::pair<std::filesystem::path, std::filesystem::path>> + files; + std::error_code error; + for (auto entry + = std::filesystem::recursive_directory_iterator(prefix, error); + !error + && entry != std::filesystem::recursive_directory_iterator(); + entry.increment(error)) + { + if (!entry->is_regular_file(error) + || entry->path().extension() != ".gcda") + { + continue; + } + // GCOV_PREFIX is put before the absolute path of the object. + const auto relative + = std::filesystem::relative(entry->path(), prefix, error); + std::filesystem::path original + = std::filesystem::path{ "/" } / relative; + files.emplace_back(entry->path(), original); + } + std::sort(files.begin(), files.end()); + return files; + } + + Expected<CoverageMap> + GcovCoverage(const Tool &gcov, const std::filesystem::path &prefix) + { + CoverageMap map; + for (const auto &[gcda, original] : GcdaFiles(prefix)) + { + auto gcno = original; + gcno.replace_extension(".gcno"); + auto beside = gcda; + beside.replace_extension(".gcno"); + std::error_code error; + std::filesystem::copy_file( + gcno, + beside, + std::filesystem::copy_options::overwrite_existing, + error); + if (error) + { + return Error{ "the notes file " + gcno.string() + + " could not be read: " + error.message() }; + } + // The data file by its absolute path, so that gcov needs no + // working directory: profiles are read on several threads, + // and a working directory is the whole process's. + // Without --branch-probabilities the JSON has no branches. + auto json + = RunTool(gcov, + { "--json-format", + "--branch-probabilities", + "--stdout", + std::filesystem::absolute(gcda).string() }); + if (!json) + { + return json.GetError(); + } + // One JSON document per data file, one per line. + for (const auto line : Split(*json, '\n')) + { + if (!line.empty()) + { + AddGcovJson(std::string{ line }, map); + } + } + } + return map; + } + } // namespace + + CoverageMap + ParseLcov(const std::string &text) + { + CoverageMap map; + FileCoverage *current = nullptr; + for (auto line : Split(text, '\n')) + { + if (!line.empty() && line.back() == '\r') + { + line.remove_suffix(1); + } + if (line.substr(0, 3) == "SF:") + { + current = &map[std::string{ line.substr(3) }]; + } + else if (line == "end_of_record") + { + current = nullptr; + } + else if (current != nullptr && line.substr(0, 3) == "DA:") + { + const auto fields = Split(line.substr(3), ','); + const auto number = Number(fields[0]); + const auto count + = fields.size() > 1 ? Number(fields[1]) : std::nullopt; + if (number && count) + { + current->lines[static_cast<std::size_t>(*number)] += *count; + } + } + else if (current != nullptr && line.substr(0, 5) == "BRDA:") + { + const auto fields = Split(line.substr(5), ','); + if (fields.size() != 4) + { + continue; + } + const auto parsed = Number(fields[0]); + if (!parsed) + { + continue; + } + const auto number = static_cast<std::size_t>(*parsed); + // The branches of a line are numbered in the order they come; + // `-` counts as never taken. + std::size_t index = 0; + for (auto it = current->branches.lower_bound({ number, 0 }); + it != current->branches.end() + && it->first.first == number; + ++it) + { + ++index; + } + current->branches[{ number, index }] + += Number(fields[3]).value_or(0); + } + } + return map; + } + + void + AddGcovJson(const std::string &text, CoverageMap &map) + { + auto document = ParseJson(text); + if (!document) + { + return; + } + for (const auto &file : (*document)["files"].Items()) + { + FileCoverage &coverage = map[file["file"].AsString()]; + for (const auto &line : file["lines"].Items()) + { + const auto number = static_cast<std::size_t>( + line["line_number"].AsUnsigned()); + coverage.lines[number] += line["count"].AsUnsigned(); + std::size_t index = 0; + for (const auto &branch : line["branches"].Items()) + { + coverage.branches[{ number, index++ }] + += branch["count"].AsUnsigned(); + } + } |