catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
Keep per-test coverage with --coverage (roadmap step 4)
- Commit
17248837a78462596809f902dd47cf113a39670b- Author
- Leitwolf11 ·
- Parents
43dabc9adb45
Changes
.github/workflows/linux-builds.yml | 52 ++- .github/workflows/mac-builds.yml | 2 + docs/command-line.md | 1 + docs/progmasoft-roadmap.md | 2 +- docs/progmasoft-runner.md | 60 +++ premake5.lua | 1 + src/CMakeLists.txt | 2 + src/Progmasoft/Catch3/Results.hpp | 3 + src/Progmasoft/Catch3/meson.build | 1 + src/Progmasoft/Runner/CoverageReport.cpp | 594 ++++++++++++++++++++++++++++++ src/Progmasoft/Runner/CoverageReport.hpp | 58 +++ src/Progmasoft/Runner/Options.cpp | 20 + src/Progmasoft/Runner/Options.hpp | 4 + src/Progmasoft/Runner/ResultsJson.cpp | 2 + src/Progmasoft/Runner/Runner.cpp | 20 + src/Progmasoft/Runner/Scheduler.cpp | 61 ++- src/Progmasoft/Runner/SchedulerModel.hpp | 4 + tests/CMakeLists.txt | 22 +- tests/Progmasoft/BUILD.bazel | 1 + tests/Progmasoft/CoverageFixture.cpp | 33 ++ tests/Progmasoft/RunnerCoverage.tests.cpp | 226 ++++++++++++ tests/meson.build | 1 + xmake.lua | 1 + 23 files changed, 1160 insertions(+), 11 deletions(-) diff --git a/.github/workflows/linux-builds.yml b/.github/workflows/linux-builds.yml index a9c38c66..02ee8d67 100644 --- a/.github/workflows/linux-builds.yml +++ b/.github/workflows/linux-builds.yml @@ -32,10 +32,10 @@ jobs: with: persist-credentials: false - - name: Install Clang, CMake and Ninja + - name: Install Clang, LLVM's coverage tools, CMake and Ninja run: | sudo apt-get update - sudo apt-get install -y clang cmake ninja-build python3 + sudo apt-get install -y clang llvm cmake ninja-build python3 - name: Configure run: | @@ -51,6 +51,8 @@ jobs: - name: Test run: ctest --test-dir build -j + env: + CATCH3_COVERAGE_REQUIRE_MAP: 1 - name: Install, and find the installed package run: | @@ -75,10 +77,10 @@ jobs: with: persist-credentials: false - - name: Install Clang, CMake and Ninja + - name: Install Clang, LLVM's coverage tools, CMake and Ninja run: | sudo apt-get update - sudo apt-get install -y clang cmake ninja-build python3 + sudo apt-get install -y clang llvm cmake ninja-build python3 - name: Configure with the sanitizers env: @@ -111,8 +113,8 @@ jobs: steps: # Git comes first: without it the checkout falls back to an archive of # the repository, which is not a checkout the tests can ask questions of. - - name: Install Git, Clang, CMake and Ninja - run: dnf install -y git clang gcc-c++ cmake ninja-build python3 + - name: Install Git, Clang, LLVM's coverage tools, CMake and Ninja + run: dnf install -y git clang llvm gcc-c++ cmake ninja-build python3 - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: @@ -132,3 +134,41 @@ jobs: - name: Test run: ctest --test-dir build -j + env: + CATCH3_COVERAGE_REQUIRE_MAP: 1 + + # Catch3 is built with Clang everywhere; GCC's stricter warnings are not + # errors here. The job checks per-test coverage in gcov's format, which + # only GCC writes. + gcc-coverage: + name: Ubuntu 26.04, GCC, per-test coverage with gcov + runs-on: ubuntu-26.04 + timeout-minutes: 30 + + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + + - name: Install GCC, CMake and Ninja + run: | + sudo apt-get update + sudo apt-get install -y g++ cmake ninja-build python3 + + - name: Configure + run: | + cmake --preset basic-tests -GNinja \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_CXX_STANDARD=17 \ + -DCATCH3_ENABLE_WERROR=OFF + + - name: Build + run: cmake --build build --target ProgmasoftSelfTest + + - name: Test per-test coverage + env: + CATCH3_RUNNER_FIXTURE: build/tests/RunnerFixture + CATCH3_COVERAGE_FIXTURE: build/tests/CoverageFixture + CATCH3_COVERAGE_REQUIRE_MAP: 1 + run: build/tests/ProgmasoftSelfTest --no-history "[coverage]" diff --git a/.github/workflows/mac-builds.yml b/.github/workflows/mac-builds.yml index c115bf00..4044db9b 100644 --- a/.github/workflows/mac-builds.yml +++ b/.github/workflows/mac-builds.yml @@ -44,3 +44,5 @@ jobs: - name: Test run: ctest --test-dir build -j + env: + CATCH3_COVERAGE_REQUIRE_MAP: 1 diff --git a/docs/command-line.md b/docs/command-line.md index 236128e1..ccca35ff 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -120,6 +120,7 @@ Without a spec, every test runs except hidden ones, whose tags start with | `--junit=FILE` | write a JUnit XML report | | `--json=FILE` | write a JSON report with every attempt and its analysis | | `--trace=FILE` | write a trace for `chrome://tracing` or Perfetto | +| `--coverage=DIR` | run every test in a process of its own and keep its coverage in DIR, with a map of what each test reached; see [Per-test coverage](progmasoft-runner.md#per-test-coverage) | | `--slowest[=N]` | list the N slowest tests (default 10) | | `--premature-exit-guard-file=FILE` | create FILE, private to the user, when the run starts and remove it when the run ends normally | diff --git a/docs/progmasoft-roadmap.md b/docs/progmasoft-roadmap.md index 81a8956f..1f61be79 100644 --- a/docs/progmasoft-roadmap.md +++ b/docs/progmasoft-roadmap.md @@ -22,7 +22,7 @@ The steps are ordered by what each one builds on. | 1 | Native property testing: choice-sequence shrinking, combinators, state machines, statistics, targeted search, example database, differential and metamorphic oracles | done; see [Native property testing](progmasoft-properties.md#top) | | 2 | The Catch3 runner | done; see [The Catch3 runner](progmasoft-runner.md#top) | | 3 | Mocking | done; see [Mocking](progmasoft-mocking.md#top) | -| 4 | Per-test code coverage | planned | +| 4 | Per-test code coverage | in progress: Linux and macOS, with Clang and GCC; see [Per-test coverage](progmasoft-runner.md#per-test-coverage). Windows with clang-cl is next | | 5 | Change-impact test selection and test prioritization | planned | | 6 | Mutation testing | planned | | 7 | Coverage-, mutation- and history-guided test generation | planned | diff --git a/docs/progmasoft-runner.md b/docs/progmasoft-runner.md index 19d1e06d..cb5f29c5 100644 --- a/docs/progmasoft-runner.md +++ b/docs/progmasoft-runner.md @@ -12,6 +12,7 @@ [Temporary files](#temporary-files)<br> [What a failure report says](#what-a-failure-report-says)<br> [Reports and traces](#reports-and-traces)<br> +[Per-test coverage](#per-test-coverage)<br> [Formats other tools read](#formats-other-tools-read)<br> [Project configuration](#project-configuration)<br> [Bazel and CTest](#bazel-and-ctest)<br> @@ -332,6 +333,65 @@ replay line runs the failure again as it happened. attempt and section. +## Per-test coverage + +`--coverage=DIR` runs every test in a process of its own and keeps what each +one reached. Build the tests with coverage instrumentation: + +* Clang: `-fprofile-instr-generate -fcoverage-mapping`, LLVM source-based + coverage; +* GCC: `--coverage`, gcov's format. + +```text +$ ./tests --coverage=coverage +Catch3: 120 test(s), up to 8 worker(s), seed 2153251184 +note: --coverage: 120 test profile(s); what each test reached is in coverage/coverage.json, the run combined in coverage/coverage.lcov + +120 test(s): 120 passed +``` + +Each attempt gets a process of its own, so the profile its process writes at +exit is the coverage of that one test: `DIR/profiles/t<index>-a<attempt>` +with `.profraw` for LLVM, or a `.gcov` directory below which gcov writes its +`.gcda` files under their absolute paths. The runner sets `LLVM_PROFILE_FILE` +and `GCOV_PREFIX` for the process; each runtime reads its own. A process +that runs other tests first, to find a test that leaves state behind, writes +beside the profiles, never into one. The profiles of an earlier run in +`DIR/profiles` are removed when the run starts, because gcov adds to the +data it finds. + +After the run, the runner reads the profile of the last attempt of every +test: + +* `DIR/coverage.json` holds, for each test, its outcome, its profile and the + files it reached, with the lines it ran and, per line, how many of its + branches it took: + + ```json + { "version": 1, "format": "llvm", "binary": "./tests", + "tests": [ { "name": "Adds one", "outcome": "passed", + "profile": "coverage/profiles/t0-a1.profraw", + "files": [ { "file": "/src/calc.cpp", "lines": [ 12, 13 ], + "branches": [ { "line": 12, "taken": 1, "total": 2 } ] } ] } ] } + ``` + +* `DIR/coverage.lcov` is the coverage of the whole run as an LCOV tracefile, + which `genhtml`, Codecov and editors read. + +LLVM profiles are read with `llvm-profdata` and `llvm-cov`, gcov data with +`gcov`. They are looked up on `PATH`, through `xcrun` on macOS, or named by +`CATCH3_LLVM_PROFDATA`, `CATCH3_LLVM_COV` and `CATCH3_GCOV`. Without them the +profiles stay in `DIR/profiles` and the run says so. A test that crashes or +is stopped at its time limit leaves no profile. The runner's own process is +instrumented too and writes its profile where any instrumented program does, +`default.profraw` in the working directory unless `LLVM_PROFILE_FILE` says +otherwise; it holds no test. + +`--coverage` cannot be used with `--in-process` or `--break`, which run the +tests in one process. Per-test coverage is tested on Linux and macOS; Windows +builds with clang-cl are next. + + ## Formats other tools read These formats carry a `format` and a `version`. Within a version they only diff --git a/premake5.lua b/premake5.lua index d4c55c9e..def0463c 100644 --- a/premake5.lua +++ b/premake5.lua @@ -92,6 +92,7 @@ project "ProgmasoftPropertyTests" "tests/Progmasoft/RunnerConfig.tests.cpp", "tests/Progmasoft/RunnerAnalysis.tests.cpp", "tests/Progmasoft/RunnerImport.tests.cpp", + "tests/Progmasoft/RunnerCoverage.tests.cpp", "tests/Progmasoft/RunnerOutput.tests.cpp", "tests/Progmasoft/Snapshot.tests.cpp", "tests/Progmasoft/Stateful.tests.cpp", diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 43754fff..fac2fb62 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -301,6 +301,7 @@ set(PROGMASOFT_RUNNER_FILES "${CATCH_DIR}/src/Progmasoft/Runner/Execution.hpp" "${CATCH_DIR}/src/Progmasoft/Runner/Expected.hpp" "${CATCH_DIR}/src/Progmasoft/Runner/Files.hpp" + "${CATCH_DIR}/src/Progmasoft/Runner/CoverageReport.hpp" "${CATCH_DIR}/src/Progmasoft/Runner/Git.hpp" "${CATCH_DIR}/src/Progmasoft/Runner/Hash.hpp" "${CATCH_DIR}/src/Progmasoft/Runner/History.hpp" @@ -320,6 +321,7 @@ set(PROGMASOFT_RUNNER_FILES "${CATCH_DIR}/src/Progmasoft/Runner/Console.cpp" "${CATCH_DIR}/src/Progmasoft/Runner/Execution.cpp" "${CATCH_DIR}/src/Progmasoft/Runner/Files.cpp" + "${CATCH_DIR}/src/Progmasoft/Runner/CoverageReport.cpp" "${CATCH_DIR}/src/Progmasoft/Runner/Git.cpp" "${CATCH_DIR}/src/Progmasoft/Runner/Hash.cpp" "${CATCH_DIR}/src/Progmasoft/Runner/History.cpp" diff --git a/src/Progmasoft/Catch3/Results.hpp b/src/Progmasoft/Catch3/Results.hpp index 4c6bb1ce..bff2a72e 100644 --- a/src/Progmasoft/Catch3/Results.hpp +++ b/src/Progmasoft/Catch3/Results.hpp @@ -153,6 +153,9 @@ namespace Progmasoft::Catch3 /// The directory the test had for its temporary files; kept when the /// attempt failed, so its contents can be inspected. std::string temporaryDirectory; + /// In a coverage run, where the profile of this attempt went, + /// without its extension (`.profraw` or `.gcov`). + std::string coverageProfile; }; /// One test of the run and everything the runner learned about it. diff --git a/src/Progmasoft/Catch3/meson.build b/src/Progmasoft/Catch3/meson.build index 8420003b..4bbab28e 100644 --- a/src/Progmasoft/Catch3/meson.build +++ b/src/Progmasoft/Catch3/meson.build @@ -288,6 +288,7 @@ runner_sources = files( '../Runner/Console.cpp', '../Runner/Execution.cpp', '../Runner/Files.cpp', + '../Runner/CoverageReport.cpp', '../Runner/Git.cpp', '../Runner/Hash.cpp', '../Runner/History.cpp', diff --git a/src/Progmasoft/Runner/CoverageReport.cpp b/src/Progmasoft/Runner/CoverageReport.cpp new file mode 100644 index 00000000..c443c158 --- /dev/null +++ b/src/Progmasoft/Runner/CoverageReport.cpp @@ -0,0 +1,594 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 +#include <algorithm> +#include <cstdlib> +#include <optional> +#include <string_view> +#include <system_error> + +#include "CoverageReport.hpp" +#include "Files.hpp" +#include "Json.hpp" +#include "Process.hpp" + +namespace Progmasoft::Catch3::Runner +{ + namespace + { + /// Reads a non-negative decimal number; nothing for anything else, + /// such as the `-` LCOV writes for a branch never reached. + std::optional<std::uint64_t> + Number(std::string_view text) + { + if (text.empty() || text.size() > 19) + { + return std::nullopt; + } + std::uint64_t value = 0; + for (const char character : text) + { + if (character < '0' || character > '9') + { + return std::nullopt; + } + value + = value * 10 + static_cast<std::uint64_t>(character - '0'); + } + return value; + } + + std::vector<std::string_view> + Split(std::string_view text, char separator) + { + std::vector<std::string_view> parts; + std::size_t start = 0; + while (true) + { + const auto end = text.find(separator, start); + parts.push_back(text.substr(start, end - start)); + if (end == std::string_view::npos) + { + return parts; + } + start = end + 1; + } + } + + /// A tool of a coverage format: the program, or the command that + /// runs it, such as `xcrun llvm-cov` on macOS. + using Tool = std::vector<std::string>; + + std::optional<Tool> + FindTool(const char *variable, const std::string &name) + { + std::vector<Tool> candidates; + if (auto configured = ReadEnvironment(variable); + !configured.empty()) + { + candidates.push_back({ std::move(configured) }); + } + candidates.push_back({ name }); +#if defined(__APPLE__) + candidates.push_back({ "xcrun", name }); +#endif + for (auto &candidate : candidates) + { + ChildOptions options; + options.arguments = candidate; + options.arguments.emplace_back("--version"); + options.searchPath = true; + if (auto run = RunAndCapture(options); + run && run->status.Succeeded()) + { + return std::move(candidate); + } + } + return std::nullopt; + } + + /// Runs `tool` with `arguments`; its output, or why it failed. + Expected<std::string> + RunTool(const Tool &tool, + const std::vector<std::string> &arguments, + const std::filesystem::path &directory = {}) + { + ChildOptions options; + options.arguments = tool; + options.arguments.insert(options.arguments.end(), + arguments.begin(), + arguments.end()); + options.searchPath = true; + options.workingDirectory = directory; + auto run = RunAndCapture(options); + if (!run) + { + return run.GetError(); + } + if (!run->status.Succeeded()) + { + std::string message = run->errors; + while (!message.empty() + && (message.back() == '\n' || message.back() == '\r')) + { + message.pop_back(); + } + return Error{ tool.back() + " failed: " + message }; + } + return std::move(run->output); + } + + void + Add(CoverageMap &total, const CoverageMap &part) + { + for (const auto &[file, coverage] : part) + { + FileCoverage &into = total[file]; + for (const auto &[line, count] : coverage.lines) + { + into.lines[line] += count; + } + for (const auto &[branch, count] : coverage.branches) + { + into.branches[branch] += count; + } + } + } + + /// What one test reached: the lines it ran and, per line, how many + /// of its branches it took. + JsonValue + ReachedJson(const CoverageMap &map) + { + JsonValue files = JsonValue::Array(); + for (const auto &[file, coverage] : map) + { + JsonValue lines = JsonValue::Array(); + for (const auto &[line, count] : coverage.lines) + { + if (count > 0) + { + lines.Append(line); + } + } + JsonValue branches = JsonValue::Array(); + std::map<std::size_t, std::pair<std::size_t, std::size_t>> + perLine; + for (const auto &[branch, count] : coverage.branches) + { + auto &[taken, total] = perLine[branch.first]; + ++total; + if (count > 0) + { + ++taken; + } + } + for (const auto &[line, counts] : perLine) + { + if (counts.first == 0) + { + continue; + } + branches.Append( + JsonValue::Object({ { "line", line }, + { "taken", counts.first }, + { "total", counts.second } })); + } + if (lines.Items().empty()) + { + continue; + } + files.Append( + JsonValue::Object({ { "file", file }, + { "lines", std::move(lines) }, + { "branches", std::move(branches) } })); + } + return files; + } + + /// The final attempt of a test that left a profile. + const Attempt * + ProfiledAttempt(const TestResult &test) + { + for (auto attempt = test.attempts.rbegin(); + attempt != test.attempts.rend(); + ++attempt) + { + if (!attempt->coverageProfile.empty()) + { + return &*attempt; + } + } + return nullptr; + } + + /// The .gcda files gcov wrote below `prefix`, each with the path the + /// object was built at, where its .gcno is. + std::vector<std::pair<std::filesystem::path, std::filesystem::path>> + GcdaFiles(const std::filesystem::path &prefix) + { + std::vector<std::pair<std::filesystem::path, std::filesystem::path>> + files; + std::error_code error; + for (auto entry + = std::filesystem::recursive_directory_iterator(prefix, error); + !error + && entry != std::filesystem::recursive_directory_iterator(); + entry.increment(error)) + { + if (!entry->is_regular_file(error) + || entry->path().extension() != ".gcda") + { + continue; + } + // GCOV_PREFIX is put before the absolute path of the object. + const auto relative + = std::filesystem::relative(entry->path(), prefix, error); + std::filesystem::path original + = std::filesystem::path{ "/" } / relative; + files.emplace_back(entry->path(), original); + } + std::sort(files.begin(), files.end()); + return files; + } + + Expected<CoverageMap> + GcovCoverage(const Tool &gcov, const std::filesystem::path &prefix) + { + CoverageMap map; + for (const auto &[gcda, original] : GcdaFiles(prefix)) + { + auto gcno = original; + gcno.replace_extension(".gcno"); + auto beside = gcda; + beside.replace_extension(".gcno"); + std::error_code error; + std::filesystem::copy_file( + gcno, + beside, + std::filesystem::copy_options::overwrite_existing, + error); + if (error) + { + return Error{ "the notes file " + gcno.string() + + " could not be read: " + error.message() }; + } + auto json = RunTool( + gcov, + { "--json-format", "--stdout", gcda.filename().string() }, + gcda.parent_path()); + if (!json) + { + return json.GetError(); + } + // One JSON document per data file, one per line. + for (const auto line : Split(*json, '\n')) + { + if (!line.empty()) + { + AddGcovJson(std::string{ line }, map); + } + } + } + return map; + } + } // namespace + + CoverageMap + ParseLcov(const std::string &text) + { + CoverageMap map; + FileCoverage *current = nullptr; + for (auto line : Split(text, '\n')) + { + if (!line.empty() && line.back() == '\r') + { + line.remove_suffix(1); + } + if (line.substr(0, 3) == "SF:") + { + current = &map[std::string{ line.substr(3) }]; + } + else if (line == "end_of_record") + { + current = nullptr; + } + else if (current != nullptr && line.substr(0, 3) == "DA:") + { + const auto fields = Split(line.substr(3), ','); + const auto number = Number(fields[0]); + const auto count + = fields.size() > 1 ? Number(fields[1]) : std::nullopt; + if (number && count) + { + current->lines[*number] += *count; + } + } + else if (current != nullptr && line.substr(0, 5) == "BRDA:") + { + const auto fields = Split(line.substr(5), ','); + if (fields.size() != 4) + { + continue; + } + const auto number = Number(fields[0]); + if (!number) + { + continue; + } + // The branches of a line are numbered in the order they come; + // `-` counts as never taken. + std::size_t index = 0; + for (auto it = current->branches.lower_bound({ *number, 0 }); + it != current->branches.end() + && it->first.first == *number; + ++it) + { + ++index; + } + current->branches[{ *number, index }] + += Number(fields[3]).value_or(0); + } + } + return map; + } + + void + AddGcovJson(const std::string &text, CoverageMap &map) + { + auto document = ParseJson(text); + if (!document) + { + return; + } + for (const auto &file : (*document)["files"].Items()) + { + FileCoverage &coverage = map[file["file"].AsString()]; + for (const auto &line : file["lines"].Items()) + { + const auto number = static_cast<std::size_t>( + line["line_number"].AsUnsigned()); + coverage.lines[number] += line["count"].AsUnsigned(); + std::size_t index = 0; + for (const auto &branch : line["branches"].Items()) + { + coverage.branches[{ number, index++ }] + += branch["count"].AsUnsigned(); + } + } + } + } + + std::string + ToLcov(const CoverageMap &map) + { + std::string text; + for (const auto &[file, coverage] : map) + { + text += "SF:" + file + "\n"; + std::size_t linesHit = 0; + for (const auto &[line, count] : coverage.lines) + { + text += "DA:" + std::to_string(line) + "," + + std::to_string(count) + "\n"; + linesHit += count > 0 ? 1 : 0; + } + std::size_t branchesHit = 0; + for (const auto &[branch, count] : coverage.branches) + { + text += "BRDA:" + std::to_string(branch.first) + ",0," + + std::to_string(branch.second) + "," + + (count > 0 ? std::to_string(count) : "-") + "\n"; + branchesHit += count > 0 ? 1 : 0; + } + text += "LF:" + std::to_string(coverage.lines.size()) + "\n"; + text += "LH:" + std::to_string(linesHit) + "\n"; + text += "BRF:" + std::to_string(coverage.branches.size()) + "\n"; + text += "BRH:" + std::to_string(branchesHit) + "\n"; + text += "end_of_record\n"; + } + return text; + } + + Status + PrepareCoverageDirectory(const std::filesystem::path &directory) + { + std::error_code error; + std::filesystem::remove_all(directory / "profiles", error); + if (error) + { + return Error{ "the profiles of an earlier run in " + + directory.string() + + " could not be removed: " + error.message() }; + } + std::filesystem::create_directories(directory / "profiles", error); + if (error) + { + return Error{ "the coverage directory " + directory.string() + + " could not be created: " + error.message() }; + } + return Status::Ok(); + } + + std::vector<std::string> + WriteCoverageReports(const std::filesystem::path &directory, + const RunResult &run, + const std::string &program) + { + std::vector<std::string> notes; + std::size_t llvmProfiles = 0; + std::size_t gcovProfiles = 0; + for (const auto &test : run.tests) + { + if (const Attempt *attempt = ProfiledAttempt(test)) + { + std::error_code error; + llvmProfiles += std::filesystem::exists(attempt->coverageProfile + + ".profraw", + error) + ? 1 + : 0; + gcovProfiles += std::filesystem::exists(attempt->coverageProfile + + ".gcov", + error) + ? 1 + : 0; + } + } + if (llvmProfiles == 0 && gcovProfiles == 0) + { + notes.emplace_back( + "--coverage: the tests wrote no coverage; build them with " + "-fprofile-instr-generate -fcoverage-mapping (Clang) or " + "--coverage (GCC)"); + return notes; + } + const bool llvm = llvmProfiles >= gcovProfiles; + std::optional<Tool> profdata; + std::optional<Tool> cov; + std::optional<Tool> gcov; + if (llvm) + { + profdata = FindTool("CATCH3_LLVM_PROFDATA", "llvm-profdata"); + cov = FindTool("CATCH3_LLVM_COV", "llvm-cov"); + } + else + { + gcov = FindTool("CATCH3_GCOV", "gcov"); + } + if (llvm ? (!profdata || !cov) : !gcov) + { + notes.push_back( + "--coverage: " + + std::to_string(llvm ? llvmProfiles : gcovProfiles) + + " profile(s) in " + (directory / "profiles").string() + "; " + + (llvm ? "llvm-profdata and llvm-cov were" : "gcov was") + + " not found, so there is no map of what each test reached"); + return notes; + } + + CoverageMap combined; + std::vector<std::string> raw; + JsonValue tests = JsonValue::Array(); + std::vector<std::string> failures; + for (const auto &test : run.tests) + { + const Attempt *attempt = ProfiledAttempt(test); + if (attempt == nullptr) + { + continue; + } + const std::string base = attempt->coverageProfile; + Expected<CoverageMap> reached = CoverageMap{}; + std::string profile; + if (llvm) + { + profile = base + ".profraw"; + std::error_code error; + if (!std::filesystem::exists(profile, error)) + { + continue; + } + raw.push_back(profile); + auto merged = RunTool( + *profdata, + { "merge", "-sparse", profile, "-o", base + ".profdata" }); + if (!merged) + { + failures.push_back(test.name + ": " + + merged.GetError().Message()); + continue; + } + auto lcov = RunTool(*cov, + { "export", + "-format=lcov", + "-instr-profile=" + base + ".profdata", + program }); + if (!lcov) + { + failures.push_back(test.name + ": " + + lcov.GetError().Message()); + continue; + } + reached = ParseLcov(*lcov); + } + else + { + profile = base + ".gcov"; + std::error_code error; + if (!std::filesystem::exists(profile, error)) + { + continue; + } + reached = GcovCoverage(*gcov, profile); + if (!reached) + { + failures.push_back(test.name + ": " + + reached.GetError().Message()); + continue; + } + Add(combined, *reached); + } + tests.Append( + JsonValue::Object({ { "name", test.name }, + { "outcome", OutcomeName(test.outcome) }, + { "profile", profile }, + { "files", ReachedJson(*reached) } })); + } + + std::string lcovText; + if (llvm && !raw.empty()) + { + std::vector<std::string> arguments{ "merge", "-sparse" }; + arguments.insert(arguments.end(), raw.begin(), raw.end()); + const std::string all = (directory / "coverage.profdata").string(); + arguments.emplace_back("-o"); + arguments.push_back(all); + auto merged = RunTool(*profdata, arguments); + auto exported = merged ? RunTool(*cov, + { "export", + "-format=lcov", + "-instr-profile=" + all, + program }) + : Expected<std::string>{ merged.GetError() }; + if (exported) + { + lcovText = *exported; + } + else + { + failures.push_back("the combined report: " + + exported.GetError().Message()); + } + } + else + { + lcovText = ToLcov(combined); + } + + JsonValue report + = JsonValue::Object({ { "version", 1 }, + { "format", llvm ? "llvm" : "gcov" }, + { "binary", program }, + { "tests", std::move(tests) } }); + const auto json = directory / "coverage.json"; + const auto lcov = directory / "coverage.lcov"; + if (auto status = WriteFileAtomically(json, report.ToPrettyString()); + !status) + { + failures.push_back(status.GetError().Message()); + } + if (auto status = WriteFileAtomically(lcov, lcovText); !status) + { + failures.push_back(status.GetError().Message()); + } + notes.push_back( + "--coverage: " + std::to_string(llvm ? llvmProfiles : gcovProfiles) + + " test profile(s); what each test reached is in " + json.string() + + ", the run combined in " + lcov.string()); + for (const auto &failure : failures) + { + notes.push_back("--coverage: " + failure); + } + return notes; + } +} // namespace Progmasoft::Catch3::Runner diff --git a/src/Progmasoft/Runner/CoverageReport.hpp b/src/Progmasoft/Runner/CoverageReport.hpp new file mode 100644 index 00000000..35f9dc75 --- /dev/null +++ b/src/Progmasoft/Runner/CoverageReport.hpp @@ -0,0 +1,58 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 +#pragma once + +#include <Progmasoft/Catch3/Results.hpp> +#include <cstdint> +#include <filesystem> +#include <map> +#include <string> +#include <utility> +#include <vector> + +#include "Expected.hpp" + +namespace Progmasoft::Catch3::Runner +{ + /// What one test, or the whole run, reached in one source file. + struct FileCoverage final + { + /// Execution count of each instrumented line. + std::map<std::size_t, std::uint64_t> lines; + /// Count of each branch, by line and by its position on the line. + std::map<std::pair<std::size_t, std::size_t>, std::uint64_t> branches; + }; + + /// Coverage by source file. + using CoverageMap = std::map<std::string, FileCoverage>; + + /// Reads an LCOV tracefile: `SF`, `DA` and `BRDA` records. + [[nodiscard]] CoverageMap + ParseLcov(const std::string &text); + + /// Reads the JSON that `gcov --json-format` writes, adding to `map`. + void + AddGcovJson(const std::string &text, CoverageMap &map); + + /// Writes `map` as an LCOV tracefile. + [[nodiscard]] std::string + ToLcov(const CoverageMap &map); + + /// Prepares the directory of a coverage run: the profiles of an earlier + /// run below it are removed, because gcov adds to the .gcda files it + /// finds instead of replacing them. + Status + PrepareCoverageDirectory(const std::filesystem::path &directory); + + /// Turns the profiles a coverage run left in `directory` into + /// `directory/coverage.json`, the lines and branches each test reached, + /// and `directory/coverage.lcov`, the run combined. LLVM profiles are + /// read with `llvm-profdata` and `llvm-cov`, gcov data with `gcov`, + /// found on PATH or named by CATCH3_LLVM_PROFDATA, CATCH3_LLVM_COV and + /// CATCH3_GCOV. Returns what the run should be told: how many profiles + /// there were and where the reports are, or why there are none. + [[nodiscard]] std::vector<std::string> + WriteCoverageReports(const std::filesystem::path &directory, + const RunResult &run, + const std::string &program); +} // namespace Progmasoft::Catch3::Runner diff --git a/src/Progmasoft/Runner/Options.cpp b/src/Progmasoft/Runner/Options.cpp index 13cf4be7..aef0df6c 100644 --- a/src/Progmasoft/Runner/Options.cpp +++ b/src/Progmasoft/Runner/Options.cpp @@ -830,6 +830,21 @@ namespace Progmasoft::Catch3::Runner options.tracePath = *value; return Status::Ok(); }}, + {"coverage", + '\0', + Takes::kValue, + "DIR", + "Output", + "run every test in a process of its own and keep its " + "coverage in DIR, with a map of what each test reached", + [](Options &options, const std::optional<std::string> &value) { + if (value->empty()) + { + return Status{ Error{ "--coverage needs a directory" } }; + } + options.coverageDirectory = *value; + return Status::Ok(); + }}, {"premature-exit-guard-file", '\0', Takes::kValue, @@ -1105,6 +1120,11 @@ namespace Progmasoft::Catch3::Runner return status.GetError(); } } + if (!options.coverageDirectory.empty() && options.inProcess) + { + return Error{ "--coverage runs every test in a process of its own, " + "so it cannot be used with --in-process or --break" }; + } return options; } diff --git a/src/Progmasoft/Runner/Options.hpp b/src/Progmasoft/Runner/Options.hpp index 256f5ac7..3d53eb42 100644 --- a/src/Progmasoft/Runner/Options.hpp +++ b/src/Progmasoft/Runner/Options.hpp @@ -190,6 +190,10 @@ namespace Progmasoft::Catch3::Runner std::string junitPath; std::string jsonPath; std::string tracePath; + /// Per-test coverage: every attempt runs in a process of its own and + /// leaves its profile below this directory, with the map of what + /// each test reached and the combined report; see CoverageReport. + std::string coverageDirectory; unsigned slowest = 0; /// Created when the run starts and removed when it ends normally, so /// a file left behind shows that the run ended early. diff --git a/src/Progmasoft/Runner/ResultsJson.cpp b/src/Progmasoft/Runner/ResultsJson.cpp index ab892cf7..c82475ec 100644 --- a/src/Progmasoft/Runner/ResultsJson.cpp +++ b/src/Progmasoft/Runner/ResultsJson.cpp @@ -203,6 +203,7 @@ namespace Progmasoft::Catch3::Runner SetText(value, "exitDescription", attempt.exitDescription); SetText(value, "lastLocation", attempt.lastLocation); SetText(value, "temporaryDirectory", attempt.temporaryDirectory); + SetText(value, "coverageProfile", attempt.coverageProfile); return value; } @@ -242,6 +243,7 @@ namespace Progmasoft::Catch3::Runner attempt.exitDescription = value["exitDescription"].AsString(); attempt.lastLocation = value["lastLocation"].AsString(); attempt.temporaryDirectory = value["temporaryDirectory"].AsString(); + attempt.coverageProfile = value["coverageProfile"].AsString(); return attempt; } diff --git a/src/Progmasoft/Runner/Runner.cpp b/src/Progmasoft/Runner/Runner.cpp index 77011322..61f8f2b7 100644 --- a/src/Progmasoft/Runner/Runner.cpp +++ b/src/Progmasoft/Runner/Runner.cpp @@ -22,6 +22,7 @@ #include <random> #include <system_error> +#include "CoverageReport.hpp" #include "Analysis.hpp" #include "Catalog.hpp" #include "Console.hpp" @@ -949,6 +950,17 @@ namespace Progmasoft::Catch3 } } + if (!options.coverageDirectory.empty()) + { + if (auto status = Runner::PrepareCoverageDirectory( + options.coverageDirectory); + !status) + { + PrintError(status.GetError().Message()); + return Catch::UnspecifiedErrorExitCode; + } + } + const auto started = std::chrono::steady_clock::now(); RunResult run; run.binary = programName; @@ -983,6 +995,14 @@ namespace Progmasoft::Catch3 run.tests.push_back(std::move(kept)); } run.notes = std::move(outcome.notes); + if (!options.coverageDirectory.empty()) + { + for (auto ¬e : Runner::WriteCoverageReports( + options.coverageDirectory, run, program.string())) + { + run.notes.push_back(std::move(note)); + } + } const bool anyFailure = std::any_of(run.tests.begin(), diff --git a/src/Progmasoft/Runner/Scheduler.cpp b/src/Progmasoft/Runner/Scheduler.cpp index 56a2aa36..263257f7 100644 --- a/src/Progmasoft/Runner/Scheduler.cpp +++ b/src/Progmasoft/Runner/Scheduler.cpp @@ -346,6 +346,10 @@ namespace Progmasoft::Catch3::Runner } attempt.temporaryDirectory = TemporaryFor(catalogIndex, attempt.number); + if (target) + { + attempt.coverageProfile = worker.coverageProfile; + } worker.running = std::move(running); if (options_.inProcess) { @@ -973,8 +977,33 @@ namespace Progmasoft::Catch3::Runner // Worker processes. + /// Where the coverage profile of the next attempt of `job` goes, + /// without its extension. Only an attempt that runs alone in its + /// process gets one: a job that runs other tests first would + /// count their lines as its own. + std::string + CoverageProfileFor(const Job &job) const + { + if (options_.coverageDirectory.empty() + || job.kind != Job::Kind::kAttempt || !job.before.empty()) + { + return {}; + } + const std::size_t catalogIndex + = plan_.tests[job.test].entry->index; + const std::size_t attempt + = states_[job.test].result.attempts.size() + 1; + return (std::filesystem::path{ options_.coverageDirectory } + / "profiles" + / ("t" + std::to_string(catalogIndex) + "-a" + + std::to_string(attempt))) + .string(); + } + Worker * - SpawnWorker(bool dedicated, std::optional<std::uint32_t> seed) + SpawnWorker(bool dedicated, + std::optional<std::uint32_t> seed, + std::string coverageProfile = {}) { if (infrastructureFailed_) { @@ -994,6 +1023,26 @@ namespace Progmasoft::Catch3::Runner const std::size_t id = ++workersStarted_; child.environment.emplace_back("CATCH3_WORKER", std::to_string(id)); + if (!options_.coverageDirectory.empty()) + { + // Each runtime reads its own variable: LLVM's profile + // runtime the file it writes at exit, gcov the directory + // its .gcda files go below. A process that is not one + // attempt alone writes beside the profiles, never into + // one of them. + const std::string base + = !coverageProfile.empty() + ? coverageProfile + : (std::filesystem::path{ + options_.coverageDirectory } + / "profiles" + / ("other-w" + std::to_string(id))) + .string(); + child.environment.emplace_back("LLVM_PROFILE_FILE", + base + ".profraw"); + child.environment.emplace_back("GCOV_PREFIX", + base + ".gcov"); + } child.keptEnvironment = options_.passedEnvironment; auto process = ChildProcess::Start(child); if (!process) @@ -1010,6 +1059,7 @@ namespace Progmasoft::Catch3::Runner worker->id = id; worker->dedicated = dedicated; worker->seed = seed; + worker->coverageProfile = std::move(coverageProfile); worker->started = Clock::now(); worker->process = std::move(*process); Worker *raw = worker.get(); @@ -1099,8 +1149,12 @@ namespace Progmasoft::Catch3::Runner return; } const PlannedTest &planned = plan_.tests[job->test]; + // A coverage run gives every attempt a process of its + // own: the profile a process writes at exit is then the + // coverage of that one test. const bool fresh = job->fresh || options_.isolate - || planned.entry->isolated; + || planned.entry->isolated + || !options_.coverageDirectory.empty(); if (!fresh && idle != nullptr) { Assign(*idle, std::move(*job)); @@ -1116,7 +1170,8 @@ namespace Progmasoft::Catch3::Runner queue_.push_front(std::move(*job)); return; } - Worker *worker = SpawnWorker(fresh, job->seed); + Worker *worker + = SpawnWorker(fresh, job->seed, CoverageProfileFor(*job)); if (worker == nullptr) { queue_.push_front(std::move(*job)); diff --git a/src/Progmasoft/Runner/SchedulerModel.hpp b/src/Progmasoft/Runner/SchedulerModel.hpp index 0d5a1c6c..581b5244 100644 --- a/src/Progmasoft/Runner/SchedulerModel.hpp +++ b/src/Progmasoft/Runner/SchedulerModel.hpp @@ -116,6 +116,10 @@ namespace Progmasoft::Catch3::Runner::Scheduling /// Catalog indices this process has run, in order. std::vector<std::size_t> history; std::string startupOutput; + /// In a coverage run, where the profile of the attempt this process + /// was started for goes, without its extension; empty for a process + /// that runs other tests first or only probes. + std::string coverageProfile; }; struct Event final diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b6b9b52c..32d1ce57 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -166,6 +166,7 @@ add_executable(ProgmasoftSelfTest Progmasoft/RunnerConfig.tests.cpp Progmasoft/RunnerAnalysis.tests.cpp Progmasoft/RunnerImport.tests.cpp + Progmasoft/RunnerCoverage.tests.cpp Progmasoft/RunnerOutput.tests.cpp Progmasoft/Snapshot.tests.cpp Progmasoft/Stateful.tests.cpp @@ -180,9 +181,28 @@ target_link_libraries(RunnerFixture PRIVATE Catch3WithMain) target_compile_features(RunnerFixture PRIVATE cxx_std_17) add_dependencies(ProgmasoftSelfTest RunnerFixture) +# Tests built with coverage instrumentation, for the integration tests of +# --coverage: LLVM source-based coverage with Clang, gcov with GCC. MSVC has +# neither, so its builds skip those tests. +set(_catch3_coverage_environment "") +if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND NOT CMAKE_CXX_SIMULATE_ID STREQUAL "MSVC") + set(_catch3_coverage_flags -fprofile-instr-generate -fcoverage-mapping) +elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(_catch3_coverage_flags --coverage) +endif() +if(_catch3_coverage_flags) + add_executable(CoverageFixture Progmasoft/CoverageFixture.cpp) + target_link_libraries(CoverageFixture PRIVATE Catch3WithMain) + target_compile_features(CoverageFixture PRIVATE cxx_std_17) + target_compile_options(CoverageFixture PRIVATE ${_catch3_coverage_flags}) + target_link_options(CoverageFixture PRIVATE ${_catch3_coverage_flags}) + add_dependencies(ProgmasoftSelfTest CoverageFixture) + set(_catch3_coverage_environment "CATCH3_COVERAGE_FIXTURE=$<TARGET_FILE:CoverageFixture>") +endif() + add_test(NAME Progmasoft::PropertyTests COMMAND $<TARGET_FILE:ProgmasoftSelfTest> --no-history) set_tests_properties(Progmasoft::PropertyTests PROPERTIES - ENVIRONMENT "CATCH3_RUNNER_FIXTURE=$<TARGET_FILE:RunnerFixture>" + ENVIRONMENT "CATCH3_RUNNER_FIXTURE=$<TARGET_FILE:RunnerFixture>;${_catch3_coverage_environment}" ) if(BUILD_SHARED_LIBS AND WIN32) add_custom_command(TARGET SelfTest PRE_LINK diff --git a/tests/Progmasoft/BUILD.bazel b/tests/Progmasoft/BUILD.bazel index 871baaa8..9b94bacf 100644 --- a/tests/Progmasoft/BUILD.bazel +++ b/tests/Progmasoft/BUILD.bazel @@ -20,6 +20,7 @@ cc_test( "RunnerConfig.tests.cpp", "RunnerAnalysis.tests.cpp", "RunnerImport.tests.cpp", + "RunnerCoverage.tests.cpp", "RunnerOutput.tests.cpp", "Snapshot.tests.cpp", "Stateful.tests.cpp", diff --git a/tests/Progmasoft/CoverageFixture.cpp b/tests/Progmasoft/CoverageFixture.cpp new file mode 100644 index 00000000..a046a296 --- /dev/null +++ b/tests/Progmasoft/CoverageFixture.cpp @@ -0,0 +1,33 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 + +// A test binary built with coverage instrumentation, for the integration +// tests of `--coverage`: each test reaches lines the other does not. The +// markers at the ends of lines let the tests find those lines. + +#include <Progmasoft/Catch3/TestMacros.hpp> + +namespace +{ + int + AddOne(int value) + { + return value + 1; // coverage: add-one + } + + int + SubtractOne(int value) + { + return value - 1; // coverage: subtract-one + } +} // namespace + +TEST_CASE("Adds one", "[coverage]") +{ + CHECK(AddOne(1) == 2); +} + +TEST_CASE("Subtracts one", "[coverage]") +{ + CHECK(SubtractOne(1) == 0); +} diff --git a/tests/Progmasoft/RunnerCoverage.tests.cpp b/tests/Progmasoft/RunnerCoverage.tests.cpp new file mode 100644 index 00000000..9dcca663 --- /dev/null +++ b/tests/Progmasoft/RunnerCoverage.tests.cpp @@ -0,0 +1,226 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 + +// Per-test coverage: `--coverage`, the formats its reports read and write, +// and a run of a binary built with coverage instrumentation. + +#include <Progmasoft/Catch3/Matchers/String.hpp> +#include <Progmasoft/Runner/CoverageReport.hpp> +#include <Progmasoft/Runner/Files.hpp> +#include <Progmasoft/Runner/Json.hpp> +#include <Progmasoft/Runner/Options.hpp> +#include <Progmasoft/Runner/Process.hpp> +#include <algorithm> +#include <filesystem> +#include <string> +#include <vector> + +#include "RunnerFixtureClient.hpp" + +namespace +{ + namespace Runner = Progmasoft::Catch3::Runner; + using Catch::Matchers::ContainsSubstring; + using FixtureClient::Scratch; + + /// The lines of `file` that end with the marker `// coverage: name`. + std::size_t + MarkedLine(const std::string &file, const std::string &name) + { + auto text = Runner::ReadFile(file); + REQUIRE(text); + std::size_t line = 1; + std::size_t start = 0; + const std::string marker = "// coverage: " + name; + while (start < text->size()) + { + auto end = text->find('\n', start); + if (end == std::string::npos) + { + end = text->size(); + } + if (text->substr(start, end - start).find(marker) + != std::string::npos) + { + return line; + } + start = end + 1; + ++line; + } + FAIL("no line of " << file << " is marked " << name); + return 0; + } + + /// The lines a test of `coverage.json` reached in the fixture's source. + std::vector<std::size_t> + ReachedLines(const Runner::JsonValue &report, + const std::string &test, + std::string &source) + { + for (const auto &entry : report["tests"].Items()) + { + if (entry["name"].AsString() != test) + { + continue; + } + for (const auto &file : entry["files"].Items()) + { + const std::string path = file["file"].AsString(); + if (path.find("CoverageFixture.cpp") == std::string::npos) + { + continue; + } + source = path; + std::vector<std::size_t> lines; + for (const auto &line : file["lines"].Items()) + { + lines.push_back( + static_cast<std::size_t>(line.AsUnsigned())); + } + return lines; + } + } + FAIL("coverage.json has no lines of the fixture for '" << test << "'"); + return {}; + } + + bool + Contains(const std::vector<std::size_t> &lines, std::size_t line) + { + return std::find(lines.begin(), lines.end(), line) != lines.end(); + } +} // namespace + +TEST_CASE("--coverage takes a directory and needs worker processes", + "[progmasoft][runner][coverage]") +{ + auto options = Runner::ParseOptions({ "--coverage=out/coverage" }); + REQUIRE(options); + CHECK(options->coverageDirectory == "out/coverage"); + + auto inProcess = Runner::ParseOptions({ "--coverage=c", "--in-process" }); + REQUIRE_FALSE(inProcess); + CHECK_THAT(inProcess.GetError().Message(), + ContainsSubstring("cannot be used with --in-process")); + + auto empty = Runner::ParseOptions({ "--coverage=" }); + REQUIRE_FALSE(empty); +} + +TEST_CASE("LCOV tracefiles are read and written", + "[progmasoft][runner][coverage]") +{ + const auto map = Runner::ParseLcov("TN:\n" + "SF:/src/a.cpp\n" + "DA:3,2\n" + "DA:4,0\n" + "BRDA:3,0,0,2\n" + "BRDA:3,0,1,-\n" + "end_of_record\r\n" + "SF:/src/b.cpp\n" + "DA:1,1\n" + "end_of_record\n"); + REQUIRE(map.size() == 2); + const auto &a = map.at("/src/a.cpp"); + CHECK(a.lines.at(3) == 2); + CHECK(a.lines.at(4) == 0); + REQUIRE(a.branches.size() == 2); + CHECK(a.branches.at({ 3, 0 }) == 2); + CHECK(a.branches.at({ 3, 1 }) == 0); + CHECK(map.at("/src/b.cpp").lines.at(1) == 1); + + const std::string written = Runner::ToLcov(map); + CHECK_THAT(written, ContainsSubstring("SF:/src/a.cpp\nDA:3,2\nDA:4,0\n")); + CHECK_THAT(written, ContainsSubstring("BRDA:3,0,1,-\n")); + CHECK_THAT(written, ContainsSubstring("LF:2\nLH:1\nBRF:2\nBRH:1\n")); + const auto again = Runner::ParseLcov(written); + CHECK(again.at("/src/a.cpp").lines == a.lines); + CHECK(again.at("/src/a.cpp").branches == a.branches); +} + +TEST_CASE("gcov JSON adds up across data files", + "[progmasoft][runner][coverage]") +{ + Runner::CoverageMap map; + const std::string json + = R"({"format_version":"2","files":[{"file":"/src/a.c","lines":[)" + R"({"line_number":2,"count":3,"branches":[{"count":3},{"count":0}]},)" + R"({"line_number":5,"count":0,"branches":[]}]}]})"; + Runner::AddGcovJson(json, map); + Runner::AddGcovJson(json, map); + const auto &a = map.at("/src/a.c"); + CHECK(a.lines.at(2) == 6); + CHECK(a.lines.at(5) == 0); + CHECK(a.branches.at({ 2, 0 }) == 6); + CHECK(a.branches.at({ 2, 1 }) == 0); + Runner::AddGcovJson("not json", map); + CHECK(map.size() == 1); +} + +TEST_CASE("Each test of a coverage run gets its own profile and map", + "[progmasoft][runner][coverage]") +{ + const auto fixture = Runner::ReadEnvironment("CATCH3_COVERAGE_FIXTURE"); + if (fixture.empty()) + { + SKIP("CATCH3_COVERAGE_FIXTURE names no binary built with coverage"); + } + const Scratch scratch; + const auto directory = scratch.Path() / "coverage"; + // A profile of an earlier run is removed, not added to. + std::filesystem::create_directories(directory / "profiles"); + REQUIRE( + Runner::WriteFileAtomically(directory / "profiles" / "stale.profraw", + "stale")); + + Runner::ChildOptions options; + options.arguments = { FixtureClient::FromRunfiles(fixture).string(), + "--coverage=" + directory.string(), + "--no-history", + "--color=never" }; + options.workingDirectory = scratch.Path(); + auto run = Runner::RunAndCapture(options); + REQUIRE(run); + const std::string output = run->output + run->errors; + INFO(output); + REQUIRE(run->status.Succeeded()); + CHECK_FALSE( + std::filesystem::exists(directory / "profiles" / "stale.profraw")); + CHECK_THAT(output, ContainsSubstring("--coverage: 2 test profile(s)")); + + if (output.find("not found, so there is no map") != std::string::npos) + { + // CI sets the variable where the tools are installed, so that a + // missing map fails there instead of passing as skipped. + if (!Runner::ReadEnvironment("CATCH3_COVERAGE_REQUIRE_MAP").empty()) + { + FAIL("the coverage tools are not on PATH"); + } + SKIP("the coverage tools of the compiler are not on PATH"); + } + auto text = Runner::ReadFile(directory / "coverage.json"); + REQUIRE(text); + auto report = Runner::ParseJson(*text); + REQUIRE(report); + CHECK(((*report)["format"].AsString() == "llvm" + || (*report)["format"].AsString() == "gcov")); + REQUIRE((*report)["tests"].Items().size() == 2); + + std::string source; + const auto adds = ReachedLines(*report, "Adds one", source); + const auto subtracts = ReachedLines(*report, "Subtracts one", source); + const auto addLine = MarkedLine(source, "add-one"); + const auto subtractLine = MarkedLine(source, "subtract-one"); + CHECK(Contains(adds, addLine)); + CHECK_FALSE(Contains(adds, subtractLine)); + CHECK(Contains(subtracts, subtractLine)); + CHECK_FALSE(Contains(subtracts, addLine)); + + auto lcov = Runner::ReadFile(directory / "coverage.lcov"); + REQUIRE(lcov); + const auto combined = Runner::ParseLcov(*lcov); + const auto file = combined.find(source); + REQUIRE(file != combined.end()); + CHECK(file->second.lines.at(addLine) > 0); + CHECK(file->second.lines.at(subtractLine) > 0); +} diff --git a/tests/meson.build b/tests/meson.build index f6b26899..f4953a19 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -84,6 +84,7 @@ progmasoft_property_tests = executable( 'Progmasoft/RunnerConfig.tests.cpp', 'Progmasoft/RunnerAnalysis.tests.cpp', 'Progmasoft/RunnerImport.tests.cpp', + 'Progmasoft/RunnerCoverage.tests.cpp', 'Progmasoft/RunnerOutput.tests.cpp', 'Progmasoft/Snapshot.tests.cpp', 'Progmasoft/Stateful.tests.cpp', diff --git a/xmake.lua b/xmake.lua index 23f5dc85..7fd7075f 100644 --- a/xmake.lua +++ b/xmake.lua @@ -63,6 +63,7 @@ target("ProgmasoftPropertyTests") "tests/Progmasoft/RunnerConfig.tests.cpp", "tests/Progmasoft/RunnerAnalysis.tests.cpp", "tests/Progmasoft/RunnerImport.tests.cpp", + "tests/Progmasoft/RunnerCoverage.tests.cpp", "tests/Progmasoft/RunnerOutput.tests.cpp", "tests/Progmasoft/Snapshot.tests.cpp", "tests/Progmasoft/Stateful.tests.cpp",