Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/src/Progmasoft/Runner/CoverageReport.cpp
diff options
context:
space:
mode:
authorLeitwolf112026-11-13 21:55:26 +0300
committerLeitwolf112026-11-13 21:55:26 +0300
commit454cca9b63991416f4d9bd1ca350f1ae14e278e7 (patch)
treed85cafe2ee3ca7c5a8897a3e0fdde844fb18c6f6 /src/Progmasoft/Runner/CoverageReport.cpp
parent55dab34729c14c9019b7062fccd11fc22372e805 (diff)
downloadcatch3-454cca9b63991416f4d9bd1ca350f1ae14e278e7.tar.gz
Fix the coverage review findings: branches, stale reports, exit status
- gcov is run with --branch-probabilities; without it the JSON has no branches, so GCC runs had none in coverage.json and coverage.lcov. - PrepareCoverageDirectory also removes coverage.json, coverage.lcov and coverage.profdata of an earlier run, so a run that writes no reports leaves none to be read as its own. - WriteCoverageReports returns notes and failures apart. A profile the tools cannot read or a report that cannot be written counts as a missing report and fails a run whose tests passed; no profiles or missing tools stay notes. The paths are announced only for the reports written. - ProfiledAttempt takes the last attempt whose profile exists, so a final retry or repeat that crashed before writing one no longer hides the coverage of the attempt before it. - The per-test llvm-profdata/llvm-cov and gcov runs use as many threads as the run had workers; results keep the order of the tests. Tests: earlier reports removed and no failure without profiles; an attempt without a profile gives way to an earlier one; the fixture gains a branch taken one way, checked in the combined LCOV for both LLVM and gcov. Amalgamation regenerated.
Diffstat (limited to 'src/Progmasoft/Runner/CoverageReport.cpp')
-rw-r--r--src/Progmasoft/Runner/CoverageReport.cpp337
1 files changed, 213 insertions, 124 deletions
diff --git a/src/Progmasoft/Runner/CoverageReport.cpp b/src/Progmasoft/Runner/CoverageReport.cpp
index 68d6e5ee..fb2760c3 100644
--- a/src/Progmasoft/Runner/CoverageReport.cpp
+++ b/src/Progmasoft/Runner/CoverageReport.cpp
@@ -1,10 +1,13 @@
// Copyright (c) 2026 Progmasoft.
// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0
#include <algorithm>
+#include <atomic>
#include <cstdlib>
+#include <initializer_list>
#include <optional>
#include <string_view>
#include <system_error>
+#include <thread>
#include "CoverageReport.hpp"
#include "Files.hpp"
@@ -185,22 +188,6 @@ namespace Progmasoft::Catch3::Runner
return files;
}
- /// The final attempt of a test that left a profile.
- const Attempt *
- ProfiledAttempt(const TestResult &test)
- {
- for (auto attempt = test.attempts.rbegin();
- attempt != test.attempts.rend();
- ++attempt)
- {
- if (!attempt->coverageProfile.empty())
- {
- return &*attempt;
- }
- }
- return nullptr;
- }
-
/// The .gcda files gcov wrote below `prefix`, each with the path the
/// object was built at, where its .gcno is.
std::vector<std::pair<std::filesystem::path, std::filesystem::path>>
@@ -254,7 +241,11 @@ namespace Progmasoft::Catch3::Runner
}
auto json = RunTool(
gcov,
- { "--json-format", "--stdout", gcda.filename().string() },
+ // Without --branch-probabilities the JSON has no branches.
+ { "--json-format",
+ "--branch-probabilities",
+ "--stdout",
+ gcda.filename().string() },
gcda.parent_path());
if (!json)
{
@@ -401,6 +392,17 @@ namespace Progmasoft::Catch3::Runner
+ directory.string()
+ " could not be removed: " + error.message() };
}
+ for (const char *report :
+ { "coverage.json", "coverage.lcov", "coverage.profdata" })
+ {
+ std::filesystem::remove(directory / report, error);
+ if (error)
+ {
+ return Error{ "the report " + (directory / report).string()
+ + " of an earlier run could not be removed: "
+ + error.message() };
+ }
+ }
std::filesystem::create_directories(directory / "profiles", error);
if (error)
{
@@ -410,142 +412,215 @@ namespace Progmasoft::Catch3::Runner
return Status::Ok();
}
- std::vector<std::string>
+ const Attempt *
+ ProfiledAttempt(const TestResult &test, const std::string &suffix)
+ {
+ for (auto attempt = test.attempts.rbegin();
+ attempt != test.attempts.rend();
+ ++attempt)
+ {
+ std::error_code error;
+ if (!attempt->coverageProfile.empty()
+ && std::filesystem::exists(attempt->coverageProfile + suffix,
+ error))
+ {
+ return &*attempt;
+ }
+ }
+ return nullptr;
+ }
+
+ namespace
+ {
+ /// The coverage tools of one profile format.
+ struct CoverageTools final
+ {
+ bool llvm = false;
+ std::optional<Tool> profdata;
+ std::optional<Tool> cov;
+ std::optional<Tool> gcov;
+ };
+
+ /// What the profile of one test gave.
+ struct TestCoverage final
+ {
+ const TestResult *test = nullptr;
+ std::string profile;
+ std::optional<CoverageMap> reached;
+ std::string failure;
+ };
+
+ TestCoverage
+ ReadTestCoverage(const TestResult &test,
+ const Attempt &attempt,
+ const CoverageTools &tools,
+ const std::string &program)
+ {
+ TestCoverage result;
+ result.test = &test;
+ const std::string base = attempt.coverageProfile;
+ if (tools.llvm)
+ {
+ result.profile = base + ".profraw";
+ auto merged = RunTool(*tools.profdata,
+ { "merge",
+ "-sparse",
+ result.profile,
+ "-o",
+ base + ".profdata" });
+ if (!merged)
+ {
+ result.failure = merged.GetError().Message();
+ return result;
+ }
+ auto lcov = RunTool(*tools.cov,
+ { "export",
+ "-format=lcov",
+ "-instr-profile=" + base + ".profdata",
+ program });
+ if (!lcov)
+ {
+ result.failure = lcov.GetError().Message();
+ return result;
+ }
+ result.reached = ParseLcov(*lcov);
+ return result;
+ }
+ result.profile = base + ".gcov";
+ auto reached = GcovCoverage(*tools.gcov, result.profile);
+ if (!reached)
+ {
+ result.failure = reached.GetError().Message();
+ return result;
+ }
+ result.reached = std::move(*reached);
+ return result;
+ }
+
+ /// Reads the profile of each of `tests`, up to `jobs` at a time;
+ /// the results keep the order of `tests`.
+ std::vector<TestCoverage>
+ ReadAll(
+ const std::vector<std::pair<const TestResult *, const Attempt *>>
+ &tests,
+ const CoverageTools &tools,
+ const std::string &program,
+ std::size_t jobs)
+ {
+ std::vector<TestCoverage> results(tests.size());
+ std::atomic<std::size_t> next{ 0 };
+ const auto work = [&] {
+ for (std::size_t index = next++; index < tests.size();
+ index = next++)
+ {
+ results[index] = ReadTestCoverage(*tests[index].first,
+ *tests[index].second,
+ tools,
+ program);
+ }
+ };
+ const std::size_t threads
+ = std::min(std::max<std::size_t>(jobs, 1), tests.size());
+ std::vector<std::thread> pool;
+ for (std::size_t thread = 1; thread < threads; ++thread)
+ {
+ pool.emplace_back(work);
+ }
+ work();
+ for (auto &thread : pool)
+ {
+ thread.join();
+ }
+ return results;
+ }
+ } // namespace
+
+ CoverageReports
WriteCoverageReports(const std::filesystem::path &directory,
const RunResult &run,
- const std::string &program)
+ const std::string &program,
+ std::size_t jobs)
{
- std::vector<std::string> notes;
- std::size_t llvmProfiles = 0;
- std::size_t gcovProfiles = 0;
+ CoverageReports reports;
+ std::vector<std::pair<const TestResult *, const Attempt *>> llvmTests;
+ std::vector<std::pair<const TestResult *, const Attempt *>> gcovTests;
for (const auto &test : run.tests)
{
- if (const Attempt *attempt = ProfiledAttempt(test))
+ if (const Attempt *attempt = ProfiledAttempt(test, ".profraw"))
{
- std::error_code error;
- llvmProfiles += std::filesystem::exists(attempt->coverageProfile
- + ".profraw",
- error)
- ? 1
- : 0;
- gcovProfiles += std::filesystem::exists(attempt->coverageProfile
- + ".gcov",
- error)
- ? 1
- : 0;
+ llvmTests.emplace_back(&test, attempt);
+ }
+ if (const Attempt *attempt = ProfiledAttempt(test, ".gcov"))
+ {
+ gcovTests.emplace_back(&test, attempt);
}
}
- if (llvmProfiles == 0 && gcovProfiles == 0)
+ if (llvmTests.empty() && gcovTests.empty())
{
- notes.emplace_back(
+ reports.notes.emplace_back(
"--coverage: the tests wrote no coverage; build them with "
"-fprofile-instr-generate -fcoverage-mapping (Clang) or "
"--coverage (GCC)");
- return notes;
+ return reports;
}
- const bool llvm = llvmProfiles >= gcovProfiles;
- std::optional<Tool> profdata;
- std::optional<Tool> cov;
- std::optional<Tool> gcov;
- if (llvm)
+ CoverageTools tools;
+ tools.llvm = llvmTests.size() >= gcovTests.size();
+ const auto &profiled = tools.llvm ? llvmTests : gcovTests;
+ if (tools.llvm)
{
- profdata = FindTool("CATCH3_LLVM_PROFDATA", "llvm-profdata");
- cov = FindTool("CATCH3_LLVM_COV", "llvm-cov");
+ tools.profdata = FindTool("CATCH3_LLVM_PROFDATA", "llvm-profdata");
+ tools.cov = FindTool("CATCH3_LLVM_COV", "llvm-cov");
}
else
{
- gcov = FindTool("CATCH3_GCOV", "gcov");
+ tools.gcov = FindTool("CATCH3_GCOV", "gcov");
}
- if (llvm ? (!profdata || !cov) : !gcov)
+ if (tools.llvm ? (!tools.profdata || !tools.cov) : !tools.gcov)
{
- notes.push_back(
- "--coverage: "
- + std::to_string(llvm ? llvmProfiles : gcovProfiles)
+ reports.notes.push_back(
+ "--coverage: " + std::to_string(profiled.size())
+ " profile(s) in " + (directory / "profiles").string() + "; "
- + (llvm ? "llvm-profdata and llvm-cov were" : "gcov was")
+ + (tools.llvm ? "llvm-profdata and llvm-cov were" : "gcov was")
+ " not found, so there is no map of what each test reached");
- return notes;
+ return reports;
}
CoverageMap combined;
std::vector<std::string> raw;
JsonValue tests = JsonValue::Array();
- std::vector<std::string> failures;
- for (const auto &test : run.tests)
+ for (auto &read : ReadAll(profiled, tools, program, jobs))
{
- const Attempt *attempt = ProfiledAttempt(test);
- if (attempt == nullptr)
+ if (!read.reached)
{
+ reports.failures.push_back(read.test->name + ": "
+ + read.failure);
continue;
}
- const std::string base = attempt->coverageProfile;
- Expected<CoverageMap> reached = CoverageMap{};
- std::string profile;
- if (llvm)
+ if (tools.llvm)
{
- profile = base + ".profraw";
- std::error_code error;
- if (!std::filesystem::exists(profile, error))
- {
- continue;
- }
- raw.push_back(profile);
- auto merged = RunTool(
- *profdata,
- { "merge", "-sparse", profile, "-o", base + ".profdata" });
- if (!merged)
- {
- failures.push_back(test.name + ": "
- + merged.GetError().Message());
- continue;
- }
- auto lcov = RunTool(*cov,
- { "export",
- "-format=lcov",
- "-instr-profile=" + base + ".profdata",
- program });
- if (!lcov)
- {
- failures.push_back(test.name + ": "
- + lcov.GetError().Message());
- continue;
- }
- reached = ParseLcov(*lcov);
+ raw.push_back(read.profile);
}
else
{
- profile = base + ".gcov";
- std::error_code error;
- if (!std::filesystem::exists(profile, error))
- {
- continue;
- }
- reached = GcovCoverage(*gcov, profile);
- if (!reached)
- {
- failures.push_back(test.name + ": "
- + reached.GetError().Message());
- continue;
- }
- Add(combined, *reached);
+ Add(combined, *read.reached);
}
- tests.Append(
- JsonValue::Object({ { "name", test.name },
- { "outcome", OutcomeName(test.outcome) },
- { "profile", profile },
- { "files", ReachedJson(*reached) } }));
+ tests.Append(JsonValue::Object(
+ { { "name", read.test->name },
+ { "outcome", OutcomeName(read.test->outcome) },
+ { "profile", read.profile },
+ { "files", ReachedJson(*read.reached) } }));
}
std::string lcovText;
- if (llvm && !raw.empty())
+ if (tools.llvm && !raw.empty())
{
std::vector<std::string> arguments{ "merge", "-sparse" };
arguments.insert(arguments.end(), raw.begin(), raw.end());
const std::string all = (directory / "coverage.profdata").string();
arguments.emplace_back("-o");
arguments.push_back(all);
- auto merged = RunTool(*profdata, arguments);
- auto exported = merged ? RunTool(*cov,
+ auto merged = RunTool(*tools.profdata, arguments);
+ auto exported = merged ? RunTool(*tools.cov,
{ "export",
"-format=lcov",
"-instr-profile=" + all,
@@ -557,8 +632,8 @@ namespace Progmasoft::Catch3::Runner
}
else
{
- failures.push_back("the combined report: "
- + exported.GetError().Message());
+ reports.failures.push_back("the combined report: "
+ + exported.GetError().Message());
}
}
else
@@ -568,28 +643,42 @@ namespace Progmasoft::Catch3::Runner
JsonValue report
= JsonValue::Object({ { "version", 1 },
- { "format", llvm ? "llvm" : "gcov" },
+ { "format", tools.llvm ? "llvm" : "gcov" },
{ "binary", program },
{ "tests", std::move(tests) } });
const auto json = directory / "coverage.json";
const auto lcov = directory / "coverage.lcov";
- if (auto status = WriteFileAtomically(json, report.ToPrettyString());
- !status)
+ const auto jsonWritten
+ = WriteFileAtomically(json, report.ToPrettyString());
+ if (!jsonWritten)
+ {
+ reports.failures.push_back(jsonWritten.GetError().Message());
+ }
+ const auto lcovWritten = WriteFileAtomically(lcov, lcovText);
+ if (!lcovWritten)
+ {
+ reports.failures.push_back(lcovWritten.GetError().Message());
+ }
+ std::string written;
+ if (jsonWritten && lcovWritten)
+ {
+ written = "; what each test reached is in " + json.string()
+ + ", the run combined in " + lcov.string();
+ }
+ else if (jsonWritten)
{
- failures.push_back(status.GetError().Message());
+ written = "; what each test reached is in " + json.string();
}
- if (auto status = WriteFileAtomically(lcov, lcovText); !status)
+ else if (lcovWritten)
{
- failures.push_back(status.GetError().Message());
+ written = "; the run combined is in " + lcov.string();
}
- notes.push_back(
- "--coverage: " + std::to_string(llvm ? llvmProfiles : gcovProfiles)
- + " test profile(s); what each test reached is in " + json.string()
- + ", the run combined in " + lcov.string());
- for (const auto &failure : failures)
+ reports.notes.push_back("--coverage: " + std::to_string(profiled.size())
+ + " test profile(s)" + written);
+ for (auto &failure : reports.failures)
{
- notes.push_back("--coverage: " + failure);
+ failure = "--coverage: " + failure;
}
- return notes;
+ return reports;
}
} // namespace Progmasoft::Catch3::Runner