diff options
| author | Leitwolf11 | 2026-11-13 21:55:26 +0300 |
|---|---|---|
| committer | Leitwolf11 | 2026-11-13 21:55:26 +0300 |
| commit | 454cca9b63991416f4d9bd1ca350f1ae14e278e7 (patch) | |
| tree | d85cafe2ee3ca7c5a8897a3e0fdde844fb18c6f6 /src/Progmasoft/Runner/CoverageReport.cpp | |
| parent | 55dab34729c14c9019b7062fccd11fc22372e805 (diff) | |
| download | catch3-454cca9b63991416f4d9bd1ca350f1ae14e278e7.tar.gz | |
Fix the coverage review findings: branches, stale reports, exit status
- gcov is run with --branch-probabilities; without it the JSON has no
branches, so GCC runs had none in coverage.json and coverage.lcov.
- PrepareCoverageDirectory also removes coverage.json, coverage.lcov and
coverage.profdata of an earlier run, so a run that writes no reports
leaves none to be read as its own.
- WriteCoverageReports returns notes and failures apart. A profile the
tools cannot read or a report that cannot be written counts as a missing
report and fails a run whose tests passed; no profiles or missing tools
stay notes. The paths are announced only for the reports written.
- ProfiledAttempt takes the last attempt whose profile exists, so a final
retry or repeat that crashed before writing one no longer hides the
coverage of the attempt before it.
- The per-test llvm-profdata/llvm-cov and gcov runs use as many threads
as the run had workers; results keep the order of the tests.
Tests: earlier reports removed and no failure without profiles; an
attempt without a profile gives way to an earlier one; the fixture gains a
branch taken one way, checked in the combined LCOV for both LLVM and gcov.
Amalgamation regenerated.
Diffstat (limited to 'src/Progmasoft/Runner/CoverageReport.cpp')
| -rw-r--r-- | src/Progmasoft/Runner/CoverageReport.cpp | 337 |
1 files changed, 213 insertions, 124 deletions
diff --git a/src/Progmasoft/Runner/CoverageReport.cpp b/src/Progmasoft/Runner/CoverageReport.cpp index 68d6e5ee..fb2760c3 100644 --- a/src/Progmasoft/Runner/CoverageReport.cpp +++ b/src/Progmasoft/Runner/CoverageReport.cpp @@ -1,10 +1,13 @@ // Copyright (c) 2026 Progmasoft. // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 #include <algorithm> +#include <atomic> #include <cstdlib> +#include <initializer_list> #include <optional> #include <string_view> #include <system_error> +#include <thread> #include "CoverageReport.hpp" #include "Files.hpp" @@ -185,22 +188,6 @@ namespace Progmasoft::Catch3::Runner return files; } - /// The final attempt of a test that left a profile. - const Attempt * - ProfiledAttempt(const TestResult &test) - { - for (auto attempt = test.attempts.rbegin(); - attempt != test.attempts.rend(); - ++attempt) - { - if (!attempt->coverageProfile.empty()) - { - return &*attempt; - } - } - return nullptr; - } - /// The .gcda files gcov wrote below `prefix`, each with the path the /// object was built at, where its .gcno is. std::vector<std::pair<std::filesystem::path, std::filesystem::path>> @@ -254,7 +241,11 @@ namespace Progmasoft::Catch3::Runner } auto json = RunTool( gcov, - { "--json-format", "--stdout", gcda.filename().string() }, + // Without --branch-probabilities the JSON has no branches. + { "--json-format", + "--branch-probabilities", + "--stdout", + gcda.filename().string() }, gcda.parent_path()); if (!json) { @@ -401,6 +392,17 @@ namespace Progmasoft::Catch3::Runner + directory.string() + " could not be removed: " + error.message() }; } + for (const char *report : + { "coverage.json", "coverage.lcov", "coverage.profdata" }) + { + std::filesystem::remove(directory / report, error); + if (error) + { + return Error{ "the report " + (directory / report).string() + + " of an earlier run could not be removed: " + + error.message() }; + } + } std::filesystem::create_directories(directory / "profiles", error); if (error) { @@ -410,142 +412,215 @@ namespace Progmasoft::Catch3::Runner return Status::Ok(); } - std::vector<std::string> + const Attempt * + ProfiledAttempt(const TestResult &test, const std::string &suffix) + { + for (auto attempt = test.attempts.rbegin(); + attempt != test.attempts.rend(); + ++attempt) + { + std::error_code error; + if (!attempt->coverageProfile.empty() + && std::filesystem::exists(attempt->coverageProfile + suffix, + error)) + { + return &*attempt; + } + } + return nullptr; + } + + namespace + { + /// The coverage tools of one profile format. + struct CoverageTools final + { + bool llvm = false; + std::optional<Tool> profdata; + std::optional<Tool> cov; + std::optional<Tool> gcov; + }; + + /// What the profile of one test gave. + struct TestCoverage final + { + const TestResult *test = nullptr; + std::string profile; + std::optional<CoverageMap> reached; + std::string failure; + }; + + TestCoverage + ReadTestCoverage(const TestResult &test, + const Attempt &attempt, + const CoverageTools &tools, + const std::string &program) + { + TestCoverage result; + result.test = &test; + const std::string base = attempt.coverageProfile; + if (tools.llvm) + { + result.profile = base + ".profraw"; + auto merged = RunTool(*tools.profdata, + { "merge", + "-sparse", + result.profile, + "-o", + base + ".profdata" }); + if (!merged) + { + result.failure = merged.GetError().Message(); + return result; + } + auto lcov = RunTool(*tools.cov, + { "export", + "-format=lcov", + "-instr-profile=" + base + ".profdata", + program }); + if (!lcov) + { + result.failure = lcov.GetError().Message(); + return result; + } + result.reached = ParseLcov(*lcov); + return result; + } + result.profile = base + ".gcov"; + auto reached = GcovCoverage(*tools.gcov, result.profile); + if (!reached) + { + result.failure = reached.GetError().Message(); + return result; + } + result.reached = std::move(*reached); + return result; + } + + /// Reads the profile of each of `tests`, up to `jobs` at a time; + /// the results keep the order of `tests`. + std::vector<TestCoverage> + ReadAll( + const std::vector<std::pair<const TestResult *, const Attempt *>> + &tests, + const CoverageTools &tools, + const std::string &program, + std::size_t jobs) + { + std::vector<TestCoverage> results(tests.size()); + std::atomic<std::size_t> next{ 0 }; + const auto work = [&] { + for (std::size_t index = next++; index < tests.size(); + index = next++) + { + results[index] = ReadTestCoverage(*tests[index].first, + *tests[index].second, + tools, + program); + } + }; + const std::size_t threads + = std::min(std::max<std::size_t>(jobs, 1), tests.size()); + std::vector<std::thread> pool; + for (std::size_t thread = 1; thread < threads; ++thread) + { + pool.emplace_back(work); + } + work(); + for (auto &thread : pool) + { + thread.join(); + } + return results; + } + } // namespace + + CoverageReports WriteCoverageReports(const std::filesystem::path &directory, const RunResult &run, - const std::string &program) + const std::string &program, + std::size_t jobs) { - std::vector<std::string> notes; - std::size_t llvmProfiles = 0; - std::size_t gcovProfiles = 0; + CoverageReports reports; + std::vector<std::pair<const TestResult *, const Attempt *>> llvmTests; + std::vector<std::pair<const TestResult *, const Attempt *>> gcovTests; for (const auto &test : run.tests) { - if (const Attempt *attempt = ProfiledAttempt(test)) + if (const Attempt *attempt = ProfiledAttempt(test, ".profraw")) { - std::error_code error; - llvmProfiles += std::filesystem::exists(attempt->coverageProfile - + ".profraw", - error) - ? 1 - : 0; - gcovProfiles += std::filesystem::exists(attempt->coverageProfile - + ".gcov", - error) - ? 1 - : 0; + llvmTests.emplace_back(&test, attempt); + } + if (const Attempt *attempt = ProfiledAttempt(test, ".gcov")) + { + gcovTests.emplace_back(&test, attempt); } } - if (llvmProfiles == 0 && gcovProfiles == 0) + if (llvmTests.empty() && gcovTests.empty()) { - notes.emplace_back( + reports.notes.emplace_back( "--coverage: the tests wrote no coverage; build them with " "-fprofile-instr-generate -fcoverage-mapping (Clang) or " "--coverage (GCC)"); - return notes; + return reports; } - const bool llvm = llvmProfiles >= gcovProfiles; - std::optional<Tool> profdata; - std::optional<Tool> cov; - std::optional<Tool> gcov; - if (llvm) + CoverageTools tools; + tools.llvm = llvmTests.size() >= gcovTests.size(); + const auto &profiled = tools.llvm ? llvmTests : gcovTests; + if (tools.llvm) { - profdata = FindTool("CATCH3_LLVM_PROFDATA", "llvm-profdata"); - cov = FindTool("CATCH3_LLVM_COV", "llvm-cov"); + tools.profdata = FindTool("CATCH3_LLVM_PROFDATA", "llvm-profdata"); + tools.cov = FindTool("CATCH3_LLVM_COV", "llvm-cov"); } else { - gcov = FindTool("CATCH3_GCOV", "gcov"); + tools.gcov = FindTool("CATCH3_GCOV", "gcov"); } - if (llvm ? (!profdata || !cov) : !gcov) + if (tools.llvm ? (!tools.profdata || !tools.cov) : !tools.gcov) { - notes.push_back( - "--coverage: " - + std::to_string(llvm ? llvmProfiles : gcovProfiles) + reports.notes.push_back( + "--coverage: " + std::to_string(profiled.size()) + " profile(s) in " + (directory / "profiles").string() + "; " - + (llvm ? "llvm-profdata and llvm-cov were" : "gcov was") + + (tools.llvm ? "llvm-profdata and llvm-cov were" : "gcov was") + " not found, so there is no map of what each test reached"); - return notes; + return reports; } CoverageMap combined; std::vector<std::string> raw; JsonValue tests = JsonValue::Array(); - std::vector<std::string> failures; - for (const auto &test : run.tests) + for (auto &read : ReadAll(profiled, tools, program, jobs)) { - const Attempt *attempt = ProfiledAttempt(test); - if (attempt == nullptr) + if (!read.reached) { + reports.failures.push_back(read.test->name + ": " + + read.failure); continue; } - const std::string base = attempt->coverageProfile; - Expected<CoverageMap> reached = CoverageMap{}; - std::string profile; - if (llvm) + if (tools.llvm) { - profile = base + ".profraw"; - std::error_code error; - if (!std::filesystem::exists(profile, error)) - { - continue; - } - raw.push_back(profile); - auto merged = RunTool( - *profdata, - { "merge", "-sparse", profile, "-o", base + ".profdata" }); - if (!merged) - { - failures.push_back(test.name + ": " - + merged.GetError().Message()); - continue; - } - auto lcov = RunTool(*cov, - { "export", - "-format=lcov", - "-instr-profile=" + base + ".profdata", - program }); - if (!lcov) - { - failures.push_back(test.name + ": " - + lcov.GetError().Message()); - continue; - } - reached = ParseLcov(*lcov); + raw.push_back(read.profile); } else { - profile = base + ".gcov"; - std::error_code error; - if (!std::filesystem::exists(profile, error)) - { - continue; - } - reached = GcovCoverage(*gcov, profile); - if (!reached) - { - failures.push_back(test.name + ": " - + reached.GetError().Message()); - continue; - } - Add(combined, *reached); + Add(combined, *read.reached); } - tests.Append( - JsonValue::Object({ { "name", test.name }, - { "outcome", OutcomeName(test.outcome) }, - { "profile", profile }, - { "files", ReachedJson(*reached) } })); + tests.Append(JsonValue::Object( + { { "name", read.test->name }, + { "outcome", OutcomeName(read.test->outcome) }, + { "profile", read.profile }, + { "files", ReachedJson(*read.reached) } })); } std::string lcovText; - if (llvm && !raw.empty()) + if (tools.llvm && !raw.empty()) { std::vector<std::string> arguments{ "merge", "-sparse" }; arguments.insert(arguments.end(), raw.begin(), raw.end()); const std::string all = (directory / "coverage.profdata").string(); arguments.emplace_back("-o"); arguments.push_back(all); - auto merged = RunTool(*profdata, arguments); - auto exported = merged ? RunTool(*cov, + auto merged = RunTool(*tools.profdata, arguments); + auto exported = merged ? RunTool(*tools.cov, { "export", "-format=lcov", "-instr-profile=" + all, @@ -557,8 +632,8 @@ namespace Progmasoft::Catch3::Runner } else { - failures.push_back("the combined report: " - + exported.GetError().Message()); + reports.failures.push_back("the combined report: " + + exported.GetError().Message()); } } else @@ -568,28 +643,42 @@ namespace Progmasoft::Catch3::Runner JsonValue report = JsonValue::Object({ { "version", 1 }, - { "format", llvm ? "llvm" : "gcov" }, + { "format", tools.llvm ? "llvm" : "gcov" }, { "binary", program }, { "tests", std::move(tests) } }); const auto json = directory / "coverage.json"; const auto lcov = directory / "coverage.lcov"; - if (auto status = WriteFileAtomically(json, report.ToPrettyString()); - !status) + const auto jsonWritten + = WriteFileAtomically(json, report.ToPrettyString()); + if (!jsonWritten) + { + reports.failures.push_back(jsonWritten.GetError().Message()); + } + const auto lcovWritten = WriteFileAtomically(lcov, lcovText); + if (!lcovWritten) + { + reports.failures.push_back(lcovWritten.GetError().Message()); + } + std::string written; + if (jsonWritten && lcovWritten) + { + written = "; what each test reached is in " + json.string() + + ", the run combined in " + lcov.string(); + } + else if (jsonWritten) { - failures.push_back(status.GetError().Message()); + written = "; what each test reached is in " + json.string(); } - if (auto status = WriteFileAtomically(lcov, lcovText); !status) + else if (lcovWritten) { - failures.push_back(status.GetError().Message()); + written = "; the run combined is in " + lcov.string(); } - notes.push_back( - "--coverage: " + std::to_string(llvm ? llvmProfiles : gcovProfiles) - + " test profile(s); what each test reached is in " + json.string() - + ", the run combined in " + lcov.string()); - for (const auto &failure : failures) + reports.notes.push_back("--coverage: " + std::to_string(profiled.size()) + + " test profile(s)" + written); + for (auto &failure : reports.failures) { - notes.push_back("--coverage: " + failure); + failure = "--coverage: " + failure; } - return notes; + return reports; } } // namespace Progmasoft::Catch3::Runner |