Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/tests/Progmasoft/RunnerCoverage.tests.cpp
diff options
context:
space:
mode:
authorLeitwolf112026-11-13 21:55:26 +0300
committerLeitwolf112026-11-13 21:55:26 +0300
commit454cca9b63991416f4d9bd1ca350f1ae14e278e7 (patch)
treed85cafe2ee3ca7c5a8897a3e0fdde844fb18c6f6 /tests/Progmasoft/RunnerCoverage.tests.cpp
parent55dab34729c14c9019b7062fccd11fc22372e805 (diff)
downloadcatch3-454cca9b63991416f4d9bd1ca350f1ae14e278e7.tar.gz
Fix the coverage review findings: branches, stale reports, exit status
- gcov is run with --branch-probabilities; without it the JSON has no branches, so GCC runs had none in coverage.json and coverage.lcov. - PrepareCoverageDirectory also removes coverage.json, coverage.lcov and coverage.profdata of an earlier run, so a run that writes no reports leaves none to be read as its own. - WriteCoverageReports returns notes and failures apart. A profile the tools cannot read or a report that cannot be written counts as a missing report and fails a run whose tests passed; no profiles or missing tools stay notes. The paths are announced only for the reports written. - ProfiledAttempt takes the last attempt whose profile exists, so a final retry or repeat that crashed before writing one no longer hides the coverage of the attempt before it. - The per-test llvm-profdata/llvm-cov and gcov runs use as many threads as the run had workers; results keep the order of the tests. Tests: earlier reports removed and no failure without profiles; an attempt without a profile gives way to an earlier one; the fixture gains a branch taken one way, checked in the combined LCOV for both LLVM and gcov. Amalgamation regenerated.
Diffstat (limited to 'tests/Progmasoft/RunnerCoverage.tests.cpp')
-rw-r--r--tests/Progmasoft/RunnerCoverage.tests.cpp67
1 files changed, 67 insertions, 0 deletions
diff --git a/tests/Progmasoft/RunnerCoverage.tests.cpp b/tests/Progmasoft/RunnerCoverage.tests.cpp
index 9dcca663..b3e10489 100644
--- a/tests/Progmasoft/RunnerCoverage.tests.cpp
+++ b/tests/Progmasoft/RunnerCoverage.tests.cpp
@@ -157,6 +157,58 @@ TEST_CASE("gcov JSON adds up across data files",
CHECK(map.size() == 1);
}
+TEST_CASE("A coverage run starts without the reports of an earlier one",
+ "[progmasoft][runner][coverage]")
+{
+ const Scratch scratch;
+ const auto directory = scratch.Path() / "coverage";
+ std::filesystem::create_directories(directory / "profiles");
+ for (const char *name :
+ { "coverage.json", "coverage.lcov", "coverage.profdata" })
+ {
+ REQUIRE(Runner::WriteFileAtomically(directory / name, "earlier"));
+ }
+ REQUIRE(Runner::WriteFileAtomically(directory / "profiles" / "t.profraw",
+ "earlier"));
+
+ REQUIRE(Runner::PrepareCoverageDirectory(directory));
+ CHECK_FALSE(std::filesystem::exists(directory / "coverage.json"));
+ CHECK_FALSE(std::filesystem::exists(directory / "coverage.lcov"));
+ CHECK_FALSE(std::filesystem::exists(directory / "coverage.profdata"));
+ CHECK_FALSE(std::filesystem::exists(directory / "profiles" / "t.profraw"));
+ CHECK(std::filesystem::is_directory(directory / "profiles"));
+
+ // With no profiles there is a note, no report and no failure.
+ Progmasoft::Catch3::RunResult run;
+ run.tests.emplace_back();
+ run.tests.back().name = "no profile";
+ const auto reports
+ = Runner::WriteCoverageReports(directory, run, "binary", 2);
+ CHECK(reports.failures.empty());
+ REQUIRE(reports.notes.size() == 1);
+ CHECK_THAT(reports.notes.front(), ContainsSubstring("wrote no coverage"));
+ CHECK_FALSE(std::filesystem::exists(directory / "coverage.json"));
+}
+
+TEST_CASE("An attempt that left no profile gives way to an earlier one",
+ "[progmasoft][runner][coverage]")
+{
+ const Scratch scratch;
+ const auto base = (scratch.Path() / "test").string();
+ Progmasoft::Catch3::TestResult test;
+ test.attempts.resize(3);
+ test.attempts[0].coverageProfile = base + "-1";
+ test.attempts[1].coverageProfile = base + "-2";
+ test.attempts[2].coverageProfile = base + "-3"; // crashed: no profile
+ REQUIRE(Runner::WriteFileAtomically(base + "-1.profraw", "1"));
+ REQUIRE(Runner::WriteFileAtomically(base + "-2.profraw", "2"));
+
+ const auto *attempt = Runner::ProfiledAttempt(test, ".profraw");
+ REQUIRE(attempt != nullptr);
+ CHECK(attempt == &test.attempts[1]);
+ CHECK(Runner::ProfiledAttempt(test, ".gcov") == nullptr);
+}
+
TEST_CASE("Each test of a coverage run gets its own profile and map",
"[progmasoft][runner][coverage]")
{
@@ -223,4 +275,19 @@ TEST_CASE("Each test of a coverage run gets its own profile and map",
REQUIRE(file != combined.end());
CHECK(file->second.lines.at(addLine) > 0);
CHECK(file->second.lines.at(subtractLine) > 0);
+
+ // Both formats carry branches: the condition of the fixture went one
+ // way only, so one of its branches was taken and one was not.
+ const auto branchLine = MarkedLine(source, "branch");
+ std::size_t taken = 0;
+ std::size_t notTaken = 0;
+ for (const auto &[position, count] : file->second.branches)
+ {
+ if (position.first == branchLine)
+ {
+ (count > 0 ? taken : notTaken) += 1;
+ }
+ }
+ CHECK(taken > 0);
+ CHECK(notTaken > 0);
}