Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

develroot / tests / Progmasoft / RunnerCoverage.tests.cpp

10.9 KiB · Raw · Log

// Copyright (c) 2026 Progmasoft.
// SPDX-License-Identifier: LGPL-3.0-or-later

// Per-test coverage: `--coverage`, the formats its reports read and write,
// and a run of a binary built with coverage instrumentation.

#include <Progmasoft/Catch3/Matchers/String.hpp>
#include <Progmasoft/Runner/CoverageReport.hpp>
#include <Progmasoft/Runner/Files.hpp>
#include <Progmasoft/Runner/Json.hpp>
#include <Progmasoft/Runner/Options.hpp>
#include <Progmasoft/Runner/Process.hpp>
#include <algorithm>
#include <filesystem>
#include <string>
#include <vector>

#include "RunnerFixtureClient.hpp"

namespace
{
    namespace Runner = Progmasoft::Catch3::Runner;
    using Catch::Matchers::ContainsSubstring;
    using FixtureClient::Scratch;

    /// The lines of `file` that end with the marker `// coverage: name`.
    std::size_t
    MarkedLine(const std::string &file, const std::string &name)
    {
        auto text = Runner::ReadFile(file);
        REQUIRE(text);
        std::size_t line = 1;
        std::size_t start = 0;
        const std::string marker = "// coverage: " + name;
        while (start < text->size())
        {
            auto end = text->find('\n', start);
            if (end == std::string::npos)
            {
                end = text->size();
            }
            if (text->substr(start, end - start).find(marker)
                != std::string::npos)
            {
                return line;
            }
            start = end + 1;
            ++line;
        }
        FAIL("no line of " << file << " is marked " << name);
        return 0;
    }

    /// The lines a test of `coverage.json` reached in the fixture's source.
    std::vector<std::size_t>
    ReachedLines(const Runner::JsonValue &report,
                 const std::string &test,
                 std::string &source)
    {
        for (const auto &entry : report["tests"].Items())
        {
            if (entry["name"].AsString() != test)
            {
                continue;
            }
            for (const auto &file : entry["files"].Items())
            {
                const std::string path = file["file"].AsString();
                if (path.find("CoverageFixture.cpp") == std::string::npos)
                {
                    continue;
                }
                source = path;
                std::vector<std::size_t> lines;
                for (const auto &line : file["lines"].Items())
                {
                    lines.push_back(
                        static_cast<std::size_t>(line.AsUnsigned()));
                }
                return lines;
            }
        }
        FAIL("coverage.json has no lines of the fixture for '" << test << "'");
        return {};
    }

    bool
    Contains(const std::vector<std::size_t> &lines, std::size_t line)
    {
        return std::find(lines.begin(), lines.end(), line) != lines.end();
    }
} // namespace

TEST_CASE("--coverage takes a directory and needs worker processes",
          "[progmasoft][runner][coverage]")
{
    auto options = Runner::ParseOptions({ "--coverage=out/coverage" });
    REQUIRE(options);
    CHECK(options->coverageDirectory == "out/coverage");

    auto inProcess = Runner::ParseOptions({ "--coverage=c", "--in-process" });
    REQUIRE_FALSE(inProcess);
    CHECK_THAT(inProcess.GetError().Message(),
               ContainsSubstring("cannot be used with --in-process"));

    auto empty = Runner::ParseOptions({ "--coverage=" });
    REQUIRE_FALSE(empty);
}

TEST_CASE("LCOV tracefiles are read and written",
          "[progmasoft][runner][coverage]")
{
    const auto map = Runner::ParseLcov("TN:\n"
                                       "SF:/src/a.cpp\n"
                                       "DA:3,2\n"
                                       "DA:4,0\n"
                                       "BRDA:3,0,0,2\n"
                                       "BRDA:3,0,1,-\n"
                                       "end_of_record\r\n"
                                       "SF:/src/b.cpp\n"
                                       "DA:1,1\n"
                                       "end_of_record\n");
    REQUIRE(map.size() == 2);
    const auto &a = map.at("/src/a.cpp");
    CHECK(a.lines.at(3) == 2);
    CHECK(a.lines.at(4) == 0);
    REQUIRE(a.branches.size() == 2);
    CHECK(a.branches.at({ 3, 0 }) == 2);
    CHECK(a.branches.at({ 3, 1 }) == 0);
    CHECK(map.at("/src/b.cpp").lines.at(1) == 1);

    const std::string written = Runner::ToLcov(map);
    CHECK_THAT(written, ContainsSubstring("SF:/src/a.cpp\nDA:3,2\nDA:4,0\n"));
    CHECK_THAT(written, ContainsSubstring("BRDA:3,0,1,-\n"));
    CHECK_THAT(written, ContainsSubstring("LF:2\nLH:1\nBRF:2\nBRH:1\n"));
    const auto again = Runner::ParseLcov(written);
    CHECK(again.at("/src/a.cpp").lines == a.lines);
    CHECK(again.at("/src/a.cpp").branches == a.branches);
}

TEST_CASE("gcov JSON adds up across data files",
          "[progmasoft][runner][coverage]")
{
    Runner::CoverageMap map;
    const std::string json
        = R"({"format_version":"2","files":[{"file":"/src/a.c","lines":[)"
          R"({"line_number":2,"count":3,"branches":[{"count":3},{"count":0}]},)"
          R"({"line_number":5,"count":0,"branches":[]}]}]})";
    Runner::AddGcovJson(json, map);
    Runner::AddGcovJson(json, map);
    const auto &a = map.at("/src/a.c");
    CHECK(a.lines.at(2) == 6);
    CHECK(a.lines.at(5) == 0);
    CHECK(a.branches.at({ 2, 0 }) == 6);
    CHECK(a.branches.at({ 2, 1 }) == 0);
    Runner::AddGcovJson("not json", map);
    CHECK(map.size() == 1);
}

TEST_CASE("A coverage run starts without the reports of an earlier one",
          "[progmasoft][runner][coverage]")
{
    const Scratch scratch;
    const auto directory = scratch.Path() / "coverage";
    std::filesystem::create_directories(directory / "profiles");
    for (const char *name :
         { "coverage.json", "coverage.lcov", "coverage.profdata" })
    {
        REQUIRE(Runner::WriteFileAtomically(directory / name, "earlier"));
    }
    REQUIRE(Runner::WriteFileAtomically(directory / "profiles" / "t.profraw",
                                        "earlier"));

    REQUIRE(Runner::PrepareCoverageDirectory(directory));
    CHECK_FALSE(std::filesystem::exists(directory / "coverage.json"));
    CHECK_FALSE(std::filesystem::exists(directory / "coverage.lcov"));
    CHECK_FALSE(std::filesystem::exists(directory / "coverage.profdata"));
    CHECK_FALSE(std::filesystem::exists(directory / "profiles" / "t.profraw"));
    CHECK(std::filesystem::is_directory(directory / "profiles"));

    // With no profiles there is a note, no report and no failure.
    Progmasoft::Catch3::RunResult run;
    run.tests.emplace_back();
    run.tests.back().name = "no profile";
    const auto reports
        = Runner::WriteCoverageReports(directory, run, "binary", 2);
    CHECK(reports.failures.empty());
    REQUIRE(reports.notes.size() == 1);
    CHECK_THAT(reports.notes.front(), ContainsSubstring("wrote no coverage"));
    CHECK_FALSE(std::filesystem::exists(directory / "coverage.json"));
}

TEST_CASE("An attempt that left no profile gives way to an earlier one",
          "[progmasoft][runner][coverage]")
{
    const Scratch scratch;
    const auto base = (scratch.Path() / "test").string();
    Progmasoft::Catch3::TestResult test;
    test.attempts.resize(3);
    test.attempts[0].coverageProfile = base + "-1";
    test.attempts[1].coverageProfile = base + "-2";
    test.attempts[2].coverageProfile = base + "-3"; // crashed: no profile
    REQUIRE(Runner::WriteFileAtomically(base + "-1.profraw", "1"));
    REQUIRE(Runner::WriteFileAtomically(base + "-2.profraw", "2"));

    const auto *attempt = Runner::ProfiledAttempt(test, ".profraw");
    REQUIRE(attempt != nullptr);
    CHECK(attempt == &test.attempts[1]);
    CHECK(Runner::ProfiledAttempt(test, ".gcov") == nullptr);
}

TEST_CASE("Each test of a coverage run gets its own profile and map",
          "[progmasoft][runner][coverage]")
{
    const auto fixture = Runner::ReadEnvironment("CATCH3_COVERAGE_FIXTURE");
    if (fixture.empty())
    {
        SKIP("CATCH3_COVERAGE_FIXTURE names no binary built with coverage");
    }
    const Scratch scratch;
    const auto directory = scratch.Path() / "coverage";
    // A profile of an earlier run is removed, not added to.
    std::filesystem::create_directories(directory / "profiles");
    REQUIRE(
        Runner::WriteFileAtomically(directory / "profiles" / "stale.profraw",
                                    "stale"));

    Runner::ChildOptions options;
    options.arguments = { FixtureClient::FromRunfiles(fixture).string(),
                          "--coverage=" + directory.string(),
                          "--no-history",
                          "--color=never" };
    options.workingDirectory = scratch.Path();
    auto run = Runner::RunAndCapture(options);
    REQUIRE(run);
    const std::string output = run->output + run->errors;
    INFO(output);
    REQUIRE(run->status.Succeeded());
    CHECK_FALSE(
        std::filesystem::exists(directory / "profiles" / "stale.profraw"));
    CHECK_THAT(output, ContainsSubstring("--coverage: 2 test profile(s)"));

    if (output.find("not found, so there is no map") != std::string::npos)
    {
        // CI sets the variable where the tools are installed, so that a
        // missing map fails there instead of passing as skipped.
        if (!Runner::ReadEnvironment("CATCH3_COVERAGE_REQUIRE_MAP").empty())
        {
            FAIL("the coverage tools are not on PATH");
        }
        SKIP("the coverage tools of the compiler are not on PATH");
    }
    auto text = Runner::ReadFile(directory / "coverage.json");
    REQUIRE(text);
    auto report = Runner::ParseJson(*text);
    REQUIRE(report);
    CHECK(((*report)["format"].AsString() == "llvm"
           || (*report)["format"].AsString() == "gcov"));
    REQUIRE((*report)["tests"].Items().size() == 2);

    std::string source;
    const auto adds = ReachedLines(*report, "Adds one", source);
    const auto subtracts = ReachedLines(*report, "Subtracts one", source);
    const auto addLine = MarkedLine(source, "add-one");
    const auto subtractLine = MarkedLine(source, "subtract-one");
    CHECK(Contains(adds, addLine));
    CHECK_FALSE(Contains(adds, subtractLine));
    CHECK(Contains(subtracts, subtractLine));
    CHECK_FALSE(Contains(subtracts, addLine));

    auto lcov = Runner::ReadFile(directory / "coverage.lcov");
    REQUIRE(lcov);
    const auto combined = Runner::ParseLcov(*lcov);
    const auto file = combined.find(source);
    REQUIRE(file != combined.end());
    CHECK(file->second.lines.at(addLine) > 0);
    CHECK(file->second.lines.at(subtractLine) > 0);

    // Both formats carry branches: the condition of the fixture went one
    // way only, so one of its branches was taken and one was not.
    const auto branchLine = MarkedLine(source, "branch");
    std::size_t taken = 0;
    std::size_t notTaken = 0;
    for (const auto &[position, count] : file->second.branches)
    {
        if (position.first == branchLine)
        {
            (count > 0 ? taken : notTaken) += 1;
        }
    }
    CHECK(taken > 0);
    CHECK(notTaken > 0);
}