catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
develroot / src / Progmasoft / Runner / ResultImportJson.cpp
// Copyright (c) 2026 Progmasoft.
// SPDX-License-Identifier: LGPL-3.0-or-later
// The JSON reports: Catch3's own and CTRF.
#include <Progmasoft/Runner/Json.hpp>
#include <Progmasoft/Runner/ResultImport.hpp>
#include <Progmasoft/Runner/ResultsJson.hpp>
#include <utility>
namespace Progmasoft::Catch3::Runner
{
namespace
{
namespace ImportJson
{
/// A format of Catch3's own or CTRF, from the parsed document.
std::optional<ImportFormat>
FormatOf(const JsonValue &value)
{
if (value["format"].AsString() == "catch3-run"
|| value["format"].AsString() == "catch3-results")
{
return ImportFormat::kCatch3;
}
if (value["results"]["tests"].IsArray())
{
return ImportFormat::kCtrf;
}
return std::nullopt;
}
/// What a test owes the verdict of an earlier run is worked out
/// again here, from this run's point of view.
void
Forget(TestResult &test)
{
test.quarantined = false;
test.history = FailureHistory::kUnknown;
test.selectionReason.clear();
test.signature.clear();
test.analysis.clear();
test.replayCommand.clear();
test.predictedDurationMicroseconds = 0;
test.predictedFailureProbability = 0.0;
}
Expected<TestResult>
ReadLine(const JsonValue &value)
{
TestResult test = TestResultFromJson(value);
Forget(test);
if (test.name.empty())
{
test.name = test.id;
}
if (test.name.empty())
{
return Error{ "a test needs an 'id' or a 'name'" };
}
const std::string &outcome = value["outcome"].AsString();
if (!OutcomeFromName(outcome))
{
return Error{ "'" + test.name + "' has the outcome '"
+ outcome + "'" };
}
if (test.attempts.empty())
{
Attempt attempt = ImportedAttempt(
test.outcome,
value["durationMicroseconds"].AsInteger(),
value["message"].AsString(),
value["details"].AsString(),
test.category);
attempt.standardOutput = value["standardOutput"].AsString();
attempt.standardError = value["standardError"].AsString();
test.attempts.push_back(std::move(attempt));
}
return test;
}
Expected<std::vector<TestResult>>
ParseLines(std::string_view text)
{
std::vector<TestResult> tests;
std::size_t number = 0;
bool started = false;
for (const auto line : ImportLines(text))
{
++number;
if (ImportTrim(line).empty())
{
continue;
}
const std::string where = "line " + std::to_string(number);
auto value = ParseJson(line);
if (!value)
{
return value.GetError().Context(where);
}
if (!started)
{
started = true;
if ((*value)["format"].AsString() != "catch3-results")
{
return Error{ "the first line must name the format "
"catch3-results" };
}
if ((*value)["version"].AsInteger()
> kResultsFormatVersion)
{
return Error{ "the format is newer than this "
"version of Catch3 reads" };
}
continue;
}
auto test = ReadLine(*value);
if (!test)
{
return test.GetError().Context(where);
}
tests.push_back(std::move(*test));
}
return tests;
}
Expected<std::vector<TestResult>>
ParseCatch3(std::string_view text)
{
auto whole = ParseJson(text);
if (!whole)
{
return ParseLines(text);
}
if ((*whole)["format"].AsString() != "catch3-run")
{
return ParseLines(text);
}
auto run = RunResultFromJson(*whole);
if (!run)
{
return run.GetError();
}
for (auto &test : run->tests)
{
Forget(test);
}
return std::move(run->tests);
}
std::string
TextOf(const JsonValue &value)
{
if (!value.IsArray())
{
return value.AsString();
}
std::string text;
for (const auto &item : value.Items())
{
text += item.AsString() + "\n";
}
return text;
}
Outcome
CtrfOutcome(const std::string &status)
{
if (status == "passed")
{
return Outcome::kPassed;
}
if (status == "failed")
{
return Outcome::kFailed;
}
if (status == "skipped" || status == "pending")
{
return Outcome::kSkipped;
}
return Outcome::kNotRun;
}
/// A test, or one retry of it: the two share these fields.
Attempt
CtrfAttempt(const JsonValue &value)
{
Attempt attempt = ImportedAttempt(
CtrfOutcome(value["status"].AsString()),
ImportedMicroseconds(value["duration"].AsNumber() / 1e3),
value["message"].AsString(),
value["trace"].AsString());
attempt.standardOutput = TextOf(value["stdout"]);
attempt.standardError = TextOf(value["stderr"]);
return attempt;
}
std::string
CtrfSuite(const JsonValue &suite)
{
if (!suite.IsArray())
{
return suite.AsString();
}
std::string text;
for (const auto &item : suite.Items())
{
text += (text.empty() ? "" : " > ") + item.AsString();
}
return text;
}
Expected<std::vector<TestResult>>
ParseCtrf(std::string_view text)
{
auto document = ParseJson(text);
if (!document)
{
return document.GetError();
}
const JsonValue &tests = (*document)["results"]["tests"];
if (!tests.IsArray())
{
return Error{ "a CTRF report has results.tests" };
}
std::vector<TestResult> results;
for (const auto &item : tests.Items())
{
const std::string &name = item["name"].AsString();
if (name.empty())
{
return Error{ "a CTRF test has no name" };
}
TestResult test
= ImportedTest(name, CtrfSuite(item["suite"]));
test.file = item["filePath"].AsString();
test.line
= static_cast<std::size_t>(item["line"].AsUnsigned());
test.flaky = item["flaky"].AsBoolean();
for (const auto &tag : item["tags"].Items())
{
test.tags.push_back(tag.AsString());
}
for (const auto &retry : item["retryAttempts"].Items())
{
test.attempts.push_back(CtrfAttempt(retry));
}
// The retries end before the last attempt, which is the
// test itself.
test.attempts.push_back(CtrfAttempt(item));
results.push_back(std::move(test));
}
return results;
}
} // namespace ImportJson
} // namespace
std::optional<ImportFormat>
ImportJsonFormat(const JsonValue &value)
{
return ImportJson::FormatOf(value);
}
Expected<std::vector<TestResult>>
ParseCatch3Report(std::string_view text)
{
return ImportJson::ParseCatch3(text);
}
Expected<std::vector<TestResult>>
ParseCtrf(std::string_view text)
{
return ImportJson::ParseCtrf(text);
}
} // namespace Progmasoft::Catch3::Runner