diff options
| author | Leitwolf11 | 2026-11-01 14:23:09 +0300 |
|---|---|---|
| committer | Leitwolf11 | 2026-11-01 14:23:09 +0300 |
| commit | 1db203d56b7c71db3d80b477da1ad49e9ae46468 (patch) | |
| tree | 48cd538e23e40ebdaae034a864082de30f68364f /src | |
| parent | 1559307a6ac7731ffd6a8e28147552b3987dbd93 (diff) | |
| download | catch3-1db203d56b7c71db3d80b477da1ad49e9ae46468.tar.gz | |
Import the results of other test frameworks
Tests Catch3 does not run had no history, flaky detection or quarantine.
--import=FILE reads a report instead of running tests and sends it through
the analysis, history and reports of a run: JUnit with Surefire's reruns,
xUnit.net v2, TRX, TAP 12 to 14, CTRF, Catch3's own --json report and
catch3-results, a small JSON Lines format for tools with none of these.
Results that share an identifier are attempts at one test, so a report of a
rerun gives a flaky test. --import-revision says what was tested: a verdict
that changes on one revision also counts as flaky. --import-name keeps the
results in a history of their own, which --analytics reads too. A TAP stream
that ends early fails instead of reading as a run that passed.
The amalgamation script now accepts a relative include of the third-party
wrappers, because pugixml is included from a second directory.
Diffstat (limited to 'src')
| -rw-r--r-- | src/CMakeLists.txt | 5 | ||||
| -rw-r--r-- | src/Progmasoft/Catch3/meson.build | 4 | ||||
| -rw-r--r-- | src/Progmasoft/PugiXml.inl | 4 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/Analysis.cpp | 5 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/Console.cpp | 15 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/Console.hpp | 4 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/Options.cpp | 52 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/Options.hpp | 25 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/ProjectConfig.cpp | 4 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/ResultImport.cpp | 305 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/ResultImport.hpp | 89 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/ResultImportJson.cpp | 279 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/ResultImportTap.cpp | 338 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/ResultImportXml.cpp | 507 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/Runner.cpp | 318 |
15 files changed, 1858 insertions, 96 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index aee8c7a3..87b19651 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -310,6 +310,7 @@ set(PROGMASOFT_RUNNER_FILES "${CATCH_DIR}/src/Progmasoft/Runner/Process.hpp" "${CATCH_DIR}/src/Progmasoft/Runner/ProjectConfig.hpp" "${CATCH_DIR}/src/Progmasoft/Runner/Protocol.hpp" + "${CATCH_DIR}/src/Progmasoft/Runner/ResultImport.hpp" "${CATCH_DIR}/src/Progmasoft/Runner/ResultsJson.hpp" "${CATCH_DIR}/src/Progmasoft/Runner/Scheduler.hpp" "${CATCH_DIR}/src/Progmasoft/Runner/SchedulerModel.hpp" @@ -329,6 +330,10 @@ set(PROGMASOFT_RUNNER_FILES "${CATCH_DIR}/src/Progmasoft/Runner/Process.cpp" "${CATCH_DIR}/src/Progmasoft/Runner/ProjectConfig.cpp" "${CATCH_DIR}/src/Progmasoft/Runner/Protocol.cpp" + "${CATCH_DIR}/src/Progmasoft/Runner/ResultImport.cpp" + "${CATCH_DIR}/src/Progmasoft/Runner/ResultImportJson.cpp" + "${CATCH_DIR}/src/Progmasoft/Runner/ResultImportTap.cpp" + "${CATCH_DIR}/src/Progmasoft/Runner/ResultImportXml.cpp" "${CATCH_DIR}/src/Progmasoft/Runner/ResultsJson.cpp" "${CATCH_DIR}/src/Progmasoft/Runner/Runner.cpp" "${CATCH_DIR}/src/Progmasoft/Runner/Scheduler.cpp" diff --git a/src/Progmasoft/Catch3/meson.build b/src/Progmasoft/Catch3/meson.build index 7b75dc6d..ccf93f29 100644 --- a/src/Progmasoft/Catch3/meson.build +++ b/src/Progmasoft/Catch3/meson.build @@ -297,6 +297,10 @@ runner_sources = files( '../Runner/Process.cpp', '../Runner/ProjectConfig.cpp', '../Runner/Protocol.cpp', + '../Runner/ResultImport.cpp', + '../Runner/ResultImportJson.cpp', + '../Runner/ResultImportTap.cpp', + '../Runner/ResultImportXml.cpp', '../Runner/ResultsJson.cpp', '../Runner/Runner.cpp', '../Runner/Scheduler.cpp', diff --git a/src/Progmasoft/PugiXml.inl b/src/Progmasoft/PugiXml.inl index 207bd17f..e93dd679 100644 --- a/src/Progmasoft/PugiXml.inl +++ b/src/Progmasoft/PugiXml.inl @@ -1,8 +1,8 @@ // Copyright (c) 2026 Progmasoft. // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 -// pugixml, compiled header-only into XmlWriter.cpp and nowhere else. Its -// namespace is renamed, so Catch3 cannot collide with a copy of pugixml that +// pugixml, compiled header-only into XmlWriter.cpp and ResultImportXml.cpp +// and nowhere else. Its namespace is renamed, so Catch3 cannot collide with a copy of pugixml that // the tests themselves link. It is vendored code: it is read as a system // header, outside the warnings Catch3 builds with. #if defined(__GNUC__) || defined(__clang__) diff --git a/src/Progmasoft/Runner/Analysis.cpp b/src/Progmasoft/Runner/Analysis.cpp index 802f1ed9..739ca2dc 100644 --- a/src/Progmasoft/Runner/Analysis.cpp +++ b/src/Progmasoft/Runner/Analysis.cpp @@ -350,6 +350,11 @@ namespace Progmasoft::Catch3::Runner test.analysis.insert( test.analysis.end(), findings.begin(), findings.end()); + // An imported failure ran somewhere else. + if (inputs.program.empty()) + { + continue; + } std::string replay = QuoteArgument(inputs.program) + " " + QuoteArgument(QuoteTestSpec(test.name)) + " --seed=" + std::to_string(failure->seed); diff --git a/src/Progmasoft/Runner/Console.cpp b/src/Progmasoft/Runner/Console.cpp index 44765311..91691967 100644 --- a/src/Progmasoft/Runner/Console.cpp +++ b/src/Progmasoft/Runner/Console.cpp @@ -203,6 +203,21 @@ namespace Progmasoft::Catch3::Runner } void + ConsoleReport::ImportStarting(std::size_t tests, std::size_t files) + { + total_ = tests; + if (options_.verbosity == Verbosity::kQuiet) + { + return; + } + Write(output_, + Paint("Catch3", kBold) + ": " + std::to_string(tests) + + " test(s) imported from " + std::to_string(files) + + " report(s)\n"); + std::fflush(output_); + } + + void ConsoleReport::Retrying(const TestResult &test, const Attempt &attempt) { if (options_.verbosity != Verbosity::kVerbose) diff --git a/src/Progmasoft/Runner/Console.hpp b/src/Progmasoft/Runner/Console.hpp index e79fcf68..32c83976 100644 --- a/src/Progmasoft/Runner/Console.hpp +++ b/src/Progmasoft/Runner/Console.hpp @@ -25,6 +25,10 @@ namespace Progmasoft::Catch3::Runner std::uint32_t seed, const std::vector<std::string> ¬es); + /// The start of `--import`: results read from `files` reports. + void + ImportStarting(std::size_t tests, std::size_t files); + void TestFinished(const TestResult &test); diff --git a/src/Progmasoft/Runner/Options.cpp b/src/Progmasoft/Runner/Options.cpp index c59d7e40..4ae99eb3 100644 --- a/src/Progmasoft/Runner/Options.cpp +++ b/src/Progmasoft/Runner/Options.cpp @@ -683,6 +683,58 @@ namespace Progmasoft::Catch3::Runner options.replayFailures = true; return Status::Ok(); }}, + {"import", + '\0', + Takes::kValue, + "FILE", + "Importing results", + "read a report of another framework instead of running tests " + "(repeatable)", + [](Options &options, const std::optional<std::string> &value) { + options.mode = Mode::kImport; + options.importFiles.push_back(*value); + return Status::Ok(); + }}, + {"import-format", + '\0', + Takes::kValue, + "FORMAT", + "Importing results", + "auto (default), catch3, junit, xunit, trx, tap or ctrf", + [](Options &options, const std::optional<std::string> &value) { + return ReadChoice<ImportFormat>( + value, + "import-format", + {{"auto", ImportFormat::kAuto}, + {"catch3", ImportFormat::kCatch3}, + {"junit", ImportFormat::kJUnit}, + {"xunit", ImportFormat::kXUnit}, + {"trx", ImportFormat::kTrx}, + {"tap", ImportFormat::kTap}, + {"ctrf", ImportFormat::kCtrf}}, + options.importFormat); + }}, + {"import-name", + '\0', + Takes::kValue, + "NAME", + "Importing results", + "the history imported results go to (default: imported); " + "--analytics reads it too", + [](Options &options, const std::optional<std::string> &value) { + options.importName = *value; + return Status::Ok(); + }}, + {"import-revision", + '\0', + Takes::kValue, + "REVISION", + "Importing results", + "what the reports tested (default: the Git commit here)", + [](Options &options, const std::optional<std::string> &value) { + options.importRevision = *value; + return Status::Ok(); + }}, {"analytics", '\0', Takes::kOptionalValue, diff --git a/src/Progmasoft/Runner/Options.hpp b/src/Progmasoft/Runner/Options.hpp index 15bba45e..9f7f826e 100644 --- a/src/Progmasoft/Runner/Options.hpp +++ b/src/Progmasoft/Runner/Options.hpp @@ -21,6 +21,8 @@ namespace Progmasoft::Catch3::Runner kListTests, kListTags, kAnalytics, + /// Read the reports of other test frameworks. + kImport, kHelp, kVersion, /// Internal: run tests on behalf of a runner, over the protocol. @@ -78,6 +80,20 @@ namespace Progmasoft::Catch3::Runner kOff, }; + /// The reports `--import` reads. + enum class ImportFormat : std::uint8_t + { + /// Whichever the contents show. + kAuto, + /// Catch3's own: `--json` reports and catch3-results JSON Lines. + kCatch3, + kJUnit, + kXUnit, + kTrx, + kTap, + kCtrf, + }; + enum class ColorMode : std::uint8_t { kAuto, @@ -157,6 +173,15 @@ namespace Progmasoft::Catch3::Runner bool replayFailures = false; bool analyticsJson = false; + // Importing the reports of other frameworks. + std::vector<std::string> importFiles; + ImportFormat importFormat = ImportFormat::kAuto; + /// The history they go to; "imported" when empty. + std::string importName; + /// What was tested, so that a verdict that changes on one revision + /// counts as flaky; the Git commit here when empty. + std::string importRevision; + // Output. Verbosity verbosity = Verbosity::kNormal; ColorMode color = ColorMode::kAuto; diff --git a/src/Progmasoft/Runner/ProjectConfig.cpp b/src/Progmasoft/Runner/ProjectConfig.cpp index 54f89d9c..e382b5c5 100644 --- a/src/Progmasoft/Runner/ProjectConfig.cpp +++ b/src/Progmasoft/Runner/ProjectConfig.cpp @@ -113,6 +113,10 @@ namespace Progmasoft::Catch3::Runner "break", "replay", "analytics", + "import", + "import-format", + "import-name", + "import-revision", "verbose", "quiet", "premature-exit-guard-file", diff --git a/src/Progmasoft/Runner/ResultImport.cpp b/src/Progmasoft/Runner/ResultImport.cpp new file mode 100644 index 00000000..10a9c5a8 --- /dev/null +++ b/src/Progmasoft/Runner/ResultImport.cpp @@ -0,0 +1,305 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 +#include <Progmasoft/Runner/History.hpp> +#include <Progmasoft/Runner/Json.hpp> +#include <Progmasoft/Runner/ResultImport.hpp> +#include <algorithm> +#include <cctype> +#include <cmath> +#include <iterator> +#include <map> +#include <utility> + +namespace Progmasoft::Catch3::Runner +{ + std::string_view + ImportTrim(std::string_view text) + { + const auto blank = [](char character) { + return std::isspace(static_cast<unsigned char>(character)) != 0; + }; + while (!text.empty() && blank(text.front())) + { + text.remove_prefix(1); + } + while (!text.empty() && blank(text.back())) + { + text.remove_suffix(1); + } + return text; + } + + bool + ImportStartsWith(std::string_view text, std::string_view prefix) + { + return text.substr(0, prefix.size()) == prefix; + } + + std::vector<std::string_view> + ImportLines(std::string_view text) + { + std::vector<std::string_view> lines; + while (!text.empty()) + { + const auto end = text.find('\n'); + auto line = text.substr(0, end); + if (!line.empty() && line.back() == '\r') + { + line.remove_suffix(1); + } + lines.push_back(line); + if (end == std::string_view::npos) + { + break; + } + text.remove_prefix(end + 1); + } + return lines; + } + + namespace + { + namespace ImportText + { + /// Without a byte order mark and the white space before the + /// first character that counts. + std::string_view + Contents(std::string_view text) + { + if (ImportStartsWith(text, "\xEF\xBB\xBF")) + { + text.remove_prefix(3); + } + return ImportTrim(text); + } + + /// The name of the root element of an XML document. + std::string_view + RootElement(std::string_view text) + { + while (!text.empty()) + { + const auto open = text.find('<'); + if (open == std::string_view::npos + || open + 1 >= text.size()) + { + return {}; + } + text.remove_prefix(open + 1); + if (text.front() == '?' || text.front() == '!') + { + const auto close = text.find('>'); + if (close == std::string_view::npos) + { + return {}; + } + text.remove_prefix(close + 1); + continue; + } + return text.substr(0, text.find_first_of(" \t\r\n/>")); + } + return {}; + } + } // namespace ImportText + } // namespace + + std::optional<ImportFormat> + DetectImportFormat(std::string_view text) + { + const std::string_view contents = ImportText::Contents(text); + if (contents.empty()) + { + return std::nullopt; + } + if (contents.front() == '<') + { + const std::string_view root = ImportText::RootElement(contents); + if (root == "testsuites" || root == "testsuite") + { + return ImportFormat::kJUnit; + } + if (root == "assemblies" || root == "assembly") + { + return ImportFormat::kXUnit; + } + if (root == "TestRun") + { + return ImportFormat::kTrx; + } + return std::nullopt; + } + if (contents.front() == '{') + { + auto whole = ParseJson(contents); + if (!whole) + { + // JSON Lines: the first line says what the others are. + whole = ParseJson(contents.substr(0, contents.find('\n'))); + } + return whole ? ImportJsonFormat(*whole) : std::nullopt; + } + for (const auto raw : ImportLines(contents)) + { + const std::string_view line = ImportTrim(raw); + if (line.empty() || line.front() == '#') + { + continue; + } + if (ImportStartsWith(line, "TAP version") + || ImportStartsWith(line, "1..") || IsTapTestLine(line)) + { + return ImportFormat::kTap; + } + break; + } + return std::nullopt; + } + + Expected<std::vector<TestResult>> + ParseImport(std::string_view text, ImportFormat format) + { + const std::string_view contents = ImportText::Contents(text); + switch (format) + { + case ImportFormat::kAuto: + break; + case ImportFormat::kCatch3: + return ParseCatch3Report(contents); + case ImportFormat::kJUnit: + return ParseJUnit(contents); + case ImportFormat::kXUnit: + return ParseXUnit(contents); + case ImportFormat::kTrx: + return ParseTrx(contents); + case ImportFormat::kTap: + return ParseTap(contents); + case ImportFormat::kCtrf: + return ParseCtrf(contents); + } + const auto detected = DetectImportFormat(contents); + if (!detected) + { + return Error{ "its format is not one Catch3 knows; " + "--import-format names it" }; + } + return ParseImport(contents, *detected); + } + + std::vector<TestResult> + MergeImported(std::vector<TestResult> results) + { + std::vector<TestResult> tests; + std::map<std::string, std::size_t> positions; + for (auto &result : results) + { + if (result.id.empty()) + { + result.id = History::KeyOf(result); + } + const auto [position, added] + = positions.emplace(result.id, tests.size()); + if (added) + { + tests.push_back(std::move(result)); + continue; + } + TestResult &test = tests[position->second]; + test.flaky = test.flaky || result.flaky; + std::move(result.attempts.begin(), + result.attempts.end(), + std::back_inserter(test.attempts)); + } + for (auto &test : tests) + { + std::size_t passes = 0; + std::size_t failures = 0; + for (std::size_t index = 0; index < test.attempts.size(); ++index) + { + Attempt &attempt = test.attempts[index]; + attempt.number = index + 1; + passes += attempt.outcome == Outcome::kPassed ? 1 : 0; + failures += IsFailure(attempt.outcome) ? 1 : 0; + } + if (test.attempts.empty()) + { + continue; + } + test.outcome = test.attempts.back().outcome; + test.category = test.attempts.back().category; + if (passes > 0 && failures > 0) + { + test.outcome = Outcome::kPassed; + test.category = FailureCategory::kNone; + test.flaky = true; + test.analysis.push_back( + "flaky: failed " + std::to_string(failures) + " of " + + std::to_string(test.attempts.size()) + " attempts"); + } + } + return tests; + } + + TestResult + ImportedTest(std::string name, std::string className) + { + TestResult test; + test.name = std::move(name); + test.className = std::move(className); + test.id = History::KeyOf(test); + return test; + } + + Attempt + ImportedAttempt(Outcome outcome, + std::int64_t durationMicroseconds, + std::string message, + std::string details, + FailureCategory category) + { + Attempt attempt; + attempt.outcome = outcome; + attempt.durationMicroseconds + = std::max<std::int64_t>(durationMicroseconds, 0); + if (outcome == Outcome::kSkipped) + { + attempt.standardOutput = std::move(message); + return attempt; + } + if (outcome != Outcome::kFailedAsExpected && !IsFailure(outcome)) + { + return attempt; + } + if (category == FailureCategory::kNone) + { + category = outcome == Outcome::kCrashed ? FailureCategory::kCrash + : outcome == Outcome::kTimedOut + ? FailureCategory::kTimeout + : FailureCategory::kAssertion; + } + if (message.empty()) + { + message = details.substr(0, details.find('\n')); + } + attempt.category = category; + attempt.assertionsFailed = 1; + AssertionRecord assertion; + assertion.message = std::move(message); + assertion.expansion = std::move(details); + assertion.category = category; + assertion.allowed = outcome == Outcome::kFailedAsExpected; + attempt.assertions.push_back(std::move(assertion)); + return attempt; + } + + std::int64_t + ImportedMicroseconds(double seconds) + { + // About thirty years; a longer duration is not a duration. + constexpr double kLongest = 1e9; + if (!std::isfinite(seconds) || seconds <= 0.0 || seconds > kLongest) + { + return 0; + } + return static_cast<std::int64_t>(std::llround(seconds * 1e6)); + } +} // namespace Progmasoft::Catch3::Runner diff --git a/src/Progmasoft/Runner/ResultImport.hpp b/src/Progmasoft/Runner/ResultImport.hpp new file mode 100644 index 00000000..b9ddff27 --- /dev/null +++ b/src/Progmasoft/Runner/ResultImport.hpp @@ -0,0 +1,89 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 +#pragma once + +#include <Progmasoft/Catch3/Results.hpp> +#include <cstdint> +#include <optional> +#include <string> +#include <string_view> +#include <vector> + +#include "Expected.hpp" +#include "Json.hpp" +#include "Options.hpp" + +namespace Progmasoft::Catch3::Runner +{ + /// The format of a report, from its contents; nothing when none fits. + [[nodiscard]] std::optional<ImportFormat> + DetectImportFormat(std::string_view text); + + /// The results a report holds, in its order. A test the report names + /// twice comes back twice; `MergeImported` makes one test of them. + [[nodiscard]] Expected<std::vector<TestResult>> + ParseImport(std::string_view text, ImportFormat format); + + /// One test per identifier: results that share one are attempts at that + /// test, in the order given, and the verdict is worked out over them. A + /// test that both failed and passed is flaky, as it is in a run of the + /// runner. + [[nodiscard]] std::vector<TestResult> + MergeImported(std::vector<TestResult> results); + + /// A result without attempts yet. + [[nodiscard]] TestResult + ImportedTest(std::string name, std::string className); + + /// One attempt as a report describes it. A failure gets an assertion + /// record with `message` and `details`, so the analysis treats it as it + /// treats a failure of Catch3's own; `category` defaults by outcome. + [[nodiscard]] Attempt + ImportedAttempt(Outcome outcome, + std::int64_t durationMicroseconds, + std::string message = {}, + std::string details = {}, + FailureCategory category = FailureCategory::kNone); + + /// Seconds as a report gives them; what makes no sense as a duration + /// counts as zero. + [[nodiscard]] std::int64_t + ImportedMicroseconds(double seconds); + + [[nodiscard]] std::string_view + ImportTrim(std::string_view text); + + [[nodiscard]] bool + ImportStartsWith(std::string_view text, std::string_view prefix); + + /// The lines of a text, without their line endings. + [[nodiscard]] std::vector<std::string_view> + ImportLines(std::string_view text); + + /// A format of Catch3's own or CTRF, from a parsed document. + [[nodiscard]] std::optional<ImportFormat> + ImportJsonFormat(const JsonValue &value); + + /// Whether a line reports a test in TAP. + [[nodiscard]] bool + IsTapTestLine(std::string_view line); + + /// A `--json` report or catch3-results lines. + [[nodiscard]] Expected<std::vector<TestResult>> + ParseCatch3Report(std::string_view text); + + [[nodiscard]] Expected<std::vector<TestResult>> + ParseCtrf(std::string_view text); + + [[nodiscard]] Expected<std::vector<TestResult>> + ParseTap(std::string_view text); + + [[nodiscard]] Expected<std::vector<TestResult>> + ParseJUnit(std::string_view text); + + [[nodiscard]] Expected<std::vector<TestResult>> + ParseXUnit(std::string_view text); + + [[nodiscard]] Expected<std::vector<TestResult>> + ParseTrx(std::string_view text); +} // namespace Progmasoft::Catch3::Runner diff --git a/src/Progmasoft/Runner/ResultImportJson.cpp b/src/Progmasoft/Runner/ResultImportJson.cpp new file mode 100644 index 00000000..2cdc2019 --- /dev/null +++ b/src/Progmasoft/Runner/ResultImportJson.cpp @@ -0,0 +1,279 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0 + +// The JSON reports: Catch3's own and CTRF. +#include <Progmasoft/Runner/Json.hpp> +#include <Progmasoft/Runner/ResultImport.hpp> +#include <Progmasoft/Runner/ResultsJson.hpp> +#include <utility> + +namespace Progmasoft::Catch3::Runner +{ + namespace + { + namespace ImportJson + { + /// A format of Catch3's own or CTRF, from the parsed document. + std::optional<ImportFormat> + FormatOf(const JsonValue &value) + { + if (value["format"].AsString() == "catch3-run" + || value["format"].AsString() == "catch3-results") + { + return ImportFormat::kCatch3; + } + if (value["results"]["tests"].IsArray()) + { + return ImportFormat::kCtrf; + } + return std::nullopt; + } + + /// What a test owes the verdict of an earlier run is worked out + /// again here, from this run's point of view. + void + Forget(TestResult &test) + { + test.quarantined = false; + test.history = FailureHistory::kUnknown; + test.selectionReason.clear(); + test.signature.clear(); + test.analysis.clear(); + test.replayCommand.clear(); + test.predictedDurationMicroseconds = 0; + test.predictedFailureProbability = 0.0; + } + + Expected<TestResult> + ReadLine(const JsonValue &value) + { + TestResult test = TestResultFromJson(value); + Forget(test); + if (test.name.empty()) + { + test.name = test.id; + } + if (test.name.empty()) + { + return Error{ "a test needs an 'id' or a 'name'" }; + } + const std::string &outcome = value["outcome"].AsString(); + if (!OutcomeFromName(outcome)) + { + return Error{ "'" + test.name + "' has the outcome '" + + outcome + "'" }; + } + if (test.attempts.empty()) + { + Attempt attempt = ImportedAttempt( + test.outcome, + value["durationMicroseconds"].AsInteger(), + value["message"].AsString(), + value["details"].AsString(), + test.category); + attempt.standardOutput = value["standardOutput"].AsString(); + attempt.standardError = value["standardError"].AsString(); + test.attempts.push_back(std::move(attempt)); + } + return test; + } + + Expected<std::vector<TestResult>> + ParseLines(std::string_view text) + { + std::vector<TestResult> tests; + std::size_t number = 0; + bool started = false; + for (const auto line : ImportLines(text)) + { + ++number; + if (ImportTrim(line).empty()) + { + continue; + } + const std::string where = "line " + std::to_string(number); + auto value = ParseJson(line); + if (!value) + { + return value.GetError().Context(where); + } + if (!started) + { + started = true; + if ((*value)["format"].AsString() != "catch3-results") + { + return Error{ "the first line must name the format " + "catch3-results" }; + } + if ((*value)["version"].AsInteger() + > kResultsFormatVersion) + { + return Error{ "the format is newer than this " + "version of Catch3 reads" }; + } + continue; + } + auto test = ReadLine(*value); + if (!test) + { + return test.GetError().Context(where); + } + tests.push_back(std::move(*test)); + } + return tests; + } + + Expected& |