diff options
| author | Leitwolf11 | 2026-11-01 14:48:55 +0300 |
|---|---|---|
| committer | Leitwolf11 | 2026-11-01 14:48:55 +0300 |
| commit | 569620dad14df7e13bb9fd31d256e9fd873ef6ec (patch) | |
| tree | f933300c29658c3eb89d126d74b6c30fdff9dd04 | |
| parent | 1db203d56b7c71db3d80b477da1ad49e9ae46468 (diff) | |
| download | catch3-feature/result-import.tar.gz | |
Keep real failures visible when importingfeature/result-import
A review of the import found ways for a failure to disappear or for the
import to reach too far.
- CTRF's retryAttempts ends before the final attempt, which was dropped when
its outcome matched the one before it.
- TAP names tests only within a stream, so a failure in one file and a pass
under the same name in another merged into a flaky pass. The path given to
--import is now the class of a stream's tests.
- A failure followed by a skip with the same identifier read as skipped.
- Directives are whole words: TODOING is not TODO. SKIPPED and SKIPPING stay
directives, as harnesses treat them.
- Verdicts that alternate count as flaky only under an explicit
--import-revision. HEAD says nothing about a dirty checkout or a report from
another one, and two such events could quarantine a real failure.
- --import-name no longer redirects the history of a normal test run.
- Importing does not depend on the tests registered in the binary.
| -rw-r--r-- | docs/progmasoft-runner.md | 18 | ||||
| -rw-r--r-- | extras/catch_amalgamated.cpp | 114 | ||||
| -rw-r--r-- | extras/catch_amalgamated.hpp | 2 | ||||
| -rw-r--r-- | extras/catch_amalgamated_third_party.hpp | 2 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/Options.cpp | 3 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/Options.hpp | 4 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/ResultImport.cpp | 22 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/ResultImport.hpp | 10 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/ResultImportJson.cpp | 11 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/ResultImportTap.cpp | 31 | ||||
| -rw-r--r-- | src/Progmasoft/Runner/Runner.cpp | 31 | ||||
| -rw-r--r-- | tests/Progmasoft/RunnerImport.tests.cpp | 121 |
12 files changed, 274 insertions, 95 deletions
diff --git a/docs/progmasoft-runner.md b/docs/progmasoft-runner.md index 486c719f..8134ae18 100644 --- a/docs/progmasoft-runner.md +++ b/docs/progmasoft-runner.md @@ -164,12 +164,18 @@ work as they do for a run. Reports that were read count as one run. A test is identified by its class and its name. Results with the same identifier, in one report or in several, are attempts at one test: a test that both failed and passed is flaky, passes and is remembered as flaky, which is how -a report of a rerun is imported. TAP has no retries, so a name that comes back -in one stream is another test and gets a number. - -A verdict that changes on one revision also counts as flaky. -`--import-revision=REVISION` names what the reports tested; it defaults to the -Git commit of the working directory. +a report of a rerun is imported. A failure stays a failure when a later report +skips the test. Reports of the same tests on different platforms are not reruns +of each other: import each with an `--import-name` of its own, or a failure on one +reads as flakiness. + +TAP names tests only within one stream, so the path given to `--import` is the +class of its tests: keep it the same from run to run for the history to follow +them. A name that comes back in one stream is another test and gets a number. + +`--import-revision=REVISION` names what the reports tested, and a verdict that +changes on one revision then counts as flaky too. Without it only retries in the +reports do, because a working directory can have edits or be another checkout. Nothing is run, so an imported failure has no replay command, and only what the report says is known: it has no order, seed or sections. A TAP stream that bails diff --git a/extras/catch_amalgamated.cpp b/extras/catch_amalgamated.cpp index caee2a35..01b07618 100644 --- a/extras/catch_amalgamated.cpp +++ b/extras/catch_amalgamated.cpp @@ -7,7 +7,7 @@ // SPDX-License-Identifier: BSL-1.0 // Catch3 v0.3.0, on the Catch2 v3.16.1 engine -// Generated: 2026-11-01 14:22:22.249604 +// Generated: 2026-11-01 14:47:44.734580 // ---------------------------------------------------------- // This file is an amalgamation of multiple different files. // You probably shouldn't edit it directly. @@ -1122,8 +1122,8 @@ namespace Progmasoft::Catch3::Runner ImportFormat importFormat = ImportFormat::kAuto; /// The history they go to; "imported" when empty. std::string importName; - /// What was tested, so that a verdict that changes on one revision - /// counts as flaky; the Git commit here when empty. + /// What was tested: a verdict that changes on one revision counts + /// as flaky. Without it, only retries in the reports do. std::string importRevision; // Output. @@ -1900,8 +1900,12 @@ namespace Progmasoft::Catch3::Runner /// The results a report holds, in its order. A test the report names /// twice comes back twice; `MergeImported` makes one test of them. + /// `source` names the report, for a format whose tests have no identity + /// of their own. [[nodiscard]] Expected<std::vector<TestResult>> - ParseImport(std::string_view text, ImportFormat format); + ParseImport(std::string_view text, + ImportFormat format, + const std::string &source = {}); /// One test per identifier: results that share one are attempts at that /// test, in the order given, and the verdict is worked out over them. A @@ -1954,8 +1958,10 @@ namespace Progmasoft::Catch3::Runner [[nodiscard]] Expected<std::vector<TestResult>> ParseCtrf(std::string_view text); + /// `source` becomes the class of its tests: TAP names them only within + /// one stream. [[nodiscard]] Expected<std::vector<TestResult>> - ParseTap(std::string_view text); + ParseTap(std::string_view text, const std::string &source); [[nodiscard]] Expected<std::vector<TestResult>> ParseJUnit(std::string_view text); @@ -13555,7 +13561,8 @@ namespace Progmasoft::Catch3::Runner Takes::kValue, "REVISION", "Importing results", - "what the reports tested (default: the Git commit here)", + "what the reports tested; a verdict that changes on one " + "revision counts as flaky", [](Options &options, const std::optional<std::string> &value) { options.importRevision = *value; return Status::Ok(); @@ -16721,7 +16728,9 @@ namespace Progmasoft::Catch3::Runner } Expected<std::vector<TestResult>> - ParseImport(std::string_view text, ImportFormat format) + ParseImport(std::string_view text, + ImportFormat format, + const std::string &source) { const std::string_view contents = ImportText::Contents(text); switch (format) @@ -16737,7 +16746,7 @@ namespace Progmasoft::Catch3::Runner case ImportFormat::kTrx: return ParseTrx(contents); case ImportFormat::kTap: - return ParseTap(contents); + return ParseTap(contents, source); case ImportFormat::kCtrf: return ParseCtrf(contents); } @@ -16747,7 +16756,7 @@ namespace Progmasoft::Catch3::Runner return Error{ "its format is not one Catch3 knows; " "--import-format names it" }; } - return ParseImport(contents, *detected); + return ParseImport(contents, *detected, source); } std::vector<TestResult> @@ -16789,8 +16798,18 @@ namespace Progmasoft::Catch3::Runner { continue; } - test.outcome = test.attempts.back().outcome; - test.category = test.attempts.back().category; + // A report of a skip says nothing about an earlier failure. + const auto verdict + = std::find_if(test.attempts.rbegin(), + test.attempts.rend(), + [](const Attempt &attempt) { + return IsFailure(attempt.outcome); + }); + const Attempt &last = verdict != test.attempts.rend() + ? *verdict + : test.attempts.back(); + test.outcome = last.outcome; + test.category = last.category; if (passes > 0 && failures > 0) { test.outcome = Outcome::kPassed; @@ -17111,14 +17130,9 @@ namespace Progmasoft::Catch3::Runner { test.attempts.push_back(CtrfAttempt(retry)); } - // The retries may or may not list the last attempt, which - // is the test itself. - Attempt last = CtrfAttempt(item); - if (test.attempts.empty() - || test.attempts.back().outcome != last.outcome) - { - test.attempts.push_back(std::move(last)); - } + // The retries end before the last attempt, which is the + // test itself. + test.attempts.push_back(CtrfAttempt(item)); results.push_back(std::move(test)); } return results; @@ -17258,15 +17272,20 @@ namespace Progmasoft::Catch3::Runner { const std::string_view rest = ImportTrim(text.substr(directive + 1)); - const std::string word = Lowered(rest.substr(0, 4)); - if (word == "skip" || word == "todo") + const auto end = rest.find_first_of(" \t:"); + const std::string word = Lowered(rest.substr(0, end)); + // Harnesses take the longer spellings of SKIP, and only + // the word TODO. + if (word == "skip" || word == "skipped" + || word == "skipping" || word == "todo") { - line.directive = word; - const auto space = rest.find_first_of(" \t"); - line.reason = space == std::string_view::npos - ? std::string{} - : std::string{ ImportTrim( - rest.substr(space)) }; + line.directive = word.substr(0, 4); + if (end != std::string_view::npos) + { + std::string_view reason = rest.substr(end); + reason.remove_prefix(reason.front() == ':' ? 1 : 0); + line.reason = std::string{ ImportTrim(reason) }; + } } } return line; @@ -17329,7 +17348,7 @@ namespace Progmasoft::Catch3::Runner } Expected<std::vector<TestResult>> - Parse(std::string_view text) + Parse(std::string_view text, const std::string &source) { std::vector<TestResult> tests; std::vector<std::string> subtests; @@ -17410,7 +17429,7 @@ namespace Progmasoft::Catch3::Runner } anything = true; topLevel += depth == 0 ? 1 : 0; - std::string parent; + std::string parent = source; for (std::size_t level = 0; level < std::min(depth, subtests.size()); ++level) @@ -17455,7 +17474,7 @@ namespace Progmasoft::Catch3::Runner } if (!incomplete.empty()) { - TestResult test = ImportedTest("the TAP stream", {}); + TestResult test = ImportedTest("the TAP stream", source); test.attempts.push_back( ImportedAttempt(Outcome::kFailed, 0, @@ -17476,9 +17495,9 @@ namespace Progmasoft::Catch3::Runner } Expected<std::vector<TestResult>> - ParseTap(std::string_view text) + ParseTap(std::string_view text, const std::string &source) { - return ImportTap::Parse(text); + return ImportTap::Parse(text, source); } } // namespace Progmasoft::Catch3::Runner // Copyright (c) 2026 Progmasoft. @@ -18985,11 +19004,13 @@ namespace Progmasoft::Catch3 const std::filesystem::path &program, const std::string &programName) { - std::string name = options.importName; - if (name.empty()) + std::string name = programName; + if (options.mode == Runner::Mode::kImport + || (options.mode == Runner::Mode::kAnalytics + && !options.importName.empty())) { - name = options.mode == Runner::Mode::kImport ? "imported" - : programName; + name = options.importName.empty() ? "imported" + : options.importName; } return Runner::HistoryFile(HistoryDirectory(options, program), name); @@ -19011,7 +19032,10 @@ namespace Progmasoft::Catch3 PrintError(text.GetError().Message()); return Catch::UnspecifiedErrorExitCode; } - auto tests = Runner::ParseImport(*text, options.importFormat); + auto tests = Runner::ParseImport( + *text, + options.importFormat, + std::filesystem::path{ file }.generic_string()); if (!tests) { PrintError(tests.GetError().Context(file).Message()); @@ -19051,11 +19075,16 @@ namespace Progmasoft::Catch3 ? Runner::CurrentCommit( std::filesystem::current_path(here)) : options.importRevision; + // Only the person importing knows that two reports tested the + // same thing: a checkout can have edits, or be another one. std::map<std::string, std::string> fingerprints; for (auto &test : run.tests) { run.wallMicroseconds += test.DurationMicroseconds(); - fingerprints[test.id] = revision; + if (!options.importRevision.empty()) + { + fingerprints[test.id] = options.importRevision; + } const Runner::TestHistory *past = history.Find(test.id); test.quarantined = options.quarantine != Runner::QuarantineMode::kOff @@ -19116,6 +19145,11 @@ namespace Progmasoft::Catch3 std::printf("Catch3 %s\n", kVersion); return 0; } + // Importing does not touch the tests of this binary. + if (options.mode == Runner::Mode::kImport) + { + return Import(options, program, programName); + } if (ReportStartupErrors()) { return Catch::UnspecifiedErrorExitCode; @@ -19130,10 +19164,6 @@ namespace Progmasoft::Catch3 { return Runner::RunWorker(options, programName); } - if (options.mode == Runner::Mode::kImport) - { - return Import(options, program, programName); - } std::vector<std::string> notes; if (!options.configPath.empty() diff --git a/extras/catch_amalgamated.hpp b/extras/catch_amalgamated.hpp index 680dae0e..249577f9 100644 --- a/extras/catch_amalgamated.hpp +++ b/extras/catch_amalgamated.hpp @@ -7,7 +7,7 @@ // SPDX-License-Identifier: BSL-1.0 // Catch3 v0.3.0, on the Catch2 v3.16.1 engine -// Generated: 2026-11-01 14:22:22.117317 +// Generated: 2026-11-01 14:47:44.469514 // ---------------------------------------------------------- // This file is an amalgamation of multiple different files. // You probably shouldn't edit it directly. diff --git a/extras/catch_amalgamated_third_party.hpp b/extras/catch_amalgamated_third_party.hpp index ff52646e..4888bc35 100644 --- a/extras/catch_amalgamated_third_party.hpp +++ b/extras/catch_amalgamated_third_party.hpp @@ -1,5 +1,5 @@ // Catch3 v0.3.0: yyjson and pugixml for catch_amalgamated.cpp -// Generated: 2026-11-01 14:22:22.379981 +// Generated: 2026-11-01 14:47:45.127927 // ---------------------------------------------------------- // This file is an amalgamation of src/Progmasoft/Runner/Yyjson.inl, // src/Progmasoft/PugiXml.inl and the libraries they include from diff --git a/src/Progmasoft/Runner/Options.cpp b/src/Progmasoft/Runner/Options.cpp index 4ae99eb3..383ed68d 100644 --- a/src/Progmasoft/Runner/Options.cpp +++ b/src/Progmasoft/Runner/Options.cpp @@ -730,7 +730,8 @@ namespace Progmasoft::Catch3::Runner Takes::kValue, "REVISION", "Importing results", - "what the reports tested (default: the Git commit here)", + "what the reports tested; a verdict that changes on one " + "revision counts as flaky", [](Options &options, const std::optional<std::string> &value) { options.importRevision = *value; return Status::Ok(); diff --git a/src/Progmasoft/Runner/Options.hpp b/src/Progmasoft/Runner/Options.hpp index 9f7f826e..35e6c302 100644 --- a/src/Progmasoft/Runner/Options.hpp +++ b/src/Progmasoft/Runner/Options.hpp @@ -178,8 +178,8 @@ namespace Progmasoft::Catch3::Runner ImportFormat importFormat = ImportFormat::kAuto; /// The history they go to; "imported" when empty. std::string importName; - /// What was tested, so that a verdict that changes on one revision - /// counts as flaky; the Git commit here when empty. + /// What was tested: a verdict that changes on one revision counts + /// as flaky. Without it, only retries in the reports do. std::string importRevision; // Output. diff --git a/src/Progmasoft/Runner/ResultImport.cpp b/src/Progmasoft/Runner/ResultImport.cpp index 10a9c5a8..cfd9a2ba 100644 --- a/src/Progmasoft/Runner/ResultImport.cpp +++ b/src/Progmasoft/Runner/ResultImport.cpp @@ -156,7 +156,9 @@ namespace Progmasoft::Catch3::Runner } Expected<std::vector<TestResult>> - ParseImport(std::string_view text, ImportFormat format) + ParseImport(std::string_view text, + ImportFormat format, + const std::string &source) { const std::string_view contents = ImportText::Contents(text); switch (format) @@ -172,7 +174,7 @@ namespace Progmasoft::Catch3::Runner case ImportFormat::kTrx: return ParseTrx(contents); case ImportFormat::kTap: - return ParseTap(contents); + return ParseTap(contents, source); case ImportFormat::kCtrf: return ParseCtrf(contents); } @@ -182,7 +184,7 @@ namespace Progmasoft::Catch3::Runner return Error{ "its format is not one Catch3 knows; " "--import-format names it" }; } - return ParseImport(contents, *detected); + return ParseImport(contents, *detected, source); } std::vector<TestResult> @@ -224,8 +226,18 @@ namespace Progmasoft::Catch3::Runner { continue; } - test.outcome = test.attempts.back().outcome; - test.category = test.attempts.back().category; + // A report of a skip says nothing about an earlier failure. + const auto verdict + = std::find_if(test.attempts.rbegin(), + test.attempts.rend(), + [](const Attempt &attempt) { + return IsFailure(attempt.outcome); + }); + const Attempt &last = verdict != test.attempts.rend() + ? *verdict + : test.attempts.back(); + test.outcome = last.outcome; + test.category = last.category; if (passes > 0 && failures > 0) { test.outcome = Outcome::kPassed; diff --git a/src/Progmasoft/Runner/ResultImport.hpp b/src/Progmasoft/Runner/ResultImport.hpp index b9ddff27..869a8449 100644 --- a/src/Progmasoft/Runner/ResultImport.hpp +++ b/src/Progmasoft/Runner/ResultImport.hpp @@ -21,8 +21,12 @@ namespace Progmasoft::Catch3::Runner /// The results a report holds, in its order. A test the report names /// twice comes back twice; `MergeImported` makes one test of them. + /// `source` names the report, for a format whose tests have no identity + /// of their own. [[nodiscard]] Expected<std::vector<TestResult>> - ParseImport(std::string_view text, ImportFormat format); + ParseImport(std::string_view text, + ImportFormat format, + const std::string &source = {}); /// One test per identifier: results that share one are attempts at that /// test, in the order given, and the verdict is worked out over them. A @@ -75,8 +79,10 @@ namespace Progmasoft::Catch3::Runner [[nodiscard]] Expected<std::vector<TestResult>> ParseCtrf(std::string_view text); + /// `source` becomes the class of its tests: TAP names them only within + /// one stream. [[nodiscard]] Expected<std::vector<TestResult>> - ParseTap(std::string_view text); + ParseTap(std::string_view text, const std::string &source); [[nodiscard]] Expected<std::vector<TestResult>> ParseJUnit(std::string_view text); diff --git a/src/Progmasoft/Runner/ResultImportJson.cpp b/src/Progmasoft/Runner/ResultImportJson.cpp index 2cdc2019..1651f938 100644 --- a/src/Progmasoft/Runner/ResultImportJson.cpp +++ b/src/Progmasoft/Runner/ResultImportJson.cpp @@ -244,14 +244,9 @@ namespace Progmasoft::Catch3::Runner { test.attempts.push_back(CtrfAttempt(retry)); } - // The retries may or may not list the last attempt, which - // is the test itself. - Attempt last = CtrfAttempt(item); - if (test.attempts.empty() - || test.attempts.back().outcome != last.outcome) - { - test.attempts.push_back(std::move(last)); - } + // The retries end before the last attempt, which is the + // test itself. + test.attempts.push_back(CtrfAttempt(item)); results.push_back(std::move(test)); } return results; diff --git a/src/Progmasoft/Runner/ResultImportTap.cpp b/src/Progmasoft/Runner/ResultImportTap.cpp index a5f44046..b127701c 100644 --- a/src/Progmasoft/Runner/ResultImportTap.cpp +++ b/src/Progmasoft/Runner/ResultImportTap.cpp @@ -113,15 +113,20 @@ namespace Progmasoft::Catch3::Runner { const std::string_view rest = ImportTrim(text.substr(directive + 1)); - const std::string word = Lowered(rest.substr(0, 4)); - if (word == "skip" || word == "todo") + const auto end = rest.find_first_of(" \t:"); + const std::string word = Lowered(rest.substr(0, end)); + // Harnesses take the longer spellings of SKIP, and only + // the word TODO. + if (word == "skip" || word == "skipped" + || word == "skipping" || word == "todo") { - line.directive = word; - const auto space = rest.find_first_of(" \t"); - line.reason = space == std::string_view::npos - ? std::string{} - : std::string{ ImportTrim( - rest.substr(space)) }; + line.directive = word.substr(0, 4); + if (end != std::string_view::npos) + { + std::string_view reason = rest.substr(end); + reason.remove_prefix(reason.front() == ':' ? 1 : 0); + line.reason = std::string{ ImportTrim(reason) }; + } } } return line; @@ -184,7 +189,7 @@ namespace Progmasoft::Catch3::Runner } Expected<std::vector<TestResult>> - Parse(std::string_view text) + Parse(std::string_view text, const std::string &source) { std::vector<TestResult> tests; std::vector<std::string> subtests; @@ -265,7 +270,7 @@ namespace Progmasoft::Catch3::Runner } anything = true; topLevel += depth == 0 ? 1 : 0; - std::string parent; + std::string parent = source; for (std::size_t level = 0; level < std::min(depth, subtests.size()); ++level) @@ -310,7 +315,7 @@ namespace Progmasoft::Catch3::Runner } if (!incomplete.empty()) { - TestResult test = ImportedTest("the TAP stream", {}); + TestResult test = ImportedTest("the TAP stream", source); test.attempts.push_back( ImportedAttempt(Outcome::kFailed, 0, @@ -331,8 +336,8 @@ namespace Progmasoft::Catch3::Runner } Expected<std::vector<TestResult>> - ParseTap(std::string_view text) + ParseTap(std::string_view text, const std::string &source) { - return ImportTap::Parse(text); + return ImportTap::Parse(text, source); } } // namespace Progmasoft::Catch3::Runner diff --git a/src/Progmasoft/Runner/Runner.cpp b/src/Progmasoft/Runner/Runner.cpp index ae711293..77011322 100644 --- a/src/Progmasoft/Runner/Runner.cpp +++ b/src/Progmasoft/Runner/Runner.cpp @@ -627,11 +627,13 @@ namespace Progmasoft::Catch3 const std::filesystem::path &program, const std::string &programName) { - std::string name = options.importName; - if (name.empty()) + std::string name = programName; + if (options.mode == Runner::Mode::kImport + || (options.mode == Runner::Mode::kAnalytics + && !options.importName.empty())) { - name = options.mode == Runner::Mode::kImport ? "imported" - : programName; + name = options.importName.empty() ? "imported" + : options.importName; } return Runner::HistoryFile(HistoryDirectory(options, program), name); @@ -653,7 +655,10 @@ namespace Progmasoft::Catch3 PrintError(text.GetError().Message()); return Catch::UnspecifiedErrorExitCode; } - auto tests = Runner::ParseImport(*text, options.importFormat); + auto tests = Runner::ParseImport( + *text, + options.importFormat, + std::filesystem::path{ file }.generic_string()); if (!tests) { PrintError(tests.GetError().Context(file).Message()); @@ -693,11 +698,16 @@ namespace Progmasoft::Catch3 ? Runner::CurrentCommit( std::filesystem::current_path(here)) : options.importRevision; + // Only the person importing knows that two reports tested the + // same thing: a checkout can have edits, or be another one. std::map<std::string, std::string> fingerprints; for (auto &test : run.tests) { run.wallMicroseconds += test.DurationMicroseconds(); - fingerprints[test.id] = revision; + if (!options.importRevision.empty()) + { + fingerprints[test.id] = options.importRevision; + } const Runner::TestHistory *past = history.Find(test.id); test.quarantined = options.quarantine != Runner::QuarantineMode::kOff @@ -758,6 +768,11 @@ namespace Progmasoft::Catch3 std::printf("Catch3 %s\n", kVersion); return 0; } + // Importing does not touch the tests of this binary. + if (options.mode == Runner::Mode::kImport) + { + return Import(options, program, programName); + } if (ReportStartupErrors()) { return Catch::UnspecifiedErrorExitCode; @@ -772,10 +787,6 @@ namespace Progmasoft::Catch3 { return Runner::RunWorker(options, programName); } - if (options.mode == Runner::Mode::kImport) - { - return Import(options, program, programName); - } std::vector<std::string> notes; if (!options.configPath.empty() diff --git a/tests/Progmasoft/RunnerImport.tests.cpp b/tests/Progmasoft/RunnerImport.tests.cpp index 4eab27f4..3872e1fd 100644 --- a/tests/Progmasoft/RunnerImport.tests.cpp +++ b/tests/Progmasoft/RunnerImport.tests.cpp @@ -160,8 +160,8 @@ ok 6 - parses { "name": "later", "status": "pending" }, { "name": "retried", "status": "passed", "flaky": true, "retries": 1, "retryAttempts": [ { "attempt": 1, "status": "failed", "message": "first" } ] }, - { "name": "listed", "status": "passed", - "retryAttempts": [ { "status": "failed" }, { "status": "passed" } ] } + { "name": "repeated", "status": "failed", "message": "second", "retries": 1, + "retryAttempts": [ { "attempt": 1, "status": "failed", "message": "first" } ] } ] } })"; @@ -399,6 +399,44 @@ TEST_CASE("TAP keeps escaped hashes in descriptions", CHECK(tests.front().outcome == Outcome::kSkipped); } +TEST_CASE("TAP directives are whole words", "[progmasoft][runner][import]") +{ + CHECK(Import("not ok 1 - a # TODOING\n").front().outcome + == Outcome::kFailed); + CHECK(Import("not ok 1 - a # skipper\n").front().outcome + == Outcome::kFailed); + CHECK(Import("not ok 1 - a # TODO: later\n").front().outcome + == Outcome::kFailedAsExpected); + CHECK(Import("not ok 1 - a # SKIPPED on Windows\n").front().outcome + == Outcome::kSkipped); + CHECK(Import("ok 1 - a # skipping\n").front().outcome == Outcome::kSkipped); + CHECK(Import("ok 1 - a # Skip: not today\n") + .front() + .attempts.front() + .standardOutput + == "not today"); +} + +TEST_CASE("TAP tests belong to their stream", "[progmasoft][runner][import]") +{ + auto first = Runner::ParseImport("not ok 1\n", + Runner::ImportFormat::kTap, + "linux.tap"); + auto second = Runner::ParseImport("ok 1\n", + Runner::ImportFormat::kTap, + "windows.tap"); + REQUIRE(first); + REQUIRE(second); + std::vector<TestResult> all = *first; + all.insert(all.end(), second->begin(), second->end()); + + const auto tests = Runner::MergeImported(std::move(all)); + REQUIRE(tests.size() == 2); + CHECK(tests.front().id == "linux.tap::test 1"); + CHECK(tests.front().outcome == Outcome::kFailed); + CHECK_FALSE(tests.front().flaky); +} + TEST_CASE("A TAP stream that ends early fails", "[progmasoft][runner][import]") { const auto bailed = Import("1..2\nok 1 - a\nBail out! no database\n"); @@ -446,8 +484,11 @@ TEST_CASE("CTRF reports are imported", "[progmasoft][runner][import]") CHECK(retried.outcome == Outcome::kPassed); CHECK(retried.attempts.size() == 2); - // A list of attempts that holds the last one does not count it twice. - CHECK(Named(tests, "listed").attempts.size() == 2); + // The retries end before the last attempt, whatever its outcome. + const auto &repeated = Named(tests, "repeated"); + CHECK(repeated.outcome == Outcome::kFailed); + REQUIRE(repeated.attempts.size() == 2); + CHECK(repeated.attempts.back().assertions.front().message == "second"); } TEST_CASE("catch3-results lines are imported", "[progmasoft][runner][import]") @@ -514,6 +555,23 @@ TEST_CASE("Results that share an identifier are attempts", CHECK_FALSE(tests.back().flaky); } +TEST_CASE("A failure is not lost to a later skip", + "[progmasoft][runner][import]") +{ + const auto tests = Import(R"(<testsuite> + <testcase name="a"><failure message="no"/></testcase> + <testcase name="a"><skipped/></testcase> + <testcase name="b" status="notrun"/> + <testcase name="b"><error message="no"/></testcase> + <testcase name="b" status="notrun"/> + </testsuite>)"); + REQUIRE(tests.size() == 2); + CHECK(tests.front().outcome == Outcome::kFailed); + CHECK(tests.front().category == FailureCategory::kAssertion); + CHECK(tests.back().category == FailureCategory::kUnexpectedException); + CHECK_FALSE(tests.front().flaky); +} + TEST_CASE("Durations that make no sense count as zero", "[progmasoft][runner][import]") { @@ -577,6 +635 |