diff options
| author | Leitwolf11 | 2026-11-01 14:48:55 +0300 |
|---|---|---|
| committer | Leitwolf11 | 2026-11-01 14:48:55 +0300 |
| commit | 569620dad14df7e13bb9fd31d256e9fd873ef6ec (patch) | |
| tree | f933300c29658c3eb89d126d74b6c30fdff9dd04 /src/Progmasoft/Runner/Runner.cpp | |
| parent | 1db203d56b7c71db3d80b477da1ad49e9ae46468 (diff) | |
| download | catch3-569620dad14df7e13bb9fd31d256e9fd873ef6ec.tar.gz | |
Keep real failures visible when importingfeature/result-import
A review of the import found ways for a failure to disappear or for the
import to reach too far.
- CTRF's retryAttempts ends before the final attempt, which was dropped when
its outcome matched the one before it.
- TAP names tests only within a stream, so a failure in one file and a pass
under the same name in another merged into a flaky pass. The path given to
--import is now the class of a stream's tests.
- A failure followed by a skip with the same identifier read as skipped.
- Directives are whole words: TODOING is not TODO. SKIPPED and SKIPPING stay
directives, as harnesses treat them.
- Verdicts that alternate count as flaky only under an explicit
--import-revision. HEAD says nothing about a dirty checkout or a report from
another one, and two such events could quarantine a real failure.
- --import-name no longer redirects the history of a normal test run.
- Importing does not depend on the tests registered in the binary.
Diffstat (limited to 'src/Progmasoft/Runner/Runner.cpp')
| -rw-r--r-- | src/Progmasoft/Runner/Runner.cpp | 31 |
1 files changed, 21 insertions, 10 deletions
diff --git a/src/Progmasoft/Runner/Runner.cpp b/src/Progmasoft/Runner/Runner.cpp index ae711293..77011322 100644 --- a/src/Progmasoft/Runner/Runner.cpp +++ b/src/Progmasoft/Runner/Runner.cpp @@ -627,11 +627,13 @@ namespace Progmasoft::Catch3 const std::filesystem::path &program, const std::string &programName) { - std::string name = options.importName; - if (name.empty()) + std::string name = programName; + if (options.mode == Runner::Mode::kImport + || (options.mode == Runner::Mode::kAnalytics + && !options.importName.empty())) { - name = options.mode == Runner::Mode::kImport ? "imported" - : programName; + name = options.importName.empty() ? "imported" + : options.importName; } return Runner::HistoryFile(HistoryDirectory(options, program), name); @@ -653,7 +655,10 @@ namespace Progmasoft::Catch3 PrintError(text.GetError().Message()); return Catch::UnspecifiedErrorExitCode; } - auto tests = Runner::ParseImport(*text, options.importFormat); + auto tests = Runner::ParseImport( + *text, + options.importFormat, + std::filesystem::path{ file }.generic_string()); if (!tests) { PrintError(tests.GetError().Context(file).Message()); @@ -693,11 +698,16 @@ namespace Progmasoft::Catch3 ? Runner::CurrentCommit( std::filesystem::current_path(here)) : options.importRevision; + // Only the person importing knows that two reports tested the + // same thing: a checkout can have edits, or be another one. std::map<std::string, std::string> fingerprints; for (auto &test : run.tests) { run.wallMicroseconds += test.DurationMicroseconds(); - fingerprints[test.id] = revision; + if (!options.importRevision.empty()) + { + fingerprints[test.id] = options.importRevision; + } const Runner::TestHistory *past = history.Find(test.id); test.quarantined = options.quarantine != Runner::QuarantineMode::kOff @@ -758,6 +768,11 @@ namespace Progmasoft::Catch3 std::printf("Catch3 %s\n", kVersion); return 0; } + // Importing does not touch the tests of this binary. + if (options.mode == Runner::Mode::kImport) + { + return Import(options, program, programName); + } if (ReportStartupErrors()) { return Catch::UnspecifiedErrorExitCode; @@ -772,10 +787,6 @@ namespace Progmasoft::Catch3 { return Runner::RunWorker(options, programName); } - if (options.mode == Runner::Mode::kImport) - { - return Import(options, program, programName); - } std::vector<std::string> notes; if (!options.configPath.empty() |