Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/src/Progmasoft/Runner/Runner.cpp
diff options
context:
space:
mode:
authorLeitwolf112026-11-01 14:48:55 +0300
committerLeitwolf112026-11-01 14:48:55 +0300
commit569620dad14df7e13bb9fd31d256e9fd873ef6ec (patch)
treef933300c29658c3eb89d126d74b6c30fdff9dd04 /src/Progmasoft/Runner/Runner.cpp
parent1db203d56b7c71db3d80b477da1ad49e9ae46468 (diff)
downloadcatch3-feature/result-import.tar.gz
Keep real failures visible when importingfeature/result-import
A review of the import found ways for a failure to disappear or for the import to reach too far. - CTRF's retryAttempts ends before the final attempt, which was dropped when its outcome matched the one before it. - TAP names tests only within a stream, so a failure in one file and a pass under the same name in another merged into a flaky pass. The path given to --import is now the class of a stream's tests. - A failure followed by a skip with the same identifier read as skipped. - Directives are whole words: TODOING is not TODO. SKIPPED and SKIPPING stay directives, as harnesses treat them. - Verdicts that alternate count as flaky only under an explicit --import-revision. HEAD says nothing about a dirty checkout or a report from another one, and two such events could quarantine a real failure. - --import-name no longer redirects the history of a normal test run. - Importing does not depend on the tests registered in the binary.
Diffstat (limited to 'src/Progmasoft/Runner/Runner.cpp')
-rw-r--r--src/Progmasoft/Runner/Runner.cpp31
1 files changed, 21 insertions, 10 deletions
diff --git a/src/Progmasoft/Runner/Runner.cpp b/src/Progmasoft/Runner/Runner.cpp
index ae711293..77011322 100644
--- a/src/Progmasoft/Runner/Runner.cpp
+++ b/src/Progmasoft/Runner/Runner.cpp
@@ -627,11 +627,13 @@ namespace Progmasoft::Catch3
const std::filesystem::path &program,
const std::string &programName)
{
- std::string name = options.importName;
- if (name.empty())
+ std::string name = programName;
+ if (options.mode == Runner::Mode::kImport
+ || (options.mode == Runner::Mode::kAnalytics
+ && !options.importName.empty()))
{
- name = options.mode == Runner::Mode::kImport ? "imported"
- : programName;
+ name = options.importName.empty() ? "imported"
+ : options.importName;
}
return Runner::HistoryFile(HistoryDirectory(options, program),
name);
@@ -653,7 +655,10 @@ namespace Progmasoft::Catch3
PrintError(text.GetError().Message());
return Catch::UnspecifiedErrorExitCode;
}
- auto tests = Runner::ParseImport(*text, options.importFormat);
+ auto tests = Runner::ParseImport(
+ *text,
+ options.importFormat,
+ std::filesystem::path{ file }.generic_string());
if (!tests)
{
PrintError(tests.GetError().Context(file).Message());
@@ -693,11 +698,16 @@ namespace Progmasoft::Catch3
? Runner::CurrentCommit(
std::filesystem::current_path(here))
: options.importRevision;
+ // Only the person importing knows that two reports tested the
+ // same thing: a checkout can have edits, or be another one.
std::map<std::string, std::string> fingerprints;
for (auto &test : run.tests)
{
run.wallMicroseconds += test.DurationMicroseconds();
- fingerprints[test.id] = revision;
+ if (!options.importRevision.empty())
+ {
+ fingerprints[test.id] = options.importRevision;
+ }
const Runner::TestHistory *past = history.Find(test.id);
test.quarantined
= options.quarantine != Runner::QuarantineMode::kOff
@@ -758,6 +768,11 @@ namespace Progmasoft::Catch3
std::printf("Catch3 %s\n", kVersion);
return 0;
}
+ // Importing does not touch the tests of this binary.
+ if (options.mode == Runner::Mode::kImport)
+ {
+ return Import(options, program, programName);
+ }
if (ReportStartupErrors())
{
return Catch::UnspecifiedErrorExitCode;
@@ -772,10 +787,6 @@ namespace Progmasoft::Catch3
{
return Runner::RunWorker(options, programName);
}
- if (options.mode == Runner::Mode::kImport)
- {
- return Import(options, program, programName);
- }
std::vector<std::string> notes;
if (!options.configPath.empty()