1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
|
// Copyright (c) 2026 Progmasoft.
// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0
#pragma once
#include <Progmasoft/Catch3/Results.hpp>
#include <chrono>
#include <cstdint>
#include <filesystem>
#include <map>
#include <optional>
#include <string>
#include <vector>
#include "Expected.hpp"
#include "Json.hpp"
namespace Progmasoft::Catch3::Runner
{
/// One result of one test in one run.
struct TestRecord final
{
std::int64_t run = 0;
Outcome outcome = Outcome::kNotRun;
FailureCategory category = FailureCategory::kNone;
std::int64_t durationMicroseconds = 0;
std::size_t attempts = 0;
bool flaky = false;
std::string signature;
/// What the test's result depended on; equal fingerprints mean the
/// same inputs.
std::string fingerprint;
};
/// How to run a failure again the way it happened.
struct ReplayRecord final
{
std::int64_t run = 0;
std::uint32_t seed = 0;
/// The tests the worker ran before, in order.
std::vector<std::string> precededBy;
bool freshProcess = false;
};
struct QuarantineRecord final
{
std::int64_t since = 0;
std::string reason;
};
/// Everything remembered about one test.
struct TestHistory final
{
/// The most recent last.
std::vector<TestRecord> records;
std::optional<QuarantineRecord> quarantine;
std::optional<ReplayRecord> lastFailure;
[[nodiscard]] const TestRecord *
Last() const noexcept;
/// A percentile of the durations of recent runs that finished on
/// their own; nothing without such runs.
[[nodiscard]] std::optional<std::int64_t>
DurationPercentile(double fraction) const;
/// The probability that the next run fails: a failure rate in which
/// each older run counts less, starting from a prior for tests with
/// little history.
[[nodiscard]] double
FailureProbability() const;
/// Flaky results, and failures that alternate with passes on the
/// same fingerprint, among the last `window` runs.
[[nodiscard]] std::size_t
FlakyEvents(std::size_t window) const;
/// Passes in a row, counting back from the latest run.
[[nodiscard]] std::size_t
PassStreak() const;
};
/// One run, as the history remembers it.
struct RunRecord final
{
std::int64_t id = 0;
std::int64_t unixMilliseconds = 0;
std::string commit;
std::string binaryFingerprint;
std::uint32_t seed = 0;
std::size_t tests = 0;
std::size_t failed = 0;
std::int64_t wallMicroseconds = 0;
};
/// When to put a test into quarantine and let it out.
struct QuarantinePolicy final
{
std::size_t window = 20;
std::size_t flakyEventsToEnter = 2;
std::size_t passesToLeave = 10;
};
/// The results of earlier runs of one test binary.
///
/// It lives in one JSON file. Several runs of the same binary may finish
/// at the same time, so `Commit` changes the file under a lock: it reads
/// it again, adds its run and replaces it in one rename.
class History final
{
public:
History() = default;
/// Reads the file; a missing file is an empty history.
[[nodiscard]] static Expected<History>
Load(const std::filesystem::path &file);
/// Adds `run` to the history in `file`, updates quarantine and
/// returns the history as written.
[[nodiscard]] static Expected<History>
Commit(const std::filesystem::path &file,
const RunResult &run,
const RunRecord &record,
const std::map<std::string, std::string> &fingerprints,
const QuarantinePolicy &policy,
std::chrono::milliseconds lockWait = std::chrono::seconds(10));
[[nodiscard]] const TestHistory *
Find(const std::string &key) const;
[[nodiscard]] const std::map<std::string, TestHistory> &
Tests() const noexcept
{
return tests_;
}
[[nodiscard]] const std::vector<RunRecord> &
Runs() const noexcept
{
return runs_;
}
[[nodiscard]] JsonValue
ToJson() const;
[[nodiscard]] static Expected<History>
FromJson(const JsonValue &value);
/// The key a test result is filed under.
[[nodiscard]] static std::string
KeyOf(const TestResult &test);
private:
void
Apply(const RunResult &run,
RunRecord record,
const std::map<std::string, std::string> &fingerprints,
const QuarantinePolicy &policy);
std::int64_t nextRun_ = 1;
std::vector<RunRecord> runs_;
std::map<std::string, TestHistory> tests_;
};
/// The file the history of a binary is kept in under `directory`.
[[nodiscard]] std::filesystem::path
HistoryFile(const std::filesystem::path &directory,
const std::string &binaryName);
} // namespace Progmasoft::Catch3::Runner
|