1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
|
// Copyright (c) 2026 Progmasoft.
// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-2.0
// What the runner prints and returns: its modes, the console report, the
// exit codes and Bazel's test environment.
#include <Progmasoft/Catch3/Version.hpp>
#include <Progmasoft/Runner/Files.hpp>
#include <Progmasoft/Runner/Json.hpp>
#include <Progmasoft/Catch3/TestMacros.hpp>
#include <Progmasoft/Catch3/Matchers/String.hpp>
#include <filesystem>
#include <fstream>
#include <string>
#include <vector>
#include "RunnerFixtureClient.hpp"
namespace
{
namespace Runner = Progmasoft::Catch3::Runner;
using Catch::Matchers::ContainsSubstring;
using FixtureClient::ExitCode;
using FixtureClient::RunFixture;
using FixtureClient::RunPlain;
using FixtureClient::Scratch;
std::int64_t
CodeOf(const Runner::CapturedRun &run)
{
INFO(run.output << run.errors);
REQUIRE(run.status.kind == Runner::ExitStatus::Kind::kExited);
return run.status.code;
}
} // namespace
TEST_CASE("The runner tells its version and how to use it",
"[progmasoft][runner][output]")
{
const auto version = RunPlain({ "--version" });
CHECK(CodeOf(version) == 0);
CHECK_THAT(version.output,
ContainsSubstring(std::string{ "Catch3 " }
+ Progmasoft::Catch3::kVersion));
const auto help = RunPlain({ "--help" });
CHECK(CodeOf(help) == 0);
CHECK_THAT(help.output, ContainsSubstring("usage: "));
CHECK_THAT(help.output, ContainsSubstring("--shard=INDEX/COUNT"));
}
TEST_CASE("Tests and tags are listed", "[progmasoft][runner][output]")
{
const auto text = RunPlain({ "--list", "[deps]" });
CHECK(CodeOf(text) == 0);
CHECK_THAT(text.output, ContainsSubstring("fixture dependent"));
CHECK_THAT(text.output, ContainsSubstring("RunnerFixture.cpp:"));
CHECK_THAT(text.output, ContainsSubstring("[!depends:fixture base]"));
CHECK_THAT(text.output, ContainsSubstring("2 matching test(s)"));
const auto names = RunPlain({ "--list=names", "[deps]" });
CHECK_THAT(names.output, ContainsSubstring("fixture base"));
CHECK(names.output.find("matching") == std::string::npos);
const auto tags = RunPlain({ "--list-tags" });
CHECK(CodeOf(tags) == 0);
CHECK_THAT(tags.output, ContainsSubstring("[fixture]"));
CHECK_THAT(tags.output, ContainsSubstring("tag(s)"));
}
TEST_CASE("Mistakes on the command line stop the run",
"[progmasoft][runner][output]")
{
const auto unknown = RunPlain({ "--jbos=2" });
CHECK(CodeOf(unknown) == 1);
CHECK_THAT(unknown.errors, ContainsSubstring("error: "));
CHECK_THAT(unknown.errors, ContainsSubstring("did you mean --jobs?"));
}
TEST_CASE("Exit codes say what kind of run it was",
"[progmasoft][runner][output]")
{
const Scratch scratch;
SECTION("nothing matched")
{
const auto run
= RunFixture(scratch, { "--no-history", "no such test" });
CHECK(ExitCode(run) == 2);
// Catch2 quotes a spec that has spaces.
CHECK_THAT(
run.output,
ContainsSubstring("warning: no test matches '\"no such test\"'"));
}
SECTION("nothing matched, and that was a warning")
{
const auto run = RunFixture(scratch,
{ "--no-history",
"--warn=unmatched-specs",
"[pass]",
"no such test" });
CHECK(ExitCode(run) == 3);
}
SECTION("nothing matched, and that was allowed")
{
const auto run = RunFixture(
scratch,
{ "--no-history", "--allow-no-tests", "no such test" });
CHECK(ExitCode(run) == 0);
}
SECTION("everything skipped")
{
const auto run = RunFixture(scratch, { "--no-history", "[skip]" });
CHECK(ExitCode(run) == 4);
}
SECTION("a report that cannot be written")
{
// A directory where the report file should go.
const auto blocked = scratch.Path() / "blocked.xml";
std::filesystem::create_directories(blocked / "inside");
const auto run = RunFixture(
scratch,
{ "--no-history", "[pass]", "--junit=" + blocked.string() });
CHECK(ExitCode(run) == 1);
CHECK_THAT(run.output, ContainsSubstring("error: "));
}
}
TEST_CASE("The console report follows the verbosity",
"[progmasoft][runner][output]")
{
const Scratch scratch;
// Passing tests get a line, and their output, only when asked for.
const auto normal = RunFixture(scratch, { "--no-history", "[pass]" });
CHECK_THAT(normal.output, ContainsSubstring("Catch3: 2 test(s)"));
CHECK(normal.output.find("PASS fixture prints") == std::string::npos);
CHECK(normal.output.find("said through std::cout") == std::string::npos);
const auto verbose
= RunFixture(scratch, { "--no-history", "-v", "[pass]" });
CHECK_THAT(verbose.output, ContainsSubstring("PASS fixture prints"));
CHECK_THAT(verbose.output, ContainsSubstring("said through std::cout"));
const auto quiet = RunFixture(scratch, { "--no-history", "-q", "[pass]" });
CHECK(ExitCode(quiet) == 0);
CHECK(quiet.output.find("Catch3: ") == std::string::npos);
const auto slowest
= RunFixture(scratch, { "--no-history", "--slowest=2", "[slow]" });
CHECK_THAT(slowest.output, ContainsSubstring("slowest tests:"));
const auto coloured
= RunPlain({ "--no-history", "--color=always", "[pass]" });
CHECK_THAT(coloured.output, ContainsSubstring("\x1b["));
const auto plain = RunPlain({ "--no-history", "--color=never", "[pass]" });
CHECK(plain.output.find("\x1b[") == std::string::npos);
}
TEST_CASE("The history is summarised on request",
"[progmasoft][runner][output]")
{
const Scratch scratch;
const auto history = "--history=" + scratch.Path().string();
for (int run = 0; run < 3; ++run)
{
static_cast<void>(
RunFixture(scratch,
{ history, "--retries=off", "[pass],[alternate]" }));
}
const auto text = RunPlain({ history, "--analytics" });
CHECK(CodeOf(text) == 0);
CHECK_THAT(text.output, ContainsSubstring("3 run(s)"));
CHECK_THAT(text.output, ContainsSubstring("flaky (last 20 runs):"));
CHECK_THAT(text.output, ContainsSubstring("fixture alternates"));
CHECK_THAT(text.output, ContainsSubstring("slowest (median):"));
const auto json = RunPlain({ history, "--analytics=json" });
CHECK(CodeOf(json) == 0);
auto parsed = Runner::ParseJson(json.output);
REQUIRE(parsed);
CHECK((*parsed)["format"].AsString() == "catch3-analytics");
CHECK((*parsed)["runs"].AsInteger() == 3);
CHECK_FALSE((*parsed)["tests"].Items().empty());
}
TEST_CASE("Bazel's test environment is honoured",
"[progmasoft][runner][output]")
{
const Scratch scratch;
const auto xml = scratch.Path() / "test.xml";
const auto outputs = scratch.Path() / "outputs";
const auto status = scratch.Path() / "shard.status";
std::filesystem::create_directories(outputs);
const auto run
= RunPlain({ "--color=never" },
{ { "BAZEL_TEST", "1" },
{ "XML_OUTPUT_FILE", xml.string() },
{ "TESTBRIDGE_TEST_ONLY", "fixture passes" },
{ "TEST_SHARD_INDEX", "0" },
{ "TEST_TOTAL_SHARDS", "1" },
{ "TEST_SHARD_STATUS_FILE", status.string() },
{ "TEST_RANDOM_SEED", "1234" },
{ "TEST_UNDECLARED_OUTPUTS_DIR", outputs.string() },
{ "CATCH3_FIXTURE_STATE", scratch.Path().string() } });
CHECK(CodeOf(run) == 0);
CHECK_THAT(run.output, ContainsSubstring("1 test(s): 1 passed"));
CHECK(std::filesystem::exists(status));
auto junit = Runner::ReadFile(xml);
REQUIRE(junit);
CHECK_THAT(*junit, ContainsSubstring("fixture passes"));
auto report = Runner::ReadFile(outputs / "catch3.json");
REQUIRE(report);
auto parsed = Runner::ParseJson(*report);
REQUIRE(parsed);
CHECK((*parsed)["seed"].AsInteger() == 1234);
// A sandboxed test keeps no history beside the binary.
const auto beside
= std::filesystem::path{ FixtureClient::Fixture() }.parent_path()
/ ".catch3";
CHECK_FALSE(std::filesystem::exists(beside));
}
|