diff options
| author | Leitwolf | 2026-10-24 19:03:40 +0300 |
|---|---|---|
| committer | GitHub | 2026-10-24 19:03:40 +0300 |
| commit | 4f1c759cffcb14bd6511e1d7c5fbace682e24f2d (patch) | |
| tree | 415fe5d48a5b48650bba3747beaf245d727c4f61 /tests/Progmasoft/Results.tests.cpp | |
| parent | 83c6fc3ac13abe83fa39f7b5f8559ccaf4412233 (diff) | |
| parent | 622188f9c08e4af35941c146979883ed4882410e (diff) | |
| download | catch3-4f1c759cffcb14bd6511e1d7c5fbace682e24f2d.tar.gz | |
Merge pull request #15 from Progmasoft/feature/catch3-runner
Replace Catch2's runner with the Catch3 runner
Diffstat (limited to 'tests/Progmasoft/Results.tests.cpp')
| -rw-r--r-- | tests/Progmasoft/Results.tests.cpp | 360 |
1 files changed, 217 insertions, 143 deletions
diff --git a/tests/Progmasoft/Results.tests.cpp b/tests/Progmasoft/Results.tests.cpp index 91b06856..1216fdd2 100644 --- a/tests/Progmasoft/Results.tests.cpp +++ b/tests/Progmasoft/Results.tests.cpp @@ -1,165 +1,239 @@ // Copyright (c) 2026 Progmasoft. // SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 -#include <Progmasoft/Catch3.hpp> +#include <Progmasoft/Catch3/Results.hpp> +#include <Progmasoft/Catch3/XmlWriter.hpp> + +#include <catch2/catch_test_macros.hpp> -#include <cmath> -#include <ios> -#include <limits> #include <sstream> -#include <stdexcept> #include <string> -#include <utility> - -namespace { - - Progmasoft::Catch3::TestCaseResult - MakeCase( std::string name, - Progmasoft::Catch3::TestCaseStatus status, - double duration ) { - Progmasoft::Catch3::TestCaseResult testCase; - testCase.Name = std::move( name ); - testCase.Status = status; - testCase.DurationSeconds = duration; - return testCase; - } -} // namespace - -TEST_CASE( "Test result summaries count outcomes and duration", - "[progmasoft][results]" ) { +namespace +{ using namespace Progmasoft::Catch3; - TestSuiteResult suite; - suite.Name = "Parser"; - suite.TestCases = { - MakeCase( "accepts input", TestCaseStatus::kPassed, 0.125 ), - MakeCase( "rejects malformed input", TestCaseStatus::kFailed, 0.250 ), - MakeCase( "handles I/O", TestCaseStatus::kErrored, 0.500 ), - MakeCase( "optional backend", TestCaseStatus::kSkipped, 0.0 ), - }; - const TestRunResult run{ "Compiler", { suite } }; - - const TestRunSummary summary = SummarizeTestRun( run ); - CHECK( summary.Tests == 4 ); - CHECK( summary.Passed() == 1 ); - CHECK( summary.Failures == 1 ); - CHECK( summary.Errors == 1 ); - CHECK( summary.Skipped == 1 ); - CHECK( summary.DurationSeconds == 0.875 ); -} + Attempt + MakeAttempt(Outcome outcome, + std::int64_t microseconds, + std::size_t number = 1) + { + Attempt attempt; + attempt.outcome = outcome; + attempt.number = number; + attempt.durationMicroseconds = microseconds; + if (IsFailure(outcome)) + { + attempt.category = outcome == Outcome::kCrashed + ? FailureCategory::kCrash + : outcome == Outcome::kTimedOut + ? FailureCategory::kTimeout + : FailureCategory::kAssertion; + } + return attempt; + } -TEST_CASE( "JUnit XML writer serializes counts, details, and escaped text", - "[progmasoft][results][xml]" ) { - using namespace Progmasoft::Catch3; + TestResult + MakeTest(const std::string &name, Outcome outcome, std::int64_t microseconds) + { + TestResult test; + test.id = name; + test.name = name; + test.file = "tests/parser.cpp"; + test.line = 12; + test.outcome = outcome; + test.attempts.push_back(MakeAttempt(outcome, microseconds)); + test.category = test.attempts.back().category; + return test; + } +} // namespace - TestSuiteResult suite; - suite.Name = "Parser & resolver"; - - auto failed = - MakeCase( "rejects <input>", TestCaseStatus::kFailed, 0.0125 ); - failed.ClassName = "CompilerTests"; - failed.SourceFile = "tests/parser.cpp"; - failed.Message = "assertion: \"expected\" & actual"; - failed.Details = "expected <node> but got <none>"; - failed.StandardOutput = "trace <start>&finish"; - suite.TestCases.push_back( std::move( failed ) ); - - auto errored = MakeCase( "throws", TestCaseStatus::kErrored, 0.25 ); - errored.Message = "runtime error"; - errored.Details = "first line\nsecond line"; - errored.StandardError = "failure detail"; - suite.TestCases.push_back( std::move( errored ) ); - - auto skipped = MakeCase( "platform-only", TestCaseStatus::kSkipped, 0.0 ); - skipped.Message = "not available"; - suite.TestCases.push_back( std::move( skipped ) ); - - suite.TestCases.push_back( - MakeCase( "passes", TestCaseStatus::kPassed, 0.001 ) ); - const TestRunResult run{ "Compiler run", { std::move( suite ) } }; - - const std::string xml = JUnitXmlWriter::ToString( run ); - CHECK( xml.starts_with( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" ) ); - CHECK( xml.find( "<testsuites name=\"Compiler run\" tests=\"4\" " ) != - std::string::npos ); - CHECK( xml.find( "failures=\"1\" errors=\"1\" skipped=\"1\"" ) != - std::string::npos ); - CHECK( xml.find( "name=\"Parser & resolver\"" ) != std::string::npos ); - CHECK( xml.find( "name=\"rejects <input>\"" ) != std::string::npos ); - CHECK( xml.find( - "message=\"assertion: "expected" & actual\"" ) != - std::string::npos ); - CHECK( xml.find( "expected <node> but got &lt;none>" ) != - std::string::npos ); - CHECK( - xml.find( "<system-out>trace <start>&finish</system-out>" ) != - std::string::npos ); - CHECK( xml.find( "<error message=\"runtime error\" type=\"TestError\">" ) != - std::string::npos ); - CHECK( xml.find( "<skipped message=\"not available\"/>" ) != - std::string::npos ); - CHECK( xml.find( "classname=\"CompilerTests\"" ) != std::string::npos ); - CHECK( xml.find( "file=\"tests/parser.cpp\"" ) != std::string::npos ); - CHECK( xml.find( "time=\"0.263500\"" ) != std::string::npos ); - CHECK( xml.ends_with( "</testsuites>\n" ) ); +TEST_CASE("A run summary counts every verdict once", + "[progmasoft][results]") +{ + RunResult run; + run.binary = "parser-tests"; + run.wallMicroseconds = 2000000; + run.tests.push_back(MakeTest("parses", Outcome::kPassed, 125000)); + run.tests.push_back(MakeTest("reports", Outcome::kFailed, 250000)); + run.tests.push_back(MakeTest("loads", Outcome::kCrashed, 500000)); + run.tests.push_back(MakeTest("waits", Outcome::kTimedOut, 1000000)); + auto flaky = MakeTest("retries", Outcome::kPassed, 0); + flaky.flaky = true; + flaky.attempts.insert(flaky.attempts.begin(), + MakeAttempt(Outcome::kFailed, 10000)); + run.tests.push_back(flaky); + auto quarantined = MakeTest("quarantined", Outcome::kFailed, 1); + quarantined.quarantined = true; + run.tests.push_back(quarantined); + + const RunSummary summary = Summarize(run); + CHECK(summary.tests == 6); + CHECK(summary.passed == 2); + CHECK(summary.failed == 2); + CHECK(summary.crashed == 1); + CHECK(summary.timedOut == 1); + CHECK(summary.flaky == 1); + CHECK(summary.quarantined == 1); + CHECK(summary.attempts == 7); + CHECK(summary.testMicroseconds == 1885001); + + CHECK(run.tests[1].FailsRun()); + CHECK_FALSE(run.tests[4].FailsRun()); + CHECK_FALSE(run.tests[5].FailsRun()); // In quarantine. } -TEST_CASE( "JUnit XML replaces malformed UTF-8 and forbidden XML controls", - "[progmasoft][results][xml]" ) { - using namespace Progmasoft::Catch3; - - auto testCase = MakeCase( "valid", TestCaseStatus::kPassed, 0.0 ); - testCase.Name = std::string( "bad\x01utf8\xF0\x28\x8C\x28", 12 ); - const TestRunResult run{ std::string( "suite\0name", 10 ), - { { "suite", { testCase } } } }; - - const std::string xml = JUnitXmlWriter::ToString( run ); - CHECK( xml.find( "name=\"bad�utf8�(�(\"" ) != - std::string::npos ); - CHECK( xml.find( "name=\"suite�name\"" ) != std::string::npos ); +TEST_CASE("Outcome and category names read back", "[progmasoft][results]") +{ + for (const Outcome outcome : {Outcome::kPassed, + Outcome::kFailed, + Outcome::kFailedAsExpected, + Outcome::kSkipped, + Outcome::kCrashed, + Outcome::kTimedOut, + Outcome::kBlocked, + Outcome::kNotRun, + Outcome::kUpToDate}) + { + CHECK(OutcomeFromName(OutcomeName(outcome)) == outcome); + } + CHECK(CategoryFromName(CategoryName(FailureCategory::kTimeout)) == + FailureCategory::kTimeout); + CHECK_FALSE(OutcomeFromName("exploded").has_value()); } -TEST_CASE( "Result summaries reject invalid durations and statuses", - "[progmasoft][results]" ) { - using namespace Progmasoft::Catch3; +TEST_CASE("A failure is described with its expansion, messages and sections", + "[progmasoft][results]") +{ + Attempt attempt = MakeAttempt(Outcome::kFailed, 1000); + AssertionRecord assertion; + assertion.file = "parser.cpp"; + assertion.line = 40; + assertion.macro = "REQUIRE"; + assertion.expression = "tokens.size() == 3"; + assertion.expansion = "2 == 3"; + assertion.message = "the comma was lost"; + assertion.context = {"input := a,b"}; + assertion.sections = {"lists", "with commas"}; + attempt.assertions.push_back(assertion); + + const std::string text = DescribeFailure(attempt); + CHECK(text.find("parser.cpp:40: FAILED:") != std::string::npos); + CHECK(text.find("REQUIRE( tokens.size() == 3 )") != std::string::npos); + CHECK(text.find("with expansion:\n 2 == 3") != std::string::npos); + CHECK(text.find("with message:\n the comma was lost") != std::string::npos); + CHECK(text.find("with context:\n input := a,b") != std::string::npos); + CHECK(text.find("in section lists / with commas") != std::string::npos); + CHECK(SummarizeFailure(attempt) == "parser.cpp:40: FAILED:"); +} - TestCaseResult invalidDuration = - MakeCase( "invalid", TestCaseStatus::kPassed, -1.0 ); - CHECK_THROWS_AS( - SummarizeTestRun( { "run", { { "suite", { invalidDuration } } } } ), - std::invalid_argument ); - - invalidDuration.DurationSeconds = std::numeric_limits<double>::infinity(); - CHECK_THROWS_AS( - SummarizeTestRun( { "run", { { "suite", { invalidDuration } } } } ), - std::invalid_argument ); - - invalidDuration.DurationSeconds = 0.0; - invalidDuration.Status = static_cast<TestCaseStatus>( 255 ); - CHECK_THROWS_AS( - SummarizeTestRun( { "run", { { "suite", { invalidDuration } } } } ), - std::invalid_argument ); +TEST_CASE("A crash and a timeout say where they stopped", + "[progmasoft][results]") +{ + Attempt crash = MakeAttempt(Outcome::kCrashed, 1000); + crash.exitDescription = "signal 11 (SIGSEGV)"; + crash.fatalMessage = "SIGSEGV - Segmentation violation signal"; + crash.lastLocation = "parser.cpp:51"; + SectionRecord open; + open.name = "deep input"; + open.durationMicroseconds = -1; + crash.sections.push_back(open); + const std::string crashText = DescribeFailure(crash); + CHECK(crashText.find("crashed with signal 11 (SIGSEGV)") != + std::string::npos); + CHECK(crashText.find("Segmentation violation") != std::string::npos); + CHECK(crashText.find("last assertion started at parser.cpp:51") != + std::string::npos); + CHECK(crashText.find("in section deep input") != std::string::npos); + + const Attempt timeout = MakeAttempt(Outcome::kTimedOut, 2500000); + CHECK(DescribeFailure(timeout).find("timed out after 2.5 s") != + std::string::npos); } -TEST_CASE( "JUnit XML writer validates results and output stream state", - "[progmasoft][results][xml]" ) { - using namespace Progmasoft::Catch3; +TEST_CASE("JUnit XML keeps every attempt and marks flaky tests", + "[progmasoft][results][xml]") +{ + RunResult run; + run.binary = "parser-tests"; + run.seed = 42; + run.workers = 4; + run.wallMicroseconds = 263500; + + auto failed = MakeTest("reports <errors>", Outcome::kFailed, 263500); + failed.className = "ParserTests"; + failed.attempts.back().standardOutput = "trace <start>&finish"; + AssertionRecord assertion; + assertion.file = "parser.cpp"; + assertion.line = 7; + assertion.expression = "ok"; + failed.attempts.back().assertions.push_back(assertion); + failed.analysis.push_back("new failure after 3 passing run(s)"); + failed.replayCommand = "parser-tests \"reports <errors>\" --seed=42"; + run.tests.push_back(failed); + + auto crashed = MakeTest("loads", Outcome::kCrashed, 1); + crashed.attempts.back().exitDescription = "signal 6 (SIGABRT)"; + run.tests.push_back(crashed); + + auto flaky = MakeTest("retries", Outcome::kPassed, 10); + flaky.flaky = true; + flaky.attempts.insert(flaky.attempts.begin(), + MakeAttempt(Outcome::kTimedOut, 5000000)); + run.tests.push_back(flaky); + + auto blocked = MakeTest("needs the parser", Outcome::kBlocked, 0); + blocked.attempts.clear(); + blocked.analysis.push_back("depends on 'reports <errors>', which failed"); + run.tests.push_back(blocked); + + const std::string xml = JUnitXmlWriter::ToString(run); + CHECK(xml.rfind("<?xml version=\"1.0\" encoding=\"UTF-8\"?>", 0) == 0); + CHECK(xml.find("tests=\"4\" failures=\"1\" errors=\"1\" skipped=\"1\"") != + std::string::npos); + CHECK(xml.find("name=\"reports <errors>\"") != std::string::npos); + CHECK(xml.find("classname=\"ParserTests\"") != std::string::npos); + CHECK(xml.find("<failure message=\"parser.cpp:7: FAILED:\" " + "type=\"assertion\">") != std::string::npos); + CHECK(xml.find("analysis: new failure after 3 passing run(s)") != + std::string::npos); + CHECK(xml.find("<system-out>trace <start>&finish</system-out>") != + std::string::npos); + CHECK(xml.find("<error message=\"crashed with signal 6 (SIGABRT)\" " + "type=\"crash\">") != std::string::npos); + CHECK(xml.find("<flakyFailure message=\"timed out after 5.0 s\" " + "type=\"timeout\">") != std::string::npos); + CHECK(xml.find("<skipped message=\"depends on 'reports <errors>" + "', which failed\"/>") != std::string::npos); + CHECK(xml.find("<property name=\"random-seed\" value=\"42\"/>") != + std::string::npos); + CHECK(xml.find("time=\"0.263500\"") != std::string::npos); + CHECK(xml.ends_with("</testsuites>\n")); +} - const TestRunResult emptyRun{ "empty", {} }; - const std::string emptyXml = JUnitXmlWriter::ToString( emptyRun ); - CHECK( emptyXml.find( "tests=\"0\" failures=\"0\" errors=\"0\" " ) != - std::string::npos ); +TEST_CASE("JUnit XML replaces malformed UTF-8 and forbidden XML controls", + "[progmasoft][results][xml]") +{ + RunResult run; + run.binary = std::string("suite\0name", 10); + auto test = MakeTest("valid", Outcome::kPassed, 0); + test.name = std::string("bad\x01utf8\xF0\x28\x8C\x28", 12); + run.tests.push_back(test); + + const std::string xml = JUnitXmlWriter::ToString(run); + CHECK(xml.find("name=\"bad�utf8�(�(\"") != + std::string::npos); + CHECK(xml.find("name=\"suite�name\"") != std::string::npos); +} - auto invalidCase = MakeCase( "bad", TestCaseStatus::kPassed, 0.0 ); - invalidCase.DurationSeconds = std::numeric_limits<double>::quiet_NaN(); - const TestRunResult invalidRun{ "invalid", - { { "suite", { invalidCase } } } }; +TEST_CASE("The JUnit writer reports a failed stream", "[progmasoft][results][xml]") +{ + const RunResult empty; std::ostringstream output; - CHECK_THROWS_AS( JUnitXmlWriter::Write( invalidRun, output ), - std::invalid_argument ); + CHECK(JUnitXmlWriter::Write(empty, output)); + CHECK(output.str().find("tests=\"0\" failures=\"0\" errors=\"0\" ") != + std::string::npos); - std::ostringstream failedOutput; - failedOutput.setstate( std::ios::badbit ); - CHECK_THROWS_AS( JUnitXmlWriter::Write( emptyRun, failedOutput ), - std::ios_base::failure ); + std::ostringstream failed; + failed.setstate(std::ios::badbit); + CHECK_FALSE(JUnitXmlWriter::Write(empty, failed)); } |