Add snapshots and JUnit result serialization0.2.0

5 files changed, 344 insertions, 3 deletions

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index d307ecd8..78e56f1d 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -157,7 +157,11 @@ target_link_libraries(SelfTest PRIVATE Catch2WithMain)
 
 # Exercise the C++20 Progmasoft layer independently so the upstream Catch2
 # SelfTest continues to compile at its supported baseline language level.
-add_executable(ProgmasoftSelfTest Progmasoft/Property.tests.cpp)
+add_executable(ProgmasoftSelfTest
+  Progmasoft/Property.tests.cpp
+  Progmasoft/Results.tests.cpp
+  Progmasoft/Snapshot.tests.cpp
+)
 target_link_libraries(ProgmasoftSelfTest PRIVATE Progmasoft::Catch3 Catch2WithMain)
 target_compile_features(ProgmasoftSelfTest PRIVATE cxx_std_20)
 add_test(NAME Progmasoft::PropertyTests COMMAND $<TARGET_FILE:ProgmasoftSelfTest>)
diff --git a/tests/Progmasoft/BUILD.bazel b/tests/Progmasoft/BUILD.bazel
index c0cb14ac..18912383 100644
--- a/tests/Progmasoft/BUILD.bazel
+++ b/tests/Progmasoft/BUILD.bazel
@@ -6,7 +6,11 @@ package(default_visibility = ["//visibility:public"])
 
 cc_test(
     name = "property_tests",
-    srcs = ["Property.tests.cpp"],
+    srcs = [
+        "Property.tests.cpp",
+        "Results.tests.cpp",
+        "Snapshot.tests.cpp",
+    ],
     copts = select({
         "@platforms//os:windows": ["/std:c++20"],
         "//conditions:default": ["-std=c++20"],
diff --git a/tests/Progmasoft/Results.tests.cpp b/tests/Progmasoft/Results.tests.cpp
new file mode 100644
index 00000000..91b06856
--- /dev/null
+++ b/tests/Progmasoft/Results.tests.cpp
@@ -0,0 +1,165 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+#include <Progmasoft/Catch3.hpp>
+
+#include <cmath>
+#include <ios>
+#include <limits>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <utility>
+
+namespace {
+
+    Progmasoft::Catch3::TestCaseResult
+    MakeCase( std::string name,
+              Progmasoft::Catch3::TestCaseStatus status,
+              double duration ) {
+        Progmasoft::Catch3::TestCaseResult testCase;
+        testCase.Name = std::move( name );
+        testCase.Status = status;
+        testCase.DurationSeconds = duration;
+        return testCase;
+    }
+
+} // namespace
+
+TEST_CASE( "Test result summaries count outcomes and duration",
+           "[progmasoft][results]" ) {
+    using namespace Progmasoft::Catch3;
+
+    TestSuiteResult suite;
+    suite.Name = "Parser";
+    suite.TestCases = {
+        MakeCase( "accepts input", TestCaseStatus::kPassed, 0.125 ),
+        MakeCase( "rejects malformed input", TestCaseStatus::kFailed, 0.250 ),
+        MakeCase( "handles I/O", TestCaseStatus::kErrored, 0.500 ),
+        MakeCase( "optional backend", TestCaseStatus::kSkipped, 0.0 ),
+    };
+    const TestRunResult run{ "Compiler", { suite } };
+
+    const TestRunSummary summary = SummarizeTestRun( run );
+    CHECK( summary.Tests == 4 );
+    CHECK( summary.Passed() == 1 );
+    CHECK( summary.Failures == 1 );
+    CHECK( summary.Errors == 1 );
+    CHECK( summary.Skipped == 1 );
+    CHECK( summary.DurationSeconds == 0.875 );
+}
+
+TEST_CASE( "JUnit XML writer serializes counts, details, and escaped text",
+           "[progmasoft][results][xml]" ) {
+    using namespace Progmasoft::Catch3;
+
+    TestSuiteResult suite;
+    suite.Name = "Parser & resolver";
+
+    auto failed =
+        MakeCase( "rejects <input>", TestCaseStatus::kFailed, 0.0125 );
+    failed.ClassName = "CompilerTests";
+    failed.SourceFile = "tests/parser.cpp";
+    failed.Message = "assertion: \"expected\" & actual";
+    failed.Details = "expected <node> but got &lt;none>";
+    failed.StandardOutput = "trace <start>&finish";
+    suite.TestCases.push_back( std::move( failed ) );
+
+    auto errored = MakeCase( "throws", TestCaseStatus::kErrored, 0.25 );
+    errored.Message = "runtime error";
+    errored.Details = "first line\nsecond line";
+    errored.StandardError = "failure detail";
+    suite.TestCases.push_back( std::move( errored ) );
+
+    auto skipped = MakeCase( "platform-only", TestCaseStatus::kSkipped, 0.0 );
+    skipped.Message = "not available";
+    suite.TestCases.push_back( std::move( skipped ) );
+
+    suite.TestCases.push_back(
+        MakeCase( "passes", TestCaseStatus::kPassed, 0.001 ) );
+    const TestRunResult run{ "Compiler run", { std::move( suite ) } };
+
+    const std::string xml = JUnitXmlWriter::ToString( run );
+    CHECK( xml.starts_with( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" ) );
+    CHECK( xml.find( "<testsuites name=\"Compiler run\" tests=\"4\" " ) !=
+           std::string::npos );
+    CHECK( xml.find( "failures=\"1\" errors=\"1\" skipped=\"1\"" ) !=
+           std::string::npos );
+    CHECK( xml.find( "name=\"Parser &amp; resolver\"" ) != std::string::npos );
+    CHECK( xml.find( "name=\"rejects &lt;input&gt;\"" ) != std::string::npos );
+    CHECK( xml.find(
+               "message=\"assertion: &quot;expected&quot; &amp; actual\"" ) !=
+           std::string::npos );
+    CHECK( xml.find( "expected &lt;node&gt; but got &amp;lt;none&gt;" ) !=
+           std::string::npos );
+    CHECK(
+        xml.find( "<system-out>trace &lt;start&gt;&amp;finish</system-out>" ) !=
+        std::string::npos );
+    CHECK( xml.find( "<error message=\"runtime error\" type=\"TestError\">" ) !=
+           std::string::npos );
+    CHECK( xml.find( "<skipped message=\"not available\"/>" ) !=
+           std::string::npos );
+    CHECK( xml.find( "classname=\"CompilerTests\"" ) != std::string::npos );
+    CHECK( xml.find( "file=\"tests/parser.cpp\"" ) != std::string::npos );
+    CHECK( xml.find( "time=\"0.263500\"" ) != std::string::npos );
+    CHECK( xml.ends_with( "</testsuites>\n" ) );
+}
+
+TEST_CASE( "JUnit XML replaces malformed UTF-8 and forbidden XML controls",
+           "[progmasoft][results][xml]" ) {
+    using namespace Progmasoft::Catch3;
+
+    auto testCase = MakeCase( "valid", TestCaseStatus::kPassed, 0.0 );
+    testCase.Name = std::string( "bad\x01utf8\xF0\x28\x8C\x28", 12 );
+    const TestRunResult run{ std::string( "suite\0name", 10 ),
+                             { { "suite", { testCase } } } };
+
+    const std::string xml = JUnitXmlWriter::ToString( run );
+    CHECK( xml.find( "name=\"bad&#xFFFD;utf8&#xFFFD;(&#xFFFD;(\"" ) !=
+           std::string::npos );
+    CHECK( xml.find( "name=\"suite&#xFFFD;name\"" ) != std::string::npos );
+}
+
+TEST_CASE( "Result summaries reject invalid durations and statuses",
+           "[progmasoft][results]" ) {
+    using namespace Progmasoft::Catch3;
+
+    TestCaseResult invalidDuration =
+        MakeCase( "invalid", TestCaseStatus::kPassed, -1.0 );
+    CHECK_THROWS_AS(
+        SummarizeTestRun( { "run", { { "suite", { invalidDuration } } } } ),
+        std::invalid_argument );
+
+    invalidDuration.DurationSeconds = std::numeric_limits<double>::infinity();
+    CHECK_THROWS_AS(
+        SummarizeTestRun( { "run", { { "suite", { invalidDuration } } } } ),
+        std::invalid_argument );
+
+    invalidDuration.DurationSeconds = 0.0;
+    invalidDuration.Status = static_cast<TestCaseStatus>( 255 );
+    CHECK_THROWS_AS(
+        SummarizeTestRun( { "run", { { "suite", { invalidDuration } } } } ),
+        std::invalid_argument );
+}
+
+TEST_CASE( "JUnit XML writer validates results and output stream state",
+           "[progmasoft][results][xml]" ) {
+    using namespace Progmasoft::Catch3;
+
+    const TestRunResult emptyRun{ "empty", {} };
+    const std::string emptyXml = JUnitXmlWriter::ToString( emptyRun );
+    CHECK( emptyXml.find( "tests=\"0\" failures=\"0\" errors=\"0\" " ) !=
+           std::string::npos );
+
+    auto invalidCase = MakeCase( "bad", TestCaseStatus::kPassed, 0.0 );
+    invalidCase.DurationSeconds = std::numeric_limits<double>::quiet_NaN();
+    const TestRunResult invalidRun{ "invalid",
+                                    { { "suite", { invalidCase } } } };
+    std::ostringstream output;
+    CHECK_THROWS_AS( JUnitXmlWriter::Write( invalidRun, output ),
+                     std::invalid_argument );
+
+    std::ostringstream failedOutput;
+    failedOutput.setstate( std::ios::badbit );
+    CHECK_THROWS_AS( JUnitXmlWriter::Write( emptyRun, failedOutput ),
+                     std::ios_base::failure );
+}
diff --git a/tests/Progmasoft/Snapshot.tests.cpp b/tests/Progmasoft/Snapshot.tests.cpp
new file mode 100644
index 00000000..65ee732e
--- /dev/null
+++ b/tests/Progmasoft/Snapshot.tests.cpp
@@ -0,0 +1,164 @@
+// Copyright (c) 2026 Progmasoft.
+// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+#include <Progmasoft/Catch3.hpp>
+
+#include <atomic>
+#include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <system_error>
+
+namespace {
+
+    class TemporarySnapshotDirectory final {
+    public:
+        TemporarySnapshotDirectory() {
+            static std::atomic<unsigned long long> nextId{ 0 };
+            const auto timestamp =
+                std::chrono::steady_clock::now().time_since_epoch().count();
+            Path = std::filesystem::temp_directory_path() /
+                   ( "progmasoft-catch3-snapshot-" +
+                     std::to_string( timestamp ) + "-" +
+                     std::to_string(
+                         nextId.fetch_add( 1, std::memory_order_relaxed ) ) );
+            std::filesystem::create_directories( Path );
+        }
+
+        ~TemporarySnapshotDirectory() {
+            std::error_code ignored;
+            std::filesystem::remove_all( Path, ignored );
+        }
+
+        std::filesystem::path Path;
+    };
+
+} // namespace
+
+TEST_CASE( "Snapshot comparison does not write during ordinary runs",
+           "[progmasoft][snapshot]" ) {
+    using Progmasoft::Catch3::CompareSnapshot;
+    using Progmasoft::Catch3::SnapshotOptions;
+    using Progmasoft::Catch3::SnapshotStatus;
+
+    const TemporarySnapshotDirectory directory;
+    const auto result =
+        CompareSnapshot( "greeting",
+                         "hello\nworld\n",
+                         SnapshotOptions{ .Directory = directory.Path } );
+
+    CHECK( result.Status == SnapshotStatus::kMismatched );
+    CHECK_FALSE( result.Succeeded() );
+    CHECK( result.DifferenceLine == 1 );
+    CHECK( result.DifferenceColumn == 1 );
+    CHECK_FALSE( std::filesystem::exists( result.File ) );
+}
+
+TEST_CASE( "Snapshots normalize line endings and report precise differences",
+           "[progmasoft][snapshot]" ) {
+    using Progmasoft::Catch3::CompareSnapshot;
+    using Progmasoft::Catch3::SnapshotOptions;
+    using Progmasoft::Catch3::SnapshotStatus;
+    using Progmasoft::Catch3::SnapshotUpdateMode;
+
+    const TemporarySnapshotDirectory directory;
+    const SnapshotOptions createOptions{
+        .Directory = directory.Path,
+        .UpdateMode = SnapshotUpdateMode::kCreateMissing,
+    };
+    const auto created =
+        CompareSnapshot( "multiline", "one\ntwo\n", createOptions );
+    REQUIRE( created.Status == SnapshotStatus::kCreated );
+    CHECK( created.Succeeded() );
+
+    const auto normalized =
+        CompareSnapshot( "multiline", "one\r\ntwo\r\n", createOptions );
+    CHECK( normalized.Status == SnapshotStatus::kMatched );
+    CHECK( normalized.Expected == "one\ntwo\n" );
+
+    const auto mismatch =
+        CompareSnapshot( "multiline",
+                         "one\ntoo\n",
+                         SnapshotOptions{ .Directory = directory.Path } );
+    CHECK( mismatch.Status == SnapshotStatus::kMismatched );
+    CHECK( mismatch.DifferenceLine == 2 );
+    CHECK( mismatch.DifferenceColumn == 2 );
+    CHECK( mismatch.Message.find( "line 2, column 2" ) != std::string::npos );
+}
+
+TEST_CASE( "Snapshot creation and replacement require explicit update modes",
+           "[progmasoft][snapshot]" ) {
+    using Progmasoft::Catch3::CompareSnapshot;
+    using Progmasoft::Catch3::SnapshotOptions;
+    using Progmasoft::Catch3::SnapshotStatus;
+    using Progmasoft::Catch3::SnapshotUpdateMode;
+
+    const TemporarySnapshotDirectory directory;
+    const SnapshotOptions createMissing{
+        .Directory = directory.Path,
+        .UpdateMode = SnapshotUpdateMode::kCreateMissing,
+    };
+    REQUIRE( CompareSnapshot( "answer", "first", createMissing ).Succeeded() );
+
+    const auto protectedMismatch =
+        CompareSnapshot( "answer", "second", createMissing );
+    CHECK( protectedMismatch.Status == SnapshotStatus::kMismatched );
+    CHECK( protectedMismatch.Expected == "first" );
+
+    const SnapshotOptions updateEverything{
+        .Directory = directory.Path,
+        .UpdateMode = SnapshotUpdateMode::kAlways,
+    };
+    const auto updated =
+        CompareSnapshot( "answer", "second", updateEverything );
+    REQUIRE( updated.Status == SnapshotStatus::kUpdated );
+    CHECK( updated.Expected == "first" );
+    CHECK( updated.Actual == "second" );
+    CHECK( CompareSnapshot( "answer", "second", createMissing ).Succeeded() );
+}
+
+TEST_CASE( "Snapshot names and I/O failures are returned as structured errors",
+           "[progmasoft][snapshot]" ) {
+    using Progmasoft::Catch3::CompareSnapshot;
+    using Progmasoft::Catch3::SnapshotOptions;
+    using Progmasoft::Catch3::SnapshotStatus;
+    using Progmasoft::Catch3::SnapshotUpdateMode;
+
+    const TemporarySnapshotDirectory directory;
+    const SnapshotOptions options{ .Directory = directory.Path };
+    CHECK( CompareSnapshot( "../outside", "text", options ).Status ==
+           SnapshotStatus::kInvalidName );
+    CHECK( CompareSnapshot( "nested/name", "text", options ).Status ==
+           SnapshotStatus::kInvalidName );
+    CHECK( CompareSnapshot( "", "text", options ).Status ==
+           SnapshotStatus::kInvalidName );
+    CHECK( CompareSnapshot( "valid", "text", SnapshotOptions{} ).Status ==
+           SnapshotStatus::kInvalidOptions );
+
+    const auto blockingFile = directory.Path / "not-a-directory";
+    {
+        std::ofstream output( blockingFile, std::ios::binary );
+        REQUIRE( output.good() );
+        output << "file";
+    }
+    const auto ioFailure = CompareSnapshot(
+        "snapshot",
+        "text",
+        SnapshotOptions{ .Directory = blockingFile,
+                         .UpdateMode = SnapshotUpdateMode::kCreateMissing } );
+    CHECK( ioFailure.Status == SnapshotStatus::kIoError );
+    CHECK_FALSE( ioFailure.Message.empty() );
+}
+
+TEST_CASE( "Snapshot assertion macro reports successful explicit updates",
+           "[progmasoft][snapshot]" ) {
+    const TemporarySnapshotDirectory directory;
+    CATCH3_CHECK_SNAPSHOT(
+        "macro",
+        "stable value",
+        ( Progmasoft::Catch3::SnapshotOptions{
+            .Directory = directory.Path,
+            .UpdateMode =
+                Progmasoft::Catch3::SnapshotUpdateMode::kCreateMissing,
+        } ) );
+}
diff --git a/tests/meson.build b/tests/meson.build
index 1d20b9ce..d96b6f9f 100644
--- a/tests/meson.build
+++ b/tests/meson.build
@@ -81,7 +81,11 @@ test('SelfTest', self_test)
 # language requirement or the upstream SelfTest compilation mode.
 progmasoft_property_tests = executable(
   'ProgmasoftPropertyTests',
-  'Progmasoft/Property.tests.cpp',
+  [
+    'Progmasoft/Property.tests.cpp',
+    'Progmasoft/Results.tests.cpp',
+    'Progmasoft/Snapshot.tests.cpp',
+  ],
   include_directories: '../src',
   dependencies: [progmasoft_catch3_dep, catch2_with_main_dep],
   override_options: ['cpp_std=c++20'],