catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
Add snapshots and JUnit result serialization
- Commit
d79b294903e4d0f4b24fdf1cc50be4b0fec7f745- Author
- Leitwolf11 ·
- Parents
9ad804cedf4c
Changes
MODULE.bazel | 2 +- docs/progmasoft-catch3.md | 41 +++++- docs/why-catch3.md | 5 +- premake5.lua | 23 ++- src/CMakeLists.txt | 16 +- src/Progmasoft/BUILD.bazel | 12 ++ src/Progmasoft/Catch3.hpp | 3 + src/Progmasoft/Catch3/Assertions.hpp | 10 ++ src/Progmasoft/Catch3/Results.hpp | 57 ++++++++ src/Progmasoft/Catch3/Snapshot.hpp | 55 +++++++ src/Progmasoft/Catch3/XmlWriter.hpp | 20 +++ src/Progmasoft/Results.cpp | 73 ++++++++++ src/Progmasoft/Snapshot.cpp | 265 +++++++++++++++++++++++++++++++++ src/Progmasoft/XmlWriter.cpp | 274 +++++++++++++++++++++++++++++++++++ src/Progmasoft/meson.build | 15 +- tests/CMakeLists.txt | 6 +- tests/Progmasoft/BUILD.bazel | 6 +- tests/Progmasoft/Results.tests.cpp | 165 +++++++++++++++++++++ tests/Progmasoft/Snapshot.tests.cpp | 164 +++++++++++++++++++++ tests/meson.build | 6 +- xmake.lua | 15 +- 21 files changed, 1216 insertions(+), 17 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 8bff2332..fcc80d58 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -1,4 +1,4 @@ -module(name = "catch3", version = "0.1.0") +module(name = "catch3", version = "0.2.0") bazel_dep(name = "bazel_skylib", version = "1.9.0") bazel_dep(name = "rules_cc", version = "0.2.17") diff --git a/docs/progmasoft-catch3.md b/docs/progmasoft-catch3.md index 96989479..43a73f03 100644 --- a/docs/progmasoft-catch3.md +++ b/docs/progmasoft-catch3.md @@ -58,9 +58,48 @@ state-machine commands, parallel trials, and adaptive shrinking are deliberately not implied by this first module; they need their own deterministic contracts and tests before being added. +## Snapshots + +`<Progmasoft/Catch3/Snapshot.hpp>` provides file-backed text snapshots through +`CompareSnapshot(name, actual, options)`. The directory is supplied explicitly; +names are restricted to portable ASCII filename characters and cannot contain +path separators. CRLF and CR are normalized to LF by default, with no trimming +or other content rewriting. + +Ordinary comparisons never write files. `SnapshotUpdateMode::kCreateMissing` +creates only absent snapshots, while `kAlways` explicitly permits replacement. +The result reports a match, create/update, mismatch with a one-based first +difference location, invalid input, or I/O failure. `CATCH3_CHECK_SNAPSHOT` +combines this result with Catch2's normal assertion reporting. Keep snapshots +under a directory owned by the test suite and enable replacement only for an +intentional update run. + +```cpp +const Progmasoft::Catch3::SnapshotOptions options{ + .Directory = "tests/snapshots", +}; +CATCH3_CHECK_SNAPSHOT("rendered-report", RenderReport(), options); +``` + +## Structured results and XML + +`<Progmasoft/Catch3/Results.hpp>` defines a runner-neutral hierarchy of +`TestRunResult`, `TestSuiteResult`, and `TestCaseResult`. `SummarizeTestRun` +validates finite, non-negative durations and computes pass/failure/error/skip +counts. `<Progmasoft/Catch3/XmlWriter.hpp>` serializes that model as deterministic +JUnit-style XML through `JUnitXmlWriter::Write` or `ToString`. It escapes XML +markup, replaces malformed UTF-8 and XML-forbidden control characters, and +omits timestamps so identical results produce identical output. Catch2's own +`--reporter xml` and `--reporter junit` remain the direct reporters for results +produced by its runner; this writer is for Progmasoft's explicit result model. + +The snapshot and result/XML APIs have compiled C++ implementations in the +`Progmasoft::Catch3` library target. Generator templates and convenience macros +remain in headers where their types or call-site expansion require it. + ## Build and integration -The CMake build exposes `Progmasoft::Catch3`, a C++20 interface target linked to +The CMake build exposes `Progmasoft::Catch3`, a compiled C++20 library linked to `Catch2::Catch2`. The existing Catch2 target retains its C++14 baseline. After installation, consumers can use: diff --git a/docs/why-catch3.md b/docs/why-catch3.md index 0b62402f..30dd5159 100644 --- a/docs/why-catch3.md +++ b/docs/why-catch3.md @@ -40,7 +40,10 @@ future Catch2 upstream change will be adopted unchanged or that all downstream projects have already been validated. The split also applies to implementation and packaging: Catch2 remains the -engine dependency, while Progmasoft's header-only API is a separate target. +engine dependency, while Progmasoft's compiled C++20 library is a separate +target. Template-based generators and call-site assertion macros remain in +headers; snapshot I/O, result aggregation, and XML serialization are compiled +implementations. Keep tests for the two areas independently identifiable so additions do not silently alter the upstream compatibility baseline. diff --git a/premake5.lua b/premake5.lua index a03b8734..9c2115d2 100644 --- a/premake5.lua +++ b/premake5.lua @@ -54,6 +54,25 @@ project "Catch2WithMain" includedirs { "src", generated_include } links { "Catch2" } +project "ProgmasoftCatch3" + kind "StaticLib" + language "C++" + cppdialect "C++20" + targetdir "build/premake/bin/%{cfg.buildcfg}" + objdir "build/premake/obj/ProgmasoftCatch3/%{cfg.buildcfg}" + files { + "src/Progmasoft/Results.cpp", + "src/Progmasoft/Results.hpp", + "src/Progmasoft/Snapshot.cpp", + "src/Progmasoft/Snapshot.hpp", + "src/Progmasoft/XmlWriter.cpp", + "src/Progmasoft/XmlWriter.hpp", + "src/Progmasoft/Catch3/**.hpp", + "src/Progmasoft/Catch3.hpp", + } + includedirs { "src", generated_include } + links { "Catch2" } + project "ProgmasoftPropertyTests" kind "ConsoleApp" language "C++" @@ -62,7 +81,9 @@ project "ProgmasoftPropertyTests" objdir "build/premake/obj/ProgmasoftPropertyTests/%{cfg.buildcfg}" files { "tests/Progmasoft/Property.tests.cpp", + "tests/Progmasoft/Results.tests.cpp", + "tests/Progmasoft/Snapshot.tests.cpp", "src/Progmasoft/**.hpp", } includedirs { "src", generated_include } - links { "Catch2WithMain", "Catch2" } + links { "ProgmasoftCatch3", "Catch2WithMain", "Catch2" } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cb8c339d..5eec18e9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -360,18 +360,22 @@ if(CATCH_ENABLE_REPRODUCIBLE_BUILD) endif() add_library(Catch2::Catch2 ALIAS Catch2) -# Progmasoft's C++20 additions are a separate target so the Catch2 -# compatibility engine can keep its existing, broader C++14 baseline. -add_library(ProgmasoftCatch3 INTERFACE) +# Progmasoft's compiled C++20 utilities are separate from Catch2's +# compatibility engine, which retains its existing C++14 baseline. +add_library(ProgmasoftCatch3 STATIC + "${CATCH_DIR}/src/Progmasoft/Results.cpp" + "${CATCH_DIR}/src/Progmasoft/Snapshot.cpp" + "${CATCH_DIR}/src/Progmasoft/XmlWriter.cpp" +) add_library(Progmasoft::Catch3 ALIAS ProgmasoftCatch3) set_target_properties(ProgmasoftCatch3 PROPERTIES EXPORT_NAME Catch3) -target_link_libraries(ProgmasoftCatch3 INTERFACE Catch2::Catch2) +target_link_libraries(ProgmasoftCatch3 PUBLIC Catch2::Catch2) target_include_directories(ProgmasoftCatch3 - INTERFACE + PUBLIC $<BUILD_INTERFACE:${CATCH_DIR}/src> $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> ) -target_compile_features(ProgmasoftCatch3 INTERFACE cxx_std_20) +target_compile_features(ProgmasoftCatch3 PUBLIC cxx_std_20) if(ANDROID) target_link_libraries(Catch2 PRIVATE log) diff --git a/src/Progmasoft/BUILD.bazel b/src/Progmasoft/BUILD.bazel index 9927aafa..642bc158 100644 --- a/src/Progmasoft/BUILD.bazel +++ b/src/Progmasoft/BUILD.bazel @@ -6,6 +6,11 @@ package(default_visibility = ["//visibility:public"]) cc_library( name = "catch3", + srcs = [ + "Results.cpp", + "Snapshot.cpp", + "XmlWriter.cpp", + ], hdrs = [ "Catch3.hpp", "Catch3/Assertions.hpp", @@ -14,7 +19,14 @@ cc_library( "Catch3/Generators/Integer.hpp", "Catch3/Property.hpp", "Catch3/Random.hpp", + "Catch3/Results.hpp", + "Catch3/Snapshot.hpp", + "Catch3/XmlWriter.hpp", ], + copts = select({ + "@platforms//os:windows": ["/std:c++20"], + "//conditions:default": ["-std=c++20"], + }), strip_include_prefix = "/src", deps = ["//:catch2"], ) diff --git a/src/Progmasoft/Catch3.hpp b/src/Progmasoft/Catch3.hpp index 1ae73e53..d2d34e74 100644 --- a/src/Progmasoft/Catch3.hpp +++ b/src/Progmasoft/Catch3.hpp @@ -11,3 +11,6 @@ #include <Progmasoft/Catch3/Generators/Integer.hpp> #include <Progmasoft/Catch3/Property.hpp> #include <Progmasoft/Catch3/Random.hpp> +#include <Progmasoft/Catch3/Results.hpp> +#include <Progmasoft/Catch3/Snapshot.hpp> +#include <Progmasoft/Catch3/XmlWriter.hpp> diff --git a/src/Progmasoft/Catch3/Assertions.hpp b/src/Progmasoft/Catch3/Assertions.hpp index 19ac13ef..668c086d 100644 --- a/src/Progmasoft/Catch3/Assertions.hpp +++ b/src/Progmasoft/Catch3/Assertions.hpp @@ -6,6 +6,7 @@ #include <catch2/catch_test_macros.hpp> #include <Progmasoft/Catch3/Property.hpp> +#include <Progmasoft/Catch3/Snapshot.hpp> /// Runs a property and reports its replay seed and minimized counterexample. #define CATCH3_CHECK_PROPERTY( ... ) \ @@ -15,3 +16,12 @@ INFO( catch3PropertyResult.Describe() ); \ CHECK( catch3PropertyResult.Succeeded() ); \ } while ( false ) + +/// Compares a value with a named snapshot and reports the first difference. +#define CATCH3_CHECK_SNAPSHOT( name, actual, options ) \ + do { \ + const auto catch3SnapshotResult = \ + ::Progmasoft::Catch3::CompareSnapshot( name, actual, options ); \ + INFO( catch3SnapshotResult.Message ); \ + CHECK( catch3SnapshotResult.Succeeded() ); \ + } while ( false ) diff --git a/src/Progmasoft/Catch3/Results.hpp b/src/Progmasoft/Catch3/Results.hpp new file mode 100644 index 00000000..b0b69b55 --- /dev/null +++ b/src/Progmasoft/Catch3/Results.hpp @@ -0,0 +1,57 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#pragma once + +#include <cstddef> +#include <string> +#include <vector> + +namespace Progmasoft::Catch3 { + + enum class TestCaseStatus : unsigned char { + kPassed, + kFailed, + kErrored, + kSkipped, + }; + + /// Framework-neutral result for one completed test case. + struct TestCaseResult final { + std::string Name; + std::string ClassName; + std::string SourceFile; + TestCaseStatus Status = TestCaseStatus::kPassed; + double DurationSeconds = 0.0; + std::string Message; + std::string Details; + std::string StandardOutput; + std::string StandardError; + }; + + struct TestSuiteResult final { + std::string Name; + std::vector<TestCaseResult> TestCases; + }; + + struct TestRunResult final { + std::string Name; + std::vector<TestSuiteResult> TestSuites; + }; + + struct TestRunSummary final { + std::size_t Tests = 0; + std::size_t Failures = 0; + std::size_t Errors = 0; + std::size_t Skipped = 0; + double DurationSeconds = 0.0; + + [[nodiscard]] std::size_t Passed() const noexcept; + }; + + /// Validates durations and statuses while aggregating nested test results. + [[nodiscard]] TestRunSummary + SummarizeTestSuite( const TestSuiteResult& result ); + [[nodiscard]] TestRunSummary + SummarizeTestRun( const TestRunResult& result ); + +} // namespace Progmasoft::Catch3 diff --git a/src/Progmasoft/Catch3/Snapshot.hpp b/src/Progmasoft/Catch3/Snapshot.hpp new file mode 100644 index 00000000..09863c1c --- /dev/null +++ b/src/Progmasoft/Catch3/Snapshot.hpp @@ -0,0 +1,55 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#pragma once + +#include <cstddef> +#include <filesystem> +#include <string> +#include <string_view> + +namespace Progmasoft::Catch3 { + + /// Updates are deliberately explicit so normal test runs never rewrite + /// files. + enum class SnapshotUpdateMode : unsigned char { + kNever, + kCreateMissing, + kAlways, + }; + + enum class SnapshotStatus : unsigned char { + kMatched, + kCreated, + kUpdated, + kMismatched, + kInvalidName, + kInvalidOptions, + kIoError, + }; + + struct SnapshotOptions final { + std::filesystem::path Directory; + SnapshotUpdateMode UpdateMode = SnapshotUpdateMode::kNever; + bool NormalizeLineEndings = true; + }; + + /// Structured comparison result; DifferenceLine/Column are one-based. + struct SnapshotResult final { + SnapshotStatus Status = SnapshotStatus::kInvalidOptions; + std::filesystem::path File; + std::string Expected; + std::string Actual; + std::string Message; + std::size_t DifferenceLine = 0; + std::size_t DifferenceColumn = 0; + + [[nodiscard]] bool Succeeded() const noexcept; + }; + + /// Compare against `<Directory>/<name>.snap`; names cannot contain paths. + [[nodiscard]] SnapshotResult + CompareSnapshot( std::string_view name, + std::string_view actual, + const SnapshotOptions& options ); + +} // namespace Progmasoft::Catch3 diff --git a/src/Progmasoft/Catch3/XmlWriter.hpp b/src/Progmasoft/Catch3/XmlWriter.hpp new file mode 100644 index 00000000..3216b5ae --- /dev/null +++ b/src/Progmasoft/Catch3/XmlWriter.hpp @@ -0,0 +1,20 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#pragma once + +#include <iosfwd> +#include <string> + +namespace Progmasoft::Catch3 { + + struct TestRunResult; + + /// Emits deterministic JUnit-style XML without timestamps or global state. + class JUnitXmlWriter final { + public: + static void Write( const TestRunResult& result, std::ostream& output ); + [[nodiscard]] static std::string + ToString( const TestRunResult& result ); + }; + +} // namespace Progmasoft::Catch3 diff --git a/src/Progmasoft/Results.cpp b/src/Progmasoft/Results.cpp new file mode 100644 index 00000000..69a819a7 --- /dev/null +++ b/src/Progmasoft/Results.cpp @@ -0,0 +1,73 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#include <Progmasoft/Catch3/Results.hpp> + +#include <cmath> +#include <limits> +#include <stdexcept> + +namespace Progmasoft::Catch3 { + + namespace { + + void AccumulateCases( TestRunSummary& summary, + const std::vector<TestCaseResult>& testCases ) { + for ( const TestCaseResult& testCase : testCases ) { + if ( !std::isfinite( testCase.DurationSeconds ) || + testCase.DurationSeconds < 0.0 ) { + throw std::invalid_argument( + "test duration must be finite and non-negative" ); + } + if ( summary.Tests == + std::numeric_limits<std::size_t>::max() ) { + throw std::overflow_error( "test count overflow" ); + } + + ++summary.Tests; + summary.DurationSeconds += testCase.DurationSeconds; + if ( !std::isfinite( summary.DurationSeconds ) ) { + throw std::overflow_error( "test duration total overflow" ); + } + + switch ( testCase.Status ) { + case TestCaseStatus::kPassed: + break; + case TestCaseStatus::kFailed: + ++summary.Failures; + break; + case TestCaseStatus::kErrored: + ++summary.Errors; + break; + case TestCaseStatus::kSkipped: + ++summary.Skipped; + break; + default: + throw std::invalid_argument( "unknown test case status" ); + } + } + } + + } // namespace + + std::size_t TestRunSummary::Passed() const noexcept { + if ( Failures > Tests || Errors > Tests - Failures ) { return 0; } + const std::size_t failuresAndErrors = Failures + Errors; + if ( Skipped > Tests - failuresAndErrors ) { return 0; } + return Tests - failuresAndErrors - Skipped; + } + + TestRunSummary SummarizeTestSuite( const TestSuiteResult& result ) { + TestRunSummary summary; + AccumulateCases( summary, result.TestCases ); + return summary; + } + + TestRunSummary SummarizeTestRun( const TestRunResult& result ) { + TestRunSummary summary; + for ( const TestSuiteResult& suite : result.TestSuites ) { + AccumulateCases( summary, suite.TestCases ); + } + return summary; + } + +} // namespace Progmasoft::Catch3 diff --git a/src/Progmasoft/Snapshot.cpp b/src/Progmasoft/Snapshot.cpp new file mode 100644 index 00000000..7278eee3 --- /dev/null +++ b/src/Progmasoft/Snapshot.cpp @@ -0,0 +1,265 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#include <Progmasoft/Catch3/Snapshot.hpp> + +#include <atomic> +#include <chrono> +#include <cstdint> +#include <fstream> +#include <iterator> +#include <system_error> + +#if defined( _WIN32 ) +# if !defined( NOMINMAX ) +# define NOMINMAX +# endif +# include <Windows.h> +#endif +#include <utility> + +namespace Progmasoft::Catch3 { + + namespace { + + bool IsSafeSnapshotName( std::string_view name ) noexcept { + if ( name.empty() || name == "." || name == ".." ) { return false; } + for ( const unsigned char character : name ) { + const bool isLetter = + ( character >= 'a' && character <= 'z' ) || + ( character >= 'A' && character <= 'Z' ); + const bool isDigit = character >= '0' && character <= '9'; + if ( !isLetter && !isDigit && character != '_' && + character != '-' && character != '.' ) { + return false; + } + } + return true; + } + + std::string NormalizeSnapshotText( std::string_view text, + bool normalizeLineEndings ) { + if ( !normalizeLineEndings ) { return std::string( text ); } + + std::string normalized; + normalized.reserve( text.size() ); + for ( std::size_t index = 0; index < text.size(); ++index ) { + if ( text[index] == '\r' ) { + if ( index + 1 < text.size() && text[index + 1] == '\n' ) { + ++index; + } + normalized.push_back( '\n' ); + } else { + normalized.push_back( text[index] ); + } + } + return normalized; + } + + void LocateDifference( SnapshotResult& result ) noexcept { + const std::size_t commonLength = + result.Expected.size() < result.Actual.size() + ? result.Expected.size() + : result.Actual.size(); + std::size_t index = 0; + while ( index < commonLength && + result.Expected[index] == result.Actual[index] ) { + ++index; + } + + result.DifferenceLine = 1; + result.DifferenceColumn = 1; + for ( std::size_t offset = 0; offset < index; ++offset ) { + if ( result.Actual[offset] == '\n' ) { + ++result.DifferenceLine; + result.DifferenceColumn = 1; + } else { + ++result.DifferenceColumn; + } + } + } + + bool WriteAtomically( const std::filesystem::path& destination, + std::string_view contents, + bool replaceExisting, + std::error_code& error ) { + // Write beside the destination so replacement stays on one volume + // and readers never observe a partially written snapshot. + static std::atomic<std::uint64_t> nextTemporaryFile{ 0 }; + const auto clockValue = + std::chrono::steady_clock::now().time_since_epoch().count(); + const auto sequence = + nextTemporaryFile.fetch_add( 1, std::memory_order_relaxed ); + std::filesystem::path temporary = destination; + temporary += ".tmp." + std::to_string( clockValue ) + "." + + std::to_string( sequence ); + + { + std::ofstream output( temporary, + std::ios::binary | std::ios::trunc ); + if ( !output ) { + error = std::make_error_code( std::errc::io_error ); + return false; + } + output.write( contents.data(), + static_cast<std::streamsize>( contents.size() ) ); + output.flush(); + if ( !output ) { + error = std::make_error_code( std::errc::io_error ); + output.close(); + std::error_code ignored; + std::filesystem::remove( temporary, ignored ); + return false; + } + } + + if ( replaceExisting ) { +#if defined( _WIN32 ) + if ( MoveFileExW( temporary.c_str(), + destination.c_str(), + MOVEFILE_REPLACE_EXISTING | + MOVEFILE_WRITE_THROUGH ) == 0 ) { + error = std::error_code( static_cast<int>( GetLastError() ), + std::system_category() ); + } +#else + std::filesystem::rename( temporary, destination, error ); +#endif + } else { + // A hard-link create is atomic and refuses a concurrent file. + std::filesystem::create_hard_link( + temporary, destination, error ); + std::error_code ignored; + std::filesystem::remove( temporary, ignored ); + } + if ( error ) { + std::error_code ignored; + std::filesystem::remove( temporary, ignored ); + return false; + } + return true; + } + + } // namespace + + bool SnapshotResult::Succeeded() const noexcept { + return Status == SnapshotStatus::kMatched || + Status == SnapshotStatus::kCreated || + Status == SnapshotStatus::kUpdated; + } + + SnapshotResult CompareSnapshot( std::string_view name, + std::string_view actual, + const SnapshotOptions& options ) { + SnapshotResult result; + if ( !IsSafeSnapshotName( name ) ) { + result.Status = SnapshotStatus::kInvalidName; + result.Message = + "snapshot name must be a non-empty ASCII filename component"; + return result; + } + if ( options.Directory.empty() || + ( options.UpdateMode != SnapshotUpdateMode::kNever && + options.UpdateMode != SnapshotUpdateMode::kCreateMissing && + options.UpdateMode != SnapshotUpdateMode::kAlways ) ) { + result.Status = SnapshotStatus::kInvalidOptions; + result.Message = "snapshot directory or update mode is invalid"; + return result; + } + + result.File = options.Directory / ( std::string( name ) + ".snap" ); + result.Actual = + NormalizeSnapshotText( actual, options.NormalizeLineEndings ); + + std::error_code error; + const std::filesystem::file_status fileStatus = + std::filesystem::symlink_status( result.File, error ); + if ( error && error != std::errc::no_such_file_or_directory ) { + result.Status = SnapshotStatus::kIoError; + result.Message = + "could not inspect snapshot file: " + error.message(); + return result; + } + + const bool exists = !error && fileStatus.type() != + std::filesystem::file_type::not_found; + if ( exists && std::filesystem::is_symlink( fileStatus ) ) { + result.Status = SnapshotStatus::kIoError; + result.Message = "snapshot files must not be symbolic links"; + return result; + } + if ( exists && !std::filesystem::is_regular_file( fileStatus ) ) { + result.Status = SnapshotStatus::kIoError; + result.Message = "snapshot path is not a regular file"; + return result; + } + + if ( !exists ) { + if ( options.UpdateMode == SnapshotUpdateMode::kNever ) { + result.Status = SnapshotStatus::kMismatched; + result.Message = "snapshot file does not exist"; + LocateDifference( result ); + return result; + } + + std::filesystem::create_directories( options.Directory, error ); + if ( error || !WriteAtomically( + result.File, result.Actual, false, error ) ) { + result.Status = SnapshotStatus::kIoError; + result.Message = + "could not create snapshot file: " + error.message(); + return result; + } + result.Status = SnapshotStatus::kCreated; + result.Message = "snapshot created"; + return result; + } + + std::ifstream input( result.File, std::ios::binary ); + if ( !input ) { + result.Status = SnapshotStatus::kIoError; + result.Message = "could not open snapshot file for reading"; + return result; + } + const std::string fileContents{ std::istreambuf_iterator<char>( input ), + std::istreambuf_iterator<char>() }; + if ( input.bad() ) { + result.Status = SnapshotStatus::kIoError; + result.Message = "could not read snapshot file"; + return result; + } + input.close(); + if ( input.fail() ) { + result.Status = SnapshotStatus::kIoError; + result.Message = "could not close snapshot file after reading"; + return result; + } + result.Expected = + NormalizeSnapshotText( fileContents, options.NormalizeLineEndings ); + + if ( result.Expected == result.Actual ) { + result.Status = SnapshotStatus::kMatched; + result.Message = "snapshot matched"; + return result; + } + + LocateDifference( result ); + if ( options.UpdateMode == SnapshotUpdateMode::kAlways ) { + if ( !WriteAtomically( result.File, result.Actual, true, error ) ) { + result.Status = SnapshotStatus::kIoError; + result.Message = + "could not update snapshot file: " + error.message(); + return result; + } + result.Status = SnapshotStatus::kUpdated; + result.Message = "snapshot updated"; + return result; + } + + result.Status = SnapshotStatus::kMismatched; + result.Message = "snapshot differs at line " + + std::to_string( result.DifferenceLine ) + ", column " + + std::to_string( result.DifferenceColumn ); + return result; + } + +} // namespace Progmasoft::Catch3 diff --git a/src/Progmasoft/XmlWriter.cpp b/src/Progmasoft/XmlWriter.cpp new file mode 100644 index 00000000..b3a5f9b1 --- /dev/null +++ b/src/Progmasoft/XmlWriter.cpp @@ -0,0 +1,274 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#include <Progmasoft/Catch3/Results.hpp> +#include <Progmasoft/Catch3/XmlWriter.hpp> + +#include <cmath> +#include <iomanip> +#include <locale> +#include <ostream> +#include <sstream> +#include <stdexcept> +#include <string_view> + +namespace Progmasoft::Catch3 { + + namespace { + + struct DecodedCharacter final { + char32_t Value = 0; + std::size_t Length = 0; + }; + + DecodedCharacter DecodeUtf8( std::string_view text, + std::size_t offset ) noexcept { + const auto first = static_cast<unsigned char>( text[offset] ); + if ( first < 0x80 ) { return { first, 1 }; } + + std::size_t length = 0; + char32_t value = 0; + char32_t minimum = 0; + if ( first >= 0xC2 && first <= 0xDF ) { + length = 2; + value = first & 0x1F; + minimum = 0x80; + } else if ( first >= 0xE0 && first <= 0xEF ) { + length = 3; + value = first & 0x0F; + minimum = 0x800; + } else if ( first >= 0xF0 && first <= 0xF4 ) { + length = 4; + value = first & 0x07; + minimum = 0x10000; + } else { + return { 0xFFFD, 0 }; + } + + if ( offset + length > text.size() ) { return { 0xFFFD, 0 }; } + for ( std::size_t index = 1; index < length; ++index ) { + const auto continuation = + static_cast<unsigned char>( text[offset + index] ); + if ( ( continuation & 0xC0 ) != 0x80 ) { return { 0xFFFD, 0 }; } + value = static_cast<char32_t>( ( value << 6 ) | + ( continuation & 0x3F ) ); + } + if ( value < minimum || value > 0x10FFFF || + ( value >= 0xD800 && value <= 0xDFFF ) ) { + return { 0xFFFD, 0 }; + } + return { value, length }; + } + + bool IsXmlCharacter( char32_t value ) noexcept { + return value == 0x09 || value == 0x0A || value == 0x0D || + ( value >= 0x20 && value <= 0xD7FF ) || + ( value >= 0xE000 && value <= 0xFFFD ) || + ( value >= 0x10000 && value <= 0x10FFFF ); + } + + void WriteEscaped( std::string_view value, + std::ostream& output, + bool attribute ) { + for ( std::size_t index = 0; index < value.size(); ) { + const DecodedCharacter character = DecodeUtf8( value, index ); + if ( character.Length == 0 || + !IsXmlCharacter( character.Value ) ) { + // Preserve parseability without silently dropping input. + output << "�"; + ++index; + continue; + } + + if ( character.Value < 0x80 ) { + switch ( static_cast<char>( character.Value ) ) { + case '&': + output << "&"; + break; + case '<': + output << "<"; + break; + case '>': + output << ">"; + break; + case '"': + if ( attribute ) { + output << """; + } else { + output.put( '"' ); + } + break; + case '\'': + if ( attribute ) { + output << "'"; + } else { + output.put( '\'' ); + } + break; + default: + output.put( static_cast<char>( character.Value ) ); + } + } else { + output.write( + value.data() + index, + static_cast<std::streamsize>( character.Length ) ); + } + index += character.Length; + } + } + + void WriteAttribute( std::ostream& output, + std::string_view name, + std::string_view value ) { + output.put( ' ' ); + output.write( name.data(), + static_cast<std::streamsize>( name.size() ) ); + output << "=\""; + WriteEscaped( value, output, true ); + output.put( '"' ); + } + + std::string FormatSeconds( double seconds ) { + std::ostringstream formatted; + formatted.imbue( std::locale::classic() ); + formatted << std::fixed << std::setprecision( 6 ) << seconds; + return formatted.str(); + } + + void WriteIndent( std::ostream& output, unsigned int level ) { + for ( unsigned int index = 0; index < level; ++index ) { + output << " "; + } + } + + void WriteTextElement( std::ostream& output, + unsigned int indent, + std::string_view name, + std::string_view value ) { + WriteIndent( output, indent ); + output.put( '<' ); + output.write( name.data(), + static_cast<std::streamsize>( name.size() ) ); + output.put( '>' ); + WriteEscaped( value, output, false ); + output << "</"; + output.write( name.data(), + static_cast<std::streamsize>( name.size() ) ); + output << ">\n"; + } + + TestRunSummary SummarizeSuite( const TestSuiteResult& suite ) { + return SummarizeTestSuite( suite ); + } + + void WriteTestCase( const TestCaseResult& testCase, + std::string_view suiteName, + std::ostream& output ) { + const bool hasBody = testCase.Status != TestCaseStatus::kPassed || + !testCase.StandardOutput.empty() || + !testCase.StandardError.empty(); + WriteIndent( output, 2 ); + output << "<testcase"; + WriteAttribute( output, + "classname", + testCase.ClassName.empty() ? suiteName + : testCase.ClassName ); + WriteAttribute( output, "name", testCase.Name ); + WriteAttribute( + output, "time", FormatSeconds( testCase.DurationSeconds ) ); + if ( !testCase.SourceFile.empty() ) { + WriteAttribute( output, "file", testCase.SourceFile ); + } + if ( !hasBody ) { + output << "/>\n"; + return; + } + + output << ">\n"; + if ( testCase.Status == TestCaseStatus::kFailed || + testCase.Status == TestCaseStatus::kErrored ) { + const bool isError = + testCase.Status == TestCaseStatus::kErrored; + WriteIndent( output, 3 ); + output << ( isError ? "<error" : "<failure" ); + WriteAttribute( output, "message", testCase.Message ); + WriteAttribute( output, + "type", + isError ? "TestError" : "AssertionFailure" ); + output.put( '>' ); + WriteEscaped( testCase.Details.empty() ? testCase.Message + : testCase.Details, + output, + false ); + output << ( isError ? "</error>\n" : "</failure>\n" ); + } else if ( testCase.Status == TestCaseStatus::kSkipped ) { + WriteIndent( output, 3 ); + output << "<skipped"; + WriteAttribute( output, "message", testCase.Message ); + output << "/>\n"; + } + if ( !testCase.StandardOutput.empty() ) { + WriteTextElement( + output, 3, "system-out", testCase.StandardOutput ); + } + if ( !testCase.StandardError.empty() ) { + WriteTextElement( + output, 3, "system-err", testCase.StandardError ); + } + WriteIndent( output, 2 ); + output << "</testcase>\n"; + } + + } // namespace + + void JUnitXmlWriter::Write( const TestRunResult& result, + std::ostream& output ) { + const TestRunSummary runSummary = SummarizeTestRun( result ); + output << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<testsuites"; + WriteAttribute( output, "name", result.Name ); + WriteAttribute( output, "tests", std::to_string( runSummary.Tests ) ); + WriteAttribute( + output, "failures", std::to_string( runSummary.Failures ) ); + WriteAttribute( output, "errors", std::to_string( runSummary.Errors ) ); + WriteAttribute( + output, "skipped", std::to_string( runSummary.Skipped ) ); + WriteAttribute( + output, "time", FormatSeconds( runSummary.DurationSeconds ) ); + output << ">\n"; + + for ( const TestSuiteResult& suite : result.TestSuites ) { + const TestRunSummary suiteSummary = SummarizeSuite( suite ); + WriteIndent( output, 1 ); + output << "<testsuite"; + WriteAttribute( output, "name", suite.Name ); + WriteAttribute( + output, "tests", std::to_string( suiteSummary.Tests ) ); + WriteAttribute( + output, "failures", std::to_string( suiteSummary.Failures ) ); + WriteAttribute( + output, "errors", std::to_string( suiteSummary.Errors ) ); + WriteAttribute( + output, "skipped", std::to_string( suiteSummary.Skipped ) ); + WriteAttribute( + output, "time", FormatSeconds( suiteSummary.DurationSeconds ) ); + output << ">\n"; + + for ( const TestCaseResult& testCase : suite.TestCases ) { + WriteTestCase( testCase, suite.Name, output ); + } + WriteIndent( output, 1 ); + output << "</testsuite>\n"; + } + output << "</testsuites>\n"; + + if ( !output ) { + throw std::ios_base::failure( "could not write JUnit XML results" ); + } + } + + std::string JUnitXmlWriter::ToString( const TestRunResult& result ) { + std::ostringstream output; + Write( result, output ); + return output.str(); + } + +} // namespace Progmasoft::Catch3 diff --git a/src/Progmasoft/meson.build b/src/Progmasoft/meson.build index 109edaa4..3e1ab458 100644 --- a/src/Progmasoft/meson.build +++ b/src/Progmasoft/meson.build @@ -1,8 +1,18 @@ # Copyright (c) 2026 Progmasoft. # SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 -# Expose the header-only C++20 layer without raising Catch2's C++14 baseline. +# Compile the non-template C++20 utilities separately while retaining Catch2's +# own C++14 baseline for existing users. +progmasoft_catch3 = static_library( + 'ProgmasoftCatch3', + ['Results.cpp', 'Snapshot.cpp', 'XmlWriter.cpp'], + include_directories: '..', + dependencies: catch2_dep, + override_options: ['cpp_std=c++20'], + install: get_option('install'), +) progmasoft_catch3_dep = declare_dependency( + link_with: progmasoft_catch3, dependencies: catch2_dep, include_directories: include_directories('..'), ) @@ -13,6 +23,9 @@ if get_option('install') install_headers('Catch3/Generator.hpp', subdir: 'Progmasoft/Catch3') install_headers('Catch3/Property.hpp', subdir: 'Progmasoft/Catch3') install_headers('Catch3/Random.hpp', subdir: 'Progmasoft/Catch3') + install_headers('Catch3/Results.hpp', subdir: 'Progmasoft/Catch3') + install_headers('Catch3/Snapshot.hpp', subdir: 'Progmasoft/Catch3') + install_headers('Catch3/XmlWriter.hpp', subdir: 'Progmasoft/Catch3') install_headers('Catch3/Generators/Boolean.hpp', subdir: 'Progmasoft/Catch3/Generators') install_headers('Catch3/Generators/Integer.hpp', subdir: 'Progmasoft/Catch3/Generators') endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d307ecd8..78e56f1d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -157,7 +157,11 @@ target_link_libraries(SelfTest PRIVATE Catch2WithMain) # Exercise the C++20 Progmasoft layer independently so the upstream Catch2 # SelfTest continues to compile at its supported baseline language level. -add_executable(ProgmasoftSelfTest Progmasoft/Property.tests.cpp) +add_executable(ProgmasoftSelfTest + Progmasoft/Property.tests.cpp + Progmasoft/Results.tests.cpp + Progmasoft/Snapshot.tests.cpp +) target_link_libraries(ProgmasoftSelfTest PRIVATE Progmasoft::Catch3 Catch2WithMain) target_compile_features(ProgmasoftSelfTest PRIVATE cxx_std_20) add_test(NAME Progmasoft::PropertyTests COMMAND $<TARGET_FILE:ProgmasoftSelfTest>) diff --git a/tests/Progmasoft/BUILD.bazel b/tests/Progmasoft/BUILD.bazel index c0cb14ac..18912383 100644 --- a/tests/Progmasoft/BUILD.bazel +++ b/tests/Progmasoft/BUILD.bazel @@ -6,7 +6,11 @@ package(default_visibility = ["//visibility:public"]) cc_test( name = "property_tests", - srcs = ["Property.tests.cpp"], + srcs = [ + "Property.tests.cpp", + "Results.tests.cpp", + "Snapshot.tests.cpp", + ], copts = select({ "@platforms//os:windows": ["/std:c++20"], "//conditions:default": ["-std=c++20"], diff --git a/tests/Progmasoft/Results.tests.cpp b/tests/Progmasoft/Results.tests.cpp new file mode 100644 index 00000000..91b06856 --- /dev/null +++ b/tests/Progmasoft/Results.tests.cpp @@ -0,0 +1,165 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#include <Progmasoft/Catch3.hpp> + +#include <cmath> +#include <ios> +#include <limits> +#include <sstream> +#include <stdexcept> +#include <string> +#include <utility> + +namespace { + + Progmasoft::Catch3::TestCaseResult + MakeCase( std::string name, + Progmasoft::Catch3::TestCaseStatus status, + double duration ) { + Progmasoft::Catch3::TestCaseResult testCase; + testCase.Name = std::move( name ); + testCase.Status = status; + testCase.DurationSeconds = duration; + return testCase; + } + +} // namespace + +TEST_CASE( "Test result summaries count outcomes and duration", + "[progmasoft][results]" ) { + using namespace Progmasoft::Catch3; + + TestSuiteResult suite; + suite.Name = "Parser"; + suite.TestCases = { + MakeCase( "accepts input", TestCaseStatus::kPassed, 0.125 ), + MakeCase( "rejects malformed input", TestCaseStatus::kFailed, 0.250 ), + MakeCase( "handles I/O", TestCaseStatus::kErrored, 0.500 ), + MakeCase( "optional backend", TestCaseStatus::kSkipped, 0.0 ), + }; + const TestRunResult run{ "Compiler", { suite } }; + + const TestRunSummary summary = SummarizeTestRun( run ); + CHECK( summary.Tests == 4 ); + CHECK( summary.Passed() == 1 ); + CHECK( summary.Failures == 1 ); + CHECK( summary.Errors == 1 ); + CHECK( summary.Skipped == 1 ); + CHECK( summary.DurationSeconds == 0.875 ); +} + +TEST_CASE( "JUnit XML writer serializes counts, details, and escaped text", + "[progmasoft][results][xml]" ) { + using namespace Progmasoft::Catch3; + + TestSuiteResult suite; + suite.Name = "Parser & resolver"; + + auto failed = + MakeCase( "rejects <input>", TestCaseStatus::kFailed, 0.0125 ); + failed.ClassName = "CompilerTests"; + failed.SourceFile = "tests/parser.cpp"; + failed.Message = "assertion: \"expected\" & actual"; + failed.Details = "expected <node> but got <none>"; + failed.StandardOutput = "trace <start>&finish"; + suite.TestCases.push_back( std::move( failed ) ); + + auto errored = MakeCase( "throws", TestCaseStatus::kErrored, 0.25 ); + errored.Message = "runtime error"; + errored.Details = "first line\nsecond line"; + errored.StandardError = "failure detail"; + suite.TestCases.push_back( std::move( errored ) ); + + auto skipped = MakeCase( "platform-only", TestCaseStatus::kSkipped, 0.0 ); + skipped.Message = "not available"; + suite.TestCases.push_back( std::move( skipped ) ); + + suite.TestCases.push_back( + MakeCase( "passes", TestCaseStatus::kPassed, 0.001 ) ); + const TestRunResult run{ "Compiler run", { std::move( suite ) } }; + + const std::string xml = JUnitXmlWriter::ToString( run ); + CHECK( xml.starts_with( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" ) ); + CHECK( xml.find( "<testsuites name=\"Compiler run\" tests=\"4\" " ) != + std::string::npos ); + CHECK( xml.find( "failures=\"1\" errors=\"1\" skipped=\"1\"" ) != + std::string::npos ); + CHECK( xml.find( "name=\"Parser & resolver\"" ) != std::string::npos ); + CHECK( xml.find( "name=\"rejects <input>\"" ) != std::string::npos ); + CHECK( xml.find( + "message=\"assertion: "expected" & actual\"" ) != + std::string::npos ); + CHECK( xml.find( "expected <node> but got &lt;none>" ) != + std::string::npos ); + CHECK( + xml.find( "<system-out>trace <start>&finish</system-out>" ) != + std::string::npos ); + CHECK( xml.find( "<error message=\"runtime error\" type=\"TestError\">" ) != + std::string::npos ); + CHECK( xml.find( "<skipped message=\"not available\"/>" ) != + std::string::npos ); + CHECK( xml.find( "classname=\"CompilerTests\"" ) != std::string::npos ); + CHECK( xml.find( "file=\"tests/parser.cpp\"" ) != std::string::npos ); + CHECK( xml.find( "time=\"0.263500\"" ) != std::string::npos ); + CHECK( xml.ends_with( "</testsuites>\n" ) ); +} + +TEST_CASE( "JUnit XML replaces malformed UTF-8 and forbidden XML controls", + "[progmasoft][results][xml]" ) { + using namespace Progmasoft::Catch3; + + auto testCase = MakeCase( "valid", TestCaseStatus::kPassed, 0.0 ); + testCase.Name = std::string( "bad\x01utf8\xF0\x28\x8C\x28", 12 ); + const TestRunResult run{ std::string( "suite\0name", 10 ), + { { "suite", { testCase } } } }; + + const std::string xml = JUnitXmlWriter::ToString( run ); + CHECK( xml.find( "name=\"bad�utf8�(�(\"" ) != + std::string::npos ); + CHECK( xml.find( "name=\"suite�name\"" ) != std::string::npos ); +} + +TEST_CASE( "Result summaries reject invalid durations and statuses", + "[progmasoft][results]" ) { + using namespace Progmasoft::Catch3; + + TestCaseResult invalidDuration = + MakeCase( "invalid", TestCaseStatus::kPassed, -1.0 ); + CHECK_THROWS_AS( + SummarizeTestRun( { "run", { { "suite", { invalidDuration } } } } ), + std::invalid_argument ); + + invalidDuration.DurationSeconds = std::numeric_limits<double>::infinity(); + CHECK_THROWS_AS( + SummarizeTestRun( { "run", { { "suite", { invalidDuration } } } } ), + std::invalid_argument ); + + invalidDuration.DurationSeconds = 0.0; + invalidDuration.Status = static_cast<TestCaseStatus>( 255 ); + CHECK_THROWS_AS( + SummarizeTestRun( { "run", { { "suite", { invalidDuration } } } } ), + std::invalid_argument ); +} + +TEST_CASE( "JUnit XML writer validates results and output stream state", + "[progmasoft][results][xml]" ) { + using namespace Progmasoft::Catch3; + + const TestRunResult emptyRun{ "empty", {} }; + const std::string emptyXml = JUnitXmlWriter::ToString( emptyRun ); + CHECK( emptyXml.find( "tests=\"0\" failures=\"0\" errors=\"0\" " ) != + std::string::npos ); + + auto invalidCase = MakeCase( "bad", TestCaseStatus::kPassed, 0.0 ); + invalidCase.DurationSeconds = std::numeric_limits<double>::quiet_NaN(); + const TestRunResult invalidRun{ "invalid", + { { "suite", { invalidCase } } } }; + std::ostringstream output; + CHECK_THROWS_AS( JUnitXmlWriter::Write( invalidRun, output ), + std::invalid_argument ); + + std::ostringstream failedOutput; + failedOutput.setstate( std::ios::badbit ); + CHECK_THROWS_AS( JUnitXmlWriter::Write( emptyRun, failedOutput ), + std::ios_base::failure ); +} diff --git a/tests/Progmasoft/Snapshot.tests.cpp b/tests/Progmasoft/Snapshot.tests.cpp new file mode 100644 index 00000000..65ee732e --- /dev/null +++ b/tests/Progmasoft/Snapshot.tests.cpp @@ -0,0 +1,164 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#include <Progmasoft/Catch3.hpp> + +#include <atomic> +#include <chrono> +#include <filesystem> +#include <fstream> +#include <string> +#include <system_error> + +namespace { + + class TemporarySnapshotDirectory final { + public: + TemporarySnapshotDirectory() { + static std::atomic<unsigned long long> nextId{ 0 }; + const auto timestamp = + std::chrono::steady_clock::now().time_since_epoch().count(); + Path = std::filesystem::temp_directory_path() / + ( "progmasoft-catch3-snapshot-" + + std::to_string( timestamp ) + "-" + + std::to_string( + nextId.fetch_add( 1, std::memory_order_relaxed ) ) ); + std::filesystem::create_directories( Path ); + } + + ~TemporarySnapshotDirectory() { + std::error_code ignored; + std::filesystem::remove_all( Path, ignored ); + } + + std::filesystem::path Path; + }; + +} // namespace + +TEST_CASE( "Snapshot comparison does not write during ordinary runs", + "[progmasoft][snapshot]" ) { + using Progmasoft::Catch3::CompareSnapshot; + using Progmasoft::Catch3::SnapshotOptions; + using Progmasoft::Catch3::SnapshotStatus; + + const TemporarySnapshotDirectory directory; + const auto result = + CompareSnapshot( "greeting", + "hello\nworld\n", + SnapshotOptions{ .Directory = directory.Path } ); + + CHECK( result.Status == SnapshotStatus::kMismatched ); + CHECK_FALSE( result.Succeeded() ); + CHECK( result.DifferenceLine == 1 ); + CHECK( result.DifferenceColumn == 1 ); + CHECK_FALSE( std::filesystem::exists( result.File ) ); +} + +TEST_CASE( "Snapshots normalize line endings and report precise differences", + "[progmasoft][snapshot]" ) { + using Progmasoft::Catch3::CompareSnapshot; + using Progmasoft::Catch3::SnapshotOptions; + using Progmasoft::Catch3::SnapshotStatus; + using Progmasoft::Catch3::SnapshotUpdateMode; + + const TemporarySnapshotDirectory directory; + const SnapshotOptions createOptions{ + .Directory = directory.Path, + .UpdateMode = SnapshotUpdateMode::kCreateMissing, + }; + const auto created = + CompareSnapshot( "multiline", "one\ntwo\n", createOptions ); + REQUIRE( created.Status == SnapshotStatus::kCreated ); + CHECK( created.Succeeded() ); + + const auto normalized = + CompareSnapshot( "multiline", "one\r\ntwo\r\n", createOptions ); + CHECK( normalized.Status == SnapshotStatus::kMatched ); + CHECK( normalized.Expected == "one\ntwo\n" ); + + const auto mismatch = + CompareSnapshot( "multiline", + "one\ntoo\n", + SnapshotOptions{ .Directory = directory.Path } ); + CHECK( mismatch.Status == SnapshotStatus::kMismatched ); + CHECK( mismatch.DifferenceLine == 2 ); + CHECK( mismatch.DifferenceColumn == 2 ); + CHECK( mismatch.Message.find( "line 2, column 2" ) != std::string::npos ); +} + +TEST_CASE( "Snapshot creation and replacement require explicit update modes", + "[progmasoft][snapshot]" ) { + using Progmasoft::Catch3::CompareSnapshot; + using Progmasoft::Catch3::SnapshotOptions; + using Progmasoft::Catch3::SnapshotStatus; + using Progmasoft::Catch3::SnapshotUpdateMode; + + const TemporarySnapshotDirectory directory; + const SnapshotOptions createMissing{ + .Directory = directory.Path, + .UpdateMode = SnapshotUpdateMode::kCreateMissing, + }; + REQUIRE( CompareSnapshot( "answer", "first", createMissing ).Succeeded() ); + + const auto protectedMismatch = + CompareSnapshot( "answer", "second", createMissing ); + CHECK( protectedMismatch.Status == SnapshotStatus::kMismatched ); + CHECK( protectedMismatch.Expected == "first" ); + + const SnapshotOptions updateEverything{ + .Directory = directory.Path, + .UpdateMode = SnapshotUpdateMode::kAlways, + }; + const auto updated = + CompareSnapshot( "answer", "second", updateEverything ); + REQUIRE( updated.Status == SnapshotStatus::kUpdated ); + CHECK( updated.Expected == "first" ); + CHECK( updated.Actual == "second" ); + CHECK( CompareSnapshot( "answer", "second", createMissing ).Succeeded() ); +} + +TEST_CASE( "Snapshot names and I/O failures are returned as structured errors", + "[progmasoft][snapshot]" ) { + using Progmasoft::Catch3::CompareSnapshot; + using Progmasoft::Catch3::SnapshotOptions; + using Progmasoft::Catch3::SnapshotStatus; + using Progmasoft::Catch3::SnapshotUpdateMode; + + const TemporarySnapshotDirectory directory; + const SnapshotOptions options{ .Directory = directory.Path }; + CHECK( CompareSnapshot( "../outside", "text", options ).Status == + SnapshotStatus::kInvalidName ); + CHECK( CompareSnapshot( "nested/name", "text", options ).Status == + SnapshotStatus::kInvalidName ); + CHECK( CompareSnapshot( "", "text", options ).Status == + SnapshotStatus::kInvalidName ); + CHECK( CompareSnapshot( "valid", "text", SnapshotOptions{} ).Status == + SnapshotStatus::kInvalidOptions ); + + const auto blockingFile = directory.Path / "not-a-directory"; + { + std::ofstream output( blockingFile, std::ios::binary ); + REQUIRE( output.good() ); + output << "file"; + } + const auto ioFailure = CompareSnapshot( + "snapshot", + "text", + SnapshotOptions{ .Directory = blockingFile, + .UpdateMode = SnapshotUpdateMode::kCreateMissing } ); + CHECK( ioFailure.Status == SnapshotStatus::kIoError ); + CHECK_FALSE( ioFailure.Message.empty() ); +} + +TEST_CASE( "Snapshot assertion macro reports successful explicit updates", + "[progmasoft][snapshot]" ) { + const TemporarySnapshotDirectory directory; + CATCH3_CHECK_SNAPSHOT( + "macro", + "stable value", + ( Progmasoft::Catch3::SnapshotOptions{ + .Directory = directory.Path, + .UpdateMode = + Progmasoft::Catch3::SnapshotUpdateMode::kCreateMissing, + } ) ); +} diff --git a/tests/meson.build b/tests/meson.build index 1d20b9ce..d96b6f9f 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -81,7 +81,11 @@ test('SelfTest', self_test) # language requirement or the upstream SelfTest compilation mode. progmasoft_property_tests = executable( 'ProgmasoftPropertyTests', - 'Progmasoft/Property.tests.cpp', + [ + 'Progmasoft/Property.tests.cpp', + 'Progmasoft/Results.tests.cpp', + 'Progmasoft/Snapshot.tests.cpp', + ], include_directories: '../src', dependencies: [progmasoft_catch3_dep, catch2_with_main_dep], override_options: ['cpp_std=c++20'], diff --git a/xmake.lua b/xmake.lua index 221351c3..8138e4e8 100644 --- a/xmake.lua +++ b/xmake.lua @@ -33,17 +33,26 @@ target("catch2_main") add_deps("catch2") add_includedirs("src", generated_include, {public = true}) --- This is a header-only C++20 layer over the compatibility engine. +-- Compile Progmasoft's implementation separately from the compatibility engine. target("ProgmasoftCatch3") - set_kind("headeronly") + set_kind("static") set_languages("cxx20") add_headerfiles("src/Progmasoft/**.hpp", {prefixdir = ""}) + add_files( + "src/Progmasoft/Results.cpp", + "src/Progmasoft/Snapshot.cpp", + "src/Progmasoft/XmlWriter.cpp" + ) add_includedirs("src", {public = true}) add_deps("catch2", {public = true}) target("ProgmasoftPropertyTests") set_kind("binary") set_languages("cxx20") - add_files("tests/Progmasoft/Property.tests.cpp") + add_files( + "tests/Progmasoft/Property.tests.cpp", + "tests/Progmasoft/Results.tests.cpp", + "tests/Progmasoft/Snapshot.tests.cpp" + ) add_includedirs("src", generated_include) add_deps("ProgmasoftCatch3", "catch2_main")