diff options
| author | Leitwolf11 | 2026-10-02 03:27:05 +0300 |
|---|---|---|
| committer | Leitwolf11 | 2026-10-02 03:27:05 +0300 |
| commit | 5d55a3226360fae1eaa5ef91908b0fa7d7d07234 (patch) | |
| tree | 61e94c3bd1684b4caa256bc34e380c272b67cdb6 /src | |
| parent | febb5b1e244174da0985b9994f8d5ed1676950e8 (diff) | |
| download | catch3-feature/property-combinators.tar.gz | |
Add composite property generators, single-trial replay and stable counterexample printingfeature/property-combinators
Property tests could only generate one integer or Boolean. This adds the
composite strategies the documentation listed as missing, without changing
any Catch2 file or the existing Progmasoft API.
Generators:
- Tuple(generators...) draws components left to right and shrinks one
component at a time, which is how a property receives several arguments.
- Vector(element, minimumSize, maximumSize) and Text(alphabet,
minimumLength, maximumLength) share one ordered shrink: shortest allowed
prefix, the two halves, single removals, then single element
simplifications. A shrunk value never falls below its minimum, each step
offers at most 256 candidates, and a size limit above 1,048,576 is
rejected.
- Element(values) picks from a fixed list and shrinks toward earlier
entries.
Runner:
- PropertyOptions::ReplayTrialSeed runs exactly one trial with a reported
replay seed, including its shrinking, so a failure from a log can be
reproduced without rerunning earlier trials. The report now ends with the
line that names this option and the seed.
- Counterexamples of tuples, pairs and vectors are printed element by
element instead of relying on Catch2's optional string makers, so the
report does not depend on configuration macros or include order.
The headers still compile for clients built without language exceptions.
Thirteen new test cases cover bounds, determinism, shrink order, candidate
limits, minimized counterexamples, replay and invalid configurations, and
every build description (Bazel, CMake, Meson, Xmake, Premake) lists the new
headers and test file.
Diffstat (limited to 'src')
| -rw-r--r-- | src/Progmasoft/BUILD.bazel | 5 | ||||
| -rw-r--r-- | src/Progmasoft/Catch3.hpp | 4 | ||||
| -rw-r--r-- | src/Progmasoft/Catch3/Detail/Sequence.hpp | 97 | ||||
| -rw-r--r-- | src/Progmasoft/Catch3/Generators/Element.hpp | 63 | ||||
| -rw-r--r-- | src/Progmasoft/Catch3/Generators/Text.hpp | 93 | ||||
| -rw-r--r-- | src/Progmasoft/Catch3/Generators/Tuple.hpp | 83 | ||||
| -rw-r--r-- | src/Progmasoft/Catch3/Generators/Vector.hpp | 60 | ||||
| -rw-r--r-- | src/Progmasoft/Catch3/Property.hpp | 97 | ||||
| -rw-r--r-- | src/Progmasoft/meson.build | 5 |
9 files changed, 503 insertions, 4 deletions
diff --git a/src/Progmasoft/BUILD.bazel b/src/Progmasoft/BUILD.bazel index ed49083c..14a67bf6 100644 --- a/src/Progmasoft/BUILD.bazel +++ b/src/Progmasoft/BUILD.bazel @@ -15,9 +15,14 @@ cc_library( "Catch3.hpp", "Catch3/Assertions.hpp", "Catch3/Detail/InvalidArgument.hpp", + "Catch3/Detail/Sequence.hpp", "Catch3/Generator.hpp", "Catch3/Generators/Boolean.hpp", + "Catch3/Generators/Element.hpp", "Catch3/Generators/Integer.hpp", + "Catch3/Generators/Text.hpp", + "Catch3/Generators/Tuple.hpp", + "Catch3/Generators/Vector.hpp", "Catch3/Property.hpp", "Catch3/Random.hpp", "Catch3/Results.hpp", diff --git a/src/Progmasoft/Catch3.hpp b/src/Progmasoft/Catch3.hpp index d2d34e74..5fca369a 100644 --- a/src/Progmasoft/Catch3.hpp +++ b/src/Progmasoft/Catch3.hpp @@ -8,7 +8,11 @@ #include <Progmasoft/Catch3/Assertions.hpp> #include <Progmasoft/Catch3/Generator.hpp> #include <Progmasoft/Catch3/Generators/Boolean.hpp> +#include <Progmasoft/Catch3/Generators/Element.hpp> #include <Progmasoft/Catch3/Generators/Integer.hpp> +#include <Progmasoft/Catch3/Generators/Text.hpp> +#include <Progmasoft/Catch3/Generators/Tuple.hpp> +#include <Progmasoft/Catch3/Generators/Vector.hpp> #include <Progmasoft/Catch3/Property.hpp> #include <Progmasoft/Catch3/Random.hpp> #include <Progmasoft/Catch3/Results.hpp> diff --git a/src/Progmasoft/Catch3/Detail/Sequence.hpp b/src/Progmasoft/Catch3/Detail/Sequence.hpp new file mode 100644 index 00000000..b1446a79 --- /dev/null +++ b/src/Progmasoft/Catch3/Detail/Sequence.hpp @@ -0,0 +1,97 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#pragma once + +#include <cstddef> +#include <iterator> +#include <utility> +#include <vector> + +namespace Progmasoft::Catch3::Detail +{ + + /// Upper bound on the candidates one sequence value offers per shrink + /// step. The property runner re-shrinks after every accepted candidate, + /// so a bounded list still reaches small counterexamples while keeping + /// each step linear in this constant instead of in the sequence length. + inline constexpr std::size_t kMaximumSequenceCandidates = 256; + + /// Largest sequence a built-in generator will create. A size range above + /// this is rejected instead of silently allocating gigabytes in a test. + inline constexpr std::size_t kMaximumSequenceLength = std::size_t{ 1 } + << 20U; + + /** + * @brief Ordered shrink candidates shared by Vector and Text. + * + * Candidates are produced in a fixed order, simplest first: + * + * 1. the shortest allowed prefix, + * 2. the first half and then the second half, + * 3. the value with one element removed, position by position, + * 4. the value with one element replaced by each of that element's own + * shrink candidates, position by position. + * + * No candidate is shorter than @p minimumSize, every candidate differs + * from @p value in length or in exactly one element, and the list is cut + * at kMaximumSequenceCandidates. The same candidate may appear twice, + * for example when removing the last element equals the shortest prefix; + * the runner simply tests it again. + */ + template<typename Sequence, typename ShrinkElement> + [[nodiscard]] std::vector<Sequence> + ShrinkSequence(const Sequence &value, + std::size_t minimumSize, + ShrinkElement &&shrinkElement) + { + using Element = typename Sequence::value_type; + using Difference = typename std::iterator_traits< + typename Sequence::const_iterator>::difference_type; + + std::vector<Sequence> candidates; + const std::size_t size = value.size(); + const auto full = [&candidates] { + return candidates.size() >= kMaximumSequenceCandidates; + }; + const auto at = [&value](std::size_t index) { + return value.begin() + static_cast<Difference>(index); + }; + + if (size > minimumSize) + { + candidates.push_back(Sequence(at(0), at(minimumSize))); + + const std::size_t kept = size - size / 2; + if (kept > minimumSize && kept < size) + { + candidates.push_back(Sequence(at(0), at(kept))); + candidates.push_back(Sequence(at(size - kept), at(size))); + } + + for (std::size_t index = 0; index < size && !full(); ++index) + { + Sequence shorter(at(0), at(index)); + shorter.insert(shorter.end(), at(index + 1), at(size)); + candidates.push_back(std::move(shorter)); + } + } + + for (std::size_t index = 0; index < size && !full(); ++index) + { + const Element current = static_cast<Element>(*at(index)); + for (const auto &simpler : shrinkElement(current)) + { + if (full()) + { + break; + } + Sequence replaced = value; + *(replaced.begin() + static_cast<Difference>(index)) + = static_cast<Element>(simpler); + candidates.push_back(std::move(replaced)); + } + } + return candidates; + } + +} // namespace Progmasoft::Catch3::Detail diff --git a/src/Progmasoft/Catch3/Generators/Element.hpp b/src/Progmasoft/Catch3/Generators/Element.hpp new file mode 100644 index 00000000..087b6355 --- /dev/null +++ b/src/Progmasoft/Catch3/Generators/Element.hpp @@ -0,0 +1,63 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#pragma once + +#include <Progmasoft/Catch3/Generator.hpp> +#include <algorithm> +#include <concepts> +#include <cstddef> +#include <initializer_list> +#include <iterator> +#include <memory> +#include <utility> +#include <vector> + +namespace Progmasoft::Catch3 +{ + + /// Picks uniformly from a fixed, non-empty list of values. + /// + /// The list order is the simplicity order: a value shrinks to the values + /// listed before it, earliest first, so place the simplest case first. + /// A value that is not in the list has no candidates. + template<typename ValueType> + requires std::equality_comparable<ValueType> + [[nodiscard]] Generator<ValueType> + Element(std::vector<ValueType> values) + { + if (values.empty()) + { + Detail::RejectInvalidArgument( + "Element requires at least one value"); + } + + // Both callbacks share one immutable copy of the list. + const auto shared + = std::make_shared<const std::vector<ValueType>>(std::move(values)); + return MakeGenerator<ValueType>( + [shared](Random &random) { + const std::size_t index + = random.Between(std::size_t{ 0 }, shared->size() - 1); + return static_cast<ValueType>((*shared)[index]); + }, + [shared](const ValueType &value) { + const auto found + = std::find(shared->begin(), shared->end(), value); + if (found == shared->end()) + { + return std::vector<ValueType>{}; + } + return std::vector<ValueType>(shared->begin(), found); + }); + } + + /// Convenience overload for a literal list of values. + template<typename ValueType> + requires std::equality_comparable<ValueType> + [[nodiscard]] Generator<ValueType> + Element(std::initializer_list<ValueType> values) + { + return Element(std::vector<ValueType>(values)); + } + +} // namespace Progmasoft::Catch3 diff --git a/src/Progmasoft/Catch3/Generators/Text.hpp b/src/Progmasoft/Catch3/Generators/Text.hpp new file mode 100644 index 00000000..05df8c00 --- /dev/null +++ b/src/Progmasoft/Catch3/Generators/Text.hpp @@ -0,0 +1,93 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#pragma once + +#include <Progmasoft/Catch3/Detail/Sequence.hpp> +#include <Progmasoft/Catch3/Generator.hpp> +#include <cstddef> +#include <memory> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + +namespace Progmasoft::Catch3 +{ + + /// The 95 printable ASCII characters, space through tilde, with the + /// lower-case letter `a` first so that it is the simplest character. + [[nodiscard]] inline std::string + PrintableAscii() + { + std::string alphabet = "a"; + for (char character = ' '; character <= '~'; ++character) + { + if (character != 'a') + { + alphabet.push_back(character); + } + } + return alphabet; + } + + /// Generates byte strings whose characters are drawn from @p alphabet + /// and whose length lies in the inclusive range. + /// + /// The alphabet order is the simplicity order: shrinking first shortens + /// the string and then replaces single characters with earlier alphabet + /// characters, earliest first. Characters are bytes; the generator does + /// not interpret an encoding, so supply an alphabet that is valid for + /// the code under test. + [[nodiscard]] inline Generator<std::string> + Text(std::string alphabet = PrintableAscii(), + std::size_t minimumLength = 0, + std::size_t maximumLength = 32) + { + if (alphabet.empty()) + { + Detail::RejectInvalidArgument( + "Text requires a non-empty alphabet"); + } + if (minimumLength > maximumLength) + { + Detail::RejectInvalidArgument( + "Text requires minimumLength <= maximumLength"); + } + if (maximumLength > Detail::kMaximumSequenceLength) + { + Detail::RejectInvalidArgument( + "Text maximumLength exceeds the supported sequence length"); + } + + const auto shared + = std::make_shared<const std::string>(std::move(alphabet)); + return MakeGenerator<std::string>( + [shared, minimumLength, maximumLength](Random &random) { + const std::size_t length + = random.Between(minimumLength, maximumLength); + std::string text; + text.reserve(length); + for (std::size_t index = 0; index < length; ++index) + { + text.push_back((*shared)[random.Between( + std::size_t{ 0 }, + shared->size() - 1)]); + } + return text; + }, + [shared, minimumLength](const std::string &value) { + return Detail::ShrinkSequence( + value, + minimumLength, + [&shared](char character) { + const std::size_t position = shared->find(character); + if (position == std::string::npos) + { + return std::string{}; + } + return shared->substr(0, position); + }); + }); + } + +} // namespace Progmasoft::Catch3 diff --git a/src/Progmasoft/Catch3/Generators/Tuple.hpp b/src/Progmasoft/Catch3/Generators/Tuple.hpp new file mode 100644 index 00000000..df58da05 --- /dev/null +++ b/src/Progmasoft/Catch3/Generators/Tuple.hpp @@ -0,0 +1,83 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#pragma once + +#include <Progmasoft/Catch3/Generator.hpp> +#include <cstddef> +#include <tuple> +#include <utility> +#include <vector> + +namespace Progmasoft::Catch3 +{ + namespace Detail + { + + template<std::size_t Index, typename... ValueTypes> + void + AppendComponentCandidates( + const std::tuple<Generator<ValueTypes>...> &generators, + const std::tuple<ValueTypes...> &value, + std::vector<std::tuple<ValueTypes...>> &candidates) + { + for (const auto &simpler : + std::get<Index>(generators).Shrink(std::get<Index>(value))) + { + std::tuple<ValueTypes...> replaced = value; + std::get<Index>(replaced) = simpler; + candidates.push_back(std::move(replaced)); + } + } + + template<typename... ValueTypes, std::size_t... Indices> + [[nodiscard]] std::vector<std::tuple<ValueTypes...>> + ShrinkTuple(const std::tuple<Generator<ValueTypes>...> &generators, + const std::tuple<ValueTypes...> &value, + std::index_sequence<Indices...>) + { + std::vector<std::tuple<ValueTypes...>> candidates; + // The comma fold visits components strictly left to right. + (AppendComponentCandidates<Indices>(generators, value, candidates), + ...); + return candidates; + } + + } // namespace Detail + + /// Combines independent generators into one generator of tuples, which + /// is how a property receives several generated arguments. + /// + /// Components are drawn strictly left to right from the trial's random + /// source, so a tuple is reproducible from its replay seed. Shrinking + /// changes one component at a time: all candidates of the first + /// component, then all of the second, and so on. Combined with the + /// runner's greedy restart this minimizes every component without + /// trying the cross product of candidates. + template<typename... ValueTypes> + requires(sizeof...(ValueTypes) > 0) + [[nodiscard]] Generator<std::tuple<ValueTypes...>> + Tuple(Generator<ValueTypes>... generators) + { + using Value = std::tuple<ValueTypes...>; + const std::tuple<Generator<ValueTypes>...> components{ std::move( + generators)... }; + + return MakeGenerator<Value>( + [components](Random &random) { + return std::apply( + [&random](const Generator<ValueTypes> &...component) { + // Braced initialization fixes the evaluation order; + // a function-call argument list would not. + return Value{ component.Generate(random)... }; + }, + components); + }, + [components](const Value &value) { + return Detail::ShrinkTuple( + components, + value, + std::index_sequence_for<ValueTypes...>{}); + }); + } + +} // namespace Progmasoft::Catch3 diff --git a/src/Progmasoft/Catch3/Generators/Vector.hpp b/src/Progmasoft/Catch3/Generators/Vector.hpp new file mode 100644 index 00000000..65c81e3b --- /dev/null +++ b/src/Progmasoft/Catch3/Generators/Vector.hpp @@ -0,0 +1,60 @@ +// Copyright (c) 2026 Progmasoft. +// SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1 +#pragma once + +#include <Progmasoft/Catch3/Detail/Sequence.hpp> +#include <Progmasoft/Catch3/Generator.hpp> +#include <cstddef> +#include <utility> +#include <vector> + +namespace Progmasoft::Catch3 +{ + + /// Generates vectors whose length lies in the inclusive size range and + /// whose elements come from @p element, drawn in index order. + /// + /// Shrinking first shortens the vector, then simplifies single elements + /// with the element generator's own candidates; see + /// Detail::ShrinkSequence for the exact order. A shrunk vector never + /// becomes shorter than @p minimumSize, so a property may rely on it. + template<typename ValueType> + [[nodiscard]] Generator<std::vector<ValueType>> + Vector(Generator<ValueType> element, + std::size_t minimumSize = 0, + std::size_t maximumSize = 32) + { + if (minimumSize > maximumSize) + { + Detail::RejectInvalidArgument( + "Vector requires minimumSize <= maximumSize"); + } + if (maximumSize > Detail::kMaximumSequenceLength) + { + Detail::RejectInvalidArgument( + "Vector maximumSize exceeds the supported sequence length"); + } + + return MakeGenerator<std::vector<ValueType>>( + [element, minimumSize, maximumSize](Random &random) { + const std::size_t size + = random.Between(minimumSize, maximumSize); + std::vector<ValueType> values; + values.reserve(size); + for (std::size_t index = 0; index < size; ++index) + { + values.push_back(element.Generate(random)); + } + return values; + }, + [element, minimumSize](const std::vector<ValueType> &value) { + return Detail::ShrinkSequence( + value, + minimumSize, + [&element](const ValueType &item) { + return element.Shrink(item); + }); + }); + } + +} // namespace Progmasoft::Catch3 diff --git a/src/Progmasoft/Catch3/Property.hpp b/src/Progmasoft/Catch3/Property.hpp index 1a853bf1..57f7ac7b 100644 --- a/src/Progmasoft/Catch3/Property.hpp +++ b/src/Progmasoft/Catch3/Property.hpp @@ -12,8 +12,10 @@ #include <optional> #include <sstream> #include <string> +#include <tuple> #include <type_traits> #include <utility> +#include <vector> namespace Progmasoft::Catch3 { @@ -24,6 +26,11 @@ namespace Progmasoft::Catch3 std::size_t Trials = 100; std::uint64_t Seed = 0xC47C4A53ULL; std::size_t MaxShrinkSteps = 1000; + /// When set, runs exactly one trial with this seed instead of + /// deriving Trials seeds from Seed. Use the "replay seed" of a + /// reported failure to reproduce that single trial, including its + /// shrinking, without rerunning the trials before it. + std::optional<std::uint64_t> ReplayTrialSeed = std::nullopt; }; enum class PropertyStatus : std::uint8_t @@ -88,7 +95,9 @@ namespace Progmasoft::Catch3 << Failure->TrialSeed << ", shrink checks " << Failure->ShrinkSteps << "\ncounterexample: " << Failure->Counterexample - << "\nreason: " << Failure->Reason; + << "\nreason: " << Failure->Reason + << "\nreplay: set PropertyOptions::ReplayTrialSeed to " + << Failure->TrialSeed; } return output.str(); } @@ -99,9 +108,85 @@ namespace Progmasoft::Catch3 template<typename ValueType> [[nodiscard]] std::string + StringifyPropertyValue(const ValueType &value); + + /// Renders one counterexample. Scalars use Catch2's StringMaker, so + /// user specializations apply. Tuples, pairs and vectors are + /// rendered here, element by element, because Catch2 only prints + /// tuples when a configuration macro is defined before its headers + /// and a counterexample must not degrade to "{?}" depending on + /// include order. + template<typename ValueType> + struct PropertyValuePrinter final + { + [[nodiscard]] static std::string + Print(const ValueType &value) + { + return Catch::StringMaker<ValueType>::convert(value); + } + }; + + template<typename... ValueTypes> + struct PropertyValuePrinter<std::tuple<ValueTypes...>> final + { + [[nodiscard]] static std::string + Print(const std::tuple<ValueTypes...> &value) + { + std::string text = "{"; + bool first = true; + std::apply( + [&text, &first](const ValueTypes &...component) { + const auto append = [&text, &first](std::string item) { + text += first ? " " : ", "; + text += item; + first = false; + }; + (append(StringifyPropertyValue(component)), ...); + }, + value); + text += first ? "}" : " }"; + return text; + } + }; + + template<typename FirstType, typename SecondType> + struct PropertyValuePrinter<std::pair<FirstType, SecondType>> final + { + [[nodiscard]] static std::string + Print(const std::pair<FirstType, SecondType> &value) + { + return "{ " + StringifyPropertyValue(value.first) + ", " + + StringifyPropertyValue(value.second) + " }"; + } + }; + + template<typename ElementType, typename AllocatorType> + struct PropertyValuePrinter<std::vector<ElementType, AllocatorType>> + final + { + [[nodiscard]] static std::string + Print(const std::vector<ElementType, AllocatorType> &value) + { + std::string text = "{"; + bool first = true; + for (const auto &element : value) + { + text += first ? " " : ", "; + // The cast turns std::vector<bool>'s proxy into a value. + text += StringifyPropertyValue( + static_cast<ElementType>(element)); + first = false; + } + text += first ? "}" : " }"; + return text; + } + }; + + template<typename ValueType> + [[nodiscard]] std::string StringifyPropertyValue(const ValueType &value) { - return Catch::StringMaker<std::remove_cvref_t<ValueType>>::convert( + return PropertyValuePrinter<std::remove_cvref_t<ValueType>>::Print( value); } @@ -178,11 +263,15 @@ namespace Progmasoft::Catch3 } PropertyResult result; - for (std::size_t trial = 0; trial < options.Trials; ++trial) + const std::size_t trials + = options.ReplayTrialSeed.has_value() ? 1 : options.Trials; + for (std::size_t trial = 0; trial < trials; ++trial) { result.TrialsRun = trial + 1; const std::uint64_t trialSeed - = Random::DeriveSeed(options.Seed, trial); + = options.ReplayTrialSeed.has_value() + ? *options.ReplayTrialSeed + : Random::DeriveSeed(options.Seed, trial); Random random(trialSeed); std::optional<ValueType> sample; diff --git a/src/Progmasoft/meson.build b/src/Progmasoft/meson.build index f9acedb2..1f9afbda 100644 --- a/src/Progmasoft/meson.build +++ b/src/Progmasoft/meson.build @@ -21,6 +21,7 @@ if get_option('install') install_headers('Catch3.hpp', subdir: 'Progmasoft') install_headers('Catch3/Assertions.hpp', subdir: 'Progmasoft/Catch3') install_headers('Catch3/Detail/InvalidArgument.hpp', subdir: 'Progmasoft/Catch3/Detail') + install_headers('Catch3/Detail/Sequence.hpp', subdir: 'Progmasoft/Catch3/Detail') install_headers('Catch3/Generator.hpp', subdir: 'Progmasoft/Catch3') install_headers('Catch3/Property.hpp', subdir: 'Progmasoft/Catch3') install_headers('Catch3/Random.hpp', subdir: 'Progmasoft/Catch3') @@ -29,4 +30,8 @@ if get_option('install') install_headers('Catch3/XmlWriter.hpp', subdir: 'Progmasoft/Catch3') install_headers('Catch3/Generators/Boolean.hpp', subdir: 'Progmasoft/Catch3/Generators') install_headers('Catch3/Generators/Integer.hpp', subdir: 'Progmasoft/Catch3/Generators') + install_headers('Catch3/Generators/Element.hpp', subdir: 'Progmasoft/Catch3/Generators') + install_headers('Catch3/Generators/Text.hpp', subdir: 'Progmasoft/Catch3/Generators') + install_headers('Catch3/Generators/Tuple.hpp', subdir: 'Progmasoft/Catch3/Generators') + install_headers('Catch3/Generators/Vector.hpp', subdir: 'Progmasoft/Catch3/Generators') endif |