catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
v2.9.0
- Commit
f9dce28e7db84f61622c9f5d254af1acf03dfcc0- Author
- Martin Hořeňovský ·
- Parents
b87caafd9117
Changes
CMakeLists.txt | 2 +- README.md | 4 +- docs/release-notes.md | 11 + include/catch.hpp | 2 +- include/internal/catch_version.cpp | 2 +- single_include/catch2/catch.hpp | 2036 +++++++++++++++++++++++++++++++----- 6 files changed, 1780 insertions(+), 277 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bf59b7d7..e1a26e05 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ if(NOT DEFINED PROJECT_NAME) set(NOT_SUBPROJECT ON) endif() -project(Catch2 LANGUAGES CXX VERSION 2.8.0) +project(Catch2 LANGUAGES CXX VERSION 2.9.0) if (CMAKE_BINARY_DIR STREQUAL CMAKE_SOURCE_DIR) message(FATAL_ERROR "Building in-source is not supported! Create a build dir and remove ${CMAKE_SOURCE_DIR}/CMakeCache.txt") diff --git a/README.md b/README.md index 47269cea..64308ee5 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,11 @@ [](https://travis-ci.org/catchorg/Catch2) [](https://ci.appveyor.com/project/catchorg/catch2) [](https://codecov.io/gh/catchorg/Catch2) -[](https://wandbox.org/permlink/wQeSudewoh6pdgRp) +[](https://wandbox.org/permlink/HbqvzsGh9s9aubN3) [](https://discord.gg/4CWS9zD) -<a href="https://github.com/catchorg/Catch2/releases/download/v2.8.0/catch.hpp">The latest version of the single header can be downloaded directly using this link</a> +<a href="https://github.com/catchorg/Catch2/releases/download/v2.9.0/catch.hpp">The latest version of the single header can be downloaded directly using this link</a> ## Catch2 is released! diff --git a/docs/release-notes.md b/docs/release-notes.md index c998b354..28388ebe 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -2,6 +2,7 @@ # Release notes **Contents**<br> +[2.9.0](#290)<br> [2.8.0](#280)<br> [2.7.2](#272)<br> [2.7.1](#271)<br> @@ -24,6 +25,16 @@ [Older versions](#older-versions)<br> [Even Older versions](#even-older-versions)<br> +## 2.9.0 + +### Improvements +* The experimental benchmarking support has been replaced by integrating Nonius code (#1616) + * This provides a much more featurefull micro-benchmarking support. + * Due to the compilation cost, it is disabled by default. See the documentation for details. + * As far as backwards compatibility is concerned, this feature is still considered experimental in that we might change the interface based on user feedback. +* `WithinULP` matcher now shows the acceptable range (#1581) +* Template test cases now support type lists (#1627) + ## 2.8.0 diff --git a/include/catch.hpp b/include/catch.hpp index 483096b1..b63cc7fe 100644 --- a/include/catch.hpp +++ b/include/catch.hpp @@ -10,7 +10,7 @@ #define TWOBLUECUBES_CATCH_HPP_INCLUDED #define CATCH_VERSION_MAJOR 2 -#define CATCH_VERSION_MINOR 8 +#define CATCH_VERSION_MINOR 9 #define CATCH_VERSION_PATCH 0 #ifdef __clang__ diff --git a/include/internal/catch_version.cpp b/include/internal/catch_version.cpp index e5f683b2..b0fcb115 100644 --- a/include/internal/catch_version.cpp +++ b/include/internal/catch_version.cpp @@ -37,7 +37,7 @@ namespace Catch { } Version const& libraryVersion() { - static Version version( 2, 8, 0, "", 0 ); + static Version version( 2, 9, 0, "", 0 ); return version; } diff --git a/single_include/catch2/catch.hpp b/single_include/catch2/catch.hpp index 02302b8d..9b145059 100644 --- a/single_include/catch2/catch.hpp +++ b/single_include/catch2/catch.hpp @@ -1,6 +1,6 @@ /* - * Catch v2.8.0 - * Generated: 2019-05-26 21:29:22.235281 + * Catch v2.9.0 + * Generated: 2019-06-16 10:41:32.122746 * ---------------------------------------------------------- * This file has been merged from multiple headers. Please don't edit it directly * Copyright (c) 2019 Two Blue Cubes Ltd. All rights reserved. @@ -14,7 +14,7 @@ #define CATCH_VERSION_MAJOR 2 -#define CATCH_VERSION_MINOR 8 +#define CATCH_VERSION_MINOR 9 #define CATCH_VERSION_PATCH 0 #ifdef __clang__ @@ -210,9 +210,9 @@ namespace Catch { // some versions of cygwin (most) do not support std::to_string. Use the libstd check. // https://gcc.gnu.org/onlinedocs/gcc-4.8.2/libstdc++/api/a01053_source.html line 2812-2813 # if !((__cplusplus >= 201103L) && defined(_GLIBCXX_USE_C99) \ - && !defined(_GLIBCXX_HAVE_BROKEN_VSWPRINTF)) + && !defined(_GLIBCXX_HAVE_BROKEN_VSWPRINTF)) -# define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING +# define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING # endif #endif // __CYGWIN__ @@ -239,7 +239,11 @@ namespace Catch { # if !defined(_MSVC_TRADITIONAL) || (defined(_MSVC_TRADITIONAL) && _MSVC_TRADITIONAL) # define CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR # endif +#endif // _MSC_VER +#if defined(_REENTRANT) || defined(_MSC_VER) +// Enable async processing, as -pthread is specified or no additional linking is required +# define CATCH_INTERNAL_CONFIG_USE_ASYNC #endif // _MSC_VER //////////////////////////////////////////////////////////////////////////////// @@ -358,6 +362,10 @@ namespace Catch { # define CATCH_CONFIG_POLYFILL_ISNAN #endif +#if defined(CATCH_INTERNAL_CONFIG_USE_ASYNC) && !defined(CATCH_CONFIG_NO_USE_ASYNC) && !defined(CATCH_CONFIG_USE_ASYNC) +# define CATCH_CONFIG_USE_ASYNC +#endif + #if !defined(CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS) # define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS # define CATCH_INTERNAL_UNSUPPRESS_PARENTHESES_WARNINGS @@ -771,6 +779,8 @@ struct is_unique<T0, T1, Rest...> : std::integral_constant constexpr auto append(L1<E1...>, L2<E2...>) noexcept -> L1<E1...,E2...> { return {}; }\ template< template<typename...> class L1, typename...E1, template<typename...> class L2, typename...E2, typename...Rest>\ constexpr auto append(L1<E1...>, L2<E2...>, Rest...) noexcept -> decltype(append(L1<E1...,E2...>{}, Rest{}...)) { return {}; }\ + template< template<typename...> class L1, typename...E1, typename...Rest>\ + constexpr auto append(L1<E1...>, TypeList<mpl_::na>, Rest...) noexcept -> L1<E1...> { return {}; }\ \ template< template<typename...> class Container, template<typename...> class List, typename...elems>\ constexpr auto rewrap(List<elems...>) noexcept -> TypeList<Container<elems...>> { return {}; }\ @@ -778,7 +788,9 @@ struct is_unique<T0, T1, Rest...> : std::integral_constant constexpr auto rewrap(List<Elems...>,Elements...) noexcept -> decltype(append(TypeList<Container<Elems...>>{}, rewrap<Container>(Elements{}...))) { return {}; }\ \ template<template <typename...> class Final, template< typename...> class...Containers, typename...Types>\ - constexpr auto create(TypeList<Types...>) noexcept -> decltype(append(Final<>{}, rewrap<Containers>(Types{}...)...)) { return {}; } + constexpr auto create(TypeList<Types...>) noexcept -> decltype(append(Final<>{}, rewrap<Containers>(Types{}...)...)) { return {}; }\ + template<template <typename...> class Final, template <typename...> class List, typename...Ts>\ + constexpr auto convert(List<Ts...>) noexcept -> decltype(append(Final<>{},TypeList<Ts>{}...)) { return {}; } #define INTERNAL_CATCH_NTTP_1(signature, ...)\ template<INTERNAL_CATCH_REMOVE_PARENS(signature)> struct Nttp{};\ @@ -886,10 +898,29 @@ struct is_unique<T0, T1, Rest...> : std::integral_constant #include <type_traits> namespace Catch { - template<typename T> - struct always_false : std::false_type {}; +template<typename T> +struct always_false : std::false_type {}; + +template <typename> struct true_given : std::true_type {}; +struct is_callable_tester { + template <typename Fun, typename... Args> + true_given<decltype(std::declval<Fun>()(std::declval<Args>()...))> static test(int); + template <typename...> + std::false_type static test(...); +}; + +template <typename T> +struct is_callable; + +template <typename Fun, typename... Args> +struct is_callable<Fun(Args...)> : decltype(is_callable_tester::test<Fun, Args...>(0)) {}; + } // namespace Catch +namespace mpl_{ + struct na; +} + // end catch_meta.hpp namespace Catch { @@ -1107,6 +1138,34 @@ struct AutoReg : NonCopyable { INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, Signature, __VA_ARGS__ ) ) #endif + #define INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_2(TestName, TestFunc, Name, Tags, TmplList)\ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + template<typename TestType> static void TestFunc(); \ + namespace {\ + namespace INTERNAL_CATCH_MAKE_NAMESPACE(TestName){\ + INTERNAL_CATCH_TYPE_GEN\ + template<typename... Types> \ + struct TestName { \ + void reg_tests() { \ + int index = 0; \ + using expander = int[]; \ + (void)expander{(Catch::AutoReg( Catch::makeTestInvoker( &TestFunc<Types> ), CATCH_INTERNAL_LINEINFO, Catch::StringRef(), Catch::NameAndTags{ Name " - " + std::string(INTERNAL_CATCH_STRINGIZE(TmplList)) + " - " + std::to_string(index), Tags } ), index++, 0)... };/* NOLINT */\ + } \ + };\ + static int INTERNAL_CATCH_UNIQUE_NAME( globalRegistrar ) = [](){ \ + using TestInit = decltype(convert<TestName>(TmplList {})); \ + TestInit t; \ + t.reg_tests(); \ + return 0; \ + }(); \ + }}\ + CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS \ + template<typename TestType> \ + static void TestFunc() + + #define INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE(Name, Tags, TmplList) \ + INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), Name, Tags, TmplList ) + #define INTERNAL_CATCH_TEMPLATE_TEST_CASE_METHOD_2( TestNameClass, TestName, ClassName, Name, Tags, Signature, ... ) \ CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS \ @@ -1202,6 +1261,37 @@ struct AutoReg : NonCopyable { INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), ClassName, Name, Tags, Signature,__VA_ARGS__ ) ) #endif + #define INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD_2( TestNameClass, TestName, ClassName, Name, Tags, TmplList) \ + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ + template<typename TestType> \ + struct TestName : INTERNAL_CATCH_REMOVE_PARENS(ClassName <TestType>) { \ + void test();\ + };\ + namespace {\ + namespace INTERNAL_CATCH_MAKE_NAMESPACE(TestName){ \ + INTERNAL_CATCH_TYPE_GEN\ + template<typename...Types>\ + struct TestNameClass{\ + void reg_tests(){\ + int index = 0;\ + using expander = int[];\ + (void)expander{(Catch::AutoReg( Catch::makeTestInvoker( &TestName<Types>::test ), CATCH_INTERNAL_LINEINFO, #ClassName, Catch::NameAndTags{ Name " - " + std::string(INTERNAL_CATCH_STRINGIZE(TmplList)) + " - " + std::to_string(index), Tags } ), index++, 0)... };/* NOLINT */ \ + }\ + };\ + static int INTERNAL_CATCH_UNIQUE_NAME( globalRegistrar ) = [](){\ + using TestInit = decltype(convert<TestNameClass>(TmplList {}));\ + TestInit t;\ + t.reg_tests();\ + return 0;\ + }(); \ + }}\ + CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS \ + template<typename TestType> \ + void TestName<TestType>::test() + +#define INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD(ClassName, Name, Tags, TmplList) \ + INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD_2( INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____ ), INTERNAL_CATCH_UNIQUE_NAME( ____C_A_T_C_H____T_E_M_P_L_A_T_E____T_E_S_T____F_U_N_C____ ), ClassName, Name, Tags, TmplList ) + // end catch_test_registry.h // start catch_capture.hpp @@ -2270,6 +2360,7 @@ namespace Catch { // start catch_interfaces_capture.h #include <string> +#include <chrono> namespace Catch { @@ -2280,14 +2371,18 @@ namespace Catch { struct MessageInfo; struct MessageBuilder; struct Counts; - struct BenchmarkInfo; - struct BenchmarkStats; struct AssertionReaction; struct SourceLineInfo; struct ITransientExpression; struct IGeneratorTracker; +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) + struct BenchmarkInfo; + template <typename Duration = std::chrono::duration<double, std::nano>> + struct BenchmarkStats; +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING + struct IResultCapture { virtual ~IResultCapture(); @@ -2299,8 +2394,12 @@ namespace Catch { virtual auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& = 0; +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) + virtual void benchmarkPreparing( std::string const& name ) = 0; virtual void benchmarkStarting( BenchmarkInfo const& info ) = 0; - virtual void benchmarkEnded( BenchmarkStats const& stats ) = 0; + virtual void benchmarkEnded( BenchmarkStats<> const& stats ) = 0; + virtual void benchmarkFailed( std::string const& error ) = 0; +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING virtual void pushScopedMessage( MessageInfo const& message ) = 0; virtual void popScopedMessage( MessageInfo const& message ) = 0; @@ -2766,52 +2865,6 @@ namespace Catch { CATCH_INTERNAL_UNSUPPRESS_UNUSED_WARNINGS // end catch_section.h -// start catch_benchmark.h - -#include <cstdint> -#include <string> - -namespace Catch { - - class BenchmarkLooper { - - std::string m_name; - std::size_t m_count = 0; - std::size_t m_iterationsToRun = 1; - uint64_t m_resolution; - Timer m_timer; - - static auto getResolution() -> uint64_t; - public: - // Keep most of this inline as it's on the code path that is being timed - BenchmarkLooper( StringRef name ) - : m_name( name ), - m_resolution( getResolution() ) - { - reportStart(); - m_timer.start(); - } - - explicit operator bool() { - if( m_count < m_iterationsToRun ) - return true; - return needsMoreIterations(); - } - - void increment() { - ++m_count; - } - - void reportStart(); - auto needsMoreIterations() -> bool; - }; - -} // end namespace Catch - -#define BENCHMARK( name ) \ - for( Catch::BenchmarkLooper looper( name ); looper; looper.increment() ) - -// end catch_benchmark.h // start catch_interfaces_exception.h // start catch_interfaces_registry_hub.h @@ -3678,7 +3731,7 @@ namespace Catch { // end catch_interfaces_generatortracker.h // start catch_enforce.h -#include <stdexcept> +#include <exception> namespace Catch { #if !defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) @@ -3691,18 +3744,30 @@ namespace Catch { [[noreturn]] void throw_exception(std::exception const& e); #endif + + [[noreturn]] + void throw_logic_error(std::string const& msg); + [[noreturn]] + void throw_domain_error(std::string const& msg); + [[noreturn]] + void throw_runtime_error(std::string const& msg); + } // namespace Catch; -#define CATCH_PREPARE_EXCEPTION( type, msg ) \ - type( ( Catch::ReusableStringStream() << msg ).str() ) -#define CATCH_INTERNAL_ERROR( msg ) \ - Catch::throw_exception(CATCH_PREPARE_EXCEPTION( std::logic_error, CATCH_INTERNAL_LINEINFO << ": Internal Catch error: " << msg)) -#define CATCH_ERROR( msg ) \ - Catch::throw_exception(CATCH_PREPARE_EXCEPTION( std::domain_error, msg )) -#define CATCH_RUNTIME_ERROR( msg ) \ - Catch::throw_exception(CATCH_PREPARE_EXCEPTION( std::runtime_error, msg )) -#define CATCH_ENFORCE( condition, msg ) \ - do{ if( !(condition) ) CATCH_ERROR( msg ); } while(false) +#define CATCH_MAKE_MSG(...) \ + (Catch::ReusableStringStream() << __VA_ARGS__).str() + +#define CATCH_INTERNAL_ERROR(...) \ + Catch::throw_logic_error(CATCH_MAKE_MSG( CATCH_INTERNAL_LINEINFO << ": Internal Catch2 error: " << __VA_ARGS__)); + +#define CATCH_ERROR(...) \ + Catch::throw_domain_error(CATCH_MAKE_MSG( __VA_ARGS__ )); + +#define CATCH_RUNTIME_ERROR(...) \ + Catch::throw_runtime_error(CATCH_MAKE_MSG( __VA_ARGS__ )); + +#define CATCH_ENFORCE( condition, ... ) \ + do{ if( !(condition) ) CATCH_ERROR( __VA_ARGS__ ); } while(false) // end catch_enforce.h #include <memory> @@ -4187,6 +4252,71 @@ namespace Catch { // end catch_context.h // start catch_interfaces_config.h +// start catch_option.hpp + +namespace Catch { + + // An optional type + template<typename T> + class Option { + public: + Option() : nullableValue( nullptr ) {} + Option( T const& _value ) + : nullableValue( new( storage ) T( _value ) ) + {} + Option( Option const& _other ) + : nullableValue( _other ? new( storage ) T( *_other ) : nullptr ) + {} + + ~Option() { + reset(); + } + + Option& operator= ( Option const& _other ) { + if( &_other != this ) { + reset(); + if( _other ) + nullableValue = new( storage ) T( *_other ); + } + return *this; + } + Option& operator = ( T const& _value ) { + reset(); + nullableValue = new( storage ) T( _value ); + return *this; + } + + void reset() { + if( nullableValue ) + nullableValue->~T(); + nullableValue = nullptr; + } + + T& operator*() { return *nullableValue; } + T const& operator*() const { return *nullableValue; } + T* operator->() { return nullableValue; } + const T* operator->() const { return nullableValue; } + + T valueOr( T const& defaultValue ) const { + return nullableValue ? *nullableValue : defaultValue; + } + + bool some() const { return nullableValue != nullptr; } + bool none() const { return nullableValue == nullptr; } + + bool operator !() const { return nullableValue == nullptr; } + explicit operator bool() const { + return some(); + } + + private: + T *nullableValue; + alignas(alignof(T)) char storage[sizeof(T)]; + }; + +} // end namespace Catch + +// end catch_option.hpp #include <iosfwd> #include <string> #include <vector> @@ -4249,10 +4379,14 @@ namespace Catch { virtual std::vector<std::string> const& getTestsOrTags() const = 0; virtual RunTests::InWhatOrder runOrder() const = 0; virtual unsigned int rngSeed() const = 0; - virtual int benchmarkResolutionMultiple() const = 0; virtual UseColour::YesOrNo useColour() const = 0; virtual std::vector<std::string> const& getSectionsToRun() const = 0; virtual Verbosity verbosity() const = 0; + + virtual bool benchmarkNoAnalysis() const = 0; + virtual int benchmarkSamples() const = 0; + virtual double benchmarkConfidenceInterval() const = 0; + virtual unsigned int benchmarkResamples() const = 0; }; using IConfigPtr = std::shared_ptr<IConfig const>; @@ -4894,7 +5028,11 @@ namespace Catch { int abortAfter = -1; unsigned int rngSeed = 0; - int benchmarkResolutionMultiple = 100; + + bool benchmarkNoAnalysis = false; + unsigned int benchmarkSamples = 100; + double benchmarkConfidenceInterval = 0.95; + unsigned int benchmarkResamples = 100000; Verbosity verbosity = Verbosity::Normal; WarnAbout::What warnings = WarnAbout::Nothing; @@ -4951,12 +5089,15 @@ namespace Catch { ShowDurations::OrNot showDurations() const override; RunTests::InWhatOrder runOrder() const override; unsigned int rngSeed() const override; - int benchmarkResolutionMultiple() const override; UseColour::YesOrNo useColour() const override; bool shouldDebugBreak() const override; int abortAfter() const override; bool showInvisibles() const override; Verbosity verbosity() const override; + bool benchmarkNoAnalysis() const override; + int benchmarkSamples() const override; + double benchmarkConfidenceInterval() const override; + unsigned int benchmarkResamples() const override; private: @@ -5017,76 +5158,59 @@ namespace Catch { } // end namespace Catch // end catch_assertionresult.h -// start catch_option.hpp - -namespace Catch { +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) +// start catch_estimate.hpp - // An optional type - template<typename T> - class Option { - public: - Option() : nullableValue( nullptr ) {} - Option( T const& _value ) - : nullableValue( new( storage ) T( _value ) ) - {} - Option( Option const& _other ) - : nullableValue( _other ? new( storage ) T( *_other ) : nullptr ) - {} + // Statistics estimates - ~Option() { - reset(); - } - Option& operator= ( Option const& _other ) { - if( &_other != this ) { - reset(); - if( _other ) - nullableValue = new( storage ) T( *_other ); +namespace Catch { + namespace Benchmark { + template <typename Duration> + struct Estimate { + Duration point; + Duration lower_bound; + Duration upper_bound; + double confidence_interval; + + template <typename Duration2> + operator Estimate<Duration2>() const { + return { point, lower_bound, upper_bound, confidence_interval }; } - return *this; - } - Option& operator = ( T const& _value ) { - reset(); - nullableValue = new( storage ) T( _value ); - return *this; - } - - void reset() { - if( nullableValue ) - nullableValue->~T(); - nullableValue = nullptr; - } - - T& operator*() { return *nullableValue; } - T const& operator*() const { return *nullableValue; } - T* operator->() { return nullableValue; } - const T* operator->() const { return nullableValue; } - - T valueOr( T const& defaultValue ) const { - return nullableValue ? *nullableValue : defaultValue; - } + }; + } // namespace Benchmark +} // namespace Catch - bool some() const { return nullableValue != nullptr; } - bool none() const { return nullableValue == nullptr; } +// end catch_estimate.hpp +// start catch_outlier_classification.hpp - bool operator !() const { return nullableValue == nullptr; } - explicit operator bool() const { - return some(); - } +// Outlier information - private: - T *nullableValue; - alignas(alignof(T)) char storage[sizeof(T)]; - }; +namespace Catch { + namespace Benchmark { + struct OutlierClassification { + int samples_seen = 0; + int low_severe = 0; // more than 3 times IQR below Q1 + int low_mild = 0; // 1.5 to 3 times IQR below Q1 + int high_mild = 0; // 1.5 to 3 times IQR above Q3 + int high_severe = 0; // more than 3 times IQR above Q3 + + int total() const { + return low_severe + low_mild + high_mild + high_severe; + } + }; + } // namespace Benchmark +} // namespace Catch -} // end namespace Catch +// end catch_outlier_classification.hpp +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING -// end catch_option.hpp #include <string> #include <iosfwd> #include <map> #include <set> #include <memory> +#include <algorithm> namespace Catch { @@ -5222,14 +5346,43 @@ namespace Catch { bool aborting; }; +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) struct BenchmarkInfo { std::string name; + double estimatedDuration; + int iterations; + int samples; + unsigned int resamples; + double clockResolution; + double clockCost; }; + + template <class Duration> struct BenchmarkStats { BenchmarkInfo info; - std::size_t iterations; - uint64_t elapsedTimeInNanoseconds; + + std::vector<Duration> samples; + Benchmark::Estimate<Duration> mean; + Benchmark::Estimate<Duration> standardDeviation; + Benchmark::OutlierClassification outliers; + double outlierVariance; + + template <typename Duration2> + operator BenchmarkStats<Duration2>() const { + std::vector<Duration2> samples2; + samples2.reserve(samples.size()); + std::transform(samples.begin(), samples.end(), std::back_inserter(samples2), [](Duration d) { return Duration2(d); }); + return { + info, + std::move(samples2), + mean, + standardDeviation, + outliers, + outlierVariance, + }; + } }; +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING struct IStreamingReporter { virtual ~IStreamingReporter() = default; @@ -5248,17 +5401,18 @@ namespace Catch { virtual void testCaseStarting( TestCaseInfo const& testInfo ) = 0; virtual void sectionStarting( SectionInfo const& sectionInfo ) = 0; - // *** experimental *** +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) + virtual void benchmarkPreparing( std::string const& ) {} virtual void benchmarkStarting( BenchmarkInfo const& ) {} + virtual void benchmarkEnded( BenchmarkStats<> const& ) {} + virtual void benchmarkFailed( std::string const& ) {} +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING virtual void assertionStarting( AssertionInfo const& assertionInfo ) = 0; // The return value indicates if the messages buffer should be cleared: virtual bool assertionEnded( AssertionStats const& assertionStats ) = 0; - // *** experimental *** - virtual void benchmarkEnded( BenchmarkStats const& ) {} - virtual void sectionEnded( SectionStats const& sectionStats ) = 0; virtual void testCaseEnded( TestCaseStats const& testCaseStats ) = 0; virtual void testGroupEnded( TestGroupStats const& testGroupStats ) = 0; @@ -5743,8 +5897,12 @@ namespace Catch { void sectionStarting(SectionInfo const& _sectionInfo) override; void sectionEnded(SectionStats const& _sectionStats) override; +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) + void benchmarkPreparing(std::string const& name) override; void benchmarkStarting(BenchmarkInfo const& info) override; - void benchmarkEnded(BenchmarkStats const& stats) override; + void benchmarkEnded(BenchmarkStats<> const& stats) override; + void benchmarkFailed(std::string const& error) override; +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING void testCaseEnded(TestCaseStats const& _testCaseStats) override; void testGroupEnded(TestGroupStats const& _testGroupStats) override; @@ -5862,121 +6020,1073 @@ namespace Catch { XmlWriter& writeBlankLine(); - void ensureTagClosed(); + void ensureTagClosed(); + + private: + + void writeDeclaration(); + + void newlineIfNecessary(); + + bool m_tagIsOpen = false; + bool m_needsNewline = false; + std::vector<std::string> m_tags; + std::string m_indent; + std::ostream& m_os; + }; + +} + +// end catch_xmlwriter.h +namespace Catch { + + class JunitReporter : public CumulativeReporterBase<JunitReporter> { + public: + JunitReporter(ReporterConfig const& _config); + + ~JunitReporter() override; + + static std::string getDescription(); + + void noMatchingTestCases(std::string const& /*spec*/) override; + + void testRunStarting(TestRunInfo const& runInfo) override; + + void testGroupStarting(GroupInfo const& groupInfo) override; + + void testCaseStarting(TestCaseInfo const& testCaseInfo) override; + bool assertionEnded(AssertionStats const& assertionStats) override; + + void testCaseEnded(TestCaseStats const& testCaseStats) override; + + void testGroupEnded(TestGroupStats const& testGroupStats) override; + + void testRunEndedCumulative() override; + + void writeGroup(TestGroupNode const& groupNode, double suiteTime); + + void writeTestCase(TestCaseNode const& testCaseNode); + + void writeSection(std::string const& className, + std::string const& rootName, + SectionNode const& sectionNode); + + void writeAssertions(SectionNode const& sectionNode); + void writeAssertion(AssertionStats const& stats); + + XmlWriter xml; + Timer suiteTimer; + std::string stdOutForSuite; + std::string stdErrForSuite; + unsigned int unexpectedExceptions = 0; + bool m_okToFail = false; + }; + +} // end namespace Catch + +// end catch_reporter_junit.h +// start catch_reporter_xml.h + +namespace Catch { + class XmlReporter : public StreamingReporterBase<XmlReporter> { + public: + XmlReporter(ReporterConfig const& _config); + + ~XmlReporter() override; + + static std::string getDescription(); + + virtual std::string getStylesheetRef() const; + + void writeSourceInfo(SourceLineInfo const& sourceInfo); + + public: // StreamingReporterBase + + void noMatchingTestCases(std::string const& s) override; + + void testRunStarting(TestRunInfo const& testInfo) override; + + void testGroupStarting(GroupInfo const& groupInfo) override; + + void testCaseStarting(TestCaseInfo const& testInfo) override; + + void sectionStarting(SectionInfo const& sectionInfo) override; + + void assertionStarting(AssertionInfo const&) override; + + bool assertionEnded(AssertionStats const& assertionStats) override; + + void sectionEnded(SectionStats const& sectionStats) override; + + void testCaseEnded(TestCaseStats const& testCaseStats) override; + + void testGroupEnded(TestGroupStats const& testGroupStats) override; + + void testRunEnded(TestRunStats const& testRunStats) override; + +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) + void benchmarkStarting(BenchmarkInfo const&) override; + void benchmarkEnded(BenchmarkStats<> const&) override; + void benchmarkFailed(std::string const&) override; +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING + + private: + Timer m_testCaseTimer; + XmlWriter m_xml; + int m_sectionDepth = 0; + }; + +} // end namespace Catch + +// end catch_reporter_xml.h + +// end catch_external_interfaces.h +#endif + +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) +// start catch_benchmark.hpp + + // Benchmark + +// start catch_chronometer.hpp + +// User-facing chronometer + + +// start catch_clock.hpp + +// Clocks + + +#include <chrono> +#include <ratio> + +namespace Catch { + namespace Benchmark { + template <typename Clock> + using ClockDuration = typename Clock::duration; + template <typename Clock> + using FloatDuration = std::chrono::duration<double, typename Clock::period>; + + template <typename Clock> + using TimePoint = typename Clock::time_point; + + using default_clock = std::chrono::steady_clock; + + template <typename Clock> + struct now { + TimePoint<Clock> operator()() const { + return Clock::now(); + } + }; + + using fp_seconds = std::chrono::duration<double, std::ratio<1>>; + } // namespace Benchmark +} // namespace Catch + +// end catch_clock.hpp +// start catch_optimizer.hpp + + // Hinting the optimizer + + +#if defined(_MSC_VER) +# include <atomic> // atomic_thread_fence +#endif + +namespace Catch { + namespace Benchmark { +#if defined(__GNUC__) || defined(__clang__) + template <typename T> + inline void keep_memory(T* p) { + asm volatile("" : : "g"(p) : "memory"); + } + inline void keep_memory() { + asm volatile("" : : : "memory"); + } + + namespace Detail { + inline void optimizer_barrier() { keep_memory(); } + } // namespace Detail +#elif defined(_MSC_VER) + +#pragma optimize("", off) + template <typename T> + inline void keep_memory(T* p) { + // thanks @milleniumbug + *reinterpret_cast<char volatile*>(p) = *reinterpret_cast<char const volatile*>(p); + } + // TODO equivalent keep_memory() +#pragma optimize("", on) + + namespace Detail { + inline void optimizer_barrier() { + std::atomic_thread_fence(std::memory_order_seq_cst); + } + } // namespace Detail + +#endif + + template <typename T> + inline void deoptimize_value(T&& x) { + keep_memory(&x); + } + + template <typename Fn, typename... Args> + inline auto invoke_deoptimized(Fn&& fn, Args&&... args) -> typename std::enable_if<!std::is_same<void, decltype(fn(args...))>::value>::type { + deoptimize_value(std::forward<Fn>(fn) (std::forward<Args...>(args...))); + } + + template <typename Fn, typename... Args> + inline auto invoke_deoptimized(Fn&& fn, Args&&... args) -> typename std::enable_if<std::is_same<void, decltype(fn(args...))>::value>::type { + std::forward<Fn>(fn) (std::forward<Args...>(args...)); + } + } // namespace Benchmark +} // namespace Catch + +// end catch_optimizer.hpp +// start catch_complete_invoke.hpp + +// Invoke with a special case for void + + +#include <type_traits> +#include <utility> + +namespace Catch { + namespace Benchmark { + namespace Detail { + template <typename T> + struct CompleteType { using type = T; }; + template <> + struct CompleteType<void> { struct type {}; }; + + template <typename T> + using CompleteType_t = typename CompleteType<T>::type; + + template <typename Result> + struct CompleteInvoker { + template <typename Fun, typename... Args> + static Result invoke(Fun&& fun, Args&&... args) { + return std::forward<Fun>(fun)(std::forward<Args>(args)...); + } + }; + template <> + struct CompleteInvoker<void> { + template <typename Fun, typename... Args> + static CompleteType_t<void> invoke(Fun&& fun, Args&&... args) { + std::forward<Fun>(fun)(std::forward<Args>(args)...); + return {}; + } + }; + template <typename Sig> + using ResultOf_t = typename std::result_of<Sig>::type; + + // invoke and not return void :( + template <typename Fun, typename... Args> + CompleteType_t<ResultOf_t<Fun(Args...)>> complete_invoke(Fun&& fun, Args&&... args) { + return CompleteInvoker<ResultOf_t<Fun(Args...)>>::invoke(std::forward<Fun>(fun), std::forward<Args>(args)...); + } + + const std::string benchmarkErrorMsg = "a benchmark failed to run successfully"; + } // namespace Detail + + template <typename Fun> + Detail::CompleteType_t<Detail::ResultOf_t<Fun()>> user_code(Fun&& fun) { + CATCH_TRY{ + return Detail::complete_invoke(std::forward<Fun>(fun)); + } CATCH_CATCH_ALL{ + getResultCapture().benchmarkFailed(translateActiveException()); + CATCH_RUNTIME_ERROR(Detail::benchmarkErrorMsg); + } + } + } // namespace Benchmark +} // namespace Catch + +// end catch_complete_invoke.hpp +namespace Catch { + namespace Benchmark { + namespace Detail { + struct ChronometerConcept { + virtual void start() = 0; + virtual void finish() = 0; + virtual ~ChronometerConcept() = default; + }; + template <typename Clock> + struct ChronometerModel final : public ChronometerConcept { + void start() override { started = Clock::now(); } + void finish() override { finished = Clock::now(); } + + ClockDuration<Clock> elapsed() const { return finished - started; } + + TimePoint<Clock> started; + TimePoint<Clock> finished; + }; + } // namespace Detail + + struct Chronometer { + public: + template <typename Fun> + void measure(Fun&& fun) { measure(std::forward<Fun>(fun), is_callable<Fun(int)>()); } + + int runs() const { return k; } + + Chronometer(Detail::ChronometerConcept& meter, int k) + : impl(&meter) + , k(k) {} + + private: + template <typename Fun> + void measure(Fun&& fun, std::false_type) { + measure([&fun](int) { return fun(); }, std::true_type()); + } + + template <typename Fun> + void measure(Fun&& fun, std::true_type) { + Detail::optimizer_barrier(); + impl->start(); + for (int i = 0; i < k; ++i) invoke_deoptimized(fun, i); + impl->finish(); + Detail::optimizer_barrier(); + } + + Detail::ChronometerConcept* impl; + int k; + }; + } // namespace Benchmark +} // namespace Catch + +// end catch_chronometer.hpp +// start catch_environment.hpp + +// Environment information + + +namespace Catch { + namespace Benchmark { + template <typename Duration> + struct EnvironmentEstimate { + Duration mean; + OutlierClassification outliers; + + template <typename Duration2> + operator EnvironmentEstimate<Duration2>() const { + return { mean, outliers }; + } + }; + template <typename Clock> + struct Environment { + using clock_type = Clock; + EnvironmentEstimate<FloatDuration<Clock>> clock_resolution; + EnvironmentEstimate<FloatDuration<Clock>> clock_cost; + }; + } // namespace Benchmark +} // namespace Catch + +// end catch_environment.hpp +// start catch_execution_plan.hpp + + // Execution plan + + +// start catch_benchmark_function.hpp + + // Dumb std::function implementation for consistent call overhead + + +#include <cassert> +#include <type_traits> +#include <utility> +#include <memory> + +namespace Catch { + namespace Benchmark { + namespace Detail { + template <typename T> + using Decay = typename std::decay<T>::type; + template <typename T, typename U> + struct is_related + : std::is_same<Decay<T>, Decay<U>> {}; + + /// We need to reinvent std::function because every piece of code that might add overhead + /// in a measurement context needs to have consistent performance characteristics so that we + /// can account for it in the measurement. + /// Implementations of std::function with optimizations that aren't always applicable, like + /// small buffer optimizations, are not uncommon. + /// This is effectively an implementation of std::function without any such optimizations; + /// it may be slow, but it is consistently slow. + struct BenchmarkFunction { + private: + struct callable { + virtual void call(Chronometer meter) const = 0; + virtual callable* clone() const = 0; + virtual ~callable() = default; + }; + template <typename Fun> + struct model : public callable { + model(Fun&& fun) : fun(std::move(fun)) {} + model(Fun const& fun) : fun(fun) {} + + model<Fun>* clone() const override { return new model<Fun>(*this); } + + void call(Chronometer meter) const override { + call(meter, is_callable<Fun(Chronometer)>()); + } + void call(Chronometer meter, std::true_type) const { + fun(meter); + } + void call(Chronometer meter, std::false_type) const { + meter.measure(fun); + } + + Fun fun; + }; + + struct do_nothing { void operator()() const {} }; + + template <typename T> + BenchmarkFunction(model<T>* c) : f(c) {} + + public: + BenchmarkFunction() + : f(new model<do_nothing>{ {} }) {} + + template <typename Fun, + typename std::enable_if<!is_related<Fun, BenchmarkFunction>::value, int>::type = 0> + BenchmarkFunction(Fun&& fun) + : f(new model<typename std::decay<Fun>::type>(std::forward<Fun>(fun))) {} + + BenchmarkFunction(BenchmarkFunction&& that) + : f(std::move(that.f)) {} + + BenchmarkFunction(BenchmarkFunction const& that) + : f(that.f->clone()) {} + + BenchmarkFunction& operator=(BenchmarkFunction&& that) { + f = std::move(that.f); + return *this; + } + + BenchmarkFunction& operator=(BenchmarkFunction const& that) { + f.reset(that.f->clone()); + return *this; + } + + void operator()(Chronometer meter) const { f->call(meter); } + + private: + std::unique_ptr<callable> f; + }; + } // namespace Detail + } // namespace Benchmark +} // namespace Catch + +// end catch_benchmark_function.hpp +// start catch_repeat.hpp + +// repeat algorithm + + +#include <type_traits> +#include <utility> + +namespace Catch { + namespace Benchmark { + namespace Detail { + template <typename Fun> + struct repeater { + void operator()(int k) const { + for (int i = 0; i < k; ++i) { + fun(); + } + } + Fun fun; + }; + template <typename Fun> + repeater<typename std::decay<Fun>::type> repeat(Fun&& fun) { + return { std::forward<Fun>(fun) }; + } + } // namespace Detail + } // namespace Benchmark +} // namespace Catch + +// end catch_repeat.hpp +// start catch_run_for_at_least.hpp + +// Run a function for a minimum amount of time + + +// start catch_measure.hpp + +// Measure + + +// start catch_timing.hpp + +// Timing + + +#include <tuple> +#include <type_traits> + +namespace Catch { + namespace Benchmark { + template <typename Duration, typename Result> + struct Timing { + Duration elapsed; + Result result; + int iterations; + }; + template <typename Clock, typename Sig> + using TimingOf = Timing<ClockDuration<Clock>, Detail::CompleteType_t<Detail::ResultOf_t<Sig>>>; + } // namespace Benchmark +} // namespace Catch + +// end catch_timing.hpp +#include <utility> + +namespace Catch { + namespace Benchmark { + namespace Detail { + template <typename Clock, typename Fun, typename... Args> + TimingOf<Clock, Fun(Args...)> measure(Fun&& fun, Args&&... args) { + auto start = Clock::now(); + auto&& r = Detail::complete_invoke(fun, std::forward<Args>(args)...); + auto end = Clock::now(); + auto delta = end - start; + return { delta, std::forward<decltype(r)>(r), 1 }; + } + } // namespace Detail + } // namespace Benchmark +} // namespace Catch + +// end catch_measure.hpp +#include <utility> +#include <type_traits> + +namespace Catch { + namespace Benchmark { + namespace Detail { + template <typename Clock, typename Fun> + TimingOf<Clock, Fun(int)> measure_one(Fun&& fun, int iters, std::false_type) { + return Detail::measure<Clock>(fun, iters); + } + template <typename Clock, typename Fun> + TimingOf<Clock, Fun(Chronometer)> measure_one(Fun&& fun, int iters, std::true_type) { + Detail::ChronometerModel<Clock> meter; + auto&& result = Detail::complete_invoke(fun, Chronometer(meter, iters)); + + return { meter.elapsed(), std::move(result), iters }; + } + + template <typename Clock, typename Fun> + using run_for_at_least_argument_t = typename std::conditional<is_callable<Fun(Chronometer)>::value, Chronometer, int>::type; + + struct optimized_away_error : std::exception { + const char* what() const noexcept override { + return "could not measure benchmark, maybe it was optimized away"; + } + }; + + template <typename Clock, typename Fun> + TimingOf<Clock, Fun(run_for_at_least_argument_t<Clock, Fun>)> run_for_at_least(ClockDuration<Clock> how_long, int seed, Fun&& fun) { + auto iters = seed; + while (iters < (1 << 30)) { + auto&& Timing = measure_one<Clock>(fun, iters, is_callable<Fun(Chronometer)>()); + + if (Timing.elapsed >= how_long) { + return { Timing.elapsed, std::move(Timing.result), iters }; + } + iters *= 2; + } + throw optimized_away_error{}; + } + } // namespace Detail + } // namespace Benchmark +} // namespace Catch + +// end catch_run_for_at_least.hpp +#include <algorithm> + +namespace Catch { + namespace Benchmark { + template <typename Duration> + struct ExecutionPlan { + int iterations_per_sample; + Duration estimated_duration; + Detail::BenchmarkFunction benchmark; + Duration warmup_time; + int warmup_iterations; + + template <typename Duration2> + operator ExecutionPlan<Duration2>() const { + return { iterations_per_sample, estimated_duration, benchmark, warmup_time, warmup_iterations }; + } + + template <typename Clock> + std::vector<FloatDuration<Clock>> run(const IConfig &cfg, Environment<FloatDuration<Clock>> env) const { + // warmup a bit + Detail::run_for_at_least<Clock>(std::chrono::duration_cast<ClockDuration<Clock>>(warmup_time), warmup_iterations, Detail::repeat(now<Clock>{})); + + std::vector<FloatDuration<Clock>> times; + times.reserve(cfg.benchmarkSamples()); + std::generate_n(std::back_inserter(times), cfg.benchmarkSamples(), [this, env] { + Detail::ChronometerModel<Clock> model; + this->benchmark(Chronometer(model, iterations_per_sample)); + auto sample_time = model.elapsed() - env.clock_cost.mean; + if (sample_time < FloatDuration<Clock>::zero()) sample_time = FloatDuration<Clock>::zero(); + return sample_time / iterations_per_sample; + }); + return times; + } + }; + } // namespace Benchmark +} // namespace Catch + +// end catch_execution_plan.hpp +// start catch_estimate_clock.hpp + + // Environment measurement + + +// start catch_stats.hpp + +// Statistical analysis tools + + +#include <algorithm> +#include <functional> +#include <vector> +#include <numeric> +#include <tuple> +#include <cmath> +#include <utility> +#include <cstddef> + +namespace Catch { + namespace Benchmark { + namespace Detail { + using sample = std::vector<double>; + + double weighted_average_quantile(int k, int q, std::vector<double>::iterator first, std::vector<double>::iterator last); + + template <typename Iterator> + OutlierClassification classify_outliers(Iterator first, Iterator last) { + std::vector<double> copy(first, last); + + auto q1 = weighted_average_quantile(1, 4, copy.begin(), copy.end()); + auto q3 = weighted_average_quantile(3, 4, copy.begin(), copy.end()); + auto iqr = q3 - q1; + auto los = q1 - (iqr * 3.); + auto lom = q1 - (iqr * 1.5); + auto him = q3 + (iqr * 1.5); + auto his = q3 + (iqr * 3.); + + OutlierClassification o; + for (; first != last; ++first) { + auto&& t = *first; + if (t < los) ++o.low_severe; + else if (t < lom) ++o.low_mild; + else if (t > his) ++o.high_severe; + else if (t > him) ++o.high_mild; + ++o.samples_seen; + } + return o; + } - private: + template <typename Iterator> + double mean(Iterator first, Iterator last) { + auto count = last - first; + double sum = std::accumulate(first, last, 0.); + return sum / count; + } - void writeDeclaration(); + template <typename URng, typename Iterator, typename Estimator> + sample resample(URng& rng, int resamples, Iterator first, Iterator last, Estimator& estimator) { + auto n = last - first; + std::uniform_int_distribution<decltype(n)> dist(0, n - 1); + + sample out; + out.reserve(resamples); + std::generate_n(std::back_inserter(out), resamples, [n, first, &estimator, &dist, &rng] { + std::vector<double> resampled; + resampled.reserve(n); + std::generate_n(std::back_inserter(resampled), n, [first, &dist, &rng] { return first[dist(rng)]; }); + return estimator(resampled.begin(), resampled.end()); + }); + std::sort(out.begin(), out.end()); + return out; + } - void newlineIfNecessary(); + template <typename Estimator, typename Iterator> + sample jackknife(Estimator&& estimator, Iterator first, Iterator last) { + auto n = last - first; + auto second = std::next(first); + sample results; + results.reserve(n); - bool m_tagIsOpen = false; - bool m_needsNewline = false; - std::vector<std::string> m_tags; - std::string m_indent; - std::ostream& m_os; - }; + for (auto it = first; it != last; ++it) { + std::iter_swap(it, first); + results.push_back(estimator(second, last)); + } -} + return results; + } -// end catch_xmlwriter.h -namespace Catch { + inline double normal_cdf(double x) { + return std::erfc(-x / std::sqrt(2.0)) / 2.0; + } - class JunitReporter : public CumulativeReporterBase<JunitReporter> { - public: - JunitReporter(ReporterConfig const& _config); + double erfc_inv(double x); + + double normal_quantile(double p); + + template <typename Iterator, typename Estimator> + Estimate<double> bootstrap(double confidence_level, Iterator first, Iterator last, sample const& resample, Estimator&& estimator) { + auto n_samples = last - first; + + double point = estimator(first, last); + // Degenerate case with a single sample + if (n_samples == 1) return { point, point, point, confidence_level }; + + sample jack = jackknife(estimator, first, last); + double jack_mean = mean(jack.begin(), jack.end()); + double sum_squares, sum_cubes; + std::tie(sum_squares, sum_cubes) = std::accumulate(jack.begin(), jack.end(), std::make_pair(0., 0.), [jack_mean](std::pair<double, double> sqcb, double x) -> std::pair<double, double> { + auto d = jack_mean - x; + auto d2 = d * d; + auto d3 = d2 * d; + return { sqcb.first + d2, sqcb.second + d3 }; + }); + + double accel = sum_cubes / (6 * std::pow(sum_squares, 1.5)); + int n = static_cast<int>(resample.size()); + double prob_n = std::count_if(resample.begin(), resample.end(), [point](double x) { return x < point; }) / (double)n; + // degenerate case with uniform samples + if (prob_n == 0) return { point, point, point, confidence_level }; + + double bias = normal_quantile(prob_n); + double z1 = normal_quantile((1. - confidence_level) / 2.); + + auto cumn = [n](double x) -> int { + return std::lround(normal_cdf(x) * n); }; + auto a = [bias, accel](double b) { return bias + b / (1. - accel * b); }; + double b1 = bias + z1; + double b2 = bias - z1; + double a1 = a(b1); + double a2 = a(b2); + auto lo = std::max(cumn(a1), 0); + auto hi = std::min(cumn(a2), n - 1); + + return { point, resample[lo], resample[hi], confidence_level }; + } - ~JunitReporter() override; + double outlier_variance(Estimate<double> mean, Estimate<double> stddev, int n); - static std::string getDescription(); + struct bootstrap_analysis { + Estimate<double> mean; + Estimate<double> standard_deviation; + double outlier_variance; + }; - void noMatchingTestCases(std::string const& /*spec*/) override; + bootstrap_analysis analyse_samples(double confidence_level, int n_resamples, std::vector<double>::iterator first, std::vector<double>::iterator last); + } // namespace Detail + } // namespace Benchmark +} // namespace Catch - void testRunStarting(TestRunInfo const& runInfo) override; +// end catch_stats.hpp +#include <algorithm> +#include <iterator> +#include <tuple> +#include <vector> +#include <cmath> - void testGroupStarting(GroupInfo const& groupInfo) override; +namespace Catch { + namespace Benchmark { + namespace Detail { + template <typename Clock> + std::vector<double> resolution(int k) { + std::vector<TimePoint<Clock>> times; + times.reserve(k + 1); + std::generate_n(std::back_inserter(times), k + 1, now<Clock>{}); + + std::vector<double> deltas; + deltas.reserve(k); + std::transform(std::next(times.begin()), times.end(), times.begin(), + std::back_inserter(deltas), + [](TimePoint<Clock> a, TimePoint<Clock> b) { return static_cast<double>((a - b).count()); }); + + return deltas; + } - void testCaseStarting(TestCaseInfo const& testCaseInfo) override; - bool assertionEnded(AssertionStats const& assertionStats) override; + const auto warmup_iterations = 10000; + const auto warmup_time = std::chrono::milliseconds(100); + const auto minimum_ticks = 1000; + const auto warmup_seed = 10000; + const auto clock_resolution_estimation_time = std::chrono::milliseconds(500); + const auto clock_cost_estimation_time_limit = std::chrono::seconds(1); + const auto clock_cost_estimation_tick_limit = 100000; + const auto clock_cost_estimation_time = std::chrono::milliseconds(10); + const auto clock_cost_estimation_iterations = 10000; + + template <typename Clock> + int warmup() { + return run_for_at_least<Clock>(std::chrono::duration_cast<ClockDuration<Clock>>(warmup_time), warmup_seed, &resolution<Clock>) + .iterations; + } + template <typename Clock> + EnvironmentEstimate<FloatDuration<Clock>> estimate_clock_resolution(int iterations) { + auto r = run_for_at_least<Clock>(std::chrono::duration_cast<ClockDuration<Clock>>(clock_resolution_estimation_time), iterations, &resolution<Clock>) + .result; + return { + FloatDuration<Clock>(mean(r.begin(), r.end())), + classify_outliers(r.begin(), r.end()), + }; + } + template <typename Clock> + EnvironmentEstimate<FloatDuration<Clock>> estimate_clock_cost(FloatDuration<Clock> resolution) { + auto time_limit = std::min(resolution * clock_cost_estimation_tick_limit, FloatDuration<Clock>(clock_cost_estimation_time_limit)); + auto time_clock = [](int k) { + return Detail::measure<Clock>([k] { + for (int i = 0; i < k; ++i) { + volatile auto ignored = Clock::now(); + (void)ignored; + } + }).elapsed; + }; + time_clock(1); + int iters = clock_cost_estimation_iterations; + auto&& r = run_for_at_least<Clock>(std::chrono::duration_cast<ClockDuration<Clock>>(clock_cost_estimation_time), iters, time_clock); + std::vector<double> times; + int nsamples = static_cast<int>(std::ceil(time_limit / r.elapsed)); + times.reserve(nsamples); + std::generate_n(std::back_inserter(times), nsamples, [time_clock, &r] { + return static_cast<double>((time_clock(r.iterations) / r.iterations).count()); + }); + return { + FloatDuration<Clock>(mean(times.begin(), times.end())), + classify_outliers(times.begin(), times.end()), + }; + } - void testCaseEnded(TestCaseStats const& testCaseStats) override; + template <typename Clock> + Environment<FloatDuration<Clock>> measure_environment() { + static Environment<FloatDuration<Clock>>* env = nullptr; + if (env) { + return *env; + } - void testGroupEnded(TestGroupStats const& testGroupStats) override; + auto iters = Detail::warmup<Clock>(); + auto resolution = Detail::estimate_clock_resolution<Clock>(iters); + auto cost = Detail::estimate_clock_cost<Clock>(resolution.mean); - void testRunEndedCumulative() override; + env = new Environment<FloatDuration<Clock>>{ resolution, cost }; + return *env; + } + } // namespace Detail + } // namespace Benchmark +} // namespace Catch - void writeGroup(TestGroupNode const& groupNode, double suiteTime); +// end catch_estimate_clock.hpp +// start catch_analyse.hpp - void writeTestCase(TestCaseNode const& testCaseNode); + // Run and analyse one benchmark - void writeSection(std::string const& className, - std::string const& rootName, - SectionNode const& sectionNode); - void writeAssertions(SectionNode const& sectionNode); - void writeAssertion(AssertionStats const& stats); +// start catch_sample_analysis.hpp - XmlWriter xml; - Timer suiteTimer; - std::string stdOutForSuite; - std::string stdErrForSuite; - unsigned int unexpectedExceptions = 0; - bool m_okToFail = false; - }; +// Benchmark results -} // end namespace Catch -// end catch_reporter_junit.h -// start catch_reporter_xml.h +#include <algorithm> +#include <vector> +#include <string> +#include <iterator> namespace Catch { - class XmlReporter : public StreamingReporterBase<XmlReporter> { - public: - XmlReporter(ReporterConfig const& _config); - - ~XmlReporter() override; + namespace Benchmark { + template <typename Duration> + struct SampleAnalysis { + std::vector<Duration> samples; + Estimate<Duration> mean; + Estimate<Duration> standard_deviation; + OutlierClassification outliers; + double outlier_variance; + + template <typename Duration2> + operator SampleAnalysis<Duration2>() const { + std::vector<Duration2> samples2; + samples2.reserve(samples.size()); + std::transform(samples.begin(), samples.end(), std::back_inserter(samples2), [](Duration d) { return Duration2(d); }); + return { + std::move(samples2), + mean, + standard_deviation, + outliers, + outlier_variance, + }; + } + }; + } // namespace Benchmark +} // namespace Catch - static std::string getDescription(); +// end catch_sample_analysis.hpp +#include <algorithm> +#include <iterator> +#include <vector> - virtual std::string getStylesheetRef() const; +namespace Catch { + namespace Benchmark { + namespace Detail { + template <typename Duration, typename Iterator> + SampleAnalysis<Duration> analyse(const IConfig &cfg, Environment<Duration>, Iterator first, Iterator last) { + if (!cfg.benchmarkNoAnalysis()) { + std::vector<double> samples; + samples.reserve(last - first); + std::transform(first, last, std::back_inserter(samples), [](Duration d) { return d.count(); }); + + auto analysis = Catch::Benchmark::Detail::analyse_samples(cfg.benchmarkConfidenceInterval(), cfg.benchmarkResamples(), samples.begin(), samples.end()); + auto outliers = Catch::Benchmark::Detail::classify_outliers(samples.begin(), samples.end()); + + auto wrap_estimate = [](Estimate<double> e) { + return Estimate<Duration> { + Duration(e.point), + Duration(e.lower_bound), + Duration(e.upper_bound), + e.confidence_interval, + }; + }; + std::vector<Duration> samples2; + samples2.reserve(samples.size()); + std::transform(samples.begin(), samples.end(), std::back_inserter(samples2), [](double d) { return Duration(d); }); + return { + std::move(samples2), + wrap_estimate(analysis.mean), + wrap_estimate(analysis.standard_deviation), + outliers, + analysis.outlier_variance, + }; + } else { + std::vector<Duration> samples; + samples.reserve(last - first); + + Duration mean = Duration(0); + int i = 0; + for (auto it = first; it < last; ++it, ++i) { + samples.push_back(Duration(*it)); + mean += Duration(*it); + } + mean /= i; + + return { + std::move(samples), + Estimate<Duration>{mean, mean, mean, 0.0}, + Estimate<Duration>{Duration(0), Duration(0), Duration(0), 0.0}, + OutlierClassification{}, + 0.0 + }; + } + } + } // namespace Detail + } // namespace Benchmark +} // namespace Catch - void writeSourceInfo(SourceLineInfo const& sourceInfo); +// end catch_analyse.hpp +#include <algorithm> +#include <functional> +#include <string> +#include <vector> +#include <cmath> - public: // StreamingReporterBase +namespace Catch { + namespace Benchmark { + struct Benchmark { + Benchmark(std::string &&name) + : name(std::move(name)) {} + + template <class FUN> + Benchmark(std::string &&name, FUN &&func) + : fun(std::move(func)), name(std::move(name)) {} + + template <typename Clock> + ExecutionPlan<FloatDuration<Clock>> prepare(const IConfig &cfg, Environment<FloatDuration<Clock>> env) const { + auto min_time = env.clock_resolution.mean * Detail::minimum_ticks; + auto run_time = std::max(min_time, std::chrono::duration_cast<decltype(min_time)>(Detail::warmup_time)); + auto&& test = Detail::run_for_at_least<Clock>(std::chrono::duration_cast<ClockDuration<Clock>>(run_time), 1, fun); + int new_iters = static_cast<int>(std::ceil(min_time * test.iterations / test.elapsed)); + return { new_iters, test.elapsed / test.iterations * new_iters * cfg.benchmarkSamples(), fun, std::chrono::duration_cast<FloatDuration<Clock>>(Detail::warmup_time), Detail::warmup_iterations }; + } - void noMatchingTestCases(std::string const& s) override; + template <typename Clock = default_clock> + void run() { + IConfigPtr cfg = getCurrentContext().getConfig(); - void testRunStarting(TestRunInfo const& testInfo) override; + auto env = Detail::measure_environment<Clock>(); - void testGroupStarting(GroupInfo const& groupInfo) override; + getResultCapture().benchmarkPreparing(name); + CATCH_TRY{ + auto plan = user_code([&] { + return prepare<Clock>(*cfg, env); + }); - void testCaseStarting(TestCaseInfo const& testInfo) override; + BenchmarkInfo info { + name, + plan.estimated_duration.count(), + plan.iterations_per_sample, + cfg->benchmarkSamples(), + cfg->benchmarkResamples(), + env.clock_resolution.mean.count(), + env.clock_cost.mean.count() + }; - void sectionStarting(SectionInfo const& sectionInfo) override; + getResultCapture().benchmarkStarting(info); - void assertionStarting(AssertionInfo const&) override; + auto samples = user_code([&] { + return plan.template run<Clock>(*cfg, env); + }); - bool assertionEnded(AssertionStats const& assertionStats) override; + auto analysis = Detail::analyse(*cfg, env, samples.begin(), samples.end()); + BenchmarkStats<std::chrono::duration<double, std::nano>> stats{ info, analysis.samples, analysis.mean, analysis.standard_deviation, analysis.outliers, analysis.outlier_variance }; + getResultCapture().benchmarkEnded(stats); - void sectionEnded(SectionStats const& sectionStats) override; + } CATCH_CATCH_ALL{ + if (translateActiveException() != Detail::benchmarkErrorMsg) // benchmark errors have been reported, otherwise rethrow. + std::rethrow_exception(std::current_exception()); + } + } - void testCaseEnded(TestCaseStats const& testCaseStats) override; + // sets lambda to be used in fun *and* executes benchmark! + template <typename Fun, + typename std::enable_if<!Detail::is_related<Fun, Benchmark>::value, int>::type = 0> + Benchmark & operator=(Fun func) { + fun = Detail::BenchmarkFunction(func); + run(); + return *this; + } - void testGroupEnded(TestGroupStats const& testGroupStats) override; + explicit operator bool() { + return true; + } - void testRunEnded(TestRunStats const& testRunStats) override; + private: + Detail::BenchmarkFunction fun; + std::string name; + }; + } +} // namespace Catch - private: - Timer m_testCaseTimer; - XmlWriter m_xml; - int m_sectionDepth = 0; - }; +#define INTERNAL_CATCH_GET_1_ARG(arg1, arg2, ...) arg1 +#define INTERNAL_CATCH_GET_2_ARG(arg1, arg2, ...) arg2 -} // end namespace Catch +#define INTERNAL_CATCH_BENCHMARK(BenchmarkName, name, benchmarkIndex)\ + if( Catch::Benchmark::Benchmark BenchmarkName{name} ) \ + BenchmarkName = [&](int benchmarkIndex) -// end catch_reporter_xml.h +#define INTERNAL_CATCH_BENCHMARK_ADVANCED(BenchmarkName, name)\ + if( Catch::Benchmark::Benchmark BenchmarkName{name} ) \ + BenchmarkName = [&] -// end catch_external_interfaces.h +// end catch_benchmark.hpp #endif #endif // ! CATCH_CONFIG_IMPL_ONLY @@ -6150,6 +7260,216 @@ namespace Catch { } // end catch_leak_detector.h // Cpp files will be included in the single-header file here +// start catch_stats.cpp + +// Statistical analysis tools + +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) + +#include <cassert> +#include <random> + +#if defined(CATCH_CONFIG_USE_ASYNC) +#include <future> +#endif + +namespace { + double erf_inv(double x) { + // Code accompanying the article "Approximating the erfinv function" in GPU Computing Gems, Volume 2 + double w, p; + + w = -log((1.0 - x) * (1.0 + x)); + + if (w < 6.250000) { + w = w - 3.125000; + p = -3.6444120640178196996e-21; + p = -1.685059138182016589e-19 + p * w; + p = 1.2858480715256400167e-18 + p * w; + p = 1.115787767802518096e-17 + p * w; + p = -1.333171662854620906e-16 + p * w; + p = 2.0972767875968561637e-17 + p * w; + p = 6.6376381343583238325e-15 + p * w; + p = -4.0545662729752068639e-14 + p * w; + p = -8.1519341976054721522e-14 + p * w; + p = 2.6335093153082322977e-12 + p * w; + p = -1.2975133253453532498e-11 + p * w; + p = -5.4154120542946279317e-11 + p * w; + p = 1.051212273321532285e-09 + p * w; + p = -4.1126339803469836976e-09 + p * w; + p = -2.9070369957882005086e-08 + p * w; + p = 4.2347877827932403518e-07 + p * w; + p = -1.3654692000834678645e-06 + p * w; + p = -1.3882523362786468719e-05 + p * w; + p = 0.0001867342080340571352 + p * w; + p = -0.00074070253416626697512 + p * w; + p = -0.0060336708714301490533 + p * w; + p = 0.24015818242558961693 + p * w; + p = 1.6536545626831027356 + p * w; + } else if (w < 16.000000) { + w = sqrt(w) - 3.250000; + p = 2.2137376921775787049e-09; + p = 9.0756561938885390979e-08 + p * w; + p = -2.7517406297064545428e-07 + p * w; + p = 1.8239629214389227755e-08 + p * w; + p = 1.5027403968909827627e-06 + p * w; + p = -4.013867526981545969e-06 + p * w; + p = 2.9234449089955446044e-06 + p * w; + p = 1.2475304481671778723e-05 + p * w; + p = -4.7318229009055733981e-05 + p * w; + p = 6.8284851459573175448e-05 + p * w; + p = 2.4031110387097893999e-05 + p * w; + p = -0.0003550375203628474796 + p * w; + p = 0.00095328937973738049703 + p * w; + p = -0.0016882755560235047313 + p * w; + p = 0.0024914420961078508066 + p * w; + p = -0.0037512085075692412107 + p * w; + p = 0.005370914553590063617 + p * w; + p = 1.0052589676941592334 + p * w; + p = 3.0838856104922207635 + p * w; + } else { + w = sqrt(w) - 5.000000; + p = -2.7109920616438573243e-11; + p = -2.5556418169965252055e-10 + p * w; + p = 1.5076572693500548083e-09 + p * w; + p = -3.7894654401267369937e-09 + p * w; + p = 7.6157012080783393804e-09 + p * w; + p = -1.4960026627149240478e-08 + p * w; + p = 2.9147953450901080826e-08 + p * w; + p = -6.7711997758452339498e-08 + p * w; + p = 2.2900482228026654717e-07 + p * w; + p = -9.9298272942317002539e-07 + p * w; + p = 4.5260625972231537039e-06 + p * w; + p = -1.9681778105531670567e-05 + p * w; + p = 7.5995277030017761139e-05 + p * w; + p = -0.00021503011930044477347 + p * w; + p = -0.00013871931833623122026 + p * w; + p = 1.0103004648645343977 + p * w; + p = 4.8499064014085844221 + p * w; + } + return p * x; + } + + double standard_deviation(std::vector<double>::iterator first, std::vector<double>::iterator last) { + auto m = Catch::Benchmark::Detail::mean(first, last); + double variance = std::accumulate(first, last, 0., [m](double a, double b) { + double diff = b - m; + return a + diff * diff; + }) / (last - first); + return std::sqrt(variance); + } + +} + +namespace Catch { + namespace Benchmark { + namespace Detail { + + double weighted_average_quantile(int k, int q, std::vector<double>::iterator first, std::vector<double>::iterator last) { + auto count = last - first; + double idx = (count - 1) * k / static_cast<double>(q); + int j = static_cast<int>(idx); + double g = idx - j; + std::nth_element(first, first + j, last); + auto xj = first[j]; + if (g == 0) return xj; + + auto xj1 = *std::min_element(first + (j + 1), last); + return xj + g * (xj1 - xj); + } + + double erfc_inv(double x) { + return erf_inv(1.0 - x); + } + + double normal_quantile(double p) { + static const double ROOT_TWO = std::sqrt(2.0); + + double result = 0.0; + assert(p >= 0 && p <= 1); + if (p < 0 || p > 1) { + return result; + } + + result = -erfc_inv(2.0 * p); + // result *= normal distribution standard deviation (1.0) * sqrt(2) + result *= /*sd * */ ROOT_TWO; + // result += normal disttribution mean (0) + return result; + } + + double outlier_variance(Estimate<double> mean, Estimate<double> stddev, int n) { + double sb = stddev.point; + double mn = mean.point / n; + double mg_min = mn / 2.; + double sg = std::min(mg_min / 4., sb / std::sqrt(n)); + double sg2 = sg * sg; + double sb2 = sb * sb; + + auto c_max = [n, mn, sb2, sg2](double x) -> double { + double k = mn - x; + double d = k * k; + double nd = n * d; + double k0 = -n * nd; + double k1 = sb2 - n * sg2 + nd; + double det = k1 * k1 - 4 * sg2 * k0; + return (int)(-2. * k0 / (k1 + std::sqrt(det))); + }; + + auto var_out = [n, sb2, sg2](double c) { + double nc = n - c; + return (nc / n) * (sb2 - nc * sg2); + }; + + return std::min(var_out(1), var_out(std::min(c_max(0.), c_max(mg_min)))) / sb2; + } + + bootstrap_analysis analyse_samples(double confidence_level, int n_resamples, std::vector<double>::iterator first, std::vector<double>::iterator last) { + CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS + static std::random_device entropy; + CATCH_INTERNAL_UNSUPPRESS_GLOBALS_WARNINGS + + auto n = static_cast<int>(last - first); // seriously, one can't use integral types without hell in C++ + + auto mean = &Detail::mean<std::vector<double>::iterator>; + auto stddev = &standard_deviation; + +#if defined(CATCH_CONFIG_USE_ASYNC) + auto Estimate = [=](double(*f)(std::vector<double>::iterator, std::vector<double>::iterator)) { + auto seed = entropy(); + return std::async(std::launch::async, [=] { + std::mt19937 rng(seed); + auto resampled = resample(rng, n_resamples, first, last, f); + return bootstrap(confidence_level, first, last, resampled, f); + }); + }; + + auto mean_future = Estimate(mean); + auto stddev_future = Estimate(stddev); + + auto mean_estimate = mean_future.get(); + auto stddev_estimate = stddev_future.get(); +#else + auto Estimate = [=](double(*f)(std::vector<double>::iterator, std::vector<double>::iterator)) { + auto seed = entropy(); + std::mt19937 rng(seed); + auto resampled = resample(rng, n_resamples, first, last, f); + return bootstrap(confidence_level, first, last, resampled, f); + }; + + auto mean_estimate = Estimate(mean); + auto stddev_estimate = Estimate(stddev); +#endif // CATCH_USE_ASYNC + + double outlier_variance = Detail::outlier_variance(mean_estimate, stddev_estimate, n); + + return { mean_estimate, stddev_estimate, outlier_variance }; + } + } // namespace Detail + } // namespace Benchmark +} // namespace Catch + +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING +// end catch_stats.cpp // start catch_approx.cpp #include <cmath> @@ -6410,8 +7730,12 @@ namespace Catch { auto acquireGeneratorTracker( SourceLineInfo const& lineInfo ) -> IGeneratorTracker& override; +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) + void benchmarkPreparing( std::string const& name ) override; void benchmarkStarting( BenchmarkInfo const& info ) override; - void benchmarkEnded( BenchmarkStats const& stats ) override; + void benchmarkEnded( BenchmarkStats<> const& stats ) override; + void benchmarkFailed( std::string const& error ) override; +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING void pushScopedMessage( MessageInfo const& message ) override; void popScopedMessage( MessageInfo const& message ) override; @@ -6673,32 +7997,6 @@ namespace Catch { } // end namespace Catch // end catch_assertionresult.cpp -// start catch_benchmark.cpp - -namespace Catch { - - auto BenchmarkLooper::getResolution() -> uint64_t { - return getEstimatedClockResolution() * getCurrentContext().getConfig()->benchmarkResolutionMultiple(); - } - - void BenchmarkLooper::reportStart() { - getResultCapture().benchmarkStarting( { m_name } ); - } - auto BenchmarkLooper::needsMoreIterations() -> bool { - auto elapsed = m_timer.getElapsedNanoseconds(); - - // Exponentially increasing iterations until we're confident in our timer resolution - if( elapsed < m_resolution ) { - m_iterationsToRun *= 10; - return true; - } - - getResultCapture().benchmarkEnded( { { m_name }, m_count, elapsed } ); - return false; - } - -} // end namespace Catch -// end catch_benchmark.cpp // start catch_capture_matchers.cpp namespace Catch { @@ -8193,11 +9491,19 @@ namespace Catch { | Opt( setWaitForKeypress, "start|exit|both" ) ["--wait-for-keypress"] ( "waits for a keypress before exiting" ) - | Opt( config.benchmarkResolutionMultiple, "multiplier" ) - ["--benchmark-resolution-multiple"] - ( "multiple of clock resolution to run benchmarks" ) - - | Arg( config.testsOrTags, "test name|pattern|tags" ) + | Opt( config.benchmarkSamples, "samples" ) + ["--benchmark-samples"] + ( "number of samples to collect (default: 100)" ) + | Opt( config.benchmarkResamples, "resamples" ) + ["--benchmark-resamples"] + ( "number of resamples for the bootstrap (default: 100000)" ) + | Opt( config.benchmarkConfidenceInterval, "confidence interval" ) + ["--benchmark-confidence-interval"] + ( "confidence interval for the bootstrap (between 0 and 1, default: 0.95)" ) + | Opt( config.benchmarkNoAnalysis ) + ["--benchmark-no-analysis"] + ( "perform only measurements; do not perform any analysis" ) + | Arg( config.testsOrTags, "test name|pattern|tags" ) ( "which test or tests to use" ); return cli; @@ -8289,13 +9595,17 @@ namespace Catch { ShowDurations::OrNot Config::showDurations() const { return m_data.showDurations; } RunTests::InWhatOrder Config::runOrder() const { return m_data.runOrder; } unsigned int Config::rngSeed() const { return m_data.rngSeed; } - int Config::benchmarkResolutionMultiple() const { return m_data.benchmarkResolutionMultiple; } UseColour::YesOrNo Config::useColour() const { return m_data.useColour; } bool Config::shouldDebugBreak() const { return m_data.shouldDebugBreak; } int Config::abortAfter() const { return m_data.abortAfter; } bool Config::showInvisibles() const { return m_data.showInvisibles; } Verbosity Config::verbosity() const { return m_data.verbosity; } + bool Config::benchmarkNoAnalysis() const { return m_data.benchmarkNoAnalysis; } + int Config::benchmarkSamples() const { return m_data.benchmarkSamples; } + double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; } + unsigned int Config::benchmarkResamples() const { return m_data.benchmarkResamples; } + IStream const* Config::openStream() { return Catch::makeStream(m_data.outputFilename); } @@ -8522,7 +9832,13 @@ namespace Catch { void Colour::use( Code _colourCode ) { static IColourImpl* impl = platformColourInstance(); - impl->use( _colourCode ); + // Strictly speaking, this cannot possibly happen. + // However, under some conditions it does happen (see #1626), + // and this change is small enough that we can let practicality + // triumph over purity in this case. + if (impl != NULL) { + impl->use( _colourCode ); + } } std::ostream& operator << ( std::ostream& os, Colour const& ) { @@ -8751,6 +10067,8 @@ namespace Catch { // end catch_decomposer.cpp // start catch_enforce.cpp +#include <stdexcept> + namespace Catch { #if defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) && !defined(CATCH_CONFIG_DISABLE_EXCEPTIONS_CUSTOM_HANDLER) [[noreturn]] @@ -8760,6 +10078,22 @@ namespace Catch { std::terminate(); } #endif + + [[noreturn]] + void throw_logic_error(std::string const& msg) { + throw_exception(std::logic_error(msg)); + } + + [[noreturn]] + void throw_domain_error(std::string const& msg) { + throw_exception(std::domain_error(msg)); + } + + [[noreturn]] + void throw_runtime_error(std::string const& msg) { + throw_exception(std::runtime_error(msg)); + } + } // namespace Catch; // end catch_enforce.cpp // start catch_enum_values_registry.cpp @@ -9205,8 +10539,12 @@ namespace Catch { static std::set<Verbosity> getSupportedVerbosities(); +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) + void benchmarkPreparing(std::string const& name) override; void benchmarkStarting( BenchmarkInfo const& benchmarkInfo ) override; - void benchmarkEnded( BenchmarkStats const& benchmarkStats ) override; + void benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) override; + void benchmarkFailed(std::string const&) override; +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING void testRunStarting( TestRunInfo const& testRunInfo ) override; void testGroupStarting( GroupInfo const& groupInfo ) override; @@ -9604,6 +10942,9 @@ namespace Catch { #include <cstdlib> #include <cstdint> #include <cstring> +#include <sstream> +#include <iomanip> +#include <limits> namespace Catch { namespace Matchers { @@ -9664,8 +11005,16 @@ bool almostEqualUlps(FP lhs, FP rhs, int maxUlpDiff) { return ulpDiff <= maxUlpDiff; } +template <typename FP> +FP step(FP start, FP direction, int steps) { + for (int i = 0; i < steps; ++i) { + start = std::nextafter(start, direction); + } + return start; } +} // end anonymous namespace + namespace Catch { namespace Matchers { namespace Floating { @@ -9713,7 +11062,29 @@ namespace Floating { #endif std::string WithinUlpsMatcher::describe() const { - return "is within " + Catch::to_string(m_ulps) + " ULPs of " + ::Catch::Detail::stringify(m_target) + ((m_type == FloatingPointKind::Float)? "f" : ""); + std::stringstream ret; + + ret << "is within " << m_ulps << " ULPs of " << ::Catch::Detail::stringify(m_target); + + if (m_type == FloatingPointKind::Float) { + ret << 'f'; + } + + ret << " (["; + ret << std::fixed << std::setprecision(std::numeric_limits<double>::max_digits10); + if (m_type == FloatingPointKind::Double) { + ret << step(m_target, static_cast<double>(-INFINITY), m_ulps) + << ", " + << step(m_target, static_cast<double>(INFINITY), m_ulps); + } else { + ret << step<float>(static_cast<float>(m_target), -INFINITY, m_ulps) + << ", " + << step<float>(static_cast<float>(m_target), INFINITY, m_ulps); + } + ret << "])"; + + return ret.str(); + //return "is within " + Catch::to_string(m_ulps) + " ULPs of " + ::Catch::Detail::stringify(m_target) + ((m_type == FloatingPointKind::Float)? "f" : ""); } }// namespace Floating @@ -10795,12 +12166,21 @@ namespace Catch { m_unfinishedSections.push_back(endInfo); } + +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) + void RunContext::benchmarkPreparing(std::string const& name) { + m_reporter->benchmarkPreparing(name); + } void RunContext::benchmarkStarting( BenchmarkInfo const& info ) { m_reporter->benchmarkStarting( info ); } - void RunContext::benchmarkEnded( BenchmarkStats const& stats ) { + void RunContext::benchmarkEnded( BenchmarkStats<> const& stats ) { m_reporter->benchmarkEnded( stats ); } + void RunContext::benchmarkFailed(std::string const & error) { + m_reporter->benchmarkFailed(error); + } +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING void RunContext::pushScopedMessage(MessageInfo const & message) { m_messages.push_back(message); @@ -11514,7 +12894,7 @@ namespace Catch { Catch::IStream::~IStream() = default; - namespace detail { namespace { + namespace Detail { namespace { template<typename WriterF, std::size_t bufferSize=256> class StreamBufImpl : public std::streambuf { char data[bufferSize]; @@ -11613,15 +12993,15 @@ namespace Catch { auto makeStream( StringRef const &filename ) -> IStream const* { if( filename.empty() ) - return new detail::CoutStream(); + return new Detail::CoutStream(); else if( filename[0] == '%' ) { if( filename == "%debug" ) - return new detail::DebugOutStream(); + return new Detail::DebugOutStream(); else CATCH_ERROR( "Unrecognised stream: '" << filename << "'" ); } else - return new detail::FileStream( filename ); + return new Detail::FileStream( filename ); } // This class encapsulates the idea of a pool of ostringstreams that can be reused. @@ -13033,7 +14413,7 @@ namespace Catch { } Version const& libraryVersion() { - static Version version( 2, 8, 0, "", 0 ); + static Version version( 2, 9, 0, "", 0 ); return version; } @@ -13710,8 +15090,13 @@ private: #if defined(_MSC_VER) #pragma warning(push) #pragma warning(disable:4061) // Not all labels are EXPLICITLY handled in switch - // Note that 4062 (not all labels are handled - // and default is missing) is enabled + // Note that 4062 (not all labels are handled and default is missing) is enabled +#endif + +#if defined(__clang__) +# pragma clang diagnostic push +// For simplicity, benchmarking-only helpers are always enabled +# pragma clang diagnostic ignored "-Wunused-function" #endif namespace Catch { @@ -13897,6 +15282,10 @@ class Duration { Unit m_units; public: + explicit Duration(double inNanoseconds, Unit units = Unit::Auto) + : Duration(static_cast<uint64_t>(inNanoseconds), units) { + } + explicit Duration(uint64_t inNanoseconds, Unit units = Unit::Auto) : m_inNanoseconds(inNanoseconds), m_units(units) { @@ -13972,9 +15361,15 @@ public: if (!m_isOpen) { m_isOpen = true; *this << RowBreak(); - for (auto const& info : m_columnInfos) - *this << info.name << ColumnBreak(); - *this << RowBreak(); + + Columns headerCols; + Spacer spacer(2); + for (auto const& info : m_columnInfos) { + headerCols += Column(info.name).width(static_cast<std::size_t>(info.width - 2)); + headerCols += spacer; + } + m_os << headerCols << "\n"; + m_os << Catch::getLineOfChars<'-'>() << "\n"; } } @@ -14029,9 +15424,9 @@ ConsoleReporter::ConsoleReporter(ReporterConfig const& config) m_tablePrinter(new TablePrinter(config.stream(), { { "benchmark name", CATCH_CONFIG_CONSOLE_WIDTH - 32, ColumnInfo::Left }, - { "iters", 8, ColumnInfo::Right }, - { "elapsed ns", 14, ColumnInfo::Right }, - { "average", 14, ColumnInfo::Right } + { "samples mean std dev", 14, ColumnInfo::Right }, + { "iterations low mean low std dev", 14, ColumnInfo::Right }, + { "estimated high mean high std dev", 14, ColumnInfo::Right } })) {} ConsoleReporter::~ConsoleReporter() = default; @@ -14063,6 +15458,7 @@ bool ConsoleReporter::assertionEnded(AssertionStats const& _assertionStats) { } void ConsoleReporter::sectionStarting(SectionInfo const& _sectionInfo) { + m_tablePrinter->close(); m_headerPrinted = false; StreamingReporterBase::sectionStarting(_sectionInfo); } @@ -14086,28 +15482,45 @@ void ConsoleReporter::sectionEnded(SectionStats const& _sectionStats) { StreamingReporterBase::sectionEnded(_sectionStats); } -void ConsoleReporter::benchmarkStarting(BenchmarkInfo const& info) { - lazyPrintWithoutClosingBenchmarkTable(); +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) +void ConsoleReporter::benchmarkPreparing(std::string const& name) { + lazyPrintWithoutClosingBenchmarkTable(); - auto nameCol = Column( info.name ).width( static_cast<std::size_t>( m_tablePrinter->columnInfos()[0].width - 2 ) ); + auto nameCol = Column(name).width(static_cast<std::size_t>(m_tablePrinter->columnInfos()[0].width - 2)); - bool firstLine = true; - for (auto line : nameCol) { - if (!firstLine) - (*m_tablePrinter) << ColumnBreak() << ColumnBreak() << ColumnBreak(); - else - firstLine = false; + bool firstLine = true; + for (auto line : nameCol) { + if (!firstLine) + (*m_tablePrinter) << ColumnBreak() << ColumnBreak() << ColumnBreak(); + else + firstLine = false; - (*m_tablePrinter) << line << ColumnBreak(); - } + (*m_tablePrinter) << line << ColumnBreak(); + } } -void ConsoleReporter::benchmarkEnded(BenchmarkStats const& stats) { - Duration average(stats.elapsedTimeInNanoseconds / stats.iterations); + +void ConsoleReporter::benchmarkStarting(BenchmarkInfo const& info) { + (*m_tablePrinter) << info.samples << ColumnBreak() + << info.iterations << ColumnBreak() + << Duration(info.estimatedDuration) << ColumnBreak(); +} +void ConsoleReporter::benchmarkEnded(BenchmarkStats<> const& stats) { + (*m_tablePrinter) << ColumnBreak() + << Duration(stats.mean.point.count()) << ColumnBreak() + << Duration(stats.mean.lower_bound.count()) << ColumnBreak() + << Duration(stats.mean.upper_bound.count()) << ColumnBreak() << ColumnBreak() + << Duration(stats.standardDeviation.point.count()) << ColumnBreak() + << Duration(stats.standardDeviation.lower_bound.count()) << ColumnBreak() + << Duration(stats.standardDeviation.upper_bound.count()) << ColumnBreak() << ColumnBreak() << ColumnBreak() << ColumnBreak() << ColumnBreak(); +} + +void ConsoleReporter::benchmarkFailed(std::string const& error) { + Colour colour(Colour::Red); (*m_tablePrinter) - << stats.iterations << ColumnBreak() - << stats.elapsedTimeInNanoseconds << ColumnBreak() - << average << ColumnBreak(); + << "Benchmark failed (" << error << ")" + << ColumnBreak() << RowBreak(); } +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING void ConsoleReporter::testCaseEnded(TestCaseStats const& _testCaseStats) { m_tablePrinter->close(); @@ -14326,6 +15739,10 @@ CATCH_REGISTER_REPORTER("console", ConsoleReporter) #if defined(_MSC_VER) #pragma warning(pop) #endif + +#if defined(__clang__) +# pragma clang diagnostic pop +#endif // end catch_reporter_console.cpp // start catch_reporter_junit.cpp @@ -14615,19 +16032,34 @@ namespace Catch { m_reporter->noMatchingTestCases( spec ); } +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) + void ListeningReporter::benchmarkPreparing( std::string const& name ) { + for (auto const& listener : m_listeners) { + listener->benchmarkPreparing(name); + } + m_reporter->benchmarkPreparing(name); + } void ListeningReporter::benchmarkStarting( BenchmarkInfo const& benchmarkInfo ) { for ( auto const& listener : m_listeners ) { listener->benchmarkStarting( benchmarkInfo ); } m_reporter->benchmarkStarting( benchmarkInfo ); } - void ListeningReporter::benchmarkEnded( BenchmarkStats const& benchmarkStats ) { + void ListeningReporter::benchmarkEnded( BenchmarkStats<> const& benchmarkStats ) { for ( auto const& listener : m_listeners ) { listener->benchmarkEnded( benchmarkStats ); } m_reporter->benchmarkEnded( benchmarkStats ); } + void ListeningReporter::benchmarkFailed( std::string const& error ) { + for (auto const& listener : m_listeners) { + listener->benchmarkFailed(error); + } + m_reporter->benchmarkFailed(error); + } +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING + void ListeningReporter::testRunStarting( TestRunInfo const& testRunInfo ) { for ( auto const& listener : m_listeners ) { listener->testRunStarting( testRunInfo ); @@ -14921,6 +16353,48 @@ namespace Catch { m_xml.endElement(); } +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) + void XmlReporter::benchmarkStarting(BenchmarkInfo const &info) { + m_xml.startElement("BenchmarkResults") + .writeAttribute("name", info.name) + .writeAttribute("samples", info.samples) + .writeAttribute("resamples", info.resamples) + .writeAttribute("iterations", info.iterations) + .writeAttribute("clockResolution", static_cast<uint64_t>(info.clockResolution)) + .writeAttribute("estimatedDuration", static_cast<uint64_t>(info.estimatedDuration)) + .writeComment("All values in nano seconds"); + } + + void XmlReporter::benchmarkEnded(BenchmarkStats<> const& benchmarkStats) { + m_xml.startElement("mean") + .writeAttribute("value", static_cast<uint64_t>(benchmarkStats.mean.point.count())) + .writeAttribute("lowerBound", static_cast<uint64_t>(benchmarkStats.mean.lower_bound.count())) + .writeAttribute("upperBound", static_cast<uint64_t>(benchmarkStats.mean.upper_bound.count())) + .writeAttribute("ci", benchmarkStats.mean.confidence_interval); + m_xml.endElement(); + m_xml.startElement("standardDeviation") + .writeAttribute("value", benchmarkStats.standardDeviation.point.count()) + .writeAttribute("lowerBound", benchmarkStats.standardDeviation.lower_bound.count()) + .writeAttribute("upperBound", benchmarkStats.standardDeviation.upper_bound.count()) + .writeAttribute("ci", benchmarkStats.standardDeviation.confidence_interval); + m_xml.endElement(); + m_xml.startElement("outliers") + .writeAttribute("variance", benchmarkStats.outlierVariance) + .writeAttribute("lowMild", benchmarkStats.outliers.low_mild) + .writeAttribute("lowSevere", benchmarkStats.outliers.low_severe) + .writeAttribute("highMild", benchmarkStats.outliers.high_mild) + .writeAttribute("highSevere", benchmarkStats.outliers.high_severe); + m_xml.endElement(); + m_xml.endElement(); + } + + void XmlReporter::benchmarkFailed(std::string const &error) { + m_xml.scopedElement("failed"). + writeAttribute("message", error); + m_xml.endElement(); + } +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING + CATCH_REGISTER_REPORTER( "xml", XmlReporter ) } // end namespace Catch @@ -15077,6 +16551,13 @@ int main (int argc, char * const argv[]) { #define CATCH_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " Then: " << desc ) #define CATCH_AND_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " And: " << desc ) +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) +#define CATCH_BENCHMARK(...) \ + INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,)) +#define CATCH_BENCHMARK_ADVANCED(name) \ + INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), name) +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING + // If CATCH_CONFIG_PREFIX_ALL is not defined then the CATCH_ prefix is not required #else @@ -15136,6 +16617,8 @@ int main (int argc, char * const argv[]) { #define TEMPLATE_PRODUCT_TEST_CASE_SIG( ... ) INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_SIG( __VA_ARGS__ ) #define TEMPLATE_PRODUCT_TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD( className, __VA_ARGS__ ) #define TEMPLATE_PRODUCT_TEST_CASE_METHOD_SIG( className, ... ) INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_SIG( className, __VA_ARGS__ ) +#define TEMPLATE_LIST_TEST_CASE( ... ) INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE(__VA_ARGS__) +#define TEMPLATE_LIST_TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD( className, __VA_ARGS__ ) #else #define TEMPLATE_TEST_CASE( ... ) INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE( __VA_ARGS__ ) ) #define TEMPLATE_TEST_CASE_SIG( ... ) INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_TEST_CASE_SIG( __VA_ARGS__ ) ) @@ -15145,6 +16628,8 @@ int main (int argc, char * const argv[]) { #define TEMPLATE_PRODUCT_TEST_CASE_SIG( ... ) INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_SIG( __VA_ARGS__ ) ) #define TEMPLATE_PRODUCT_TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD( className, __VA_ARGS__ ) ) #define TEMPLATE_PRODUCT_TEST_CASE_METHOD_SIG( className, ... ) INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_PRODUCT_TEST_CASE_METHOD_SIG( className, __VA_ARGS__ ) ) +#define TEMPLATE_LIST_TEST_CASE( ... ) INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE( __VA_ARGS__ ) ) +#define TEMPLATE_LIST_TEST_CASE_METHOD( className, ... ) INTERNAL_CATCH_EXPAND_VARGS( INTERNAL_CATCH_TEMPLATE_LIST_TEST_CASE_METHOD( className, __VA_ARGS__ ) ) #endif #if !defined(CATCH_CONFIG_RUNTIME_STATIC_REQUIRE) @@ -15170,6 +16655,13 @@ int main (int argc, char * const argv[]) { #define THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " Then: " << desc ) #define AND_THEN( desc ) INTERNAL_CATCH_DYNAMIC_SECTION( " And: " << desc ) +#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING) +#define BENCHMARK(...) \ + INTERNAL_CATCH_BENCHMARK(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), INTERNAL_CATCH_GET_1_ARG(__VA_ARGS__,,), INTERNAL_CATCH_GET_2_ARG(__VA_ARGS__,,)) +#define BENCHMARK_ADVANCED(name) \ + INTERNAL_CATCH_BENCHMARK_ADVANCED(INTERNAL_CATCH_UNIQUE_NAME(____C_A_T_C_H____B_E_N_C_H____), name) +#endif // CATCH_CONFIG_ENABLE_BENCHMARKING + using Catch::Detail::Approx; #else // CATCH_CONFIG_DISABLE