Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

Added run without -s to approvals

Commit
239fa28e4679b2889b715f98243d2af380beadaf
Author
Phil Nash ·
Parents
47751e4df3af

Files · Download .tar.gz

Changes

 projects/SelfTest/Baselines/approvedResults.txt | 681 ++++++++++++++++++++++++
 scripts/approvalTests.py                        |   1 +
 2 files changed, 682 insertions(+)

diff --git a/projects/SelfTest/Baselines/approvedResults.txt b/projects/SelfTest/Baselines/approvedResults.txt
index 8279682b..1de7f385 100644
--- a/projects/SelfTest/Baselines/approvedResults.txt
+++ b/projects/SelfTest/Baselines/approvedResults.txt
@@ -1,4 +1,685 @@
 
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+CatchSelfTest is a Catch v1.0 b1 host application.
+Run with -? for options
+
+-------------------------------------------------------------------------------
+./failing/TestClass/failingCase
+-------------------------------------------------------------------------------
+ClassTests.cpp:35
+...............................................................................
+
+ClassTests.cpp:28: FAILED:
+  REQUIRE( s == "world" )
+with expansion:
+  "hello" == "world"
+
+-------------------------------------------------------------------------------
+./failing/Fixture/failingCase
+-------------------------------------------------------------------------------
+ClassTests.cpp:53
+...............................................................................
+
+ClassTests.cpp:55: FAILED:
+  REQUIRE( m_a == 2 )
+with expansion:
+  1 == 2
+
+-------------------------------------------------------------------------------
+./failing/conditions/equality
+-------------------------------------------------------------------------------
+ConditionTests.cpp:67
+...............................................................................
+
+ConditionTests.cpp:71: FAILED:
+  CHECK( data.int_seven == 6 )
+with expansion:
+  7 == 6
+
+ConditionTests.cpp:72: FAILED:
+  CHECK( data.int_seven == 8 )
+with expansion:
+  7 == 8
+
+ConditionTests.cpp:73: FAILED:
+  CHECK( data.int_seven == 0 )
+with expansion:
+  7 == 0
+
+ConditionTests.cpp:74: FAILED:
+  CHECK( data.float_nine_point_one == Approx( 9.11f ) )
+with expansion:
+  9.1 == Approx( 9.11 )
+
+ConditionTests.cpp:75: FAILED:
+  CHECK( data.float_nine_point_one == Approx( 9.0f ) )
+with expansion:
+  9.1 == Approx( 9 )
+
+ConditionTests.cpp:76: FAILED:
+  CHECK( data.float_nine_point_one == Approx( 1 ) )
+with expansion:
+  9.1 == Approx( 1 )
+
+ConditionTests.cpp:77: FAILED:
+  CHECK( data.float_nine_point_one == Approx( 0 ) )
+with expansion:
+  9.1 == Approx( 0 )
+
+ConditionTests.cpp:78: FAILED:
+  CHECK( data.double_pi == Approx( 3.1415 ) )
+with expansion:
+  3.1415926535 == Approx( 3.1415 )
+
+ConditionTests.cpp:79: FAILED:
+  CHECK( data.str_hello == "goodbye" )
+with expansion:
+  "hello" == "goodbye"
+
+ConditionTests.cpp:80: FAILED:
+  CHECK( data.str_hello == "hell" )
+with expansion:
+  "hello" == "hell"
+
+ConditionTests.cpp:81: FAILED:
+  CHECK( data.str_hello == "hello1" )
+with expansion:
+  "hello" == "hello1"
+
+ConditionTests.cpp:82: FAILED:
+  CHECK( data.str_hello.size() == 6 )
+with expansion:
+  5 == 6
+
+ConditionTests.cpp:85: FAILED:
+  CHECK( x == Approx( 1.301 ) )
+with expansion:
+  1.3 == Approx( 1.301 )
+
+-------------------------------------------------------------------------------
+./failing/conditions/inequality
+-------------------------------------------------------------------------------
+ConditionTests.cpp:107
+...............................................................................
+
+ConditionTests.cpp:111: FAILED:
+  CHECK( data.int_seven != 7 )
+with expansion:
+  7 != 7
+
+ConditionTests.cpp:112: FAILED:
+  CHECK( data.float_nine_point_one != Approx( 9.1f ) )
+with expansion:
+  9.1 != Approx( 9.1 )
+
+ConditionTests.cpp:113: FAILED:
+  CHECK( data.double_pi != Approx( 3.1415926535 ) )
+with expansion:
+  3.1415926535 != Approx( 3.14159 )
+
+ConditionTests.cpp:114: FAILED:
+  CHECK( data.str_hello != "hello" )
+with expansion:
+  "hello" != "hello"
+
+ConditionTests.cpp:115: FAILED:
+  CHECK( data.str_hello.size() != 5 )
+with expansion:
+  5 != 5
+
+-------------------------------------------------------------------------------
+./failing/conditions/ordered
+-------------------------------------------------------------------------------
+ConditionTests.cpp:148
+...............................................................................
+
+ConditionTests.cpp:152: FAILED:
+  CHECK( data.int_seven > 7 )
+with expansion:
+  7 > 7
+
+ConditionTests.cpp:153: FAILED:
+  CHECK( data.int_seven < 7 )
+with expansion:
+  7 < 7
+
+ConditionTests.cpp:154: FAILED:
+  CHECK( data.int_seven > 8 )
+with expansion:
+  7 > 8
+
+ConditionTests.cpp:155: FAILED:
+  CHECK( data.int_seven < 6 )
+with expansion:
+  7 < 6
+
+ConditionTests.cpp:156: FAILED:
+  CHECK( data.int_seven < 0 )
+with expansion:
+  7 < 0
+
+ConditionTests.cpp:157: FAILED:
+  CHECK( data.int_seven < -1 )
+with expansion:
+  7 < -1
+
+ConditionTests.cpp:159: FAILED:
+  CHECK( data.int_seven >= 8 )
+with expansion:
+  7 >= 8
+
+ConditionTests.cpp:160: FAILED:
+  CHECK( data.int_seven <= 6 )
+with expansion:
+  7 <= 6
+
+ConditionTests.cpp:162: FAILED:
+  CHECK( data.float_nine_point_one < 9 )
+with expansion:
+  9.1 < 9
+
+ConditionTests.cpp:163: FAILED:
+  CHECK( data.float_nine_point_one > 10 )
+with expansion:
+  9.1 > 10
+
+ConditionTests.cpp:164: FAILED:
+  CHECK( data.float_nine_point_one > 9.2 )
+with expansion:
+  9.1 > 9.199999999999999
+
+ConditionTests.cpp:166: FAILED:
+  CHECK( data.str_hello > "hello" )
+with expansion:
+  "hello" > "hello"
+
+ConditionTests.cpp:167: FAILED:
+  CHECK( data.str_hello < "hello" )
+with expansion:
+  "hello" < "hello"
+
+ConditionTests.cpp:168: FAILED:
+  CHECK( data.str_hello > "hellp" )
+with expansion:
+  "hello" > "hellp"
+
+ConditionTests.cpp:169: FAILED:
+  CHECK( data.str_hello > "z" )
+with expansion:
+  "hello" > "z"
+
+ConditionTests.cpp:170: FAILED:
+  CHECK( data.str_hello < "hellm" )
+with expansion:
+  "hello" < "hellm"
+
+ConditionTests.cpp:171: FAILED:
+  CHECK( data.str_hello < "a" )
+with expansion:
+  "hello" < "a"
+
+ConditionTests.cpp:173: FAILED:
+  CHECK( data.str_hello >= "z" )
+with expansion:
+  "hello" >= "z"
+
+ConditionTests.cpp:174: FAILED:
+  CHECK( data.str_hello <= "a" )
+with expansion:
+  "hello" <= "a"
+
+-------------------------------------------------------------------------------
+./failing/conditions/not
+-------------------------------------------------------------------------------
+ConditionTests.cpp:333
+...............................................................................
+
+ConditionTests.cpp:337: FAILED:
+  CHECK( false != false )
+
+ConditionTests.cpp:338: FAILED:
+  CHECK( true != true )
+
+ConditionTests.cpp:339: FAILED:
+  CHECK( !true )
+with expansion:
+  false
+
+ConditionTests.cpp:340: FAILED:
+  CHECK_FALSE( true )
+
+ConditionTests.cpp:342: FAILED:
+  CHECK( !trueValue )
+with expansion:
+  false
+
+ConditionTests.cpp:343: FAILED:
+  CHECK_FALSE( trueValue )
+with expansion:
+  !true
+
+ConditionTests.cpp:345: FAILED:
+  CHECK( !(1 == 1) )
+with expansion:
+  false
+
+ConditionTests.cpp:346: FAILED:
+  CHECK_FALSE( 1 == 1 )
+with expansion:
+  !(1 == 1)
+
+-------------------------------------------------------------------------------
+./failing/exceptions/explicit
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:38
+...............................................................................
+
+ExceptionTests.cpp:40: FAILED:
+  CHECK_THROWS_AS( thisThrows() )
+due to unexpected exception with message:
+  expected exception
+
+ExceptionTests.cpp:41: FAILED:
+  CHECK_THROWS_AS( thisDoesntThrow() )
+because no exception was thrown where one was expected:
+
+ExceptionTests.cpp:42: FAILED:
+  CHECK_NOTHROW( thisThrows() )
+due to unexpected exception with message:
+  expected exception
+
+-------------------------------------------------------------------------------
+./failing/exceptions/implicit
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:45
+...............................................................................
+
+ExceptionTests.cpp:45: FAILED:
+due to unexpected exception with message:
+  unexpected exception
+
+-------------------------------------------------------------------------------
+./failing/exceptions/implicit/2
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:51
+...............................................................................
+
+ExceptionTests.cpp:53: FAILED:
+  {Unknown expression after the reported line}
+due to unexpected exception with message:
+  unexpected exception
+
+-------------------------------------------------------------------------------
+./failing/exceptions/implicit/3
+  section name
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:59
+...............................................................................
+
+ExceptionTests.cpp:59: FAILED:
+due to unexpected exception with message:
+  unexpected exception
+
+-------------------------------------------------------------------------------
+./failing/exceptions/implicit/4
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:66
+...............................................................................
+
+ExceptionTests.cpp:68: FAILED:
+  CHECK( thisThrows() == 0 )
+due to unexpected exception with message:
+  expected exception
+
+-------------------------------------------------------------------------------
+./failing/exceptions/custom
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:108
+...............................................................................
+
+ExceptionTests.cpp:108: FAILED:
+due to unexpected exception with message:
+  custom exception
+
+-------------------------------------------------------------------------------
+./failing/exceptions/custom/nothrow
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:119
+...............................................................................
+
+ExceptionTests.cpp:121: FAILED:
+  REQUIRE_NOTHROW( throwCustom() )
+due to unexpected exception with message:
+  custom exception - not std
+
+-------------------------------------------------------------------------------
+./failing/exceptions/custom/throw
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:124
+...............................................................................
+
+ExceptionTests.cpp:126: FAILED:
+  REQUIRE_THROWS_AS( throwCustom() )
+due to unexpected exception with message:
+  custom exception - not std
+
+-------------------------------------------------------------------------------
+./failing/exceptions/custom/double
+-------------------------------------------------------------------------------
+ExceptionTests.cpp:130
+...............................................................................
+
+ExceptionTests.cpp:130: FAILED:
+due to unexpected exception with message:
+  3.14
+
+-------------------------------------------------------------------------------
+./failing/message/info/1
+-------------------------------------------------------------------------------
+MessageTests.cpp:21
+...............................................................................
+
+MessageTests.cpp:26: FAILED:
+  REQUIRE( a == 1 )
+with expansion:
+  2 == 1
+with messages:
+  this message should be logged
+  so should this
+
+-------------------------------------------------------------------------------
+./mixed/message/info/2
+-------------------------------------------------------------------------------
+MessageTests.cpp:29
+...............................................................................
+
+MessageTests.cpp:37: FAILED:
+  CHECK( a == 1 )
+with expansion:
+  2 == 1
+with message:
+  this message should be logged
+
+MessageTests.cpp:41: FAILED:
+  CHECK( a == 0 )
+with expansion:
+  2 == 0
+with message:
+  and this, but later
+
+-------------------------------------------------------------------------------
+./failing/message/fail
+-------------------------------------------------------------------------------
+MessageTests.cpp:48
+...............................................................................
+
+MessageTests.cpp:51: FAILED:
+explicitly with message:
+  This is a failure
+
+-------------------------------------------------------------------------------
+./failing/message/sections
+  one
+-------------------------------------------------------------------------------
+MessageTests.cpp:56
+...............................................................................
+
+MessageTests.cpp:58: FAILED:
+explicitly with message:
+  Message from section one
+
+-------------------------------------------------------------------------------
+./failing/message/sections
+  two
+-------------------------------------------------------------------------------
+MessageTests.cpp:61
+...............................................................................
+
+MessageTests.cpp:63: FAILED:
+explicitly with message:
+  Message from section two
+
+Message from section one
+Message from section two
+-------------------------------------------------------------------------------
+./mixed/message/scoped
+-------------------------------------------------------------------------------
+MessageTests.cpp:80
+...............................................................................
+
+MessageTests.cpp:86: FAILED:
+  REQUIRE( i < 10 )
+with expansion:
+  10 < 10
+with messages:
+  current counter 10
+  i := 10
+
+-------------------------------------------------------------------------------
+just failure
+-------------------------------------------------------------------------------
+MessageTests.cpp:99
+...............................................................................
+
+MessageTests.cpp:101: FAILED:
+explicitly with message:
+  Previous info should not be seen
+
+-------------------------------------------------------------------------------
+./mixed/Misc/Sections/nested2
+  s1
+  s2
+-------------------------------------------------------------------------------
+MiscTests.cpp:55
+...............................................................................
+
+MiscTests.cpp:57: FAILED:
+  REQUIRE( a == b )
+with expansion:
+  1 == 2
+
+-------------------------------------------------------------------------------
+./mixed/Misc/Sections/loops
+  s1
+-------------------------------------------------------------------------------
+MiscTests.cpp:97
+...............................................................................
+
+MiscTests.cpp:99: FAILED:
+  CHECK( b > a )
+with expansion:
+  0 > 1
+
+-------------------------------------------------------------------------------
+./mixed/Misc/loops
+-------------------------------------------------------------------------------
+MiscTests.cpp:104
+...............................................................................
+
+MiscTests.cpp:111: FAILED:
+  CHECK( ( fib[i] % 2 ) == 0 )
+with expansion:
+  1 == 0
+with message:
+  Testing if fib[0] (1) is even
+
+MiscTests.cpp:111: FAILED:
+  CHECK( ( fib[i] % 2 ) == 0 )
+with expansion:
+  1 == 0
+with message:
+  Testing if fib[1] (1) is even
+
+MiscTests.cpp:111: FAILED:
+  CHECK( ( fib[i] % 2 ) == 0 )
+with expansion:
+  1 == 0
+with message:
+  Testing if fib[3] (3) is even
+
+MiscTests.cpp:111: FAILED:
+  CHECK( ( fib[i] % 2 ) == 0 )
+with expansion:
+  1 == 0
+with message:
+  Testing if fib[4] (5) is even
+
+MiscTests.cpp:111: FAILED:
+  CHECK( ( fib[i] % 2 ) == 0 )
+with expansion:
+  1 == 0
+with message:
+  Testing if fib[6] (13) is even
+
+MiscTests.cpp:111: FAILED:
+  CHECK( ( fib[i] % 2 ) == 0 )
+with expansion:
+  1 == 0
+with message:
+  Testing if fib[7] (21) is even
+
+Some information
+An error
+-------------------------------------------------------------------------------
+./failing/info
+-------------------------------------------------------------------------------
+MiscTests.cpp:133
+...............................................................................
+
+MiscTests.cpp:138: FAILED:
+  REQUIRE( false )
+with messages:
+  hi
+  i := 7
+
+-------------------------------------------------------------------------------
+./failing/checkedif
+-------------------------------------------------------------------------------
+MiscTests.cpp:154
+...............................................................................
+
+MiscTests.cpp:143: FAILED:
+  CHECKED_IF( flag )
+with expansion:
+  false
+
+MiscTests.cpp:156: FAILED:
+  REQUIRE( testCheckedIf( false ) )
+with expansion:
+  false
+
+-------------------------------------------------------------------------------
+./failing/checkedelse
+-------------------------------------------------------------------------------
+MiscTests.cpp:172
+...............................................................................
+
+MiscTests.cpp:161: FAILED:
+  CHECKED_ELSE( flag )
+with expansion:
+  false
+
+MiscTests.cpp:174: FAILED:
+  REQUIRE( testCheckedElse( false ) )
+with expansion:
+  false
+
+-------------------------------------------------------------------------------
+./manual/onechar
+-------------------------------------------------------------------------------
+MiscTests.cpp:189
+...............................................................................
+
+MiscTests.cpp:192: FAILED:
+  REQUIRE( false )
+with message:
+  3
+
+-------------------------------------------------------------------------------
+./failing/matchers/Contains
+-------------------------------------------------------------------------------
+MiscTests.cpp:219
+...............................................................................
+
+MiscTests.cpp:221: FAILED:
+  CHECK_THAT( testStringForMatching() Contains( "not there" ) )
+with expansion:
+  "this string contains 'abc' as a substring" contains: "not there"
+
+-------------------------------------------------------------------------------
+./failing/matchers/StartsWith
+-------------------------------------------------------------------------------
+MiscTests.cpp:224
+...............................................................................
+
+MiscTests.cpp:226: FAILED:
+  CHECK_THAT( testStringForMatching() StartsWith( "string" ) )
+with expansion:
+  "this string contains 'abc' as a substring" starts with: "string"
+
+-------------------------------------------------------------------------------
+./failing/matchers/EndsWith
+-------------------------------------------------------------------------------
+MiscTests.cpp:229
+...............................................................................
+
+MiscTests.cpp:231: FAILED:
+  CHECK_THAT( testStringForMatching() EndsWith( "this" ) )
+with expansion:
+  "this string contains 'abc' as a substring" ends with: "this"
+
+-------------------------------------------------------------------------------
+./failing/matchers/Equals
+-------------------------------------------------------------------------------
+MiscTests.cpp:234
+...............................................................................
+
+MiscTests.cpp:236: FAILED:
+  CHECK_THAT( testStringForMatching() Equals( "something else" ) )
+with expansion:
+  "this string contains 'abc' as a substring" equals: "something else"
+
+Message from section one
+Message from section two
+Some information
+An error
+Message from section one
+Message from section two
+Some information
+An error
+hello
+hello
+-------------------------------------------------------------------------------
+./failing/Tricky/non streamable type
+-------------------------------------------------------------------------------
+TrickyTests.cpp:88
+...............................................................................
+
+TrickyTests.cpp:95: FAILED:
+  CHECK( &o1 == &o2 )
+with expansion:
+  0x<hex digits> == 0x<hex digits>
+
+TrickyTests.cpp:96: FAILED:
+  CHECK( o1 == o2 )
+with expansion:
+  {?} == {?}
+
+-------------------------------------------------------------------------------
+./failing/string literals
+-------------------------------------------------------------------------------
+TrickyTests.cpp:104
+...............................................................................
+
+TrickyTests.cpp:106: FAILED:
+  REQUIRE( std::string( "first" ) == "second" )
+with expansion:
+  "first" == "second"
+
+===============================================================================
+113 test cases - 34 failed (686 assertions - 87 failed)
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 CatchSelfTest is a Catch v1.0 b1 host application.
 Run with -? for options
diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py
index 14a5354e..d0eba1cf 100644
--- a/scripts/approvalTests.py
+++ b/scripts/approvalTests.py
@@ -20,6 +20,7 @@ else:
 	cmdPath = os.path.join( catchPath, 'projects/XCode4/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' )
 
 f = open( rawResultsPath, 'w' )
+subprocess.call([ cmdPath, "~dummy", "-r", "console" ], stdout=f, stderr=f )
 subprocess.call([ cmdPath, "~dummy", "-s", "-w", "NoAssertions", "-r", "console" ], stdout=f, stderr=f )
 subprocess.call([ cmdPath, "~dummy", "-s", "-w", "NoAssertions", "-r", "console", "-a", "4" ], stdout=f, stderr=f )
 subprocess.call([ cmdPath, "~dummy", "-s", "-w", "NoAssertions", "-r", "junit" ], stdout=f, stderr=f )