Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

Approval tests can use Catch path fixed in env. var, and convert nullptr and __null to 0 for comparison

Commit
976a6554960e2cd3c5d39c7d84bbef1af4d762e5
Author
Phil Nash ·
Parents
37e1e2430930

Files · Download .tar.gz

Changes

 .../SelfTest/Baselines/console.sw.approved.txt     |  50 +++---
 projects/SelfTest/Baselines/junit.sw.approved.txt  |   2 +-
 projects/SelfTest/Baselines/xml.sw.approved.txt    |  50 +++---
 scripts/approvalTests.py                           | 199 +++++++++++----------
 scripts/scriptCommon.py                            |   7 +-
 5 files changed, 163 insertions(+), 145 deletions(-)

diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt
index b9fdceab..6c982790 100644
--- a/projects/SelfTest/Baselines/console.sw.approved.txt
+++ b/projects/SelfTest/Baselines/console.sw.approved.txt
@@ -923,7 +923,7 @@ MiscTests.cpp:<line number>
 
 MiscTests.cpp:<line number>:
 PASSED:
-  REQUIRE_THAT( "", Equals(nullptr) )
+  REQUIRE_THAT( "", Equals(0) )
 with expansion:
   "" equals: ""
 
@@ -5446,9 +5446,9 @@ ConditionTests.cpp:<line number>
 
 ConditionTests.cpp:<line number>:
 PASSED:
-  REQUIRE( p == nullptr )
+  REQUIRE( p == 0 )
 with expansion:
-  NULL == nullptr
+  NULL == 0
 
 ConditionTests.cpp:<line number>:
 PASSED:
@@ -5458,39 +5458,39 @@ with expansion:
 
 ConditionTests.cpp:<line number>:
 PASSED:
-  REQUIRE( p != nullptr )
+  REQUIRE( p != 0 )
 with expansion:
-  0x<hex digits> != nullptr
+  0x<hex digits> != 0
 
 ConditionTests.cpp:<line number>:
 PASSED:
-  REQUIRE( cp != nullptr )
+  REQUIRE( cp != 0 )
 with expansion:
-  0x<hex digits> != nullptr
+  0x<hex digits> != 0
 
 ConditionTests.cpp:<line number>:
 PASSED:
-  REQUIRE( cpc != nullptr )
+  REQUIRE( cpc != 0 )
 with expansion:
-  0x<hex digits> != nullptr
+  0x<hex digits> != 0
 
 ConditionTests.cpp:<line number>:
 PASSED:
-  REQUIRE( returnsNull() == nullptr )
+  REQUIRE( returnsNull() == 0 )
 with expansion:
-  {null string} == nullptr
+  {null string} == 0
 
 ConditionTests.cpp:<line number>:
 PASSED:
-  REQUIRE( returnsConstNull() == nullptr )
+  REQUIRE( returnsConstNull() == 0 )
 with expansion:
-  {null string} == nullptr
+  {null string} == 0
 
 ConditionTests.cpp:<line number>:
 PASSED:
-  REQUIRE( nullptr != p )
+  REQUIRE( 0 != p )
 with expansion:
-  nullptr != 0x<hex digits>
+  0 != 0x<hex digits>
 
 -------------------------------------------------------------------------------
 Pointers can be converted to strings
@@ -7886,9 +7886,9 @@ TrickyTests.cpp:<line number>
 
 TrickyTests.cpp:<line number>:
 PASSED:
-  REQUIRE( obj.prop != nullptr )
+  REQUIRE( obj.prop != 0 )
 with expansion:
-  0x<hex digits> != nullptr
+  0x<hex digits> != 0
 
 -------------------------------------------------------------------------------
 checkedElse
@@ -8253,13 +8253,13 @@ MiscTests.cpp:<line number>
 
 MiscTests.cpp:<line number>:
 PASSED:
-  REQUIRE( makeString( false ) != static_cast<char*>(nullptr) )
+  REQUIRE( makeString( false ) != static_cast<char*>(0) )
 with expansion:
   "valid string" != {null string}
 
 MiscTests.cpp:<line number>:
 PASSED:
-  REQUIRE( makeString( true ) == static_cast<char*>(nullptr) )
+  REQUIRE( makeString( true ) == static_cast<char*>(0) )
 with expansion:
   {null string} == {null string}
 
@@ -8271,9 +8271,9 @@ TrickyTests.cpp:<line number>
 
 TrickyTests.cpp:<line number>:
 PASSED:
-  REQUIRE( ptr.get() == nullptr )
+  REQUIRE( ptr.get() == 0 )
 with expansion:
-  NULL == nullptr
+  NULL == 0
 
 -------------------------------------------------------------------------------
 pair<pair<int,const char *,pair<std::string,int> > -> toString
@@ -8797,18 +8797,18 @@ with expansion:
   "{ 0 }" == "{ 0 }"
 
 -------------------------------------------------------------------------------
-tuple<nullptr,int,const char *>
+tuple<0,int,const char *>
 -------------------------------------------------------------------------------
 ToStringTuple.cpp:<line number>
 ...............................................................................
 
 ToStringTuple.cpp:<line number>:
 PASSED:
-  CHECK( "{ nullptr, 42, \"Catch me\" }" == Catch::toString(value) )
+  CHECK( "{ 0, 42, \"Catch me\" }" == Catch::toString(value) )
 with expansion:
-  "{ nullptr, 42, "Catch me" }"
+  "{ 0, 42, "Catch me" }"
   ==
-  "{ nullptr, 42, "Catch me" }"
+  "{ 0, 42, "Catch me" }"
 
 -------------------------------------------------------------------------------
 tuple<string,string>
diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt
index 02ca9f68..490c810e 100644
--- a/projects/SelfTest/Baselines/junit.sw.approved.txt
+++ b/projects/SelfTest/Baselines/junit.sw.approved.txt
@@ -661,7 +661,7 @@ TrickyTests.cpp:<line number>
     <testcase classname="global" name="tuple&lt;>" time="{duration}"/>
     <testcase classname="global" name="tuple&lt;float,int>" time="{duration}"/>
     <testcase classname="global" name="tuple&lt;int>" time="{duration}"/>
-    <testcase classname="global" name="tuple&lt;nullptr,int,const char *>" time="{duration}"/>
+    <testcase classname="global" name="tuple&lt;0,int,const char *>" time="{duration}"/>
     <testcase classname="global" name="tuple&lt;string,string>" time="{duration}"/>
     <testcase classname="global" name="tuple&lt;tuple&lt;int>,tuple&lt;>,float>" time="{duration}"/>
     <testcase classname="global" name="vec&lt;vec&lt;string,alloc>> -> toString" time="{duration}"/>
diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt
index 3dd64def..b031e4d9 100644
--- a/projects/SelfTest/Baselines/xml.sw.approved.txt
+++ b/projects/SelfTest/Baselines/xml.sw.approved.txt
@@ -1015,7 +1015,7 @@
     <TestCase name="Equals string matcher, with NULL">
       <Expression success="true" type="REQUIRE_THAT" filename="projects/SelfTest/MiscTests.cpp" >
         <Original>
-          "", Equals(nullptr)
+          "", Equals(0)
         </Original>
         <Expanded>
           "" equals: ""
@@ -5709,10 +5709,10 @@ four"
     <TestCase name="Pointers can be compared to null">
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
         <Original>
-          p == nullptr
+          p == 0
         </Original>
         <Expanded>
-          NULL == nullptr
+          NULL == 0
         </Expanded>
       </Expression>
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
@@ -5725,50 +5725,50 @@ four"
       </Expression>
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
         <Original>
-          p != nullptr
+          p != 0
         </Original>
         <Expanded>
-          0x<hex digits> != nullptr
+          0x<hex digits> != 0
         </Expanded>
       </Expression>
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
         <Original>
-          cp != nullptr
+          cp != 0
         </Original>
         <Expanded>
-          0x<hex digits> != nullptr
+          0x<hex digits> != 0
         </Expanded>
       </Expression>
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
         <Original>
-          cpc != nullptr
+          cpc != 0
         </Original>
         <Expanded>
-          0x<hex digits> != nullptr
+          0x<hex digits> != 0
         </Expanded>
       </Expression>
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
         <Original>
-          returnsNull() == nullptr
+          returnsNull() == 0
         </Original>
         <Expanded>
-          {null string} == nullptr
+          {null string} == 0
         </Expanded>
       </Expression>
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
         <Original>
-          returnsConstNull() == nullptr
+          returnsConstNull() == 0
         </Original>
         <Expanded>
-          {null string} == nullptr
+          {null string} == 0
         </Expanded>
       </Expression>
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/ConditionTests.cpp" >
         <Original>
-          nullptr != p
+          0 != p
         </Original>
         <Expanded>
-          nullptr != 0x<hex digits>
+          0 != 0x<hex digits>
         </Expanded>
       </Expression>
       <OverallResult success="true"/>
@@ -8318,10 +8318,10 @@ there"
     <TestCase name="boolean member">
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/TrickyTests.cpp" >
         <Original>
-          obj.prop != nullptr
+          obj.prop != 0
         </Original>
         <Expanded>
-          0x<hex digits> != nullptr
+          0x<hex digits> != 0
         </Expanded>
       </Expression>
       <OverallResult success="true"/>
@@ -8709,7 +8709,7 @@ there"
     <TestCase name="null strings">
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/MiscTests.cpp" >
         <Original>
-          makeString( false ) != static_cast&lt;char*>(nullptr)
+          makeString( false ) != static_cast&lt;char*>(0)
         </Original>
         <Expanded>
           "valid string" != {null string}
@@ -8717,7 +8717,7 @@ there"
       </Expression>
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/MiscTests.cpp" >
         <Original>
-          makeString( true ) == static_cast&lt;char*>(nullptr)
+          makeString( true ) == static_cast&lt;char*>(0)
         </Original>
         <Expanded>
           {null string} == {null string}
@@ -8728,10 +8728,10 @@ there"
     <TestCase name="null_ptr">
       <Expression success="true" type="REQUIRE" filename="projects/SelfTest/TrickyTests.cpp" >
         <Original>
-          ptr.get() == nullptr
+          ptr.get() == 0
         </Original>
         <Expanded>
-          NULL == nullptr
+          NULL == 0
         </Expanded>
       </Expression>
       <OverallResult success="true"/>
@@ -9264,15 +9264,15 @@ there"
       </Expression>
       <OverallResult success="true"/>
     </TestCase>
-    <TestCase name="tuple&lt;nullptr,int,const char *>">
+    <TestCase name="tuple&lt;0,int,const char *>">
       <Expression success="true" type="CHECK" filename="projects/SelfTest/ToStringTuple.cpp" >
         <Original>
-          "{ nullptr, 42, \"Catch me\" }" == Catch::toString(value)
+          "{ 0, 42, \"Catch me\" }" == Catch::toString(value)
         </Original>
         <Expanded>
-          "{ nullptr, 42, "Catch me" }"
+          "{ 0, 42, "Catch me" }"
 ==
-"{ nullptr, 42, "Catch me" }"
+"{ 0, 42, "Catch me" }"
         </Expanded>
       </Expression>
       <OverallResult success="true"/>
diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py
index 2fe3245d..f665c317 100644
--- a/scripts/approvalTests.py
+++ b/scripts/approvalTests.py
@@ -1,116 +1,129 @@
-from  __future__ import  print_function
+from  __future__ import print_function
 
 import os
 import sys
 import subprocess
 import re
 
+import scriptCommon
 from scriptCommon import catchPath
 
-rootPath = os.path.join( catchPath, 'projects/SelfTest/Baselines' )
+rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines')
 
-filenameParser = re.compile( r'(.*)/(.*\..pp:)(.*)' )
-filelineParser = re.compile( r'(.*\..pp:)([0-9]*)(.*)' )
-pathParser = re.compile( r'(.*?)/(.*\..pp)(.*)' )
-lineNumberParser = re.compile( r'(.*)line="[0-9]*"(.*)' )
-hexParser = re.compile( r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)' )
-durationsParser = re.compile( r'(.*)time="[0-9]*\.[0-9]*"(.*)' )
-versionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)' )
-devVersionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)' )
+filenameParser = re.compile(r'(.*)/(.*\..pp:)(.*)')
+filelineParser = re.compile(r'(.*\..pp:)([0-9]*)(.*)')
+pathParser = re.compile(r'(.*?)/(.*\..pp)(.*)')
+lineNumberParser = re.compile(r'(.*)line="[0-9]*"(.*)')
+hexParser = re.compile(r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)')
+durationsParser = re.compile(r'(.*)time="[0-9]*\.[0-9]*"(.*)')
+versionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)')
+devVersionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)')
+nullParser = re.compile(r'(.*?)\b(__null|nullptr)\b(.*)')
 
 if len(sys.argv) == 2:
-	cmdPath = sys.argv[1]
+    cmdPath = sys.argv[1]
 else:
-	cmdPath = os.path.join( catchPath, 'projects/XCode/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' )
-#	cmdPath = os.path.join( catchPath, 'projects/CMake/cmake-build-debug/SelfTest' )
+    cmdPath = scriptCommon.getBuildExecutable()
+
+if not cmdPath.startswith("/"):
+    cmdPath = os.path.join(catchPath, cmdPath)
 
 overallResult = 0
 
-def filterLine( line ):
-	m = filenameParser.match( line )
-	if m:
-		line = m.group(2) + m.group(3)
-		m2 = filelineParser.match( line )
-		if m2:
-			line = m2.group(1) + "<line number>" + m2.group(3)
-	else:
-		m2 = lineNumberParser.match( line )
-		if m2:
-			line = m2.group(1) + m2.group(2)
-	m = pathParser.match( line )
-	if m:
-		path = "/" + m.group(2)
-		if path.startswith( catchPath ):
-			path = path[1+len(catchPath):]
-		line = m.group(1) + path + m.group(3)
-	m = devVersionParser.match( line )
-	if m:
-		line = m.group(1) + "<version>" + m.group(2)
-	else:
-		m = versionParser.match( line )
-		if m:
-			line = m.group(1) + "<version>" + m.group(2)
-
-	while True:
-		m = hexParser.match( line )
-		if m:
-			line = m.group(1) + "0x<hex digits>" + m.group(3)
-		else:
-			break
-	m = durationsParser.match( line )
-	if m:
-		line = m.group(1) + 'time="{duration}"' + m.group(2)
-	return line
-
-def approve( baseName, args ):
-	global overallResult
-	args[0:0] = [cmdPath]
-	if not os.path.exists( cmdPath ):
-			raise Exception( "Executable doesn't exist at " + cmdPath )
-	baselinesPath = os.path.join( rootPath, '{0}.approved.txt'.format( baseName ) )
-	rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) )
-	filteredResultsPath = os.path.join( rootPath, '{0}.unapproved.txt'.format( baseName ) )
-
-	f = open( rawResultsPath, 'w' )
-	subprocess.call( args, stdout=f, stderr=f )
-	f.close()
-
-	rawFile = open( rawResultsPath, 'r' )
-	filteredFile = open( filteredResultsPath, 'w' )
-	for line in rawFile:
-		filteredFile.write( filterLine( line ).rstrip() + "\n" )
-	filteredFile.close()
-	rawFile.close()
-
-	os.remove( rawResultsPath )
-	print()
-	print( baseName + ":" )
-	if os.path.exists( baselinesPath ):
-		diffResult = subprocess.call([ "diff", baselinesPath, filteredResultsPath ] )
-		if diffResult == 0:
-			os.remove( filteredResultsPath )
-			print( "  \033[92mResults matched" )
-		else:
-			print( "  \n****************************\n  \033[91mResults differed" )
-			if diffResult > overallResult:
-				overallResult = diffResult
-		print( "\033[0m" )
-	else:
-		print( "  first approval" )
-		if overallResult == 0:
-			overallResult = 1
+def filterLine(line):
+    m = filenameParser.match(line)
+    if m:
+        line = m.group(2) + m.group(3)
+        m2 = filelineParser.match(line)
+        if m2:
+            line = m2.group(1) + "<line number>" + m2.group(3)
+    else:
+        m2 = lineNumberParser.match(line)
+        if m2:
+            line = m2.group(1) + m2.group(2)
+    m = pathParser.match(line)
+    if m:
+        path = "/" + m.group(2)
+        if path.startswith(catchPath):
+            path = path[1 + len(catchPath):]
+        line = m.group(1) + path + m.group(3)
+    m = devVersionParser.match(line)
+    if m:
+        line = m.group(1) + "<version>" + m.group(2)
+    else:
+        m = versionParser.match(line)
+        if m:
+            line = m.group(1) + "<version>" + m.group(2)
+
+    m = nullParser.match(line)
+    if m:
+        line = m.group(1) + "0" + m.group(3)
+
+    while True:
+        m = hexParser.match(line)
+        if m:
+            line = m.group(1) + "0x<hex digits>" + m.group(3)
+        else:
+            break
+    m = durationsParser.match(line)
+    if m:
+        line = m.group(1) + 'time="{duration}"' + m.group(2)
+    return line
+
+
+def approve(baseName, args):
+    global overallResult
+    args[0:0] = [cmdPath]
+    if not os.path.exists(cmdPath):
+        raise Exception("Executable doesn't exist at " + cmdPath)
+    baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
+    rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName))
+    filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
+
+    f = open(rawResultsPath, 'w')
+    subprocess.call(args, stdout=f, stderr=f)
+    f.close()
+
+    rawFile = open(rawResultsPath, 'r')
+    filteredFile = open(filteredResultsPath, 'w')
+    for line in rawFile:
+        filteredFile.write(filterLine(line).rstrip() + "\n")
+    filteredFile.close()
+    rawFile.close()
+
+    os.remove(rawResultsPath)
+    print()
+    print(baseName + ":")
+    if os.path.exists(baselinesPath):
+        diffResult = subprocess.call(["diff", baselinesPath, filteredResultsPath])
+        if diffResult == 0:
+            os.remove(filteredResultsPath)
+            print("  \033[92mResults matched")
+        else:
+            print("  \n****************************\n  \033[91mResults differed")
+            if diffResult > overallResult:
+                overallResult = diffResult
+        print("\033[0m")
+    else:
+        print("  first approval")
+        if overallResult == 0:
+            overallResult = 1
+
+
+print("Running approvals against executable:")
+print("  " + cmdPath)
 
 # Standard console reporter
-approve( "console.std", ["~_", "--order", "lex"] )
+approve("console.std", ["~_", "--order", "lex"])
 # console reporter, include passes, warn about No Assertions
-approve( "console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"] )
+approve("console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"])
 # console reporter, include passes, warn about No Assertions, limit failures to first 4
-approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"] )
+approve("console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
 # junit reporter, include passes, warn about No Assertions
-approve( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"] )
+approve("junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
 # xml reporter, include passes, warn about No Assertions
-approve( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"] )
+approve("xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
 
 if overallResult != 0:
-	print( "run approve.py to approve new baselines" )
-exit( overallResult)
+    print("If these differenecs are expected run approve.py to approve new baselines")
+exit(overallResult)
diff --git a/scripts/scriptCommon.py b/scripts/scriptCommon.py
index 6ac381ad..9adadb6d 100644
--- a/scripts/scriptCommon.py
+++ b/scripts/scriptCommon.py
@@ -2,8 +2,13 @@ import os
 import sys
 import subprocess
 
+
 catchPath = os.path.dirname(os.path.realpath( os.path.dirname(sys.argv[0])))
 
+def getBuildExecutable():
+    dir = os.environ.get('CATCH_DEV_OUT_DIR', "cmake-build-debug/SelfTest")
+    return dir
+
 def runAndCapture( args ):
     child = subprocess.Popen(" ".join( args ), shell=True, stdout=subprocess.PIPE)
     lines = []
@@ -18,4 +23,4 @@ def runAndCapture( args ):
                 line = ""
             else:
                 line = line + out
-    return lines
\ No newline at end of file
+    return lines