Approval tests can use Catch path fixed in env. var, and convert nullptr and __null to 0 for comparison

2 files changed, 112 insertions, 94 deletions

diff --git a/scripts/approvalTests.py b/scripts/approvalTests.py
index 2fe3245d..f665c317 100644
--- a/scripts/approvalTests.py
+++ b/scripts/approvalTests.py
@@ -1,116 +1,129 @@
-from  __future__ import  print_function
+from  __future__ import print_function
 
 import os
 import sys
 import subprocess
 import re
 
+import scriptCommon
 from scriptCommon import catchPath
 
-rootPath = os.path.join( catchPath, 'projects/SelfTest/Baselines' )
+rootPath = os.path.join(catchPath, 'projects/SelfTest/Baselines')
 
-filenameParser = re.compile( r'(.*)/(.*\..pp:)(.*)' )
-filelineParser = re.compile( r'(.*\..pp:)([0-9]*)(.*)' )
-pathParser = re.compile( r'(.*?)/(.*\..pp)(.*)' )
-lineNumberParser = re.compile( r'(.*)line="[0-9]*"(.*)' )
-hexParser = re.compile( r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)' )
-durationsParser = re.compile( r'(.*)time="[0-9]*\.[0-9]*"(.*)' )
-versionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)' )
-devVersionParser = re.compile( r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)' )
+filenameParser = re.compile(r'(.*)/(.*\..pp:)(.*)')
+filelineParser = re.compile(r'(.*\..pp:)([0-9]*)(.*)')
+pathParser = re.compile(r'(.*?)/(.*\..pp)(.*)')
+lineNumberParser = re.compile(r'(.*)line="[0-9]*"(.*)')
+hexParser = re.compile(r'(.*)\b(0[xX][0-9a-fA-F]+)\b(.*)')
+durationsParser = re.compile(r'(.*)time="[0-9]*\.[0-9]*"(.*)')
+versionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*(.*)')
+devVersionParser = re.compile(r'(.*?)Catch v[0-9]*\.[0-9]*\.[0-9]*-develop\.[0-9]*(.*)')
+nullParser = re.compile(r'(.*?)\b(__null|nullptr)\b(.*)')
 
 if len(sys.argv) == 2:
-	cmdPath = sys.argv[1]
+    cmdPath = sys.argv[1]
 else:
-	cmdPath = os.path.join( catchPath, 'projects/XCode/CatchSelfTest/DerivedData/CatchSelfTest/Build/Products/Debug/CatchSelfTest' )
-#	cmdPath = os.path.join( catchPath, 'projects/CMake/cmake-build-debug/SelfTest' )
+    cmdPath = scriptCommon.getBuildExecutable()
+
+if not cmdPath.startswith("/"):
+    cmdPath = os.path.join(catchPath, cmdPath)
 
 overallResult = 0
 
-def filterLine( line ):
-	m = filenameParser.match( line )
-	if m:
-		line = m.group(2) + m.group(3)
-		m2 = filelineParser.match( line )
-		if m2:
-			line = m2.group(1) + "<line number>" + m2.group(3)
-	else:
-		m2 = lineNumberParser.match( line )
-		if m2:
-			line = m2.group(1) + m2.group(2)
-	m = pathParser.match( line )
-	if m:
-		path = "/" + m.group(2)
-		if path.startswith( catchPath ):
-			path = path[1+len(catchPath):]
-		line = m.group(1) + path + m.group(3)
-	m = devVersionParser.match( line )
-	if m:
-		line = m.group(1) + "<version>" + m.group(2)
-	else:
-		m = versionParser.match( line )
-		if m:
-			line = m.group(1) + "<version>" + m.group(2)
-
-	while True:
-		m = hexParser.match( line )
-		if m:
-			line = m.group(1) + "0x<hex digits>" + m.group(3)
-		else:
-			break
-	m = durationsParser.match( line )
-	if m:
-		line = m.group(1) + 'time="{duration}"' + m.group(2)
-	return line
-
-def approve( baseName, args ):
-	global overallResult
-	args[0:0] = [cmdPath]
-	if not os.path.exists( cmdPath ):
-			raise Exception( "Executable doesn't exist at " + cmdPath )
-	baselinesPath = os.path.join( rootPath, '{0}.approved.txt'.format( baseName ) )
-	rawResultsPath = os.path.join( rootPath, '_{0}.tmp'.format( baseName ) )
-	filteredResultsPath = os.path.join( rootPath, '{0}.unapproved.txt'.format( baseName ) )
-
-	f = open( rawResultsPath, 'w' )
-	subprocess.call( args, stdout=f, stderr=f )
-	f.close()
-
-	rawFile = open( rawResultsPath, 'r' )
-	filteredFile = open( filteredResultsPath, 'w' )
-	for line in rawFile:
-		filteredFile.write( filterLine( line ).rstrip() + "\n" )
-	filteredFile.close()
-	rawFile.close()
-
-	os.remove( rawResultsPath )
-	print()
-	print( baseName + ":" )
-	if os.path.exists( baselinesPath ):
-		diffResult = subprocess.call([ "diff", baselinesPath, filteredResultsPath ] )
-		if diffResult == 0:
-			os.remove( filteredResultsPath )
-			print( "  \033[92mResults matched" )
-		else:
-			print( "  \n****************************\n  \033[91mResults differed" )
-			if diffResult > overallResult:
-				overallResult = diffResult
-		print( "\033[0m" )
-	else:
-		print( "  first approval" )
-		if overallResult == 0:
-			overallResult = 1
+def filterLine(line):
+    m = filenameParser.match(line)
+    if m:
+        line = m.group(2) + m.group(3)
+        m2 = filelineParser.match(line)
+        if m2:
+            line = m2.group(1) + "<line number>" + m2.group(3)
+    else:
+        m2 = lineNumberParser.match(line)
+        if m2:
+            line = m2.group(1) + m2.group(2)
+    m = pathParser.match(line)
+    if m:
+        path = "/" + m.group(2)
+        if path.startswith(catchPath):
+            path = path[1 + len(catchPath):]
+        line = m.group(1) + path + m.group(3)
+    m = devVersionParser.match(line)
+    if m:
+        line = m.group(1) + "<version>" + m.group(2)
+    else:
+        m = versionParser.match(line)
+        if m:
+            line = m.group(1) + "<version>" + m.group(2)
+
+    m = nullParser.match(line)
+    if m:
+        line = m.group(1) + "0" + m.group(3)
+
+    while True:
+        m = hexParser.match(line)
+        if m:
+            line = m.group(1) + "0x<hex digits>" + m.group(3)
+        else:
+            break
+    m = durationsParser.match(line)
+    if m:
+        line = m.group(1) + 'time="{duration}"' + m.group(2)
+    return line
+
+
+def approve(baseName, args):
+    global overallResult
+    args[0:0] = [cmdPath]
+    if not os.path.exists(cmdPath):
+        raise Exception("Executable doesn't exist at " + cmdPath)
+    baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
+    rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName))
+    filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
+
+    f = open(rawResultsPath, 'w')
+    subprocess.call(args, stdout=f, stderr=f)
+    f.close()
+
+    rawFile = open(rawResultsPath, 'r')
+    filteredFile = open(filteredResultsPath, 'w')
+    for line in rawFile:
+        filteredFile.write(filterLine(line).rstrip() + "\n")
+    filteredFile.close()
+    rawFile.close()
+
+    os.remove(rawResultsPath)
+    print()
+    print(baseName + ":")
+    if os.path.exists(baselinesPath):
+        diffResult = subprocess.call(["diff", baselinesPath, filteredResultsPath])
+        if diffResult == 0:
+            os.remove(filteredResultsPath)
+            print("  \033[92mResults matched")
+        else:
+            print("  \n****************************\n  \033[91mResults differed")
+            if diffResult > overallResult:
+                overallResult = diffResult
+        print("\033[0m")
+    else:
+        print("  first approval")
+        if overallResult == 0:
+            overallResult = 1
+
+
+print("Running approvals against executable:")
+print("  " + cmdPath)
 
 # Standard console reporter
-approve( "console.std", ["~_", "--order", "lex"] )
+approve("console.std", ["~_", "--order", "lex"])
 # console reporter, include passes, warn about No Assertions
-approve( "console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"] )
+approve("console.sw", ["~_", "-s", "-w", "NoAssertions", "--order", "lex"])
 # console reporter, include passes, warn about No Assertions, limit failures to first 4
-approve( "console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"] )
+approve("console.swa4", ["~_", "-s", "-w", "NoAssertions", "-x", "4", "--order", "lex"])
 # junit reporter, include passes, warn about No Assertions
-approve( "junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"] )
+approve("junit.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "junit", "--order", "lex"])
 # xml reporter, include passes, warn about No Assertions
-approve( "xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"] )
+approve("xml.sw", ["~_", "-s", "-w", "NoAssertions", "-r", "xml", "--order", "lex"])
 
 if overallResult != 0:
-	print( "run approve.py to approve new baselines" )
-exit( overallResult)
+    print("If these differenecs are expected run approve.py to approve new baselines")
+exit(overallResult)
diff --git a/scripts/scriptCommon.py b/scripts/scriptCommon.py
index 6ac381ad..9adadb6d 100644
--- a/scripts/scriptCommon.py
+++ b/scripts/scriptCommon.py
@@ -2,8 +2,13 @@ import os
 import sys
 import subprocess
 
+
 catchPath = os.path.dirname(os.path.realpath( os.path.dirname(sys.argv[0])))
 
+def getBuildExecutable():
+    dir = os.environ.get('CATCH_DEV_OUT_DIR', "cmake-build-debug/SelfTest")
+    return dir
+
 def runAndCapture( args ):
     child = subprocess.Popen(" ".join( args ), shell=True, stdout=subprocess.PIPE)
     lines = []
@@ -18,4 +23,4 @@ def runAndCapture( args ):
                 line = ""
             else:
                 line = line + out
-    return lines
\ No newline at end of file
+    return lines