Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/tools
diff options
context:
space:
mode:
authorMartin Hořeňovský2022-01-01 16:19:31 +0100
committerGitHub2022-01-01 16:19:31 +0100
commitf2f585b4788b61ed202e78b73b105f099d9cce7b (patch)
tree174f72ba09e22e564a571f96c21a175a68d85103 /tools
parent6b55f5d780c5d8e935720f9d7d8ed28a3ba8617f (diff)
parentb5547f2ef6be18f70d2e9638d8f1eb0d8d42d2ba (diff)
downloadcatch3-f2f585b4788b61ed202e78b73b105f099d9cce7b.tar.gz
Merge pull request #2183 from mjerabek/dev-multiple-reporters
Add support for multiple reporters
Diffstat (limited to 'tools')
-rwxr-xr-xtools/scripts/approvalTests.py45
1 files changed, 40 insertions, 5 deletions
diff --git a/tools/scripts/approvalTests.py b/tools/scripts/approvalTests.py
index 99b94152..1057c4ad 100755
--- a/tools/scripts/approvalTests.py
+++ b/tools/scripts/approvalTests.py
@@ -164,19 +164,36 @@ def filterLine(line, isCompact):
return line
-def approve(baseName, args):
- global overallResult
+def get_rawResultsPath(baseName):
+ return os.path.join(rootPath, '_{0}.tmp'.format(baseName))
+
+
+def get_baselinesPath(baseName):
+ return os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
+
+
+def get_filteredResultsPath(baseName):
+ return os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
+
+
+def run_test(baseName, args):
args[0:0] = [cmdPath]
if not os.path.exists(cmdPath):
raise Exception("Executable doesn't exist at " + cmdPath)
- baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
- rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName))
- filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
+ print(args)
+ rawResultsPath = get_rawResultsPath(baseName)
f = open(rawResultsPath, 'w')
subprocess.call(args, stdout=f, stderr=f)
f.close()
+
+def check_outputs(baseName):
+ global overallResult
+ rawResultsPath = get_rawResultsPath(baseName)
+ baselinesPath = get_baselinesPath(baseName)
+ filteredResultsPath = get_filteredResultsPath(baseName)
+
rawFile = io.open(rawResultsPath, 'r', encoding='utf-8', errors='surrogateescape')
filteredFile = io.open(filteredResultsPath, 'w', encoding='utf-8', errors='surrogateescape')
for line in rawFile:
@@ -204,6 +221,11 @@ def approve(baseName, args):
overallResult = 1
+def approve(baseName, args):
+ run_test(baseName, args)
+ check_outputs(baseName)
+
+
print("Running approvals against executable:")
print(" " + cmdPath)
@@ -222,6 +244,19 @@ for reporter in reporters:
reporter_args = ['-r', reporter]
approve(filename, common_args + reporter_args)
+## All reporters at the same time
+
+common_args = ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "--order", "lex", "--rng-seed", "1"]
+filenames = ['{}.sw.multi'.format(reporter) for reporter in reporters]
+reporter_args = []
+for reporter, filename in zip(reporters, filenames):
+ reporter_args += ['-r', '{}::{}'.format(reporter, get_rawResultsPath(filename))]
+
+run_test("default.sw.multi", common_args + reporter_args)
+check_outputs("default.sw.multi")
+for reporter, filename in zip(reporters, filenames):
+ check_outputs(filename)
+
if overallResult != 0:
print("If these differences are expected, run approve.py to approve new baselines.")