Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/tools/scripts/approvalTests.py
diff options
context:
space:
mode:
authorMartin Jeřábek2021-02-06 20:12:07 +0100
committerMartin Hořeňovský2022-01-01 14:02:23 +0100
commitccd67b293d7595a0cf2dc6c4bfb2017fc18c1f14 (patch)
treeba5f166d1cf867ad97617656083354fbddfc5d9f /tools/scripts/approvalTests.py
parent6b55f5d780c5d8e935720f9d7d8ed28a3ba8617f (diff)
downloadcatch3-ccd67b293d7595a0cf2dc6c4bfb2017fc18c1f14.tar.gz
Add support for multiple parallel reporters
This requires a bunch of different changes across the reporter subsystem. * We need to handle multiple reporters and their differing preferences in `ListeningReporter`, e.g. what to do when we mix reporters that capture and don't capture stdout. * We need to change how the reporter is given output and how we parse reporter's output destination from CLI. * Approval tests need to handle multireporter option
Diffstat (limited to 'tools/scripts/approvalTests.py')
-rwxr-xr-xtools/scripts/approvalTests.py45
1 files changed, 40 insertions, 5 deletions
diff --git a/tools/scripts/approvalTests.py b/tools/scripts/approvalTests.py
index 99b94152..1057c4ad 100755
--- a/tools/scripts/approvalTests.py
+++ b/tools/scripts/approvalTests.py
@@ -164,19 +164,36 @@ def filterLine(line, isCompact):
return line
-def approve(baseName, args):
- global overallResult
+def get_rawResultsPath(baseName):
+ return os.path.join(rootPath, '_{0}.tmp'.format(baseName))
+
+
+def get_baselinesPath(baseName):
+ return os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
+
+
+def get_filteredResultsPath(baseName):
+ return os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
+
+
+def run_test(baseName, args):
args[0:0] = [cmdPath]
if not os.path.exists(cmdPath):
raise Exception("Executable doesn't exist at " + cmdPath)
- baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
- rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName))
- filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName))
+ print(args)
+ rawResultsPath = get_rawResultsPath(baseName)
f = open(rawResultsPath, 'w')
subprocess.call(args, stdout=f, stderr=f)
f.close()
+
+def check_outputs(baseName):
+ global overallResult
+ rawResultsPath = get_rawResultsPath(baseName)
+ baselinesPath = get_baselinesPath(baseName)
+ filteredResultsPath = get_filteredResultsPath(baseName)
+
rawFile = io.open(rawResultsPath, 'r', encoding='utf-8', errors='surrogateescape')
filteredFile = io.open(filteredResultsPath, 'w', encoding='utf-8', errors='surrogateescape')
for line in rawFile:
@@ -204,6 +221,11 @@ def approve(baseName, args):
overallResult = 1
+def approve(baseName, args):
+ run_test(baseName, args)
+ check_outputs(baseName)
+
+
print("Running approvals against executable:")
print(" " + cmdPath)
@@ -222,6 +244,19 @@ for reporter in reporters:
reporter_args = ['-r', reporter]
approve(filename, common_args + reporter_args)
+## All reporters at the same time
+
+common_args = ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "--order", "lex", "--rng-seed", "1"]
+filenames = ['{}.sw.multi'.format(reporter) for reporter in reporters]
+reporter_args = []
+for reporter, filename in zip(reporters, filenames):
+ reporter_args += ['-r', '{}::{}'.format(reporter, get_rawResultsPath(filename))]
+
+run_test("default.sw.multi", common_args + reporter_args)
+check_outputs("default.sw.multi")
+for reporter, filename in zip(reporters, filenames):
+ check_outputs(filename)
+
if overallResult != 0:
print("If these differences are expected, run approve.py to approve new baselines.")