diff options
| author | Martin Jeřábek | 2021-02-06 20:12:07 +0100 |
|---|---|---|
| committer | Martin Hořeňovský | 2022-01-01 14:02:23 +0100 |
| commit | ccd67b293d7595a0cf2dc6c4bfb2017fc18c1f14 (patch) | |
| tree | ba5f166d1cf867ad97617656083354fbddfc5d9f /tools | |
| parent | 6b55f5d780c5d8e935720f9d7d8ed28a3ba8617f (diff) | |
| download | catch3-ccd67b293d7595a0cf2dc6c4bfb2017fc18c1f14.tar.gz | |
Add support for multiple parallel reporters
This requires a bunch of different changes across the reporter
subsystem.
* We need to handle multiple reporters and their differing
preferences in `ListeningReporter`, e.g. what to do when
we mix reporters that capture and don't capture stdout.
* We need to change how the reporter is given output and
how we parse reporter's output destination from CLI.
* Approval tests need to handle multireporter option
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/scripts/approvalTests.py | 45 |
1 files changed, 40 insertions, 5 deletions
diff --git a/tools/scripts/approvalTests.py b/tools/scripts/approvalTests.py index 99b94152..1057c4ad 100755 --- a/tools/scripts/approvalTests.py +++ b/tools/scripts/approvalTests.py @@ -164,19 +164,36 @@ def filterLine(line, isCompact): return line -def approve(baseName, args): - global overallResult +def get_rawResultsPath(baseName): + return os.path.join(rootPath, '_{0}.tmp'.format(baseName)) + + +def get_baselinesPath(baseName): + return os.path.join(rootPath, '{0}.approved.txt'.format(baseName)) + + +def get_filteredResultsPath(baseName): + return os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName)) + + +def run_test(baseName, args): args[0:0] = [cmdPath] if not os.path.exists(cmdPath): raise Exception("Executable doesn't exist at " + cmdPath) - baselinesPath = os.path.join(rootPath, '{0}.approved.txt'.format(baseName)) - rawResultsPath = os.path.join(rootPath, '_{0}.tmp'.format(baseName)) - filteredResultsPath = os.path.join(rootPath, '{0}.unapproved.txt'.format(baseName)) + print(args) + rawResultsPath = get_rawResultsPath(baseName) f = open(rawResultsPath, 'w') subprocess.call(args, stdout=f, stderr=f) f.close() + +def check_outputs(baseName): + global overallResult + rawResultsPath = get_rawResultsPath(baseName) + baselinesPath = get_baselinesPath(baseName) + filteredResultsPath = get_filteredResultsPath(baseName) + rawFile = io.open(rawResultsPath, 'r', encoding='utf-8', errors='surrogateescape') filteredFile = io.open(filteredResultsPath, 'w', encoding='utf-8', errors='surrogateescape') for line in rawFile: @@ -204,6 +221,11 @@ def approve(baseName, args): overallResult = 1 +def approve(baseName, args): + run_test(baseName, args) + check_outputs(baseName) + + print("Running approvals against executable:") print(" " + cmdPath) @@ -222,6 +244,19 @@ for reporter in reporters: reporter_args = ['-r', reporter] approve(filename, common_args + reporter_args) +## All reporters at the same time + +common_args = ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "--order", "lex", "--rng-seed", "1"] +filenames = ['{}.sw.multi'.format(reporter) for reporter in reporters] +reporter_args = [] +for reporter, filename in zip(reporters, filenames): + reporter_args += ['-r', '{}::{}'.format(reporter, get_rawResultsPath(filename))] + +run_test("default.sw.multi", common_args + reporter_args) +check_outputs("default.sw.multi") +for reporter, filename in zip(reporters, filenames): + check_outputs(filename) + if overallResult != 0: print("If these differences are expected, run approve.py to approve new baselines.") |