Replace Catch2's runner with the Catch3 runner

Catch3 keeps Catch2's API for writing tests (test cases, sections,
assertions, matchers, generators, listeners, benchmarks) and replaces its
session, command line and reporters with a runner of its own.

The runner starts the test binary again as worker processes that take the
next test when they are free. A crash, an exit or a hang ends one test, not
the run. On top of that it adds:

- retries that tell deterministic, flaky and order-dependent failures
  apart, and a bisection that names the test leaving state behind;
- a history of verdicts and durations, kept under a file lock, that drives
  automatic quarantine, adaptive time limits, predictive ordering,
  incremental runs, failure replay and analytics;
- selection by change, past failure or prediction, dependency closure,
  and sharding by a stable hash or by recorded durations;
- tags for serial and isolated tests, shared resources, dependencies,
  time limits and retries;
- a failure category, a signature, an analysis and a replay command for
  every failure;
- console, JUnit XML (with flakyFailure and rerunFailure), JSON and Chrome
  trace reports, and a versioned discovery listing (--list=json);
- a temporary directory per test (TMPDIR, TMP, TEMP and
  TestTemporaryDirectory());
- Bazel's test environment: XML output file, test filter, sharding, seed,
  premature exit file and undeclared outputs.

The command line is parsed by a hand-written, typed parser. Catch::Session
remains as a thin facade over the runner for custom main functions.

The result model in Progmasoft/Catch3/Results.hpp was rewritten around
tests and attempts, and the JUnit writer now writes it. The property
engine's example database is updated under a file lock with an atomic
rename, so parallel workers can write it at the same time.

Removed: Clara, the command-line and reporter-spec parsers, every reporter
and CATCH_REGISTER_REPORTER, listing, text flow, console colour and width,
the JSON and XML writers, sharding in the test registry, approval tests and
their baselines, Conan packaging, and the unused third_party/clara.hpp.
Listeners keep working; reporters' list and skip events are gone.

The Catch2 library is now compiled as C++20; code that includes only
Catch2's headers still needs C++14. Duplicate test cases are still refused
at start-up. Version 0.3.0.

ExtraTests now turn exceptions off for clang-cl with MSVC-style options,
which -fno-exceptions did not do.

3 files changed, 53 insertions, 275 deletions

diff --git a/tools/scripts/approvalTests.py b/tools/scripts/approvalTests.py
deleted file mode 100755
index 72ff26ca..00000000
--- a/tools/scripts/approvalTests.py
+++ /dev/null
@@ -1,243 +0,0 @@
-#!/usr/bin/env python3
-
-import io
-import os
-import sys
-import subprocess
-import re
-import difflib
-import shutil
-
-import scriptCommon
-from scriptCommon import catchPath
-
-if os.name == 'nt':
-    # Enable console colours on windows
-    os.system('')
-
-rootPath = os.path.join(catchPath, 'tests/SelfTest/Baselines')
-# Init so it is guaranteed to fail loudly if the scoping gets messed up
-outputDirPath = None
-
-if len(sys.argv) == 3:
-    cmdPath = sys.argv[1]
-    outputDirBasePath = sys.argv[2]
-    outputDirPath = os.path.join(outputDirBasePath, 'ApprovalTests')
-    if not os.path.isdir(outputDirPath):
-        os.mkdir(outputDirPath)
-else:
-    print('Usage: {} path-to-SelfTest-executable path-to-temp-output-dir'.format(sys.argv[0]))
-    exit(1)
-
-
-
-def get_rawResultsPath(baseName):
-    return os.path.join(outputDirPath, '_{0}.tmp'.format(baseName))
-
-def get_baselinesPath(baseName):
-    return os.path.join(rootPath, '{0}.approved.txt'.format(baseName))
-
-def _get_unapprovedPath(path, baseName):
-    return os.path.join(path, '{0}.unapproved.txt'.format(baseName))
-
-def get_filteredResultsPath(baseName):
-    return _get_unapprovedPath(outputDirPath, baseName)
-
-def get_unapprovedResultsPath(baseName):
-    return _get_unapprovedPath(rootPath, baseName)
-
-langFilenameParser = re.compile(r'(.+\.[ch]pp)')
-filelocParser = re.compile(r'''
-    (?P<path_prefix>tests/SelfTest/(?:\w+/)*)  # We separate prefix and fname, so that
-    (?P<filename>\w+\.tests\.[ch]pp)           # we can keep only filename
-    (?::|\()                                   # Linux has : as separator between fname and line number, Windows uses (
-    (\d*)                                      # line number
-    \)?                                        # Windows also uses an ending separator, )
-''', re.VERBOSE)
-lineNumberParser = re.compile(r' line="[0-9]*"')
-hexParser = re.compile(r'\b(0[xX][0-9a-fA-F]+)\b')
-# Note: junit must serialize time with 3 (or less) decimal places
-#       before generalizing this parser, make sure that this is checked
-#       in other places too.
-junitDurationsParser = re.compile(r' time="[0-9]+\.[0-9]{3}"')
-durationParser = re.compile(r''' duration=['"][0-9]+['"]''')
-timestampsParser = re.compile(r'\d{4}-\d{2}-\d{2}T\d{2}\:\d{2}\:\d{2}Z')
-versionParser = re.compile(r'[0-9]+\.[0-9]+\.[0-9]+(-\w*\.[0-9]+)?')
-nullParser = re.compile(r'\b(__null|nullptr)\b')
-exeNameParser = re.compile(r'''
-    \b
-    SelfTest                  # Expected executable name
-    (?:.exe)?                 # Executable name contains .exe on Windows.
-    \b
-''', re.VERBOSE)
-# This is a hack until something more reasonable is figured out
-specialCaseParser = re.compile(r'file\((\d+)\)')
-
-sinceEpochParser = re.compile(r'\d+ .+ since epoch')
-
-# The weird OR is there to always have at least empty string for group 1
-tapTestNumParser = re.compile(r'^((?:not ok)|(?:ok)|(?:warning)|(?:info)) (\d+) -')
-
-overallResult = 0
-
-def diffFiles(fileA, fileB):
-    with io.open(fileA, 'r', encoding='utf-8', errors='surrogateescape') as file:
-        aLines = [line.rstrip() for line in file.readlines()]
-    with io.open(fileB, 'r', encoding='utf-8', errors='surrogateescape') as file:
-        bLines = [line.rstrip() for line in file.readlines()]
-
-    shortenedFilenameA = fileA.rsplit(os.sep, 1)[-1]
-    shortenedFilenameB = fileB.rsplit(os.sep, 1)[-1]
-
-    diff = difflib.unified_diff(aLines, bLines, fromfile=shortenedFilenameA, tofile=shortenedFilenameB, n=0)
-    return [line for line in diff if line[0] in ('+', '-')]
-
-
-def normalizeFilepath(line):
-    # Sometimes the path separators used by compiler and Python can differ,
-    # so we try to match the path with both forward and backward path
-    # separators, to make the paths relative to Catch2 repo root.
-    forwardSlashPath = catchPath.replace('\\', '/')
-    if forwardSlashPath in line:
-        line = line.replace(forwardSlashPath + '/', '')
-    backwardSlashPath = catchPath.replace('/', '\\')
-    if backwardSlashPath in line:
-        line = line.replace(backwardSlashPath + '\\', '')
-
-    m = langFilenameParser.match(line)
-    if m:
-        filepath = m.group(0)
-        # go from \ in windows paths to /
-        filepath = filepath.replace('\\', '/')
-        # remove start of relative path
-        filepath = filepath.replace('../', '')
-        line = line[:m.start()] + filepath + line[m.end():]
-
-    return line
-
-def filterLine(line, isCompact):
-    line = normalizeFilepath(line)
-
-    # strip source line numbers
-    # Note that this parser assumes an already normalized filepath from above,
-    # and might break terribly if it is moved around before the normalization.
-    line = filelocParser.sub(r'\g<filename>:<line number>', line)
-
-    line = lineNumberParser.sub(" ", line)
-
-    if isCompact:
-        line = line.replace(': FAILED', ': failed')
-        line = line.replace(': PASSED', ': passed')
-
-    # strip out the test order number in TAP to avoid massive diffs for every change
-    line = tapTestNumParser.sub(r"\g<1> {test-number} -", line)
-
-    # strip Catch2 version number
-    line = versionParser.sub("<version>", line)
-
-    # replace *null* with 0
-    line = nullParser.sub("0", line)
-
-    # strip executable name
-    line = exeNameParser.sub("<exe-name>", line)
-
-    # strip hexadecimal numbers (presumably pointer values)
-    line = hexParser.sub("0x<hex digits>", line)
-
-    # strip durations and timestamps
-    line = junitDurationsParser.sub(' time="{duration}"', line)
-    line = durationParser.sub(' duration="{duration}"', line)
-    line = timestampsParser.sub('{iso8601-timestamp}', line)
-    line = specialCaseParser.sub(r'file:\g<1>', line)
-    line = sinceEpochParser.sub('{since-epoch-report}', line)
-    return line
-
-
-def run_test(baseName, args):
-    args[0:0] = [cmdPath]
-    if not os.path.exists(cmdPath):
-        raise Exception("Executable doesn't exist at " + cmdPath)
-
-    print(args)
-    rawResultsPath = get_rawResultsPath(baseName)
-    f = open(rawResultsPath, 'w')
-    subprocess.call(args, stdout=f, stderr=f)
-    f.close()
-
-
-def check_outputs(baseName):
-    global overallResult
-    rawResultsPath = get_rawResultsPath(baseName)
-    baselinesPath = get_baselinesPath(baseName)
-    filteredResultsPath = get_filteredResultsPath(baseName)
-
-    rawFile = io.open(rawResultsPath, 'r', encoding='utf-8', errors='surrogateescape')
-    filteredFile = io.open(filteredResultsPath, 'w', encoding='utf-8', errors='surrogateescape')
-    for line in rawFile:
-        filteredFile.write(filterLine(line, 'compact' in baseName).rstrip() + "\n")
-    filteredFile.close()
-    rawFile.close()
-
-    os.remove(rawResultsPath)
-    print()
-    print(baseName + ":")
-    if not os.path.exists(baselinesPath):
-        print(  'first approval')
-        overallResult += 1
-        return
-
-    diffResult = diffFiles(baselinesPath, filteredResultsPath)
-    if diffResult:
-        print('\n'.join(diffResult))
-        print("  \n****************************\n  \033[91mResults differed\033[0m")
-        overallResult += 1
-        shutil.move(filteredResultsPath, get_unapprovedResultsPath(baseName))
-    else:
-        os.remove(filteredResultsPath)
-        print("  \033[92mResults matched\033[0m")
-
-
-def approve(baseName, args):
-    run_test(baseName, args)
-    check_outputs(baseName)
-
-
-print("Running approvals against executable:")
-print("  " + cmdPath)
-
-
-base_args = ["--order", "lex", "--rng-seed", "1", "--colour-mode", "none"]
-
-## special cases first:
-# Standard console reporter
-approve("console.std", ["~[!nonportable]~[!benchmark]~[approvals] *"] + base_args)
-
-# console reporter, include passes, warn about No Assertions, limit failures to first 4
-approve("console.swa4", ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions", "-x", "4"] + base_args)
-
-## Common reporter checks: include passes, warn about No Assertions
-reporters = ('console', 'junit', 'xml', 'compact', 'sonarqube', 'tap', 'teamcity', 'automake')
-for reporter in reporters:
-    filename = '{}.sw'.format(reporter)
-    common_args = ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions"] + base_args
-    reporter_args = ['-r', reporter]
-    approve(filename, common_args + reporter_args)
-
-
-## All reporters at the same time
-common_args = ["~[!nonportable]~[!benchmark]~[approvals] *", "-s", "-w", "NoAssertions"] + base_args
-filenames = ['{}.sw.multi'.format(reporter) for reporter in reporters]
-reporter_args = []
-for reporter, filename in zip(reporters, filenames):
-    reporter_args += ['-r', '{}::out={}'.format(reporter, get_rawResultsPath(filename))]
-
-run_test("default.sw.multi", common_args + reporter_args)
-
-check_outputs("default.sw.multi")
-for reporter, filename in zip(reporters, filenames):
-    check_outputs(filename)
-
-
-if overallResult != 0:
-    print("If these differences are expected, run approve.py to approve new baselines.")
-    exit(2)
diff --git a/tools/scripts/approve.py b/tools/scripts/approve.py
deleted file mode 100755
index 6d73be5c..00000000
--- a/tools/scripts/approve.py
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/usr/bin/env python3
-
-import os
-import sys
-import shutil
-import glob
-from scriptCommon import catchPath
-
-rootPath = os.path.join( catchPath, 'tests/SelfTest/Baselines' )
-
-if len(sys.argv) > 1:
-    files = [os.path.join( rootPath, f ) for f in sys.argv[1:]]
-else:
-    files = glob.glob( os.path.join( rootPath, "*.unapproved.txt" ) )
-
-
-def approveFile( approvedFile, unapprovedFile ):
-    justFilename = unapprovedFile[len(rootPath)+1:]
-    if os.path.exists( unapprovedFile ):
-        if os.path.exists( approvedFile ):
-            os.remove( approvedFile )
-        os.rename( unapprovedFile, approvedFile )
-        print( "approved " + justFilename )
-    else:
-        print( "approval file " + justFilename + " does not exist" )
-
-if files:
-    for unapprovedFile in files:
-        approveFile( unapprovedFile.replace( "unapproved.txt", "approved.txt" ), unapprovedFile )
-else:
-    print( "no files to approve" )
diff --git a/tools/scripts/generateAmalgamatedFiles.py b/tools/scripts/generateAmalgamatedFiles.py
index e3e86aab..5fed3358 100755
--- a/tools/scripts/generateAmalgamatedFiles.py
+++ b/tools/scripts/generateAmalgamatedFiles.py
@@ -44,6 +44,10 @@ file_header = '''\
 //  ----------------------------------------------------------
 //  This file is an amalgamation of multiple different files.
 //  You probably shouldn't edit it directly.
+//
+//  The parts that come from src/Progmasoft, the Catch3 runner, keep
+//  their own copyright and license lines (MPL-2.0 WITH
+//  AdditionRef-Progmasoft-Exception-1.1). The runner needs C++20.
 //  ----------------------------------------------------------
 '''
 
@@ -68,6 +72,9 @@ def concatenate_file(out, filename: str, expand_headers: bool) -> int:
             if line in copyright_lines:
                 continue
             m = internal_include_parser.match(line)
+            # The runner's headers are expanded at the start of the .cpp file.
+            if not m and not expand_headers and progmasoft_include_parser.match(line):
+                continue
             # anything that isn't a Catch2 header can just be copied to
             # the resulting file
             if not m:
@@ -112,16 +119,61 @@ def generate_header():
         print('Concatenated {} headers'.format(concatenate_file(header, starting_header, True)))
         header.write('#endif // CATCH_AMALGAMATED_HPP_INCLUDED\n')
 
+# The runner's files include each other as <Progmasoft/...> or, inside one
+# directory, as "Name.hpp".
+progmasoft_include_parser = re.compile(r'\s*#include (?:<(Progmasoft/[^>]+)>|"([A-Za-z/]+\.hpp)").*')
+concatenated_runner_headers = set()
+
+
+def concatenate_runner_file(out, filename: str, expand_headers: bool):
+    """Copies a file of the runner, expanding the runner's headers it
+    includes once, or dropping those includes from the .cpp files."""
+    directory = os.path.dirname(filename)
+    with open(filename, mode='r', encoding='utf-8') as input:
+        for line in input:
+            if line.strip() == '#pragma once':
+                continue
+            # Catch2's own headers are all in the amalgamated header.
+            if internal_include_parser.match(line):
+                continue
+            m = progmasoft_include_parser.match(line)
+            if not m:
+                out.write(line)
+                continue
+            if m.group(1):
+                header = os.path.normpath(os.path.join(root_path, m.group(1)))
+            else:
+                header = os.path.normpath(os.path.join(directory, m.group(2)))
+            if not expand_headers or header in concatenated_runner_headers:
+                continue
+            concatenated_runner_headers.add(header)
+            concatenate_runner_file(out, header, True)
+
+
 def generate_cpp():
     from glob import glob
     cpp_files = sorted(glob(os.path.join(root_path, 'catch2', '**/*.cpp'), recursive=True))
+    runner_headers = sorted(glob(os.path.join(root_path, 'Progmasoft', 'Runner', '*.hpp'))) + [
+        os.path.join(root_path, 'Progmasoft', 'Catch3', name)
+        for name in ('Results.hpp', 'Runner.hpp', 'Version.hpp', 'XmlWriter.hpp')]
+    runner_files = [os.path.join(root_path, 'Progmasoft', 'Results.cpp'),
+                    os.path.join(root_path, 'Progmasoft', 'XmlWriter.cpp')] + \
+        sorted(glob(os.path.join(root_path, 'Progmasoft', 'Runner', '*.cpp')))
     with open(output_cpp, mode='w', encoding='utf-8') as cpp:
         cpp.write(formatted_file_header(Version()))
         cpp.write('\n#include "catch_amalgamated.hpp"\n')
         concatenate_file(cpp, os.path.join(root_path, 'catch2/internal/catch_windows_h_proxy.hpp'), False)
+        # The runner's headers come first: Catch::Session reaches the runner.
+        for header in runner_headers:
+            header = os.path.normpath(header)
+            if header not in concatenated_runner_headers:
+                concatenated_runner_headers.add(header)
+                concatenate_runner_file(cpp, header, True)
         for file in cpp_files:
             concatenate_file(cpp, file, False)
-    print('Concatenated {} cpp files'.format(len(cpp_files)))
+        for file in runner_files:
+            concatenate_runner_file(cpp, file, False)
+    print('Concatenated {} cpp files'.format(len(cpp_files) + len(runner_files)))
 
 if __name__ == "__main__":
     generate_header()