Replace Catch2's runner with the Catch3 runner

Catch3 keeps Catch2's API for writing tests (test cases, sections,
assertions, matchers, generators, listeners, benchmarks) and replaces its
session, command line and reporters with a runner of its own.

The runner starts the test binary again as worker processes that take the
next test when they are free. A crash, an exit or a hang ends one test, not
the run. On top of that it adds:

- retries that tell deterministic, flaky and order-dependent failures
  apart, and a bisection that names the test leaving state behind;
- a history of verdicts and durations, kept under a file lock, that drives
  automatic quarantine, adaptive time limits, predictive ordering,
  incremental runs, failure replay and analytics;
- selection by change, past failure or prediction, dependency closure,
  and sharding by a stable hash or by recorded durations;
- tags for serial and isolated tests, shared resources, dependencies,
  time limits and retries;
- a failure category, a signature, an analysis and a replay command for
  every failure;
- console, JUnit XML (with flakyFailure and rerunFailure), JSON and Chrome
  trace reports, and a versioned discovery listing (--list=json);
- a temporary directory per test (TMPDIR, TMP, TEMP and
  TestTemporaryDirectory());
- Bazel's test environment: XML output file, test filter, sharding, seed,
  premature exit file and undeclared outputs.

The command line is parsed by a hand-written, typed parser. Catch::Session
remains as a thin facade over the runner for custom main functions.

The result model in Progmasoft/Catch3/Results.hpp was rewritten around
tests and attempts, and the JUnit writer now writes it. The property
engine's example database is updated under a file lock with an atomic
rename, so parallel workers can write it at the same time.

Removed: Clara, the command-line and reporter-spec parsers, every reporter
and CATCH_REGISTER_REPORTER, listing, text flow, console colour and width,
the JSON and XML writers, sharding in the test registry, approval tests and
their baselines, Conan packaging, and the unused third_party/clara.hpp.
Listeners keep working; reporters' list and skip events are gone.

The Catch2 library is now compiled as C++20; code that includes only
Catch2's headers still needs C++14. Duplicate test cases are still refused
at start-up. Version 0.3.0.

ExtraTests now turn exceptions off for clang-cl with MSVC-style options,
which -fno-exceptions did not do.

11 files changed, 211 insertions, 679 deletions

diff --git a/tests/TestScripts/DiscoverTests/VerifyRegistration.py b/tests/TestScripts/DiscoverTests/VerifyRegistration.py
index e213bd3d..cfa2bac6 100644
--- a/tests/TestScripts/DiscoverTests/VerifyRegistration.py
+++ b/tests/TestScripts/DiscoverTests/VerifyRegistration.py
@@ -78,24 +78,19 @@ def get_test_names(build_path: str) -> List[TestInfo]:
     config_path = "Debug" if os.name == 'nt' else ""
     full_path = os.path.join(build_path, config_path, 'tests')
 
-    with tempfile.TemporaryDirectory() as tmpdir:
-        fname = f'{tmpdir}/listing-output.json'
-        cmd = [full_path,
-          '--list-tests',
-          '--reporter', 'json',
-          '--out', fname
-        ]
-        result = subprocess.run(cmd,
-                                capture_output = False,
-                                check = True,
-                                text = True)
-        with open(fname, mode='r', encoding='utf-8') as file:
-            test_listing = json.load(file)
+    cmd = [full_path, '--list=json']
+    result = subprocess.run(cmd,
+                            capture_output = True,
+                            check = True,
+                            text = True,
+                            encoding = 'utf-8')
+    test_listing = json.loads(result.stdout[result.stdout.index('{'):])
 
-    assert test_listing['version'] == 2, test_listing['version']
+    assert test_listing['format'] == 'catch3-tests', test_listing['format']
+    assert test_listing['version'] == 1, test_listing['version']
 
     tests = []
-    for test in test_listing['listings']['tests']:
+    for test in test_listing['tests']:
         test_name = test['name']
         tags = test['tags']
         tests.append(TestInfo(test_name, tags))
@@ -129,11 +124,10 @@ def extract_tests_from_ctest(ctest_output) -> List[TestInfo]:
     test_infos = []
     for test in tests:
         test_command = test['command']
-        # First part of the command is the binary, second is the filter,
-        # third is the `--order decl` arg for faster startup.
+        # First part of the command is the binary, second is the filter.
         # If there are less, registration has failed. If there are more,
         # registration has changed and the script needs updating.
-        assert len(test_command) == 3, test_command
+        assert len(test_command) == 2, test_command
         test_name = test_command[1]
         labels = []
         for prop in test['properties']:
diff --git a/tests/TestScripts/testBazelExitGuardFile.py b/tests/TestScripts/testBazelExitGuardFile.py
index c1186689..376049d2 100755
--- a/tests/TestScripts/testBazelExitGuardFile.py
+++ b/tests/TestScripts/testBazelExitGuardFile.py
@@ -7,119 +7,78 @@
 
 # SPDX-License-Identifier: BSL-1.0
 
-import os
-import stat
-import sys
-import subprocess
-
-def generate_path_suffix() -> str:
-    return os.urandom(16).hex()[:16]
+"""
+Checks the premature-exit guard file: the runner creates it when the run
+starts and removes it when the run ends. A test that ends its process is a
+crash the runner reports, so the guard is removed then too. The guard is
+created only the user can read, and a symbolic link in its place is refused.
 
+Arguments: the test binary and a directory for the guard files.
+"""
 
-def run_common(cmd, env = None):
-    cmd_env = env if env is not None else os.environ.copy()
-    print('Running:', cmd)
-
-    try:
-        ret = subprocess.run(
-            cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            check=True,
-            universal_newlines=True,
-            env=cmd_env
-        )
-    except subprocess.SubprocessError as ex:
-        print('Could not run "{}"'.format(cmd))
-        print("Return code: {}".format(ex.returncode))
-        print("stdout: {}".format(ex.stdout))
-        print("stderr: {}".format(ex.stderr))
-        raise
-
-
-def test_bazel_env_vars(bin_path, guard_path):
-    env = os.environ.copy()
-    env["TEST_PREMATURE_EXIT_FILE"] = guard_path
-    env["BAZEL_TEST"] = '1'
-    run_common([bin_path], env)
+import os
+import subprocess
+import sys
 
-def test_cli_parameter(bin_path, guard_path):
-    # Reuse a deliberately permissive stale guard file to verify that the
-    # implementation tightens its permissions before truncating it.
-    if os.name != 'nt':
-        with open(guard_path, 'w'):
-            pass
-        os.chmod(guard_path, 0o666)
-    cmd = [
-      bin_path,
-      '--premature-exit-guard-file',
-      guard_path
-    ]
-    run_common(cmd)
 
-    if os.name != 'nt':
-        target_path = guard_path + '.target'
-        symlink_path = guard_path + '.symlink'
-        sentinel = 'do not truncate a symlink target\n'
-        with open(target_path, 'w') as target:
-            target.write(sentinel)
-        os.symlink(target_path, symlink_path)
-        try:
-            result = subprocess.run(
-                [bin_path, '--premature-exit-guard-file', symlink_path],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                universal_newlines=True,
-            )
-            assert result.returncode != 0, (
-                'A symlink was accepted as a premature-exit guard file'
-            )
-            with open(target_path, 'r') as target:
-                assert target.read() == sentinel, (
-                    'Opening the guard-file symlink modified its target'
-                )
-        finally:
-            os.remove(symlink_path)
-            os.remove(target_path)
+def guard_path(directory: str) -> str:
+    return os.path.join(directory, 'guard_file.' + os.urandom(8).hex())
 
-def test_no_crash(bin_path, guard_path):
-    cmd = [
-      bin_path,
-      '--premature-exit-guard-file',
-      guard_path,
-      # Disable the quick-exit test
-      '~[quick_exit]'
-    ]
-    run_common(cmd)
 
-checks = {
-  'bazel': (test_bazel_env_vars, True),
-  'cli': (test_cli_parameter, True),
-  'no-crash': (test_no_crash, False),
-}
+def run(cmd, env = None, expected = 0):
+    print('Running:', cmd)
+    result = subprocess.run(cmd,
+                            stdout = subprocess.PIPE,
+                            stderr = subprocess.PIPE,
+                            universal_newlines = True,
+                            env = env)
+    if result.returncode != expected:
+        print('Return code {} instead of {}'.format(result.returncode, expected))
+        print('stdout:', result.stdout)
+        print('stderr:', result.stderr)
+        exit(1)
+    return result
 
 
 bin_path = os.path.abspath(sys.argv[1])
 output_dir = os.path.abspath(sys.argv[2])
-test_kind = sys.argv[3]
-guard_file_path = os.path.join(output_dir, f"guard_file.{generate_path_suffix()}")
-print(f'Guard file path: "{guard_file_path}"')
-
-check_func, file_should_exist = checks[test_kind]
-check_func(bin_path, guard_file_path)
 
-assert os.path.exists(guard_file_path) == file_should_exist
-if file_should_exist and os.name != 'nt':
-    guard_mode = stat.S_IMODE(os.stat(guard_file_path).st_mode)
-    assert guard_mode & 0o077 == 0, (
-        f'Guard file permissions are accessible to group/others: {guard_mode:o}'
-    )
-# With randomly generated file suffix, we should not run into name conflicts.
-# However, we try to cleanup anyway, to avoid having infinity files in
-# long living build directories.
-if file_should_exist:
+# Bazel names the guard in the environment.
+path = guard_path(output_dir)
+env = os.environ.copy()
+env['BAZEL_TEST'] = '1'
+env['TEST_PREMATURE_EXIT_FILE'] = path
+run([bin_path, 'pass'], env)
+assert not os.path.exists(path), 'the guard was left behind after a normal run'
+
+# A test that exits the process is a crash; the run still ends normally.
+path = guard_path(output_dir)
+run([bin_path, '--retries=off', '--premature-exit-guard-file=' + path], expected = 42)
+assert not os.path.exists(path), 'the guard was left behind after a crash'
+
+if os.name != 'nt':
+    # A stale guard with loose permissions is tightened while the run lasts.
+    path = guard_path(output_dir)
+    with open(path, 'w'):
+        pass
+    os.chmod(path, 0o666)
+    run([bin_path, 'pass', '--premature-exit-guard-file=' + path])
+
+    target_path = path + '.target'
+    symlink_path = path + '.symlink'
+    sentinel = 'do not truncate a symlink target\n'
+    with open(target_path, 'w') as target:
+        target.write(sentinel)
+    os.symlink(target_path, symlink_path)
     try:
-        os.remove(guard_file_path)
-    except Exception as ex:
-        print(f'Could not remove file {guard_file_path} because: {ex}')
-
+        result = subprocess.run(
+            [bin_path, 'pass', '--premature-exit-guard-file=' + symlink_path],
+            stdout = subprocess.PIPE,
+            stderr = subprocess.PIPE,
+            universal_newlines = True)
+        assert result.returncode != 0, 'a symlink was accepted as the guard file'
+        with open(target_path, 'r') as target:
+            assert target.read() == sentinel, 'the symlink target was modified'
+    finally:
+        os.remove(symlink_path)
+        os.remove(target_path)
diff --git a/tests/TestScripts/testBazelReporter.py b/tests/TestScripts/testBazelReporter.py
index 573eafd2..62600a39 100644
--- a/tests/TestScripts/testBazelReporter.py
+++ b/tests/TestScripts/testBazelReporter.py
@@ -14,12 +14,13 @@ import xml.etree.ElementTree as ET
 import subprocess
 
 """
-Test that Catch2 recognizes `XML_OUTPUT_FILE` env variable and creates
-a junit reporter that writes to the provided path.
+Test that the runner recognizes the `XML_OUTPUT_FILE` environment variable
+of Bazel and writes a JUnit report to the path it names.
 
-Requires 2 arguments, path to Catch2 binary configured with
-`CATCH_CONFIG_BAZEL_SUPPORT`, and the output directory for the output file.
+Requires 2 arguments, path to the test binary and the output directory for
+the report.
 """
+
 if len(sys.argv) != 3:
     print("Wrong number of arguments: {}".format(len(sys.argv)))
     print("Usage: {} test-bin-path output-dir".format(sys.argv[0]))
@@ -39,31 +40,21 @@ print('xml out path:', xml_out_path)
 
 env = os.environ.copy()
 env["XML_OUTPUT_FILE"] = xml_out_path
-test_passing = True
 
-try:
-    ret = subprocess.run(
-        bin_path,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        check=True,
-        universal_newlines=True,
-        env=env
-    )
-    stdout = ret.stdout
-except subprocess.SubprocessError as ex:
-    if ex.returncode == 42:
-        # The test cases are allowed to fail.
-        test_passing = False
-        stdout = ex.stdout
-    else:
-        print('Could not run "{}"'.format(bin_path))
-        print("Return code: {}".format(ex.returncode))
-        print("stdout: {}".format(ex.stdout))
-        print("stderr: {}".format(ex.stderr))
-        raise
-
-# Check for valid XML output
+ret = subprocess.run(
+    [bin_path, '--retries=off'],
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE,
+    universal_newlines=True,
+    env=env
+)
+# One of the tests fails on purpose.
+if ret.returncode != 42:
+    print("Unexpected return code {}".format(ret.returncode))
+    print("stdout: {}".format(ret.stdout))
+    print("stderr: {}".format(ret.stderr))
+    exit(1)
+
 try:
     tree = ET.parse(xml_out_path)
 except ET.ParseError as ex:
@@ -73,24 +64,17 @@ except FileNotFoundError as ex:
     print("Could not find '{}'".format(xml_out_path))
     raise
 
-bin_name = os.path.basename(bin_path)
-# Check for matching testsuite
-if not tree.find('.//testsuite[@name="{}"]'.format(bin_name)):
+bin_name = os.path.splitext(os.path.basename(bin_path))[0]
+if tree.find('.//testsuite[@name="{}"]'.format(bin_name)) is None:
     print("Could not find '{}' testsuite".format(bin_name))
     exit(2)
 
-# Check that we haven't disabled the default reporter
-summary_test_cases = re.findall(r'test cases: \d* \| \d* passed \| \d* failed', stdout)
-if len(summary_test_cases) == 0:
-    print("Could not find test summary in {}".format(stdout))
-    exit(2)
-
-total, passed, failed = [int(s) for s in summary_test_cases[0].split() if s.isdigit()]
-
-if failed == 0 and not test_passing:
-    print("Expected at least 1 test failure!")
+summary = re.search(r'(\d+) test\(s\): (\d+) passed, (\d+) failed', ret.stdout)
+if summary is None:
+    print("Could not find test summary in {}".format(ret.stdout))
     exit(2)
 
+total, passed, failed = [int(value) for value in summary.groups()]
 if len(tree.findall('.//testcase')) != total:
     print("Unexpected number of test cases!")
     exit(2)
diff --git a/tests/TestScripts/testBazelSharding.py b/tests/TestScripts/testBazelSharding.py
index 23594461..ba550350 100755
--- a/tests/TestScripts/testBazelSharding.py
+++ b/tests/TestScripts/testBazelSharding.py
@@ -37,44 +37,44 @@ if os.path.isfile(info_file_path):
 print('bin path:', bin_path)
 print('shard confirmation path:', info_file_path)
 
-env = os.environ.copy()
-# We will run only one shard, and it should have the passing test.
-# This simplifies our work a bit, and if we have any regression in this
-# functionality we can make more complex tests later.
-env["BAZEL_TEST"] = "1"
-env["TEST_SHARD_INDEX"] = "0"
-env["TEST_TOTAL_SHARDS"] = "2"
-env["TEST_SHARD_STATUS_FILE"] = info_file_path
+import re
 
-cmd = [
-  bin_path,
-  # We rely on the test order being deterministic
-  '--order', 'decl'
-]
+# The two shards together run the two tests of the binary, each once.
+total = 0
+for index in range(2):
+    if os.path.isfile(info_file_path):
+        os.remove(info_file_path)
+    env = os.environ.copy()
+    env["BAZEL_TEST"] = "1"
+    env["TEST_SHARD_INDEX"] = str(index)
+    env["TEST_TOTAL_SHARDS"] = "2"
+    env["TEST_SHARD_STATUS_FILE"] = info_file_path
 
-try:
     ret = subprocess.run(
-        cmd,
+        [bin_path, '--allow-no-tests', '--retries=off'],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
-        check=True,
         universal_newlines=True,
         env=env
     )
-    stdout = ret.stdout
-except subprocess.SubprocessError as ex:
-    print('Could not run "{}"'.format(bin_path))
-    print("Return code: {}".format(ex.returncode))
-    print("stdout: {}".format(ex.stdout))
-    print("stderr: {}".format(ex.stderr))
-    raise
+    if ret.returncode not in (0, 42):
+        print('Could not run "{}"'.format(bin_path))
+        print("Return code: {}".format(ret.returncode))
+        print("stdout: {}".format(ret.stdout))
+        print("stderr: {}".format(ret.stderr))
+        exit(1)
 
+    summary = re.search(r'(\d+) test\(s\):', ret.stdout)
+    if summary is None:
+        print("Did not find the summary in stdout.")
+        print("stdout:\n{}".format(ret.stdout))
+        exit(1)
+    total += int(summary.group(1))
 
-if not "All tests passed (1 assertion in 1 test case)" in stdout:
-    print("Did not find expected output in stdout.")
-    print("stdout:\n{}".format(stdout))
-    exit(1)
+    if not os.path.isfile(info_file_path):
+        print("The runner did not create the expected file at path '{}'".format(info_file_path))
+        exit(2)
 
-if not os.path.isfile(info_file_path):
-    print("Catch2 did not create expected file at path '{}'".format(info_file_path))
-    exit(2)
+if total != 2:
+    print("The shards ran {} tests together instead of 2".format(total))
+    exit(3)
diff --git a/tests/TestScripts/testConfigureDefaultReporter.py b/tests/TestScripts/testConfigureDefaultReporter.py
deleted file mode 100644
index 119e1ca2..00000000
--- a/tests/TestScripts/testConfigureDefaultReporter.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-
-#              Copyright Catch2 Authors
-# Distributed under the Boost Software License, Version 1.0.
-#   (See accompanying file LICENSE.txt or copy at
-#        https://www.boost.org/LICENSE_1_0.txt)
-
-# SPDX-License-Identifier: BSL-1.0
-
-from ConfigureTestsCommon import configure_and_build, run_and_return_output
-
-import os
-import re
-import sys
-
-"""
-Tests the CMake configure option for CATCH_CONFIG_DEFAULT_REPORTER
-
-Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
-exists, and path to where the output files should be stored.
-"""
-
-if len(sys.argv) != 3:
-    print('Wrong number of arguments: {}'.format(len(sys.argv)))
-    print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
-    exit(1)
-
-catch2_source_path = os.path.abspath(sys.argv[1])
-build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DefaultReporter')
-
-output_file = f"{build_dir_path}/foo.xml"
-# We need to escape backslashes in Windows paths, because otherwise they
-# are interpreted as escape characters in strings, and cause compilation
-# error.
-escaped_output_file = output_file.replace('\\', '\\\\')
-configure_and_build(catch2_source_path,
-                    build_dir_path,
-                    [("CATCH_CONFIG_DEFAULT_REPORTER", f"xml::out={escaped_output_file}")])
-
-stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTest', ['[approx][custom]'])
-
-if not os.path.exists(output_file):
-    print(f'Did not find the {output_file} file')
-    exit(2)
-
-xml_tag = '</Catch2TestRun>'
-with open(output_file, 'r', encoding='utf-8') as file:
-    if xml_tag not in file.read():
-        print(f"Could not find '{xml_tag}' in the file")
-        exit(3)
diff --git a/tests/TestScripts/testConfigureDisable.py b/tests/TestScripts/testConfigureDisable.py
index b0247700..ce9b2f9f 100644
--- a/tests/TestScripts/testConfigureDisable.py
+++ b/tests/TestScripts/testConfigureDisable.py
@@ -38,10 +38,10 @@ configure_and_build(catch2_source_path,
 
 stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
                                   'SelfTest',
-                                  ['--allow-running-no-tests'])
+                                  ['--allow-no-tests', '--no-history'])
 
 
-summary_line = 'No tests ran'
+summary_line = '0 test(s): none ran'
 if not summary_line in stdout:
     print("Could not find '{}' in the stdout".format(summary_line))
     print('stdout: "{}"'.format(stdout))
diff --git a/tests/TestScripts/testConfigureExperimentalRedirect.py b/tests/TestScripts/testConfigureExperimentalRedirect.py
index 09ff51e2..cf659e9e 100644
--- a/tests/TestScripts/testConfigureExperimentalRedirect.py
+++ b/tests/TestScripts/testConfigureExperimentalRedirect.py
@@ -9,6 +9,7 @@
 
 from ConfigureTestsCommon import configure_and_build, run_and_return_output
 
+import json
 import os
 import re
 import sys
@@ -32,18 +33,17 @@ configure_and_build(catch2_source_path,
                     build_dir_path,
                     [("CATCH_CONFIG_EXPERIMENTAL_REDIRECT", "ON")])
 
-stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
-                                  'SelfTest',
-                                  ['-r', 'xml', '"has printf"'])
-
-
-# The print from printf must be within the XML's reporter stdout tag.
-required_output = '''\
-      <StdOut>
-loose text artifact
-      </StdOut>
-'''
-if not required_output in stdout:
-    print("Could not find '{}' in the stdout".format(required_output))
-    print('stdout: "{}"'.format(stdout))
+report_path = os.path.join(build_dir_path, 'has-printf.json')
+run_and_return_output(os.path.join(build_dir_path, 'tests'),
+                      'SelfTest',
+                      ['--no-history', '--json=' + report_path, 'has printf'])
+
+# The text printf writes must be captured as the test's standard output.
+with open(report_path, encoding='utf-8') as report_file:
+    report = json.load(report_file)
+outputs = [attempt.get('standardOutput', '')
+           for test in report['tests']
+           for attempt in test['attempts']]
+if outputs != ['loose text artifact\n']:
+    print('Unexpected captured output: {}'.format(outputs))
     exit(2)
diff --git a/tests/TestScripts/testPartialTestCaseEvent.py b/tests/TestScripts/testPartialTestCaseEvent.py
index 55c6e299..fb3932cc 100755
--- a/tests/TestScripts/testPartialTestCaseEvent.py
+++ b/tests/TestScripts/testPartialTestCaseEvent.py
@@ -8,9 +8,9 @@
 # SPDX-License-Identifier: BSL-1.0
 
 """
-This test script verifies that the testCasePartial{Starting,Ended} reporter
+This test script verifies that the testCasePartial{Starting,Ended} listener
 events fire properly. This is done by calling a test binary compiled with
-reporter that reports specifically testCase* events, and verifying the
+a listener that records specifically testCase* events, and verifying the
 outputs match what we expect.
 """
 
@@ -46,21 +46,20 @@ TestCaseEnded: generator
 
 from typing import List
 
-def get_test_output(test_exe: str, sections: bool) -> List[str]:
-    cmd = [test_exe, '--reporter', 'partial']
-    if sections:
-        cmd.append('section')
-    else:
-        cmd.append('generator')
-
+def get_test_output(test_exe: str, sections: bool) -> str:
+    cmd = [test_exe, '--no-history', 'section' if sections else 'generator']
     ret = subprocess.run(cmd,
                          stdout = subprocess.PIPE,
                          stderr = subprocess.PIPE,
-                         timeout = 10,
+                         timeout = 30,
                          check = True,
                          universal_newlines = True)
-
-    return ret.stdout
+    # The listener prints the events when the run ends, outside any test;
+    # the runner passes that output on.
+    output = ret.stdout + ret.stderr
+    start = output.index('BEGIN EVENTS\n') + len('BEGIN EVENTS\n')
+    end = output.index('END EVENTS', start)
+    return output[start:end].replace('\r\n', '\n')
 
 def main():
     test_exe, = sys.argv[1:]
diff --git a/tests/TestScripts/testRandomOrder.py b/tests/TestScripts/testRandomOrder.py
deleted file mode 100755
index 2368423e..00000000
--- a/tests/TestScripts/testRandomOrder.py
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/env python3
-
-#              Copyright Catch2 Authors
-# Distributed under the Boost Software License, Version 1.0.
-#   (See accompanying file LICENSE.txt or copy at
-#        https://www.boost.org/LICENSE_1_0.txt)
-
-# SPDX-License-Identifier: BSL-1.0
-
-"""
-This test script verifies that the random ordering of tests inside
-Catch2 is invariant in regards to subsetting. This is done by running
-the binary 3 times, once with all tests selected, and twice with smaller
-subsets of tests selected, and verifying that the selected tests are in
-the same relative order.
-"""
-
-import subprocess
-import sys
-import random
-import xml.etree.ElementTree as ET
-
-def none_to_empty_str(e):
-    if e is None:
-        return ""
-    assert type(e) is str
-    return e
-
-def list_tests(self_test_exe, tags, rng_seed):
-    cmd = [self_test_exe, '--reporter', 'xml', '--list-tests', '--order', 'rand',
-            '--rng-seed', str(rng_seed)]
-    tags_arg = ','.join('[{}]~[.]'.format(t) for t in tags)
-    if tags_arg:
-        cmd.append(tags_arg)
-    process = subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = process.communicate()
-    if stderr:
-        raise RuntimeError("Unexpected error output:\n" + process.stderr)
-
-    root = ET.fromstring(stdout)
-    result = [(none_to_empty_str(tc.find('Name').text),
-               none_to_empty_str(tc.find('Tags').text),
-               none_to_empty_str(tc.find('ClassName').text)) for tc in root.findall('./TestCase')]
-
-    if len(result) < 2:
-        raise RuntimeError("Unexpectedly few tests listed (got {})".format(
-            len(result)))
-    return result
-
-def check_is_sublist_of(shorter, longer):
-    assert len(shorter) < len(longer)
-    assert len(set(longer)) == len(longer)
-
-    indexes_in_longer = {s: i for i, s in enumerate(longer)}
-    for s1, s2 in zip(shorter, shorter[1:]):
-        assert indexes_in_longer[s1] < indexes_in_longer[s2], (
-                '{} comes before {} in longer list.\n'
-                'Longer: {}\nShorter: {}'.format(s2, s1, longer, shorter))
-
-def main():
-    self_test_exe, = sys.argv[1:]
-
-    # We want a random seed for the test, but want to avoid 0,
-    # because it has special meaning
-    seed = random.randint(1, 2 ** 32 - 1)
-
-    list_one_tag = list_tests(self_test_exe, ['generators'], seed)
-    list_two_tags = list_tests(self_test_exe, ['generators', 'matchers'], seed)
-    list_all = list_tests(self_test_exe, [], seed)
-
-    # First, verify that restricting to a subset yields the same order
-    check_is_sublist_of(list_two_tags, list_all)
-    check_is_sublist_of(list_one_tag, list_two_tags)
-
-if __name__ == '__main__':
-    sys.exit(main())
diff --git a/tests/TestScripts/testSectionFiltering.py b/tests/TestScripts/testSectionFiltering.py
deleted file mode 100755
index 4f56df53..00000000
--- a/tests/TestScripts/testSectionFiltering.py
+++ /dev/null
@@ -1,189 +0,0 @@
-#!/usr/bin/env python3
-
-#              Copyright Catch2 Authors
-# Distributed under the Boost Software License, Version 1.0.
-#   (See accompanying file LICENSE.txt or copy at
-#        https://www.boost.org/LICENSE_1_0.txt)
-
-# SPDX-License-Identifier: BSL-1.0
-
-"""
-This test script verifies the behaviour of the legacy section filtering
-using `-c`, `--section` CLI parameters.
-
-This is done by having a hardcoded set of test filter + section filter
-combinations, together with the expected number of assertions that will
-be run inside the test for given filter combo.
-"""
-
-import os
-import subprocess
-import sys
-from typing import Tuple, List
-import xml.etree.ElementTree as ET
-
-def make_old_filters(section_names: Tuple[str, ...]) -> List[str]:
-    final = []
-    for name in section_names:
-        final.append('--section')
-        final.append(name)
-    return final
-
-def make_new_filters(path_filters: Tuple[str, ...]) -> List[str]:
-    final = []
-    for component in path_filters:
-        final.append('--path-filter')
-        final.append(component)
-    return final
-
-def run_one_test(binary_path: str,
-                 test_name: str,
-                 section_names: Tuple[str, ...],
-                 expected_assertions: int,
-                 new_filters: bool):
-    cmd = [
-      binary_path,
-      '--reporter', 'xml',
-      test_name
-    ]
-    if new_filters:
-        cmd.extend(make_new_filters(section_names))
-    else:
-        cmd.extend(make_old_filters(section_names))
-    try:
-        ret = subprocess.run(
-            cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            check=True,
-            universal_newlines=True,
-        )
-        stdout = ret.stdout
-    except subprocess.SubprocessError as ex:
-        print('Could not run "{}"'.format(cmd))
-        print("Return code: {}".format(ex.returncode))
-        print("stdout: {}".format(ex.stdout))
-        print("stderr: {}".format(ex.stderr))
-        raise
-
-    try:
-        tree = ET.fromstring(stdout)
-    except ET.ParseError as ex:
-        print("Invalid XML: '{}'".format(ex))
-        raise
-
-    # Validate that we ran exactly 1 test case, and it passed
-    test_case_stats = tree.find('OverallResultsCases')
-    expected_testcases = {'successes' : '1', 'failures' : '0', 'expectedFailures': '0', 'skips': '0'}
-    assert test_case_stats.attrib == expected_testcases, f'We did not run single passing test case as expected. {test_name}: {test_case_stats.attrib}'
-
-    # Validate that we got exactly the expected number of passing assertions
-    expected_assertions = {'successes' : str(expected_assertions), 'failures' : '0', 'expectedFailures': '0', 'skips': '0'}
-    assertion_stats = tree.find('OverallResults')
-    assert assertion_stats.attrib == expected_assertions, f'"{test_name}": {assertion_stats.attrib} vs {expected_assertions}'
-
-
-# Inputs taken from issue #3038
-old_section_tests = {
-  'foo': (
-    ((), 3),
-    (('A',), 3),
-    (('A', 'B'), 0),
-    (('A', 'B1'), 1),
-    (('A', 'B2'), 1),
-    (('A', 'B1', 'B2'), 1),
-    (('A', 'B2', 'XXXX'), 1),
-  ),
-  'bar': (
-    ((), 9),
-    (('A',), 9),
-    (('A', 'B1'), 3),
-    (('XXXX',), 2),
-    (('B1',), 2),
-    (('A', 'B1', 'B2'), 3),
-  ),
-  'baz': (
-    ((), 4),
-    (('A',), 1),
-    (('A', 'B'), 1),
-    (('A', 'XXXX'), 1),
-    (('B',), 3),
-    (('XXXX',), 0),
-  ),
-  'qux': (
-    ((), 12),
-    (('A',), 7),
-    (('B',), 9),
-    (('B', 'XXXX'), 9),
-    (('XXXX',), 6),
-  ),
-  'corge': (
-    ((), 12),
-    (('i=2',), 7),
-    (('i=3',), 7),
-  ),
-  'grault': (
-    ((), 12),
-    (('A',), 3),
-    (('B',), 9),
-    (('B', 'i=1'), 7)