Merge pull request #15 from Progmasoft/feature/catch3-runner

Replace Catch2's runner with the Catch3 runner

11 files changed, 211 insertions, 679 deletions

diff --git a/tests/TestScripts/DiscoverTests/VerifyRegistration.py b/tests/TestScripts/DiscoverTests/VerifyRegistration.py
index e213bd3d..cfa2bac6 100644
--- a/tests/TestScripts/DiscoverTests/VerifyRegistration.py
+++ b/tests/TestScripts/DiscoverTests/VerifyRegistration.py
@@ -78,24 +78,19 @@ def get_test_names(build_path: str) -> List[TestInfo]:
     config_path = "Debug" if os.name == 'nt' else ""
     full_path = os.path.join(build_path, config_path, 'tests')
 
-    with tempfile.TemporaryDirectory() as tmpdir:
-        fname = f'{tmpdir}/listing-output.json'
-        cmd = [full_path,
-          '--list-tests',
-          '--reporter', 'json',
-          '--out', fname
-        ]
-        result = subprocess.run(cmd,
-                                capture_output = False,
-                                check = True,
-                                text = True)
-        with open(fname, mode='r', encoding='utf-8') as file:
-            test_listing = json.load(file)
+    cmd = [full_path, '--list=json']
+    result = subprocess.run(cmd,
+                            capture_output = True,
+                            check = True,
+                            text = True,
+                            encoding = 'utf-8')
+    test_listing = json.loads(result.stdout[result.stdout.index('{'):])
 
-    assert test_listing['version'] == 2, test_listing['version']
+    assert test_listing['format'] == 'catch3-tests', test_listing['format']
+    assert test_listing['version'] == 1, test_listing['version']
 
     tests = []
-    for test in test_listing['listings']['tests']:
+    for test in test_listing['tests']:
         test_name = test['name']
         tags = test['tags']
         tests.append(TestInfo(test_name, tags))
@@ -129,11 +124,10 @@ def extract_tests_from_ctest(ctest_output) -> List[TestInfo]:
     test_infos = []
     for test in tests:
         test_command = test['command']
-        # First part of the command is the binary, second is the filter,
-        # third is the `--order decl` arg for faster startup.
+        # First part of the command is the binary, second is the filter.
         # If there are less, registration has failed. If there are more,
         # registration has changed and the script needs updating.
-        assert len(test_command) == 3, test_command
+        assert len(test_command) == 2, test_command
         test_name = test_command[1]
         labels = []
         for prop in test['properties']:
diff --git a/tests/TestScripts/testBazelExitGuardFile.py b/tests/TestScripts/testBazelExitGuardFile.py
index c1186689..376049d2 100755
--- a/tests/TestScripts/testBazelExitGuardFile.py
+++ b/tests/TestScripts/testBazelExitGuardFile.py
@@ -7,119 +7,78 @@
 
 # SPDX-License-Identifier: BSL-1.0
 
-import os
-import stat
-import sys
-import subprocess
-
-def generate_path_suffix() -> str:
-    return os.urandom(16).hex()[:16]
+"""
+Checks the premature-exit guard file: the runner creates it when the run
+starts and removes it when the run ends. A test that ends its process is a
+crash the runner reports, so the guard is removed then too. The guard is
+created only the user can read, and a symbolic link in its place is refused.
 
+Arguments: the test binary and a directory for the guard files.
+"""
 
-def run_common(cmd, env = None):
-    cmd_env = env if env is not None else os.environ.copy()
-    print('Running:', cmd)
-
-    try:
-        ret = subprocess.run(
-            cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            check=True,
-            universal_newlines=True,
-            env=cmd_env
-        )
-    except subprocess.SubprocessError as ex:
-        print('Could not run "{}"'.format(cmd))
-        print("Return code: {}".format(ex.returncode))
-        print("stdout: {}".format(ex.stdout))
-        print("stderr: {}".format(ex.stderr))
-        raise
-
-
-def test_bazel_env_vars(bin_path, guard_path):
-    env = os.environ.copy()
-    env["TEST_PREMATURE_EXIT_FILE"] = guard_path
-    env["BAZEL_TEST"] = '1'
-    run_common([bin_path], env)
+import os
+import subprocess
+import sys
 
-def test_cli_parameter(bin_path, guard_path):
-    # Reuse a deliberately permissive stale guard file to verify that the
-    # implementation tightens its permissions before truncating it.
-    if os.name != 'nt':
-        with open(guard_path, 'w'):
-            pass
-        os.chmod(guard_path, 0o666)
-    cmd = [
-      bin_path,
-      '--premature-exit-guard-file',
-      guard_path
-    ]
-    run_common(cmd)
 
-    if os.name != 'nt':
-        target_path = guard_path + '.target'
-        symlink_path = guard_path + '.symlink'
-        sentinel = 'do not truncate a symlink target\n'
-        with open(target_path, 'w') as target:
-            target.write(sentinel)
-        os.symlink(target_path, symlink_path)
-        try:
-            result = subprocess.run(
-                [bin_path, '--premature-exit-guard-file', symlink_path],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                universal_newlines=True,
-            )
-            assert result.returncode != 0, (
-                'A symlink was accepted as a premature-exit guard file'
-            )
-            with open(target_path, 'r') as target:
-                assert target.read() == sentinel, (
-                    'Opening the guard-file symlink modified its target'
-                )
-        finally:
-            os.remove(symlink_path)
-            os.remove(target_path)
+def guard_path(directory: str) -> str:
+    return os.path.join(directory, 'guard_file.' + os.urandom(8).hex())
 
-def test_no_crash(bin_path, guard_path):
-    cmd = [
-      bin_path,
-      '--premature-exit-guard-file',
-      guard_path,
-      # Disable the quick-exit test
-      '~[quick_exit]'
-    ]
-    run_common(cmd)
 
-checks = {
-  'bazel': (test_bazel_env_vars, True),
-  'cli': (test_cli_parameter, True),
-  'no-crash': (test_no_crash, False),
-}
+def run(cmd, env = None, expected = 0):
+    print('Running:', cmd)
+    result = subprocess.run(cmd,
+                            stdout = subprocess.PIPE,
+                            stderr = subprocess.PIPE,
+                            universal_newlines = True,
+                            env = env)
+    if result.returncode != expected:
+        print('Return code {} instead of {}'.format(result.returncode, expected))
+        print('stdout:', result.stdout)
+        print('stderr:', result.stderr)
+        exit(1)
+    return result
 
 
 bin_path = os.path.abspath(sys.argv[1])
 output_dir = os.path.abspath(sys.argv[2])
-test_kind = sys.argv[3]
-guard_file_path = os.path.join(output_dir, f"guard_file.{generate_path_suffix()}")
-print(f'Guard file path: "{guard_file_path}"')
-
-check_func, file_should_exist = checks[test_kind]
-check_func(bin_path, guard_file_path)
 
-assert os.path.exists(guard_file_path) == file_should_exist
-if file_should_exist and os.name != 'nt':
-    guard_mode = stat.S_IMODE(os.stat(guard_file_path).st_mode)
-    assert guard_mode & 0o077 == 0, (
-        f'Guard file permissions are accessible to group/others: {guard_mode:o}'
-    )
-# With randomly generated file suffix, we should not run into name conflicts.
-# However, we try to cleanup anyway, to avoid having infinity files in
-# long living build directories.
-if file_should_exist:
+# Bazel names the guard in the environment.
+path = guard_path(output_dir)
+env = os.environ.copy()
+env['BAZEL_TEST'] = '1'
+env['TEST_PREMATURE_EXIT_FILE'] = path
+run([bin_path, 'pass'], env)
+assert not os.path.exists(path), 'the guard was left behind after a normal run'
+
+# A test that exits the process is a crash; the run still ends normally.
+path = guard_path(output_dir)
+run([bin_path, '--retries=off', '--premature-exit-guard-file=' + path], expected = 42)
+assert not os.path.exists(path), 'the guard was left behind after a crash'
+
+if os.name != 'nt':
+    # A stale guard with loose permissions is tightened while the run lasts.
+    path = guard_path(output_dir)
+    with open(path, 'w'):
+        pass
+    os.chmod(path, 0o666)
+    run([bin_path, 'pass', '--premature-exit-guard-file=' + path])
+
+    target_path = path + '.target'
+    symlink_path = path + '.symlink'
+    sentinel = 'do not truncate a symlink target\n'
+    with open(target_path, 'w') as target:
+        target.write(sentinel)
+    os.symlink(target_path, symlink_path)
     try:
-        os.remove(guard_file_path)
-    except Exception as ex:
-        print(f'Could not remove file {guard_file_path} because: {ex}')
-
+        result = subprocess.run(
+            [bin_path, 'pass', '--premature-exit-guard-file=' + symlink_path],
+            stdout = subprocess.PIPE,
+            stderr = subprocess.PIPE,
+            universal_newlines = True)
+        assert result.returncode != 0, 'a symlink was accepted as the guard file'
+        with open(target_path, 'r') as target:
+            assert target.read() == sentinel, 'the symlink target was modified'
+    finally:
+        os.remove(symlink_path)
+        os.remove(target_path)
diff --git a/tests/TestScripts/testBazelReporter.py b/tests/TestScripts/testBazelReporter.py
index 573eafd2..62600a39 100644
--- a/tests/TestScripts/testBazelReporter.py
+++ b/tests/TestScripts/testBazelReporter.py
@@ -14,12 +14,13 @@ import xml.etree.ElementTree as ET
 import subprocess
 
 """
-Test that Catch2 recognizes `XML_OUTPUT_FILE` env variable and creates
-a junit reporter that writes to the provided path.
+Test that the runner recognizes the `XML_OUTPUT_FILE` environment variable
+of Bazel and writes a JUnit report to the path it names.
 
-Requires 2 arguments, path to Catch2 binary configured with
-`CATCH_CONFIG_BAZEL_SUPPORT`, and the output directory for the output file.
+Requires 2 arguments, path to the test binary and the output directory for
+the report.
 """
+
 if len(sys.argv) != 3:
     print("Wrong number of arguments: {}".format(len(sys.argv)))
     print("Usage: {} test-bin-path output-dir".format(sys.argv[0]))
@@ -39,31 +40,21 @@ print('xml out path:', xml_out_path)
 
 env = os.environ.copy()
 env["XML_OUTPUT_FILE"] = xml_out_path
-test_passing = True
 
-try:
-    ret = subprocess.run(
-        bin_path,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        check=True,
-        universal_newlines=True,
-        env=env
-    )
-    stdout = ret.stdout
-except subprocess.SubprocessError as ex:
-    if ex.returncode == 42:
-        # The test cases are allowed to fail.
-        test_passing = False
-        stdout = ex.stdout
-    else:
-        print('Could not run "{}"'.format(bin_path))
-        print("Return code: {}".format(ex.returncode))
-        print("stdout: {}".format(ex.stdout))
-        print("stderr: {}".format(ex.stderr))
-        raise
-
-# Check for valid XML output
+ret = subprocess.run(
+    [bin_path, '--retries=off'],
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE,
+    universal_newlines=True,
+    env=env
+)
+# One of the tests fails on purpose.
+if ret.returncode != 42:
+    print("Unexpected return code {}".format(ret.returncode))
+    print("stdout: {}".format(ret.stdout))
+    print("stderr: {}".format(ret.stderr))
+    exit(1)
+
 try:
     tree = ET.parse(xml_out_path)
 except ET.ParseError as ex:
@@ -73,24 +64,17 @@ except FileNotFoundError as ex:
     print("Could not find '{}'".format(xml_out_path))
     raise
 
-bin_name = os.path.basename(bin_path)
-# Check for matching testsuite
-if not tree.find('.//testsuite[@name="{}"]'.format(bin_name)):
+bin_name = os.path.splitext(os.path.basename(bin_path))[0]
+if tree.find('.//testsuite[@name="{}"]'.format(bin_name)) is None:
     print("Could not find '{}' testsuite".format(bin_name))
     exit(2)
 
-# Check that we haven't disabled the default reporter
-summary_test_cases = re.findall(r'test cases: \d* \| \d* passed \| \d* failed', stdout)
-if len(summary_test_cases) == 0:
-    print("Could not find test summary in {}".format(stdout))
-    exit(2)
-
-total, passed, failed = [int(s) for s in summary_test_cases[0].split() if s.isdigit()]
-
-if failed == 0 and not test_passing:
-    print("Expected at least 1 test failure!")
+summary = re.search(r'(\d+) test\(s\): (\d+) passed, (\d+) failed', ret.stdout)
+if summary is None:
+    print("Could not find test summary in {}".format(ret.stdout))
     exit(2)
 
+total, passed, failed = [int(value) for value in summary.groups()]
 if len(tree.findall('.//testcase')) != total:
     print("Unexpected number of test cases!")
     exit(2)
diff --git a/tests/TestScripts/testBazelSharding.py b/tests/TestScripts/testBazelSharding.py
index 23594461..ba550350 100755
--- a/tests/TestScripts/testBazelSharding.py
+++ b/tests/TestScripts/testBazelSharding.py
@@ -37,44 +37,44 @@ if os.path.isfile(info_file_path):
 print('bin path:', bin_path)
 print('shard confirmation path:', info_file_path)
 
-env = os.environ.copy()
-# We will run only one shard, and it should have the passing test.
-# This simplifies our work a bit, and if we have any regression in this
-# functionality we can make more complex tests later.
-env["BAZEL_TEST"] = "1"
-env["TEST_SHARD_INDEX"] = "0"
-env["TEST_TOTAL_SHARDS"] = "2"
-env["TEST_SHARD_STATUS_FILE"] = info_file_path
+import re
 
-cmd = [
-  bin_path,
-  # We rely on the test order being deterministic
-  '--order', 'decl'
-]
+# The two shards together run the two tests of the binary, each once.
+total = 0
+for index in range(2):
+    if os.path.isfile(info_file_path):
+        os.remove(info_file_path)
+    env = os.environ.copy()
+    env["BAZEL_TEST"] = "1"
+    env["TEST_SHARD_INDEX"] = str(index)
+    env["TEST_TOTAL_SHARDS"] = "2"
+    env["TEST_SHARD_STATUS_FILE"] = info_file_path
 
-try:
     ret = subprocess.run(
-        cmd,
+        [bin_path, '--allow-no-tests', '--retries=off'],
         stdout=subprocess.PIPE,
         stderr=subprocess.PIPE,
-        check=True,
         universal_newlines=True,
         env=env
     )
-    stdout = ret.stdout
-except subprocess.SubprocessError as ex:
-    print('Could not run "{}"'.format(bin_path))
-    print("Return code: {}".format(ex.returncode))
-    print("stdout: {}".format(ex.stdout))
-    print("stderr: {}".format(ex.stderr))
-    raise
+    if ret.returncode not in (0, 42):
+        print('Could not run "{}"'.format(bin_path))
+        print("Return code: {}".format(ret.returncode))
+        print("stdout: {}".format(ret.stdout))
+        print("stderr: {}".format(ret.stderr))
+        exit(1)
 
+    summary = re.search(r'(\d+) test\(s\):', ret.stdout)
+    if summary is None:
+        print("Did not find the summary in stdout.")
+        print("stdout:\n{}".format(ret.stdout))
+        exit(1)
+    total += int(summary.group(1))
 
-if not "All tests passed (1 assertion in 1 test case)" in stdout:
-    print("Did not find expected output in stdout.")
-    print("stdout:\n{}".format(stdout))
-    exit(1)
+    if not os.path.isfile(info_file_path):
+        print("The runner did not create the expected file at path '{}'".format(info_file_path))
+        exit(2)
 
-if not os.path.isfile(info_file_path):
-    print("Catch2 did not create expected file at path '{}'".format(info_file_path))
-    exit(2)
+if total != 2:
+    print("The shards ran {} tests together instead of 2".format(total))
+    exit(3)
diff --git a/tests/TestScripts/testConfigureDefaultReporter.py b/tests/TestScripts/testConfigureDefaultReporter.py
deleted file mode 100644
index 119e1ca2..00000000
--- a/tests/TestScripts/testConfigureDefaultReporter.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-
-#              Copyright Catch2 Authors
-# Distributed under the Boost Software License, Version 1.0.
-#   (See accompanying file LICENSE.txt or copy at
-#        https://www.boost.org/LICENSE_1_0.txt)
-
-# SPDX-License-Identifier: BSL-1.0
-
-from ConfigureTestsCommon import configure_and_build, run_and_return_output
-
-import os
-import re
-import sys
-
-"""
-Tests the CMake configure option for CATCH_CONFIG_DEFAULT_REPORTER
-
-Requires 2 arguments, path folder where the Catch2's main CMakeLists.txt
-exists, and path to where the output files should be stored.
-"""
-
-if len(sys.argv) != 3:
-    print('Wrong number of arguments: {}'.format(len(sys.argv)))
-    print('Usage: {} catch2-top-level-dir base-build-output-dir'.format(sys.argv[0]))
-    exit(1)
-
-catch2_source_path = os.path.abspath(sys.argv[1])
-build_dir_path = os.path.join(os.path.abspath(sys.argv[2]), 'CMakeConfigTests', 'DefaultReporter')
-
-output_file = f"{build_dir_path}/foo.xml"
-# We need to escape backslashes in Windows paths, because otherwise they
-# are interpreted as escape characters in strings, and cause compilation
-# error.
-escaped_output_file = output_file.replace('\\', '\\\\')
-configure_and_build(catch2_source_path,
-                    build_dir_path,
-                    [("CATCH_CONFIG_DEFAULT_REPORTER", f"xml::out={escaped_output_file}")])
-
-stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'), 'SelfTest', ['[approx][custom]'])
-
-if not os.path.exists(output_file):
-    print(f'Did not find the {output_file} file')
-    exit(2)
-
-xml_tag = '</Catch2TestRun>'
-with open(output_file, 'r', encoding='utf-8') as file:
-    if xml_tag not in file.read():
-        print(f"Could not find '{xml_tag}' in the file")
-        exit(3)
diff --git a/tests/TestScripts/testConfigureDisable.py b/tests/TestScripts/testConfigureDisable.py
index b0247700..ce9b2f9f 100644
--- a/tests/TestScripts/testConfigureDisable.py
+++ b/tests/TestScripts/testConfigureDisable.py
@@ -38,10 +38,10 @@ configure_and_build(catch2_source_path,
 
 stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
                                   'SelfTest',
-                                  ['--allow-running-no-tests'])
+                                  ['--allow-no-tests', '--no-history'])
 
 
-summary_line = 'No tests ran'
+summary_line = '0 test(s): none ran'
 if not summary_line in stdout:
     print("Could not find '{}' in the stdout".format(summary_line))
     print('stdout: "{}"'.format(stdout))
diff --git a/tests/TestScripts/testConfigureExperimentalRedirect.py b/tests/TestScripts/testConfigureExperimentalRedirect.py
index 09ff51e2..cf659e9e 100644
--- a/tests/TestScripts/testConfigureExperimentalRedirect.py
+++ b/tests/TestScripts/testConfigureExperimentalRedirect.py
@@ -9,6 +9,7 @@
 
 from ConfigureTestsCommon import configure_and_build, run_and_return_output
 
+import json
 import os
 import re
 import sys
@@ -32,18 +33,17 @@ configure_and_build(catch2_source_path,
                     build_dir_path,
                     [("CATCH_CONFIG_EXPERIMENTAL_REDIRECT", "ON")])
 
-stdout, _ = run_and_return_output(os.path.join(build_dir_path, 'tests'),
-                                  'SelfTest',
-                                  ['-r', 'xml', '"has printf"'])
-
-
-# The print from printf must be within the XML's reporter stdout tag.
-required_output = '''\
-      <StdOut>
-loose text artifact
-      </StdOut>
-'''
-if not required_output in stdout:
-    print("Could not find '{}' in the stdout".format(required_output))
-    print('stdout: "{}"'.format(stdout))
+report_path = os.path.join(build_dir_path, 'has-printf.json')
+run_and_return_output(os.path.join(build_dir_path, 'tests'),
+                      'SelfTest',
+                      ['--no-history', '--json=' + report_path, 'has printf'])
+
+# The text printf writes must be captured as the test's standard output.
+with open(report_path, encoding='utf-8') as report_file:
+    report = json.load(report_file)
+outputs = [attempt.get('standardOutput', '')
+           for test in report['tests']
+           for attempt in test['attempts']]
+if outputs != ['loose text artifact\n']:
+    print('Unexpected captured output: {}'.format(outputs))
     exit(2)
diff --git a/tests/TestScripts/testPartialTestCaseEvent.py b/tests/TestScripts/testPartialTestCaseEvent.py
index 55c6e299..fb3932cc 100755
--- a/tests/TestScripts/testPartialTestCaseEvent.py
+++ b/tests/TestScripts/testPartialTestCaseEvent.py
@@ -8,9 +8,9 @@
 # SPDX-License-Identifier: BSL-1.0
 
 """
-This test script verifies that the testCasePartial{Starting,Ended} reporter
+This test script verifies that the testCasePartial{Starting,Ended} listener
 events fire properly. This is done by calling a test binary compiled with
-reporter that reports specifically testCase* events, and verifying the
+a listener that records specifically testCase* events, and verifying the
 outputs match what we expect.
 """
 
@@ -46,21 +46,20 @@ TestCaseEnded: generator
 
 from typing import List
 
-def get_test_output(test_exe: str, sections: bool) -> List[str]:
-    cmd = [test_exe, '--reporter', 'partial']
-    if sections:
-        cmd.append('section')
-    else:
-        cmd.append('generator')
-
+def get_test_output(test_exe: str, sections: bool) -> str:
+    cmd = [test_exe, '--no-history', 'section' if sections else 'generator']
     ret = subprocess.run(cmd,
                          stdout = subprocess.PIPE,
                          stderr = subprocess.PIPE,
-                         timeout = 10,
+                         timeout = 30,
                          check = True,
                          universal_newlines = True)
-
-    return ret.stdout
+    # The listener prints the events when the run ends, outside any test;
+    # the runner passes that output on.
+    output = ret.stdout + ret.stderr
+    start = output.index('BEGIN EVENTS\n') + len('BEGIN EVENTS\n')
+    end = output.index('END EVENTS', start)
+    return output[start:end].replace('\r\n', '\n')
 
 def main():
     test_exe, = sys.argv[1:]
diff --git a/tests/TestScripts/testRandomOrder.py b/tests/TestScripts/testRandomOrder.py
deleted file mode 100755
index 2368423e..00000000
--- a/tests/TestScripts/testRandomOrder.py
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/env python3
-
-#              Copyright Catch2 Authors
-# Distributed under the Boost Software License, Version 1.0.
-#   (See accompanying file LICENSE.txt or copy at
-#        https://www.boost.org/LICENSE_1_0.txt)
-
-# SPDX-License-Identifier: BSL-1.0
-
-"""
-This test script verifies that the random ordering of tests inside
-Catch2 is invariant in regards to subsetting. This is done by running
-the binary 3 times, once with all tests selected, and twice with smaller
-subsets of tests selected, and verifying that the selected tests are in
-the same relative order.
-"""
-
-import subprocess
-import sys
-import random
-import xml.etree.ElementTree as ET
-
-def none_to_empty_str(e):
-    if e is None:
-        return ""
-    assert type(e) is str
-    return e
-
-def list_tests(self_test_exe, tags, rng_seed):
-    cmd = [self_test_exe, '--reporter', 'xml', '--list-tests', '--order', 'rand',
-            '--rng-seed', str(rng_seed)]
-    tags_arg = ','.join('[{}]~[.]'.format(t) for t in tags)
-    if tags_arg:
-        cmd.append(tags_arg)
-    process = subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    stdout, stderr = process.communicate()
-    if stderr:
-        raise RuntimeError("Unexpected error output:\n" + process.stderr)
-
-    root = ET.fromstring(stdout)
-    result = [(none_to_empty_str(tc.find('Name').text),
-               none_to_empty_str(tc.find('Tags').text),
-               none_to_empty_str(tc.find('ClassName').text)) for tc in root.findall('./TestCase')]
-
-    if len(result) < 2:
-        raise RuntimeError("Unexpectedly few tests listed (got {})".format(
-            len(result)))
-    return result
-
-def check_is_sublist_of(shorter, longer):
-    assert len(shorter) < len(longer)
-    assert len(set(longer)) == len(longer)
-
-    indexes_in_longer = {s: i for i, s in enumerate(longer)}
-    for s1, s2 in zip(shorter, shorter[1:]):
-        assert indexes_in_longer[s1] < indexes_in_longer[s2], (
-                '{} comes before {} in longer list.\n'
-                'Longer: {}\nShorter: {}'.format(s2, s1, longer, shorter))
-
-def main():
-    self_test_exe, = sys.argv[1:]
-
-    # We want a random seed for the test, but want to avoid 0,
-    # because it has special meaning
-    seed = random.randint(1, 2 ** 32 - 1)
-
-    list_one_tag = list_tests(self_test_exe, ['generators'], seed)
-    list_two_tags = list_tests(self_test_exe, ['generators', 'matchers'], seed)
-    list_all = list_tests(self_test_exe, [], seed)
-
-    # First, verify that restricting to a subset yields the same order
-    check_is_sublist_of(list_two_tags, list_all)
-    check_is_sublist_of(list_one_tag, list_two_tags)
-
-if __name__ == '__main__':
-    sys.exit(main())
diff --git a/tests/TestScripts/testSectionFiltering.py b/tests/TestScripts/testSectionFiltering.py
deleted file mode 100755
index 4f56df53..00000000
--- a/tests/TestScripts/testSectionFiltering.py
+++ /dev/null
@@ -1,189 +0,0 @@
-#!/usr/bin/env python3
-
-#              Copyright Catch2 Authors
-# Distributed under the Boost Software License, Version 1.0.
-#   (See accompanying file LICENSE.txt or copy at
-#        https://www.boost.org/LICENSE_1_0.txt)
-
-# SPDX-License-Identifier: BSL-1.0
-
-"""
-This test script verifies the behaviour of the legacy section filtering
-using `-c`, `--section` CLI parameters.
-
-This is done by having a hardcoded set of test filter + section filter
-combinations, together with the expected number of assertions that will
-be run inside the test for given filter combo.
-"""
-
-import os
-import subprocess
-import sys
-from typing import Tuple, List
-import xml.etree.ElementTree as ET
-
-def make_old_filters(section_names: Tuple[str, ...]) -> List[str]:
-    final = []
-    for name in section_names:
-        final.append('--section')
-        final.append(name)
-    return final
-
-def make_new_filters(path_filters: Tuple[str, ...]) -> List[str]:
-    final = []
-    for component in path_filters:
-        final.append('--path-filter')
-        final.append(component)
-    return final
-
-def run_one_test(binary_path: str,
-                 test_name: str,
-                 section_names: Tuple[str, ...],
-                 expected_assertions: int,
-                 new_filters: bool):
-    cmd = [
-      binary_path,
-      '--reporter', 'xml',
-      test_name
-    ]
-    if new_filters:
-        cmd.extend(make_new_filters(section_names))
-    else:
-        cmd.extend(make_old_filters(section_names))
-    try:
-        ret = subprocess.run(
-            cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            check=True,
-            universal_newlines=True,
-        )
-        stdout = ret.stdout
-    except subprocess.SubprocessError as ex:
-        print('Could not run "{}"'.format(cmd))
-        print("Return code: {}".format(ex.returncode))
-        print("stdout: {}".format(ex.stdout))
-        print("stderr: {}".format(ex.stderr))
-        raise
-
-    try:
-        tree = ET.fromstring(stdout)
-    except ET.ParseError as ex:
-        print("Invalid XML: '{}'".format(ex))
-        raise
-
-    # Validate that we ran exactly 1 test case, and it passed
-    test_case_stats = tree.find('OverallResultsCases')
-    expected_testcases = {'successes' : '1', 'failures' : '0', 'expectedFailures': '0', 'skips': '0'}
-    assert test_case_stats.attrib == expected_testcases, f'We did not run single passing test case as expected. {test_name}: {test_case_stats.attrib}'
-
-    # Validate that we got exactly the expected number of passing assertions
-    expected_assertions = {'successes' : str(expected_assertions), 'failures' : '0', 'expectedFailures': '0', 'skips': '0'}
-    assertion_stats = tree.find('OverallResults')
-    assert assertion_stats.attrib == expected_assertions, f'"{test_name}": {assertion_stats.attrib} vs {expected_assertions}'
-
-
-# Inputs taken from issue #3038
-old_section_tests = {
-  'foo': (
-    ((), 3),
-    (('A',), 3),
-    (('A', 'B'), 0),
-    (('A', 'B1'), 1),
-    (('A', 'B2'), 1),
-    (('A', 'B1', 'B2'), 1),
-    (('A', 'B2', 'XXXX'), 1),
-  ),
-  'bar': (
-    ((), 9),
-    (('A',), 9),
-    (('A', 'B1'), 3),
-    (('XXXX',), 2),
-    (('B1',), 2),
-    (('A', 'B1', 'B2'), 3),
-  ),
-  'baz': (
-    ((), 4),
-    (('A',), 1),
-    (('A', 'B'), 1),
-    (('A', 'XXXX'), 1),
-    (('B',), 3),
-    (('XXXX',), 0),
-  ),
-  'qux': (
-    ((), 12),
-    (('A',), 7),
-    (('B',), 9),
-    (('B', 'XXXX'), 9),
-    (('XXXX',), 6),
-  ),
-  'corge': (
-    ((), 12),
-    (('i=2',), 7),
-    (('i=3',), 7),
-  ),
-  'grault': (
-    ((), 12),
-    (('A',), 3),
-    (('B',), 9),
-    (('B', 'i=1'), 7),
-    (('B', 'XXXX'), 6),
-  ),
-}
-
-new_filter_tests = {
-  'grault': (
-    ((), 12),
-    (('c:B', 'g:1'), 3),
-    (('c:B', 'g:1', 'c:i=2'), 3),
-    # We filter the generator to different element than the dynamic section
-    (('c:B', 'g:1', 'c:i=1'), 2),
-  ),
-  'garply': (
-    ((), 3),
-    (('g:1',), 1),
-    (('g:*',), 3),
-    # Followup section filters do not change results (there are no sections underneath)
-    (('g:1', 'c:abcdef'), 1),
-    (('g:*', 'c:abcdef'), 3),
-  ),
-  'waldo': (
-    ((), 9),
-    (('g:1',), 3),
-    (('g:*',), 9),
-    (('g:1', 'g:2'), 1),
-    (('g:*', 'g:1'), 3),
-    (('g:0', 'g:*'), 3),
-    (('g:*', 'g:*'), 9),
-    (('g:*', 'g:1', 'c:abcdef'), 3),
-  ),
-  'fred': (
-    ((), 24),
-    # No sections entered, but we still hit assertion at the start of test
-    (('c:abcdef',), 1),
-    (('c:A', ), 12),
-    (('c:A', 'g:*', 'c:A1'), 6),
-    (('c:A', 'g:1', 'c:A1'), 2),
-    (('c:A', 'g:1'), 4),
-    # Section 'A3' does not exist
-    (('c:A', 'g:*', 'c:A3'), 3),
-    (('c:A', 'g:1', 'c:A3'), 1),
-    # We would lik