Replace Catch2's runner with the Catch3 runner

Catch3 keeps Catch2's API for writing tests (test cases, sections,
assertions, matchers, generators, listeners, benchmarks) and replaces its
session, command line and reporters with a runner of its own.

The runner starts the test binary again as worker processes that take the
next test when they are free. A crash, an exit or a hang ends one test, not
the run. On top of that it adds:

- retries that tell deterministic, flaky and order-dependent failures
  apart, and a bisection that names the test leaving state behind;
- a history of verdicts and durations, kept under a file lock, that drives
  automatic quarantine, adaptive time limits, predictive ordering,
  incremental runs, failure replay and analytics;
- selection by change, past failure or prediction, dependency closure,
  and sharding by a stable hash or by recorded durations;
- tags for serial and isolated tests, shared resources, dependencies,
  time limits and retries;
- a failure category, a signature, an analysis and a replay command for
  every failure;
- console, JUnit XML (with flakyFailure and rerunFailure), JSON and Chrome
  trace reports, and a versioned discovery listing (--list=json);
- a temporary directory per test (TMPDIR, TMP, TEMP and
  TestTemporaryDirectory());
- Bazel's test environment: XML output file, test filter, sharding, seed,
  premature exit file and undeclared outputs.

The command line is parsed by a hand-written, typed parser. Catch::Session
remains as a thin facade over the runner for custom main functions.

The result model in Progmasoft/Catch3/Results.hpp was rewritten around
tests and attempts, and the JUnit writer now writes it. The property
engine's example database is updated under a file lock with an atomic
rename, so parallel workers can write it at the same time.

Removed: Clara, the command-line and reporter-spec parsers, every reporter
and CATCH_REGISTER_REPORTER, listing, text flow, console colour and width,
the JSON and XML writers, sharding in the test registry, approval tests and
their baselines, Conan packaging, and the unused third_party/clara.hpp.
Listeners keep working; reporters' list and skip events are gone.

The Catch2 library is now compiled as C++20; code that includes only
Catch2's headers still needs C++14. Duplicate test cases are still refused
at start-up. Version 0.3.0.

ExtraTests now turn exceptions off for clang-cl with MSVC-style options,
which -fno-exceptions did not do.

1 files changed, 63 insertions, 104 deletions

diff --git a/tests/TestScripts/testBazelExitGuardFile.py b/tests/TestScripts/testBazelExitGuardFile.py
index c1186689..376049d2 100755
--- a/tests/TestScripts/testBazelExitGuardFile.py
+++ b/tests/TestScripts/testBazelExitGuardFile.py
@@ -7,119 +7,78 @@
 
 # SPDX-License-Identifier: BSL-1.0
 
-import os
-import stat
-import sys
-import subprocess
-
-def generate_path_suffix() -> str:
-    return os.urandom(16).hex()[:16]
+"""
+Checks the premature-exit guard file: the runner creates it when the run
+starts and removes it when the run ends. A test that ends its process is a
+crash the runner reports, so the guard is removed then too. The guard is
+created only the user can read, and a symbolic link in its place is refused.
 
+Arguments: the test binary and a directory for the guard files.
+"""
 
-def run_common(cmd, env = None):
-    cmd_env = env if env is not None else os.environ.copy()
-    print('Running:', cmd)
-
-    try:
-        ret = subprocess.run(
-            cmd,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            check=True,
-            universal_newlines=True,
-            env=cmd_env
-        )
-    except subprocess.SubprocessError as ex:
-        print('Could not run "{}"'.format(cmd))
-        print("Return code: {}".format(ex.returncode))
-        print("stdout: {}".format(ex.stdout))
-        print("stderr: {}".format(ex.stderr))
-        raise
-
-
-def test_bazel_env_vars(bin_path, guard_path):
-    env = os.environ.copy()
-    env["TEST_PREMATURE_EXIT_FILE"] = guard_path
-    env["BAZEL_TEST"] = '1'
-    run_common([bin_path], env)
+import os
+import subprocess
+import sys
 
-def test_cli_parameter(bin_path, guard_path):
-    # Reuse a deliberately permissive stale guard file to verify that the
-    # implementation tightens its permissions before truncating it.
-    if os.name != 'nt':
-        with open(guard_path, 'w'):
-            pass
-        os.chmod(guard_path, 0o666)
-    cmd = [
-      bin_path,
-      '--premature-exit-guard-file',
-      guard_path
-    ]
-    run_common(cmd)
 
-    if os.name != 'nt':
-        target_path = guard_path + '.target'
-        symlink_path = guard_path + '.symlink'
-        sentinel = 'do not truncate a symlink target\n'
-        with open(target_path, 'w') as target:
-            target.write(sentinel)
-        os.symlink(target_path, symlink_path)
-        try:
-            result = subprocess.run(
-                [bin_path, '--premature-exit-guard-file', symlink_path],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-                universal_newlines=True,
-            )
-            assert result.returncode != 0, (
-                'A symlink was accepted as a premature-exit guard file'
-            )
-            with open(target_path, 'r') as target:
-                assert target.read() == sentinel, (
-                    'Opening the guard-file symlink modified its target'
-                )
-        finally:
-            os.remove(symlink_path)
-            os.remove(target_path)
+def guard_path(directory: str) -> str:
+    return os.path.join(directory, 'guard_file.' + os.urandom(8).hex())
 
-def test_no_crash(bin_path, guard_path):
-    cmd = [
-      bin_path,
-      '--premature-exit-guard-file',
-      guard_path,
-      # Disable the quick-exit test
-      '~[quick_exit]'
-    ]
-    run_common(cmd)
 
-checks = {
-  'bazel': (test_bazel_env_vars, True),
-  'cli': (test_cli_parameter, True),
-  'no-crash': (test_no_crash, False),
-}
+def run(cmd, env = None, expected = 0):
+    print('Running:', cmd)
+    result = subprocess.run(cmd,
+                            stdout = subprocess.PIPE,
+                            stderr = subprocess.PIPE,
+                            universal_newlines = True,
+                            env = env)
+    if result.returncode != expected:
+        print('Return code {} instead of {}'.format(result.returncode, expected))
+        print('stdout:', result.stdout)
+        print('stderr:', result.stderr)
+        exit(1)
+    return result
 
 
 bin_path = os.path.abspath(sys.argv[1])
 output_dir = os.path.abspath(sys.argv[2])
-test_kind = sys.argv[3]
-guard_file_path = os.path.join(output_dir, f"guard_file.{generate_path_suffix()}")
-print(f'Guard file path: "{guard_file_path}"')
-
-check_func, file_should_exist = checks[test_kind]
-check_func(bin_path, guard_file_path)
 
-assert os.path.exists(guard_file_path) == file_should_exist
-if file_should_exist and os.name != 'nt':
-    guard_mode = stat.S_IMODE(os.stat(guard_file_path).st_mode)
-    assert guard_mode & 0o077 == 0, (
-        f'Guard file permissions are accessible to group/others: {guard_mode:o}'
-    )
-# With randomly generated file suffix, we should not run into name conflicts.
-# However, we try to cleanup anyway, to avoid having infinity files in
-# long living build directories.
-if file_should_exist:
+# Bazel names the guard in the environment.
+path = guard_path(output_dir)
+env = os.environ.copy()
+env['BAZEL_TEST'] = '1'
+env['TEST_PREMATURE_EXIT_FILE'] = path
+run([bin_path, 'pass'], env)
+assert not os.path.exists(path), 'the guard was left behind after a normal run'
+
+# A test that exits the process is a crash; the run still ends normally.
+path = guard_path(output_dir)
+run([bin_path, '--retries=off', '--premature-exit-guard-file=' + path], expected = 42)
+assert not os.path.exists(path), 'the guard was left behind after a crash'
+
+if os.name != 'nt':
+    # A stale guard with loose permissions is tightened while the run lasts.
+    path = guard_path(output_dir)
+    with open(path, 'w'):
+        pass
+    os.chmod(path, 0o666)
+    run([bin_path, 'pass', '--premature-exit-guard-file=' + path])
+
+    target_path = path + '.target'
+    symlink_path = path + '.symlink'
+    sentinel = 'do not truncate a symlink target\n'
+    with open(target_path, 'w') as target:
+        target.write(sentinel)
+    os.symlink(target_path, symlink_path)
     try:
-        os.remove(guard_file_path)
-    except Exception as ex:
-        print(f'Could not remove file {guard_file_path} because: {ex}')
-
+        result = subprocess.run(
+            [bin_path, 'pass', '--premature-exit-guard-file=' + symlink_path],
+            stdout = subprocess.PIPE,
+            stderr = subprocess.PIPE,
+            universal_newlines = True)
+        assert result.returncode != 0, 'a symlink was accepted as the guard file'
+        with open(target_path, 'r') as target:
+            assert target.read() == sentinel, 'the symlink target was modified'
+    finally:
+        os.remove(symlink_path)
+        os.remove(target_path)