Replace Catch2's runner with the Catch3 runner

Catch3 keeps Catch2's API for writing tests (test cases, sections,
assertions, matchers, generators, listeners, benchmarks) and replaces its
session, command line and reporters with a runner of its own.

The runner starts the test binary again as worker processes that take the
next test when they are free. A crash, an exit or a hang ends one test, not
the run. On top of that it adds:

- retries that tell deterministic, flaky and order-dependent failures
  apart, and a bisection that names the test leaving state behind;
- a history of verdicts and durations, kept under a file lock, that drives
  automatic quarantine, adaptive time limits, predictive ordering,
  incremental runs, failure replay and analytics;
- selection by change, past failure or prediction, dependency closure,
  and sharding by a stable hash or by recorded durations;
- tags for serial and isolated tests, shared resources, dependencies,
  time limits and retries;
- a failure category, a signature, an analysis and a replay command for
  every failure;
- console, JUnit XML (with flakyFailure and rerunFailure), JSON and Chrome
  trace reports, and a versioned discovery listing (--list=json);
- a temporary directory per test (TMPDIR, TMP, TEMP and
  TestTemporaryDirectory());
- Bazel's test environment: XML output file, test filter, sharding, seed,
  premature exit file and undeclared outputs.

The command line is parsed by a hand-written, typed parser. Catch::Session
remains as a thin facade over the runner for custom main functions.

The result model in Progmasoft/Catch3/Results.hpp was rewritten around
tests and attempts, and the JUnit writer now writes it. The property
engine's example database is updated under a file lock with an atomic
rename, so parallel workers can write it at the same time.

Removed: Clara, the command-line and reporter-spec parsers, every reporter
and CATCH_REGISTER_REPORTER, listing, text flow, console colour and width,
the JSON and XML writers, sharding in the test registry, approval tests and
their baselines, Conan packaging, and the unused third_party/clara.hpp.
Listeners keep working; reporters' list and skip events are gone.

The Catch2 library is now compiled as C++20; code that includes only
Catch2's headers still needs C++14. Duplicate test cases are still refused
at start-up. Version 0.3.0.

ExtraTests now turn exceptions off for clang-cl with MSVC-style options,
which -fno-exceptions did not do.

1 files changed, 54 insertions, 142 deletions

diff --git a/tests/TestScripts/testSharding.py b/tests/TestScripts/testSharding.py
index fa6f94d3..c9a25fe7 100755
--- a/tests/TestScripts/testSharding.py
+++ b/tests/TestScripts/testSharding.py
@@ -8,158 +8,70 @@
 # SPDX-License-Identifier: BSL-1.0
 
 """
-This test script verifies that sharding tests does change which tests are run.
-This is done by running the binary multiple times, once to list all the tests,
-once per shard to list the tests for that shard, and once again per shard to
-execute the tests. The sharded lists are compared to the full list to ensure
-none are skipped, duplicated, and that the order remains the same.
+This test script verifies that sharding splits the tests without losing or
+repeating any. It lists the tests once without sharding and once per shard,
+runs every shard, and checks that the shards together are exactly all tests
+and that each shard runs what it lists.
 """
 
-import random
+import json
+import os
 import subprocess
 import sys
-import xml.etree.ElementTree as ET
+import tempfile
+from typing import List
 
-from collections import namedtuple
-
-from typing import List, Dict
-
-seed = random.randint(0, 2 ** 32 - 1)
 number_of_shards = 5
-
-def make_base_commandline(self_test_exe):
-    return [
-        self_test_exe,
-        '--reporter', 'xml',
-        '--order', 'rand',
-        '--rng-seed', str(seed),
-        "[generators]~[benchmarks]~[.]"
-    ]
-
-
-def list_tests(self_test_exe: str, extra_args: List[str] = None):
-    cmd = make_base_commandline(self_test_exe) + ['--list-tests']
-    if extra_args:
-        cmd.extend(extra_args)
-
-    try:
-        ret = subprocess.run(cmd,
-                             stdout = subprocess.PIPE,
-                             stderr = subprocess.PIPE,
-                             timeout = 10,
-                             check = True,
-                             universal_newlines = True)
-    except subprocess.CalledProcessError as ex:
-        print('Could not list tests:\n{}'.format(ex.stderr))
-
-    if ret.stderr:
-        raise RuntimeError("Unexpected error output:\n" + ret.stderr)
-
-    root = ET.fromstring(ret.stdout)
-    result = [elem.text for elem in root.findall('./TestCase/Name')]
-
-    if len(result) < 2:
-        raise RuntimeError("Unexpectedly few tests listed (got {})".format(
-            len(result)))
-
-
-    return result
-
-
-def execute_tests(self_test_exe: str, extra_args: List[str] = None):
-    cmd = make_base_commandline(self_test_exe)
-    if extra_args:
-        cmd.extend(extra_args)
-
-    try:
-        ret = subprocess.run(cmd,
-                             stdout = subprocess.PIPE,
-                             stderr = subprocess.PIPE,
-                             timeout = 10,
-                             check = True,
-                             universal_newlines = True)
-    except subprocess.CalledProcessError as ex:
-        print('Could not list tests:\n{}'.format(ex.stderr))
-
-    if ret.stderr:
-        raise RuntimeError("Unexpected error output:\n" + process.stderr)
-
-    root = ET.fromstring(ret.stdout)
-    result = [elem.attrib["name"] for elem in root.findall('./TestCase')]
-
-    if len(result) < 2:
-        raise RuntimeError("Unexpectedly few tests listed (got {})".format(
-            len(result)))
-
-    return result
-
-
-def test_sharded_listing(self_test_exe: str) -> Dict[int, List[str]]:
-    """
-    Asks the test binary for list of all tests, and also for lists of
-    tests from shards.
-
-    The combination of shards is then checked whether it corresponds to
-    the full list of all tests.
-
-    Returns the dictionary of shard-index => listed tests for later use.
-    """
-    all_tests = list_tests(self_test_exe)
-    big_shard_tests = list_tests(self_test_exe, ['--shard-count', '1', '--shard-index', '0'])
-
-    assert all_tests == big_shard_tests, (
-        "No-sharding test list does not match the listing of big shard:\nNo shard:\n{}\n\nWith shard:\n{}\n".format(
-            '\n'.join(all_tests),
-            '\n'.join(big_shard_tests)
-        )
-    )
-
-    shard_listings = dict()
-    for shard_idx in range(number_of_shards):
-        shard_listings[shard_idx] = list_tests(self_test_exe, ['--shard-count', str(number_of_shards), '--shard-index', str(shard_idx)])
-
-    shard_sizes = [len(v) for v in shard_listings.values()]
-    assert len(all_tests) == sum(shard_sizes)
-
-    # Check that the shards have roughly the right sizes (e.g. we don't
-    # have all tests in single shard and the others are empty)
-    differences = [abs(x1 - x2) for x1, x2 in zip(shard_sizes, shard_sizes[1:])]
-    assert all(diff <= 1 for diff in differences), "A shard has weird size: {}".format(shard_sizes)
-
-    combined_shards = [inner for outer in shard_listings.values() for inner in outer]
-    assert all_tests == combined_shards, (
-        "All tests and combined shards disagree.\nNo shard:\n{}\n\nCombined:\n{}\n\n".format(
-            '\n'.join(all_tests),
-            '\n'.join(combined_shards)
-        )
-    )
-    shard_listings[-1] = all_tests
-
-    return shard_listings
-
-
-def test_sharded_execution(self_test_exe: str, listings: Dict[int, List[str]]):
-    """
-    Runs the test binary and checks that the executed tests match the
-    previously listed tests.
-
-    Also does this for various shard indices, and that the combination
-    of all shards matches the full run/listing.
-    """
-    all_tests = execute_tests(self_test_exe)
-    big_shard_tests = execute_tests(self_test_exe, ['--shard-count', '1', '--shard-index', '0'])
-    assert all_tests == big_shard_tests
-
-    assert listings[-1] == all_tests
-
-    for shard_idx in range(number_of_shards):
-        assert listings[shard_idx] == execute_tests(self_test_exe, ['--shard-count', str(number_of_shards), '--shard-index', str(shard_idx)])
+spec = "[generators]~[benchmarks]"
+
+
+def list_tests(self_test_exe: str, extra_args: List[str] = None) -> List[str]:
+    cmd = [self_test_exe, '--list=names', spec] + (extra_args or [])
+    ret = subprocess.run(cmd,
+                         stdout = subprocess.PIPE,
+                         stderr = subprocess.PIPE,
+                         timeout = 60,
+                         check = True,
+                         universal_newlines = True)
+    return [line for line in ret.stdout.splitlines() if line]
+
+
+def execute_tests(self_test_exe: str, extra_args: List[str]) -> List[str]:
+    with tempfile.TemporaryDirectory() as directory:
+        report = os.path.join(directory, 'report.json')
+        cmd = [self_test_exe, '--no-history', '--allow-no-tests',
+               '--json=' + report, spec] + extra_args
+        subprocess.run(cmd,
+                       stdout = subprocess.PIPE,
+                       stderr = subprocess.PIPE,
+                       timeout = 120,
+                       check = True,
+                       universal_newlines = True)
+        with open(report, encoding = 'utf-8') as file:
+            run = json.load(file)
+    return [test['name'] for test in run['tests']]
 
 
 def main():
     self_test_exe, = sys.argv[1:]
-    listings = test_sharded_listing(self_test_exe)
-    test_sharded_execution(self_test_exe, listings)
+    all_tests = list_tests(self_test_exe)
+    if len(all_tests) < 2:
+        raise RuntimeError("Unexpectedly few tests listed (got {})".format(len(all_tests)))
+    assert all_tests == list_tests(self_test_exe, ['--shard=0/1'])
+
+    combined = []
+    for index in range(number_of_shards):
+        shard = '--shard={}/{}'.format(index, number_of_shards)
+        listed = list_tests(self_test_exe, [shard])
+        executed = execute_tests(self_test_exe, [shard])
+        assert sorted(listed) == sorted(executed), (
+            "Shard {} lists\n{}\nbut runs\n{}".format(index, listed, executed))
+        combined += listed
+
+    assert sorted(all_tests) == sorted(combined), (
+        "All tests and combined shards disagree.\nNo shard:\n{}\n\nCombined:\n{}\n".format(
+            '\n'.join(all_tests), '\n'.join(combined)))
+
 
 if __name__ == '__main__':
     sys.exit(main())