Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/tests/TestScripts
diff options
context:
space:
mode:
authorBen Dunkin2021-07-11 12:46:05 -0700
committerMartin Hořeňovský2021-10-27 17:24:30 +0200
commit3087e19cc72ccb6caddc42ec2f39eebc56160754 (patch)
tree36d3adb84f1876a7136c169b219637343055a44d /tests/TestScripts
parent6456ee8b01c6133e553b99da2f7420e7b20da7ec (diff)
downloadcatch3-3087e19cc72ccb6caddc42ec2f39eebc56160754.tar.gz
Allow test sharding for e.g. Bazel test sharding feature
This greatly simplifies running Catch2 tests in single binary in parallel from external test runners. Instead of having to shard the tests by tags/test names, an external test runner can now just ask for test shard 2 (out of X), and execute that in single process, without having to know what tests are actually in the shard. Note that sharding also applies to test listing, and happens after tests were ordered according to the `--order` feature.
Diffstat (limited to 'tests/TestScripts')
-rw-r--r--tests/TestScripts/testSharding.py90
1 files changed, 90 insertions, 0 deletions
diff --git a/tests/TestScripts/testSharding.py b/tests/TestScripts/testSharding.py
new file mode 100644
index 00000000..fcea03e2
--- /dev/null
+++ b/tests/TestScripts/testSharding.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+
+"""
+This test script verifies that sharding tests does change which tests are run.
+This is done by running the binary multiple times, once to list all the tests,
+once per shard to list the tests for that shard, and once again per shard to
+execute the tests. The sharded lists are compared to the full list to ensure
+none are skipped, duplicated, and that the order remains the same. This process
+is repeated for multiple command line argument combinations to ensure sharding
+works with different filters and test orderings.
+"""
+
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+
+from collections import namedtuple
+
+def make_base_commandline(self_test_exe):
+ return [
+ self_test_exe,
+ '--reporter', 'xml',
+ "--shard-count", "5",
+ "--shard-index", "2",
+ "[generators]~[benchmarks]~[.]"
+ ]
+
+def list_tests(self_test_exe):
+ cmd = make_base_commandline(self_test_exe) + ['--list-tests']
+
+ process = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = process.communicate()
+ if stderr:
+ raise RuntimeError("Unexpected error output:\n" + process.stderr)
+
+ root = ET.fromstring(stdout)
+ result = [elem.text for elem in root.findall('./TestCase/Name')]
+
+ if len(result) < 2:
+ raise RuntimeError("Unexpectedly few tests listed (got {})".format(
+ len(result)))
+
+ return result
+
+
+def execute_tests(self_test_exe):
+ cmd = make_base_commandline(self_test_exe)
+
+ process = subprocess.Popen(
+ cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = process.communicate()
+ if stderr:
+ raise RuntimeError("Unexpected error output:\n" + process.stderr)
+
+ root = ET.fromstring(stdout)
+ result = [elem.attrib["name"] for elem in root.findall('./TestCase')]
+
+ if len(result) < 2:
+ raise RuntimeError("Unexpectedly few tests listed (got {})".format(
+ len(result)))
+ return result
+
+
+def check_listed_and_executed_tests_match(listed_tests, executed_tests):
+ listed_names = set(listed_tests)
+ executed_names = set(executed_tests)
+
+ listed_string = "\n".join(listed_names)
+ exeucted_string = "\n".join(executed_names)
+
+ assert listed_names == executed_names, (
+ "Executed tests do not match the listed tests:\nExecuted:\n{}\n\nListed:\n{}".format(exeucted_string, listed_string)
+ )
+
+
+def test_sharding(self_test_exe):
+ listed_tests = list_tests(self_test_exe)
+ executed_tests = execute_tests(self_test_exe)
+
+ check_listed_and_executed_tests_match(listed_tests, executed_tests)
+
+
+def main():
+ self_test_exe, = sys.argv[1:]
+
+ test_sharding(self_test_exe)
+
+if __name__ == '__main__':
+ sys.exit(main())