diff options
| author | Martin Hořeňovský | 2020-04-14 16:38:07 +0200 |
|---|---|---|
| committer | Martin Hořeňovský | 2020-05-01 14:45:36 +0200 |
| commit | 9b5fc9eaeacbe0bac5575824368c4a693a9a58a1 (patch) | |
| tree | edf89b07f668d11c2004cf5736302dafea788eb0 /tests/TestScripts | |
| parent | 630ba2627802f2832929dc52d3a69ddc83546959 (diff) | |
| download | catch3-9b5fc9eaeacbe0bac5575824368c4a693a9a58a1.tar.gz | |
Randomize test for subset invariant random ordering of tests
Also removed the iterative checking that seeds 1-100 do not create
the same output, because it used too much runtime.
Diffstat (limited to 'tests/TestScripts')
| -rwxr-xr-x | tests/TestScripts/testRandomOrder.py | 26 |
1 files changed, 16 insertions, 10 deletions
diff --git a/tests/TestScripts/testRandomOrder.py b/tests/TestScripts/testRandomOrder.py index 888903cf..cfd867b9 100755 --- a/tests/TestScripts/testRandomOrder.py +++ b/tests/TestScripts/testRandomOrder.py @@ -1,7 +1,16 @@ #!/usr/bin/env python3 +""" +This test script verifies that the random ordering of tests inside +Catch2 is invariant in regards to subsetting. This is done by running +the binary 3 times, once with all tests selected, and twice with smaller +subsets of tests selected, and verifying that the selected tests are in +the same relative order. +""" + import subprocess import sys +import random import xml.etree.ElementTree as ET def list_tests(self_test_exe, tags, rng_seed): @@ -37,20 +46,17 @@ def check_is_sublist_of(shorter, longer): def main(): self_test_exe, = sys.argv[1:] - list_one_tag = list_tests(self_test_exe, ['generators'], 1) - list_two_tags = list_tests(self_test_exe, ['generators', 'matchers'], 1) - list_all = list_tests(self_test_exe, [], 1) + # We want a random seed for the test, but want to avoid 0, + # because it has special meaning + seed = random.randint(1, 2 ** 32 - 1) + + list_one_tag = list_tests(self_test_exe, ['generators'], seed) + list_two_tags = list_tests(self_test_exe, ['generators', 'matchers'], seed) + list_all = list_tests(self_test_exe, [], seed) # First, verify that restricting to a subset yields the same order check_is_sublist_of(list_two_tags, list_all) check_is_sublist_of(list_one_tag, list_two_tags) - # Second, verify that different seeds yield different orders - num_seeds = 100 - seeds = range(1, num_seeds + 1) # Avoid zero since that's special - lists = {tuple(list_tests(self_test_exe, [], seed)) for seed in seeds} - assert len(lists) == num_seeds, ( - 'Got {} distict lists from {} seeds'.format(len(lists), num_seeds)) - if __name__ == '__main__': sys.exit(main()) |