1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
|
#!/usr/bin/env python3
# Copyright Catch2 Authors
# Distributed under the Boost Software License, Version 1.0.
# (See accompanying file LICENSE.txt or copy at
# https://www.boost.org/LICENSE_1_0.txt)
# SPDX-License-Identifier: BSL-1.0
"""
This test script verifies the behaviour of the legacy section filtering
using `-c`, `--section` CLI parameters.
This is done by having a hardcoded set of test filter + section filter
combinations, together with the expected number of assertions that will
be run inside the test for given filter combo.
"""
import os
import subprocess
import sys
from typing import Tuple, List
import xml.etree.ElementTree as ET
def make_old_filters(section_names: Tuple[str, ...]) -> List[str]:
final = []
for name in section_names:
final.append('--section')
final.append(name)
return final
def make_new_filters(path_filters: Tuple[str, ...]) -> List[str]:
final = []
for component in path_filters:
final.append('--path-filter')
final.append(component)
return final
def run_one_test(binary_path: str,
test_name: str,
section_names: Tuple[str, ...],
expected_assertions: int,
new_filters: bool):
cmd = [
binary_path,
'--reporter', 'xml',
test_name
]
if new_filters:
cmd.extend(make_new_filters(section_names))
else:
cmd.extend(make_old_filters(section_names))
try:
ret = subprocess.run(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=True,
universal_newlines=True,
)
stdout = ret.stdout
except subprocess.SubprocessError as ex:
print('Could not run "{}"'.format(cmd))
print("Return code: {}".format(ex.returncode))
print("stdout: {}".format(ex.stdout))
print("stderr: {}".format(ex.stderr))
raise
try:
tree = ET.fromstring(stdout)
except ET.ParseError as ex:
print("Invalid XML: '{}'".format(ex))
raise
# Validate that we ran exactly 1 test case, and it passed
test_case_stats = tree.find('OverallResultsCases')
expected_testcases = {'successes' : '1', 'failures' : '0', 'expectedFailures': '0', 'skips': '0'}
assert test_case_stats.attrib == expected_testcases, f'We did not run single passing test case as expected. {test_name}: {test_case_stats.attrib}'
# Validate that we got exactly the expected number of passing assertions
expected_assertions = {'successes' : str(expected_assertions), 'failures' : '0', 'expectedFailures': '0', 'skips': '0'}
assertion_stats = tree.find('OverallResults')
assert assertion_stats.attrib == expected_assertions, f'"{test_name}": {assertion_stats.attrib} vs {expected_assertions}'
# Inputs taken from issue #3038
old_section_tests = {
'foo': (
((), 3),
(('A',), 3),
(('A', 'B'), 0),
(('A', 'B1'), 1),
(('A', 'B2'), 1),
(('A', 'B1', 'B2'), 1),
(('A', 'B2', 'XXXX'), 1),
),
'bar': (
((), 9),
(('A',), 9),
(('A', 'B1'), 3),
(('XXXX',), 2),
(('B1',), 2),
(('A', 'B1', 'B2'), 3),
),
'baz': (
((), 4),
(('A',), 1),
(('A', 'B'), 1),
(('A', 'XXXX'), 1),
(('B',), 3),
(('XXXX',), 0),
),
'qux': (
((), 12),
(('A',), 7),
(('B',), 9),
(('B', 'XXXX'), 9),
(('XXXX',), 6),
),
'corge': (
((), 12),
(('i=2',), 7),
(('i=3',), 7),
),
'grault': (
((), 12),
(('A',), 3),
(('B',), 9),
(('B', 'i=1'), 7),
(('B', 'XXXX'), 6),
),
}
new_filter_tests = {
'grault': (
((), 12),
(('c:B', 'g:1'), 3),
(('c:B', 'g:1', 'c:i=2'), 3),
# We filter the generator to different element than the dynamic section
(('c:B', 'g:1', 'c:i=1'), 2),
),
'garply': (
((), 3),
(('g:1',), 1),
(('g:*',), 3),
# Followup section filters do not change results (there are no sections underneath)
(('g:1', 'c:abcdef'), 1),
(('g:*', 'c:abcdef'), 3),
),
'waldo': (
((), 9),
(('g:1',), 3),
(('g:*',), 9),
(('g:1', 'g:2'), 1),
(('g:*', 'g:1'), 3),
(('g:0', 'g:*'), 3),
(('g:*', 'g:*'), 9),
(('g:*', 'g:1', 'c:abcdef'), 3),
),
'fred': (
((), 24),
# No sections entered, but we still hit assertion at the start of test
(('c:abcdef',), 1),
(('c:A', ), 12),
(('c:A', 'g:*', 'c:A1'), 6),
(('c:A', 'g:1', 'c:A1'), 2),
(('c:A', 'g:1'), 4),
# Section 'A3' does not exist
(('c:A', 'g:*', 'c:A3'), 3),
(('c:A', 'g:1', 'c:A3'), 1),
# We would like something like 'c:*', 'g:2', but we do not support globs
# for sections (yet?)
),
}
if len(sys.argv) != 2:
print("Wrong number of arguments, expected just the path to Catch2 SelfTest binary")
exit(1)
bin_path = os.path.abspath(sys.argv[1])
for test_filter, specs in old_section_tests.items():
for section_path, expected_assertions in specs:
run_one_test(bin_path, test_filter, section_path, expected_assertions, new_filters=False)
for test_filter, specs in new_filter_tests.items():
for section_path, expected_assertions in specs:
run_one_test(bin_path, test_filter, section_path, expected_assertions, new_filters=True)
|