Merge pull request #15 from Progmasoft/feature/catch3-runner

Replace Catch2's runner with the Catch3 runner

1 files changed, 25 insertions, 41 deletions

diff --git a/tests/TestScripts/testBazelReporter.py b/tests/TestScripts/testBazelReporter.py
index 573eafd2..62600a39 100644
--- a/tests/TestScripts/testBazelReporter.py
+++ b/tests/TestScripts/testBazelReporter.py
@@ -14,12 +14,13 @@ import xml.etree.ElementTree as ET
 import subprocess
 
 """
-Test that Catch2 recognizes `XML_OUTPUT_FILE` env variable and creates
-a junit reporter that writes to the provided path.
+Test that the runner recognizes the `XML_OUTPUT_FILE` environment variable
+of Bazel and writes a JUnit report to the path it names.
 
-Requires 2 arguments, path to Catch2 binary configured with
-`CATCH_CONFIG_BAZEL_SUPPORT`, and the output directory for the output file.
+Requires 2 arguments, path to the test binary and the output directory for
+the report.
 """
+
 if len(sys.argv) != 3:
     print("Wrong number of arguments: {}".format(len(sys.argv)))
     print("Usage: {} test-bin-path output-dir".format(sys.argv[0]))
@@ -39,31 +40,21 @@ print('xml out path:', xml_out_path)
 
 env = os.environ.copy()
 env["XML_OUTPUT_FILE"] = xml_out_path
-test_passing = True
 
-try:
-    ret = subprocess.run(
-        bin_path,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        check=True,
-        universal_newlines=True,
-        env=env
-    )
-    stdout = ret.stdout
-except subprocess.SubprocessError as ex:
-    if ex.returncode == 42:
-        # The test cases are allowed to fail.
-        test_passing = False
-        stdout = ex.stdout
-    else:
-        print('Could not run "{}"'.format(bin_path))
-        print("Return code: {}".format(ex.returncode))
-        print("stdout: {}".format(ex.stdout))
-        print("stderr: {}".format(ex.stderr))
-        raise
-
-# Check for valid XML output
+ret = subprocess.run(
+    [bin_path, '--retries=off'],
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE,
+    universal_newlines=True,
+    env=env
+)
+# One of the tests fails on purpose.
+if ret.returncode != 42:
+    print("Unexpected return code {}".format(ret.returncode))
+    print("stdout: {}".format(ret.stdout))
+    print("stderr: {}".format(ret.stderr))
+    exit(1)
+
 try:
     tree = ET.parse(xml_out_path)
 except ET.ParseError as ex:
@@ -73,24 +64,17 @@ except FileNotFoundError as ex:
     print("Could not find '{}'".format(xml_out_path))
     raise
 
-bin_name = os.path.basename(bin_path)
-# Check for matching testsuite
-if not tree.find('.//testsuite[@name="{}"]'.format(bin_name)):
+bin_name = os.path.splitext(os.path.basename(bin_path))[0]
+if tree.find('.//testsuite[@name="{}"]'.format(bin_name)) is None:
     print("Could not find '{}' testsuite".format(bin_name))
     exit(2)
 
-# Check that we haven't disabled the default reporter
-summary_test_cases = re.findall(r'test cases: \d* \| \d* passed \| \d* failed', stdout)
-if len(summary_test_cases) == 0:
-    print("Could not find test summary in {}".format(stdout))
-    exit(2)
-
-total, passed, failed = [int(s) for s in summary_test_cases[0].split() if s.isdigit()]
-
-if failed == 0 and not test_passing:
-    print("Expected at least 1 test failure!")
+summary = re.search(r'(\d+) test\(s\): (\d+) passed, (\d+) failed', ret.stdout)
+if summary is None:
+    print("Could not find test summary in {}".format(ret.stdout))
     exit(2)
 
+total, passed, failed = [int(value) for value in summary.groups()]
 if len(tree.findall('.//testcase')) != total:
     print("Unexpected number of test cases!")
     exit(2)