catch_discover_tests uses tempfile to retrieve JSON from the binary

This allows it to deal with badly behaved code, where 3rd party
dependencies write into stdout during global construction.

Closes #3162
Closes #3166

2 files changed, 44 insertions, 12 deletions

diff --git a/tests/TestScripts/DiscoverTests/VerifyRegistration.py b/tests/TestScripts/DiscoverTests/VerifyRegistration.py
index 7d9862cf..c281a403 100644
--- a/tests/TestScripts/DiscoverTests/VerifyRegistration.py
+++ b/tests/TestScripts/DiscoverTests/VerifyRegistration.py
@@ -12,6 +12,7 @@ import subprocess
 import sys
 import re
 import json
+import tempfile
 from collections import namedtuple
 from typing import List
 
@@ -72,14 +73,19 @@ def get_test_names(build_path: str) -> List[TestInfo]:
     config_path = "Debug" if os.name == 'nt' else ""
     full_path = os.path.join(build_path, config_path, 'tests')
 
-
-    cmd = [full_path, '--reporter', 'json', '--list-tests']
-    result = subprocess.run(cmd,
-                            capture_output = True,
-                            check = True,
-                            text = True)
-
-    test_listing = json.loads(result.stdout)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        fname = f'{tmpdir}/listing-output.json'
+        cmd = [full_path,
+          '--list-tests',
+          '--reporter', 'json',
+          '--out', fname
+        ]
+        result = subprocess.run(cmd,
+                                capture_output = False,
+                                check = True,
+                                text = True)
+        with open(fname, mode='r', encoding='utf-8') as file:
+            test_listing = json.load(file)
 
     assert test_listing['version'] == 1
 
@@ -96,10 +102,18 @@ def get_ctest_listing(build_path):
     os.chdir(build_path)
 
     cmd = ['ctest', '-C', 'debug', '--show-only=json-v1']
-    result = subprocess.run(cmd,
-                            capture_output = True,
-                            check = True,
-                            text = True)
+    try:
+        result = subprocess.run(cmd,
+                                capture_output = True,
+                                check = True,
+                                text = True)
+    except subprocess.CalledProcessError as err:
+        print('Error when getting output from CTest')
+        print(f'cmd: {err.cmd}')
+        print(f'stderr: {err.stderr}')
+        print(f'stdout: {err.stdout}')
+        exit(4)
+
     os.chdir(old_path)
     return result.stdout
 
diff --git a/tests/TestScripts/DiscoverTests/register-tests.cpp b/tests/TestScripts/DiscoverTests/register-tests.cpp
index be533ab6..8b071196 100644
--- a/tests/TestScripts/DiscoverTests/register-tests.cpp
+++ b/tests/TestScripts/DiscoverTests/register-tests.cpp
@@ -8,6 +8,24 @@
 
 #include <catch2/catch_test_macros.hpp>
 
+#include <cstdio>
+#include <iostream>
+
+namespace {
+
+struct PrintsWhenConstructed {
+    PrintsWhenConstructed() {
+        std::cout << "Hello\n";
+        std::cerr << "Holla\n";
+        std::fprintf(stdout, "Hullo\n");
+        std::fprintf(stderr, "Hillo\n");
+    }
+};
+
+static PrintsWhenConstructed instance;
+
+}
+
 TEST_CASE("@Script[C:\\EPM1A]=x;\"SCALA_ZERO:\"", "[script regressions]"){}
 TEST_CASE("Some test") {}
 TEST_CASE( "Let's have a test case with a long name. Longer. No, even longer. "