Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

Add codecov.io coverage collection from AppVeyor

Also had to add new project to redirect CTest output, add
separate batch scripts for AppVeyor because it doesn't handle
multi-line batch scripts in yaml properly, and other helper
scripts.
Commit
45a465713e522a0ae818b78c8b3e9f149fa50da0
Author
Martin Hořeňovský ·
Parents
dfa817ae7348

Files · Download .tar.gz

Changes

 CMakeLists.txt                            |   4 +-
 appveyor.yml                              |  15 +++--
 codecov.yml                               |   4 --
 misc/CMakeLists.txt                       |  11 ++++
 misc/appveyorBuildConfigurationScript.bat |  15 +++++
 misc/appveyorMergeCoverageScript.py       |   9 +++
 misc/appveyorTestRunScript.bat            |  11 ++++
 misc/coverage-helper.cpp                  | 100 ++++++++++++++++++++++++++++++
 misc/installOpenCppCoverage.ps1           |  19 ++++++
 9 files changed, 176 insertions(+), 12 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9d667e51..2ff2f54e 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -325,7 +325,7 @@ if (NOT NO_SELFTEST)
 
 
     # configure unit tests via CTest
-    enable_testing()
+    include(CTest)
     add_test(NAME RunTests COMMAND $<TARGET_FILE:SelfTest>)
 
     add_test(NAME ListTests COMMAND $<TARGET_FILE:SelfTest> --list-tests --verbosity high)
@@ -341,6 +341,7 @@ if (NOT NO_SELFTEST)
     set_tests_properties(ListTestNamesOnly PROPERTIES PASS_REGULAR_EXPRESSION "Regex string matcher")
 
 
+
     # AppVeyor has a Python 2.7 in path, but doesn't have .py files as autorunnable
     add_test(NAME ApprovalTests COMMAND python ${CMAKE_CURRENT_SOURCE_DIR}/scripts/approvalTests.py $<TARGET_FILE:SelfTest>)
     set_tests_properties(ApprovalTests PROPERTIES FAIL_REGULAR_EXPRESSION "Results differed")
@@ -360,7 +361,6 @@ if(BUILD_EXAMPLES)
     add_subdirectory(examples)
 endif()
 
-
 install(DIRECTORY "single_include/" DESTINATION "include/catch")
 
 ## Provide some pkg-config integration
diff --git a/appveyor.yml b/appveyor.yml
index 08a609dd..95846d38 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -30,8 +30,9 @@ init:
   # Set build version to git commit-hash
   - ps: Update-AppveyorBuild -Version "$($env:APPVEYOR_REPO_BRANCH) - $($env:APPVEYOR_REPO_COMMIT)"
 
-# fetch repository as zip archive
-shallow_clone: true
+install:
+  - ps: if (($env:CONFIGURATION) -eq "Debug" ) { python -m pip install codecov }
+  - ps: if (($env:CONFIGURATION) -eq "Debug" ) { .\misc\installOpenCppCoverage.ps1 }
 
 # Win32 and x64 are CMake-compatible solution platform names.
 # This allows us to pass %PLATFORM% to CMake -A.
@@ -46,9 +47,12 @@ configuration:
 
 #Cmake will autodetect the compiler, but we set the arch
 before_build:
-  - python scripts/generateSingleHeader.py
   - set CXXFLAGS=%additional_flags%
-  - cmake -H. -BBuild -A%PLATFORM% -DUSE_WMAIN=%wmain% -DBUILD_EXAMPLES=ON
+  # Indirection because appveyor doesn't handle multiline batch scripts properly
+  # https://stackoverflow.com/questions/37627248/how-to-split-a-command-over-multiple-lines-in-appveyor-yml/37647169#37647169
+  # https://help.appveyor.com/discussions/questions/3888-multi-line-cmd-or-powershell-warning-ignore
+  - cmd: .\misc\appveyorBuildConfigurationScript.bat
+
 
 # build with MSBuild
 build:
@@ -57,6 +61,5 @@ build:
   verbosity: normal                     # MSBuild verbosity level {quiet|minimal|normal|detailed}
 
 test_script:
-  - cd Build
   - set CTEST_OUTPUT_ON_FAILURE=1
-  - ctest -j 2 -C %CONFIGURATION%
+  - cmd: .\misc\appveyorTestRunScript.bat
diff --git a/codecov.yml b/codecov.yml
index 18ff598e..54226c2a 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -1,9 +1,5 @@
 codecov:
   branch: master
-  ci:
-    # Don't wait for AppVeyor build to finalize coverage information
-    # We don't have a way to generate coverage from it anyway
-    - !appveyor
 
 coverage:
   ignore:
diff --git a/misc/CMakeLists.txt b/misc/CMakeLists.txt
new file mode 100644
index 00000000..bf80846c
--- /dev/null
+++ b/misc/CMakeLists.txt
@@ -0,0 +1,11 @@
+cmake_minimum_required(VERSION 3.0)
+
+project(CatchCoverageHelper)
+
+add_executable(CoverageHelper coverage-helper.cpp)
+set_property(TARGET CoverageHelper PROPERTY CXX_STANDARD 11)
+set_property(TARGET CoverageHelper PROPERTY CXX_STANDARD_REQUIRED ON)
+set_property(TARGET CoverageHelper PROPERTY CXX_EXTENSIONS OFF)
+if (MSVC)
+    target_compile_options( CoverageHelper PRIVATE /W4 /w44265 /WX /w44061 /w44062 )
+endif()
diff --git a/misc/appveyorBuildConfigurationScript.bat b/misc/appveyorBuildConfigurationScript.bat
new file mode 100644
index 00000000..f21e8c18
--- /dev/null
+++ b/misc/appveyorBuildConfigurationScript.bat
@@ -0,0 +1,15 @@
+@REM  # In debug build, we want to
+@REM  # 1) Prebuild memcheck redirecter
+@REM  # 2) Regenerate single header include for examples
+@REM  # 3) Enable building examples
+if "%CONFIGURATION%"=="Debug" (
+  echo "buildConfiguration.bat thinks this is a Debug build"
+  python scripts\generateSingleHeader.py
+  cmake -Hmisc -Bbuild-misc -A%PLATFORM%
+  cmake --build build-misc
+  cmake -H. -BBuild -A%PLATFORM% -DUSE_WMAIN=%wmain% -DBUILD_EXAMPLES=ON -DMEMORYCHECK_COMMAND=build-misc\Debug\CoverageHelper.exe -DMEMORYCHECK_COMMAND_OPTIONS=--sep-- -DMEMORYCHECK_TYPE=Valgrind
+)
+if "%CONFIGURATION%"=="Release" (
+  echo "buildConfiguration.bat thinks this is a Release build"
+  cmake -H. -BBuild -A%PLATFORM% -DUSE_WMAIN=%wmain%
+)
diff --git a/misc/appveyorMergeCoverageScript.py b/misc/appveyorMergeCoverageScript.py
new file mode 100644
index 00000000..a74e6e0e
--- /dev/null
+++ b/misc/appveyorMergeCoverageScript.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python2
+
+import glob
+import subprocess
+
+if __name__ == '__main__':
+    cov_files = list(glob.glob('cov-report*.bin'))
+    base_cmd = ['OpenCppCoverage', '--quiet', '--export_type=cobertura:cobertura.xml'] + ['--input_coverage={}'.format(f) for f in cov_files]
+    subprocess.call(base_cmd)
diff --git a/misc/appveyorTestRunScript.bat b/misc/appveyorTestRunScript.bat
new file mode 100644
index 00000000..0c8d9928
--- /dev/null
+++ b/misc/appveyorTestRunScript.bat
@@ -0,0 +1,11 @@
+cd Build
+if "%CONFIGURATION%"=="Debug" (
+  echo "appveyorTestScript.bat thinks this is a Debug build"
+  ctest -j 2 -C %CONFIGURATION% -D ExperimentalMemCheck
+  python ..\misc\appveyorMergeCoverageScript.py
+  codecov --root .. --no-color --disable gcov -f cobertura.xml -t %CODECOV_TOKEN%
+)
+if "%CONFIGURATION%"=="Release" (
+  echo "appveyorTestScript.bat thinks this is a Release build"
+  ctest -j 2 -C %CONFIGURATION%
+)
diff --git a/misc/coverage-helper.cpp b/misc/coverage-helper.cpp
new file mode 100644
index 00000000..9783d693
--- /dev/null
+++ b/misc/coverage-helper.cpp
@@ -0,0 +1,100 @@
+#include <algorithm>
+#include <array>
+#include <cassert>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <numeric>
+#include <regex>
+#include <string>
+#include <vector>
+
+
+void create_empty_file(std::string const& path) {
+    std::ofstream ofs(path);
+    ofs << '\n';
+}
+
+const std::string separator = "--sep--";
+const std::string logfile_prefix = "--log-file=";
+
+bool starts_with(std::string const& str, std::string const& pref) {
+    return str.find(pref) == 0;
+}
+
+int parse_log_file_arg(std::string const& arg) {
+    assert(starts_with(arg, logfile_prefix) && "Attempting to parse incorrect arg!");
+    auto fname = arg.substr(logfile_prefix.size());
+    create_empty_file(fname);
+    std::regex regex("MemoryChecker\\.(\\d+)\\.log", std::regex::icase);
+    std::smatch match;
+    if (std::regex_search(fname, match, regex)) {
+        return std::stoi(match[1]);
+    } else {
+        throw std::domain_error("Couldn't find desired expression in string: " + fname);
+    }
+}
+
+std::string catch_path(std::string path) {
+    auto start = path.find("catch");
+    // try capitalized instead
+    if (start == std::string::npos) {
+        start = path.find("Catch");
+    }
+    if (start == std::string::npos) {
+        throw std::domain_error("Couldn't find Catch's base path");
+    }
+    auto end = path.find_first_of("\\/", start);
+    return path.substr(0, end);
+}
+
+std::string windowsify_path(std::string path) {
+    for (auto& c : path) {
+        if (c == '/') {
+            c = '\\';
+        }
+    }
+    return path;
+}
+
+void exec_cmd(std::string const& cmd, int log_num, std::string const& path) {
+    std::array<char, 128> buffer;
+#if defined(_WIN32)
+    auto real_cmd = "OpenCppCoverage --export_type binary:cov-report" + std::to_string(log_num)
+        + ".bin --quiet " + "--sources " + path + " --cover_children -- " + cmd;
+    std::cout << "=== Marker ===: Cmd: " << real_cmd << '\n';
+    std::shared_ptr<FILE> pipe(_popen(real_cmd.c_str(), "r"), _pclose);
+#else // Just for testing, in the real world we will always work under WIN32
+    (void)log_num; (void)path;
+    std::shared_ptr<FILE> pipe(popen(cmd.c_str(), "r"), pclose);
+#endif
+
+    if (!pipe) {
+        throw std::runtime_error("popen() failed!");
+    }
+    while (!feof(pipe.get())) {
+        if (fgets(buffer.data(), 128, pipe.get()) != nullptr) {
+            std::cout << buffer.data();
+        }
+    }
+}
+
+// argv should be:
+// [0]: our path
+// [1]: "--log-file=<path>"
+// [2]: "--sep--"
+// [3]+: the actual command
+
+int main(int argc, char** argv) {
+    std::vector<std::string> args(argv, argv + argc);
+    auto sep = std::find(begin(args), end(args), separator);
+    assert(sep - begin(args) == 2 && "Structure differs from expected!");
+
+    auto num = parse_log_file_arg(args[1]);
+
+    auto cmdline = std::accumulate(++sep, end(args), std::string{}, [] (const std::string& lhs, const std::string& rhs) {
+        return lhs + ' ' + rhs;
+    });
+
+    exec_cmd(cmdline, num, windowsify_path(catch_path(args[0])));
+}
diff --git a/misc/installOpenCppCoverage.ps1 b/misc/installOpenCppCoverage.ps1
new file mode 100644
index 00000000..9dbffca7
--- /dev/null
+++ b/misc/installOpenCppCoverage.ps1
@@ -0,0 +1,19 @@
+# Downloads are done from the oficial github release page links
+$downloadUrl = "https://github.com/OpenCppCoverage/OpenCppCoverage/releases/download/release-0.9.6.1/OpenCppCoverageSetup-x64-0.9.6.1.exe"
+$installerPath = [System.IO.Path]::Combine($Env:USERPROFILE, "Downloads", "OpenCppCoverageSetup.exe")
+
+if(-Not (Test-Path $installerPath)) {
+    Write-Host -ForegroundColor White ("Downloading OpenCppCoverage from: " + $downloadUrl)
+    Start-FileDownload $downloadUrl -FileName $installerPath
+}
+
+Write-Host -ForegroundColor White "About to install OpenCppCoverage..."
+
+$installProcess = (Start-Process $installerPath -ArgumentList '/VERYSILENT' -PassThru -Wait)
+if($installProcess.ExitCode -ne 0) {
+    throw [System.String]::Format("Failed to install OpenCppCoverage, ExitCode: {0}.", $installProcess.ExitCode)
+}
+
+# Assume standard, boring, installation path of ".../Program Files/OpenCppCoverage"
+$installPath = [System.IO.Path]::Combine(${Env:ProgramFiles}, "OpenCppCoverage")
+$env:Path="$env:Path;$installPath"