Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

Separated getting test cases to run from running them

Commit
c1dbc9c22bda91b586919333fd0b946450ad3067
Author
Phil Nash ·
Parents
d0a5461addb2

Files · Download .tar.gz

Changes

 include/catch_runner.hpp                           | 12 ++--
 include/internal/catch_interfaces_runner.h         | 15 ++++-
 include/internal/catch_interfaces_testcase.h       |  3 +
 include/internal/catch_runner_impl.hpp             | 65 +++++++++-------------
 include/internal/catch_test_case_registry_impl.hpp | 22 ++++++--
 include/internal/catch_totals.hpp                  |  8 ++-
 projects/SelfTest/TestMain.cpp                     | 18 +++---
 projects/SelfTest/catch_self_test.cpp              | 11 ++--
 projects/SelfTest/catch_self_test.hpp              | 12 +---
 9 files changed, 89 insertions(+), 77 deletions(-)

diff --git a/include/catch_runner.hpp b/include/catch_runner.hpp
index ca63aa0d..eed0fca7 100644
--- a/include/catch_runner.hpp
+++ b/include/catch_runner.hpp
@@ -83,22 +83,22 @@ namespace Catch {
         {
             Runner runner( configWrapper, reporter );
 
+            Totals totals;
             // Run test specs specified on the command line - or default to all
             if( config.testSpecs.empty() ) {
-                runner.runAll();
+                totals = runner.runAllNonHidden();
             }
             else {
-                // !TBD We should get all the testcases upfront, report any missing,
-                // then just run them
                 std::vector<std::string>::const_iterator it = config.testSpecs.begin();
                 std::vector<std::string>::const_iterator itEnd = config.testSpecs.end();
                 for(; it != itEnd; ++it ) {
-                    if( runner.runMatching( *it ) == 0 ) {
+                    Totals groupTotals = runner.runMatching( *it );
+                    if( groupTotals.testCases.total() == 0 )
                         std::cerr << "\n[No test cases matched with: " << *it << "]" << std::endl;
-                    }
+                    totals += groupTotals;
                 }
             }
-            result = static_cast<int>( runner.getTotals().assertions.failed );
+            result = static_cast<int>( totals.assertions.failed );
         }
         Catch::cleanUp();
         return result;
diff --git a/include/internal/catch_interfaces_runner.h b/include/internal/catch_interfaces_runner.h
index b932e65f..f3825626 100644
--- a/include/internal/catch_interfaces_runner.h
+++ b/include/internal/catch_interfaces_runner.h
@@ -17,9 +17,18 @@ namespace Catch {
     
     struct IRunner {
         virtual ~IRunner();
-        virtual void runAll( bool runHiddenTests = false ) = 0;        
-        virtual std::size_t runMatching( const std::string& rawTestSpec ) = 0;        
-        virtual Totals getTotals() const = 0;
+
+        /// Runs all tests, even if hidden
+        virtual Totals runAll() = 0;
+
+        /// Runs all tests unless 'hidden' by ./ prefix
+        virtual Totals runAllNonHidden() = 0;
+
+        /// Runs all test that match the spec string
+        virtual Totals runMatching( const std::string& rawTestSpec ) = 0;
+
+        /// Runs all the tests passed in
+        virtual Totals runTests( const std::string& groupName, const std::vector<TestCaseInfo>& testCases ) = 0;
     };
 }
 
diff --git a/include/internal/catch_interfaces_testcase.h b/include/internal/catch_interfaces_testcase.h
index 0781e646..992b71d3 100644
--- a/include/internal/catch_interfaces_testcase.h
+++ b/include/internal/catch_interfaces_testcase.h
@@ -22,7 +22,10 @@ namespace Catch {
     struct ITestCaseRegistry {
         virtual ~ITestCaseRegistry();
         virtual const std::vector<TestCaseInfo>& getAllTests() const = 0;
+        virtual const std::vector<TestCaseInfo>& getAllNonHiddenTests() const = 0;        
+
         virtual std::vector<TestCaseInfo> getMatchingTestCases( const std::string& rawTestSpec ) const = 0;
+        virtual void getMatchingTestCases( const std::string& rawTestSpec, std::vector<TestCaseInfo>& matchingTestsOut ) const = 0;
     };
 }
 
diff --git a/include/internal/catch_runner_impl.hpp b/include/internal/catch_runner_impl.hpp
index da60a90c..7a22b3b9 100644
--- a/include/internal/catch_runner_impl.hpp
+++ b/include/internal/catch_runner_impl.hpp
@@ -79,45 +79,37 @@ namespace Catch {
             m_context.setConfig( m_prevConfig );
         }
         
-        virtual void runAll( bool runHiddenTests = false ) {
-            m_reporter->StartGroup( "" );
-            const std::vector<TestCaseInfo>& allTests = getRegistryHub().getTestCaseRegistry().getAllTests();
-            for( std::size_t i=0; i < allTests.size(); ++i ) {
-                if( runHiddenTests || !allTests[i].isHidden() ) {
-                    if( aborting() ) {
-                        m_reporter->Aborted();
-                        break;
-                    }
-                    runTest( allTests[i] );
-                }
-            }
-            m_reporter->EndGroup( "", getTotals() );
+        virtual Totals runAll() {
+            return runTests( "", getRegistryHub().getTestCaseRegistry().getAllTests() );
         }
-        
-        virtual std::size_t runMatching( const std::string& rawTestSpec ) {
 
-            Totals prevTotals = getTotals();
-            m_reporter->StartGroup( rawTestSpec );
+        virtual Totals runAllNonHidden() {
+            return runTests( "", getRegistryHub().getTestCaseRegistry().getAllNonHiddenTests() );
+        }
 
-            TestSpec testSpec( rawTestSpec );
-            
-            const std::vector<TestCaseInfo>& allTests = getRegistryHub().getTestCaseRegistry().getAllTests();
-            std::size_t testsRun = 0;
-            for( std::size_t i=0; i < allTests.size(); ++i ) {
-                if( testSpec.matches( allTests[i].getName() ) ) {
-                    if( aborting() ) {
-                        m_reporter->Aborted();
-                        break;
-                    }
-                    runTest( allTests[i] );
-                    testsRun++;
+        virtual Totals runMatching( const std::string& rawTestSpec ) {
+
+            const std::vector<TestCaseInfo>& matchingTests = getRegistryHub().getTestCaseRegistry().getMatchingTestCases( rawTestSpec );
+            return runTests( rawTestSpec, matchingTests );
+        }
+
+        virtual Totals runTests( const std::string& groupName, const std::vector<TestCaseInfo>& testCases ) {
+
+            Totals totals;
+            m_reporter->StartGroup( groupName );
+
+            for( std::size_t i=0; i < testCases.size(); ++i ) {
+                if( aborting() ) {
+                    m_reporter->Aborted();
+                    break;
                 }
+                totals += runTest( testCases[i] );
             }
-            m_reporter->EndGroup( rawTestSpec, getTotals() - prevTotals );
-            return testsRun;
+            m_reporter->EndGroup( groupName, totals );
+            return totals;
         }
-        
-        void runTest( const TestCaseInfo& testInfo ) {
+
+        Totals runTest( const TestCaseInfo& testInfo ) {
             Totals prevTotals = m_totals;
 
             std::string redirectedCout;
@@ -129,10 +121,8 @@ namespace Catch {
 
             do {
                 do {
-//                    m_reporter->StartGroup( "test case run" );
                     m_currentResult.setLineInfo( m_runningTest->getTestCaseInfo().getLineInfo() );
                     runCurrentTest( redirectedCout, redirectedCerr );
-//                    m_reporter->EndGroup( "test case run", m_totals.delta( prevTotals ) );
                 }
                 while( m_runningTest->hasUntestedSections() && !aborting() );
             }
@@ -144,12 +134,9 @@ namespace Catch {
             Totals deltaTotals = m_totals.delta( prevTotals );
             m_totals.testCases += deltaTotals.testCases;            
             m_reporter->EndTestCase( testInfo, deltaTotals, redirectedCout, redirectedCerr );
+            return deltaTotals;
         }
 
-        virtual Totals getTotals() const {
-            return m_totals;
-        }
-        
         const Config& config() const {
             return m_config;
         }
diff --git a/include/internal/catch_test_case_registry_impl.hpp b/include/internal/catch_test_case_registry_impl.hpp
index 10202f64..a17e6033 100644
--- a/include/internal/catch_test_case_registry_impl.hpp
+++ b/include/internal/catch_test_case_registry_impl.hpp
@@ -7,6 +7,7 @@
  */
 #include "catch_test_registry.hpp"
 #include "catch_test_case_info.h"
+#include "catch_test_spec.h"
 #include "catch_context.h"
 
 #include <vector>
@@ -31,6 +32,8 @@ namespace Catch {
             if( m_functions.find( testInfo ) == m_functions.end() ) {
                 m_functions.insert( testInfo );
                 m_functionsInOrder.push_back( testInfo );
+                if( !testInfo.isHidden() )
+                    m_nonHiddenFunctions.push_back( testInfo );
             }
             else {
                 const TestCaseInfo& prev = *m_functions.find( testInfo );
@@ -45,24 +48,35 @@ namespace Catch {
             return m_functionsInOrder;
         }
 
+        virtual const std::vector<TestCaseInfo>& getAllNonHiddenTests() const {
+            return m_nonHiddenFunctions;
+        }
+
         virtual std::vector<TestCaseInfo> getMatchingTestCases( const std::string& rawTestSpec ) const {
             TestSpec testSpec( rawTestSpec );
             
-            std::vector<TestCaseInfo> testList;
+            std::vector<TestCaseInfo> matchingTests;
+            getMatchingTestCases( rawTestSpec, matchingTests );
+            return matchingTests;
+        }
+
+        virtual void getMatchingTestCases( const std::string& rawTestSpec, std::vector<TestCaseInfo>& matchingTestsOut ) const {
+            TestSpec testSpec( rawTestSpec );
+
             std::vector<TestCaseInfo>::const_iterator it = m_functionsInOrder.begin();
             std::vector<TestCaseInfo>::const_iterator itEnd = m_functionsInOrder.end();
             for(; it != itEnd; ++it ) {
                 if( testSpec.matches( it->getName() ) ) {
-                    testList.push_back( *it );
+                    matchingTestsOut.push_back( *it );
                 }
             }
-            return testList;
         }
-        
+
     private:
         
         std::set<TestCaseInfo> m_functions;
         std::vector<TestCaseInfo> m_functionsInOrder;
+        std::vector<TestCaseInfo> m_nonHiddenFunctions;        
         size_t m_unnamedCount;
     };
 
diff --git a/include/internal/catch_totals.hpp b/include/internal/catch_totals.hpp
index 557288dd..8940d0fc 100644
--- a/include/internal/catch_totals.hpp
+++ b/include/internal/catch_totals.hpp
@@ -50,7 +50,13 @@ namespace Catch {
                 ++diff.testCases.passed;
             return diff;
         }
-        
+
+        Totals& operator += ( const Totals& other ) {
+            assertions += other.assertions;
+            testCases += other.testCases;
+            return *this;
+        }
+
         Counts assertions;
         Counts testCases;
     };
diff --git a/projects/SelfTest/TestMain.cpp b/projects/SelfTest/TestMain.cpp
index 324fc28c..4dd59c8f 100644
--- a/projects/SelfTest/TestMain.cpp
+++ b/projects/SelfTest/TestMain.cpp
@@ -34,16 +34,16 @@ TEST_CASE( "selftest/main", "Runs all Catch self tests and checks their results"
         
         SECTION(    "selftest/test counts/succeeding tests", 
                     "Number of 'succeeding' tests is fixed" ) {
-            runner.runMatching( "./succeeding/*" );
-            CHECK( runner.getTotals().assertions.passed == 285 );
-            CHECK( runner.getTotals().assertions.failed == 0 );
+            Totals totals = runner.runMatching( "./succeeding/*" );
+            CHECK( totals.assertions.passed == 285 );
+            CHECK( totals.assertions.failed == 0 );
         }
 
         SECTION(    "selftest/test counts/failing tests", 
                     "Number of 'failing' tests is fixed" ) {
-            runner.runMatching( "./failing/*" );        
-            CHECK( runner.getTotals().assertions.passed == 0 );
-            CHECK( runner.getTotals().assertions.failed == 72 );
+            Totals totals = runner.runMatching( "./failing/*" );        
+            CHECK( totals.assertions.passed == 0 );
+            CHECK( totals.assertions.failed == 72 );
         }
     }
 }
@@ -51,9 +51,9 @@ TEST_CASE( "selftest/main", "Runs all Catch self tests and checks their results"
 TEST_CASE( "meta/Misc/Sections", "looped tests" ) {
     Catch::EmbeddedRunner runner;
     
-    runner.runMatching( "./mixed/Misc/Sections/nested2" );
-    CHECK( runner.getTotals().assertions.passed == 2 );
-    CHECK( runner.getTotals().assertions.failed == 1 );
+    Catch::Totals totals = runner.runMatching( "./mixed/Misc/Sections/nested2" );
+    CHECK( totals.assertions.passed == 2 );
+    CHECK( totals.assertions.failed == 1 );
 }
 
 #pragma clang diagnostic ignored "-Wweak-vtables"
diff --git a/projects/SelfTest/catch_self_test.cpp b/projects/SelfTest/catch_self_test.cpp
index d0065a78..a5d9cb34 100644
--- a/projects/SelfTest/catch_self_test.cpp
+++ b/projects/SelfTest/catch_self_test.cpp
@@ -13,21 +13,20 @@
 
 namespace Catch{
     
-    std::size_t EmbeddedRunner::runMatching( const std::string& rawTestSpec, const std::string& ) {
+    Totals EmbeddedRunner::runMatching( const std::string& rawTestSpec, const std::string& ) {
         std::ostringstream oss;
         Config config;
         config.setStreamBuf( oss.rdbuf() );
         
-        std::size_t result;
-        
+        Totals totals;
+
         // Scoped because Runner doesn't report EndTesting until its destructor
         {
             Runner runner( config, m_reporter.get() );
-            result = runner.runMatching( rawTestSpec );
-            m_totals = runner.getTotals();
+            totals = runner.runMatching( rawTestSpec );
         }
         m_output = oss.str();
-        return result;
+        return totals;
     }
     
     void MockReporter::Result( const ResultInfo& resultInfo ) {
diff --git a/projects/SelfTest/catch_self_test.hpp b/projects/SelfTest/catch_self_test.hpp
index 1b79a44f..c223e931 100644
--- a/projects/SelfTest/catch_self_test.hpp
+++ b/projects/SelfTest/catch_self_test.hpp
@@ -114,17 +114,13 @@ namespace Catch {
     public:
         EmbeddedRunner() : m_reporter( new MockReporter() ) {}
         
-        std::size_t runMatching(    const std::string& rawTestSpec,
-                                    const std::string& reporter = "basic" );
+        Totals runMatching( const std::string& rawTestSpec,
+                            const std::string& reporter = "basic" );
         
         std::string getOutput() {
             return m_output;
         }
 
-        const Totals& getTotals() const {
-            return m_totals;
-        }
-
         void addRecorder( const std::string& recorder ) {
             m_reporter->addRecorder( recorder );
         }
@@ -134,7 +130,6 @@ namespace Catch {
         }
 
     private:
-        Totals m_totals;
         std::string m_output;
         Ptr<MockReporter> m_reporter;
     };
@@ -157,8 +152,7 @@ namespace Catch {
         
         void operator()( const TestCaseInfo& testCase ) {
             EmbeddedRunner runner;
-            runner.runMatching( testCase.getName() );
-            Totals totals = runner.getTotals();
+            Totals totals = runner.runMatching( testCase.getName() );
             switch( m_expectedResult ) {
                 case Expected::ToSucceed:
                     if( totals.assertions.failed > 0 ) {