Progmasoft Git

catch3

Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.

Clonegit clone https://git.progmasoft.com/catch3

Add --min-duration option

A test runner already has a --durations option to print durations.
However, this isn't entirely satisfactory.

When there are many tests, this produces output spam which makes it hard
to find the test failure output.  Nevertheless, it is helpful to be
informed of tests which are unusually slow.

Therefore, introduce a new option --min-duration that causes all
durations above a certain threshold to be printed.  This allows slow
tests to be visible without mentioning every test.
Commit
46fde0c5978d0c138f9309a4cebfc8ff9c44e286
Author
John Bytheway ·
Committer
Martin Hořeňovský
Parents
de103db6968c

Files · Download .tar.gz

Changes

 docs/command-line.md                         | 4 ++++
 include/internal/catch_commandline.cpp       | 3 +++
 include/internal/catch_config.cpp            | 1 +
 include/internal/catch_config.hpp            | 2 ++
 include/internal/catch_interfaces_config.h   | 1 +
 include/reporters/catch_reporter_bases.cpp   | 7 +++++++
 include/reporters/catch_reporter_bases.hpp   | 2 ++
 include/reporters/catch_reporter_compact.cpp | 5 +++--
 include/reporters/catch_reporter_console.cpp | 5 +++--
 projects/CMakeLists.txt                      | 3 +++
 single_include/catch2/catch.hpp              | 1 +
 11 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/docs/command-line.md b/docs/command-line.md
index 1741eb8f..77e969b0 100644
--- a/docs/command-line.md
+++ b/docs/command-line.md
@@ -222,6 +222,10 @@ available warnings
 
 When set to ```yes``` Catch will report the duration of each test case, in milliseconds. Note that it does this regardless of whether a test case passes or fails. Note, also, the certain reporters (e.g. Junit) always report test case durations regardless of this option being set or not.
 
+<pre>-D, --min-duration &lt;value></pre>
+
+When set, Catch will report the duration of each test case that took more than &lt;value> seconds, in milliseconds.
+
 <a id="input-file"></a>
 ## Load test names to run from a file
 <pre>-f, --input-file &lt;filename></pre>
diff --git a/include/internal/catch_commandline.cpp b/include/internal/catch_commandline.cpp
index a4010850..673d3b70 100644
--- a/include/internal/catch_commandline.cpp
+++ b/include/internal/catch_commandline.cpp
@@ -170,6 +170,9 @@ namespace Catch {
             | Opt( [&]( bool flag ) { config.showDurations = flag ? ShowDurations::Always : ShowDurations::Never; }, "yes|no" )
                 ["-d"]["--durations"]
                 ( "show test durations" )
+            | Opt( config.minDuration, "seconds" )
+                ["-D"]["--min-duration"]
+                ( "show test durations for tests taking at least the given number of seconds" )
             | Opt( loadTestNamesFromFile, "filename" )
                 ["-f"]["--input-file"]
                 ( "load test names to run from a file" )
diff --git a/include/internal/catch_config.cpp b/include/internal/catch_config.cpp
index 21cd6296..252bec62 100644
--- a/include/internal/catch_config.cpp
+++ b/include/internal/catch_config.cpp
@@ -64,6 +64,7 @@ namespace Catch {
     bool Config::warnAboutMissingAssertions() const    { return !!(m_data.warnings & WarnAbout::NoAssertions); }
     bool Config::warnAboutNoTests() const              { return !!(m_data.warnings & WarnAbout::NoTests); }
     ShowDurations::OrNot Config::showDurations() const { return m_data.showDurations; }
+    double Config::minDuration() const                 { return m_data.minDuration; }
     RunTests::InWhatOrder Config::runOrder() const     { return m_data.runOrder; }
     unsigned int Config::rngSeed() const               { return m_data.rngSeed; }
     UseColour::YesOrNo Config::useColour() const       { return m_data.useColour; }
diff --git a/include/internal/catch_config.hpp b/include/internal/catch_config.hpp
index 64d2c035..fd7de5b2 100644
--- a/include/internal/catch_config.hpp
+++ b/include/internal/catch_config.hpp
@@ -52,6 +52,7 @@ namespace Catch {
         Verbosity verbosity = Verbosity::Normal;
         WarnAbout::What warnings = WarnAbout::Nothing;
         ShowDurations::OrNot showDurations = ShowDurations::DefaultForReporter;
+        double minDuration = -1;
         RunTests::InWhatOrder runOrder = RunTests::InDeclarationOrder;
         UseColour::YesOrNo useColour = UseColour::Auto;
         WaitForKeypress::When waitForKeypress = WaitForKeypress::Never;
@@ -103,6 +104,7 @@ namespace Catch {
         bool warnAboutMissingAssertions() const override;
         bool warnAboutNoTests() const override;
         ShowDurations::OrNot showDurations() const override;
+        double minDuration() const override;
         RunTests::InWhatOrder runOrder() const override;
         unsigned int rngSeed() const override;
         UseColour::YesOrNo useColour() const override;
diff --git a/include/internal/catch_interfaces_config.h b/include/internal/catch_interfaces_config.h
index 8fb986be..e9c2cf96 100644
--- a/include/internal/catch_interfaces_config.h
+++ b/include/internal/catch_interfaces_config.h
@@ -69,6 +69,7 @@ namespace Catch {
         virtual int abortAfter() const = 0;
         virtual bool showInvisibles() const = 0;
         virtual ShowDurations::OrNot showDurations() const = 0;
+        virtual double minDuration() const = 0;
         virtual TestSpec const& testSpec() const = 0;
         virtual bool hasTestFilters() const = 0;
         virtual std::vector<std::string> const& getTestsOrTags() const = 0;
diff --git a/include/reporters/catch_reporter_bases.cpp b/include/reporters/catch_reporter_bases.cpp
index fcbafef5..271f90ca 100644
--- a/include/reporters/catch_reporter_bases.cpp
+++ b/include/reporters/catch_reporter_bases.cpp
@@ -41,6 +41,13 @@ namespace Catch {
         return std::string(buffer);
     }
 
+    bool shouldShowDuration( IConfig const& config, double duration ) {
+      if( config.showDurations() == ShowDurations::Always )
+        return true;
+      double min = config.minDuration();
+      return min >= 0 && duration >= min;
+    }
+
     std::string serializeFilters( std::vector<std::string> const& container ) {
         ReusableStringStream oss;
         bool first = true;
diff --git a/include/reporters/catch_reporter_bases.hpp b/include/reporters/catch_reporter_bases.hpp
index 66a891ff..f53794f4 100644
--- a/include/reporters/catch_reporter_bases.hpp
+++ b/include/reporters/catch_reporter_bases.hpp
@@ -25,6 +25,8 @@ namespace Catch {
     // Returns double formatted as %.3f (format expected on output)
     std::string getFormattedDuration( double duration );
 
+    bool shouldShowDuration( IConfig const &, double duration );
+
     std::string serializeFilters( std::vector<std::string> const& container );
 
     template<typename DerivedT>
diff --git a/include/reporters/catch_reporter_compact.cpp b/include/reporters/catch_reporter_compact.cpp
index 017f521f..e9c44ea6 100644
--- a/include/reporters/catch_reporter_compact.cpp
+++ b/include/reporters/catch_reporter_compact.cpp
@@ -275,8 +275,9 @@ private:
         }
 
         void CompactReporter::sectionEnded(SectionStats const& _sectionStats) {
-            if (m_config->showDurations() == ShowDurations::Always) {
-                stream << getFormattedDuration(_sectionStats.durationInSeconds) << " s: " << _sectionStats.sectionInfo.name << std::endl;
+            double dur = _sectionStats.durationInSeconds;
+            if ( shouldShowDuration( *m_config, dur ) ) {
+                stream << getFormattedDuration( dur ) << " s: " << _sectionStats.sectionInfo.name << std::endl;
             }
         }
 
diff --git a/include/reporters/catch_reporter_console.cpp b/include/reporters/catch_reporter_console.cpp
index 0fa399d5..4f74ccb3 100644
--- a/include/reporters/catch_reporter_console.cpp
+++ b/include/reporters/catch_reporter_console.cpp
@@ -418,8 +418,9 @@ void ConsoleReporter::sectionEnded(SectionStats const& _sectionStats) {
             stream << "\nNo assertions in test case";
         stream << " '" << _sectionStats.sectionInfo.name << "'\n" << std::endl;
     }
-    if (m_config->showDurations() == ShowDurations::Always) {
-        stream << getFormattedDuration(_sectionStats.durationInSeconds) << " s: " << _sectionStats.sectionInfo.name << std::endl;
+    double dur = _sectionStats.durationInSeconds;
+    if (shouldShowDuration(*m_config, dur)) {
+        stream << getFormattedDuration(dur) << " s: " << _sectionStats.sectionInfo.name << std::endl;
     }
     if (m_headerPrinted) {
         m_headerPrinted = false;
diff --git a/projects/CMakeLists.txt b/projects/CMakeLists.txt
index 08badb80..1e50e9cc 100644
--- a/projects/CMakeLists.txt
+++ b/projects/CMakeLists.txt
@@ -454,6 +454,9 @@ set_tests_properties(TestsInFile::InvalidTestNames-2 PROPERTIES PASS_REGULAR_EXP
 add_test(NAME RandomTestOrdering COMMAND ${PYTHON_EXECUTABLE}
   ${CATCH_DIR}/projects/TestScripts/testRandomOrder.py $<TARGET_FILE:SelfTest>)
 
+add_test(NAME TestTimeThreshold COMMAND ${PYTHON_EXECUTABLE}
+  ${CATCH_DIR}/projects/TestScripts/testTimeThreshold.py $<TARGET_FILE:SelfTest>)
+
 if (CATCH_USE_VALGRIND)
     add_test(NAME ValgrindRunTests COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest>)
     add_test(NAME ValgrindListTests COMMAND valgrind --leak-check=full --error-exitcode=1 $<TARGET_FILE:SelfTest> --list-tests --verbosity high)
diff --git a/single_include/catch2/catch.hpp b/single_include/catch2/catch.hpp
index b066b71d..fcd5c253 100644
--- a/single_include/catch2/catch.hpp
+++ b/single_include/catch2/catch.hpp
@@ -4522,6 +4522,7 @@ namespace Catch {
         virtual int abortAfter() const = 0;
         virtual bool showInvisibles() const = 0;
         virtual ShowDurations::OrNot showDurations() const = 0;
+        virtual double minDuration() const = 0;
         virtual TestSpec const& testSpec() const = 0;
         virtual bool hasTestFilters() const = 0;
         virtual std::vector<std::string> const& getTestsOrTags() const = 0;