catch3
Progmasoft Catch3: an independently maintained C++17 test framework with a parallel runner, property checks, snapshots and structured results.
Clonegit clone https://git.progmasoft.com/catch3
Mark !mayfail tests as skipped in the JUnit reporter
- Commit
eea3e9a5b5b0a405369961eb6601a56512a87ce8- Author
- Martin Hořeňovský ·
- Parents
7727c15290ce
Changes
include/reporters/catch_reporter_junit.cpp | 19 ++++++--- include/reporters/catch_reporter_junit.h | 7 ++-- .../SelfTest/Baselines/compact.sw.approved.txt | 4 ++ .../SelfTest/Baselines/console.std.approved.txt | 44 ++++++++++++++++++++- .../SelfTest/Baselines/console.sw.approved.txt | 44 ++++++++++++++++++++- projects/SelfTest/Baselines/junit.sw.approved.txt | 35 ++++++++++++++++- .../SelfTest/Baselines/sonarqube.sw.approved.txt | 24 ++++++++++++ projects/SelfTest/Baselines/xml.sw.approved.txt | 45 ++++++++++++++++++++-- projects/SelfTest/UsageTests/Condition.tests.cpp | 15 +++++++- 9 files changed, 218 insertions(+), 19 deletions(-) diff --git a/include/reporters/catch_reporter_junit.cpp b/include/reporters/catch_reporter_junit.cpp index a50ead5f..ceb941be 100644 --- a/include/reporters/catch_reporter_junit.cpp +++ b/include/reporters/catch_reporter_junit.cpp @@ -182,12 +182,13 @@ namespace Catch { if ( !m_config->name().empty() ) className = m_config->name() + "." + className; - writeSection( className, "", rootSection ); + writeSection( className, "", rootSection, stats.testInfo.okToFail() ); } - void JunitReporter::writeSection( std::string const& className, - std::string const& rootName, - SectionNode const& sectionNode ) { + void JunitReporter::writeSection( std::string const& className, + std::string const& rootName, + SectionNode const& sectionNode, + bool testOkToFail) { std::string name = trim( sectionNode.stats.sectionInfo.name ); if( !rootName.empty() ) name = rootName + '/' + name; @@ -211,8 +212,14 @@ namespace Catch { // events and write those out appropriately. xml.writeAttribute( "status", "run" ); + if (sectionNode.stats.assertions.failedButOk) { + xml.scopedElement("skipped") + .writeAttribute("message", "TEST_CASE tagged with !mayfail"); + } + writeAssertions( sectionNode ); + if( !sectionNode.stdOut.empty() ) xml.scopedElement( "system-out" ).writeText( trim( sectionNode.stdOut ), XmlFormatting::Newline ); if( !sectionNode.stdErr.empty() ) @@ -220,9 +227,9 @@ namespace Catch { } for( auto const& childNode : sectionNode.childSections ) if( className.empty() ) - writeSection( name, "", *childNode ); + writeSection( name, "", *childNode, testOkToFail ); else - writeSection( className, name, *childNode ); + writeSection( className, name, *childNode, testOkToFail ); } void JunitReporter::writeAssertions( SectionNode const& sectionNode ) { diff --git a/include/reporters/catch_reporter_junit.h b/include/reporters/catch_reporter_junit.h index 5ee3a570..da5b5721 100644 --- a/include/reporters/catch_reporter_junit.h +++ b/include/reporters/catch_reporter_junit.h @@ -41,9 +41,10 @@ namespace Catch { void writeTestCase(TestCaseNode const& testCaseNode); - void writeSection(std::string const& className, - std::string const& rootName, - SectionNode const& sectionNode); + void writeSection( std::string const& className, + std::string const& rootName, + SectionNode const& sectionNode, + bool testOkToFail ); void writeAssertions(SectionNode const& sectionNode); void writeAssertion(AssertionStats const& stats); diff --git a/projects/SelfTest/Baselines/compact.sw.approved.txt b/projects/SelfTest/Baselines/compact.sw.approved.txt index efefa833..0d12ebb9 100644 --- a/projects/SelfTest/Baselines/compact.sw.approved.txt +++ b/projects/SelfTest/Baselines/compact.sw.approved.txt @@ -846,6 +846,10 @@ Matchers.tests.cpp:<line number>: passed: testStringForMatching(), (Contains("st Matchers.tests.cpp:<line number>: failed: testStringForMatching(), (Contains("string") || Contains("different")) && Contains("random") for: "this string contains 'abc' as a substring" ( ( contains: "string" or contains: "different" ) and contains: "random" ) Matchers.tests.cpp:<line number>: passed: testStringForMatching(), !Contains("different") for: "this string contains 'abc' as a substring" not contains: "different" Matchers.tests.cpp:<line number>: failed: testStringForMatching(), !Contains("substring") for: "this string contains 'abc' as a substring" not contains: "substring" +Condition.tests.cpp:<line number>: failed: explicitly +Condition.tests.cpp:<line number>: failed: explicitly +Condition.tests.cpp:<line number>: failed: explicitly +Condition.tests.cpp:<line number>: failed: explicitly Exception.tests.cpp:<line number>: passed: thisThrows(), "expected exception" for: "expected exception" equals: "expected exception" Exception.tests.cpp:<line number>: failed: thisThrows(), "should fail" for: "expected exception" equals: "should fail" Generators.tests.cpp:<line number>: passed: values > -6 for: 3 > -6 diff --git a/projects/SelfTest/Baselines/console.std.approved.txt b/projects/SelfTest/Baselines/console.std.approved.txt index cfcdc72e..9de49d9a 100644 --- a/projects/SelfTest/Baselines/console.std.approved.txt +++ b/projects/SelfTest/Baselines/console.std.approved.txt @@ -694,6 +694,46 @@ Matchers.tests.cpp:<line number>: FAILED: with expansion: "this string contains 'abc' as a substring" not contains: "substring" +------------------------------------------------------------------------------- +Mayfail test case with nested sections + A + 1 +------------------------------------------------------------------------------- +Condition.tests.cpp:<line number> +............................................................................... + +Condition.tests.cpp:<line number>: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + A + 2 +------------------------------------------------------------------------------- +Condition.tests.cpp:<line number> +............................................................................... + +Condition.tests.cpp:<line number>: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + B + 1 +------------------------------------------------------------------------------- +Condition.tests.cpp:<line number> +............................................................................... + +Condition.tests.cpp:<line number>: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + B + 2 +------------------------------------------------------------------------------- +Condition.tests.cpp:<line number> +............................................................................... + +Condition.tests.cpp:<line number>: FAILED: + ------------------------------------------------------------------------------- Mismatching exception messages failing the test ------------------------------------------------------------------------------- @@ -1380,6 +1420,6 @@ due to unexpected exception with message: Why would you throw a std::string? =============================================================================== -test cases: 322 | 248 passed | 70 failed | 4 failed as expected -assertions: 1759 | 1607 passed | 131 failed | 21 failed as expected +test cases: 323 | 248 passed | 70 failed | 5 failed as expected +assertions: 1763 | 1607 passed | 131 failed | 25 failed as expected diff --git a/projects/SelfTest/Baselines/console.sw.approved.txt b/projects/SelfTest/Baselines/console.sw.approved.txt index 3c09bf31..7f663129 100644 --- a/projects/SelfTest/Baselines/console.sw.approved.txt +++ b/projects/SelfTest/Baselines/console.sw.approved.txt @@ -6492,6 +6492,46 @@ Matchers.tests.cpp:<line number>: FAILED: with expansion: "this string contains 'abc' as a substring" not contains: "substring" +------------------------------------------------------------------------------- +Mayfail test case with nested sections + A + 1 +------------------------------------------------------------------------------- +Condition.tests.cpp:<line number> +............................................................................... + +Condition.tests.cpp:<line number>: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + A + 2 +------------------------------------------------------------------------------- +Condition.tests.cpp:<line number> +............................................................................... + +Condition.tests.cpp:<line number>: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + B + 1 +------------------------------------------------------------------------------- +Condition.tests.cpp:<line number> +............................................................................... + +Condition.tests.cpp:<line number>: FAILED: + +------------------------------------------------------------------------------- +Mayfail test case with nested sections + B + 2 +------------------------------------------------------------------------------- +Condition.tests.cpp:<line number> +............................................................................... + +Condition.tests.cpp:<line number>: FAILED: + ------------------------------------------------------------------------------- Mismatching exception messages failing the test ------------------------------------------------------------------------------- @@ -14138,6 +14178,6 @@ Misc.tests.cpp:<line number> Misc.tests.cpp:<line number>: PASSED: =============================================================================== -test cases: 322 | 232 passed | 86 failed | 4 failed as expected -assertions: 1776 | 1607 passed | 148 failed | 21 failed as expected +test cases: 323 | 232 passed | 86 failed | 5 failed as expected +assertions: 1780 | 1607 passed | 148 failed | 25 failed as expected diff --git a/projects/SelfTest/Baselines/junit.sw.approved.txt b/projects/SelfTest/Baselines/junit.sw.approved.txt index 89a0b438..b7e55e04 100644 --- a/projects/SelfTest/Baselines/junit.sw.approved.txt +++ b/projects/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="UTF-8"?> <testsuitesloose text artifact > - <testsuite name="<exe-name>" errors="17" failures="132" tests="1777" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> + <testsuite name="<exe-name>" errors="17" failures="132" tests="1781" hostname="tbd" time="{duration}" timestamp="{iso8601-timestamp}"> <properties> <property name="filters" value="~[!nonportable]~[!benchmark]~[approvals]"/> <property name="random-seed" value="1"/> @@ -48,6 +48,7 @@ Nor would this <testcase classname="<exe-name>.global" name="#1954 - 7 arg template test case sig compiles - 5, 1, 1, 1, 1, 0, 0" time="{duration}" status="run"/> <testcase classname="<exe-name>.global" name="#1954 - 7 arg template test case sig compiles - 5, 3, 1, 1, 1, 0, 0" time="{duration}" status="run"/> <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/outside assertions" time="{duration}" status="run"> + <skipped message="TEST_CASE tagged with !mayfail"/> <error type="TEST_CASE"> FAILED: expected exception @@ -56,6 +57,7 @@ Exception.tests.cpp:<line number> </error> </testcase> <testcase classname="<exe-name>.global" name="#748 - captures with unexpected exceptions/inside REQUIRE_NOTHROW" time="{duration}" status="run"> + <skipped message="TEST_CASE tagged with !mayfail"/> <error message="thisThrows()" type="REQUIRE_NOTHROW"> FAILED: REQUIRE_NOTHROW( thisThrows() ) @@ -68,6 +70,7 @@ Exception.tests.cpp:<line number> <testcase classname="<exe-name>.global" name="#809" time="{duration}" status="run"/> <testcase classname="<exe-name>.global" name="#833" time="{duration}" status="run"/> <testcase classname="<exe-name>.global" name="#835 -- errno should not be touched by Catch" time="{duration}" status="run"> + <skipped message="TEST_CASE tagged with !mayfail"/> <failure message="f() == 0" type="CHECK"> FAILED: CHECK( f() == 0 ) @@ -443,6 +446,7 @@ Matchers.tests.cpp:<line number> <testcase classname="<exe-name>.global" name="Enums in namespaces can quickly have stringification enabled using REGISTER_ENUM" time="{duration}" status="run"/> <testcase classname="<exe-name>.global" name="Epsilon only applies to Approx's value" time="{duration}" status="run"/> <testcase classname="<exe-name>.global" name="Equality checks that should fail" time="{duration}" status="run"> + <skipped message="TEST_CASE tagged with !mayfail"/> <failure message="data.int_seven == 6" type="CHECK"> FAILED: CHECK( data.int_seven == 6 ) @@ -736,6 +740,7 @@ Message.tests.cpp:<line number> </failure> </testcase> <testcase classname="<exe-name>.global" name="Inequality checks that should fail" time="{duration}" status="run"> + <skipped message="TEST_CASE tagged with !mayfail"/> <failure message="data.int_seven != 7" type="CHECK"> FAILED: CHECK( data.int_seven != 7 ) @@ -799,6 +804,34 @@ with expansion: Matchers.tests.cpp:<line number> </failure> </testcase> + <testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/1/A" time="{duration}" status="run"> + <skipped message="TEST_CASE tagged with !mayfail"/> + <failure type="FAIL"> +FAILED: +Condition.tests.cpp:<line number> + </failure> + </testcase> + <testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/2/A" time="{duration}" status="run"> + <skipped message="TEST_CASE tagged with !mayfail"/> + <failure type="FAIL"> +FAILED: +Condition.tests.cpp:<line number> + </failure> + </testcase> + <testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/1/B" time="{duration}" status="run"> + <skipped message="TEST_CASE tagged with !mayfail"/> + <failure type="FAIL"> +FAILED: +Condition.tests.cpp:<line number> + </failure> + </testcase> + <testcase classname="<exe-name>.global" name="Mayfail test case with nested sections/2/B" time="{duration}" status="run"> + <skipped message="TEST_CASE tagged with !mayfail"/> + <failure type="FAIL"> +FAILED: +Condition.tests.cpp:<line number> + </failure> + </testcase> <testcase classname="<exe-name>.global" name="Mismatching exception messages failing the test" time="{duration}" status="run"> <failure message="thisThrows(), "should fail"" type="REQUIRE_THROWS_WITH"> FAILED: diff --git a/projects/SelfTest/Baselines/sonarqube.sw.approved.txt b/projects/SelfTest/Baselines/sonarqube.sw.approved.txt index 12a4e5be..1b4248d3 100644 --- a/projects/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/projects/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -575,6 +575,30 @@ Condition.tests.cpp:<line number> </skipped> </testCase> <testCase name="Inequality checks that should succeed" duration="{duration}"/> + <testCase name="Mayfail test case with nested sections/1/A" duration="{duration}"> + <skipped message="FAIL()"> +FAILED: +Condition.tests.cpp:<line number> + </skipped> + </testCase> + <testCase name="Mayfail test case with nested sections/2/A" duration="{duration}"> + <skipped message="FAIL()"> +FAILED: +Condition.tests.cpp:<line number> + </skipped> + </testCase> + <testCase name="Mayfail test case with nested sections/1/B" duration="{duration}"> + <skipped message="FAIL()"> +FAILED: +Condition.tests.cpp:<line number> + </skipped> + </testCase> + <testCase name="Mayfail test case with nested sections/2/B" duration="{duration}"> + <skipped message="FAIL()"> +FAILED: +Condition.tests.cpp:<line number> + </skipped> + </testCase> <testCase name="Ordering comparison checks that should fail" duration="{duration}"> <failure message="CHECK(data.int_seven > 7)"> FAILED: diff --git a/projects/SelfTest/Baselines/xml.sw.approved.txt b/projects/SelfTest/Baselines/xml.sw.approved.txt index 2b5da8d3..0a2d3377 100644 --- a/projects/SelfTest/Baselines/xml.sw.approved.txt +++ b/projects/SelfTest/Baselines/xml.sw.approved.txt @@ -7956,6 +7956,43 @@ Nor would this </Expression> <OverallResult success="false"/> </TestCase> + <TestCase name="Mayfail test case with nested sections" tags="[!mayfail]" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" > + <Section name="A" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" > + <Section name="1" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" > + <Failure filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" /> + <OverallResults successes="0" failures="0" expectedFailures="1"/> + </Section> + <OverallResults successes="0" failures="0" expectedFailures="1"/> + </Section> + <Section name="A" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" > + <Section name="2" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" > + <Failure filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" /> + <OverallResults successes="0" failures="0" expectedFailures="1"/> + </Section> + <OverallResults successes="0" failures="0" expectedFailures="1"/> + </Section> + <Section name="A" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" > + <OverallResults successes="0" failures="0" expectedFailures="0"/> + </Section> + <Section name="B" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" > + <Section name="1" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" > + <Failure filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" /> + <OverallResults successes="0" failures="0" expectedFailures="1"/> + </Section> + <OverallResults successes="0" failures="0" expectedFailures="1"/> + </Section> + <Section name="B" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" > + <Section name="2" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" > + <Failure filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" /> + <OverallResults successes="0" failures="0" expectedFailures="1"/> + </Section> + <OverallResults successes="0" failures="0" expectedFailures="1"/> + </Section> + <Section name="B" filename="projects/<exe-name>/UsageTests/Condition.tests.cpp" > + <OverallResults successes="0" failures="0" expectedFailures="0"/> + </Section> + <OverallResult success="true"/> + </TestCase> <TestCase name="Mismatching exception messages failing the test" tags="[!hide][!throws][.][failing]" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" > <Expression success="true" type="REQUIRE_THROWS_WITH" filename="projects/<exe-name>/UsageTests/Exception.tests.cpp" > <Original> @@ -16722,9 +16759,9 @@ loose text artifact </Section> <OverallResult success="true"/> </TestCase> - <OverallResults successes="1607" failures="149" expectedFailures="21"/> - <OverallResultsCases successes="232" failures="86" expectedFailures="4"/> + <OverallResults successes="1607" failures="149" expectedFailures="25"/> + <OverallResultsCases successes="232" failures="86" expectedFailures="5"/> </Group> - <OverallResults successes="1607" failures="148" expectedFailures="21"/> - <OverallResultsCases successes="232" failures="86" expectedFailures="4"/> + <OverallResults successes="1607" failures="148" expectedFailures="25"/> + <OverallResultsCases successes="232" failures="86" expectedFailures="5"/> </Catch> diff --git a/projects/SelfTest/UsageTests/Condition.tests.cpp b/projects/SelfTest/UsageTests/Condition.tests.cpp index 9aed5e2c..fa533095 100644 --- a/projects/SelfTest/UsageTests/Condition.tests.cpp +++ b/projects/SelfTest/UsageTests/Condition.tests.cpp @@ -11,7 +11,7 @@ // Wdouble-promotion is not supported until 3.8 # if (__clang_major__ > 3) || (__clang_major__ == 3 && __clang_minor__ > 7) # pragma clang diagnostic ignored "-Wdouble-promotion" -# endif +# endif #endif #include "catch.hpp" @@ -89,6 +89,19 @@ TEST_CASE( "Equality checks that should fail", "[.][failing][!mayfail]" ) CHECK( x == Approx( 1.301 ) ); } +// Needed to test junit reporter's handling of mayfail test cases and sections +TEST_CASE("Mayfail test case with nested sections", "[!mayfail]") { + SECTION("A") { + SECTION("1") { FAIL(); } + SECTION("2") { FAIL(); } + } + SECTION("B") { + SECTION("1") { FAIL(); } + SECTION("2") { FAIL(); } + } +} + + TEST_CASE( "Inequality checks that should succeed" ) { TestData data;