Replace Catch2's runner with the Catch3 runner

Catch3 keeps Catch2's API for writing tests (test cases, sections,
assertions, matchers, generators, listeners, benchmarks) and replaces its
session, command line and reporters with a runner of its own.

The runner starts the test binary again as worker processes that take the
next test when they are free. A crash, an exit or a hang ends one test, not
the run. On top of that it adds:

- retries that tell deterministic, flaky and order-dependent failures
  apart, and a bisection that names the test leaving state behind;
- a history of verdicts and durations, kept under a file lock, that drives
  automatic quarantine, adaptive time limits, predictive ordering,
  incremental runs, failure replay and analytics;
- selection by change, past failure or prediction, dependency closure,
  and sharding by a stable hash or by recorded durations;
- tags for serial and isolated tests, shared resources, dependencies,
  time limits and retries;
- a failure category, a signature, an analysis and a replay command for
  every failure;
- console, JUnit XML (with flakyFailure and rerunFailure), JSON and Chrome
  trace reports, and a versioned discovery listing (--list=json);
- a temporary directory per test (TMPDIR, TMP, TEMP and
  TestTemporaryDirectory());
- Bazel's test environment: XML output file, test filter, sharding, seed,
  premature exit file and undeclared outputs.

The command line is parsed by a hand-written, typed parser. Catch::Session
remains as a thin facade over the runner for custom main functions.

The result model in Progmasoft/Catch3/Results.hpp was rewritten around
tests and attempts, and the JUnit writer now writes it. The property
engine's example database is updated under a file lock with an atomic
rename, so parallel workers can write it at the same time.

Removed: Clara, the command-line and reporter-spec parsers, every reporter
and CATCH_REGISTER_REPORTER, listing, text flow, console colour and width,
the JSON and XML writers, sharding in the test registry, approval tests and
their baselines, Conan packaging, and the unused third_party/clara.hpp.
Listeners keep working; reporters' list and skip events are gone.

The Catch2 library is now compiled as C++20; code that includes only
Catch2's headers still needs C++14. Duplicate test cases are still refused
at start-up. Version 0.3.0.

ExtraTests now turn exceptions off for clang-cl with MSVC-style options,
which -fno-exceptions did not do.

1 files changed, 299 insertions, 198 deletions

diff --git a/src/Progmasoft/XmlWriter.cpp b/src/Progmasoft/XmlWriter.cpp
index b3a5f9b1..f01d4c45 100644
--- a/src/Progmasoft/XmlWriter.cpp
+++ b/src/Progmasoft/XmlWriter.cpp
@@ -3,272 +3,373 @@
 #include <Progmasoft/Catch3/Results.hpp>
 #include <Progmasoft/Catch3/XmlWriter.hpp>
 
-#include <cmath>
-#include <iomanip>
-#include <locale>
 #include <ostream>
 #include <sstream>
-#include <stdexcept>
 #include <string_view>
 
-namespace Progmasoft::Catch3 {
-
-    namespace {
-
-        struct DecodedCharacter final {
-            char32_t Value = 0;
-            std::size_t Length = 0;
+namespace Progmasoft::Catch3
+{
+    namespace
+    {
+        struct DecodedCharacter final
+        {
+            char32_t value = 0;
+            std::size_t length = 0;
         };
 
-        DecodedCharacter DecodeUtf8( std::string_view text,
-                                     std::size_t offset ) noexcept {
-            const auto first = static_cast<unsigned char>( text[offset] );
-            if ( first < 0x80 ) { return { first, 1 }; }
-
+        DecodedCharacter
+        DecodeUtf8(std::string_view text, std::size_t offset) noexcept
+        {
+            const auto first = static_cast<unsigned char>(text[offset]);
+            if (first < 0x80)
+            {
+                return {first, 1};
+            }
             std::size_t length = 0;
             char32_t value = 0;
             char32_t minimum = 0;
-            if ( first >= 0xC2 && first <= 0xDF ) {
+            if (first >= 0xC2 && first <= 0xDF)
+            {
                 length = 2;
                 value = first & 0x1F;
                 minimum = 0x80;
-            } else if ( first >= 0xE0 && first <= 0xEF ) {
+            }
+            else if (first >= 0xE0 && first <= 0xEF)
+            {
                 length = 3;
                 value = first & 0x0F;
                 minimum = 0x800;
-            } else if ( first >= 0xF0 && first <= 0xF4 ) {
+            }
+            else if (first >= 0xF0 && first <= 0xF4)
+            {
                 length = 4;
                 value = first & 0x07;
                 minimum = 0x10000;
-            } else {
-                return { 0xFFFD, 0 };
             }
-
-            if ( offset + length > text.size() ) { return { 0xFFFD, 0 }; }
-            for ( std::size_t index = 1; index < length; ++index ) {
+            else
+            {
+                return {0xFFFD, 0};
+            }
+            if (offset + length > text.size())
+            {
+                return {0xFFFD, 0};
+            }
+            for (std::size_t index = 1; index < length; ++index)
+            {
                 const auto continuation =
-                    static_cast<unsigned char>( text[offset + index] );
-                if ( ( continuation & 0xC0 ) != 0x80 ) { return { 0xFFFD, 0 }; }
-                value = static_cast<char32_t>( ( value << 6 ) |
-                                               ( continuation & 0x3F ) );
+                    static_cast<unsigned char>(text[offset + index]);
+                if ((continuation & 0xC0) != 0x80)
+                {
+                    return {0xFFFD, 0};
+                }
+                value = static_cast<char32_t>((value << 6) |
+                                              (continuation & 0x3F));
             }
-            if ( value < minimum || value > 0x10FFFF ||
-                 ( value >= 0xD800 && value <= 0xDFFF ) ) {
-                return { 0xFFFD, 0 };
+            if (value < minimum || value > 0x10FFFF ||
+                (value >= 0xD800 && value <= 0xDFFF))
+            {
+                return {0xFFFD, 0};
             }
-            return { value, length };
+            return {value, length};
         }
 
-        bool IsXmlCharacter( char32_t value ) noexcept {
+        bool
+        IsXmlCharacter(char32_t value) noexcept
+        {
             return value == 0x09 || value == 0x0A || value == 0x0D ||
-                   ( value >= 0x20 && value <= 0xD7FF ) ||
-                   ( value >= 0xE000 && value <= 0xFFFD ) ||
-                   ( value >= 0x10000 && value <= 0x10FFFF );
+                   (value >= 0x20 && value <= 0xD7FF) ||
+                   (value >= 0xE000 && value <= 0xFFFD) ||
+                   (value >= 0x10000 && value <= 0x10FFFF);
         }
 
-        void WriteEscaped( std::string_view value,
-                           std::ostream& output,
-                           bool attribute ) {
-            for ( std::size_t index = 0; index < value.size(); ) {
-                const DecodedCharacter character = DecodeUtf8( value, index );
-                if ( character.Length == 0 ||
-                     !IsXmlCharacter( character.Value ) ) {
-                    // Preserve parseability without silently dropping input.
+        void
+        WriteEscaped(std::string_view value, std::ostream &output, bool attribute)
+        {
+            for (std::size_t index = 0; index < value.size();)
+            {
+                const DecodedCharacter character = DecodeUtf8(value, index);
+                if (character.length == 0 || !IsXmlCharacter(character.value))
+                {
+                    // A character XML cannot hold is replaced, not dropped,
+                    // so the reader sees that something was there.
                     output << "&#xFFFD;";
                     ++index;
                     continue;
                 }
-
-                if ( character.Value < 0x80 ) {
-                    switch ( static_cast<char>( character.Value ) ) {
-                    case '&':
-                        output << "&amp;";
-                        break;
-                    case '<':
-                        output << "&lt;";
-                        break;
-                    case '>':
-                        output << "&gt;";
-                        break;
-                    case '"':
-                        if ( attribute ) {
-                            output << "&quot;";
-                        } else {
-                            output.put( '"' );
-                        }
-                        break;
-                    case '\'':
-                        if ( attribute ) {
-                            output << "&apos;";
-                        } else {
-                            output.put( '\'' );
-                        }
-                        break;
-                    default:
-                        output.put( static_cast<char>( character.Value ) );
+                if (character.value < 0x80)
+                {
+                    switch (static_cast<char>(character.value))
+                    {
+                        case '&':
+                            output << "&amp;";
+                            break;
+                        case '<':
+                            output << "&lt;";
+                            break;
+                        case '>':
+                            output << "&gt;";
+                            break;
+                        case '"':
+                            output << (attribute ? "&quot;" : "\"");
+                            break;
+                        case '\'':
+                            output << (attribute ? "&apos;" : "'");
+                            break;
+                        case '\n':
+                            output << (attribute ? "&#10;" : "\n");
+                            break;
+                        default:
+                            output.put(static_cast<char>(character.value));
                     }
-                } else {
-                    output.write(
-                        value.data() + index,
-                        static_cast<std::streamsize>( character.Length ) );
                 }
-                index += character.Length;
+                else
+                {
+                    output.write(value.data() + index,
+                                 static_cast<std::streamsize>(character.length));
+                }
+                index += character.length;
             }
         }
 
-        void WriteAttribute( std::ostream& output,
-                             std::string_view name,
-                             std::string_view value ) {
-            output.put( ' ' );
-            output.write( name.data(),
-                          static_cast<std::streamsize>( name.size() ) );
-            output << "=\"";
-            WriteEscaped( value, output, true );
-            output.put( '"' );
+        void
+        WriteAttribute(std::ostream &output,
+                       std::string_view name,
+                       std::string_view value)
+        {
+            output << ' ' << name << "=\"";
+            WriteEscaped(value, output, true);
+            output << '"';
         }
 
-        std::string FormatSeconds( double seconds ) {
-            std::ostringstream formatted;
-            formatted.imbue( std::locale::classic() );
-            formatted << std::fixed << std::setprecision( 6 ) << seconds;
-            return formatted.str();
+        std::string
+        JUnitSeconds(std::int64_t microseconds)
+        {
+            const bool negative = microseconds < 0;
+            const auto magnitude = negative ? -microseconds : microseconds;
+            std::string fraction = std::to_string(magnitude % 1000000);
+            fraction.insert(0, 6 - fraction.size(), '0');
+            return (negative ? "-" : "") + std::to_string(magnitude / 1000000) +
+                   "." + fraction;
         }
 
-        void WriteIndent( std::ostream& output, unsigned int level ) {
-            for ( unsigned int index = 0; index < level; ++index ) {
+        void
+        WriteIndent(std::ostream &output, unsigned level)
+        {
+            for (unsigned index = 0; index < level; ++index)
+            {
                 output << "  ";
             }
         }
 
-        void WriteTextElement( std::ostream& output,
-                               unsigned int indent,
-                               std::string_view name,
-                               std::string_view value ) {
-            WriteIndent( output, indent );
-            output.put( '<' );
-            output.write( name.data(),
-                          static_cast<std::streamsize>( name.size() ) );
-            output.put( '>' );
-            WriteEscaped( value, output, false );
-            output << "</";
-            output.write( name.data(),
-                          static_cast<std::streamsize>( name.size() ) );
-            output << ">\n";
+        void
+        WriteTextElement(std::ostream &output,
+                         unsigned indent,
+                         std::string_view name,
+                         std::string_view value)
+        {
+            WriteIndent(output, indent);
+            output << '<' << name << '>';
+            WriteEscaped(value, output, false);
+            output << "</" << name << ">\n";
         }
 
-        TestRunSummary SummarizeSuite( const TestSuiteResult& suite ) {
-            return SummarizeTestSuite( suite );
+        std::string
+        Output(const Attempt &attempt, bool errors)
+        {
+            if (errors)
+            {
+                return attempt.standardError + attempt.processOutput;
+            }
+            return attempt.standardOutput;
         }
 
-        void WriteTestCase( const TestCaseResult& testCase,
-                            std::string_view suiteName,
-                            std::ostream& output ) {
-            const bool hasBody = testCase.Status != TestCaseStatus::kPassed ||
-                                 !testCase.StandardOutput.empty() ||
-                                 !testCase.StandardError.empty();
-            WriteIndent( output, 2 );
-            output << "<testcase";
-            WriteAttribute( output,
-                            "classname",
-                            testCase.ClassName.empty() ? suiteName
-                                                       : testCase.ClassName );
-            WriteAttribute( output, "name", testCase.Name );
-            WriteAttribute(
-                output, "time", FormatSeconds( testCase.DurationSeconds ) );
-            if ( !testCase.SourceFile.empty() ) {
-                WriteAttribute( output, "file", testCase.SourceFile );
+        /// An earlier attempt: `<rerunFailure>` before a final failure,
+        /// `<flakyFailure>` before a final pass.
+        void
+        WriteEarlierAttempt(std::ostream &output,
+                            const Attempt &attempt,
+                            std::string_view element)
+        {
+            WriteIndent(output, 3);
+            output << '<' << element;
+            WriteAttribute(output, "message", SummarizeFailure(attempt));
+            WriteAttribute(output, "type", CategoryName(attempt.category));
+            output << ">\n";
+            WriteTextElement(output, 4, "stackTrace", DescribeFailure(attempt));
+            if (!attempt.standardOutput.empty())
+            {
+                WriteTextElement(
+                    output, 4, "system-out", Output(attempt, false));
             }
-            if ( !hasBody ) {
-                output << "/>\n";
-                return;
+            if (!Output(attempt, true).empty())
+            {
+                WriteTextElement(output, 4, "system-err", Output(attempt, true));
+            }
+            WriteIndent(output, 3);
+            output << "</" << element << ">\n";
+        }
+
+        std::string
+        SkipReason(const TestResult &test)
+        {
+            switch (test.outcome)
+            {
+                case Outcome::kBlocked:
+                    return test.analysis.empty() ? "a dependency did not pass"
+                                                 : test.analysis.front();
+                case Outcome::kNotRun:
+                    return "the run stopped before this test";
+                case Outcome::kUpToDate:
+                    return "passed before; nothing it depends on changed";
+                default:
+                    return "skipped";
             }
+        }
 
+        void
+        WriteTestCase(std::ostream &output,
+                      const TestResult &test,
+                      std::string_view suiteName)
+        {
+            WriteIndent(output, 2);
+            output << "<testcase";
+            WriteAttribute(output,
+                           "classname",
+                           test.className.empty() ? suiteName : test.className);
+            WriteAttribute(output, "name", test.name);
+            WriteAttribute(output, "time", JUnitSeconds(test.DurationMicroseconds()));
+            if (!test.file.empty())
+            {
+                WriteAttribute(output, "file", test.file);
+                WriteAttribute(output, "line", std::to_string(test.line));
+            }
             output << ">\n";
-            if ( testCase.Status == TestCaseStatus::kFailed ||
-                 testCase.Status == TestCaseStatus::kErrored ) {
-                const bool isError =
-                    testCase.Status == TestCaseStatus::kErrored;
-                WriteIndent( output, 3 );
-                output << ( isError ? "<error" : "<failure" );
-                WriteAttribute( output, "message", testCase.Message );
-                WriteAttribute( output,
-                                "type",
-                                isError ? "TestError" : "AssertionFailure" );
-                output.put( '>' );
-                WriteEscaped( testCase.Details.empty() ? testCase.Message
-                                                       : testCase.Details,
-                              output,
-                              false );
-                output << ( isError ? "</error>\n" : "</failure>\n" );
-            } else if ( testCase.Status == TestCaseStatus::kSkipped ) {
-                WriteIndent( output, 3 );
+
+            const Attempt *last =
+                test.attempts.empty() ? nullptr : &test.attempts.back();
+            const bool failedRun = IsFailure(test.outcome);
+            if (failedRun && last != nullptr)
+            {
+                const bool error = test.outcome != Outcome::kFailed ||
+                                   last->category ==
+                                       FailureCategory::kInfrastructure;
+                const std::string_view element = error ? "error" : "failure";
+                WriteIndent(output, 3);
+                output << '<' << element;
+                WriteAttribute(output, "message", SummarizeFailure(*last));
+                WriteAttribute(output, "type", CategoryName(test.category));
+                output << '>';
+                std::string details = DescribeFailure(*last);
+                for (const auto &finding : test.analysis)
+                {
+                    details += "analysis: " + finding + "\n";
+                }
+                if (!test.replayCommand.empty())
+                {
+                    details += "replay: " + test.replayCommand + "\n";
+                }
+                WriteEscaped(details, output, false);
+                output << "</" << element << ">\n";
+            }
+            else if (test.outcome == Outcome::kSkipped ||
+                     test.outcome == Outcome::kBlocked ||
+                     test.outcome == Outcome::kNotRun ||
+                     test.outcome == Outcome::kUpToDate)
+            {
+                WriteIndent(output, 3);
                 output << "<skipped";
-                WriteAttribute( output, "message", testCase.Message );
+                WriteAttribute(output, "message", SkipReason(test));
                 output << "/>\n";
             }
-            if ( !testCase.StandardOutput.empty() ) {
-                WriteTextElement(
-                    output, 3, "system-out", testCase.StandardOutput );
+            // Every earlier failed attempt, so retries stay visible.
+            for (std::size_t index = 0; index + 1 < test.attempts.size();
+                 ++index)
+            {
+                const Attempt &attempt = test.attempts[index];
+                if (!IsFailure(attempt.outcome))
+                {
+                    continue;
+                }
+                WriteEarlierAttempt(output,
+                                    attempt,
+                                    failedRun ? "rerunFailure" : "flakyFailure");
             }
-            if ( !testCase.StandardError.empty() ) {
-                WriteTextElement(
-                    output, 3, "system-err", testCase.StandardError );
+            if (last != nullptr && !last->standardOutput.empty())
+            {
+                WriteTextElement(output, 3, "system-out", Output(*last, false));
+            }
+            if (last != nullptr && !Output(*last, true).empty())
+            {
+                WriteTextElement(output, 3, "system-err", Output(*last, true));
             }
-            WriteIndent( output, 2 );
+            WriteIndent(output, 2);
             output << "</testcase>\n";
         }
-
     } // namespace
 
-    void JUnitXmlWriter::Write( const TestRunResult& result,
-                                std::ostream& output ) {
-        const TestRunSummary runSummary = SummarizeTestRun( result );
+    bool
+    JUnitXmlWriter::Write(const RunResult &run, std::ostream &output)
+    {
+        const RunSummary summary = Summarize(run);
+        const std::size_t errors = summary.crashed + summary.timedOut;
+        const std::size_t failures = summary.failed;
+        const std::size_t skipped =
+            summary.skipped + summary.blocked + summary.notRun +
+            summary.upToDate;
+        const std::string time = JUnitSeconds(run.wallMicroseconds);
         output << "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<testsuites";
-        WriteAttribute( output, "name", result.Name );
-        WriteAttribute( output, "tests", std::to_string( runSummary.Tests ) );
-        WriteAttribute(
-            output, "failures", std::to_string( runSummary.Failures ) );
-        WriteAttribute( output, "errors", std::to_string( runSummary.Errors ) );
-        WriteAttribute(
-            output, "skipped", std::to_string( runSummary.Skipped ) );
-        WriteAttribute(
-            output, "time", FormatSeconds( runSummary.DurationSeconds ) );
+        WriteAttribute(output, "name", run.binary);
+        WriteAttribute(output, "tests", std::to_string(summary.tests));
+        WriteAttribute(output, "failures", std::to_string(failures));
+        WriteAttribute(output, "errors", std::to_string(errors));
+        WriteAttribute(output, "skipped", std::to_string(skipped));
+        WriteAttribute(output, "time", time);
         output << ">\n";
-
-        for ( const TestSuiteResult& suite : result.TestSuites ) {
-            const TestRunSummary suiteSummary = SummarizeSuite( suite );
-            WriteIndent( output, 1 );
-            output << "<testsuite";
-            WriteAttribute( output, "name", suite.Name );
-            WriteAttribute(
-                output, "tests", std::to_string( suiteSummary.Tests ) );
-            WriteAttribute(
-                output, "failures", std::to_string( suiteSummary.Failures ) );
-            WriteAttribute(
-                output, "errors", std::to_string( suiteSummary.Errors ) );
-            WriteAttribute(
-                output, "skipped", std::to_string( suiteSummary.Skipped ) );
-            WriteAttribute(
-                output, "time", FormatSeconds( suiteSummary.DurationSeconds ) );
-            output << ">\n";
-
-            for ( const TestCaseResult& testCase : suite.TestCases ) {
-                WriteTestCase( testCase, suite.Name, output );
-            }
-            WriteIndent( output, 1 );
-            output << "</testsuite>\n";
+        WriteIndent(output, 1);
+        output << "<testsuite";
+        WriteAttribute(output, "name", run.binary);
+        WriteAttribute(output, "tests", std::to_string(summary.tests));
+        WriteAttribute(output, "failures", std::to_string(failures));
+        WriteAttribute(output, "errors", std::to_string(errors));
+        WriteAttribute(output, "skipped", std::to_string(skipped));
+        WriteAttribute(output, "time", time);
+        output << ">\n";
+        WriteIndent(output, 2);
+        output << "<properties>\n";
+        const auto property = [&](std::string_view name,
+                                  const std::string &value) {
+            WriteIndent(output, 3);
+            output << "<property";
+            WriteAttribute(output, "name", name);
+            WriteAttribute(output, "value", value);
+            output << "/>\n";
+        };
+        property("random-seed", std::to_string(run.seed));
+        property("workers", std::to_string(run.workers));
+        if (run.shardCount > 1)
+        {
+            property("shard",
+                     std::to_string(run.shardIndex) + "/" +
+                         std::to_string(run.shardCount));
         }
-        output << "</testsuites>\n";
-
-        if ( !output ) {
-            throw std::ios_base::failure( "could not write JUnit XML results" );
+        property("flaky", std::to_string(summary.flaky));
+        property("quarantined", std::to_string(summary.quarantined));
+        WriteIndent(output, 2);
+        output << "</properties>\n";
+        for (const auto &test : run.tests)
+        {
+            WriteTestCase(output, test, run.binary);
         }
+        WriteIndent(output, 1);
+        output << "</testsuite>\n</testsuites>\n";
+        return static_cast<bool>(output);
     }
 
-    std::string JUnitXmlWriter::ToString( const TestRunResult& result ) {
+    std::string
+    JUnitXmlWriter::ToString(const RunResult &run)
+    {
         std::ostringstream output;
-        Write( result, output );
+        static_cast<void>(Write(run, output));
         return output.str();
     }
-
 } // namespace Progmasoft::Catch3