Read-only mirrors. Development, issues and pull requests are on GitHub.

summaryrefslogtreecommitdiffhomepage
path: root/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
diff options
context:
space:
mode:
authorMartin Hořeňovský2019-12-05 16:00:20 +0100
committerMartin Hořeňovský2019-12-05 16:00:20 +0100
commit0fea081ad1a678b379182930a3ec22a3e275dbf0 (patch)
tree3afa04ce6805d7a86e9c59f87ac68cb1d21eb8b2 /tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
parent90e2549cecc2e886d563b7a88e752c9b4104f798 (diff)
downloadcatch3-0fea081ad1a678b379182930a3ec22a3e275dbf0.tar.gz
Move tests from projects/ to tests/
Diffstat (limited to 'tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp')
-rw-r--r--tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp516
1 files changed, 516 insertions, 0 deletions
diff --git a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
new file mode 100644
index 00000000..fe3b95c9
--- /dev/null
+++ b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
@@ -0,0 +1,516 @@
+/*
+ * Created by Phil on 13/5/2013.
+ * Copyright 2014 Two Blue Cubes Ltd. All rights reserved.
+ *
+ * Distributed under the Boost Software License, Version 1.0. (See accompanying
+ * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+ */
+
+#include <catch2/catch.hpp>
+#include <catch2/catch_test_spec_parser.h>
+#include <catch2/catch_test_case_info.h>
+#include <catch2/catch_config.hpp>
+#include <catch2/catch_commandline.h>
+
+#ifdef __clang__
+# pragma clang diagnostic ignored "-Wc++98-compat"
+#endif
+
+namespace {
+ auto fakeTestCase(const char* name, const char* desc = "") { return Catch::makeTestCaseInfo("", { name, desc }, CATCH_INTERNAL_LINEINFO); }
+}
+
+TEST_CASE( "Parse test names and tags" ) {
+
+ using Catch::parseTestSpec;
+ using Catch::TestSpec;
+
+ auto tcA = fakeTestCase( "a" );
+ auto tcB = fakeTestCase( "b", "[one][x]" );
+ auto tcC = fakeTestCase( "longer name with spaces", "[two][three][.][x]" );
+ auto tcD = fakeTestCase( "zlonger name with spacesz" );
+
+ SECTION( "Empty test spec should have no filters" ) {
+ TestSpec spec;
+ CHECK( spec.hasFilters() == false );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ }
+
+ SECTION( "Test spec from empty string should have no filters" ) {
+ TestSpec spec = parseTestSpec( "" );
+ CHECK( spec.hasFilters() == false );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ }
+
+ SECTION( "Test spec from just a comma should have no filters" ) {
+ TestSpec spec = parseTestSpec( "," );
+ CHECK( spec.hasFilters() == false );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ }
+
+ SECTION( "Test spec from name should have one filter" ) {
+ TestSpec spec = parseTestSpec( "b" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == true );
+ }
+
+ SECTION( "Test spec from quoted name should have one filter" ) {
+ TestSpec spec = parseTestSpec( "\"b\"" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == true );
+ }
+
+ SECTION( "Test spec from name should have one filter" ) {
+ TestSpec spec = parseTestSpec( "b" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == true );
+ CHECK( spec.matches( *tcC ) == false );
+ }
+
+ SECTION( "Wildcard at the start" ) {
+ TestSpec spec = parseTestSpec( "*spaces" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == true );
+ CHECK( spec.matches( *tcD ) == false );
+ CHECK( parseTestSpec( "*a" ).matches( *tcA ) == true );
+ }
+ SECTION( "Wildcard at the end" ) {
+ TestSpec spec = parseTestSpec( "long*" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == true );
+ CHECK( spec.matches( *tcD ) == false );
+ CHECK( parseTestSpec( "a*" ).matches( *tcA ) == true );
+ }
+ SECTION( "Wildcard at both ends" ) {
+ TestSpec spec = parseTestSpec( "*name*" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == true );
+ CHECK( spec.matches( *tcD ) == true );
+ CHECK( parseTestSpec( "*a*" ).matches( *tcA ) == true );
+ }
+ SECTION( "Redundant wildcard at the start" ) {
+ TestSpec spec = parseTestSpec( "*a" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == true );
+ CHECK( spec.matches( *tcB ) == false );
+ }
+ SECTION( "Redundant wildcard at the end" ) {
+ TestSpec spec = parseTestSpec( "a*" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == true );
+ CHECK( spec.matches( *tcB ) == false );
+ }
+ SECTION( "Redundant wildcard at both ends" ) {
+ TestSpec spec = parseTestSpec( "*a*" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == true );
+ CHECK( spec.matches( *tcB ) == false );
+ }
+ SECTION( "Wildcard at both ends, redundant at start" ) {
+ TestSpec spec = parseTestSpec( "*longer*" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == true );
+ CHECK( spec.matches( *tcD ) == true );
+ }
+ SECTION( "Just wildcard" ) {
+ TestSpec spec = parseTestSpec( "*" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == true );
+ CHECK( spec.matches( *tcB ) == true );
+ CHECK( spec.matches( *tcC ) == true );
+ CHECK( spec.matches( *tcD ) == true );
+ }
+
+ SECTION( "Single tag" ) {
+ TestSpec spec = parseTestSpec( "[one]" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == true );
+ CHECK( spec.matches( *tcC ) == false );
+ }
+ SECTION( "Single tag, two matches" ) {
+ TestSpec spec = parseTestSpec( "[x]" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == true );
+ CHECK( spec.matches( *tcC ) == true );
+ }
+ SECTION( "Two tags" ) {
+ TestSpec spec = parseTestSpec( "[two][x]" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == true );
+ }
+ SECTION( "Two tags, spare separated" ) {
+ TestSpec spec = parseTestSpec( "[two] [x]" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == true );
+ }
+ SECTION( "Wildcarded name and tag" ) {
+ TestSpec spec = parseTestSpec( "*name*[x]" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == true );
+ CHECK( spec.matches( *tcD ) == false );
+ }
+ SECTION( "Single tag exclusion" ) {
+ TestSpec spec = parseTestSpec( "~[one]" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == true );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == false );
+ }
+ SECTION( "One tag exclusion and one tag inclusion" ) {
+ TestSpec spec = parseTestSpec( "~[two][x]" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == true );
+ CHECK( spec.matches( *tcC ) == false );
+ }
+ SECTION( "One tag exclusion and one wldcarded name inclusion" ) {
+ TestSpec spec = parseTestSpec( "~[two]*name*" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == false );
+ CHECK( spec.matches( *tcD ) == true );
+ }
+ SECTION( "One tag exclusion, using exclude:, and one wldcarded name inclusion" ) {
+ TestSpec spec = parseTestSpec( "exclude:[two]*name*" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == false );
+ CHECK( spec.matches( *tcD ) == true );
+ }
+ SECTION( "name exclusion" ) {
+ TestSpec spec = parseTestSpec( "~b" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == true );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == false );
+ CHECK( spec.matches( *tcD ) == true );
+ }
+ SECTION( "wildcarded name exclusion" ) {
+ TestSpec spec = parseTestSpec( "~*name*" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == true );
+ CHECK( spec.matches( *tcB ) == true );
+ CHECK( spec.matches( *tcC ) == false );
+ CHECK( spec.matches( *tcD ) == false );
+ }
+ SECTION( "wildcarded name exclusion with tag inclusion" ) {
+ TestSpec spec = parseTestSpec( "~*name*,[three]" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == true );
+ CHECK( spec.matches( *tcB ) == true );
+ CHECK( spec.matches( *tcC ) == true );
+ CHECK( spec.matches( *tcD ) == false );
+ }
+ SECTION( "wildcarded name exclusion, using exclude:, with tag inclusion" ) {
+ TestSpec spec = parseTestSpec( "exclude:*name*,[three]" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == true );
+ CHECK( spec.matches( *tcB ) == true );
+ CHECK( spec.matches( *tcC ) == true );
+ CHECK( spec.matches( *tcD ) == false );
+ }
+ SECTION( "two wildcarded names" ) {
+ TestSpec spec = parseTestSpec( "\"longer*\"\"*spaces\"" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == true );
+ CHECK( spec.matches( *tcD ) == false );
+ }
+ SECTION( "empty tag" ) {
+ TestSpec spec = parseTestSpec( "[]" );
+ CHECK( spec.hasFilters() == false );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == false );
+ CHECK( spec.matches( *tcD ) == false );
+ }
+ SECTION( "empty quoted name" ) {
+ TestSpec spec = parseTestSpec( "\"\"" );
+ CHECK( spec.hasFilters() == false );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == false );
+ CHECK( spec.matches( *tcD ) == false );
+ }
+ SECTION( "quoted string followed by tag exclusion" ) {
+ TestSpec spec = parseTestSpec( "\"*name*\"~[.]" );
+ CHECK( spec.hasFilters() == true );
+ CHECK( spec.matches( *tcA ) == false );
+ CHECK( spec.matches( *tcB ) == false );
+ CHECK( spec.matches( *tcC ) == false );
+ CHECK( spec.matches( *tcD ) == true );
+ }
+ SECTION( "Leading and trailing spaces in test spec" ) {
+ TestSpec spec = parseTestSpec( "\" aardvark \"" );
+ CHECK( spec.matches( *fakeTestCase( " aardvark " ) ) );
+ CHECK( spec.matches( *fakeTestCase( " aardvark" ) ) );
+ CHECK( spec.matches( *fakeTestCase( " aardvark " ) ) );
+ CHECK( spec.matches( *fakeTestCase( "aardvark " ) ) );
+ CHECK( spec.matches( *fakeTestCase( "aardvark" ) ) );
+
+ }
+ SECTION( "Leading and trailing spaces in test name" ) {
+ TestSpec spec = parseTestSpec( "aardvark" );
+ CHECK( spec.matches( *fakeTestCase( " aardvark " ) ) );
+ CHECK( spec.matches( *fakeTestCase( " aardvark" ) ) );
+ CHECK( spec.matches( *fakeTestCase( " aardvark " ) ) );
+ CHECK( spec.matches( *fakeTestCase( "aardvark " ) ) );
+ CHECK( spec.matches( *fakeTestCase( "aardvark" ) ) );
+
+ }
+}
+
+TEST_CASE( "Process can be configured on command line", "[config][command-line]" ) {
+
+#ifndef CATCH_CONFIG_DISABLE_MATCHERS
+ using namespace Catch::Matchers;
+#endif
+
+ Catch::ConfigData config;
+ auto cli = Catch::makeCommandLineParser(config);
+
+ SECTION("empty args don't cause a crash") {
+ auto result = cli.parse({""});
+ CHECK(result);
+ CHECK(config.processName == "");
+ }
+
+ SECTION("default - no arguments") {
+ auto result = cli.parse({"test"});
+ CHECK(result);
+ CHECK(config.processName == "test");
+ CHECK(config.shouldDebugBreak == false);
+ CHECK(config.abortAfter == -1);
+ CHECK(config.noThrow == false);
+ CHECK(config.reporterName == "console");
+
+ Catch::Config cfg(config);
+ CHECK_FALSE(cfg.hasTestFilters());
+ }
+
+ SECTION("test lists") {
+ SECTION("Specify one test case using") {
+ auto result = cli.parse({"test", "test1"});
+ CHECK(result);
+
+ Catch::Config cfg(config);
+ REQUIRE(cfg.hasTestFilters());
+ REQUIRE(cfg.testSpec().matches(*fakeTestCase("notIncluded")) == false);
+ REQUIRE(cfg.testSpec().matches(*fakeTestCase("test1")));
+ }
+ SECTION("Specify one test case exclusion using exclude:") {
+ auto result = cli.parse({"test", "exclude:test1"});
+ CHECK(result);
+
+ Catch::Config cfg(config);
+ REQUIRE(cfg.hasTestFilters());
+ REQUIRE(cfg.testSpec().matches(*fakeTestCase("test1")) == false);
+ REQUIRE(cfg.testSpec().matches(*fakeTestCase("alwaysIncluded")));
+ }
+
+ SECTION("Specify one test case exclusion using ~") {
+ auto result = cli.parse({"test", "~test1"});
+ CHECK(result);
+
+ Catch::Config cfg(config);
+ REQUIRE(cfg.hasTestFilters());
+ REQUIRE(cfg.testSpec().matches(*fakeTestCase("test1")) == false);
+ REQUIRE(cfg.testSpec().matches(*fakeTestCase("alwaysIncluded")));
+ }
+
+ }
+
+ SECTION("reporter") {
+ SECTION("-r/console") {
+ CHECK(cli.parse({"test", "-r", "console"}));
+
+ REQUIRE(config.reporterName == "console");
+ }
+ SECTION("-r/xml") {
+ CHECK(cli.parse({"test", "-r", "xml"}));
+
+ REQUIRE(config.reporterName == "xml");
+ }
+ SECTION("--reporter/junit") {
+ CHECK(cli.parse({"test", "--reporter", "junit"}));
+
+ REQUIRE(config.reporterName == "junit");
+ }
+ SECTION("Only one reporter is accepted") {
+ REQUIRE_FALSE(cli.parse({ "test", "-r", "xml", "-r", "junit" }));
+ }
+ SECTION("must match one of the available ones") {
+ auto result = cli.parse({"test", "--reporter", "unsupported"});
+ CHECK(!result);
+
+#ifndef CATCH_CONFIG_DISABLE_MATCHERS
+ REQUIRE_THAT(result.errorMessage(), Contains("Unrecognized reporter"));
+#endif
+ }
+ }
+
+ SECTION("debugger") {
+ SECTION("-b") {
+ CHECK(cli.parse({"test", "-b"}));
+
+ REQUIRE(config.shouldDebugBreak == true);
+ }
+ SECTION("--break") {
+ CHECK(cli.parse({"test", "--break"}));
+
+ REQUIRE(config.shouldDebugBreak);
+ }
+ }
+
+
+ SECTION("abort") {
+ SECTION("-a aborts after first failure") {
+ CHECK(cli.parse({"test", "-a"}));
+
+ REQUIRE(config.abortAfter == 1);
+ }
+ SECTION("-x 2 aborts after two failures") {
+ CHECK(cli.parse({"test", "-x", "2"}));
+
+ REQUIRE(config.abortAfter == 2);
+ }
+ SECTION("-x must be numeric") {
+ auto result = cli.parse({"test", "-x", "oops"});
+ CHECK(!result);
+
+#ifndef CATCH_CONFIG_DISABLE_MATCHERS
+ REQUIRE_THAT(result.errorMessage(), Contains("convert") && Contains("oops"));
+#endif
+ }
+ }
+
+ SECTION("nothrow") {
+ SECTION("-e") {
+ CHECK(cli.parse({"test", "-e"}));
+
+ REQUIRE(config.noThrow);
+ }
+ SECTION("--nothrow") {
+ CHECK(cli.parse({"test", "--nothrow"}));
+
+ REQUIRE(config.noThrow);
+ }
+ }
+
+ SECTION("output filename") {
+ SECTION("-o filename") {
+ CHECK(cli.parse({"test", "-o", "filename.ext"}));
+
+ REQUIRE(config.outputFilename == "filename.ext");
+ }
+ SECTION("--out") {
+ CHECK(cli.parse({"test", "--out", "filename.ext"}));
+
+ REQUIRE(config.outputFilename == "filename.ext");
+ }
+ }
+
+ SECTION("combinations") {
+ SECTION("Single character flags can be combined") {
+ CHECK(cli.parse({"test", "-abe"}));
+
+ CHECK(config.abortAfter == 1);
+ CHECK(config.shouldDebugBreak);
+ CHECK(config.noThrow == true);
+ }
+ }
+
+
+ SECTION( "use-colour") {
+
+ using Catch::UseColour;
+
+ SECTION( "without option" ) {
+ CHECK(cli.parse({"test"}));
+
+ REQUIRE( config.useColour == UseColour::Auto );
+ }
+
+ SECTION( "auto" ) {
+ CHECK(cli.parse({"test", "--use-colour", "auto"}));
+
+ REQUIRE( config.useColour == UseColour::Auto );
+ }
+
+ SECTION( "yes" ) {
+ CHECK(cli.parse({"test", "--use-colour", "yes"}));
+
+ REQUIRE( config.useColour == UseColour::Yes );
+ }
+
+ SECTION( "no" ) {
+ CHECK(cli.parse({"test", "--use-colour", "no"}));
+
+ REQUIRE( config.useColour == UseColour::No );
+ }
+
+ SECTION( "error" ) {
+ auto result = cli.parse({"test", "--use-colour", "wrong"});
+ CHECK( !result );
+#ifndef CATCH_CONFIG_DISABLE_MATCHERS
+ CHECK_THAT( result.errorMessage(), Contains( "colour mode must be one of" ) );
+#endif
+ }
+ }
+
+ SECTION("Benchmark options") {
+ SECTION("samples") {
+ CHECK(cli.parse({ "test", "--benchmark-samples=200" }));
+
+ REQUIRE(config.benchmarkSamples == 200);
+ }
+
+ SECTION("resamples") {
+ CHECK(cli.parse({ "test", "--benchmark-resamples=20000" }));
+
+ REQUIRE(config.benchmarkResamples == 20000);
+ }
+
+ SECTION("resamples") {
+ CHECK(cli.parse({ "test", "--benchmark-confidence-interval=0.99" }));
+
+ REQUIRE(config.benchmarkConfidenceInterval == Catch::Detail::Approx(0.99));
+ }
+
+ SECTION("resamples") {
+ CHECK(cli.parse({ "test", "--benchmark-no-analysis" }));
+
+ REQUIRE(config.benchmarkNoAnalysis);
+ }
+ }
+}
+
+TEST_CASE("Test with special, characters \"in name", "[cli][regression]") {
+ // This test case succeeds if we can invoke it from the CLI
+ SUCCEED();
+}