Emit warning when using --shard-count (> 1) with --order rand

2 files changed, 51 insertions, 1 deletions

diff --git a/tests/ExtraTests/CMakeLists.txt b/tests/ExtraTests/CMakeLists.txt
index 21075a5b..a7b78748 100644
--- a/tests/ExtraTests/CMakeLists.txt
+++ b/tests/ExtraTests/CMakeLists.txt
@@ -21,6 +21,37 @@ set_tests_properties(TestSharding::OverlyLargeShardIndex
     PASS_REGULAR_EXPRESSION "The shard count \\(5\\) must be greater than the shard index \\(5\\)"
 )
 
+set(CATCH_SHARDING_WARNING_REGEX "Warning: using sharding .* with random order")
+
+add_test(
+  NAME TestSharding::WarningOnRandomOrderWithoutFixedSeed
+  COMMAND $<TARGET_FILE:SelfTest> --shard-index 0 --shard-count 2 --list-tests
+)
+set_tests_properties(TestSharding::WarningOnRandomOrderWithoutFixedSeed
+  PROPERTIES
+    PASS_REGULAR_EXPRESSION "${CATCH_SHARDING_WARNING_REGEX}"
+)
+
+add_test(
+  NAME TestSharding::NoWarningOnRandomOrderWithFixedSeed
+  COMMAND $<TARGET_FILE:SelfTest> --shard-index 0 --shard-count 2 --rng-seed 12345 --list-tests
+)
+set_tests_properties(TestSharding::NoWarningOnRandomOrderWithFixedSeed
+  PROPERTIES
+    FAIL_REGULAR_EXPRESSION "${CATCH_SHARDING_WARNING_REGEX}"
+)
+
+foreach(shardOrder decl lex)
+  add_test(
+    NAME TestSharding::NoWarningOn${shardOrder}OrderWithoutFixedSeed
+    COMMAND $<TARGET_FILE:SelfTest> --shard-index 0 --shard-count 2 --order ${shardOrder} --list-tests
+  )
+  set_tests_properties(TestSharding::NoWarningOn${shardOrder}OrderWithoutFixedSeed
+    PROPERTIES
+      FAIL_REGULAR_EXPRESSION "${CATCH_SHARDING_WARNING_REGEX}"
+  )
+endforeach()
+
 # The MinDuration reporting tests do not need separate compilation, but
 # they have non-trivial execution time, so they are categorized as
 # extra tests, so that they are run less.
diff --git a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
index 0717ae2e..71ddc6fd 100644
--- a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
+++ b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp
@@ -473,15 +473,34 @@ TEST_CASE( "Parse rng seed in different formats", "[approvals][cli][rng-seed]" )
         CAPTURE( seed_string );
 
         auto result = cli.parse( { "tests", "--rng-seed", seed_string } );
+        REQUIRE( result );
 
+        Catch::Config cfg{config};
+        REQUIRE( cfg.rngSeed() == seed_value );
+        REQUIRE( cfg.rngSeedWasFixed() );
+    }
+    SECTION( "time seed is not considered fixed" ) {
+        auto result = cli.parse( { "tests", "--rng-seed", "time" } );
         REQUIRE( result );
-        REQUIRE( config.rngSeed == seed_value );
+
+        Catch::Config cfg{config};
+        REQUIRE_FALSE( cfg.rngSeedWasFixed() );
+    }
+    SECTION( "random-device seed is not considered fixed" ) {
+        auto result = cli.parse( { "tests", "--rng-seed", "random-device" } );
+        REQUIRE( result );
+
+        Catch::Config cfg{config};
+        REQUIRE_FALSE( cfg.rngSeedWasFixed() );
     }
     SECTION( "Error cases" ) {
         auto seed_string =
             GENERATE( "0xSEED", "999999999999", "08888", "BEEF", "123 456" );
         CAPTURE( seed_string );
         REQUIRE_FALSE( cli.parse( { "tests", "--rng-seed", seed_string } ) );
+
+        Catch::Config cfg{config};
+        REQUIRE_FALSE( cfg.rngSeedWasFixed() );
     }
 }