Fix counting of UTF-8 codepoints in TextFlow

For line-wrapping bytes were counted instead of codepoints. This
resulted in line-breaks being inserted at the wrong position and also
breaking UTF-8 characters.

Fixes #1022.

1 files changed, 75 insertions, 0 deletions

diff --git a/tests/SelfTest/IntrospectiveTests/TextFlow.tests.cpp b/tests/SelfTest/IntrospectiveTests/TextFlow.tests.cpp
index de03ed09..fc344c0d 100644
--- a/tests/SelfTest/IntrospectiveTests/TextFlow.tests.cpp
+++ b/tests/SelfTest/IntrospectiveTests/TextFlow.tests.cpp
@@ -381,6 +381,81 @@ TEST_CASE( "TextFlow::AnsiSkippingString substrings properly",
     }
 }
 
+TEST_CASE( "TextFlow::AnsiSkippingString counts UTF-8 codepoints",
+           "[TextFlow][ansiskippingstring][approvals]" ) {
+    SECTION( "2-byte codepoints" ) {
+        AnsiSkippingString str( "\xC3\xA4\xC3\xB6\xC3\xBC" ); // äöü
+        CHECK( str.size() == 3 );
+    }
+    SECTION( "3-byte codepoints" ) {
+        AnsiSkippingString str( "\xE4\xB8\xAD\xE6\x96\x87" ); // 中文
+        CHECK( str.size() == 2 );
+    }
+    SECTION( "4-byte codepoints" ) {
+        // U+1F600 U+1F60E
+        AnsiSkippingString str( "\xF0\x9F\x98\x80\xF0\x9F\x98\x8E" );
+        CHECK( str.size() == 2 );
+    }
+    SECTION( "mixed ASCII and UTF-8" ) {
+        AnsiSkippingString str( "a\xC3\xA4" "b" ); // aäb
+        CHECK( str.size() == 3 );
+    }
+    SECTION( "UTF-8 with ANSI escapes" ) {
+        AnsiSkippingString str( "\033[31m\xC3\xA4\xC3\xB6\xC3\xBC\033[0m" );
+        CHECK( str.size() == 3 );
+    }
+}
+
+TEST_CASE( "TextFlow::AnsiSkippingString iterates UTF-8 codepoints",
+           "[TextFlow][ansiskippingstring][approvals]" ) {
+    // "aäb" = 'a' (0x61), 'ä' (0xC3 0xA4), 'b' (0x62)
+    std::string text = "a\xC3\xA4" "b";
+    AnsiSkippingString str( text );
+
+    SECTION( "forward iteration has correct count" ) {
+        int count = 0;
+        for ( auto it = str.begin(); it != str.end(); ++it ) {
+            ++count;
+        }
+        CHECK( count == 3 );
+    }
+    SECTION( "backward iteration has correct count" ) {
+        auto it = str.end();
+        int count = 0;
+        while ( it != str.begin() ) {
+            --it;
+            ++count;
+        }
+        CHECK( count == 3 );
+    }
+    SECTION( "substring preserves full UTF-8 bytes" ) {
+        auto a = str.begin();
+        auto b = str.begin();
+        ++b; // past 'a'
+        ++b; // past 'ä'
+        CHECK( str.substring( a, b ) == "a\xC3\xA4" );
+    }
+}
+
+TEST_CASE( "TextFlow::Column wraps UTF-8 text correctly",
+           "[TextFlow][column][approvals]" ) {
+    // "äöü äöü äöü" = 11 codepoints, 17 bytes
+    Column col( "\xC3\xA4\xC3\xB6\xC3\xBC \xC3\xA4\xC3\xB6\xC3\xBC \xC3\xA4\xC3\xB6\xC3\xBC" );
+
+    SECTION( "width=8" ) {
+        col.width( 8 );
+        // 7 visible codepoints "äöü äöü" fit, then wrap
+        REQUIRE( as_written( col ) ==
+                 "\xC3\xA4\xC3\xB6\xC3\xBC \xC3\xA4\xC3\xB6\xC3\xBC\n"
+                 "\xC3\xA4\xC3\xB6\xC3\xBC" );
+    }
+    SECTION( "width=80" ) {
+        col.width( 80 );
+        REQUIRE( as_written( col ) ==
+                 "\xC3\xA4\xC3\xB6\xC3\xBC \xC3\xA4\xC3\xB6\xC3\xBC \xC3\xA4\xC3\xB6\xC3\xBC" );
+    }
+}
+
 TEST_CASE( "TextFlow::Column skips ansi escape sequences",
            "[TextFlow][column][approvals]" ) {
     std::string text = "\033[38;2;98;174;239m\033[38;2;198;120;221mThe quick brown \033[38;2;198;120;221mfox jumped over the lazy dog\033[0m";