bugfix: sf.net #897 length of literal wstring is miscalculated at compile time. Compile time evaluation of len(!"\u1234") was using the internal escaped string length instead of the actual codepoint length

Author: jayrm

changelog.txt

@@ -10,6 +10,7 @@ Version 1.07.0
 - push/pop correct GL_PROJECTION matrix and GL_MODELVIEW matrix when setting up graphics screen (gothon)
 - fix char & wchar concatentation.  For char double byte characters (such as Chinese characters), the "wstr = char & wchar" would become "wstr = char & chrw(0) & wchar" (SkyFish)
 - sf.net #794: literal tabs in string literals miscompiled if followed by 0-9 numeric chars
+- sf.net #897: length of literal wstring is miscalculated at compile time.  Compile time evaluation of len(!"\u1234") was using the internal escaped string length instead of the actual codepoint length
 
 
 Version 1.06.0

src/compiler/parser-quirk-math.bas

@@ -74,7 +74,10 @@ private function hLen _
 		end if
 
 		'' String literal, evaluate at compile-time
-		lgt = symbGetWstrLen( litsym ) - 1
+		'' symbGetStrLen( litsym ) will return the number of codepoints
+		'' that are used to store the escaped WSTRING literal, when what
+		'' we really want is the number od codepoints unescaped.
+		lgt = len( *hUnescapeW( symbGetVarLitTextW( litsym ) ) )
 
 	case FB_DATATYPE_FIXSTR
 		'' len( fixstr ) returns the N from STRING * N, i.e. it works

tests/wstring/len.bas

@@ -2,27 +2,70 @@
 
 SUITE( fbc_tests.wstring_.len_ )
 
-	const TEST_LIT = "abcdef"
+	'' len( [w]str( literal ) ) is evaluated at compile time
+	'' check that the compile time calculation matches the
+	'' run time library calculation
+
+	const LIT_A1 = "abcdef"
+	const LIT_W1 = wstr( "abcdef" )
+	const LIT_W2 = !"bcd\u0065\u0066\u0067"
+	const LIT_W3 = !"\u0063\u0064\u0065\u0065\u0067\u0068"
+
+	const CODE_LEN = 6
+
 	const TEST_LEN = 32
 	const TEST_SIZ = TEST_LEN * len( wstring )
 
-	TEST( default )
-
-		dim s as wstring * TEST_LEN
-		dim ps as wstring ptr
-		
-		s = TEST_LIT
-		ps = @s
-		
-		CU_ASSERT( len( s ) = len( TEST_LIT ) )
-		
-		CU_ASSERT( sizeof( s ) = TEST_SIZ )
-		
-		CU_ASSERT( len( *ps ) = len( TEST_LIT ) )
-		
-		CU_ASSERT( len( ps ) = len( any ptr ) )
-		
-		CU_ASSERT( sizeof( ps ) = sizeof( any ptr ) )
+	#macro do_test( init_string )
+
+		scope
+			dim s as wstring * TEST_LEN
+			dim ps as wstring ptr
+			
+			s = init_string
+			ps = @s
+			
+			CU_ASSERT( len( s ) = CODE_LEN )
+			CU_ASSERT( len( *ps ) = CODE_LEN )
+
+			CU_ASSERT( len( s ) = len( init_string ) )
+			
+			CU_ASSERT( sizeof( s ) = TEST_SIZ )
+			
+			CU_ASSERT( len( *ps ) = len( init_string ) )
+			
+			CU_ASSERT( len( ps ) = len( any ptr ) )
+			
+			CU_ASSERT( sizeof( ps ) = sizeof( any ptr ) )
+		end scope
+
+	#endmacro
+
+	TEST( literal )
+
+		do_test( "abcdef" )
+		do_test( wstr( "abcdef" ) )
+		do_test( !"bcd\u0065\u0066\u0067" )
+		do_test( !"\u0063\u0064\u0065\u0065\u0067\u0068" )
+
+		do_test( LIT_A1 )
+		do_test( LIT_W1 )
+		do_test( LIT_W2 )
+		do_test( LIT_W3 )
+
+	END_TEST
+
+	TEST( literal_const )
+
+		do_test( "abcdef" )
+		do_test( wstr( "abcdef" ) )
+		do_test( !"bcd\u0065\u0066\u0067" )
+		do_test( !"\u0063\u0064\u0065\u0065\u0067\u0068" )
+
+		do_test( LIT_A1 )
+		do_test( LIT_W1 )
+		do_test( LIT_W2 )
+		do_test( LIT_W3 )
 
 	END_TEST