rtlib: Fix wstring-to-zstring conversion functions

This fixes fb_wstr_ConvToA() and also improves fb_wstr_ConvFromA() in the
process.

1. fb_wstr_ConvToA() was definitely broken - it should use wcstombs(), but
   that code was commented out, and it only converted ASCII chars, turning
   everything else into '?' chars.
   Looks like the only reason it was commented out is that the call might
   fail. Well, we can use it, and in case it fails, then we can fall back
   to the ASCII-only conversion.

2. fb_wstr_ConvFromA() used mbstowcs(), but didn't do ASCII-only conversion
   as a fallback. Now it does - seems like it's better to return something
   instead of nothing.

3. FB_WSTR_WCHARTOCHAR macro stuff removed, the two functions are now
   implemented in the proper module instead of fb_unicode.h, parameters
   re-ordered to indicate that dst/dst_chars belong together, while src
   must be null-terminated.

4. fb_WstrAssignToAEx() now allocates a big dst string, in the hopes to
   have enough room for the decoded multi-byte string (which can be UTF8
   on GNU/Linux). Previously it would simply allocate the same amount of
   "chars" for dst as were present in src, but the conversion isn't
   guaranteed to be 1:1 in terms of chars. Now it will probably often
   allocate more memory than needed, but in exchange the string is much
   less likely to be truncated.

Unfortunately it is nearly impossible to test this stuff in a portable way.
The Unicode <-> codepage conversions are lossy and depend on the system
codepage at run-time.

Author: dkl

changelog.txt

@@ -14,6 +14,7 @@ Version 1.05.0
 - BYREF fixed-length strings (e.g. BYREF myParameter AS ZSTRING * 10) now trigger a compiler error since they are unsupported
 - #print typeof() output now differentiates between ZSTRING and ZSTRING * N (ZSTRING without size is produced by dereferencing a ZSTRING PTR, or BYREF AS ZSTRING)
 - Context-specific keywords, e.g. graphics PUT modes, must now be given as keywords (e.g. PSET), string literals (e.g. "PSET") are no longer accepted.
+- Wstring-to-Zstring conversions didn't use the system's Unicode <-> codepage conversion function, and only converted ASCII characters. Now it will try to convert the Unicode chars to codepage chars.
 
 
 Version 1.04.0

src/gfxlib2/gfx_print_wstr.c

@@ -9,7 +9,7 @@ void fb_GfxPrintBufferWstrEx(const FB_WCHAR *buffer, size_t len, int mask)
 	char temp[len + 1];
 
 	if( len > 0 )
-		fb_wstr_ConvToA( temp, buffer, len );
+		fb_wstr_ConvToA( temp, len, buffer );
 	else
 		*temp = '\0';
 

src/rtlib/dev_file_write_wstr.c

@@ -24,7 +24,7 @@ int fb_DevFileWriteWstr( FB_FILE *handle, const FB_WCHAR* src, size_t chars )
 
 	/* convert to ascii, file should be opened with the ENCODING option
 	   to allow UTF characters to be written */
-	fb_wstr_ConvToA( buffer, src, chars );
+	fb_wstr_ConvToA( buffer, chars, src );
 
 	/* do write */
 	res = fwrite( (void *)buffer, 1, chars, fp ) == chars;

src/rtlib/dos/io_printbuff_wstr.c

@@ -10,7 +10,7 @@ void fb_ConsolePrintBufferWstrEx( const FB_WCHAR *buffer, size_t len, int mask )
     char *temp = alloca( len + 1 );
 
     if( len > 0 )
-    	fb_wstr_ConvToA( temp, buffer, len );
+        fb_wstr_ConvToA( temp, len, buffer );
     else
     	*temp = '\0';
 

src/rtlib/dos/io_printer.c

@@ -47,7 +47,7 @@ int fb_PrinterWriteWstr( DEV_LPT_INFO *devInfo, const FB_WCHAR *data, size_t len
     char *temp = alloca( length + 1 );
 
     if( length > 0 )
-    	fb_wstr_ConvToA( temp, data, length );
+        fb_wstr_ConvToA( temp, length, data );
     else
     	*temp = '\0';
 

src/rtlib/error_assert_wstr.c

@@ -6,7 +6,7 @@
 
 static void hConvToA( char *buffer, FB_WCHAR *expression )
 {
-	fb_wstr_ConvToA( buffer, expression, BUFFER_SIZE-1 );
+	fb_wstr_ConvToA( buffer, BUFFER_SIZE-1, expression );
 	buffer[BUFFER_SIZE-1] = 0; /* null terminator */
 }
 

src/rtlib/fb_unicode.h

@@ -39,7 +39,6 @@ typedef uint8_t  UTF_8;
 #	define wcstoull strtoull
 #	define wcschr   strchr
 #	define wcscspn  strcspn
-#	define FB_WSTR_WCHARTOCHAR fb_wstr_WcharToChar
 	static __inline__ size_t __dos_mbstowcs(FB_WCHAR *wcstr, const char *mbstr, size_t count)
 	{
 		return memcpy(wcstr,mbstr,count), count;
@@ -48,10 +47,6 @@ typedef uint8_t  UTF_8;
 	{
 		return memcpy(mbstr,wcstr,count), count;
 	}
-	static __inline__ void fb_wstr_WcharToChar( char *dst, const FB_WCHAR *src, ssize_t chars )
-	{
-		memcpy(dst,src,chars);
-	}
 	static __inline__ int swprintf(FB_WCHAR *buffer, size_t n, const FB_WCHAR *format, ...)
 	{
 		int result;
@@ -78,19 +73,6 @@ typedef uint8_t  UTF_8;
 #	else
 #		define FB_WEOF ((FB_WCHAR)-1)
 #	endif
-#	define FB_WSTR_WCHARTOCHAR fb_wstr_WcharToChar
-	static __inline__ void fb_wstr_WcharToChar( char *dst, const FB_WCHAR *src, ssize_t chars )
-	{
-		while( chars-- ) {
-			UTF_16 c = *src++;
-			if( c > 255 ) {
-				if( c >= UTF16_SUR_HIGH_START && c <= UTF16_SUR_HIGH_END )
-					++src;
-				c = '?';
-			}
-			*dst++ = c;
-		}
-	}
 #else
 #	define __USE_ISOC99 1
 #	define __USE_ISOC95 1
@@ -141,20 +123,6 @@ typedef uint8_t  UTF_8;
     swprintf( buffer, 16+8 + 1, _LC("%.16g"), (double) (num) )
 #endif
 
-#ifndef FB_WSTR_WCHARTOCHAR
-#define FB_WSTR_WCHARTOCHAR fb_wstr_WcharToChar
-static __inline__ void fb_wstr_WcharToChar( char *dst, const FB_WCHAR *src, ssize_t chars )
-{
-	while( chars ) {
-		UTF_32 c = *src++;
-		if( c > 255 )
-			c = '?';
-		*dst++ = c;
-		--chars;
-	}
-}
-#endif
-
 /* Calculate the number of characters between two pointers. */
 static __inline__ ssize_t fb_wstr_CalcDiff( const FB_WCHAR *ini, const FB_WCHAR *end )
 {
@@ -179,52 +147,8 @@ static __inline__ ssize_t fb_wstr_Len( const FB_WCHAR *s )
 	return wcslen( s );
 }
 
-static __inline__ void fb_wstr_ConvFromA( FB_WCHAR *dst, ssize_t dst_chars, const char *src )
-{
-	ssize_t chars;
-
-	/* NULL? */
-	if( src == NULL ) {
-		chars = -1;
-	} else {
-		/* plus the null-term (note: "n" in chars, not bytes!) */
-		chars = mbstowcs( dst, src, dst_chars + 1 );
-	}
-
-	/* error? */
-	if( chars == -1 )
-		*dst = _LC('\0');
-
-	/* if there's no enough space in dst the null-term won't be added? */
-	else if( chars == (dst_chars + 1) )
-		dst[dst_chars] = _LC('\0');
-}
-
-static __inline__ void fb_wstr_ConvToA( char *dst, const FB_WCHAR *src, ssize_t chars )
-{
-	/* !!!FIXME!!! wcstombs() will fail and not emit '?' or such if the
-	   characters are above 255 and can't be converted? not good.. */
-#if 0
-	ssize_t bytes;
-
-	/* plus the null-term */
-	bytes = wcstombs( dst, src, chars + 1 );
-
-	/* error? */
-	if( bytes == -1 )
-		*dst = '\0';
-
-	/* if there's no enough space in dst the null-term won't be added? */
-	else if( bytes == chars + 1 )
-		dst[src_chars] = '\0';
-
-#else
-	FB_WSTR_WCHARTOCHAR( dst, src, chars );
-
-	/* plus the null-term */
-	dst[chars] = '\0';
-#endif
-}
+ssize_t fb_wstr_ConvFromA( FB_WCHAR *dst, int dst_chars, const char *src );
+ssize_t fb_wstr_ConvToA( char *dst, int dst_chars, const FB_WCHAR *src );
 
 static __inline__ int fb_wstr_IsLower( FB_WCHAR c )
 {

src/rtlib/strw_convassign.c

@@ -38,6 +38,13 @@ FBCALL FB_WCHAR *fb_WstrAssignFromA
 	return dst;
 }
 
+/* We'll convert wide-string to multi-byte string -- we don't know
+   how big the result will be, but we can make a good guess.
+   In the worst case, we'll allocate too much. */
+/* 4 bytes per char - allowing for UTF8 multi-byte strings, happens on GNU/Linux.
+   On Windows there can be double-byte encodings etc. too. */
+#define getALenForWLen(wlen) ((wlen) * 4)
+
 FBCALL void *fb_WstrAssignToAEx
 	(
 		void *dst,
@@ -77,44 +84,44 @@ FBCALL void *fb_WstrAssignToAEx
 		else
 		{
         	/* realloc dst if needed and copy src */
+			dst_chars = getALenForWLen(src_chars);
 			if( is_init == FB_FALSE )
 			{
-				if( FB_STRSIZE( dst ) != src_chars )
-					fb_hStrRealloc( (FBSTRING *)dst, src_chars, FB_FALSE );
+				if( FB_STRSIZE( dst ) != dst_chars )
+					fb_hStrRealloc( (FBSTRING *)dst, dst_chars, FB_FALSE );
 			}
 			else
 			{
-				fb_hStrAlloc( (FBSTRING *)dst, src_chars );
+				fb_hStrAlloc( (FBSTRING *)dst, dst_chars );
 			}
 
-			fb_wstr_ConvToA( ((FBSTRING *)dst)->data, src, src_chars );
+			ssize_t writtenchars = fb_wstr_ConvToA( ((FBSTRING *)dst)->data, dst_chars, src );
+			fb_hStrSetLength( dst, writtenchars );
 		}
 	}
 	/* fixed-len or zstring.. */
 	else
 	{
 		/* src NULL? */
-		if( src_chars == 0 )
-		{
-			*(char *)dst = '\0';
-		}
-		else
-		{
-			/* byte ptr? as in C, assume dst is large enough */
-			if( dst_chars == 0 )
-				dst_chars = src_chars;
-
-			fb_wstr_ConvToA( (char *)dst,
-							 src,
-							 (dst_chars <= src_chars? dst_chars : src_chars) );
-		}
-
-		/* fill reminder with null's */
-		if( fill_rem != 0 )
-		{
-			dst_chars -= src_chars;
-			if( dst_chars > 0 )
-				memset( &(((char *)dst)[src_chars]), 0, dst_chars );
+		if( src_chars == 0 ) {
+			if( fill_rem && dst_chars > 0 ) {
+				memset( dst, 0, dst_chars );
+			} else {
+				*(char *)dst = '\0';
+			}
+		/* byte ptr? as in C, assume dst is large enough */
+		} else if( dst_chars == 0 ) {
+			dst_chars = getALenForWLen(src_chars);
+			fb_wstr_ConvToA( (char *)dst, dst_chars, src );
+		} else {
+			dst_chars -= 1; /* null terminator */
+			ssize_t writtenchars = fb_wstr_ConvToA( (char *)dst, dst_chars, src );
+
+			/* fill remainder with null's */
+			if( fill_rem && writtenchars < dst_chars ) {
+				/* + 1 to fill behind null terminator. There is room for dst_chars + 1. */
+				memset( ((char *)dst) + writtenchars + 1, 0, dst_chars - writtenchars );
+			}
 		}
 	}
 

src/rtlib/strw_convfrom_str.c

@@ -2,6 +2,57 @@
 
 #include "fb.h"
 
+/* dst_chars == room in dst buffer without null terminator. Thus, the dst buffer
+   must be at least (dst_chars + 1) * sizeof(FB_WCHAR).
+   src must be null-terminated.
+   result = number of chars written, excluding null terminator that is always written */
+ssize_t fb_wstr_ConvFromA(FB_WCHAR *dst, ssize_t dst_chars, const char *src)
+{
+	if (src == NULL) {
+		*dst = _LC('\0');
+		return 0;
+	}
+
+#if defined HOST_DOS
+	ssize_t chars = strlen(src);
+	if (chars > dst_chars)
+		chars = dst_chars;
+
+	memcpy(dst, src, chars + 1);
+	return chars;
+#else
+	/* plus the null-term (note: "n" in chars, not bytes!) */
+	ssize_t chars = mbstowcs(dst, src, dst_chars + 1);
+
+	/* worked? */
+	if (chars >= 0) {
+		/* a null terminator won't be added if there was not
+		   enough space, so do it manually (this will cut off the last
+		   char, but what can you do) */
+		if (chars == (dst_chars + 1)) {
+			dst[dst_chars] = _LC('\0');
+			return dst_chars - 1;
+		}
+		return chars;
+	}
+
+	/* mbstowcs() failed; translate at least ASCII chars
+	   and write out '?' for the others */
+	FB_WCHAR *origdst = dst;
+	FB_WCHAR *dstlimit = dst + dst_chars;
+	while (dst < dstlimit) {
+		unsigned char c = *src++;
+		if (c == 0)
+			break;
+		if (c > 127)
+			c = '?';
+		*dst++ = c;
+	}
+	*dst = _LC('\0');
+	return dst - origdst;
+#endif
+}
+
 FBCALL FB_WCHAR *fb_StrToWstr( const char *src )
 {
 	FB_WCHAR *dst;

src/rtlib/strw_convto_str.c

@@ -2,6 +2,68 @@
 
 #include "fb.h"
 
+/* dst_chars == room in dst buffer without null terminator. Thus, the dst buffer
+   must be at least dst_chars+1 bytes.
+   src must be null-terminated.
+   result = number of chars written, excluding null terminator that is always written */
+ssize_t fb_wstr_ConvToA(char *dst, ssize_t dst_chars, const FB_WCHAR *src)
+{
+	if (src == NULL) {
+		*dst = '\0';
+		return 0;
+	}
+
+#if defined HOST_DOS
+	ssize_t chars = strlen(src);
+	if (chars > dst_chars)
+		chars = dst_chars;
+
+	memcpy(dst, src, chars + 1);
+	return chars;
+#else
+	/* plus the null-term */
+	ssize_t chars = wcstombs(dst, src, dst_chars + 1);
+
+	/* worked? */
+	if (chars >= 0) {
+		/* a null terminator won't be added if there was not
+		   enough space, so do it manually (this will cut off the last
+		   char, but what can you do) */
+		if (chars == (dst_chars + 1)) {
+			dst[dst_chars] = '\0';
+			return dst_chars - 1;
+		}
+		return chars;
+	}
+
+	/* wcstombs() failed; translate at least ASCII chars
+	   and write out '?' for the others */
+	char *origdst = dst;
+	char *dstlimit = dst + dst_chars;
+	while (dst < dstlimit) {
+#if defined HOST_WIN32
+		UTF_16 c = *src++;
+		if (c == 0)
+			break;
+		if (c > 127) {
+			if (c >= UTF16_SUR_HIGH_START && c <= UTF16_SUR_HIGH_END)
+				src++;
+			c = '?';
+		}
+#else
+		UTF_32 c = *src++;
+		if (c == 0)
+			break;
+		if (c > 127)
+			c = '?';
+#endif
+		*dst++ = c;
+	}
+	*dst = '\0';
+	return dst - origdst;
+#endif
+}
+
 FBCALL FBSTRING *fb_WstrToStr( const FB_WCHAR *src )
 {
 	FBSTRING *dst;
@@ -18,7 +80,7 @@ FBCALL FBSTRING *fb_WstrToStr( const FB_WCHAR *src )
 	if( dst == NULL )
 		return &__fb_ctx.null_desc;
 
-	fb_wstr_ConvToA( dst->data, src, chars );
+	fb_wstr_ConvToA( dst->data, chars, src );
 
 	return dst;
 }