Merge pull request #78 from Wires77/utf_learnings

Author: Wires77

CMakeLists.txt

@@ -54,8 +54,6 @@ set(SIMPLEGRAPHIC_SOURCES
     "engine/render/r_main.h"
     "engine/render/r_texture.cpp"
     "engine/render/r_texture.h"
-    # "engine/system/win/sys_console.cpp"
-    "engine/system/win/sys_console_unix.cpp"
     "engine/system/win/sys_local.h"
     "engine/system/win/sys_main.cpp"
     "engine/system/win/sys_opengl.cpp"
@@ -85,6 +83,16 @@ if (APPLE)
     )
 endif()
 
+if (WIN32)
+    set (SIMPLEGRAPHIC_PLATFORM_SOURCES
+        "engine/system/win/sys_console.cpp"
+    )
+else()
+    set (SIMPLEGRAPHIC_PLATFORM_SOURCES
+        "engine/system/win/sys_console_unix.cpp"
+    )
+endif()
+
 add_library(SimpleGraphic SHARED
     ${SIMPLEGRAPHIC_SOURCES}
     ${SIMPLEGRAPHIC_PLATFORM_SOURCES}

config.h

@@ -9,7 +9,7 @@
 #define CFG_LOGFILE		"SimpleGraphic/SimpleGraphic.log"
 #define CFG_DATAPATH	"SimpleGraphic/"
 
-#define CFG_SCON_TITLE	"SimpleGraphic Console"
+#define CFG_SCON_TITLE	L"SimpleGraphic Console"
 #define CFG_SCON_TEXTCOL 0x00FFFFFF	// 255 255 255
 #define CFG_SCON_TEXTBG 0x001A3400	//   0  52  26
 #define CFG_SCON_WINBG 0x00000000	//   0   0   0

engine/common/common.cpp

@@ -479,7 +479,7 @@ IndexedUTF32String IndexUTF8ToUTF32(std::string_view input)
 			auto p0 = (uint32_t)b[0] & 0b111;
 			auto p1 = (uint32_t)b[1] & 0b11'1111;
 			auto p2 = (uint32_t)b[2] & 0b11'1111;
-			auto p3 = (uint32_t)b[2] & 0b11'1111;
+			auto p3 = (uint32_t)b[3] & 0b11'1111;
 			codepoint = p0 << 18 | p1 << 12 | p2 << 6 | p3;
 			byteIdx += 4;
 		}

engine/system/win/sys_console.cpp

@@ -48,7 +48,7 @@ class sys_console_c: public sys_IConsole, public conPrintHook_c, public thread_c
 	void	RunMessages(HWND hwnd = nullptr);
 	void	ThreadProc();
 
-	void	Print(const char* text);
+	void	Print(std::u32string_view text);
 	void	CopyToClipboard();
 
 	void	ConPrintHook(const char* text);
@@ -118,16 +118,16 @@ void sys_console_c::ThreadProc()
 	if (RegisterClass(&conClass) == 0) exit(0);
 
 	// Create the system console window
-	hwMain = CreateWindowEx(
+	hwMain = CreateWindowExW(
 		0, CFG_SCON_TITLE " Class", CFG_SCON_TITLE, SCON_STYLE, 
 		wrec.left, wrec.top, wrec.right - wrec.left, wrec.bottom - wrec.top,
 		NULL, NULL, sys->hinst, NULL
 	);
 	SetWindowLongPtr(hwMain, GWLP_USERDATA, (LONG_PTR)this);
 
 	// Populate window
-	hwOut = CreateWindowEx(
-		WS_EX_CLIENTEDGE, "EDIT", "", WS_VISIBLE | WS_CHILD | WS_BORDER | WS_VSCROLL | ES_MULTILINE | ES_READONLY,
+	hwOut = CreateWindowExW(
+		WS_EX_CLIENTEDGE, L"EDIT", L"", WS_VISIBLE | WS_CHILD | WS_BORDER | WS_VSCROLL | ES_MULTILINE | ES_READONLY,
 		10, 10, SCON_WIDTH - 20, SCON_HEIGHT - 20, 
 		hwMain, NULL, sys->hinst, NULL
 	);
@@ -138,7 +138,7 @@ void sys_console_c::ThreadProc()
 		FW_LIGHT, FALSE, FALSE, FALSE, 
 		DEFAULT_CHARSET, 
 		OUT_DEFAULT_PRECIS, CLIP_DEFAULT_PRECIS, DEFAULT_QUALITY, 
-		FIXED_PITCH|FF_MODERN, "Lucida Console"
+		FIXED_PITCH|FF_MODERN, L"Lucida Console"
 	);
 	SetWindowFont(hwOut, font, FALSE);
 	Edit_LimitText(hwOut, 0xFFFF);
@@ -173,7 +173,7 @@ void sys_console_c::ThreadProc()
 	isRunning = false;
 
 	// Flush windowless messages (Like WM_QUIT)
-	sys->RunMessages();
+	RunMessages();
 }
 
 sys_console_c::~sys_console_c()
@@ -227,9 +227,11 @@ void sys_console_c::SetVisible(bool show)
 			SetForegroundWindow(hwMain);
 
 			// Select all text and replace with full text
-			Edit_SetText(hwOut, "");
+			Edit_SetText(hwOut, L"");
 			char* buffer = sys->con->BuildBuffer();
-			Print(buffer);
+			std::u32string u32_text = IndexUTF8ToUTF32(buffer).text;
+
+			Print(u32_text);
 			delete buffer;
 
 			RunMessages(hwMain);
@@ -256,54 +258,72 @@ bool sys_console_c::IsVisible()
 
 void sys_console_c::SetTitle(const char* title)
 {
-	SetWindowText(hwMain, (title && *title)? title : CFG_SCON_TITLE);
+	WCHAR* text = WidenUTF8String(title);
+	SetWindowText(hwMain, (text && *text)? text : CFG_SCON_TITLE);
 }
 
-void sys_console_c::Print(const char* text)
+void sys_console_c::Print(std::u32string_view string_view_text)
 {
 	if ( !shown ) {
 		return;
 	}
 
 	int escLen;
+	const char32_t* text = string_view_text.data();
 
 	// Find the required buffer length
 	int len = 0;
 	for (int b = 0; text[b]; b++) {
-		if (text[b] == '\n') {
+		if (text[b] == U'\n') {
 			// Newline takes 2 characters
 			len+= 2;
 		} else if (escLen = IsColorEscape(&text[b])) {
 			// Skip colour escapes
 			b+= escLen - 1;
 		} else {
-			len++;
+			if (text[b] > UINT16_MAX) {
+				// Higher codepoints will be separated into surrogate pairs
+				len += 2;
+			}
+			else {
+				len++;
+			}
 		}
 	}
 
 	// Parse into the buffer
-	char* winText = AllocStringLen(len);
-	char* p = winText;
+	char16_t* winText = new char16_t[len + 1];
+	char16_t* p = winText;
 	for (int b = 0; text[b]; b++) {
-		if (text[b] == '\n') {
+		if (text[b] == L'\n') {
 			// Append newline
-			*(p++) = '\r';
-			*(p++) = '\n';
+			*(p++) = L'\r';
+			*(p++) = L'\n';
 		} else if (escLen = IsColorEscape(&text[b])) {
 			// Skip colour escapes
 			b+= escLen - 1;
 		} else {
 			// Add character
-			*(p++) = text[b];
+			if (text[b] > UINT16_MAX) { // Outside the BMP
+				char16_t high_surrogate = ((text[b] - 0x10000) / 0x400) + 0xD800;
+				char16_t low_surrogate = ((text[b] - 0x10000) % 0x400) + 0xDC00;
+				*(p++) = high_surrogate;
+				*(p++) = low_surrogate;
+			}
+			else {
+				*(p++) = (char16_t)text[b];
+			}
+			
 		}
 	}
+	winText[len] = 0;
 
 	// Append to the output
 	Edit_SetSel(hwOut, Edit_GetTextLength(hwOut), -1);
 	Edit_ReplaceSel(hwOut, winText);
 	Edit_Scroll(hwOut, 0xFFFF, 0);
 	RunMessages(hwMain);
-	delete winText;
+	delete[] winText;
 }
 
 void sys_console_c::CopyToClipboard()
@@ -312,8 +332,8 @@ void sys_console_c::CopyToClipboard()
 	if (len) {
 		HGLOBAL hg = GlobalAlloc(GMEM_MOVEABLE, len + 1);
 		if ( !hg ) return;
-		char* cp = (char*)GlobalLock(hg);
-		GetWindowText(hwOut, cp, len + 1);
+		WCHAR* cp = (WCHAR*)GlobalLock(hg);
+		GetWindowTextW(hwOut, cp, len + 1);
 		GlobalUnlock(hg);
 		OpenClipboard(hwMain);
 		EmptyClipboard();
@@ -324,10 +344,10 @@ void sys_console_c::CopyToClipboard()
 
 void sys_console_c::ConPrintHook(const char* text)
 {
-	Print(text);
+	Print(IndexUTF8ToUTF32(text).text);
 }
 
 void sys_console_c::ConPrintClear()
 {
-	Edit_SetText(hwOut, "");
+	Edit_SetText(hwOut, L"");
 }