Merge pull request #140 from NuiCpp/feat/use-utf8cpp

Replaced std codecvt with utfcpp library

Author: 5cript

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.16)
 
-project(nui VERSION 0.10.2)
+project(nui VERSION 1.1.0)
 
 set(NUI_SOURCE_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} CACHE INTERNAL "NUI source directory")
 
@@ -15,6 +15,7 @@ include(${CMAKE_CURRENT_LIST_DIR}/cmake/dependencies/fmt.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/cmake/dependencies/describe.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/cmake/dependencies/mp11.cmake)
 include(${CMAKE_CURRENT_LIST_DIR}/cmake/dependencies/nui_traits.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/cmake/dependencies/utfcpp.cmake)
 if (EMSCRIPTEN)
     include(${CMAKE_CURRENT_LIST_DIR}/cmake/frontend/emscripten.cmake)
 else()

README.md

@@ -53,3 +53,4 @@ Nui uses the following dependencies:
 - portable-file-dialogs https://github.com/samhocevar/portable-file-dialogs - [WTFPL LICENSE](https://github.com/samhocevar/portable-file-dialogs/blob/main/COPYING)
 - 5cript/roar https://github.com/5cript/roar - [BSL-1.0 LICENSE](https://github.com/5cript/roar/blob/master/LICENSE)
 - Nui/traits https://github.com/NuiCpp/traits - [CC0-1.0 LICENSE](https://github.com/NuiCpp/traits/blob/main/LICENSE)
+- nemtrif/utfcpp https://github.com/nemtrif/utfcpp - [BSL-1.0 license](https://github.com/nemtrif/utfcpp/blob/master/LICENSE)

cmake/dependencies/utfcpp.cmake

@@ -0,0 +1,14 @@
+option(NUI_FETCH_UTFCPP "Try FetchContent for utfcpp" ON)
+
+set(NUI_UTFCPP_GIT_REPOSITORY "https://github.com/nemtrif/utfcpp.git" CACHE STRING "utfcpp git repository")
+set(NUI_UTFCPP_GIT_TAG "v4.0.8" CACHE STRING "utfcpp git tag")
+
+include("${CMAKE_CURRENT_LIST_DIR}/../fetcher.cmake")
+
+nui_fetch_dependency(
+    LIBRARY_NAME utf8cpp
+    FIND OFF # Developer does not support this library being found
+    FETCH ${NUI_FETCH_UTFCPP}
+    GIT_REPOSITORY ${NUI_UTFCPP_GIT_REPOSITORY}
+    GIT_TAG ${NUI_UTFCPP_GIT_TAG}
+)

nui/include/nui/utility/utf.hpp

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <utf8.h>
+
+#include <iterator>
+
+namespace Nui
+{
+    namespace Detail
+    {
+        template <typename T>
+        concept IsStringCharTypeUtf16Compatible = requires(T str) {
+            typename T::value_type;
+            requires(sizeof(typename T::value_type) > 1);
+        };
+
+        template <typename T>
+        concept IsStringCharTypeUtf8Compatible = requires(T str) {
+            typename T::value_type;
+            requires(sizeof(typename T::value_type) == 1);
+        };
+    }
+
+    template <typename Utf16StringType, typename Utf8StringType>
+    requires(Detail::IsStringCharTypeUtf16Compatible<Utf16StringType>) &&
+        (Detail::IsStringCharTypeUtf8Compatible<Utf8StringType>)
+    Utf16StringType utf8ToUtf16(Utf8StringType const& str)
+    {
+        Utf16StringType result;
+        utf8::utf8to16(str.begin(), str.end(), std::back_inserter(result));
+        return result;
+    }
+
+    template <typename Utf16StringType, typename Utf8StringType>
+    requires(Detail::IsStringCharTypeUtf16Compatible<Utf16StringType>) &&
+        (Detail::IsStringCharTypeUtf8Compatible<Utf8StringType>)
+    Utf8StringType utf16ToUtf8(Utf16StringType const& str)
+    {
+        Utf8StringType result;
+        utf8::utf16to8(str.begin(), str.end(), std::back_inserter(result));
+        return result;
+    }
+}

nui/include/nui/utility/widen.hpp

@@ -30,6 +30,7 @@ target_link_libraries(
     nui-backend
     PRIVATE
         project-settings
+        utf8cpp
     PUBLIC
         traits-library
         fmt

nui/src/nui/backend/CMakeLists.txt

@@ -8,13 +8,13 @@ namespace Nui
 
         if (options.browserArguments)
         {
-            const auto wideArgs = widenString(*options.browserArguments);
+            const auto wideArgs = utf8ToUtf16<std::wstring, std::string>(*options.browserArguments);
             environmentOptions->put_AdditionalBrowserArguments(wideArgs.c_str());
         }
 
         if (options.language)
         {
-            const auto wideLanguage = widenString(*options.language);
+            const auto wideLanguage = utf8ToUtf16<std::wstring, std::string>(*options.language);
             environmentOptions->put_Language(wideLanguage.c_str());
         }
 
@@ -32,15 +32,15 @@ namespace Nui
 
             for (const auto& customScheme : options.customSchemes)
             {
-                wideSchemes.push_back(widenString(customScheme.scheme));
+                wideSchemes.push_back(utf8ToUtf16<std::wstring, std::string>(customScheme.scheme));
                 customSchemeRegistrations.push_back(
                     Microsoft::WRL::Make<CoreWebView2CustomSchemeRegistration>(wideSchemes.back().c_str()));
                 auto& customSchemeRegistration = customSchemeRegistrations.back();
 
                 allowedOrigins.push_back({});
                 allowedOrigins.back().reserve(customScheme.allowedOrigins.size());
                 for (const auto& allowedOrigin : customScheme.allowedOrigins)
-                    allowedOrigins.back().push_back(widenString(allowedOrigin));
+                    allowedOrigins.back().push_back(utf8ToUtf16<std::wstring, std::string>(allowedOrigin));
 
                 allowedOriginsRaw.push_back({});
                 allowedOriginsRaw.back().reserve(allowedOrigins.back().size());

nui/src/nui/backend/environment_options_from_window_options.ipp

@@ -3,8 +3,7 @@
 #include <nui/backend/filesystem/special_paths.hpp>
 #include <nui/backend/filesystem/file_dialog.hpp>
 #include <nui/utility/scope_exit.hpp>
-#include <nui/utility/widen.hpp>
-#include <nui/utility/scope_exit.hpp>
+#include <nui/utility/utf.hpp>
 #include <nui/data_structures/selectables_registry.hpp>
 #include <nui/screen.hpp>
 #include "load_file.hpp"
@@ -347,7 +346,7 @@ namespace Nui
                 windowsServeThroughAuthority = std::string{windowsServeAuthority};
             const auto authority = *windowsServeThroughAuthority;
 
-            const std::wstring filter = widenString("https://"s + authority + "/index.html");
+            const auto filter = utf8ToUtf16<std::wstring, std::string>("https://"s + authority + "/index.html");
 
             auto result = webView->AddWebResourceRequestedFilter(
                 filter.c_str(), static_cast<COREWEBVIEW2_WEB_RESOURCE_CONTEXT>(NuiCoreWebView2WebResourceContext::All));
@@ -370,7 +369,7 @@ namespace Nui
                             webViewRequest->get_Uri(&uri);
                             std::wstring uriW{uri};
                             CoTaskMemFree(uri);
-                            return shortenString(uriW);
+                            return utf16ToUtf8<std::wstring, std::string>(uriW);
                         }();
 
                         if (uri != compareUri)
@@ -697,8 +696,8 @@ namespace Nui
                 break;
         }
 
-        const auto wideHost = widenString(hostName);
-        const auto widePath = widenString(folderPath.string());
+        const auto wideHost = utf8ToUtf16<std::wstring, std::string>(hostName);
+        const auto widePath = utf8ToUtf16<std::wstring, std::string>(folderPath.string());
         wv23->SetVirtualHostNameToFolderMapping(wideHost.c_str(), widePath.c_str(), nativeAccessKind);
 #elif defined(__APPLE__)
         (void)accessKind;

nui/src/nui/backend/window.cpp

@@ -39,7 +39,7 @@ namespace Nui
 
         for (auto const& customScheme : options.customSchemes)
         {
-            const std::wstring filter = widenString(customScheme.scheme + ":*");
+            const std::wstring filter = utf8ToUtf16<std::wstring, std::string>(customScheme.scheme + ":*");
 
             auto result = webView->AddWebResourceRequestedFilter(
                 filter.c_str(), static_cast<COREWEBVIEW2_WEB_RESOURCE_CONTEXT>(NuiCoreWebView2WebResourceContext::All));
@@ -79,7 +79,7 @@ namespace Nui
             webViewRequest->get_Uri(&uri);
             std::wstring uriW{uri};
             CoTaskMemFree(uri);
-            return shortenString(uriW);
+            return utf16ToUtf8<std::wstring, std::string>(uriW);
         }();
 
         const auto customScheme = [&schemes, &uri]() -> std::optional<CustomScheme> {
@@ -126,7 +126,8 @@ namespace Nui
 
         std::wstring responseHeaders;
         for (auto const& [key, value] : responseData.headers)
-            responseHeaders += widenString(key) + L": " + widenString(value) + L"\r\n";
+            responseHeaders += utf8ToUtf16<std::wstring, std::string>(key) + L": " +
+                utf8ToUtf16<std::wstring, std::string>(value) + L"\r\n";
         if (!responseHeaders.empty())
         {
             responseHeaders.pop_back();
@@ -137,7 +138,7 @@ namespace Nui
         stream.Attach(SHCreateMemStream(
             reinterpret_cast<const BYTE*>(responseData.body.data()), static_cast<UINT>(responseData.body.size())));
 
-        const auto phrase = widenString(responseData.reasonPhrase);
+        const auto phrase = utf8ToUtf16<std::wstring, std::string>(responseData.reasonPhrase);
         result = environment->CreateWebResourceResponse(
             stream.Get(), responseData.statusCode, phrase.c_str(), responseHeaders.c_str(), &response);
 
@@ -191,7 +192,9 @@ namespace Nui
                         CoTaskMemFree(name);
                         CoTaskMemFree(value);
 
-                        headersMap.emplace(shortenString(nameW), shortenString(valueW));
+                        headersMap.emplace(
+                            utf16ToUtf8<std::wstring, std::string>(nameW),
+                            utf16ToUtf8<std::wstring, std::string>(valueW));
 
                         BOOL hasNext = FALSE;
                         if (FAILED(iterator->MoveNext(&hasNext)) || !hasNext)
@@ -206,7 +209,7 @@ namespace Nui
                     webViewRequest->get_Method(&method);
                     std::wstring methodW{method};
                     CoTaskMemFree(method);
-                    return shortenString(methodW);
+                    return utf16ToUtf8<std::wstring, std::string>(methodW);
                 }(),
             .resourceContext = static_cast<NuiCoreWebView2WebResourceContext>(resourceContext),
         };