fix(frontend): handle conversion between utf8 and utf16 strings based on llvm::ConvertUTF (#116)

Author: KKiiim

frontend/CompilerImpl.cpp

@@ -9,7 +9,6 @@
 #include <map>
 #include <optional>
 #include <regex>
-#include <sstream>
 #include <string>
 #include <string_view>
 #include <utility>
@@ -18,6 +17,7 @@
 #include "ASC/ASC.hpp"
 #include "CompilerImpl.hpp"
 #include "LinkedAPI.hpp"
+#include "llvm/Support/ConvertUTF.h"
 #include "warpo/frontend/Compiler.hpp"
 #include "warpo/support/Debug.hpp"
 #include "warpo/support/FileSystem.hpp"
@@ -42,15 +42,14 @@ enum WasmFFIBool : uint32_t { WASM_FALSE = 0, WASM_TRUE = 1 };
 
 } // namespace
 int32_t FrontendCompiler::allocString(std::string_view str) {
-  // FIXME: convert utf8 to utf16 need library
-  int32_t const ptr = m.callExportedFunctionWithName<1>(stackTop, "__new", static_cast<int32_t>(str.size() * 2U),
+  std::u16string utf16Str = utf8ToUtf16(std::string(str));
+  int32_t const ptr = m.callExportedFunctionWithName<1>(stackTop, "__new", static_cast<int32_t>(utf16Str.size() * 2U),
                                                         static_cast<int32_t>(2))[0]
                           .i32;
   m.callExportedFunctionWithName<1>(stackTop, "__pin", ptr);
-  uint8_t *const stringBegin = m.getLinearMemoryRegion(static_cast<uint32_t>(ptr), str.size() * 2U);
-  for (size_t i = 0; i < str.size(); i++) {
-    stringBegin[i * 2U] = str[i];
-  }
+  uint8_t *const stringBegin = m.getLinearMemoryRegion(static_cast<uint32_t>(ptr), utf16Str.size());
+  std::memcpy(stringBegin, utf16Str.data(), utf16Str.size() * sizeof(char16_t));
+
   return ptr;
 }
 
@@ -70,14 +69,47 @@ std::string FrontendCompiler::getAsString(int32_t ptr) {
   uint32_t size = 0;
   std::memcpy(&size, header + 16, sizeof(size));
   uint8_t const *content = m.getLinearMemoryRegion(ptr, size);
-  size /= 2U;
-  std::stringstream ss{};
-  for (uint32_t i = 0; i < size; ++i) {
-    ss << content[i * 2U];
-  }
-  return std::move(ss).str();
+
+  std::u16string utf16Str;
+  utf16Str.resize(size / 2);
+  std::memcpy(utf16Str.data(), content, size);
+  return utf16ToUtf8(utf16Str);
 };
 
+std::u16string FrontendCompiler::utf8ToUtf16(std::string const &utf8Str) {
+  if (utf8Str.empty())
+    return std::u16string();
+  const llvm::UTF8 *src = reinterpret_cast<const llvm::UTF8 *>(utf8Str.data());
+  const llvm::UTF8 *srcEnd = src + utf8Str.size();
+  std::u16string utf16Str;
+  utf16Str.resize(utf8Str.size());
+  llvm::UTF16 *dst = reinterpret_cast<llvm::UTF16 *>(utf16Str.data());
+  llvm::UTF16 *dstEnd = dst + utf16Str.size();
+
+  if (llvm::ConvertUTF8toUTF16(&src, srcEnd, &dst, dstEnd, llvm::strictConversion) != llvm::conversionOK)
+    throw std::runtime_error("UTF8 to UTF16 conversion failed");
+  // Resize the string to the actual number of UTF-16 code units written
+  utf16Str.resize(dst - reinterpret_cast<llvm::UTF16 *>(utf16Str.data()));
+  return utf16Str;
+}
+
+std::string FrontendCompiler::utf16ToUtf8(std::u16string const &utf16Str) {
+  if (utf16Str.empty())
+    return std::string();
+  const llvm::UTF16 *src = reinterpret_cast<const llvm::UTF16 *>(utf16Str.data());
+  const llvm::UTF16 *srcEnd = src + utf16Str.size();
+  std::string utf8Str;
+  utf8Str.resize(utf16Str.size() * 4); // UTF-8 can be up to 4 bytes per Unicode code point
+  llvm::UTF8 *dst = reinterpret_cast<llvm::UTF8 *>(utf8Str.data());
+  llvm::UTF8 *dstEnd = dst + utf8Str.size();
+
+  if (llvm::ConvertUTF16toUTF8(&src, srcEnd, &dst, dstEnd, llvm::strictConversion) != llvm::conversionOK)
+    throw std::runtime_error("UTF16 to UTF8 conversion failed");
+  // Resize the string to the actual number of UTF-8 bytes written
+  utf8Str.resize(dst - reinterpret_cast<llvm::UTF8 *>(utf8Str.data()));
+  return utf8Str;
+}
+
 using PackageResolveResult = std::optional<std::pair<std::string, std::optional<std::string>>>;
 
 static PackageResolveResult getPackageName(std::string const &fileInternalPath) {

frontend/CompilerImpl.hpp

@@ -31,6 +31,8 @@ class FrontendCompiler {
   std::string errorMessage_;
 
   int32_t allocString(std::string_view str);
+  std::u16string utf8ToUtf16(std::string const &utf8Str);
+  std::string utf16ToUtf8(std::u16string const &utf16Str);
 
   void parseFile(int32_t const program, std::optional<std::string> const &code, std::string_view path, IsEntry isEntry);
 
@@ -60,7 +62,6 @@ class FrontendCompiler {
   FrontendCompiler(Config const &config);
   ~FrontendCompiler();
 
-
   CompilationResult compile(std::vector<std::string> const &entryFilePaths, Config const &config);
 
   std::set<void *> allocedPtrs_;