Fixed utf-8 issue (#166)

* Added user provided unit test

* Fix issue with escape_json

---------

Co-authored-by: Denis Avvakumov <[email protected]>

Author: bakwc

binlog_json_parser/CMakeLists.txt

@@ -5,9 +5,19 @@ project(binlog_json_parser)
 set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
+include(CheckIPOSupported)
+include(CheckCSourceCompiles)
+check_ipo_supported(RESULT IPO_SUPPORTED OUTPUT IPO_OUTPUT)
+
+if(IPO_SUPPORTED)
+    set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+    message(STATUS "Interprocedural optimization (IPO/LTO) enabled globally.")
+else()
+    message(STATUS "IPO/LTO is not supported: ${IPO_OUTPUT}")
+endif()
+
 # Check if the build type is Release
 if(CMAKE_BUILD_TYPE STREQUAL "Release")
-
     # Set optimization level to -O3 for release builds
     if(NOT CMAKE_CXX_FLAGS_RELEASE MATCHES "-O")
         set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
@@ -28,28 +38,47 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
                 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -march=skylake")
             endif()
         endif()
+    elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64|ARM64")
+        if(USE_MARCH_NATIVE)
+            set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -march=native")
+        else()
+            if(NOT CMAKE_CXX_FLAGS_RELEASE MATCHES "march=")
+                set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -march=armv8.2-a")
+            endif()
+        endif()
     else()
-        message(WARNING "The -march option will not be set because the system is not x86 or x64.")
-    endif()
-
-    # Check for LTO support
-    include(CheckCXXCompilerFlag)
-
-    check_cxx_compiler_flag("-flto" COMPILER_SUPPORTS_LTO)
-
-    if(COMPILER_SUPPORTS_LTO)
-        message(STATUS "Link Time Optimization (LTO) is supported by the compiler.")
-        set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -flto")
-        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -flto")
-        set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -flto")
-    else()
-        message(WARNING "Link Time Optimization (LTO) is not supported by the compiler.")
+        message(WARNING "The -march option will not be set because the system is not x86, x64, or ARM64.")
     endif()
 
     # Export compile flags to a file
     file(WRITE "${CMAKE_BINARY_DIR}/compile_flags.txt" "CXXFLAGS: ${CMAKE_CXX_FLAGS_RELEASE}\n")
     file(APPEND "${CMAKE_BINARY_DIR}/compile_flags.txt" "LINKER_FLAGS: ${CMAKE_EXE_LINKER_FLAGS}\n")
-
 endif()
 
-add_library(mysqljsonparse SHARED mysqljsonparse.cpp mysql_json_parser.cpp)
+check_c_source_compiles("
+#include <features.h>
+#if defined(__GLIBC__)
+#error \"This is glibc, not musl\"
+#endif
+#include <stdio.h>
+int main() { return 0; }
+" IS_MUSL)
+
+option(ALPINE_STATIC "Force fully static build when using musl/Alpine" ${IS_MUSL})
+
+if(ALPINE_STATIC)
+    add_definitions(-D_FORTIFY_SOURCE=0)
+    message(STATUS "musl detected → producing shared library with static musl linking")
+    
+    add_library(mysqljsonparse SHARED mysqljsonparse.cpp mysql_json_parser.cpp)
+    target_link_options(mysqljsonparse PRIVATE 
+        -static
+        -static-libgcc 
+        -static-libstdc++
+        -fPIC
+    )
+    target_compile_options(mysqljsonparse PRIVATE -fPIC)
+else()
+    message(STATUS "musl not detected → building shared library with dynamic glibc")
+    add_library(mysqljsonparse SHARED mysqljsonparse.cpp mysql_json_parser.cpp)
+endif()

binlog_json_parser/Dockerfile

@@ -0,0 +1,31 @@
+FROM alpine:3.22 AS build
+
+RUN apk add --no-cache \
+        build-base \
+        clang \
+        lld \
+        llvm \
+        cmake \
+        ninja \
+        musl-dev \
+        binutils
+
+WORKDIR /src
+
+COPY . .
+
+RUN cmake -S . -B build -G Ninja \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_C_COMPILER=clang \
+        -DCMAKE_CXX_COMPILER=clang++ \
+        -DCMAKE_EXE_LINKER_FLAGS="-fuse-ld=lld" \
+        -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" \
+        && ninja -C build -v \
+        && strip --strip-unneeded build/libmysqljsonparse.so \
+        && echo "Library dependencies:" && ldd build/libmysqljsonparse.so
+
+FROM scratch AS artifact
+
+COPY --from=build /src/build/libmysqljsonparse.so /
+
+CMD [""]

binlog_json_parser/build_static.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+IMAGE_TAG=${1:-mysqljsonparse:alpine-static}
+ARTIFACT_BASE=${2:-libmysqljsonparse.so}
+
+if ! docker buildx version >/dev/null 2>&1; then
+    echo "[ERROR] Docker buildx is not available. Please install Docker Desktop or enable buildx." >&2
+    exit 1
+fi
+
+BUILDER_NAME="multiarch-builder"
+if ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then
+    echo "[INFO] Creating buildx builder '$BUILDER_NAME'..."
+    docker buildx create --name "$BUILDER_NAME" --driver docker-container --use
+fi
+
+docker buildx use "$BUILDER_NAME"
+
+extract_artifact() {
+    local platform=$1
+    local artifact_name=$2
+    local platform_tag="${IMAGE_TAG}-${platform//\//-}"
+    
+    echo "[INFO] Building single-platform image for $platform..."
+    docker buildx build --platform "$platform" -t "$platform_tag" --load .
+    
+    echo "[INFO] Creating temporary container from $platform image..."
+    local cid=$(docker create "$platform_tag")
+    trap "docker rm -fv '$cid' >/dev/null" EXIT
+    
+    echo "[INFO] Copying '$artifact_name' from $platform container to host..."
+    if docker cp "${cid}:/${ARTIFACT_BASE}" "./${artifact_name}"; then
+        echo "[SUCCESS] Artifact '$artifact_name' extracted to $(pwd)"
+    else
+        echo "[ERROR] Failed to find '${ARTIFACT_BASE}' inside the $platform image." >&2
+        return 1
+    fi
+}
+
+# Extract ARM64 artifact
+extract_artifact "linux/arm64" "libmysqljsonparse.so"
+
+# Extract AMD64 artifact  
+extract_artifact "linux/amd64" "libmysqljsonparse_x86_64.so"
+
+echo "[SUCCESS] Both artifacts built successfully:"
+echo "  - libmysqljsonparse.so (ARM64)"
+echo "  - libmysqljsonparse_x86_64.so (AMD64)" 

binlog_json_parser/mysql_json_parser.cpp

@@ -120,24 +120,28 @@ static bool read_variable_length(const char *data, size_t data_length,
 
 std::string escape_json(const std::string &s) {
   std::ostringstream o;
-  for (auto c = s.cbegin(); c != s.cend(); c++) {
-    switch (*c) {
-      case '"': o << "\\\""; break;
+
+  for (const unsigned char uc : s) {
+    switch (uc) {
+      case '"':  o << "\\\""; break;
       case '\\': o << "\\\\"; break;
-      case '\b': o << "\\b"; break;
-      case '\f': o << "\\f"; break;
-      case '\n': o << "\\n"; break;
-      case '\r': o << "\\r"; break;
-      case '\t': o << "\\t"; break;
+      case '\b': o << "\\b";   break;
+      case '\f': o << "\\f";   break;
+      case '\n': o << "\\n";   break;
+      case '\r': o << "\\r";   break;
+      case '\t': o << "\\t";   break;
+
       default:
-        if (*c <= '\x1f') {
+        if (uc <= 0x1F) {
           o << "\\u"
-            << std::hex << std::setw(4) << std::setfill('0') << static_cast<int>(*c);
+            << std::hex << std::setw(4) << std::setfill('0') << static_cast<int>(uc)
+            << std::dec;
         } else {
-          o << *c;
+          o << static_cast<char>(uc);
         }
     }
   }
+
   return o.str();
 }
 

mysql_ch_replicator/pymysqlreplication/libmysqljsonparse.so

@@ -2747,3 +2747,60 @@ def test_truncate_operation_bug_issue_155():
     # Clean up
     db_replicator_runner.stop()
     binlog_replicator_runner.stop()
+
+def test_json2():
+    cfg = config.Settings()
+    cfg.load(CONFIG_FILE)
+
+    mysql = mysql_api.MySQLApi(
+        database=None,
+        mysql_settings=cfg.mysql,
+    )
+
+    ch = clickhouse_api.ClickhouseApi(
+        database=TEST_DB_NAME,
+        clickhouse_settings=cfg.clickhouse,
+    )
+
+    prepare_env(cfg, mysql, ch)
+
+    mysql.execute("SET sql_mode = 'ALLOW_INVALID_DATES';")
+
+    mysql.execute(f'''
+CREATE TABLE `{TEST_TABLE_NAME}` (
+    `id` int unsigned NOT NULL AUTO_INCREMENT,
+    name varchar(255),
+    data json,
+    PRIMARY KEY (id)
+); 
+    ''')
+
+    mysql.execute(
+        f"INSERT INTO `{TEST_TABLE_NAME}` (name, data) VALUES " +
+        """('Ivan', '{"а": "б", "в": [1,2,3]}');""",
+        commit=True,
+    )
+
+    binlog_replicator_runner = BinlogReplicatorRunner()
+    binlog_replicator_runner.run()
+    db_replicator_runner = DbReplicatorRunner(TEST_DB_NAME)
+    db_replicator_runner.run()
+
+    assert_wait(lambda: TEST_DB_NAME in ch.get_databases())
+
+    ch.execute_command(f'USE `{TEST_DB_NAME}`')
+
+    assert_wait(lambda: TEST_TABLE_NAME in ch.get_tables())
+    assert_wait(lambda: len(ch.select(TEST_TABLE_NAME)) == 1)
+
+    mysql.execute(
+        f"INSERT INTO `{TEST_TABLE_NAME}` (name, data) VALUES " +
+        """('Peter', '{"в": "б", "а": [3,2,1]}');""",
+        commit=True,
+    )
+    assert_wait(lambda: len(ch.select(TEST_TABLE_NAME)) == 2)
+
+    assert json.loads(ch.select(TEST_TABLE_NAME, "name='Ivan'")[0]['data'])['в'] == [1, 2, 3]
+    assert json.loads(ch.select(TEST_TABLE_NAME, "name='Peter'")[0]['data'])['в'] == 'б'
+    db_replicator_runner.stop()
+    binlog_replicator_runner.stop()