Merge remote-tracking branch 'asf/main' into sync_code_to_0.13

Author: chaokunyang

.github/workflows/ci.yml

@@ -288,7 +288,7 @@ jobs:
     name: Rust CI
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-13, macos-14]  # macos-13: x86, macos-14: arm64
+        os: [ubuntu-latest, macos-14, macos-latest]  # macos-14: arm64
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
     steps:
@@ -316,7 +316,7 @@ jobs:
     name: C++ CI
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-13, macos-14, windows-2022]  # macos-13: x86, macos-14: arm64
+        os: [ubuntu-latest, macos-14, macos-latest, windows-2022]  # macos-13: x86, macos-14: arm64
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v5

AGENTS.md

@@ -389,14 +389,14 @@ Fory python has two implementations for the protocol:
 
 Code structure:
 
-- `python/pyfory/_serialization.pyx`: Core serialization logic and entry point for cython mode based on `xlang serialization format`
+- `python/pyfory/serialization.pyx`: Core serialization logic and entry point for cython mode based on `xlang serialization format`
 - `python/pyfory/_fory.py`: Serialization entry point for pure python mode based on `xlang serialization format`
 - `python/pyfory/_registry.py`: Type registry, resolution and serializer dispatch for pure python mode, which is also used by cython mode. Cython mode use a cache to reduce invocations to this module.
 - `python/pyfory/serializer.py`: Serializers for non-internal types
 - `python/pyfory/includes`: Cython headers for `c++` functions and classes.
 - `python/pyfory/resolver.py`: resolving shared/circular references when ref tracking is enabled in pure python mode
 - `python/pyfory/format`: Fory row format encoding and decoding, arrow columnar format interoperation
-- `python/pyfory/_util.pyx`: Buffer for reading/writing data, string utilities. Used by `_serialization.pyx` and `python/pyfory/format` at the same time.
+- `python/pyfory/_util.pyx`: Buffer for reading/writing data, string utilities. Used by `serialization.pyx` and `python/pyfory/format` at the same time.
 
 #### Go
 

BUILD

@@ -51,11 +51,11 @@ pyx_library(
 )
 
 pyx_library(
-    name = "_serialization",
+    name = "serialization",
     srcs = glob([
         "python/pyfory/includes/*.pxd",
         "python/pyfory/_util.pxd",
-        "python/pyfory/_serialization.pyx",
+        "python/pyfory/serialization.pyx",
         "python/pyfory/__init__.py",
     ]),
     cc_kwargs = dict(
@@ -96,7 +96,7 @@ genrule(
         ":python/pyfory/_util.so",
         ":python/pyfory/lib/mmh3/mmh3.so",
         ":python/pyfory/format/_format.so",
-        ":python/pyfory/_serialization.so",
+        ":python/pyfory/serialization.so",
     ],
     outs = [
         "cp_fory_py_generated.out",
@@ -111,12 +111,12 @@ genrule(
             cp -f $(location python/pyfory/_util.so) "$$WORK_DIR/python/pyfory/_util.pyd"
             cp -f $(location python/pyfory/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfory/lib/mmh3/mmh3.pyd"
             cp -f $(location python/pyfory/format/_format.so) "$$WORK_DIR/python/pyfory/format/_format.pyd"
-            cp -f $(location python/pyfory/_serialization.so) "$$WORK_DIR/python/pyfory/_serialization.pyd"
+            cp -f $(location python/pyfory/serialization.so) "$$WORK_DIR/python/pyfory/serialization.pyd"
         else
             cp -f $(location python/pyfory/_util.so) "$$WORK_DIR/python/pyfory"
             cp -f $(location python/pyfory/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfory/lib/mmh3"
             cp -f $(location python/pyfory/format/_format.so) "$$WORK_DIR/python/pyfory/format"
-            cp -f $(location python/pyfory/_serialization.so) "$$WORK_DIR/python/pyfory"
+            cp -f $(location python/pyfory/serialization.so) "$$WORK_DIR/python/pyfory"
         fi
         echo $$(date) > $@
     """,

CONTRIBUTING.md

@@ -155,7 +155,7 @@ cd python
 python setup.py develop
 ```
 
-- Use `cython --cplus -a  pyfory/_serialization.pyx` to produce an annotated HTML file of the source code. Then you can analyze interaction between Python objects and Python's C API.
+- Use `cython --cplus -a  pyfory/serialization.pyx` to produce an annotated HTML file of the source code. Then you can analyze interaction between Python objects and Python's C API.
 - Read more: https://cython.readthedocs.io/en/latest/src/userguide/debugging.html
 
 ```bash

ci/release.py

@@ -54,8 +54,19 @@ def build(v: str):
     if os.path.exists("dist"):
         shutil.rmtree("dist")
     os.mkdir("dist")
-    subprocess.check_call(f"git checkout releases-{v}", shell=True)
     branch = f"releases-{v}"
+    # Check if branch exists, if not create it
+    result = subprocess.run(
+        f"git show-ref --verify --quiet refs/heads/{branch}",
+        shell=True,
+        capture_output=True,
+    )
+    if result.returncode == 0:
+        # Branch exists, checkout
+        subprocess.check_call(f"git checkout {branch}", shell=True)
+    else:
+        # Branch doesn't exist, create it
+        subprocess.check_call(f"git checkout -b {branch}", shell=True)
     src_tar = f"apache-fory-{v}-src.tar.gz"
     _check_all_committed()
     _strip_unnecessary_license()

ci/run_ci.py

@@ -293,7 +293,7 @@ def parse_args():
         if USE_PYTHON_GO:
             func()
         else:
-            # run_shell_script("go")
+            run_shell_script("go")
             pass
     elif command == "format":
         if USE_PYTHON_FORMAT:

ci/run_ci.sh

@@ -372,11 +372,11 @@ case $1 in
     ;;
     go)
       echo "Executing fory go tests for go"
-      cd "$ROOT/go/fory"
-      go install ./cmd/fory
-      cd "$ROOT/go/fory/tests"
-      go generate
-      go test -v
+      # cd "$ROOT/go/fory"
+      # go install ./cmd/fory
+      # cd "$ROOT/go/fory/tests"
+      # go generate
+      # go test -v
       cd "$ROOT/go/fory"
       go test -v
       echo "Executing fory go tests succeeds"

cpp/fory/benchmark/benchmark_string_util.cc

@@ -347,6 +347,20 @@ bool utf16HasSurrogatePairs_BaseLine(const std::u16string &str) {
   return false;
 }
 
+// Generate test strings of various sizes for threshold benchmarking
+const std::vector<std::u16string> test_utf16_strings_small_16 =
+    generateUTF16String(num_tests, 16);
+const std::vector<std::u16string> test_utf16_strings_small_32 =
+    generateUTF16String(num_tests, 32);
+const std::vector<std::u16string> test_utf16_strings_small_64 =
+    generateUTF16String(num_tests, 64);
+const std::vector<std::u16string> test_utf16_strings_medium_128 =
+    generateUTF16String(num_tests, 128);
+const std::vector<std::u16string> test_utf16_strings_medium_256 =
+    generateUTF16String(num_tests, 256);
+const std::vector<std::u16string> test_utf16_strings_large_512 =
+    generateUTF16String(num_tests, 512);
+
 // Benchmark function for checking if a UTF-16 string contains surrogate pairs
 static void BM_Utf16HasSurrogatePairs_BaseLine(benchmark::State &state) {
   for (auto _ : state) {
@@ -372,6 +386,142 @@ static void BM_Utf16HasSurrogatePairs_FORY(benchmark::State &state) {
 
 BENCHMARK(BM_Utf16HasSurrogatePairs_FORY);
 
+// Benchmarks for different string sizes to determine optimal threshold
+static void BM_Utf16HasSurrogatePairs_BaseLine_Size16(benchmark::State &state) {
+  for (auto _ : state) {
+    for (const std::u16string &str : test_utf16_strings_small_16) {
+      bool result = utf16HasSurrogatePairs_BaseLine(str);
+      benchmark::DoNotOptimize(result);
+    }
+  }
+}
+
+BENCHMARK(BM_Utf16HasSurrogatePairs_BaseLine_Size16);
+
+static void BM_Utf16HasSurrogatePairs_FORY_Size16(benchmark::State &state) {
+  for (auto _ : state) {
+    for (const std::u16string &str : test_utf16_strings_small_16) {
+      bool result = fory::utf16HasSurrogatePairs(str);
+      benchmark::DoNotOptimize(result);
+    }
+  }
+}
+
+BENCHMARK(BM_Utf16HasSurrogatePairs_FORY_Size16);
+
+static void BM_Utf16HasSurrogatePairs_BaseLine_Size32(benchmark::State &state) {
+  for (auto _ : state) {
+    for (const std::u16string &str : test_utf16_strings_small_32) {
+      bool result = utf16HasSurrogatePairs_BaseLine(str);
+      benchmark::DoNotOptimize(result);
+    }
+  }
+}
+
+BENCHMARK(BM_Utf16HasSurrogatePairs_BaseLine_Size32);
+
+static void BM_Utf16HasSurrogatePairs_FORY_Size32(benchmark::State &state) {
+  for (auto _ : state) {
+    for (const std::u16string &str : test_utf16_strings_small_32) {
+      bool result = fory::utf16HasSurrogatePairs(str);
+      benchmark::DoNotOptimize(result);
+    }
+  }
+}
+
+BENCHMARK(BM_Utf16HasSurrogatePairs_FORY_Size32);
+
+static void BM_Utf16HasSurrogatePairs_BaseLine_Size64(benchmark::State &state) {
+  for (auto _ : state) {
+    for (const std::u16string &str : test_utf16_strings_small_64) {
+      bool result = utf16HasSurrogatePairs_BaseLine(str);
+      benchmark::DoNotOptimize(result);
+    }
+  }
+}
+
+BENCHMARK(BM_Utf16HasSurrogatePairs_BaseLine_Size64);
+
+static void BM_Utf16HasSurrogatePairs_FORY_Size64(benchmark::State &state) {
+  for (auto _ : state) {
+    for (const std::u16string &str : test_utf16_strings_small_64) {
+      bool result = fory::utf16HasSurrogatePairs(str);
+      benchmark::DoNotOptimize(result);
+    }
+  }
+}
+
+BENCHMARK(BM_Utf16HasSurrogatePairs_FORY_Size64);
+
+static void
+BM_Utf16HasSurrogatePairs_BaseLine_Size128(benchmark::State &state) {
+  for (auto _ : state) {
+    for (const std::u16string &str : test_utf16_strings_medium_128) {
+      bool result = utf16HasSurrogatePairs_BaseLine(str);
+      benchmark::DoNotOptimize(result);
+    }
+  }
+}
+
+BENCHMARK(BM_Utf16HasSurrogatePairs_BaseLine_Size128);
+
+static void BM_Utf16HasSurrogatePairs_FORY_Size128(benchmark::State &state) {
+  for (auto _ : state) {
+    for (const std::u16string &str : test_utf16_strings_medium_128) {
+      bool result = fory::utf16HasSurrogatePairs(str);
+      benchmark::DoNotOptimize(result);
+    }
+  }
+}
+
+BENCHMARK(BM_Utf16HasSurrogatePairs_FORY_Size128);
+
+static void
+BM_Utf16HasSurrogatePairs_BaseLine_Size256(benchmark::State &state) {
+  for (auto _ : state) {
+    for (const std::u16string &str : test_utf16_strings_medium_256) {
+      bool result = utf16HasSurrogatePairs_BaseLine(str);
+      benchmark::DoNotOptimize(result);
+    }
+  }
+}
+
+BENCHMARK(BM_Utf16HasSurrogatePairs_BaseLine_Size256);
+
+static void BM_Utf16HasSurrogatePairs_FORY_Size256(benchmark::State &state) {
+  for (auto _ : state) {
+    for (const std::u16string &str : test_utf16_strings_medium_256) {
+      bool result = fory::utf16HasSurrogatePairs(str);
+      benchmark::DoNotOptimize(result);
+    }
+  }
+}
+
+BENCHMARK(BM_Utf16HasSurrogatePairs_FORY_Size256);
+
+static void
+BM_Utf16HasSurrogatePairs_BaseLine_Size512(benchmark::State &state) {
+  for (auto _ : state) {
+    for (const std::u16string &str : test_utf16_strings_large_512) {
+      bool result = utf16HasSurrogatePairs_BaseLine(str);
+      benchmark::DoNotOptimize(result);
+    }
+  }
+}
+
+BENCHMARK(BM_Utf16HasSurrogatePairs_BaseLine_Size512);
+
+static void BM_Utf16HasSurrogatePairs_FORY_Size512(benchmark::State &state) {
+  for (auto _ : state) {
+    for (const std::u16string &str : test_utf16_strings_large_512) {
+      bool result = fory::utf16HasSurrogatePairs(str);
+      benchmark::DoNotOptimize(result);
+    }
+  }
+}
+
+BENCHMARK(BM_Utf16HasSurrogatePairs_FORY_Size512);
+
 /*
  * TEST Utf16ToUtf8
  */

cpp/fory/util/string_util.h

@@ -133,23 +133,11 @@ inline bool isLatin1(const uint16_t *data, size_t length) {
                           length % VECTOR_SIZE);
 }
 inline bool utf16HasSurrogatePairs(const uint16_t *data, size_t length) {
-  constexpr size_t VECTOR_SIZE = 16;
-  const auto *ptr = reinterpret_cast<const __m256i *>(data);
-  const auto *end = ptr + length / VECTOR_SIZE;
-  const __m256i lower_bound = _mm256_set1_epi16(0xD800);
-  const __m256i higher_bound = _mm256_set1_epi16(0xDFFF);
-
-  for (; ptr < end; ++ptr) {
-    __m256i vec = _mm256_loadu_si256(ptr);
-    __m256i mask1 = _mm256_cmpgt_epi16(vec, lower_bound);
-    __m256i mask2 = _mm256_cmpgt_epi16(higher_bound, vec);
-    __m256i result = _mm256_and_si256(mask1, mask2);
-    if (!_mm256_testz_si256(result, result))
-      return true;
-  }
-
-  return hasSurrogatePairFallback(data + (length / VECTOR_SIZE) * VECTOR_SIZE,
-                                  length % VECTOR_SIZE);
+  // Direct fallback implementation - SIMD versions were consistently slower
+  // due to early-exit characteristics: surrogate pairs are rare and when
+  // present, often appear early in strings, making SIMD setup overhead
+  // outweigh any vectorization benefits.
+  return hasSurrogatePairFallback(data, length);
 }
 
 #elif defined(FORY_HAS_NEON)
@@ -182,18 +170,11 @@ inline bool isLatin1(const uint16_t *data, size_t length) {
 }
 
 inline bool utf16HasSurrogatePairs(const uint16_t *data, size_t length) {
-  size_t i = 0;
-  uint16x8_t lower_bound = vdupq_n_u16(0xD800);
-  uint16x8_t higher_bound = vdupq_n_u16(0xDFFF);
-  for (; i + 7 < length; i += 8) {
-    uint16x8_t chunk = vld1q_u16(data + i);
-    uint16x8_t mask1 = vcgeq_u16(chunk, lower_bound);
-    uint16x8_t mask2 = vcleq_u16(chunk, higher_bound);
-    if (vmaxvq_u16(mask1 & mask2)) {
-      return true; // Detected a high surrogate
-    }
-  }
-  return hasSurrogatePairFallback(data + i, length - i);
+  // Direct fallback implementation - SIMD versions were consistently slower
+  // due to early-exit characteristics: surrogate pairs are rare and when
+  // present, often appear early in strings, making SIMD setup overhead
+  // outweigh any vectorization benefits.
+  return hasSurrogatePairFallback(data, length);
 }
 #elif defined(FORY_HAS_SSE2)
 inline bool isAscii(const char *data, size_t length) {
@@ -227,19 +208,11 @@ inline bool isLatin1(const uint16_t *data, size_t length) {
 }
 
 inline bool utf16HasSurrogatePairs(const uint16_t *data, size_t length) {
-  size_t i = 0;
-  __m128i lower_bound = _mm_set1_epi16(0xd7ff);
-  __m128i higher_bound = _mm_set1_epi16(0xe000);
-  for (; i + 7 < length; i += 8) {
-    __m128i chunk =
-        _mm_loadu_si128(reinterpret_cast<const __m128i *>(data + i));
-    __m128i cmp1 = _mm_cmpgt_epi16(chunk, lower_bound);
-    __m128i cmp2 = _mm_cmpgt_epi16(higher_bound, chunk);
-    if (_mm_movemask_epi8(_mm_and_si128(cmp1, cmp2)) != 0) {
-      return true; // Detected a surrogate
-    }
-  }
-  return hasSurrogatePairFallback(data + i, length - i);
+  // Direct fallback implementation - SIMD versions were consistently slower
+  // due to early-exit characteristics: surrogate pairs are rare and when
+  // present, often appear early in strings, making SIMD setup overhead
+  // outweigh any vectorization benefits.
+  return hasSurrogatePairFallback(data, length);
 }
 #else
 inline bool isAscii(const char *data, size_t length) {
@@ -266,10 +239,14 @@ inline bool isLatin1(const std::u16string &str) {
 }
 
 inline bool utf16HasSurrogatePairs(const std::u16string &str) {
-  // Get the data pointer
-  const std::uint16_t *data =
-      reinterpret_cast<const std::uint16_t *>(str.data());
-  return utf16HasSurrogatePairs(data, str.size());
+  // Inline implementation for best performance
+  for (size_t i = 0; i < str.size(); ++i) {
+    auto c = str[i];
+    if (c >= 0xD800 && c <= 0xDFFF) {
+      return true;
+    }
+  }
+  return false;
 }
 
 } // namespace fory