feat(xlang): support serialization for unsigned types and field encoding config

.github/workflows/build-native-pr.yml

@@ -168,7 +168,7 @@ jobs:
           X86_DIR=$(ls -d unpacked/x86_64/pyfory-*)
           UNIVERSAL_DIR="unpacked/pyfory-universal2"
           cp -R "$ARM_DIR" "$UNIVERSAL_DIR"
-          for so in pyfory/_util.so pyfory/serialization.so pyfory/format/_format.so pyfory/lib/mmh3/mmh3.so; do
+          for so in pyfory/buffer.so pyfory/serialization.so pyfory/format/_format.so pyfory/lib/mmh3/mmh3.so; do
             lipo -create "$ARM_DIR/$so" "$X86_DIR/$so" -output "$UNIVERSAL_DIR/$so"
           done
           WHEEL_FILE=$(ls "$UNIVERSAL_DIR"/pyfory-*.dist-info/WHEEL)
@@ -187,7 +187,7 @@ jobs:
           VERIFY_DIR=$(ls -d verify/pyfory-*)
           WHEEL_FILE=$(ls "$VERIFY_DIR"/pyfory-*.dist-info/WHEEL)
           grep -q "macosx_11_0_universal2" "$WHEEL_FILE"
-          for so in pyfory/_util.so pyfory/serialization.so pyfory/format/_format.so pyfory/lib/mmh3/mmh3.so; do
+          for so in pyfory/buffer.so pyfory/serialization.so pyfory/format/_format.so pyfory/lib/mmh3/mmh3.so; do
             echo "$so: $(lipo -archs "$VERIFY_DIR/$so")"
           done
       - name: Upload universal2 wheel

.github/workflows/build-native-release.yml

@@ -206,7 +206,7 @@ jobs:
           X86_DIR=$(ls -d unpacked/x86_64/pyfory-*)
           UNIVERSAL_DIR="unpacked/pyfory-universal2"
           cp -R "$ARM_DIR" "$UNIVERSAL_DIR"
-          for so in pyfory/_util.so pyfory/serialization.so pyfory/format/_format.so pyfory/lib/mmh3/mmh3.so; do
+          for so in pyfory/buffer.so pyfory/serialization.so pyfory/format/_format.so pyfory/lib/mmh3/mmh3.so; do
             lipo -create "$ARM_DIR/$so" "$X86_DIR/$so" -output "$UNIVERSAL_DIR/$so"
           done
           WHEEL_FILE=$(ls "$UNIVERSAL_DIR"/pyfory-*.dist-info/WHEEL)
@@ -225,7 +225,7 @@ jobs:
           VERIFY_DIR=$(ls -d verify/pyfory-*)
           WHEEL_FILE=$(ls "$VERIFY_DIR"/pyfory-*.dist-info/WHEEL)
           grep -q "macosx_11_0_universal2" "$WHEEL_FILE"
-          for so in pyfory/_util.so pyfory/serialization.so pyfory/format/_format.so pyfory/lib/mmh3/mmh3.so; do
+          for so in pyfory/ pyfory/serialization.so pyfory/format/_format.so pyfory/lib/mmh3/mmh3.so; do
             echo "$so: $(lipo -archs "$VERIFY_DIR/$so")"
           done
       - name: Upload universal2 wheel

AGENTS.md

@@ -13,6 +13,7 @@ While working on Fory, please remember:
 - **Git-Tracked Files**: When reading code, skip all files not tracked by git by default unless generated by yourself.
 - **Cross-Language Consistency**: Maintain consistency across language implementations while respecting language-specific idioms.
 - **Graalvm Support using fory codegen**: For graalvm, please use `fory codegen` to generate the serializer when building graalvm native image, do not use graallvm reflect-related configuration unless for JDK `proxy`.
+- **Xlang Type System**: Java `native mode(xlang=false)` shares same type systems between type id from `Types.BOOL~Types.STRING` with `xlang mode(xlang=true)`, but for other types, java `native mode` has different type ids.
 
 ## Build and Development Commands
 
@@ -125,7 +126,7 @@ cd java
 mvn -T16 install -DskipTests
 cd fory-core
 # disable fory cython for faster debugging
-FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=0 mvn -T16 test -Dtest=org.apache.fory.xlang.PythonXlangTest
+FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=0 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.xlang.PythonXlangTest
 # enable fory cython
 FORY_PYTHON_JAVA_CI=1 ENABLE_FORY_CYTHON_SERIALIZATION=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn -T16 test -Dtest=org.apache.fory.xlang.PythonXlangTest
 ```
@@ -215,7 +216,7 @@ Run Rust xlang tests:
 cd java
 mvn -T16 install -DskipTests
 cd fory-core
-FORY_RUST_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.xlang.RustXlangTest
+RUST_BACKTRACE=1 FORY_PANIC_ON_ERROR=1 FORY_RUST_JAVA_CI=1 ENABLE_FORY_DEBUG_OUTPUT=1 mvn test -Dtest=org.apache.fory.xlang.RustXlangTest
 ```
 
 ### JavaScript/TypeScript Development
@@ -445,12 +446,12 @@ Code structure:
 
 - `python/pyfory/serialization.pyx`: Core serialization logic and entry point for cython mode based on `xlang serialization format`
 - `python/pyfory/_fory.py`: Serialization entry point for pure python mode based on `xlang serialization format`
-- `python/pyfory/_registry.py`: Type registry, resolution and serializer dispatch for pure python mode, which is also used by cython mode. Cython mode use a cache to reduce invocations to this module.
+- `python/pyfory/registry.py`: Type registry, resolution and serializer dispatch for pure python mode, which is also used by cython mode. Cython mode use a cache to reduce invocations to this module.
 - `python/pyfory/serializer.py`: Serializers for non-internal types
 - `python/pyfory/includes`: Cython headers for `c++` functions and classes.
 - `python/pyfory/resolver.py`: resolving shared/circular references when ref tracking is enabled in pure python mode
 - `python/pyfory/format`: Fory row format encoding and decoding, arrow columnar format interoperation
-- `python/pyfory/_util.pyx`: Buffer for reading/writing data, string utilities. Used by `serialization.pyx` and `python/pyfory/format` at the same time.
+- `python/pyfory/buffer.pyx`: Buffer for reading/writing data, string utilities. Used by `serialization.pyx` and `python/pyfory/format` at the same time.
 
 #### Go
 

BUILD

@@ -20,11 +20,11 @@ load("@hedron_compile_commands//:refresh_compile_commands.bzl", "refresh_compile
 
 
 pyx_library(
-    name = "_util",
+    name = "buffer",
     srcs = glob([
         "python/pyfory/includes/*.pxd",
-        "python/pyfory/_util.pxd",
-        "python/pyfory/_util.pyx",
+        "python/pyfory/buffer.pxd",
+        "python/pyfory/buffer.pyx",
         "python/pyfory/__init__.py",
     ]),
     cc_kwargs = dict(
@@ -54,7 +54,7 @@ pyx_library(
     name = "serialization",
     srcs = glob([
         "python/pyfory/includes/*.pxd",
-        "python/pyfory/_util.pxd",
+        "python/pyfory/buffer.pxd",
         "python/pyfory/serialization.pyx",
         "python/pyfory/*.pxi",
         "python/pyfory/__init__.py",
@@ -76,7 +76,7 @@ pyx_library(
         [
             "python/pyfory/__init__.py",
             "python/pyfory/includes/*.pxd",
-            "python/pyfory/_util.pxd",
+            "python/pyfory/buffer.pxd",
             "python/pyfory/*.pxi",
             "python/pyfory/format/_format.pyx",
             "python/pyfory/format/__init__.py",
@@ -95,7 +95,7 @@ pyx_library(
 genrule(
     name = "cp_fory_so",
     srcs = [
-        ":python/pyfory/_util.so",
+        ":python/pyfory/buffer.so",
         ":python/pyfory/lib/mmh3/mmh3.so",
         ":python/pyfory/format/_format.so",
         ":python/pyfory/serialization.so",
@@ -110,12 +110,12 @@ genrule(
         u_name=`uname -s`
         if [ "$${u_name: 0: 4}" == "MING" ] || [ "$${u_name: 0: 4}" == "MSYS" ]
         then
-            cp -f $(location python/pyfory/_util.so) "$$WORK_DIR/python/pyfory/_util.pyd"
+            cp -f $(location python/pyfory/buffer.so) "$$WORK_DIR/python/pyfory/buffer.pyd"
             cp -f $(location python/pyfory/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfory/lib/mmh3/mmh3.pyd"
             cp -f $(location python/pyfory/format/_format.so) "$$WORK_DIR/python/pyfory/format/_format.pyd"
             cp -f $(location python/pyfory/serialization.so) "$$WORK_DIR/python/pyfory/serialization.pyd"
         else
-            cp -f $(location python/pyfory/_util.so) "$$WORK_DIR/python/pyfory"
+            cp -f $(location python/pyfory/buffer.so) "$$WORK_DIR/python/pyfory"
             cp -f $(location python/pyfory/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfory/lib/mmh3"
             cp -f $(location python/pyfory/format/_format.so) "$$WORK_DIR/python/pyfory/format"
             cp -f $(location python/pyfory/serialization.so) "$$WORK_DIR/python/pyfory"
@@ -131,4 +131,4 @@ refresh_compile_commands(
     name = "refresh_compile_commands",
     exclude_headers = "all",
     exclude_external_sources = True,
-)
+)

benchmarks/cpp_benchmark/README.md

@@ -31,12 +31,12 @@ Note: Protobuf is fetched automatically via CMake FetchContent, so no manual ins
 
 | Datatype     | Operation   | Fory TPS   | Protobuf TPS | Faster      |
 | ------------ | ----------- | ---------- | ------------ | ----------- |
-| Mediacontent | Serialize   | 2,312,522  | 501,867      | Fory (4.6x) |
-| Mediacontent | Deserialize | 769,157    | 398,960      | Fory (1.9x) |
-| Sample       | Serialize   | 5,046,250  | 3,182,176    | Fory (1.6x) |
-| Sample       | Deserialize | 941,637    | 721,614      | Fory (1.3x) |
-| Struct       | Serialize   | 21,424,386 | 6,024,856    | Fory (3.6x) |
-| Struct       | Deserialize | 7,904,533  | 6,515,853    | Fory (1.2x) |
+| Mediacontent | Serialize   | 2,430,924  | 484,368      | Fory (5.0x) |
+| Mediacontent | Deserialize | 740,074    | 387,522      | Fory (1.9x) |
+| Sample       | Serialize   | 4,813,270  | 3,021,968    | Fory (1.6x) |
+| Sample       | Deserialize | 915,554    | 684,675      | Fory (1.3x) |
+| Struct       | Serialize   | 18,105,957 | 5,788,186    | Fory (3.1x) |
+| Struct       | Deserialize | 7,495,726  | 5,932,982    | Fory (1.3x) |
 
 ## Quick Start
 

benchmarks/rust_benchmark/src/models/complex.rs

@@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
 // Fory models
-#[derive(ForyObject, Debug, Clone, PartialEq)]
+#[derive(ForyObject, Debug, Clone, PartialEq, Default)]
 pub struct ForyProduct {
     pub id: String,
     pub name: String,
@@ -39,7 +39,7 @@ pub struct ForyOrderItem {
     pub customizations: HashMap<String, String>,
 }
 
-#[derive(ForyObject, Debug, Clone, PartialEq)]
+#[derive(ForyObject, Debug, Clone, PartialEq, Default)]
 pub struct ForyCustomer {
     pub id: String,
     pub name: String,

benchmarks/rust_benchmark/src/models/medium.rs

@@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
 // Fory models
-#[derive(ForyObject, Debug, Clone, PartialEq)]
+#[derive(ForyObject, Debug, Clone, PartialEq, Default)]
 pub struct ForyAddress {
     pub street: String,
     pub city: String,