feat(xlang/java): refactor java native serialization type system and streaming type info for xlang (#3153)

## Why?

Fory needs streaming-friendly shared type meta so readers can
deserialize without seeking. The current work also unifies native/xlang
type IDs and removes legacy paths that were causing inconsistent type
resolution and CI failures across languages.

## What does this PR do?

- Switch shared type meta to the streaming wire format (inline TypeDef
bytes with index marker) and remove legacy meta-offset handling.
- Unify native and xlang type ID handling in Java resolvers, simplify
dispatch paths, and remove outdated compatibility branches.
- Add `registerSerializerAndType` API for explicit type+serializer
registration flow.
- Fix Go skip path to consume streaming shared type meta in compatible
struct skipping and add a regression test.
- Keep resolver fast paths by inlining class-info reads and avoiding
redundant serializer creation.

## Related issues

Closes #3157 
Closes #3119

#2609 #2818

## Does this PR introduce any user-facing change?

<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fory/issues/new/choose) describing the
need to do so and update the document if necessary.

Delete section if not applicable.
-->

- [x] Does this PR introduce any public API change?
- [x] Does this PR introduce any binary protocol compatibility change?

## Benchmark

Not run.

Author: chaokunyang

.github/workflows/ci.yml

@@ -324,6 +324,7 @@ jobs:
       - name: Run Rust Xlang Test
         env:
           FORY_RUST_JAVA_CI: "1"
+          ENABLE_FORY_DEBUG_OUTPUT: "1"
         run: |
           cd java
           mvn -T16 --no-transfer-progress clean install -DskipTests
@@ -410,6 +411,7 @@ jobs:
       - name: Run CPP Xlang Test
         env:
           FORY_CPP_JAVA_CI: "1"
+          ENABLE_FORY_DEBUG_OUTPUT: "1"
         run: |
           cd java
           mvn -T16 --no-transfer-progress clean install -DskipTests
@@ -531,6 +533,7 @@ jobs:
       - name: Run Python Xlang Test
         env:
           FORY_PYTHON_JAVA_CI: "1"
+          ENABLE_FORY_DEBUG_OUTPUT: "1"
         run: |
           cd java
           mvn -T16 --no-transfer-progress clean install -DskipTests
@@ -599,6 +602,7 @@ jobs:
       - name: Run Go Xlang Test
         env:
           FORY_GO_JAVA_CI: "1"
+          ENABLE_FORY_DEBUG_OUTPUT: "1"
         run: |
           cd java
           mvn -T16 --no-transfer-progress clean install -DskipTests

AGENTS.md

@@ -14,6 +14,8 @@ While working on Fory, please remember:
 - **Cross-Language Consistency**: Maintain consistency across language implementations while respecting language-specific idioms.
 - **Graalvm Support using fory codegen**: For graalvm, please use `fory codegen` to generate the serializer when building graalvm native image, do not use graallvm reflect-related configuration unless for JDK `proxy`.
 - **Xlang Type System**: Java `native mode(xlang=false)` shares same type systems between type id from `Types.BOOL~Types.STRING` with `xlang mode(xlang=true)`, but for other types, java `native mode` has different type ids.
+- **Remote git repository**: `git@github.com:apache/fory.git` is remote repository, do not use other remote repository when you want to check code under `main` branch.
+- **Contributor git repository**: A contributor should fork the `git@github.com:apache/fory.git` repo, and git push the code changes into their forked repo, then create a pull request from the branch in their forked repo into `git@github.com:apache/fory.git`.
 
 ## Build and Development Commands
 
@@ -295,9 +297,6 @@ it_dir=$(pwd)
 # Run graalvm tests
 cd $it_dir/graalvm_tests && mvn -T16 -DskipTests=true -Pnative package && target/main
 
-# Run latest_jdk_tests
-cd $it_dir/latest_jdk_tests && mvn -T16 test
-
 # Run JDK compatibility tests
 cd $it_dir/jdk_compatibility_tests && mvn -T16 test
 
@@ -498,7 +497,6 @@ Fory rust provides macro-based serialization and deserialization. Fory rust cons
   - Note that fory use codegen to support graalvm instead of reflection, fory don't use `reflect-config.json` for
     serialization, this is the core advantage of compared to graalvm JDK serialization.
 - **jdk_compatibility_tests**: test suite for fory serialization compatibility between multiple JDK versions
-- **latest_jdk_tests**: test suite for `jdk17+` versions
 
 ## Key Development Guidelines
 

benchmarks/java_benchmark/src/main/java/org/apache/fory/benchmark/UserTypeDeserializeSuite.java

@@ -147,7 +147,7 @@ public Object flatbuffers_deserialize(FlatBuffersState.FlatBuffersUserTypeState
   public static void main(String[] args) throws IOException {
     if (args.length == 0) {
       String commandLine =
-          "org.apache.fory.*UserTypeDeserializeSuite.fory* -f 1 -wi 5 -i 10 -t 1 -w 2s -r 2s -rf csv "
+          "org.apache.fory.*UserTypeDeserializeSuite.* -f 1 -wi 3 -i 3 -t 1 -w 2s -r 2s -rf csv "
               + "-p objectType=MEDIA_CONTENT -p bufferType=array -p references=false";
       System.out.println(commandLine);
       args = commandLine.split(" ");

benchmarks/java_benchmark/src/main/java/org/apache/fory/benchmark/UserTypeSerializeSuite.java

@@ -157,7 +157,7 @@ public Object flatbuffers_serialize(FlatBuffersState.FlatBuffersUserTypeState st
   public static void main(String[] args) throws IOException {
     if (args.length == 0) {
       String commandLine =
-          "org.apache.fory.*UserTypeSerializeSuite.fory_serialize_compatible -f 1 -wi 5 -i 10 -t 1 -w 200s -r 2s -rf csv "
+          "org.apache.fory.*UserTypeSerializeSuite.* -f 1 -wi 3 -i 3 -t 1 -w 2s -r 2s -rf csv "
               + "-p objectType=MEDIA_CONTENT -p bufferType=array -p references=false";
       System.out.println(commandLine);
       args = commandLine.split(" ");

ci/format.sh

@@ -143,7 +143,7 @@ format_java() {
       cd "$ROOT/benchmarks/java_benchmark"
       mvn -T10 --no-transfer-progress spotless:apply
       cd "$ROOT/integration_tests"
-      dirs=("graalvm_tests" "jdk_compatibility_tests" "latest_jdk_tests")
+      dirs=("graalvm_tests" "jdk_compatibility_tests")
       for d in "${dirs[@]}" ; do
         pushd "$d"
           mvn -T10 --no-transfer-progress spotless:apply

ci/release.py

@@ -186,15 +186,14 @@ def bump_java_version(new_version):
         "integration_tests/graalvm_tests",
         "integration_tests/jdk_compatibility_tests",
         "integration_tests/jpms_tests",
-        "integration_tests/latest_jdk_tests",
-        "integration_tests/latest_jdk_tests",
         "benchmarks/java_benchmark",
         "java/fory-core",
         "java/fory-format",
         "java/fory-simd",
         "java/fory-extensions",
         "java/fory-test-core",
         "java/fory-testsuite",
+        "java/fory-latest-jdk-tests",
     ]:
         _bump_version(p, "pom.xml", new_version, _update_pom_parent_version)
     # mvn versions:set too slow

ci/run_ci.sh

@@ -182,12 +182,6 @@ integration_tests() {
   echo "benchmark tests"
   cd "$ROOT"/benchmarks/java_benchmark
   mvn -T10 -B --no-transfer-progress clean test install -Pjmh
-  echo "Start latest jdk tests"
-  cd "$ROOT"/integration_tests/latest_jdk_tests
-  echo "latest_jdk_tests: JDK 21"
-  export JAVA_HOME="$ROOT/zulu21.28.85-ca-jdk21.0.0-linux_x64"
-  export PATH=$JAVA_HOME/bin:$PATH
-  mvn -T10 -B --no-transfer-progress clean test
   echo "Start JPMS tests"
   cd "$ROOT"/integration_tests/jpms_tests
   mvn -T10 -B --no-transfer-progress clean compile
@@ -220,10 +214,6 @@ jdk17_plus_tests() {
     exit $testcode
   fi
   echo "Executing fory java tests succeeds"
-  echo "Executing latest_jdk_tests"
-  cd "$ROOT"/integration_tests/latest_jdk_tests
-  mvn -T10 -B --no-transfer-progress clean test
-  echo "Executing latest_jdk_tests succeeds"
 }
 
 kotlin_tests() {

ci/tasks/java.py

@@ -175,12 +175,6 @@ def run_jdk17_plus(java_version="17"):
     common.exec_cmd("mvn -T10 --batch-mode --no-transfer-progress install")
 
     logging.info("Executing fory java tests succeeds")
-    logging.info("Executing latest_jdk_tests")
-
-    common.cd_project_subdir("integration_tests/latest_jdk_tests")
-    common.exec_cmd("mvn -T10 -B --no-transfer-progress clean test")
-
-    logging.info("Executing latest_jdk_tests succeeds")
 
 
 def run_windows_java21():
@@ -218,17 +212,6 @@ def run_integration_tests():
     common.cd_project_subdir("benchmarks/java_benchmark")
     common.exec_cmd("mvn -T10 -B --no-transfer-progress clean test install -Pjmh")
 
-    logging.info("Start latest jdk tests")
-    common.cd_project_subdir("integration_tests/latest_jdk_tests")
-    logging.info("latest_jdk_tests: JDK 21")
-
-    # Set Java 21 as the current JDK
-    java_home = os.path.join(common.PROJECT_ROOT_DIR, JDKS["21"])
-    os.environ["JAVA_HOME"] = java_home
-    os.environ["PATH"] = f"{java_home}/bin:{os.environ.get('PATH', '')}"
-
-    common.exec_cmd("mvn -T10 -B --no-transfer-progress clean test")
-
     logging.info("Start JPMS tests")
     common.cd_project_subdir("integration_tests/jpms_tests")
     common.exec_cmd("mvn -T10 -B --no-transfer-progress clean compile")