Skip to content

Commit 597f2ae

Browse files
chaokunyangchaokunyang
authored
Merge branch 'main' into fix/sliceSerializer-Partial
2 parents dec1efd + aae11ea commit 597f2ae

File tree

39 files changed

+909
-769
lines changed

39 files changed

+909
-769
lines changed

‎.github/scripts/add_doc_headers.sh‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,14 @@ EOF
9595
)
9696

9797
# Process Python guide
98+
rm -rf docs/guide/python_guide.md
9899
add_header "python/README.md" "docs/guide/python_guide.md" "$PYTHON_HEADER"
99100

100101
# Process Rust guide
101-
add_header "rust/README.md" "docs/guide/rust_guide.md" "$RUST_HEADER"
102+
rm -rf docs/guide/rust_guide.md
103+
add_header "rust/README.md" "docs/guide/rust_guide.md" "$RUST_HEADER"
104+
git config --global user.email "[email protected]"
105+
git config --global user.name "Apache Fory"
106+
git add docs/guide/rust_guide.md
107+
git add docs/guide/python_guide.md
108+
git commit -m "Added rust and python docs"

‎.github/workflows/ci.yml‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,13 @@ jobs:
5454
with:
5555
java-version: ${{ matrix.java-version }}
5656
distribution: "temurin"
57+
- name: Cache Maven local repository
58+
uses: actions/cache@v4
59+
with:
60+
path: ~/.m2/repository
61+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
62+
restore-keys: |
63+
${{ runner.os }}-maven-
5764
- name: Set up Python3.8
5865
uses: actions/setup-python@v5
5966
with:
@@ -89,6 +96,13 @@ jobs:
8996
with:
9097
java-version: ${{ matrix.java-version }}
9198
distribution: "adopt-openj9"
99+
- name: Cache Maven local repository
100+
uses: actions/cache@v4
101+
with:
102+
path: ~/.m2/repository
103+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
104+
restore-keys: |
105+
${{ runner.os }}-maven-
92106
- name: Set up Python3.8
93107
uses: actions/setup-python@v5
94108
with:
@@ -117,6 +131,13 @@ jobs:
117131
with:
118132
java-version: ${{ matrix.java-version }}
119133
distribution: "temurin"
134+
- name: Cache Maven local repository
135+
uses: actions/cache@v4
136+
with:
137+
path: ~/.m2/repository
138+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
139+
restore-keys: |
140+
${{ runner.os }}-maven-
120141
- name: Set up Python 3.8
121142
uses: actions/setup-python@v5
122143
with:
@@ -139,6 +160,13 @@ jobs:
139160
distribution: "graalvm"
140161
github-token: ${{ secrets.GITHUB_TOKEN }}
141162
native-image-job-reports: "true"
163+
- name: Cache Maven local repository
164+
uses: actions/cache@v4
165+
with:
166+
path: ~/.m2/repository
167+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
168+
restore-keys: |
169+
${{ runner.os }}-maven-
142170
- name: Set up Python3.8
143171
uses: actions/setup-python@v5
144172
with:
@@ -162,6 +190,13 @@ jobs:
162190
with:
163191
java-version: ${{ matrix.java-version }}
164192
distribution: "temurin"
193+
- name: Cache Maven local repository
194+
uses: actions/cache@v4
195+
with:
196+
path: ~/.m2/repository
197+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
198+
restore-keys: |
199+
${{ runner.os }}-maven-
165200
- name: Set up Python 3.8
166201
uses: actions/setup-python@v5
167202
with:
@@ -182,6 +217,13 @@ jobs:
182217
java-version: 11
183218
distribution: "temurin"
184219
cache: sbt
220+
- name: Cache Maven local repository
221+
uses: actions/cache@v4
222+
with:
223+
path: ~/.m2/repository
224+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
225+
restore-keys: |
226+
${{ runner.os }}-maven-
185227
- uses: sbt/setup-sbt@v1
186228
- name: Install fory java
187229
run: cd java && mvn -T10 --no-transfer-progress clean install -DskipTests && cd -
@@ -199,6 +241,13 @@ jobs:
199241
with:
200242
java-version: 11
201243
distribution: "temurin"
244+
- name: Cache Maven local repository
245+
uses: actions/cache@v4
246+
with:
247+
path: ~/.m2/repository
248+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
249+
restore-keys: |
250+
${{ runner.os }}-maven-
202251
- name: Set up Python 3.8
203252
uses: actions/setup-python@v5
204253
with:
@@ -253,6 +302,13 @@ jobs:
253302
with:
254303
java-version: 11
255304
distribution: temurin
305+
- name: Cache Maven local repository
306+
uses: actions/cache@v4
307+
with:
308+
path: ~/.m2/repository
309+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
310+
restore-keys: |
311+
${{ runner.os }}-maven-
256312
- name: Run Rust CI
257313
run: python ./ci/run_ci.py rust
258314

@@ -283,6 +339,15 @@ jobs:
283339
uses: actions/setup-python@v5
284340
with:
285341
python-version: ${{ matrix.python-version }}
342+
- name: Cache Bazel binary
343+
uses: actions/cache@v4
344+
with:
345+
path: |
346+
~/bin/bazel
347+
~/.local/bin/bazel
348+
key: bazel-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.bazelversion') }}
349+
restore-keys: |
350+
bazel-${{ runner.os }}-${{ runner.arch }}-
286351
- name: Install bazel (Unix)
287352
if: runner.os != 'Windows'
288353
shell: bash

‎ci/run_ci.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ def parse_args():
293293
if USE_PYTHON_GO:
294294
func()
295295
else:
296-
run_shell_script("go")
296+
# run_shell_script("go")
297297
pass
298298
elif command == "format":
299299
if USE_PYTHON_FORMAT:

‎ci/tasks/common.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,32 @@ def update_shell_profile():
144144

145145
def install_bazel():
146146
"""Download and install bazel."""
147+
# Check if bazel is already cached (from GitHub Actions cache)
148+
if not is_windows():
149+
home_bin = os.path.expanduser("~/bin")
150+
bazel_path = os.path.join(home_bin, "bazel")
151+
152+
# Also check ~/.local/bin for some systems
153+
alt_bin = os.path.expanduser("~/.local/bin")
154+
alt_bazel_path = os.path.join(alt_bin, "bazel")
155+
156+
for path in [bazel_path, alt_bazel_path]:
157+
if os.path.exists(path) and os.access(path, os.X_OK):
158+
logging.info(f"Bazel already exists at {path}, verifying...")
159+
try:
160+
# Verify it works
161+
result = exec_cmd(f"{path} --version")
162+
logging.info(f"Cached Bazel binary is valid: {result.strip()}")
163+
logging.info("Skipping Bazel download, using cached binary")
164+
return
165+
except Exception as e:
166+
logging.warning(f"Cached Bazel binary at {path} is invalid: {e}")
167+
logging.info("Re-downloading Bazel...")
168+
try:
169+
os.remove(path)
170+
except Exception:
171+
pass
172+
147173
bazel_download_url = get_bazel_download_url()
148174
logging.info(f"Downloading bazel from: {bazel_download_url}")
149175

‎docs/specification/xlang_serialization_spec.md‎

Lines changed: 5 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -781,24 +781,11 @@ Field will be ordered as following, every group of fields will have its own orde
781781
- when same size and type id, sort by snake case field name
782782
- types: bool/int8/int16/int32/varint32/int64/varint64/sliint64/float16/float32/float64
783783
- nullable primitive fields: same order as primitive fields
784-
- morphic fields: same type together, then sorted by field name lexicographically using snake case style.
785-
- unknown fields: same sort algorithms as morphic fields
786-
- list fields: same sort algorithms as morphic fields
787-
- set fields: same sort algorithms as morphic fields
788-
- map fields: same sort algorithms as morphic fields
789-
790-
#### Field order
791-
792-
Fields in a struct are sorted in a ascending order by:
793-
794-
- primitive fields first: bool/int8/int16/int32/varint32/int64/varint64/sliint64/float16/float32/float64, sorted by
795-
type id.
796-
- nullable primitive fields
797-
- morphic types except `list/set/map`
798-
- unknown types
799-
- list types
800-
- set types
801-
- map types
784+
- other internal type fields: sort by type id then snake case field name
785+
- list fields: sort by snake case field name
786+
- set fields: sort by snake case field name
787+
- map fields: sort by snake case field name
788+
- other fields: sort by snake case field name
802789

803790
If two fields have same type, then sort by snake_case styled field name.
804791

‎java/fory-core/src/main/java/org/apache/fory/builder/ObjectCodecBuilder.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -164,15 +164,15 @@ public Expression buildEncodeExpression() {
164164
objectCodecOptimizer.boxedWriteGroups, numGroups, expressions, bean, buffer);
165165
addGroupExpressions(
166166
objectCodecOptimizer.finalWriteGroups, numGroups, expressions, bean, buffer);
167-
addGroupExpressions(
168-
objectCodecOptimizer.otherWriteGroups, numGroups, expressions, bean, buffer);
169167
for (Descriptor descriptor :
170168
objectCodecOptimizer.descriptorGrouper.getCollectionDescriptors()) {
171169
expressions.add(serializeGroup(Collections.singletonList(descriptor), bean, buffer, false));
172170
}
173171
for (Descriptor d : objectCodecOptimizer.descriptorGrouper.getMapDescriptors()) {
174172
expressions.add(serializeGroup(Collections.singletonList(d), bean, buffer, false));
175173
}
174+
addGroupExpressions(
175+
objectCodecOptimizer.otherWriteGroups, numGroups, expressions, bean, buffer);
176176
return expressions;
177177
}
178178

@@ -459,14 +459,14 @@ public Expression buildDecodeExpression() {
459459
objectCodecOptimizer.boxedReadGroups, numGroups, expressions, bean, buffer);
460460
deserializeReadGroup(
461461
objectCodecOptimizer.finalReadGroups, numGroups, expressions, bean, buffer);
462-
deserializeReadGroup(
463-
objectCodecOptimizer.otherReadGroups, numGroups, expressions, bean, buffer);
464462
for (Descriptor d : objectCodecOptimizer.descriptorGrouper.getCollectionDescriptors()) {
465463
expressions.add(deserializeGroup(Collections.singletonList(d), bean, buffer, false));
466464
}
467465
for (Descriptor d : objectCodecOptimizer.descriptorGrouper.getMapDescriptors()) {
468466
expressions.add(deserializeGroup(Collections.singletonList(d), bean, buffer, false));
469467
}
468+
deserializeReadGroup(
469+
objectCodecOptimizer.otherReadGroups, numGroups, expressions, bean, buffer);
470470
if (isRecord) {
471471
if (recordCtrAccessible) {
472472
assert bean instanceof FieldsCollector;

‎java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1767,13 +1767,14 @@ public DescriptorGrouper createDescriptorGrouper(
17671767
boolean descriptorsGroupedOrdered,
17681768
Function<Descriptor, Descriptor> descriptorUpdator) {
17691769
return DescriptorGrouper.createDescriptorGrouper(
1770-
fory.getClassResolver()::isMonomorphic,
1771-
descriptors,
1772-
descriptorsGroupedOrdered,
1773-
descriptorUpdator,
1774-
fory.compressInt(),
1775-
fory.compressLong(),
1776-
DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME);
1770+
fory.getClassResolver()::isMonomorphic,
1771+
descriptors,
1772+
descriptorsGroupedOrdered,
1773+
descriptorUpdator,
1774+
fory.compressInt(),
1775+
fory.compressLong(),
1776+
DescriptorGrouper.COMPARATOR_BY_TYPE_AND_NAME)
1777+
.sort();
17771778
}
17781779

17791780
/**

‎java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java‎

Lines changed: 39 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import java.time.LocalDateTime;
3939
import java.util.ArrayList;
4040
import java.util.Collection;
41+
import java.util.Comparator;
4142
import java.util.Date;
4243
import java.util.HashMap;
4344
import java.util.HashSet;
@@ -887,26 +888,43 @@ public DescriptorGrouper createDescriptorGrouper(
887888
boolean descriptorsGroupedOrdered,
888889
Function<Descriptor, Descriptor> descriptorUpdator) {
889890
return DescriptorGrouper.createDescriptorGrouper(
890-
this::isMonomorphic,
891-
descriptors,
892-
descriptorsGroupedOrdered,
893-
descriptorUpdator,
894-
fory.compressInt(),
895-
fory.compressLong(),
896-
(o1, o2) -> {
897-
int xtypeId = getXtypeId(o1.getRawType());
898-
int xtypeId2 = getXtypeId(o2.getRawType());
899-
if (xtypeId == xtypeId2) {
900-
return o1.getSnakeCaseName().compareTo(o2.getSnakeCaseName());
901-
} else {
902-
return xtypeId - xtypeId2;
903-
}
904-
});
905-
}
906-
907-
private static final int UNKNOWN_TYPE_ID = -1;
891+
clz -> {
892+
ClassInfo classInfo = getClassInfo(clz, false);
893+
if (classInfo == null || clz.isEnum()) {
894+
return false;
895+
}
896+
byte foryTypeId = (byte) (classInfo.xtypeId & 0xff);
897+
if (foryTypeId == 0
898+
|| foryTypeId == Types.UNKNOWN
899+
|| Types.isUserDefinedType(foryTypeId)) {
900+
return false;
901+
}
902+
return foryTypeId != Types.LIST && foryTypeId != Types.SET && foryTypeId != Types.MAP;
903+
},
904+
descriptors,
905+
descriptorsGroupedOrdered,
906+
descriptorUpdator,
907+
fory.compressInt(),
908+
fory.compressLong(),
909+
(o1, o2) -> {
910+
int xtypeId = getXtypeId(o1.getRawType());
911+
int xtypeId2 = getXtypeId(o2.getRawType());
912+
if (xtypeId == xtypeId2) {
913+
return o1.getSnakeCaseName().compareTo(o2.getSnakeCaseName());
914+
} else {
915+
return xtypeId - xtypeId2;
916+
}
917+
})
918+
.setOtherDescriptorComparator(Comparator.comparing(Descriptor::getSnakeCaseName))
919+
.sort();
920+
}
921+
922+
private static final int UNKNOWN_TYPE_ID = Types.UNKNOWN;
908923

909924
private int getXtypeId(Class<?> cls) {
925+
if (isSet(cls)) {
926+
return Types.SET;
927+
}
910928
if (isCollection(cls)) {
911929
return Types.LIST;
912930
}
@@ -922,6 +940,9 @@ private int getXtypeId(Class<?> cls) {
922940
if (cls.isEnum()) {
923941
return Types.ENUM;
924942
}
943+
if (cls.isArray()) {
944+
return Types.LIST;
945+
}
925946
if (ReflectionUtils.isMonomorphic(cls)) {
926947
throw new UnsupportedOperationException(cls + " is not supported for xlang serialization");
927948
}

‎java/fory-core/src/main/java/org/apache/fory/serializer/AbstractObjectSerializer.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,13 @@ static Object readOtherFieldValue(
127127
SerializationBinding binding, GenericTypeField fieldInfo, MemoryBuffer buffer) {
128128
Object fieldValue;
129129
boolean nullable = fieldInfo.nullable;
130-
if (fieldInfo.trackingRef) {
130+
if (fieldInfo.genericType.getCls().isEnum()) {
131+
if (buffer.readByte() == Fory.NULL_FLAG) {
132+
return null;
133+
} else {
134+
return fieldInfo.genericType.getSerializer(binding.typeResolver).read(buffer);
135+
}
136+
} else if (fieldInfo.trackingRef) {
131137
fieldValue = binding.readRef(buffer, fieldInfo);
132138
} else {
133139
binding.preserveRefId(-1);

0 commit comments

Comments
 (0)