Skip to content

Commit cc91f02

Browse files
chaokunyangCherishCaistevenschlanskerpenguin-wwywenyangwang
authored
chore: cp more commits to 0.11.0 (#2319)
<!-- **Thanks for contributing to Fory.** **If this is your first time opening a PR on fory, you can refer to [CONTRIBUTING.md](https://github.com/apache/fory/blob/main/CONTRIBUTING.md).** Contribution Checklist - The **Apache Fory (incubating)** community has restrictions on the naming of pr titles. You can also find instructions in [CONTRIBUTING.md](https://github.com/apache/fory/blob/main/CONTRIBUTING.md). - Fory has a strong focus on performance. If the PR you submit will have an impact on performance, please benchmark it first and provide the benchmark result here. --> ## What does this PR do? <!-- Describe the purpose of this PR. --> ## Related issues <!-- Is there any related issue? Please attach here. - #xxxx0 - #xxxx1 - #xxxx2 --> ## Does this PR introduce any user-facing change? <!-- If any user-facing interface changes, please [open an issue](https://github.com/apache/fory/issues/new/choose) describing the need to do so and update the document if necessary. --> - [ ] Does this PR introduce any public API change? - [ ] Does this PR introduce any binary protocol compatibility change? ## Benchmark <!-- When the PR has an impact on performance (if you don't know whether the PR will have an impact on performance, you can submit the PR first, and if it will have impact on performance, the code reviewer will explain it), be sure to attach a benchmark data here. --> --------- Co-authored-by: hongwen.chw <785427346@qq.com> Co-authored-by: Steven Schlansker <stevenschlansker@gmail.com> Co-authored-by: penguin_wwy <940375606@qq.com> Co-authored-by: wenyangwang <wenyangwang@tencent.com> Co-authored-by: OmCheeLin <19563671928@163.com>
1 parent d9e0ed3 commit cc91f02

File tree

19 files changed

+542
-172
lines changed

19 files changed

+542
-172
lines changed

‎bazel/fory_deps_setup.bzl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -145,8 +145,8 @@ def setup_deps():
145145
auto_http_archive(
146146
name = "cython",
147147
build_file = "@com_github_grpc_grpc//third_party:cython.BUILD",
148-
url = "https://github.com/cython/cython/releases/download/3.1.0a1/cython-3.1.0a1.tar.gz",
149-
sha256 = "35b53f6947c3133452b84f0f9703f222deb9b02874861427a45e63c891379440",
148+
url = "https://github.com/cython/cython/releases/download/3.1.1/cython-3.1.1.tar.gz",
149+
sha256 = "505ccd413669d5132a53834d792c707974248088c4f60c497deb1b416e366397",
150150
)
151151
auto_http_archive(
152152
name = "com_google_googletest",

‎integration_tests/cpython_benchmark/fory_benchmark.py‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
# under the License.
1717

1818
import argparse
19-
import array
2019
from dataclasses import dataclass
2120
import datetime
2221
import os
@@ -146,8 +145,7 @@ class ComplexObject2:
146145
f8=2**63 - 1,
147146
f9=1.0 / 2,
148147
f10=1 / 3.0,
149-
f11=array.array("h", [1, 2]),
150-
f12=[-1, 4],
148+
f11=[-1, 4],
151149
)
152150

153151

@@ -157,6 +155,13 @@ def fory_object(language, ref_tracking, obj):
157155
fory.deserialize(binary)
158156

159157

158+
def fory_data_class(language, ref_tracking, obj, register_callable):
159+
fory = pyfory.Fory(language=language, ref_tracking=ref_tracking)
160+
register_callable(fory)
161+
binary = fory.serialize(obj)
162+
fory.deserialize(binary)
163+
164+
160165
def benchmark_args():
161166
parser = argparse.ArgumentParser(description="Fory Benchmark")
162167
parser.add_argument("--xlang", action="store_true", default=False)
@@ -190,7 +195,6 @@ def micro_benchmark():
190195
runner.bench_func(
191196
"fory_large_tuple", fory_object, language, not args.no_ref, LARGE_TUPLE
192197
)
193-
runner.bench_func("fory_list", fory_object, language, not args.no_ref, LIST)
194198
runner.bench_func(
195199
"fory_large_float_tuple",
196200
fory_object,
@@ -209,8 +213,22 @@ def micro_benchmark():
209213
runner.bench_func(
210214
"fory_large_list", fory_object, language, not args.no_ref, LARGE_LIST
211215
)
216+
217+
def register_complex(fory):
218+
if args.xlang:
219+
fory.register_type(ComplexObject1, typename="example.ComplexObject1")
220+
fory.register_type(ComplexObject2, typename="example.ComplexObject2")
221+
else:
222+
fory.register_type(ComplexObject1)
223+
fory.register_type(ComplexObject2)
224+
212225
runner.bench_func(
213-
"fory_complex", fory_object, language, not args.no_ref, COMPLEX_OBJECT
226+
"fory_complex",
227+
fory_data_class,
228+
language,
229+
not args.no_ref,
230+
COMPLEX_OBJECT,
231+
register_complex,
214232
)
215233

216234

‎java/fory-core/src/main/java/org/apache/fory/meta/ClassDef.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -843,6 +843,9 @@ public ArrayFieldType(
843843

844844
@Override
845845
public TypeRef<?> toTypeToken(TypeResolver classResolver, TypeRef<?> declared) {
846+
while (declared != null && declared.isArray()) {
847+
declared = declared.getComponentType();
848+
}
846849
TypeRef<?> componentTypeRef = componentType.toTypeToken(classResolver, declared);
847850
Class<?> componentRawType = componentTypeRef.getRawType();
848851
if (NonexistentClass.class.isAssignableFrom(componentRawType)) {
@@ -993,6 +996,9 @@ private static FieldType buildFieldType(TypeResolver resolver, GenericType gener
993996
}
994997
if (rawType.isArray()) {
995998
Class<?> elemType = rawType.getComponentType();
999+
while (elemType.isArray()) {
1000+
elemType = elemType.getComponentType();
1001+
}
9961002
if (isXlang && !elemType.isPrimitive()) {
9971003
return new CollectionFieldType(
9981004
xtypeId,

‎java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,9 @@ private static class ExtRegistry {
255255
new IdentityMap<>(estimatedNumRegistered);
256256
private final BiMap<String, Class<?>> registeredClasses =
257257
HashBiMap.create(estimatedNumRegistered);
258+
// cache absClassInfo, support customized serializer for abstract or interface.
259+
private final IdentityMap<Class<?>, ClassInfo> absClassInfo =
260+
new IdentityMap<>(estimatedNumRegistered, foryMapLoadFactor);
258261
// avoid potential recursive call for seq codec generation.
259262
// ex. A->field1: B, B.field1: A
260263
private final Set<Class<?>> getClassCtx = new HashSet<>();
@@ -466,7 +469,7 @@ public void register(Class<?> cls, int classId) {
466469
classInfo = new ClassInfo(this, cls, null, id, NOT_SUPPORT_XLANG);
467470
// make `extRegistry.registeredClassIdMap` and `classInfoMap` share same classInfo
468471
// instances.
469-
classInfoMap.put(cls, classInfo);
472+
setClassInfo(cls, classInfo);
470473
}
471474
// serializer will be set lazily in `addSerializer` method if it's null.
472475
registeredId2ClassInfo[id] = classInfo;
@@ -514,7 +517,7 @@ public void register(Class<?> cls, String namespace, String name) {
514517
MetaStringBytes nameBytes = metaStringResolver.getOrCreateMetaStringBytes(encodeTypeName(name));
515518
ClassInfo classInfo =
516519
new ClassInfo(cls, fullNameBytes, nsBytes, nameBytes, false, null, NO_CLASS_ID, (short) -1);
517-
classInfoMap.put(cls, classInfo);
520+
setClassInfo(cls, classInfo);
518521
compositeNameBytes2ClassInfo.put(
519522
new TypeNameBytes(nsBytes.hashCode, nameBytes.hashCode), classInfo);
520523
extRegistry.registeredClasses.put(fullname, cls);
@@ -857,7 +860,7 @@ private void addSerializer(Class<?> type, Serializer<?> serializer) {
857860

858861
if (classInfo == null || classId != classInfo.classId) {
859862
classInfo = new ClassInfo(this, type, null, classId, (short) 0);
860-
classInfoMap.put(type, classInfo);
863+
setClassInfo(type, classInfo);
861864
if (registered) {
862865
registeredId2ClassInfo[classId] = classInfo;
863866
}
@@ -1294,6 +1297,10 @@ public ClassInfo getClassInfo(Class<?> cls, boolean createClassInfoIfNotFound) {
12941297

12951298
void setClassInfo(Class<?> cls, ClassInfo classInfo) {
12961299
classInfoMap.put(cls, classInfo);
1300+
// in order to support customized serializer for abstract or interface.
1301+
if (!cls.isPrimitive() && (ReflectionUtils.isAbstract(cls) || cls.isInterface())) {
1302+
extRegistry.absClassInfo.put(cls, classInfo);
1303+
}
12971304
}
12981305

12991306
@Internal
@@ -1363,6 +1370,23 @@ private Serializer createSerializer(Class<?> cls) {
13631370
return shimSerializer;
13641371
}
13651372

1373+
// support customized serializer for abstract or interface.
1374+
if (!extRegistry.absClassInfo.isEmpty()) {
1375+
Class<?> tmpCls = cls;
1376+
while (tmpCls != null && tmpCls != Object.class) {
1377+
ClassInfo absClass = null;
1378+
if ((absClass = extRegistry.absClassInfo.get(tmpCls.getSuperclass())) != null) {
1379+
return absClass.serializer;
1380+
}
1381+
for (Class<?> tmpI : tmpCls.getInterfaces()) {
1382+
if ((absClass = extRegistry.absClassInfo.get(tmpI)) != null) {
1383+
return absClass.serializer;
1384+
}
1385+
}
1386+
tmpCls = tmpCls.getSuperclass();
1387+
}
1388+
}
1389+
13661390
Class<? extends Serializer> serializerClass = getSerializerClass(cls);
13671391
Serializer serializer = Serializers.newSerializer(fory, cls, serializerClass);
13681392
if (ForyCopyable.class.isAssignableFrom(cls)) {
@@ -1772,7 +1796,7 @@ public void writeClassInternal(MemoryBuffer buffer, Class<?> cls) {
17721796
classInfo =
17731797
new ClassInfo(
17741798
this, cls, null, classId == null ? NO_CLASS_ID : classId, NOT_SUPPORT_XLANG);
1775-
classInfoMap.put(cls, classInfo);
1799+
setClassInfo(cls, classInfo);
17761800
}
17771801
writeClassInternal(buffer, classInfo);
17781802
}
@@ -1953,7 +1977,7 @@ private ClassInfo populateBytesToClassInfo(
19531977
// don't create serializer here, if the class is an interface,
19541978
// there won't be serializer since interface has no instance.
19551979
if (!classInfoMap.containsKey(cls)) {
1956-
classInfoMap.put(cls, classInfo);
1980+
setClassInfo(cls, classInfo);
19571981
}
19581982
}
19591983
compositeNameBytes2ClassInfo.put(typeNameBytes, classInfo);

‎java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -431,7 +431,7 @@ private ClassInfo buildClassInfo(Class<?> cls) {
431431
serializer = getCollectionSerializer(cls);
432432
}
433433
xtypeId = Types.LIST;
434-
} else if (cls.isArray() && !TypeUtils.getArrayComponent(cls).isPrimitive()) {
434+
} else if (cls.isArray() && !cls.getComponentType().isPrimitive()) {
435435
serializer = new ArraySerializers.ObjectArraySerializer(fory, cls);
436436
xtypeId = Types.LIST;
437437
} else if (classResolver.isMap(cls)) {

‎java/fory-core/src/main/java/org/apache/fory/type/CustomTypeRegistry.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,24 +20,32 @@
2020
package org.apache.fory.type;
2121

2222
import org.apache.fory.annotation.Internal;
23+
import org.apache.fory.reflect.TypeRef;
2324

2425
@Internal
2526
public interface CustomTypeRegistry {
2627
CustomTypeRegistry EMPTY =
2728
new CustomTypeRegistry() {
2829
@Override
29-
public boolean hasCodec(final Class<?> beanType, final Class<?> fieldType) {
30-
return false;
30+
public TypeRef<?> replacementTypeFor(final Class<?> beanType, final Class<?> fieldType) {
31+
return null;
3132
}
3233

3334
@Override
3435
public boolean canConstructCollection(
3536
final Class<?> collectionType, final Class<?> elementType) {
3637
return false;
3738
}
39+
40+
@Override
41+
public boolean isExtraSupportedType(final TypeRef<?> type) {
42+
return false;
43+
}
3844
};
3945

40-
boolean hasCodec(Class<?> beanType, Class<?> fieldType);
46+
TypeRef<?> replacementTypeFor(Class<?> beanType, Class<?> fieldType);
4147

4248
boolean canConstructCollection(Class<?> collectionType, Class<?> elementType);
49+
50+
boolean isExtraSupportedType(TypeRef<?> type);
4351
}

‎java/fory-core/src/main/java/org/apache/fory/type/TypeUtils.java‎

Lines changed: 39 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -605,6 +605,10 @@ public static boolean isBean(TypeRef<?> typeRef, TypeResolutionContext ctx) {
605605
if (Modifier.isAbstract(cls.getModifiers()) || Modifier.isInterface(cls.getModifiers())) {
606606
return false;
607607
}
608+
if (ctx.getWalkedTypePath().contains(typeRef)
609+
|| ctx.getCustomTypeRegistry().isExtraSupportedType(typeRef)) {
610+
return false;
611+
}
608612
// since we need to access class in generated code in our package, the class must be public
609613
// if ReflectionUtils.hasNoArgConstructor(cls) return false, we use Unsafe to create object.
610614
if (Modifier.isPublic(cls.getModifiers())) {
@@ -624,17 +628,21 @@ public static boolean isBean(TypeRef<?> typeRef, TypeResolutionContext ctx) {
624628
&& !ITERABLE_TYPE.isSupertypeOf(typeRef)
625629
&& !MAP_TYPE.isSupertypeOf(typeRef);
626630
if (maybe) {
627-
return Descriptor.getDescriptors(cls).stream()
628-
.allMatch(
629-
d -> {
630-
TypeRef<?> t = d.getTypeRef();
631-
// do field modifiers and getter/setter validation here, not in getDescriptors.
632-
// If Modifier.isFinal(d.getModifiers()), use reflection
633-
// private field that doesn't have getter/setter will be handled by reflection.
634-
return ctx.getCustomTypeRegistry().hasCodec(cls, t.getRawType())
635-
|| isSupported(t, newTypePath)
636-
|| isBean(t, newTypePath);
637-
});
631+
for (Descriptor d : Descriptor.getDescriptors(cls)) {
632+
TypeRef<?> t = d.getTypeRef();
633+
// do field modifiers and getter/setter validation here, not in getDescriptors.
634+
// If Modifier.isFinal(d.getModifiers()), use reflection
635+
// private field that doesn't have getter/setter will be handled by reflection.
636+
TypeRef<?> replacementType =
637+
ctx.getCustomTypeRegistry().replacementTypeFor(cls, t.getRawType());
638+
if (replacementType != null) {
639+
t = replacementType;
640+
}
641+
if (!isSupported(t, newTypePath)) {
642+
return false;
643+
}
644+
}
645+
return true;
638646
} else {
639647
return false;
640648
}
@@ -658,10 +666,16 @@ private static boolean isSupported(TypeRef<?> typeRef, TypeResolutionContext ctx
658666
// box.
659667
return true;
660668
}
661-
if (SUPPORTED_TYPES.contains(typeRef)) {
669+
TypeRef<?> replacementType =
670+
ctx.getCustomTypeRegistry()
671+
.replacementTypeFor(ctx.getEnclosingType().getRawType(), typeRef.getRawType());
672+
if (replacementType != null) {
673+
return isSupported(replacementType, ctx);
674+
} else if (SUPPORTED_TYPES.contains(typeRef)
675+
|| ctx.getCustomTypeRegistry().isExtraSupportedType(typeRef)) {
662676
return true;
663677
} else if (typeRef.isArray()) {
664-
return isSupported(Objects.requireNonNull(typeRef.getComponentType()));
678+
return isSupported(Objects.requireNonNull(typeRef.getComponentType()), ctx);
665679
} else if (ITERABLE_TYPE.isSupertypeOf(typeRef)) {
666680
TypeRef<?> elementType = getElementType(typeRef);
667681
boolean isSuperOfArrayList = cls.isAssignableFrom(ArrayList.class);
@@ -672,7 +686,7 @@ private static boolean isSupported(TypeRef<?> typeRef, TypeResolutionContext ctx
672686
.canConstructCollection(typeRef.getRawType(), elementType.getRawType())) {
673687
return false;
674688
}
675-
return isSupported(getElementType(typeRef));
689+
return isSupported(elementType, ctx);
676690
} else if (MAP_TYPE.isSupertypeOf(typeRef)) {
677691
boolean isSuperOfHashMap = cls.isAssignableFrom(HashMap.class);
678692
if (!isSuperOfHashMap && (cls.isInterface() || Modifier.isAbstract(cls.getModifiers()))) {
@@ -684,7 +698,7 @@ private static boolean isSupported(TypeRef<?> typeRef, TypeResolutionContext ctx
684698
return true;
685699
} else {
686700
ctx.checkNoCycle(typeRef);
687-
return isBean(typeRef, ctx.appendTypePath(typeRef));
701+
return isBean(typeRef, ctx);
688702
}
689703
}
690704

@@ -715,30 +729,32 @@ private static LinkedHashSet<Class<?>> listBeansRecursiveInclusive(
715729
LinkedHashSet<Class<?>> beans = new LinkedHashSet<>();
716730
Class<?> enclosingType = ctx.getEnclosingType().getRawType();
717731
Class<?> type = typeRef.getRawType();
718-
TypeResolutionContext newCtx = ctx;
719-
if (ctx.getCustomTypeRegistry().hasCodec(enclosingType, type)) {
732+
TypeRef<?> replacementType =
733+
ctx.getCustomTypeRegistry().replacementTypeFor(enclosingType, type);
734+
if (replacementType != null && !replacementType.equals(typeRef)) {
735+
beans.addAll(listBeansRecursiveInclusive(replacementType, ctx));
720736
return beans;
721737
} else if (type == Optional.class) {
722738
TypeRef<?> elemType = getTypeArguments(typeRef).get(0);
723-
beans.addAll(listBeansRecursiveInclusive(elemType, newCtx));
739+
beans.addAll(listBeansRecursiveInclusive(elemType, ctx));
724740
} else if (isCollection(type) || Iterable.class == type) {
725741
TypeRef<?> elementType = getElementType(typeRef);
726-
beans.addAll(listBeansRecursiveInclusive(elementType, newCtx));
742+
beans.addAll(listBeansRecursiveInclusive(elementType, ctx));
727743
} else if (isMap(type)) {
728744
Tuple2<TypeRef<?>, TypeRef<?>> mapKeyValueType = getMapKeyValueType(typeRef);
729-
TypeResolutionContext mapCtx = newCtx;
745+
TypeResolutionContext mapCtx = ctx;
730746
beans.addAll(listBeansRecursiveInclusive(mapKeyValueType.f0, mapCtx));
731747
beans.addAll(listBeansRecursiveInclusive(mapKeyValueType.f1, mapCtx));
732748
} else if (type.isArray()) {
733749
Class<?> arrayComponent = getArrayComponent(type);
734-
beans.addAll(listBeansRecursiveInclusive(TypeRef.of(arrayComponent), newCtx));
735-
} else if (isBean(type, newCtx)) {
750+
beans.addAll(listBeansRecursiveInclusive(TypeRef.of(arrayComponent), ctx));
751+
} else if (isBean(type, ctx)) {
736752
List<Descriptor> descriptors = Descriptor.getDescriptors(type);
737753
beans.add(type);
738754
for (Descriptor descriptor : descriptors) {
739755
ctx.checkNoCycle(typeRef);
740756
beans.addAll(
741-
listBeansRecursiveInclusive(descriptor.getTypeRef(), newCtx.appendTypePath(typeRef)));
757+
listBeansRecursiveInclusive(descriptor.getTypeRef(), ctx.appendTypePath(typeRef)));
742758
}
743759
}
744760
return beans;

0 commit comments

Comments
 (0)