Skip to content
Closed
3 changes: 3 additions & 0 deletions java/fory-core/src/main/java/org/apache/fory/Fory.java
Original file line number Diff line number Diff line change
Expand Up @@ -590,6 +590,8 @@ public void xwriteData(MemoryBuffer buffer, ClassInfo classInfo, Object obj) {
buffer.writeInt16((Short) obj);
break;
case Types.INT32:
buffer.writeInt32((Integer) obj);
break;
case Types.VAR_INT32:
// TODO(chaokunyang) support other encoding
buffer.writeVarInt32((Integer) obj);
Expand Down Expand Up @@ -1106,6 +1108,7 @@ public Object xreadNonRef(MemoryBuffer buffer, ClassInfo classInfo) {
case Types.INT16:
return buffer.readInt16();
case Types.INT32:
return buffer.readInt32();
case Types.VAR_INT32:
// TODO(chaokunyang) support other encoding
return buffer.readVarInt32();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -421,7 +421,7 @@ private void finish() {
if (language != Language.JAVA) {
stringRefIgnored = true;
longEncoding = LongEncoding.PVL;
compressInt = true;
// compressInt = true;
compressString = true;
}
if (ENABLE_CLASS_REGISTRATION_FORCIBLY) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -586,7 +586,11 @@ private void registerDefaultTypes() {
registerDefaultTypes(Types.BOOL, Boolean.class, boolean.class, AtomicBoolean.class);
registerDefaultTypes(Types.INT8, Byte.class, byte.class);
registerDefaultTypes(Types.INT16, Short.class, short.class);
registerDefaultTypes(Types.INT32, Integer.class, int.class, AtomicInteger.class);
if (this.fory.compressInt()) {
registerDefaultTypes(Types.VAR_INT32, Integer.class, int.class, AtomicInteger.class);
} else {
registerDefaultTypes(Types.INT32, Integer.class, int.class, AtomicInteger.class);
}
registerDefaultTypes(Types.INT64, Long.class, long.class, AtomicLong.class);
registerDefaultTypes(Types.FLOAT32, Float.class, float.class);
registerDefaultTypes(Types.FLOAT64, Double.class, double.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ public static boolean isPrimitiveType(int typeId) {
case INT8:
case INT16:
case INT32:
case VAR_INT32:
case INT64:
case FLOAT32:
case FLOAT64:
Expand Down
47 changes: 47 additions & 0 deletions java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -877,6 +877,53 @@ public void testStructVersionCheck() throws java.io.IOException {
Assert.assertEquals(fory.deserialize(buffer2), obj);
}

@Data
static class IntWrapper {
int f1;

IntWrapper(int f1) {
this.f1 = f1;
}
}

@Test
public void testCompressInt() throws java.io.IOException {
String caseName = "test_compress_int";
List<String> command = setTestCase(caseName);
Fory fory =
Fory.builder()
.withLanguage(Language.XLANG)
.withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT)
.withCodegen(false)
.withClassVersionCheck(true)
.withIntCompressed(false)
.build();
Fory foryCompressed =
Fory.builder()
.withLanguage(Language.XLANG)
.withCompatibleMode(CompatibleMode.SCHEMA_CONSISTENT)
.withCodegen(false)
.withClassVersionCheck(true)
.withIntCompressed(true)
.build();
fory.register(IntWrapper.class, 100);
foryCompressed.register(IntWrapper.class, 101);

IntWrapper item = new IntWrapper(42);
IntWrapper itemCompressed = new IntWrapper(43);

MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32);
fory.serialize(buffer, item);
foryCompressed.serialize(buffer, itemCompressed);
Path dataFile = Files.createTempFile(caseName, "data");
Pair<Map<String, String>, File> env_workdir =
setFilePath(dataFile, buffer.getBytes(0, buffer.writerIndex()));
Assert.assertTrue(executeCommand(command, 30, env_workdir.getLeft(), env_workdir.getRight()));
MemoryBuffer buffer2 = MemoryUtils.wrap(Files.readAllBytes(dataFile));
Assert.assertEquals(fory.deserialize(buffer2), item);
Assert.assertEquals(foryCompressed.deserialize(buffer2), itemCompressed);
}

/**
* Execute an external command.
*
Expand Down
11 changes: 11 additions & 0 deletions python/pyfory/_fory.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ class Fory:
"depth",
"field_nullable",
"policy",
"compress_int",
)

def __init__(
Expand Down Expand Up @@ -216,6 +217,12 @@ def __init__(
(xlang=False), regardless of Optional annotation. Ignored in cross-language
mode.

compress_int: Enable integer compression using varint encoding (default: True).
When enabled, integers are encoded using variable-length encoding
for more efficient serialization of small integers. This reduces
the serialized size but may have minor performance impact on
serialization/deserialization.

Example:
>>> # Python-native mode with reference tracking
>>> fory = Fory(ref=True)
Expand All @@ -237,6 +244,10 @@ def __init__(
if kwargs.get("require_type_registration") is not None:
strict = kwargs.get("require_type_registration")
self.strict = _ENABLE_TYPE_REGISTRATION_FORCIBLY or strict
if kwargs.get("compress_int") is not None:
self.compress_int = kwargs.get("compress_int")
else:
self.compress_int = True
self.policy = policy or DEFAULT_POLICY
self.compatible = compatible
self.field_nullable = field_nullable if self.is_py else False
Expand Down
5 changes: 4 additions & 1 deletion python/pyfory/_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,10 @@ def _initialize_common(self):
register(bool, type_id=TypeId.BOOL, serializer=BooleanSerializer)
register(int8, type_id=TypeId.INT8, serializer=ByteSerializer)
register(int16, type_id=TypeId.INT16, serializer=Int16Serializer)
register(int32, type_id=TypeId.INT32, serializer=Int32Serializer)
int32_type_id = TypeId.INT32
if self.fory.compress_int:
int32_type_id = TypeId.VAR_INT32
register(int32, type_id=int32_type_id, serializer=Int32Serializer)
register(int64, type_id=TypeId.INT64, serializer=Int64Serializer)
register(int, type_id=TypeId.INT64, serializer=Int64Serializer)
register(
Expand Down
10 changes: 8 additions & 2 deletions python/pyfory/_serializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,10 +122,16 @@ def read(self, buffer):

class Int32Serializer(XlangCompatibleSerializer):
def write(self, buffer, value):
buffer.write_varint32(value)
if self.fory.compress_int:
buffer.write_varint32(value)
else:
buffer.write_int32(value)

def read(self, buffer):
return buffer.read_varint32()
if self.fory.compress_int:
return buffer.read_varint32()
else:
return buffer.read_int32()


class Int64Serializer(Serializer):
Expand Down
21 changes: 19 additions & 2 deletions python/pyfory/serialization.pyx
Original file line number Diff line number Diff line change
Expand Up @@ -902,6 +902,7 @@ cdef class Fory:
cdef readonly c_bool compatible
cdef readonly c_bool field_nullable
cdef readonly object policy
cdef readonly c_bool compress_int
cdef readonly MapRefResolver ref_resolver
cdef readonly TypeResolver type_resolver
cdef readonly MetaStringResolver metastring_resolver
Expand Down Expand Up @@ -964,6 +965,12 @@ cdef class Fory:
(xlang=False), regardless of Optional annotation. Ignored in cross-language
mode.

compress_int: Enable integer compression using varint encoding (default: True).
When enabled, integers are encoded using variable-length encoding
for more efficient serialization of small integers. This reduces
the serialized size but may have minor performance impact on
serialization/deserialization.

Example:
>>> # Python-native mode with reference tracking
>>> fory = Fory(ref=True)
Expand All @@ -982,6 +989,10 @@ cdef class Fory:
self.strict = True
else:
self.strict = False
if kwargs.get("compress_int") is not None:
self.compress_int = kwargs.get("compress_int")
else:
self.compress_int = True
self.policy = policy or DEFAULT_POLICY
self.compatible = compatible
self.ref_tracking = ref
Expand Down Expand Up @@ -1719,10 +1730,16 @@ cdef class Int16Serializer(XlangCompatibleSerializer):
@cython.final
cdef class Int32Serializer(XlangCompatibleSerializer):
cpdef inline write(self, Buffer buffer, value):
buffer.write_varint32(value)
if self.fory.compress_int:
buffer.write_varint32(value)
else:
buffer.write_int32(value)

cpdef inline read(self, Buffer buffer):
return buffer.read_varint32()
if self.fory.compress_int:
return buffer.read_varint32()
else:
return buffer.read_int32()


@cython.final
Expand Down
20 changes: 16 additions & 4 deletions python/pyfory/serializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,10 @@ def _get_write_stmt_for_codegen(self, serializer, buffer, field_value):
elif isinstance(serializer, Int16Serializer):
return f"{buffer}.write_int16({field_value})"
elif isinstance(serializer, Int32Serializer):
return f"{buffer}.write_varint32({field_value})"
if self.fory.compress_int:
return f"{buffer}.write_varint32({field_value})"
else:
return f"{buffer}.write_int32({field_value})"
elif isinstance(serializer, Int64Serializer):
return f"{buffer}.write_varint64({field_value})"
elif isinstance(serializer, Float32Serializer):
Expand All @@ -367,7 +370,10 @@ def _get_read_stmt_for_codegen(self, serializer, buffer, field_value):
elif isinstance(serializer, Int16Serializer):
return f"{field_value} = {buffer}.read_int16()"
elif isinstance(serializer, Int32Serializer):
return f"{field_value} = {buffer}.read_varint32()"
if self.fory.compress_int:
return f"{field_value} = {buffer}.read_varint32()"
else:
return f"{field_value} = {buffer}.read_int32()"
elif isinstance(serializer, Int64Serializer):
return f"{field_value} = {buffer}.read_varint64()"
elif isinstance(serializer, Float32Serializer):
Expand All @@ -388,7 +394,10 @@ def _write_non_nullable_field(self, buffer, field_value, serializer):
elif isinstance(serializer, Int16Serializer):
buffer.write_int16(field_value)
elif isinstance(serializer, Int32Serializer):
buffer.write_varint32(field_value)
if self.fory.compress_int:
buffer.write_varint32(field_value)
else:
buffer.write_int32(field_value)
elif isinstance(serializer, Int64Serializer):
buffer.write_varint64(field_value)
elif isinstance(serializer, Float32Serializer):
Expand All @@ -409,7 +418,10 @@ def _read_non_nullable_field(self, buffer, serializer):
elif isinstance(serializer, Int16Serializer):
return buffer.read_int16()
elif isinstance(serializer, Int32Serializer):
return buffer.read_varint32()
if self.fory.compress_int:
return buffer.read_varint32()
else:
return buffer.read_int32()
elif isinstance(serializer, Int64Serializer):
return buffer.read_varint64()
elif isinstance(serializer, Float32Serializer):
Expand Down
1 change: 1 addition & 0 deletions python/pyfory/type.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,7 @@ def is_namespaced_type(type_id: int) -> bool:
TypeId.INT8,
TypeId.INT16,
TypeId.INT32,
TypeId.VAR_INT32,
TypeId.INT64,
TypeId.FLOAT16,
TypeId.FLOAT32,
Expand Down
Loading
Loading