Skip to content

feat(Rust): support sli_int64 compression #2914

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feat(Rust): support sli_int64 compression #2914

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
67b7f70
support sli_i64 in rust
urlyy Nov 15, 2025
b2b7a0a
commit rust change
urlyy Nov 15, 2025
fa9a21b
fix lint
urlyy Nov 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 33 additions & 12 deletions java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ public void testBufferVar() throws IOException {
List<String> command = setTestCase(caseName);
Path dataFile = Files.createTempFile(caseName, "data");
MemoryBuffer buffer = MemoryUtils.buffer(100);
int[] varInt32Values = {
int[] int32Values = {
Integer.MIN_VALUE,
Integer.MIN_VALUE + 1,
-1000000,
Expand All @@ -161,11 +161,14 @@ public void testBufferVar() throws IOException {
Integer.MAX_VALUE - 1,
Integer.MAX_VALUE
};
for (int value : varInt32Values) {
for (int value : int32Values) {
buffer.writeInt32(value);
}
for (int value : int32Values) {
buffer.writeVarInt32(value);
}

int[] varUint32Values = {
int[] uint32Values = {
0,
1,
127,
Expand All @@ -179,11 +182,11 @@ public void testBufferVar() throws IOException {
Integer.MAX_VALUE - 1,
Integer.MAX_VALUE
};
for (int value : varUint32Values) {
for (int value : uint32Values) {
buffer.writeVarUint32(value);
}

long[] varUint64Values = {
long[] uint64Values = {
0L,
1L,
127L,
Expand All @@ -204,11 +207,11 @@ public void testBufferVar() throws IOException {
72057594037927936L,
Long.MAX_VALUE,
};
for (long value : varUint64Values) {
for (long value : uint64Values) {
buffer.writeVarUint64(value);
}

long[] varInt64Values = {
long[] int64Values = {
Long.MIN_VALUE,
Long.MIN_VALUE + 1,
-1000000000000L,
Expand All @@ -225,7 +228,13 @@ public void testBufferVar() throws IOException {
Long.MAX_VALUE - 1,
Long.MAX_VALUE
};
for (long value : varInt64Values) {
for (long value : int64Values) {
buffer.writeInt64(value);
}
for (long value : int64Values) {
buffer.writeSliInt64(value);
}
for (long value : int64Values) {
buffer.writeVarInt64(value);
}

Expand All @@ -234,19 +243,31 @@ public void testBufferVar() throws IOException {
Assert.assertTrue(executeCommand(command, 30, env_workdir.getLeft(), env_workdir.getRight()));

buffer = MemoryUtils.wrap(Files.readAllBytes(dataFile));
for (int expected : varInt32Values) {
for (int expected : int32Values) {
int actual = buffer.readInt32();
Assert.assertEquals(actual, expected);
}
for (int expected : int32Values) {
int actual = buffer.readVarInt32();
Assert.assertEquals(actual, expected);
}
for (int expected : varUint32Values) {
for (int expected : uint32Values) {
int actual = buffer.readVarUint32();
Assert.assertEquals(actual, expected);
}
for (long expected : varUint64Values) {
for (long expected : uint64Values) {
long actual = buffer.readVarUint64();
Assert.assertEquals(actual, expected);
}
for (long expected : varInt64Values) {
for (long expected : int64Values) {
long actual = buffer.readInt64();
Assert.assertEquals(actual, expected);
}
for (long expected : int64Values) {
long actual = buffer.readSliInt64();
Assert.assertEquals(actual, expected);
}
for (long expected : int64Values) {
long actual = buffer.readVarInt64();
Assert.assertEquals(actual, expected);
}
Expand Down
27 changes: 27 additions & 0 deletions rust/fory-core/src/buffer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@ use std::cmp::max;
/// Threshold for using SIMD optimizations in string operations.
/// For buffers smaller than this, direct copy is faster than SIMD setup overhead.
const SIMD_THRESHOLD: usize = 128;
const HALF_MAX_INT: i64 = i32::MAX as i64 / 2;
const HALF_MIN_INT: i64 = i32::MIN as i64 / 2;
const SLI_BIG_LONG_FLAG: u8 = 0b1;

pub struct Writer<'a> {
pub(crate) bf: &'a mut Vec<u8>,
Expand Down Expand Up @@ -174,6 +177,17 @@ impl<'a> Writer<'a> {
}
}

#[inline(always)]
pub fn write_sliint64(&mut self, value: i64) {
if (HALF_MIN_INT..=HALF_MAX_INT).contains(&value) {
Copy link
Collaborator

@chaokunyang chaokunyang Nov 15, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why use this range instead of conditional?

Copy link
Contributor Author

@urlyy urlyy Nov 15, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the lint error and hint I got:

$ cargo clippy --all-targets --all-features -- -D warnings

error: manual `RangeInclusive::contains` implementation                                                                                                                                                                            
   --> fory-core\src\buffer.rs:182:12
    |
182 |         if value >= HALF_MIN_INT && value <= HALF_MAX_INT {
    |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: use: `(HALF_MIN_INT..=HALF_MAX_INT).contains(&value)`
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#manual_range_contains
    = note: `-D clippy::manual-range-contains` implied by `-D warnings`
    = help: to override `-D warnings` add `#[allow(clippy::manual_range_contains)]`

You can see detail at https://rust-lang.github.io/rust-clippy/master/index.html#manual_range_contains

Maybe more Rust-idiomatic, but actually, I think the condition is still more intuitive.

let v = (value as i32) << 1;
self.bf.extend_from_slice(&v.to_le_bytes());
} else {
self.bf.push(SLI_BIG_LONG_FLAG);
self.bf.extend_from_slice(&value.to_le_bytes());
}
}

#[inline(always)]
pub fn write_usize(&mut self, value: usize) {
self.write_u64(value as u64);
Expand Down Expand Up @@ -549,6 +563,19 @@ impl<'a> Reader<'a> {
Ok(self.read_u64()? as i64)
}

#[inline(always)]
pub fn read_sliint64(&mut self) -> Result<i64, Error> {
let i = LittleEndian::read_i32(&self.bf[self.cursor..self.cursor + 4]);
if (i & 0b1) != 0b1 {
self.cursor += 4;
Ok((i >> 1) as i64)
} else {
let val = LittleEndian::read_i64(&self.bf[self.cursor + 1..self.cursor + 9]);
self.cursor += 9;
Ok(val)
}
}

#[inline(always)]
pub fn read_f32(&mut self) -> Result<f32, Error> {
let slice = self.slice_after_cursor();
Expand Down
45 changes: 33 additions & 12 deletions rust/tests/tests/test_cross_language.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ fn test_buffer_var() {
let bytes = fs::read(&data_file_path).unwrap();
let mut reader = Reader::new(bytes.as_slice());

let varint32_values = vec![
let int32_values = vec![
i32::MIN,
i32::MIN + 1,
-1000000,
Expand All @@ -124,11 +124,15 @@ fn test_buffer_var() {
i32::MAX - 1,
i32::MAX,
];
for &expected in &varint32_values {
for &expected in &int32_values {
let value = reader.read_i32().unwrap();
assert_eq!(expected, value, "i32 value mismatch");
}
for &expected in &int32_values {
let value = reader.read_varint32().unwrap();
assert_eq!(expected, value, "varint32 value mismatch");
}
let varuint32_values = vec![
let uint32_values = vec![
0,
1,
127,
Expand All @@ -142,11 +146,11 @@ fn test_buffer_var() {
i32::MAX - 1,
i32::MAX,
];
for &expected in &varuint32_values {
for &expected in &uint32_values {
let value = reader.read_varuint32().unwrap();
assert_eq!(expected, value as i32, "varuint32 value mismatch");
}
let varuint64_values = vec![
let uint64_values = vec![
0u64,
1,
127,
Expand All @@ -167,11 +171,11 @@ fn test_buffer_var() {
72057594037927936,
i64::MAX as u64,
];
for &expected in &varuint64_values {
for &expected in &uint64_values {
let value = reader.read_varuint64().unwrap();
assert_eq!(expected, value, "varuint64 value mismatch");
}
let varint64_values = vec![
let int64_values = vec![
i64::MIN,
i64::MIN + 1,
-1000000000000,
Expand All @@ -188,23 +192,40 @@ fn test_buffer_var() {
i64::MAX - 1,
i64::MAX,
];
for &expected in &varint64_values {
for &expected in &int64_values {
let value = reader.read_i64().unwrap();
assert_eq!(expected, value, "i64 value mismatch");
}
for &expected in &int64_values {
let value = reader.read_sliint64().unwrap();
assert_eq!(expected, value, "sli_i64 value mismatch");
}
for &expected in &int64_values {
let value = reader.read_varint64().unwrap();
assert_eq!(expected, value, "varint64 value mismatch");
}

let mut buffer = vec![];
let mut writer = Writer::from_buffer(&mut buffer);
for &value in &varint32_values {
for &value in &int32_values {
writer.write_i32(value);
}
for &value in &int32_values {
writer.write_varint32(value);
}
for &value in &varuint32_values {
for &value in &uint32_values {
writer.write_varuint32(value as u32);
}
for &value in &varuint64_values {
for &value in &uint64_values {
writer.write_varuint64(value);
}
for &value in &varint64_values {
for &value in &int64_values {
writer.write_i64(value);
}
for &value in &int64_values {
writer.write_sliint64(value);
}
for &value in &int64_values {
writer.write_varint64(value);
}
fs::write(data_file_path, writer.dump()).unwrap();
Expand Down
Loading