Skip to content

Implement remaining tagless uints #993

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
12 changes: 10 additions & 2 deletions src/lazy/binary/raw/v1_1/binary_buffer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -260,12 +260,20 @@ impl<'a> BinaryBuffer<'a> {

pub fn read_fixed_uint_as_lazy_value(self, size_in_bytes: usize) -> ParseResult<'a, LazyRawBinaryValue_1_1<'a>> {
if self.len() < size_in_bytes {
return IonResult::incomplete("a uint8", self.offset());
return IonResult::incomplete("a uint", self.offset());
}

let encoding = match size_in_bytes {
1 => BinaryValueEncoding::UInt8,
2 => BinaryValueEncoding::UInt16,
4 => BinaryValueEncoding::UInt32,
8 => BinaryValueEncoding::UInt64,
_ => return IonResult::decoding_error(format!("Invalid byte size for fixed uint: {size_in_bytes}")),
};

let matched_input = self.slice(0, size_in_bytes);
let remaining_input = self.slice_to_end(size_in_bytes);
let value = LazyRawBinaryValue_1_1::for_fixed_uint8(matched_input);
let value = LazyRawBinaryValue_1_1::for_fixed_uint(matched_input, encoding);
Ok((value, remaining_input))
}

Expand Down
42 changes: 42 additions & 0 deletions src/lazy/binary/raw/v1_1/e_expression.rs
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,48 @@
remaining,
)
}
ParameterEncoding::UInt16 => {
let (fixed_uint_lazy_value, remaining) = try_or_some_err! {
self.remaining_args_buffer.read_fixed_uint_as_lazy_value(2)
};
let value_ref = &*self
.remaining_args_buffer
.context()
.allocator()
.alloc_with(|| fixed_uint_lazy_value);
(
EExpArg::new(parameter, EExpArgExpr::ValueLiteral(value_ref)),
remaining,
)
}
ParameterEncoding::UInt32 => {
let (fixed_uint_lazy_value, remaining) = try_or_some_err! {
self.remaining_args_buffer.read_fixed_uint_as_lazy_value(4)
};
let value_ref = &*self
.remaining_args_buffer
.context()
.allocator()
.alloc_with(|| fixed_uint_lazy_value);
(
EExpArg::new(parameter, EExpArgExpr::ValueLiteral(value_ref)),
remaining,
)
}
ParameterEncoding::UInt64 => {
let (fixed_uint_lazy_value, remaining) = try_or_some_err! {
self.remaining_args_buffer.read_fixed_uint_as_lazy_value(8)
};
let value_ref = &*self
.remaining_args_buffer
.context()
.allocator()
.alloc_with(|| fixed_uint_lazy_value);
(
EExpArg::new(parameter, EExpArgExpr::ValueLiteral(value_ref)),
remaining,
)
}
ParameterEncoding::MacroShaped(_macro_ref) => {
todo!("macro-shaped parameter encoding")
} // TODO: The other tagless encodings
Expand Down Expand Up @@ -445,7 +487,7 @@

#[derive(Debug, Copy, Clone)]
pub struct BinaryEExpArgGroup<'top> {
parameter: &'top Parameter,

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-arm, default)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-arm, experimental)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-arm, experimental-ion-hash)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-arm, all)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-x86, default)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-x86, experimental-ion-hash)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (macos, default)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (macos, experimental-ion-hash)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (ubuntu, default)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (macos, all)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-x86, experimental)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-x86, all)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (ubuntu, experimental-ion-hash)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (ubuntu, experimental)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (macos, experimental)

field `parameter` is never read

Check failure on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (ubuntu, all)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (ubuntu, all)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (windows, experimental-ion-hash)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (windows, default)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (windows, all)

field `parameter` is never read

Check warning on line 490 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (windows, experimental)

field `parameter` is never read
input: BinaryBuffer<'top>,
header_size: u8,
// If this is a delimited arg group, this cache will hold the expressions found during parsing.
Expand Down Expand Up @@ -567,7 +609,7 @@
}

#[derive(Debug, Clone)]
pub struct RawBinarySequenceCacheIterator_1_1<'top> {

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-arm, default)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-arm, experimental)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-arm, experimental-ion-hash)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-arm, all)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-x86, default)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-x86, experimental-ion-hash)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (macos, default)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (macos, experimental-ion-hash)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (ubuntu, default)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (macos, all)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-x86, experimental)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (al2-x86, all)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (ubuntu, experimental-ion-hash)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (ubuntu, experimental)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (macos, experimental)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check failure on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (ubuntu, all)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (ubuntu, all)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (windows, experimental-ion-hash)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (windows, default)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (windows, all)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed

Check warning on line 612 in src/lazy/binary/raw/v1_1/e_expression.rs

View workflow job for this annotation

GitHub Actions / Build and Test (windows, experimental)

struct `RawBinarySequenceCacheIterator_1_1` is never constructed
child_exprs: &'top [LazyRawValueExpr<'top, v1_1::Binary>],
index: usize,
}
Expand Down
7 changes: 5 additions & 2 deletions src/lazy/binary/raw/v1_1/value.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,9 @@ pub enum BinaryValueEncoding {
Tagged,
FlexUInt,
UInt8,
UInt16,
UInt32,
UInt64,
}

#[derive(Debug, Copy, Clone)]
Expand Down Expand Up @@ -361,9 +364,9 @@ impl<'top> LazyRawBinaryValue_1_1<'top> {
}
}

pub(crate) fn for_fixed_uint8(input: BinaryBuffer<'top>) -> Self {
pub(crate) fn for_fixed_uint(input: BinaryBuffer<'top>, encoding: BinaryValueEncoding) -> Self {
let encoded_value = EncodedBinaryValue {
encoding: BinaryValueEncoding::UInt8,
encoding,
header: Header {
ion_type: IonType::Int,
ion_type_code: OpcodeType::Nop,
Expand Down
15 changes: 9 additions & 6 deletions src/lazy/encoder/binary/v1_1/fixed_uint.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use std::io::Write;

use num_traits::{PrimInt, Unsigned};
use ice_code::ice as cold_path;

use crate::decimal::coefficient::Coefficient;
Expand Down Expand Up @@ -66,17 +67,19 @@ impl FixedUInt {
self.size_in_bytes
}

/// Write the provided UInt-like as a uint8 ensuring that the value fits in 8bits.
#[inline]
pub(crate) fn write_as_uint8<W: Write>(output: &mut W, value: impl Into<UInt>) -> IonResult<()> {
let value = value.into().data;
pub(crate) fn write_as_uint<I: PrimInt + Unsigned>(output: &mut impl Write, value: impl Into<UInt>) -> IonResult<()> {
let size_in_bytes = std::mem::size_of::<I>();
let value: u128 = value.into().data;
let encoded_bytes = value.to_le_bytes();
let max_value: u128 = num_traits::cast::cast(I::max_value())
.ok_or(IonError::encoding_error("Unable to represent bounds for value as 128bit value"))?;

if !(0..256u128).contains(&value) {
return IonResult::encoding_error("provided unsigned integer value does not fit within 1 byte");
if !(0..=max_value).contains(&value) {
return IonResult::encoding_error(format!("provided unsigned integer value does not fit within {size_in_bytes} byte(s)"));
}

output.write_all(&encoded_bytes[..1])?;
output.write_all(&encoded_bytes[..size_in_bytes])?;
Ok(())
}
}
Expand Down
22 changes: 20 additions & 2 deletions src/lazy/encoder/binary/v1_1/value_writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1076,7 +1076,16 @@ impl<'value, 'top> ValueWriter for BinaryEExpParameterValueWriter_1_1<'value, 't
let result = match param.encoding() {
PE::UInt8 => value
.try_into()
.and_then(|uint: UInt| FixedUInt::write_as_uint8(self.buffer, uint)),
.and_then(|uint: UInt| FixedUInt::write_as_uint::<u8>(self.buffer, uint)),
PE::UInt16 => value
.try_into()
.and_then(|uint: UInt| FixedUInt::write_as_uint::<u16>(self.buffer, uint)),
PE::UInt32 => value
.try_into()
.and_then(|uint: UInt| FixedUInt::write_as_uint::<u32>(self.buffer, uint)),
PE::UInt64 => value
.try_into()
.and_then(|uint: UInt| FixedUInt::write_as_uint::<u64>(self.buffer, uint)),
PE::FlexUInt => value
.try_into()
.and_then(|uint: UInt| FlexUInt::write(self.buffer, uint))
Expand Down Expand Up @@ -1116,7 +1125,16 @@ impl<'value, 'top> ValueWriter for BinaryEExpParameterValueWriter_1_1<'value, 't
let result = match param.encoding() {
PE::UInt8 => value
.try_into()
.and_then(|uint: UInt| FixedUInt::write_as_uint8(self.buffer, uint)),
.and_then(|uint: UInt| FixedUInt::write_as_uint::<u8>(self.buffer, uint)),
PE::UInt16 => value
.try_into()
.and_then(|uint: UInt| FixedUInt::write_as_uint::<u16>(self.buffer, uint)),
PE::UInt32 => value
.try_into()
.and_then(|uint: UInt| FixedUInt::write_as_uint::<u32>(self.buffer, uint)),
PE::UInt64 => value
.try_into()
.and_then(|uint: UInt| FixedUInt::write_as_uint::<u64>(self.buffer, uint)),
PE::FlexUInt => value
.try_into()
.and_then(|uint: UInt| FlexUInt::write(self.buffer, uint))
Expand Down
93 changes: 93 additions & 0 deletions src/lazy/encoder/writer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1375,6 +1375,8 @@ mod tests {

mod eexp_parameter_validation {
use super::*;
use num_traits::{PrimInt, Unsigned};
use rstest::*;

#[test]
fn accept_valid_parameter_encoding() -> IonResult<()> {
Expand Down Expand Up @@ -1500,5 +1502,96 @@ mod tests {

Ok(())
}

#[rstest]
#[case::uint8("(macro foo (uint8::x) (%x))", 5, "5")]
#[case::uint16("(macro foo (uint16::x) (%x))", 5, "5")]
#[case::uint32("(macro foo (uint32::x) (%x))", 5, "5")]
#[case::uint64("(macro foo (uint64::x) (%x))", 5, "5")]
fn tagless_uint_encoding(#[case] macro_source: &str, #[case] input: i64, #[case] expected: &str) -> IonResult<()> {
use crate::{Int, Element};

// write_int

let mut writer = Writer::new(v1_1::Binary, Vec::new())?;
let foo = writer.compile_macro(macro_source)?;
let mut eexp_writer = writer.eexp_writer(&foo)?;
let int: Int = input.into();
eexp_writer.write_int(&int)?;
eexp_writer.close()?;

let output = writer.close()?;
let actual = Element::read_all(&output)?;
let exp_elem = Element::read_all(expected)?;
assert_eq!(actual, exp_elem);

// write_i64

let mut writer = Writer::new(v1_1::Binary, Vec::new())?;
let foo = writer.compile_macro(macro_source)?;
let mut eexp_writer = writer.eexp_writer(&foo)?;
eexp_writer.write_i64(input)?;
eexp_writer.close()?;

let output = writer.close()?;
let actual = Element::read_all(&output)?;
let exp_elem = Element::read_all(expected)?;
assert_eq!(actual, exp_elem);

Ok(())
}

#[rstest]
#[case::uint8("(macro foo (uint8::x) (%x))", 5u8)]
#[case::uint16("(macro foo (uint16::x) (%x))", 5u16)]
#[case::uint32("(macro foo (uint32::x) (%x))", 5u32)]
#[case::uint64("(macro foo (uint64::x) (%x))", 5u64)]
fn tagless_uint_encoding_write_int_fails<T: PrimInt + Unsigned>(#[case] macro_source: &str, #[case] input: T) -> IonResult<()> {
let max_int = T::max_value();
let max_int_plus_one = num_traits::cast::cast::<_, i128>(max_int).unwrap() + 1i128;
let neg_input = -num_traits::cast::cast::<_, i128>(input).unwrap();

let mut writer = Writer::new(v1_1::Binary, Vec::new())?;
let foo = writer.compile_macro(macro_source)?;
let mut eexp_writer = writer.eexp_writer(&foo)?;
let result = eexp_writer.write_int(&max_int_plus_one.into());
assert!(result.is_err(), "unexpected success");

// Ensure we cannot write a negative value..
let mut writer = Writer::new(v1_1::Binary, Vec::new())?;
let foo = writer.compile_macro(macro_source)?;
let mut eexp_writer = writer.eexp_writer(&foo)?;
let result = eexp_writer.write_int(&neg_input.into());
assert!(result.is_err(), "unexpected success");

Ok(())
}

#[rstest]
#[case::uint8("(macro foo (uint8::x) (%x))", 5u8)]
#[case::uint16("(macro foo (uint16::x) (%x))", 5u16)]
#[case::uint32("(macro foo (uint32::x) (%x))", 5u32)]
fn tagless_uint_encoding_write_i64_fails<T: PrimInt + Unsigned>(#[case] macro_source: &str, #[case] input: T) -> IonResult<()> {
let max_int = T::max_value();
let max_int_plus_one = num_traits::cast::cast::<_, i128>(max_int).unwrap() + 1i128;
let neg_input = -num_traits::cast::cast::<_, i128>(input).unwrap();

let mut writer = Writer::new(v1_1::Binary, Vec::new())?;
let foo = writer.compile_macro(macro_source)?;
let mut eexp_writer = writer.eexp_writer(&foo)?;
// eexp_writer should do the "right thing" given the parameter's encoding.
let result = eexp_writer.write_i64(max_int_plus_one.try_into().unwrap());
// the "right thing" should be to error, since `x` can only be an 8bit unsigned int.
assert!(result.is_err(), "unexpected success");

// Ensure we cannot write a negative value..
let mut writer = Writer::new(v1_1::Binary, Vec::new())?;
let foo = writer.compile_macro(macro_source)?;
let mut eexp_writer = writer.eexp_writer(&foo)?;
let result = eexp_writer.write_i64(neg_input.try_into().unwrap());
assert!(result.is_err(), "unexpected success");

Ok(())
}
}
}
3 changes: 3 additions & 0 deletions src/lazy/expanded/compiler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,9 @@ impl TemplateCompiler {
return match encoding_name {
"flex_uint" => Ok(ParameterEncoding::FlexUInt),
"uint8" => Ok(ParameterEncoding::UInt8),
"uint16" => Ok(ParameterEncoding::UInt16),
"uint32" => Ok(ParameterEncoding::UInt32),
"uint64" => Ok(ParameterEncoding::UInt64),
_ => IonResult::decoding_error(format!(
"unrecognized encoding '{encoding_name}' specified for parameter"
)),
Expand Down
9 changes: 9 additions & 0 deletions src/lazy/expanded/macro_table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,15 @@ fn write_macro_signature_as_ion<V: ValueWriter>(
ParameterEncoding::UInt8 => value_writer
.with_annotations("uint8")?
.write_symbol(param.name())?,
ParameterEncoding::UInt16 => value_writer
.with_annotations("uint16")?
.write_symbol(param.name())?,
ParameterEncoding::UInt32 => value_writer
.with_annotations("uint32")?
.write_symbol(param.name())?,
ParameterEncoding::UInt64 => value_writer
.with_annotations("uint64")?
.write_symbol(param.name())?,
ParameterEncoding::MacroShaped(_) => todo!(),
};
let cardinality_modifier = match param.cardinality() {
Expand Down
6 changes: 6 additions & 0 deletions src/lazy/expanded/template.rs
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,9 @@ pub enum ParameterEncoding {
Tagged,
FlexUInt,
UInt8,
UInt16,
UInt32,
UInt64,
// TODO: tagless types, including fixed-width types and macros
MacroShaped(Arc<MacroDef>),
}
Expand All @@ -219,6 +222,9 @@ impl Display for ParameterEncoding {
Tagged => write!(f, "tagged"),
FlexUInt => write!(f, "flex_uint"),
UInt8 => write!(f, "uint8"),
UInt16 => write!(f, "uint16"),
UInt32 => write!(f, "uint32"),
UInt64 => write!(f, "uint64"),
MacroShaped(m) => write!(f, "{}", m.name().unwrap_or("<anonymous>")),
}
}
Expand Down
Loading