Skip to content

Commit 22ffdbd

Browse files
srhsrh
committed
chore(cubestore): Upgrade DF: Make Int96 and Decimal96 reading and migration working
Note: We treat Int96 as Decimal128(38, 0), which brings changes to arithmetic behavior. Updates arrow-rs pointers.
1 parent 16c1c9d commit 22ffdbd

File tree

24 files changed

+655
-360
lines changed

24 files changed

+655
-360
lines changed

‎rust/cubestore/Cargo.lock‎

Lines changed: 17 additions & 17 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎rust/cubestore/cubestore/src/cachestore/cache_rocksstore.rs‎

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -420,23 +420,7 @@ impl RocksCacheStore {
420420
.join("testing-fixtures")
421421
.join(remote_fixtures);
422422

423-
fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> std::io::Result<()> {
424-
std::fs::create_dir_all(&dst)?;
425-
426-
for entry in std::fs::read_dir(src)? {
427-
let entry = entry?;
428-
let ty = entry.file_type()?;
429-
if ty.is_dir() {
430-
copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
431-
} else {
432-
std::fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
433-
}
434-
}
435-
436-
Ok(())
437-
}
438-
439-
copy_dir_all(&fixtures_path, store_path.join("cachestore")).unwrap();
423+
crate::util::copy_dir_all(&fixtures_path, store_path.join("cachestore")).unwrap();
440424

441425
Self::prepare_test_cachestore_impl(test_name, store_path, config)
442426
}

‎rust/cubestore/cubestore/src/queryplanner/partition_filter.rs‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -414,8 +414,7 @@ impl Builder<'_> {
414414
}
415415
match t {
416416
t if Self::is_signed_int(t) => Self::extract_signed_int(v),
417-
// TODO upgrade DF
418-
// DataType::Int64Decimal(scale) => Self::extract_decimal(v, *scale),
417+
DataType::Decimal128(_precision, scale) => Self::extract_decimal(v, *scale),
419418
DataType::Boolean => Self::extract_bool(v),
420419
DataType::Utf8 => Self::extract_string(v),
421420
_ => None,
@@ -457,12 +456,11 @@ impl Builder<'_> {
457456
Some(TableValue::String(s.unwrap()))
458457
}
459458

460-
fn extract_decimal(v: &ScalarValue, scale: usize) -> Option<TableValue> {
459+
fn extract_decimal(v: &ScalarValue, scale: i8) -> Option<TableValue> {
461460
let decimal_value = match v {
462-
// TODO upgrade DF
463-
// ScalarValue::Int64Decimal(v, input_scale) => {
464-
// Builder::int_to_decimal_value(v.unwrap(), scale as i64 - (*input_scale as i64))
465-
// }
461+
ScalarValue::Decimal128(v, _input_precision, input_scale) => {
462+
Builder::int_to_decimal_value(v.unwrap() as i128, scale as i64 - (*input_scale as i64))
463+
}
466464
ScalarValue::Int16(v) => {
467465
Builder::int_to_decimal_value(v.unwrap() as i128, scale as i64)
468466
}

‎rust/cubestore/cubestore/src/queryplanner/query_executor.rs‎

Lines changed: 0 additions & 108 deletions
Original file line numberDiff line numberDiff line change
@@ -1786,9 +1786,6 @@ pub fn batches_to_dataframe(batches: Vec<RecordBatch>) -> Result<DataFrame, Cube
17861786
DataType::Int16 => convert_array!(array, num_rows, rows, Int16Array, Int, i64),
17871787
DataType::Int32 => convert_array!(array, num_rows, rows, Int32Array, Int, i64),
17881788
DataType::Int64 => convert_array!(array, num_rows, rows, Int64Array, Int, i64),
1789-
// DataType::Int96 => {
1790-
// convert_array!(array, num_rows, rows, Int96Array, Int96, (Int96))
1791-
// }
17921789
DataType::Float64 => {
17931790
let a = array.as_any().downcast_ref::<Float64Array>().unwrap();
17941791
for i in 0..num_rows {
@@ -1800,114 +1797,9 @@ pub fn batches_to_dataframe(batches: Vec<RecordBatch>) -> Result<DataFrame, Cube
18001797
});
18011798
}
18021799
}
1803-
// TODO upgrade DF
18041800
DataType::Decimal128(_, _) => {
18051801
convert_array!(array, num_rows, rows, Decimal128Array, Decimal, (Decimal))
18061802
}
1807-
// DataType::Int64Decimal(1) => convert_array!(
1808-
// array,
1809-
// num_rows,
1810-
// rows,
1811-
// Int64Decimal1Array,
1812-
// Decimal,
1813-
// (Decimal)
1814-
// ),
1815-
// DataType::Int64Decimal(2) => convert_array!(
1816-
// array,
1817-
// num_rows,
1818-
// rows,
1819-
// Int64Decimal2Array,
1820-
// Decimal,
1821-
// (Decimal)
1822-
// ),
1823-
// DataType::Int64Decimal(3) => convert_array!(
1824-
// array,
1825-
// num_rows,
1826-
// rows,
1827-
// Int64Decimal3Array,
1828-
// Decimal,
1829-
// (Decimal)
1830-
// ),
1831-
// DataType::Int64Decimal(4) => convert_array!(
1832-
// array,
1833-
// num_rows,
1834-
// rows,
1835-
// Int64Decimal4Array,
1836-
// Decimal,
1837-
// (Decimal)
1838-
// ),
1839-
// DataType::Int64Decimal(5) => convert_array!(
1840-
// array,
1841-
// num_rows,
1842-
// rows,
1843-
// Int64Decimal5Array,
1844-
// Decimal,
1845-
// (Decimal)
1846-
// ),
1847-
// DataType::Int64Decimal(10) => convert_array!(
1848-
// array,
1849-
// num_rows,
1850-
// rows,
1851-
// Int64Decimal10Array,
1852-
// Decimal,
1853-
// (Decimal)
1854-
// ),
1855-
// DataType::Int96Decimal(0) => convert_array!(
1856-
// array,
1857-
// num_rows,
1858-
// rows,
1859-
// Int96Decimal0Array,
1860-
// Decimal96,
1861-
// (Decimal96)
1862-
// ),
1863-
// DataType::Int96Decimal(1) => convert_array!(
1864-
// array,
1865-
// num_rows,
1866-
// rows,
1867-
// Int96Decimal1Array,
1868-
// Decimal96,
1869-
// (Decimal96)
1870-
// ),
1871-
// DataType::Int96Decimal(2) => convert_array!(
1872-
// array,
1873-
// num_rows,
1874-
// rows,
1875-
// Int96Decimal2Array,
1876-
// Decimal96,
1877-
// (Decimal96)
1878-
// ),
1879-
// DataType::Int96Decimal(3) => convert_array!(
1880-
// array,
1881-
// num_rows,
1882-
// rows,
1883-
// Int96Decimal3Array,
1884-
// Decimal96,
1885-
// (Decimal96)
1886-
// ),
1887-
// DataType::Int96Decimal(4) => convert_array!(
1888-
// array,
1889-
// num_rows,
1890-
// rows,
1891-
// Int96Decimal4Array,
1892-
// Decimal96,
1893-
// (Decimal96)
1894-
// ),
1895-
// DataType::Int96Decimal(5) => convert_array!(
1896-
// array,
1897-
// num_rows,
1898-
// rows,
1899-
// Int96Decimal5Array,
1900-
// Decimal96,
1901-
// (Decimal96)
1902-
// ),
1903-
// DataType::Int96Decimal(10) => convert_array!(
1904-
// array,
1905-
// num_rows,
1906-
// rows,
1907-
// Int96Decimal10Array,
1908-
// Decimal96,
1909-
// (Decimal96)
1910-
// ),
19111803
DataType::Timestamp(TimeUnit::Microsecond, None) => {
19121804
let a = array
19131805
.as_any()

0 commit comments

Comments
 (0)