Skip to content

Commit 9c6065c

Browse files
authored
Add benchmarks for reading struct arrays from parquet (#9210)
# Which issue does this PR close? <!-- We generally require a GitHub issue to be filed for all bug fixes and enhancements and this helps us generate change logs for our releases. You can link an issue to this PR using the GitHub syntax. --> - Closes #9209. # Rationale for this change Improve the performance benchmark coverage. # What changes are included in this PR? # Are these changes tested? Benchmark results on my machine, without target specific optimizations: ``` arrow_array_reader/Int32Array/plain encoded, mandatory, no NULLs time: [8.5123 µs 8.5218 µs 8.5321 µs] arrow_array_reader/Int32Array/plain encoded, optional, no NULLs time: [12.843 µs 12.863 µs 12.885 µs] arrow_array_reader/Int32Array/plain encoded, optional, half NULLs time: [64.977 µs 65.108 µs 65.267 µs] arrow_array_reader/struct/Int32Array/plain encoded, mandatory struct, optional data, no NULLs time: [14.456 µs 14.473 µs 14.491 µs] arrow_array_reader/struct/Int32Array/plain encoded, optional struct, optional data, no NULLs time: [173.14 µs 173.68 µs 174.23 µs] arrow_array_reader/struct/Int32Array/plain encoded, mandatory struct, optional data, half NULLs time: [65.636 µs 65.780 µs 65.906 µs] arrow_array_reader/struct/Int32Array/plain encoded, optional struct, optional data, half NULLs time: [433.95 µs 434.29 µs 434.67 µs] ``` I think these show that a mandatory struct has negligible overhead compared to a non-nested primitive column. But an optional struct, even if its definition level is always valid, has a significant overhead. # Are there any user-facing changes? no
1 parent 191aa7c commit 9c6065c

File tree

1 file changed

+110
-0
lines changed

1 file changed

+110
-0
lines changed

parquet/benches/arrow_reader.rs

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,12 @@ fn build_test_schema() -> SchemaDescPtr {
8888
OPTIONAL INT32 optional_int16_leaf (INTEGER(16, true));
8989
REQUIRED INT64 mandatory_uint64_leaf (INTEGER(64, false));
9090
OPTIONAL INT64 optional_uint64_leaf (INTEGER(64, false));
91+
REQUIRED GROUP mandatory_struct_optional_int32_leaf {
92+
OPTIONAL INT32 element;
93+
}
94+
OPTIONAL GROUP optional_struct_optional_int32_leaf {
95+
OPTIONAL INT32 element;
96+
}
9197
}
9298
";
9399
parse_message_type(message_type)
@@ -1174,6 +1180,98 @@ fn bench_primitive<T>(
11741180
});
11751181
}
11761182

1183+
// Benchmark reading a struct with a single primitive field.
1184+
// No need to bench all encodings for the data, as that should already be covered by `bench_primitive`.
1185+
// The only performance difference should be caused by the additional definition level.
1186+
fn bench_struct_primitive<T>(
1187+
group: &mut BenchmarkGroup<WallTime>,
1188+
mandatory_column_desc: &ColumnDescPtr,
1189+
optional_column_desc: &ColumnDescPtr,
1190+
min: usize,
1191+
max: usize,
1192+
) where
1193+
T: parquet::data_type::DataType,
1194+
T::T: SampleUniform + FromPrimitive + Copy,
1195+
{
1196+
let mut count: usize = 0;
1197+
1198+
let data = build_encoded_primitive_page_iterator::<T>(
1199+
mandatory_column_desc.clone(),
1200+
0.0,
1201+
Encoding::PLAIN,
1202+
min,
1203+
max,
1204+
);
1205+
group.bench_function(
1206+
"plain encoded, mandatory struct, optional data, no NULLs",
1207+
|b| {
1208+
b.iter(|| {
1209+
let array_reader =
1210+
create_primitive_array_reader(data.clone(), mandatory_column_desc.clone());
1211+
count = bench_array_reader(array_reader);
1212+
});
1213+
assert_eq!(count, EXPECTED_VALUE_COUNT);
1214+
},
1215+
);
1216+
1217+
let data = build_encoded_primitive_page_iterator::<T>(
1218+
optional_column_desc.clone(),
1219+
0.0,
1220+
Encoding::PLAIN,
1221+
min,
1222+
max,
1223+
);
1224+
group.bench_function(
1225+
"plain encoded, optional struct, optional data, no NULLs",
1226+
|b| {
1227+
b.iter(|| {
1228+
let array_reader =
1229+
create_primitive_array_reader(data.clone(), optional_column_desc.clone());
1230+
count = bench_array_reader(array_reader);
1231+
});
1232+
assert_eq!(count, EXPECTED_VALUE_COUNT);
1233+
},
1234+
);
1235+
1236+
let data = build_encoded_primitive_page_iterator::<T>(
1237+
mandatory_column_desc.clone(),
1238+
0.5,
1239+
Encoding::PLAIN,
1240+
min,
1241+
max,
1242+
);
1243+
group.bench_function(
1244+
"plain encoded, mandatory struct, optional data, half NULLs",
1245+
|b| {
1246+
b.iter(|| {
1247+
let array_reader =
1248+
create_primitive_array_reader(data.clone(), mandatory_column_desc.clone());
1249+
count = bench_array_reader(array_reader);
1250+
});
1251+
assert_eq!(count, EXPECTED_VALUE_COUNT);
1252+
},
1253+
);
1254+
1255+
let data = build_encoded_primitive_page_iterator::<T>(
1256+
optional_column_desc.clone(),
1257+
0.5,
1258+
Encoding::PLAIN,
1259+
min,
1260+
max,
1261+
);
1262+
group.bench_function(
1263+
"plain encoded, optional struct, optional data, half NULLs",
1264+
|b| {
1265+
b.iter(|| {
1266+
let array_reader =
1267+
create_primitive_array_reader(data.clone(), optional_column_desc.clone());
1268+
count = bench_array_reader(array_reader);
1269+
});
1270+
assert_eq!(count, EXPECTED_VALUE_COUNT);
1271+
},
1272+
);
1273+
}
1274+
11771275
fn float16_benches(c: &mut Criterion) {
11781276
let schema = build_test_schema();
11791277

@@ -1304,6 +1402,8 @@ fn add_benches(c: &mut Criterion) {
13041402
let optional_int16_column_desc = schema.column(36);
13051403
let mandatory_uint64_column_desc = schema.column(37);
13061404
let optional_uint64_column_desc = schema.column(38);
1405+
let mandatory_struct_optional_in32_column_desc = schema.column(39);
1406+
let optional_struct_optional_in32_column_desc = schema.column(40);
13071407

13081408
// primitive / int32 benchmarks
13091409
// =============================
@@ -1397,6 +1497,16 @@ fn add_benches(c: &mut Criterion) {
13971497
);
13981498
group.finish();
13991499

1500+
let mut group = c.benchmark_group("arrow_array_reader/struct/Int32Array");
1501+
bench_struct_primitive::<Int32Type>(
1502+
&mut group,
1503+
&mandatory_struct_optional_in32_column_desc,
1504+
&optional_struct_optional_in32_column_desc,
1505+
0,
1506+
1000,
1507+
);
1508+
group.finish();
1509+
14001510
// string benchmarks
14011511
//==============================
14021512

0 commit comments

Comments
 (0)