Skip to content

fix: skip serializing file scan task partition #109

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
chenzl25 merged 1 commit into from
Jan 14, 2026
Merged

fix: skip serializing file scan task partition #109

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 7 additions & 31 deletions crates/iceberg/src/scan/task.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
use std::sync::Arc;

use futures::stream::BoxStream;
use serde::{Deserialize, Serialize, Serializer};
use serde::{Deserialize, Serialize};

use crate::Result;
use crate::expr::BoundPredicate;
Expand All @@ -30,24 +30,6 @@ use crate::spec::{
/// A stream of [`FileScanTask`].
pub type FileScanTaskStream = BoxStream<'static, Result<FileScanTask>>;

/// Serialization helper that always returns NotImplementedError.
/// Used for fields that should not be serialized but we want to be explicit about it.
fn serialize_not_implemented<S, T>(_: &T, _: S) -> std::result::Result<S::Ok, S::Error>
where S: Serializer {
Err(serde::ser::Error::custom(
"Serialization not implemented for this field",
))
}

/// Deserialization helper that always returns NotImplementedError.
/// Used for fields that should not be deserialized but we want to be explicit about it.
fn deserialize_not_implemented<'de, D, T>(_: D) -> std::result::Result<T, D::Error>
where D: serde::Deserializer<'de> {
Err(serde::de::Error::custom(
"Deserialization not implemented for this field",
))
}

/// A task to scan part of file.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct FileScanTask {
Expand Down Expand Up @@ -92,28 +74,22 @@ pub struct FileScanTask {
/// Partition data from the manifest entry, used to identify which columns can use
/// constant values from partition metadata vs. reading from the data file.
/// Per the Iceberg spec, only identity-transformed partition fields should use constants.
#[serde(default)]
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(serialize_with = "serialize_not_implemented")]
#[serde(deserialize_with = "deserialize_not_implemented")]
#[serde(skip_serializing)]
#[serde(skip_deserializing)]
pub partition: Option<Struct>,

/// The partition spec for this file, used to distinguish identity transforms
/// (which use partition metadata constants) from non-identity transforms like
/// bucket/truncate (which must read source columns from the data file).
#[serde(default)]
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(serialize_with = "serialize_not_implemented")]
#[serde(deserialize_with = "deserialize_not_implemented")]
#[serde(skip_serializing)]
#[serde(skip_deserializing)]
pub partition_spec: Option<Arc<PartitionSpec>>,

/// Name mapping from table metadata (property: schema.name-mapping.default),
/// used to resolve field IDs from column names when Parquet files lack field IDs
/// or have field ID conflicts.
#[serde(default)]
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(serialize_with = "serialize_not_implemented")]
#[serde(deserialize_with = "deserialize_not_implemented")]
#[serde(skip_serializing)]
#[serde(skip_deserializing)]
pub name_mapping: Option<Arc<NameMapping>>,
}

Expand Down
Loading