Add metadata column helper functions

Author: gbrgr

crates/iceberg/src/arrow/reader.rs

@@ -54,7 +54,7 @@ use crate::expr::visitors::page_index_evaluator::PageIndexEvaluator;
 use crate::expr::visitors::row_group_metrics_evaluator::RowGroupMetricsEvaluator;
 use crate::expr::{BoundPredicate, BoundReference};
 use crate::io::{FileIO, FileMetadata, FileRead};
-use crate::metadata_columns::{RESERVED_FIELD_ID_FILE, is_reserved_field};
+use crate::metadata_columns::{RESERVED_FIELD_ID_FILE, is_metadata_field};
 use crate::scan::{ArrowRecordBatchStream, FileScanTask, FileScanTaskStream};
 use crate::spec::{Datum, NestedField, PrimitiveLiteral, PrimitiveType, Schema, Type};
 use crate::utils::available_parallelism;
@@ -225,7 +225,7 @@ impl ArrowReader {
         let project_field_ids_without_reserved: Vec<i32> = task
             .project_field_ids
             .iter()
-            .filter(|&&id| !is_reserved_field(id))
+            .filter(|&&id| !is_metadata_field(id))
             .copied()
             .collect();
         // so we must use position-based projection instead of field-ID matching

crates/iceberg/src/arrow/record_batch_transformer.rs

@@ -30,7 +30,7 @@ use arrow_schema::{
 use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
 
 use crate::arrow::schema_to_arrow_schema;
-use crate::metadata_columns::get_reserved_field_name;
+use crate::metadata_columns::get_metadata_column_name;
 use crate::spec::{Literal, PrimitiveLiteral, Schema as IcebergSchema};
 use crate::{Error, ErrorKind, Result};
 
@@ -217,7 +217,7 @@ impl RecordBatchTransformer {
                 if let Some(constant_value) = constants_map.get(field_id) {
                     // Create a field for the virtual column based on the constant type
                     let arrow_type = Self::primitive_literal_to_arrow_type(constant_value)?;
-                    let field_name = get_reserved_field_name(*field_id)?;
+                    let field_name = get_metadata_column_name(*field_id)?;
                     Ok(Arc::new(
                         Field::new(field_name, arrow_type, false).with_metadata(HashMap::from([(
                             PARQUET_FIELD_ID_META_KEY.to_string(),

crates/iceberg/src/metadata_columns.rs

@@ -30,31 +30,60 @@ pub const RESERVED_FIELD_ID_FILE: i32 = 2147483646;
 /// Reserved column name for the file path metadata column
 pub const RESERVED_COL_NAME_FILE: &str = "_file";
 
-/// Returns the field name for a reserved field ID.
+/// Returns the column name for a metadata field ID.
 ///
 /// # Arguments
-/// * `field_id` - The reserved field ID
+/// * `field_id` - The metadata field ID
 ///
 /// # Returns
-/// The name of the reserved field, or an error if the field ID is not recognized
-pub fn get_reserved_field_name(field_id: i32) -> Result<&'static str> {
+/// The name of the metadata column, or an error if the field ID is not recognized
+pub fn get_metadata_column_name(field_id: i32) -> Result<&'static str> {
     match field_id {
         RESERVED_FIELD_ID_FILE => Ok(RESERVED_COL_NAME_FILE),
         _ => Err(Error::new(
             ErrorKind::Unexpected,
-            format!("Unknown reserved field ID: {field_id}"),
+            format!("Unknown metadata field ID: {field_id}"),
         )),
     }
 }
 
-/// Checks if a field ID is a reserved (virtual/metadata) field.
+/// Returns the field ID for a metadata column name.
+///
+/// # Arguments
+/// * `column_name` - The metadata column name
+///
+/// # Returns
+/// The field ID of the metadata column, or an error if the column name is not recognized
+pub fn get_metadata_field_id(column_name: &str) -> Result<i32> {
+    match column_name {
+        RESERVED_COL_NAME_FILE => Ok(RESERVED_FIELD_ID_FILE),
+        _ => Err(Error::new(
+            ErrorKind::Unexpected,
+            format!("Unknown metadata column name: {column_name}"),
+        )),
+    }
+}
+
+/// Checks if a field ID is a metadata field.
 ///
 /// # Arguments
 /// * `field_id` - The field ID to check
 ///
 /// # Returns
-/// `true` if the field ID is reserved, `false` otherwise
-pub fn is_reserved_field(field_id: i32) -> bool {
+/// `true` if the field ID is a metadata field, `false` otherwise
+pub fn is_metadata_field(field_id: i32) -> bool {
     field_id == RESERVED_FIELD_ID_FILE
-    // Additional reserved fields can be checked here in the future
+    // Additional metadata fields can be checked here in the future
+}
+
+/// Checks if a column name is a metadata column.
+///
+/// # Arguments
+/// * `column_name` - The column name to check
+///
+/// # Returns
+/// `true` if the column name is a metadata column, `false` otherwise
+pub fn is_metadata_column_name(column_name: &str) -> bool {
+    column_name == RESERVED_COL_NAME_FILE
+    // Additional metadata column names can be checked here in the future
 }

crates/iceberg/src/scan/mod.rs

@@ -36,7 +36,7 @@ use crate::delete_file_index::DeleteFileIndex;
 use crate::expr::visitors::inclusive_metrics_evaluator::InclusiveMetricsEvaluator;
 use crate::expr::{Bind, BoundPredicate, Predicate};
 use crate::io::FileIO;
-use crate::metadata_columns::{RESERVED_COL_NAME_FILE, RESERVED_FIELD_ID_FILE};
+use crate::metadata_columns::{get_metadata_field_id, is_metadata_column_name};
 use crate::runtime::spawn;
 use crate::spec::{DataContentType, SnapshotRef};
 use crate::table::Table;
@@ -222,7 +222,7 @@ impl<'a> TableScanBuilder<'a> {
         if let Some(column_names) = self.column_names.as_ref() {
             for column_name in column_names {
                 // Skip reserved columns that don't exist in the schema
-                if column_name == RESERVED_COL_NAME_FILE {
+                if is_metadata_column_name(column_name) {
                     continue;
                 }
                 if schema.field_by_name(column_name).is_none() {
@@ -245,9 +245,9 @@ impl<'a> TableScanBuilder<'a> {
         });
 
         for column_name in column_names.iter() {
-            // Handle special reserved column "_file"
-            if column_name == RESERVED_COL_NAME_FILE {
-                field_ids.push(RESERVED_FIELD_ID_FILE);
+            // Handle metadata columns (like "_file")
+            if is_metadata_column_name(column_name) {
+                field_ids.push(get_metadata_field_id(column_name)?);
                 continue;
             }
 
@@ -588,7 +588,8 @@ pub mod tests {
     use crate::arrow::ArrowReaderBuilder;
     use crate::expr::{BoundPredicate, Reference};
     use crate::io::{FileIO, OutputFile};
-    use crate::scan::{FileScanTask, RESERVED_COL_NAME_FILE};
+    use crate::metadata_columns::RESERVED_COL_NAME_FILE;
+    use crate::scan::FileScanTask;
     use crate::spec::{
         DataContentType, DataFileBuilder, DataFileFormat, Datum, Literal, ManifestEntry,
         ManifestListWriter, ManifestStatus, ManifestWriterBuilder, NestedField, PartitionSpec,