basic read support for orc iceberg table

Author: Tom McCormick

pyiceberg/io/pyarrow.py

@@ -197,6 +197,8 @@
 ICEBERG_SCHEMA = b"iceberg.schema"
 # The PARQUET: in front means that it is Parquet specific, in this case the field_id
 PYARROW_PARQUET_FIELD_ID_KEY = b"PARQUET:field_id"
+# ORC stores IDs as string metadata
+ORC_FIELD_ID_KEY = b"iceberg.id"
 PYARROW_FIELD_DOC_KEY = b"doc"
 LIST_ELEMENT_NAME = "element"
 MAP_KEY_NAME = "key"
@@ -1180,11 +1182,21 @@ def primitive(self, primitive: pa.DataType) -> T:
 
 
 def _get_field_id(field: pa.Field) -> Optional[int]:
-    return (
-        int(field_id_str.decode())
-        if (field.metadata and (field_id_str := field.metadata.get(PYARROW_PARQUET_FIELD_ID_KEY)))
-        else None
-    )
+    """Return the Iceberg field ID from Parquet or ORC metadata if available."""
+    if not field.metadata:
+        return None
+
+    # Try Parquet field ID first
+    field_id_bytes = field.metadata.get(PYARROW_PARQUET_FIELD_ID_KEY)
+    if field_id_bytes:
+        return int(field_id_bytes.decode())
+
+    # Fallback: try ORC field ID
+    field_id_bytes = field.metadata.get(ORC_FIELD_ID_KEY)
+    if field_id_bytes:
+        return int(field_id_bytes.decode())
+
+    return None
 
 
 class _HasIds(PyArrowSchemaVisitor[bool]):

tests/io/test_pyarrow_visitor.py

@@ -45,6 +45,7 @@
     pyarrow_to_schema,
     schema_to_pyarrow,
     visit_pyarrow,
+    ORC_FIELD_ID_KEY,
 )
 from pyiceberg.schema import Accessor, Schema, visit
 from pyiceberg.table.name_mapping import MappedField, NameMapping
@@ -239,6 +240,64 @@ def test_pyarrow_variable_binary_to_iceberg(pyarrow_type: pa.DataType) -> None:
     assert visit(converted_iceberg_type, _ConvertToArrowSchema()) == pa.large_binary()
 
 
+def test_orc_vs_parquet_metadata_handling() -> None:
+    """Test that ORC and Parquet metadata use different keys but produce equivalent schemas."""
+    # ORC struct with ORC metadata
+    orc_struct = pa.struct(
+        [
+            pa.field("foo", pa.string(), nullable=True, metadata={ORC_FIELD_ID_KEY.decode(): "1", "doc": "foo doc"}),
+            pa.field("bar", pa.int32(), nullable=False, metadata={ORC_FIELD_ID_KEY.decode(): "2"}),
+        ]
+    )
+    
+    # Parquet struct with Parquet metadata (same structure, different metadata keys)
+    parquet_struct = pa.struct(
+        [
+            pa.field("foo", pa.string(), nullable=True, metadata={"PARQUET:field_id": "1", "doc": "foo doc"}),
+            pa.field("bar", pa.int32(), nullable=False, metadata={"PARQUET:field_id": "2"}),
+        ]
+    )
+    
+    # Both should produce the same Iceberg schema
+    orc_result = visit_pyarrow(orc_struct, _ConvertToIceberg())
+    parquet_result = visit_pyarrow(parquet_struct, _ConvertToIceberg())
+    
+    assert orc_result == parquet_result
+    
+    # Verify they use different metadata keys
+    assert ORC_FIELD_ID_KEY.decode() == "iceberg.id"
+    assert "PARQUET:field_id" != ORC_FIELD_ID_KEY.decode()
+    
+    # Verify the metadata keys are present in the respective structs (using bytes keys)
+    assert ORC_FIELD_ID_KEY in orc_struct[0].metadata  # b'iceberg.id'
+    assert b"PARQUET:field_id" in parquet_struct[0].metadata
+    assert ORC_FIELD_ID_KEY not in parquet_struct[0].metadata
+    assert b"PARQUET:field_id" not in orc_struct[0].metadata
+
+
+def test_pyarrow_struct_to_iceberg_orc() -> None:
+    pyarrow_struct = pa.struct(
+        [
+            pa.field("foo", pa.string(), nullable=True, metadata={ORC_FIELD_ID_KEY.decode(): "1", "doc": "foo doc"}),
+            pa.field("bar", pa.int32(), nullable=False, metadata={ORC_FIELD_ID_KEY.decode(): "2"}),
+            pa.field("baz", pa.bool_(), nullable=True, metadata={ORC_FIELD_ID_KEY.decode(): "3"}),
+        ]
+    )
+    expected = StructType(
+        NestedField(id=1, name="foo", field_type=StringType(), required=False, doc="foo doc"),
+        NestedField(id=2, name="bar", field_type=IntegerType(), required=True),
+        NestedField(id=3, name="baz", field_type=BooleanType(), required=False),
+    )
+    result = visit_pyarrow(pyarrow_struct, _ConvertToIceberg())
+    assert result == expected
+    
+    # Validate that field IDs were extracted from ORC metadata
+    assert result.fields[0].field_id == 1  # foo
+    assert result.fields[1].field_id == 2  # bar  
+    assert result.fields[2].field_id == 3  # baz
+    
+
+
 def test_pyarrow_struct_to_iceberg() -> None:
     pyarrow_struct = pa.struct(
         [