cache manifests

Author: chinmay-bhat

pyiceberg/catalog/__init__.py

@@ -835,7 +835,7 @@ def purge_table(self, identifier: Union[str, Identifier]) -> None:
         manifest_lists_to_delete = set()
         manifests_to_delete: List[ManifestFile] = []
         for snapshot in metadata.snapshots:
-            manifests_to_delete += snapshot.manifests(io)
+            manifests_to_delete += snapshot.manifests(io, snapshot.manifest_list)
             if snapshot.manifest_list is not None:
                 manifest_lists_to_delete.add(snapshot.manifest_list)
 

pyiceberg/cli/output.py

@@ -144,7 +144,7 @@ def files(self, table: Table, history: bool) -> None:
             manifest_list_str = f": {snapshot.manifest_list}" if snapshot.manifest_list else ""
             list_tree = snapshot_tree.add(f"Snapshot {snapshot.snapshot_id}, schema {snapshot.schema_id}{manifest_list_str}")
 
-            manifest_list = snapshot.manifests(io)
+            manifest_list = snapshot.manifests(io, manifest_list_str)
             for manifest in manifest_list:
                 manifest_tree = list_tree.add(f"Manifest: {manifest.manifest_path}")
                 for manifest_entry in manifest.fetch_manifest_entry(io, discard_deleted=False):

pyiceberg/table/snapshots.py

@@ -19,6 +19,7 @@
 import time
 from collections import defaultdict
 from enum import Enum
+from functools import lru_cache
 from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Iterable, List, Mapping, Optional
 
 from pydantic import Field, PrivateAttr, model_serializer
@@ -249,9 +250,12 @@ def __str__(self) -> str:
         result_str = f"{operation}id={self.snapshot_id}{parent_id}{schema_id}"
         return result_str
 
-    def manifests(self, io: FileIO) -> List[ManifestFile]:
-        if self.manifest_list is not None:
-            file = io.new_input(self.manifest_list)
+    @staticmethod
+    @lru_cache
+    def manifests(io: FileIO, manifest_list: str) -> List[ManifestFile]:
+        """Return the manifests for the given snapshot."""
+        if manifest_list not in (None, ""):
+            file = io.new_input(manifest_list)
             return list(read_manifest_list(file))
         return []
 

tests/integration/test_partitioning_key.py

@@ -763,10 +763,14 @@ def test_partition_key(
         snapshot = iceberg_table.current_snapshot()
         assert snapshot
         spark_partition_for_justification = (
-            snapshot.manifests(iceberg_table.io)[0].fetch_manifest_entry(iceberg_table.io)[0].data_file.partition
+            snapshot.manifests(iceberg_table.io, snapshot.manifest_list)[0]
+            .fetch_manifest_entry(iceberg_table.io)[0]
+            .data_file.partition
         )
         spark_path_for_justification = (
-            snapshot.manifests(iceberg_table.io)[0].fetch_manifest_entry(iceberg_table.io)[0].data_file.file_path
+            snapshot.manifests(iceberg_table.io, snapshot.manifest_list)[0]
+            .fetch_manifest_entry(iceberg_table.io)[0]
+            .data_file.file_path
         )
         assert spark_partition_for_justification == expected_partition_record
         assert expected_hive_partition_path_slice in spark_path_for_justification

tests/integration/test_rest_manifest.py

@@ -75,7 +75,7 @@ def test_write_sample_manifest(table_test_all_types: Table) -> None:
     if test_snapshot is None:
         raise ValueError("Table has no current snapshot, check the docker environment")
     io = table_test_all_types.io
-    test_manifest_file = test_snapshot.manifests(io)[0]
+    test_manifest_file = test_snapshot.manifests(io, test_snapshot.manifest_list)[0]
     test_manifest_entries = test_manifest_file.fetch_manifest_entry(io)
     entry = test_manifest_entries[0]
     test_schema = table_test_all_types.schema()

tests/utils/test_manifest.py

@@ -217,7 +217,7 @@ def test_read_manifest_v1(generated_manifest_file_file_v1: str) -> None:
         summary=Summary(Operation.APPEND),
         schema_id=3,
     )
-    manifest_list = snapshot.manifests(io)[0]
+    manifest_list = snapshot.manifests(io, snapshot.manifest_list)[0]
 
     assert manifest_list.manifest_length == 7989
     assert manifest_list.partition_spec_id == 0
@@ -267,7 +267,7 @@ def test_read_manifest_v2(generated_manifest_file_file_v2: str) -> None:
         summary=Summary(Operation.APPEND),
         schema_id=3,
     )
-    manifest_list = snapshot.manifests(io)[0]
+    manifest_list = snapshot.manifests(io, manifest_list=snapshot.manifest_list)[0]
 
     assert manifest_list.manifest_length == 7989
     assert manifest_list.partition_spec_id == 0
@@ -336,7 +336,7 @@ def test_write_manifest(
         summary=Summary(Operation.APPEND),
         schema_id=3,
     )
-    demo_manifest_file = snapshot.manifests(io)[0]
+    demo_manifest_file = snapshot.manifests(io, snapshot.manifest_list)[0]
     manifest_entries = demo_manifest_file.fetch_manifest_entry(io)
     test_schema = Schema(
         NestedField(1, "VendorID", IntegerType(), False), NestedField(2, "tpep_pickup_datetime", IntegerType(), False)
@@ -508,7 +508,7 @@ def test_write_manifest_list(
         schema_id=3,
     )
 
-    demo_manifest_list = snapshot.manifests(io)
+    demo_manifest_list = snapshot.manifests(io, snapshot.manifest_list)
     with TemporaryDirectory() as tmp_dir:
         path = tmp_dir + "/manifest-list.avro"
         output = io.new_output(path)