From 2e00b0bd4ec0441139b61fe50cc5fca1f0bd5361 Mon Sep 17 00:00:00 2001 From: chinmay-bhat <12948588+chinmay-bhat@users.noreply.github.com> Date: Mon, 27 May 2024 22:55:24 +0530 Subject: [PATCH 1/6] cache manifests --- pyiceberg/catalog/__init__.py | 2 +- pyiceberg/cli/output.py | 2 +- pyiceberg/table/__init__.py | 12 ++++++------ pyiceberg/table/snapshots.py | 10 +++++++--- tests/integration/test_partitioning_key.py | 8 ++++++-- tests/integration/test_rest_manifest.py | 2 +- tests/utils/test_manifest.py | 8 ++++---- 7 files changed, 26 insertions(+), 18 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index 9a951b5c8e..f6c15698de 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -717,7 +717,7 @@ def purge_table(self, identifier: Union[str, Identifier]) -> None: manifest_lists_to_delete = set() manifests_to_delete: List[ManifestFile] = [] for snapshot in metadata.snapshots: - manifests_to_delete += snapshot.manifests(io) + manifests_to_delete += snapshot.manifests(io, snapshot.manifest_list) if snapshot.manifest_list is not None: manifest_lists_to_delete.add(snapshot.manifest_list) diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index 56b544c99f..e9ba377453 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -144,7 +144,7 @@ def files(self, table: Table, history: bool) -> None: manifest_list_str = f": {snapshot.manifest_list}" if snapshot.manifest_list else "" list_tree = snapshot_tree.add(f"Snapshot {snapshot.snapshot_id}, schema {snapshot.schema_id}{manifest_list_str}") - manifest_list = snapshot.manifests(io) + manifest_list = snapshot.manifests(io, manifest_list_str) for manifest in manifest_list: manifest_tree = list_tree.add(f"Manifest: {manifest.manifest_path}") for manifest_entry in manifest.fetch_manifest_entry(io, discard_deleted=False): diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 2d4b342461..7e4307f499 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -1708,7 +1708,7 @@ def plan_files(self) -> Iterable[FileScanTask]: manifests = [ manifest_file - for manifest_file in snapshot.manifests(self.io) + for manifest_file in snapshot.manifests(self.io, snapshot.manifest_list) if manifest_evaluators[manifest_file.partition_spec_id](manifest_file) ] @@ -2941,7 +2941,7 @@ def _existing_manifests(self) -> List[ManifestFile]: if previous_snapshot is None: raise ValueError(f"Snapshot could not be found: {self._parent_snapshot_id}") - for manifest in previous_snapshot.manifests(io=self._io): + for manifest in previous_snapshot.manifests(io=self._io, manifest_list=previous_snapshot.manifest_list): if manifest.has_added_files() or manifest.has_existing_files() or manifest.added_snapshot_id == self._snapshot_id: existing_manifests.append(manifest) @@ -2992,7 +2992,7 @@ def _get_entries(manifest: ManifestFile) -> List[ManifestEntry]: if entry.data_file.content == DataFileContent.DATA ] - list_of_entries = executor.map(_get_entries, previous_snapshot.manifests(self._io)) + list_of_entries = executor.map(_get_entries, previous_snapshot.manifests(self._io, previous_snapshot.manifest_list)) return list(chain(*list_of_entries)) else: return [] @@ -3384,7 +3384,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: entries = [] snapshot = self._get_snapshot(snapshot_id) - for manifest in snapshot.manifests(self.tbl.io): + for manifest in snapshot.manifests(self.tbl.io, snapshot.manifest_list): for entry in manifest.fetch_manifest_entry(io=self.tbl.io): column_sizes = entry.data_file.column_sizes or {} value_counts = entry.data_file.value_counts or {} @@ -3546,7 +3546,7 @@ def update_partitions_map( partitions_map: Dict[Tuple[str, Any], Any] = {} snapshot = self._get_snapshot(snapshot_id) - for manifest in snapshot.manifests(self.tbl.io): + for manifest in snapshot.manifests(self.tbl.io, snapshot.manifest_list): for entry in manifest.fetch_manifest_entry(io=self.tbl.io): partition = entry.data_file.partition partition_record_dict = { @@ -3624,7 +3624,7 @@ def _partition_summaries_to_rows( specs = self.tbl.metadata.specs() manifests = [] if snapshot := self.tbl.metadata.current_snapshot(): - for manifest in snapshot.manifests(self.tbl.io): + for manifest in snapshot.manifests(self.tbl.io, snapshot.manifest_list): is_data_file = manifest.content == ManifestContent.DATA is_delete_file = manifest.content == ManifestContent.DELETES manifests.append({ diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index b21a0f5613..6e8096f95c 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -19,6 +19,7 @@ import time from collections import defaultdict from enum import Enum +from functools import lru_cache from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Iterable, List, Mapping, Optional from pydantic import Field, PrivateAttr, model_serializer @@ -247,9 +248,12 @@ def __str__(self) -> str: result_str = f"{operation}id={self.snapshot_id}{parent_id}{schema_id}" return result_str - def manifests(self, io: FileIO) -> List[ManifestFile]: - if self.manifest_list is not None: - file = io.new_input(self.manifest_list) + @staticmethod + @lru_cache + def manifests(io: FileIO, manifest_list: str) -> List[ManifestFile]: + """Return the manifests for the given snapshot.""" + if manifest_list not in (None, ""): + file = io.new_input(manifest_list) return list(read_manifest_list(file)) return [] diff --git a/tests/integration/test_partitioning_key.py b/tests/integration/test_partitioning_key.py index 29f664909c..f45223a224 100644 --- a/tests/integration/test_partitioning_key.py +++ b/tests/integration/test_partitioning_key.py @@ -763,10 +763,14 @@ def test_partition_key( snapshot = iceberg_table.current_snapshot() assert snapshot spark_partition_for_justification = ( - snapshot.manifests(iceberg_table.io)[0].fetch_manifest_entry(iceberg_table.io)[0].data_file.partition + snapshot.manifests(iceberg_table.io, snapshot.manifest_list)[0] + .fetch_manifest_entry(iceberg_table.io)[0] + .data_file.partition ) spark_path_for_justification = ( - snapshot.manifests(iceberg_table.io)[0].fetch_manifest_entry(iceberg_table.io)[0].data_file.file_path + snapshot.manifests(iceberg_table.io, snapshot.manifest_list)[0] + .fetch_manifest_entry(iceberg_table.io)[0] + .data_file.file_path ) assert spark_partition_for_justification == expected_partition_record assert expected_hive_partition_path_slice in spark_path_for_justification diff --git a/tests/integration/test_rest_manifest.py b/tests/integration/test_rest_manifest.py index 82c41cfd93..8c4b2aaf64 100644 --- a/tests/integration/test_rest_manifest.py +++ b/tests/integration/test_rest_manifest.py @@ -75,7 +75,7 @@ def test_write_sample_manifest(table_test_all_types: Table) -> None: if test_snapshot is None: raise ValueError("Table has no current snapshot, check the docker environment") io = table_test_all_types.io - test_manifest_file = test_snapshot.manifests(io)[0] + test_manifest_file = test_snapshot.manifests(io, test_snapshot.manifest_list)[0] test_manifest_entries = test_manifest_file.fetch_manifest_entry(io) entry = test_manifest_entries[0] test_schema = table_test_all_types.schema() diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index 8bb03cd80e..df373bef13 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -217,7 +217,7 @@ def test_read_manifest_v1(generated_manifest_file_file_v1: str) -> None: summary=Summary(Operation.APPEND), schema_id=3, ) - manifest_list = snapshot.manifests(io)[0] + manifest_list = snapshot.manifests(io, snapshot.manifest_list)[0] assert manifest_list.manifest_length == 7989 assert manifest_list.partition_spec_id == 0 @@ -267,7 +267,7 @@ def test_read_manifest_v2(generated_manifest_file_file_v2: str) -> None: summary=Summary(Operation.APPEND), schema_id=3, ) - manifest_list = snapshot.manifests(io)[0] + manifest_list = snapshot.manifests(io, manifest_list=snapshot.manifest_list)[0] assert manifest_list.manifest_length == 7989 assert manifest_list.partition_spec_id == 0 @@ -319,7 +319,7 @@ def test_write_manifest( summary=Summary(Operation.APPEND), schema_id=3, ) - demo_manifest_file = snapshot.manifests(io)[0] + demo_manifest_file = snapshot.manifests(io, snapshot.manifest_list)[0] manifest_entries = demo_manifest_file.fetch_manifest_entry(io) test_schema = Schema( NestedField(1, "VendorID", IntegerType(), False), NestedField(2, "tpep_pickup_datetime", IntegerType(), False) @@ -491,7 +491,7 @@ def test_write_manifest_list( schema_id=3, ) - demo_manifest_list = snapshot.manifests(io) + demo_manifest_list = snapshot.manifests(io, snapshot.manifest_list) with TemporaryDirectory() as tmp_dir: path = tmp_dir + "/manifest-list.avro" output = io.new_output(path) From da2a295c5290e6b8511370974545507a31b1de44 Mon Sep 17 00:00:00 2001 From: chinmay-bhat <12948588+chinmay-bhat@users.noreply.github.com> Date: Tue, 4 Jun 2024 12:59:11 +0530 Subject: [PATCH 2/6] update API --- pyiceberg/catalog/__init__.py | 2 +- pyiceberg/cli/output.py | 2 +- pyiceberg/table/__init__.py | 12 ++++++------ pyiceberg/table/snapshots.py | 6 +++++- tests/integration/test_partitioning_key.py | 8 ++------ tests/integration/test_rest_manifest.py | 2 +- tests/utils/test_manifest.py | 8 ++++---- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index f6c15698de..9a951b5c8e 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -717,7 +717,7 @@ def purge_table(self, identifier: Union[str, Identifier]) -> None: manifest_lists_to_delete = set() manifests_to_delete: List[ManifestFile] = [] for snapshot in metadata.snapshots: - manifests_to_delete += snapshot.manifests(io, snapshot.manifest_list) + manifests_to_delete += snapshot.manifests(io) if snapshot.manifest_list is not None: manifest_lists_to_delete.add(snapshot.manifest_list) diff --git a/pyiceberg/cli/output.py b/pyiceberg/cli/output.py index e9ba377453..56b544c99f 100644 --- a/pyiceberg/cli/output.py +++ b/pyiceberg/cli/output.py @@ -144,7 +144,7 @@ def files(self, table: Table, history: bool) -> None: manifest_list_str = f": {snapshot.manifest_list}" if snapshot.manifest_list else "" list_tree = snapshot_tree.add(f"Snapshot {snapshot.snapshot_id}, schema {snapshot.schema_id}{manifest_list_str}") - manifest_list = snapshot.manifests(io, manifest_list_str) + manifest_list = snapshot.manifests(io) for manifest in manifest_list: manifest_tree = list_tree.add(f"Manifest: {manifest.manifest_path}") for manifest_entry in manifest.fetch_manifest_entry(io, discard_deleted=False): diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 7e4307f499..2d4b342461 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -1708,7 +1708,7 @@ def plan_files(self) -> Iterable[FileScanTask]: manifests = [ manifest_file - for manifest_file in snapshot.manifests(self.io, snapshot.manifest_list) + for manifest_file in snapshot.manifests(self.io) if manifest_evaluators[manifest_file.partition_spec_id](manifest_file) ] @@ -2941,7 +2941,7 @@ def _existing_manifests(self) -> List[ManifestFile]: if previous_snapshot is None: raise ValueError(f"Snapshot could not be found: {self._parent_snapshot_id}") - for manifest in previous_snapshot.manifests(io=self._io, manifest_list=previous_snapshot.manifest_list): + for manifest in previous_snapshot.manifests(io=self._io): if manifest.has_added_files() or manifest.has_existing_files() or manifest.added_snapshot_id == self._snapshot_id: existing_manifests.append(manifest) @@ -2992,7 +2992,7 @@ def _get_entries(manifest: ManifestFile) -> List[ManifestEntry]: if entry.data_file.content == DataFileContent.DATA ] - list_of_entries = executor.map(_get_entries, previous_snapshot.manifests(self._io, previous_snapshot.manifest_list)) + list_of_entries = executor.map(_get_entries, previous_snapshot.manifests(self._io)) return list(chain(*list_of_entries)) else: return [] @@ -3384,7 +3384,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: entries = [] snapshot = self._get_snapshot(snapshot_id) - for manifest in snapshot.manifests(self.tbl.io, snapshot.manifest_list): + for manifest in snapshot.manifests(self.tbl.io): for entry in manifest.fetch_manifest_entry(io=self.tbl.io): column_sizes = entry.data_file.column_sizes or {} value_counts = entry.data_file.value_counts or {} @@ -3546,7 +3546,7 @@ def update_partitions_map( partitions_map: Dict[Tuple[str, Any], Any] = {} snapshot = self._get_snapshot(snapshot_id) - for manifest in snapshot.manifests(self.tbl.io, snapshot.manifest_list): + for manifest in snapshot.manifests(self.tbl.io): for entry in manifest.fetch_manifest_entry(io=self.tbl.io): partition = entry.data_file.partition partition_record_dict = { @@ -3624,7 +3624,7 @@ def _partition_summaries_to_rows( specs = self.tbl.metadata.specs() manifests = [] if snapshot := self.tbl.metadata.current_snapshot(): - for manifest in snapshot.manifests(self.tbl.io, snapshot.manifest_list): + for manifest in snapshot.manifests(self.tbl.io): is_data_file = manifest.content == ManifestContent.DATA is_delete_file = manifest.content == ManifestContent.DELETES manifests.append({ diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 6e8096f95c..d5721f9618 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -250,13 +250,17 @@ def __str__(self) -> str: @staticmethod @lru_cache - def manifests(io: FileIO, manifest_list: str) -> List[ManifestFile]: + def _manifests(io: FileIO, manifest_list: str) -> List[ManifestFile]: """Return the manifests for the given snapshot.""" if manifest_list not in (None, ""): file = io.new_input(manifest_list) return list(read_manifest_list(file)) return [] + def manifests(self, io: FileIO) -> List[ManifestFile]: + """Return the manifests for the given snapshot.""" + return Snapshot._manifests(io, self.manifest_list) + class MetadataLogEntry(IcebergBaseModel): metadata_file: str = Field(alias="metadata-file") diff --git a/tests/integration/test_partitioning_key.py b/tests/integration/test_partitioning_key.py index f45223a224..29f664909c 100644 --- a/tests/integration/test_partitioning_key.py +++ b/tests/integration/test_partitioning_key.py @@ -763,14 +763,10 @@ def test_partition_key( snapshot = iceberg_table.current_snapshot() assert snapshot spark_partition_for_justification = ( - snapshot.manifests(iceberg_table.io, snapshot.manifest_list)[0] - .fetch_manifest_entry(iceberg_table.io)[0] - .data_file.partition + snapshot.manifests(iceberg_table.io)[0].fetch_manifest_entry(iceberg_table.io)[0].data_file.partition ) spark_path_for_justification = ( - snapshot.manifests(iceberg_table.io, snapshot.manifest_list)[0] - .fetch_manifest_entry(iceberg_table.io)[0] - .data_file.file_path + snapshot.manifests(iceberg_table.io)[0].fetch_manifest_entry(iceberg_table.io)[0].data_file.file_path ) assert spark_partition_for_justification == expected_partition_record assert expected_hive_partition_path_slice in spark_path_for_justification diff --git a/tests/integration/test_rest_manifest.py b/tests/integration/test_rest_manifest.py index 8c4b2aaf64..82c41cfd93 100644 --- a/tests/integration/test_rest_manifest.py +++ b/tests/integration/test_rest_manifest.py @@ -75,7 +75,7 @@ def test_write_sample_manifest(table_test_all_types: Table) -> None: if test_snapshot is None: raise ValueError("Table has no current snapshot, check the docker environment") io = table_test_all_types.io - test_manifest_file = test_snapshot.manifests(io, test_snapshot.manifest_list)[0] + test_manifest_file = test_snapshot.manifests(io)[0] test_manifest_entries = test_manifest_file.fetch_manifest_entry(io) entry = test_manifest_entries[0] test_schema = table_test_all_types.schema() diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index df373bef13..8bb03cd80e 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -217,7 +217,7 @@ def test_read_manifest_v1(generated_manifest_file_file_v1: str) -> None: summary=Summary(Operation.APPEND), schema_id=3, ) - manifest_list = snapshot.manifests(io, snapshot.manifest_list)[0] + manifest_list = snapshot.manifests(io)[0] assert manifest_list.manifest_length == 7989 assert manifest_list.partition_spec_id == 0 @@ -267,7 +267,7 @@ def test_read_manifest_v2(generated_manifest_file_file_v2: str) -> None: summary=Summary(Operation.APPEND), schema_id=3, ) - manifest_list = snapshot.manifests(io, manifest_list=snapshot.manifest_list)[0] + manifest_list = snapshot.manifests(io)[0] assert manifest_list.manifest_length == 7989 assert manifest_list.partition_spec_id == 0 @@ -319,7 +319,7 @@ def test_write_manifest( summary=Summary(Operation.APPEND), schema_id=3, ) - demo_manifest_file = snapshot.manifests(io, snapshot.manifest_list)[0] + demo_manifest_file = snapshot.manifests(io)[0] manifest_entries = demo_manifest_file.fetch_manifest_entry(io) test_schema = Schema( NestedField(1, "VendorID", IntegerType(), False), NestedField(2, "tpep_pickup_datetime", IntegerType(), False) @@ -491,7 +491,7 @@ def test_write_manifest_list( schema_id=3, ) - demo_manifest_list = snapshot.manifests(io, snapshot.manifest_list) + demo_manifest_list = snapshot.manifests(io) with TemporaryDirectory() as tmp_dir: path = tmp_dir + "/manifest-list.avro" output = io.new_output(path) From 81a85abba4e4f528c7e264a6b7f21715a96d56a4 Mon Sep 17 00:00:00 2001 From: chinmay-bhat <12948588+chinmay-bhat@users.noreply.github.com> Date: Tue, 4 Jun 2024 13:30:59 +0530 Subject: [PATCH 3/6] small fix --- pyiceberg/table/snapshots.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index d5721f9618..a810196cc4 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -250,10 +250,10 @@ def __str__(self) -> str: @staticmethod @lru_cache - def _manifests(io: FileIO, manifest_list: str) -> List[ManifestFile]: + def _manifests(io: FileIO, manifest_list: Optional[str]) -> List[ManifestFile]: """Return the manifests for the given snapshot.""" if manifest_list not in (None, ""): - file = io.new_input(manifest_list) + file = io.new_input(manifest_list) # type: ignore return list(read_manifest_list(file)) return [] From 5d943a299b4223a6517346d322c458aac525094f Mon Sep 17 00:00:00 2001 From: HonahX Date: Tue, 11 Jun 2024 23:43:36 -0700 Subject: [PATCH 4/6] update --- pyiceberg/table/snapshots.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index a810196cc4..767f5e92f2 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -229,6 +229,15 @@ def __eq__(self, other: Any) -> bool: ) +@lru_cache +def _manifests(io: FileIO, manifest_list: Optional[str]) -> List[ManifestFile]: + """Return the manifests for the given snapshot.""" + if manifest_list not in (None, ""): + file = io.new_input(manifest_list) # type: ignore + return list(read_manifest_list(file)) + return [] + + class Snapshot(IcebergBaseModel): snapshot_id: int = Field(alias="snapshot-id") parent_snapshot_id: Optional[int] = Field(alias="parent-snapshot-id", default=None) @@ -248,15 +257,6 @@ def __str__(self) -> str: result_str = f"{operation}id={self.snapshot_id}{parent_id}{schema_id}" return result_str - @staticmethod - @lru_cache - def _manifests(io: FileIO, manifest_list: Optional[str]) -> List[ManifestFile]: - """Return the manifests for the given snapshot.""" - if manifest_list not in (None, ""): - file = io.new_input(manifest_list) # type: ignore - return list(read_manifest_list(file)) - return [] - def manifests(self, io: FileIO) -> List[ManifestFile]: """Return the manifests for the given snapshot.""" return Snapshot._manifests(io, self.manifest_list) From 237ad2b43650d5ec61ef4fd9a1a5dc263e80b108 Mon Sep 17 00:00:00 2001 From: HonahX Date: Wed, 12 Jun 2024 01:14:51 -0700 Subject: [PATCH 5/6] update --- pyiceberg/table/snapshots.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 767f5e92f2..123611a9f5 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -259,7 +259,7 @@ def __str__(self) -> str: def manifests(self, io: FileIO) -> List[ManifestFile]: """Return the manifests for the given snapshot.""" - return Snapshot._manifests(io, self.manifest_list) + return _manifests(io, self.manifest_list) class MetadataLogEntry(IcebergBaseModel): From 32bf266b459567b7656c7176ef983da7e4c2ab42 Mon Sep 17 00:00:00 2001 From: HonahX Date: Wed, 12 Jun 2024 01:29:34 -0700 Subject: [PATCH 6/6] update --- pyiceberg/table/snapshots.py | 15 +++------------ pyiceberg/utils/manifest_util.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 pyiceberg/utils/manifest_util.py diff --git a/pyiceberg/table/snapshots.py b/pyiceberg/table/snapshots.py index 123611a9f5..b8d4073bd4 100644 --- a/pyiceberg/table/snapshots.py +++ b/pyiceberg/table/snapshots.py @@ -19,15 +19,15 @@ import time from collections import defaultdict from enum import Enum -from functools import lru_cache from typing import TYPE_CHECKING, Any, DefaultDict, Dict, Iterable, List, Mapping, Optional from pydantic import Field, PrivateAttr, model_serializer from pyiceberg.io import FileIO -from pyiceberg.manifest import DataFile, DataFileContent, ManifestFile, read_manifest_list +from pyiceberg.manifest import DataFile, DataFileContent, ManifestFile from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionSpec from pyiceberg.schema import Schema +from pyiceberg.utils.manifest_util import fetch_manifests if TYPE_CHECKING: from pyiceberg.table.metadata import TableMetadata @@ -229,15 +229,6 @@ def __eq__(self, other: Any) -> bool: ) -@lru_cache -def _manifests(io: FileIO, manifest_list: Optional[str]) -> List[ManifestFile]: - """Return the manifests for the given snapshot.""" - if manifest_list not in (None, ""): - file = io.new_input(manifest_list) # type: ignore - return list(read_manifest_list(file)) - return [] - - class Snapshot(IcebergBaseModel): snapshot_id: int = Field(alias="snapshot-id") parent_snapshot_id: Optional[int] = Field(alias="parent-snapshot-id", default=None) @@ -259,7 +250,7 @@ def __str__(self) -> str: def manifests(self, io: FileIO) -> List[ManifestFile]: """Return the manifests for the given snapshot.""" - return _manifests(io, self.manifest_list) + return fetch_manifests(io, self.manifest_list) class MetadataLogEntry(IcebergBaseModel): diff --git a/pyiceberg/utils/manifest_util.py b/pyiceberg/utils/manifest_util.py new file mode 100644 index 0000000000..c4dbc243ef --- /dev/null +++ b/pyiceberg/utils/manifest_util.py @@ -0,0 +1,14 @@ +from functools import lru_cache +from typing import List, Optional + +from pyiceberg.io import FileIO +from pyiceberg.manifest import ManifestFile, read_manifest_list + + +@lru_cache +def fetch_manifests(io: FileIO, manifest_list: Optional[str]) -> List[ManifestFile]: + """Return the manifests for the given snapshot.""" + if manifest_list not in (None, ""): + file = io.new_input(manifest_list) # type: ignore + return list(read_manifest_list(file)) + return []