Skip to content

Commit d3ad61c

Browse files
authored
Remove record_fields from the Record class (#580)
First step towards #579
1 parent e08cc9d commit d3ad61c

File tree

5 files changed

+12
-11
lines changed

5 files changed

+12
-11
lines changed

pyiceberg/manifest.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import math
2020
from abc import ABC, abstractmethod
21+
from copy import copy
2122
from enum import Enum
2223
from types import TracebackType
2324
from typing import (
@@ -909,7 +910,7 @@ def __init__(self, output_file: OutputFile, snapshot_id: int, parent_snapshot_id
909910
self._sequence_number = sequence_number
910911

911912
def prepare_manifest(self, manifest_file: ManifestFile) -> ManifestFile:
912-
wrapped_manifest_file = ManifestFile(*manifest_file.record_fields())
913+
wrapped_manifest_file = copy(manifest_file)
913914

914915
if wrapped_manifest_file.sequence_number == UNASSIGNED_SEQ:
915916
# if the sequence number is being assigned here, then the manifest must be created by the current operation.

pyiceberg/partitioning.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -229,9 +229,9 @@ def partition_to_path(self, data: Record, schema: Schema) -> str:
229229

230230
field_strs = []
231231
value_strs = []
232-
for pos, value in enumerate(data.record_fields()):
232+
for pos in range(len(self.fields)):
233233
partition_field = self.fields[pos]
234-
value_str = partition_field.transform.to_human_string(field_types[pos].field_type, value=value)
234+
value_str = partition_field.transform.to_human_string(field_types[pos].field_type, value=data[pos])
235235

236236
value_str = quote(value_str, safe='')
237237
value_strs.append(value_str)

pyiceberg/table/snapshots.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -274,14 +274,14 @@ def set_partition_summary_limit(self, limit: int) -> None:
274274

275275
def add_file(self, data_file: DataFile, schema: Schema, partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC) -> None:
276276
self.metrics.add_file(data_file)
277-
if len(data_file.partition.record_fields()) != 0:
277+
if len(data_file.partition) > 0:
278278
self.update_partition_metrics(partition_spec=partition_spec, file=data_file, is_add_file=True, schema=schema)
279279

280280
def remove_file(
281281
self, data_file: DataFile, schema: Schema, partition_spec: PartitionSpec = UNPARTITIONED_PARTITION_SPEC
282282
) -> None:
283283
self.metrics.remove_file(data_file)
284-
if len(data_file.partition.record_fields()) != 0:
284+
if len(data_file.partition) > 0:
285285
self.update_partition_metrics(partition_spec=partition_spec, file=data_file, is_add_file=False, schema=schema)
286286

287287
def update_partition_metrics(self, partition_spec: PartitionSpec, file: DataFile, is_add_file: bool, schema: Schema) -> None:

pyiceberg/typedef.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
Callable,
2626
Dict,
2727
Generic,
28-
List,
2928
Literal,
3029
Optional,
3130
Protocol,
@@ -198,9 +197,9 @@ def __repr__(self) -> str:
198197
"""Return the string representation of the Record class."""
199198
return f"{self.__class__.__name__}[{', '.join(f'{key}={repr(value)}' for key, value in self.__dict__.items() if not key.startswith('_'))}]"
200199

201-
def record_fields(self) -> List[str]:
202-
"""Return values of all the fields of the Record class except those specified in skip_fields."""
203-
return [self.__getattribute__(v) if hasattr(self, v) else None for v in self._position_to_field_name]
200+
def __len__(self) -> int:
201+
"""Return the number of fields in the Record class."""
202+
return len(self._position_to_field_name)
204203

205204
def __hash__(self) -> int:
206205
"""Return hash value of the Record class."""

tests/integration/test_rest_manifest.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
# pylint:disable=redefined-outer-name
1818

1919
import inspect
20+
from copy import copy
2021
from enum import Enum
2122
from tempfile import TemporaryDirectory
2223
from typing import Any
@@ -26,7 +27,7 @@
2627

2728
from pyiceberg.catalog import Catalog, load_catalog
2829
from pyiceberg.io.pyarrow import PyArrowFileIO
29-
from pyiceberg.manifest import DataFile, ManifestEntry, write_manifest
30+
from pyiceberg.manifest import DataFile, write_manifest
3031
from pyiceberg.table import Table
3132
from pyiceberg.utils.lazydict import LazyDict
3233

@@ -99,7 +100,7 @@ def test_write_sample_manifest(table_test_all_types: Table) -> None:
99100
sort_order_id=entry.data_file.sort_order_id,
100101
spec_id=entry.data_file.spec_id,
101102
)
102-
wrapped_entry_v2 = ManifestEntry(*entry.record_fields())
103+
wrapped_entry_v2 = copy(entry)
103104
wrapped_entry_v2.data_file = wrapped_data_file_v2_debug
104105
wrapped_entry_v2_dict = todict(wrapped_entry_v2)
105106
# This one should not be written

0 commit comments

Comments
 (0)