Skip to content

Commit 8f2e787

Browse files
authored
Fix overwrite when filtering complete files (apache#1023)
1 parent 159805d commit 8f2e787

File tree

2 files changed

+29
-2
lines changed

2 files changed

+29
-2
lines changed

pyiceberg/table/__init__.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -593,7 +593,9 @@ def delete(self, delete_filter: Union[str, BooleanExpression], snapshot_properti
593593
filtered_df = df.filter(preserve_row_filter)
594594

595595
# Only rewrite if there are records being deleted
596-
if len(df) != len(filtered_df):
596+
if len(filtered_df) == 0:
597+
replaced_files.append((original_file.file, []))
598+
elif len(df) != len(filtered_df):
597599
replaced_files.append((
598600
original_file.file,
599601
list(

tests/integration/test_writes/test_writes.py

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,13 @@
3838
from pyiceberg.catalog.rest import RestCatalog
3939
from pyiceberg.catalog.sql import SqlCatalog
4040
from pyiceberg.exceptions import NoSuchTableError
41+
from pyiceberg.expressions import In
4142
from pyiceberg.io.pyarrow import _dataframe_to_data_files
4243
from pyiceberg.partitioning import PartitionField, PartitionSpec
4344
from pyiceberg.schema import Schema
4445
from pyiceberg.table import TableProperties
4546
from pyiceberg.transforms import IdentityTransform
46-
from pyiceberg.types import IntegerType, LongType, NestedField
47+
from pyiceberg.types import IntegerType, LongType, NestedField, StringType
4748
from utils import _create_table
4849

4950

@@ -1309,3 +1310,27 @@ def test_table_v1_with_null_nested_namespace(session_catalog: Catalog, arrow_tab
13091310

13101311
# We expect no error here
13111312
session_catalog.drop_table(identifier)
1313+
1314+
1315+
@pytest.mark.integration
1316+
def test_overwrite_all_data_with_filter(session_catalog: Catalog) -> None:
1317+
schema = Schema(
1318+
NestedField(1, "id", StringType(), required=True),
1319+
NestedField(2, "name", StringType(), required=False),
1320+
identifier_field_ids=[1],
1321+
)
1322+
1323+
data = pa.Table.from_pylist(
1324+
[
1325+
{"id": "1", "name": "Amsterdam"},
1326+
{"id": "2", "name": "San Francisco"},
1327+
{"id": "3", "name": "Drachten"},
1328+
],
1329+
schema=schema.as_arrow(),
1330+
)
1331+
1332+
identifier = "default.test_overwrite_all_data_with_filter"
1333+
tbl = _create_table(session_catalog, identifier, data=[data], schema=schema)
1334+
tbl.overwrite(data, In("id", ["1", "2", "3"]))
1335+
1336+
assert len(tbl.scan().to_arrow()) == 3

0 commit comments

Comments
 (0)