We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 76d02ad commit 1588701Copy full SHA for 1588701
pyiceberg/io/pyarrow.py
@@ -1441,11 +1441,15 @@ def _task_to_record_batches(
1441
1442
# Apply the user filter
1443
if pyarrow_filter is not None:
1444
- current_batch = current_batch.filter(pyarrow_filter)
+ # Temporary fix until PyArrow 21 is released ( https://github.com/apache/arrow/pull/46057 )
1445
+ table = pa.Table.from_batches([current_batch])
1446
+ table = table.filter(pyarrow_filter)
1447
# skip empty batches
- if current_batch.num_rows == 0:
1448
+ if table.num_rows == 0:
1449
continue
1450
1451
+ current_batch = table.combine_chunks().to_batches()[0]
1452
+
1453
result_batch = _to_requested_schema(
1454
projected_schema,
1455
file_project_schema,
0 commit comments