Skip to content

Commit 37068da

Browse files
committed
we shouldn't try to act on equality delete files
1 parent c8c53af commit 37068da

1 file changed

Lines changed: 6 additions & 1 deletion

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1727,7 +1727,12 @@ def _task_to_record_batches(
17271727

17281728
def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[str, list[ChunkedArray]]:
17291729
deletes_per_file: dict[str, list[ChunkedArray]] = {}
1730-
unique_deletes = set(itertools.chain.from_iterable([task.delete_files for task in tasks]))
1730+
unique_deletes = {
1731+
delete_file
1732+
for task in tasks
1733+
for delete_file in task.delete_files
1734+
if delete_file.content == DataFileContent.POSITION_DELETES
1735+
}
17311736
if len(unique_deletes) > 0:
17321737
executor = ExecutorFactory.get_or_create()
17331738
deletes_per_files: Iterator[dict[str, ChunkedArray]] = executor.map(

0 commit comments

Comments
 (0)