Skip to content

Commit f6fd1d8

Browse files
committed
GH-3226: Avoid LongStream usage reading files
and use an ad-hoc Long Iterator
1 parent 1f1e07b commit f6fd1d8

File tree

1 file changed

+24
-4
lines changed

1 file changed

+24
-4
lines changed

parquet-hadoop/src/main/java/org/apache/parquet/hadoop/InternalParquetRecordReader.java

Lines changed: 24 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
import java.util.Optional;
3131
import java.util.PrimitiveIterator;
3232
import java.util.Set;
33-
import java.util.stream.LongStream;
3433
import org.apache.hadoop.conf.Configuration;
3534
import org.apache.parquet.ParquetReadOptions;
3635
import org.apache.parquet.column.page.PageReadStore;
@@ -320,21 +319,42 @@ private void resetRowIndexIterator(PageReadStore pages) {
320319
if (pages.getRowIndexes().isPresent()) {
321320
rowIdxInRowGroupItr = pages.getRowIndexes().get();
322321
} else {
323-
rowIdxInRowGroupItr = LongStream.range(0, pages.getRowCount()).iterator();
322+
rowIdxInRowGroupItr = new LongIterator(pages.getRowCount());
324323
}
325324
// Adjust the row group offset in the `rowIndexWithinRowGroupIterator` iterator.
325+
final long rowGroupRowIdxOffsetValue = rowGroupRowIdxOffset.get();
326326
this.rowIdxInFileItr = new PrimitiveIterator.OfLong() {
327327
public long nextLong() {
328-
return rowGroupRowIdxOffset.get() + rowIdxInRowGroupItr.nextLong();
328+
return rowGroupRowIdxOffsetValue + rowIdxInRowGroupItr.nextLong();
329329
}
330330

331331
public boolean hasNext() {
332332
return rowIdxInRowGroupItr.hasNext();
333333
}
334334

335335
public Long next() {
336-
return rowGroupRowIdxOffset.get() + rowIdxInRowGroupItr.next();
336+
return rowGroupRowIdxOffsetValue + rowIdxInRowGroupItr.next();
337337
}
338338
};
339339
}
340+
341+
private static class LongIterator implements PrimitiveIterator.OfLong {
342+
343+
private final long maxValue;
344+
private long currentValue = 0;
345+
346+
public LongIterator(long maxValue) {
347+
this.maxValue = maxValue;
348+
}
349+
350+
@Override
351+
public long nextLong() {
352+
return currentValue++;
353+
}
354+
355+
@Override
356+
public boolean hasNext() {
357+
return currentValue < maxValue;
358+
}
359+
}
340360
}

0 commit comments

Comments
 (0)