Skip to content

Commit 3e1d861

Browse files
committed
ORC-2049: Move MAX_ARRAY_SIZE to RecordReaderUtils from IOUtils
### What changes were proposed in this pull request? This PR aims to move `MAX_ARRAY_SIZE` to `RecordReaderUtils` from `IOUtils`. ### Why are the changes needed? Only `RecordReaderUtils` uses `IOUtils.MAX_ARRAY_SIZE`. We had better move it to `RecordReaderUtils` to improve modularity. ### How was this patch tested? Pass the CIs. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #2472 from dongjoon-hyun/ORC-2049. Authored-by: Dongjoon Hyun <dongjoon@apache.org> Signed-off-by: Dongjoon Hyun <dongjoon@apache.org>
1 parent 75bde1f commit 3e1d861

File tree

3 files changed

+20
-19
lines changed

3 files changed

+20
-19
lines changed

java/core/src/java/org/apache/orc/impl/IOUtils.java

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,6 @@ public final class IOUtils {
3030

3131
public static final int DEFAULT_BUFFER_SIZE = 8192;
3232

33-
/**
34-
* The maximum size of array to allocate, value being the same as {@link java.util.Hashtable}
35-
*/
36-
public static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;
37-
3833
/**
3934
* Returns a new byte array of size {@link #DEFAULT_BUFFER_SIZE}.
4035
*

java/core/src/java/org/apache/orc/impl/RecordReaderUtils.java

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,11 @@
4747
* Stateless methods shared between RecordReaderImpl and EncodedReaderImpl.
4848
*/
4949
public class RecordReaderUtils {
50+
/**
51+
* The maximum size of array to allocate, value being the same as {@link java.util.Hashtable}
52+
*/
53+
public static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;
54+
5055
private static final HadoopShims SHIMS = HadoopShimsFactory.get();
5156
private static final boolean supportVectoredIO =
5257
SHIMS.supportVectoredIO(VersionInfo.getVersion());
@@ -836,12 +841,12 @@ static ChunkReader create(BufferChunk from, BufferChunk to) {
836841
current = (BufferChunk) current.next;
837842
}
838843
reqBytes += currentEnd - currentStart;
839-
if (reqBytes > IOUtils.MAX_ARRAY_SIZE) {
840-
throw new IllegalArgumentException("invalid reqBytes value " + reqBytes + ",out of bounds " + IOUtils.MAX_ARRAY_SIZE);
844+
if (reqBytes > MAX_ARRAY_SIZE) {
845+
throw new IllegalArgumentException("invalid reqBytes value " + reqBytes + ",out of bounds " + MAX_ARRAY_SIZE);
841846
}
842847
long readBytes = end - start;
843-
if (readBytes > IOUtils.MAX_ARRAY_SIZE) {
844-
throw new IllegalArgumentException("invalid readBytes value " + readBytes + ",out of bounds " + IOUtils.MAX_ARRAY_SIZE);
848+
if (readBytes > MAX_ARRAY_SIZE) {
849+
throw new IllegalArgumentException("invalid readBytes value " + readBytes + ",out of bounds " + MAX_ARRAY_SIZE);
845850
}
846851
return new ChunkReader(from, to, (int) readBytes, (int) reqBytes);
847852
}

java/core/src/test/org/apache/orc/impl/TestRecordReaderUtils.java

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -209,30 +209,31 @@ public void testChunkReaderCreateOffsetExceedsMaxInt() {
209209

210210
@Test
211211
public void testChunkReaderCreateReqBytesAndReadBytesValidation() {
212+
int MAX_ARRAY_SIZE = RecordReaderUtils.MAX_ARRAY_SIZE;
212213
BufferChunkList rangeList = new TestOrcLargeStripe.RangeBuilder()
213-
.range(0, IOUtils.MAX_ARRAY_SIZE)
214-
.range(1L + IOUtils.MAX_ARRAY_SIZE, IOUtils.MAX_ARRAY_SIZE + 1)
215-
.range(2L * IOUtils.MAX_ARRAY_SIZE, IOUtils.MAX_ARRAY_SIZE - 4)
216-
.range(3L * IOUtils.MAX_ARRAY_SIZE, 2)
214+
.range(0, MAX_ARRAY_SIZE)
215+
.range(1L + MAX_ARRAY_SIZE, MAX_ARRAY_SIZE + 1)
216+
.range(2L * MAX_ARRAY_SIZE, MAX_ARRAY_SIZE - 4)
217+
.range(3L * MAX_ARRAY_SIZE, 2)
217218
.build();
218219

219220
// reqBytes,readBytes boundary value
220221
RecordReaderUtils.ChunkReader chunkReader = RecordReaderUtils.ChunkReader.create(rangeList.get(0), 0);
221-
assertEquals(chunkReader.getReadBytes(), IOUtils.MAX_ARRAY_SIZE);
222-
assertEquals(chunkReader.getReqBytes(), IOUtils.MAX_ARRAY_SIZE);
222+
assertEquals(chunkReader.getReadBytes(), MAX_ARRAY_SIZE);
223+
assertEquals(chunkReader.getReqBytes(), MAX_ARRAY_SIZE);
223224

224-
// reqBytes > IOUtils.MAX_ARRAY_SIZE validation
225+
// reqBytes > MAX_ARRAY_SIZE validation
225226
assertThrowsExactly(IllegalArgumentException.class,
226227
() -> RecordReaderUtils.ChunkReader.create(rangeList.get(1), 0),
227228
() -> String.format("invalid reqBytes value %d,out of bounds %d",
228-
rangeList.get(1).getLength(), IOUtils.MAX_ARRAY_SIZE)
229+
rangeList.get(1).getLength(), MAX_ARRAY_SIZE)
229230
);
230231

231-
// readBytes > IOUtils.MAX_ARRAY_SIZE validation
232+
// readBytes > MAX_ARRAY_SIZE validation
232233
assertThrowsExactly(IllegalArgumentException.class,
233234
() -> RecordReaderUtils.ChunkReader.create(rangeList.get(2), 100),
234235
() -> String.format("invalid readBytes value %d,out of bounds %d",
235-
rangeList.get(3).getEnd() - rangeList.get(2).getOffset(), IOUtils.MAX_ARRAY_SIZE)
236+
rangeList.get(3).getEnd() - rangeList.get(2).getOffset(), MAX_ARRAY_SIZE)
236237
);
237238
}
238239

0 commit comments

Comments
 (0)