Skip to content

Commit 27665e7

Browse files
committed
#809 Fix PR suggestions (Thanks @coderabbitai).
1 parent 7cfb193 commit 27665e7

File tree

3 files changed

+10
-9
lines changed

3 files changed

+10
-9
lines changed

cobol-parser/src/main/scala/za/co/absa/cobrix/cobol/reader/VarLenNestedReader.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,9 @@ class VarLenNestedReader[T: ClassTag](copybookContents: Seq[String],
153153
logger.info(s"Input split size = ${inputSplitSizeRecords.get} records")
154154
} else {
155155
if (inputSplitSizeMB.nonEmpty) {
156-
if (inputSplitSizeMB.get < 1 || inputSplitSizeMB.get > 200000) {
157-
throw new IllegalArgumentException(s"Invalid input split size of ${inputSplitSizeMB.get} MB.")
156+
val maxSplitSizeMB = if (dataStream.isCompressed) 200000 else 2000
157+
if (inputSplitSizeMB.get < 1 || inputSplitSizeMB.get > maxSplitSizeMB) {
158+
throw new IllegalArgumentException(s"Invalid input split size of ${inputSplitSizeMB.get} MB (max allowed: $maxSplitSizeMB MB).")
158159
}
159160
logger.info(s"Input split size = ${inputSplitSizeMB.get} MB")
160161
}

spark-cobol/src/main/scala/za/co/absa/cobrix/spark/cobol/source/streaming/BufferedFSDataInputStream.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,9 +90,9 @@ class BufferedFSDataInputStream(filePath: Path, hadoopConfig: Configuration, sta
9090
val available = bufferContainBytes - bufferPos
9191
val bytesToCopy = Math.min(lengthLeft, available)
9292
System.arraycopy(buffer, bufferPos, b, offsetLeft, bytesToCopy)
93-
bufferPos += bufferContainBytes
94-
offsetLeft += bufferContainBytes
95-
lengthLeft -= bufferContainBytes
93+
bufferPos += bytesToCopy
94+
offsetLeft += bytesToCopy
95+
lengthLeft -= bytesToCopy
9696
}
9797
}
9898
}
@@ -136,7 +136,7 @@ class BufferedFSDataInputStream(filePath: Path, hadoopConfig: Configuration, sta
136136

137137
if (startOffset > 0) {
138138
if (codec == null) {
139-
baseStream.seek(startOffset)
139+
fsIn.seek(startOffset)
140140
} else {
141141
var toSkip = startOffset
142142
while (toSkip > 0) {

spark-cobol/src/test/scala/za/co/absa/cobrix/spark/cobol/source/integration/Test40CompressesFilesSpec.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -161,11 +161,11 @@ class Test40CompressesFilesSpec extends AnyFunSuite with SparkTestBase with Bina
161161
}
162162

163163
test("Test compressed EBCDIC gzip file with indexes") {
164-
testCompressedFile("../data/test40_data/example.dat.gz")
164+
testCompressedFile("../data/test40_data/example.dat.gz", useIndexes = true)
165165
}
166166

167-
test("Test compressed EBCDIC bzip2 file with indexes") {
168-
testCompressedFile("../data/test40_data/example.dat.bz2")
167+
test("Test compressed EBCDIC bzip2 file with indexes") {
168+
testCompressedFile("../data/test40_data/example.dat.bz2", useIndexes = true)
169169
}
170170

171171
test("read mixed compressed EBCDIC files") {

0 commit comments

Comments
 (0)