Skip to content

Commit d213dee

Browse files
committed
MLE-24717 Some Polaris fixes
1 parent 00c2dc2 commit d213dee

File tree

3 files changed

+18
-2
lines changed

3 files changed

+18
-2
lines changed

marklogic-spark-connector/src/main/java/com/marklogic/spark/reader/JsonRowDeserializer.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import java.util.ArrayList;
2121
import java.util.HashMap;
2222
import java.util.List;
23+
import java.util.Objects;
2324

2425
/**
2526
* Handles deserializing a JSON object into a Spark InternalRow. This is accomplished via Spark's JacksonParser.
@@ -44,7 +45,9 @@ public JsonRowDeserializer(StructType schema) {
4445
}
4546

4647
public InternalRow deserializeJson(String json) {
47-
return this.jacksonParser.parse(json, this.jsonParserCreator, this.utf8StringCreator).head();
48+
var parsedResult = this.jacksonParser.parse(json, this.jsonParserCreator, this.utf8StringCreator);
49+
Objects.requireNonNull(parsedResult, "The output from parsing the JSON should never be null");
50+
return parsedResult.head();
4851
}
4952

5053
private JacksonParser newJacksonParser(StructType schema) {

marklogic-spark-connector/src/main/java/com/marklogic/spark/reader/file/FileUtil.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,18 @@ static ZipEntry findNextFileEntry(ZipInputStream zipInputStream) throws IOExcept
4747
}
4848

4949
static FilePartition[] makeFilePartitions(String[] files, int numPartitions) {
50-
int filesPerPartition = (int) Math.ceil((double) files.length / (double) numPartitions);
50+
if (numPartitions <= 0) {
51+
// Divide-by-zero protection.
52+
numPartitions = 1;
53+
}
54+
55+
final int filesPerPartition = (int) Math.ceil((double) files.length / (double) numPartitions);
5156
if (files.length < numPartitions) {
5257
numPartitions = files.length;
5358
}
59+
5460
final FilePartition[] partitions = new FilePartition[numPartitions];
61+
5562
List<String> currentPartition = new ArrayList<>();
5663
int partitionIndex = 0;
5764
for (int i = 0; i < files.length; i++) {

marklogic-spark-connector/src/test/java/com/marklogic/spark/reader/file/MakeFilePartitionsTest.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,4 +38,10 @@ void morePartitionsThanFiles() {
3838
assertEquals("B", partitions[1].getPaths().get(0));
3939
assertEquals("C", partitions[2].getPaths().get(0));
4040
}
41+
42+
@Test
43+
void zeroPartitions() {
44+
FilePartition[] partitions = FileUtil.makeFilePartitions(new String[]{"A", "B", "C"}, 0);
45+
assertEquals(1, partitions.length, "If a value less than 1 is passed in, partitions should default to 1.");
46+
}
4147
}

0 commit comments

Comments
 (0)