diff --git a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java index 47cf1af74..117ee7489 100644 --- a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java +++ b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java @@ -1,101 +1,232 @@ package io.delta.core.data; -import java.util.HashMap; -import java.util.Map; +import java.util.*; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.delta.core.types.LongType; -import io.delta.core.types.StringType; -import io.delta.core.types.StructField; -import io.delta.core.types.StructType; +import io.delta.core.types.*; public class JsonRow implements Row { - // TODO: we can do this cleaner / smarter / better - private static Object parse(ObjectNode rootNode, StructField field) { - if (rootNode.get(field.name) == null) { - if (field.nullable) { - return null; - } + //////////////////////////////////////////////////////////////////////////////// + // Static Methods + //////////////////////////////////////////////////////////////////////////////// - throw new NullPointerException( - String.format("Root node at key %s is null but field %s isn't nullable", rootNode, field) - ); + private static Object decodeElement(JsonNode jsonValue, DataType dataType) { + if (dataType instanceof UnresolvedDataType) { + if (jsonValue.isTextual()) { + return jsonValue.textValue(); + } else if (jsonValue instanceof ObjectNode) { + throw new RuntimeException("TODO handle UnresolvedDataType of type object"); + } } - JsonNode jsonValue = rootNode.get(field.name); + if (dataType instanceof BooleanType) { + if (!jsonValue.isBoolean()) { + throw new RuntimeException( + String.format("Couldn't decode %s, expected a boolean", jsonValue) + ); + } + return jsonValue.booleanValue(); + } - if (field.dataType instanceof LongType) { - assert (jsonValue.isLong()) : - String.format("RootNode at %s isn't a long", field.name); - return jsonValue.longValue(); + if (dataType instanceof IntegerType) { + // TODO: handle other number cases (e.g. short) and throw on invalid cases (e.g. long) + if (!jsonValue.isInt()) { + throw new RuntimeException( + String.format("Couldn't decode %s, expected an int", jsonValue) + ); + } + return jsonValue.intValue(); } - if (field.dataType instanceof StructType) { - assert (jsonValue.isObject()) : - String.format("RootNode at %s isn't a object", field.name); - return (ObjectNode) jsonValue; + if (dataType instanceof LongType) { + if (!jsonValue.isNumber()) { + throw new RuntimeException( + String.format("Couldn't decode %s, expected a long", jsonValue) + ); + } + return jsonValue.numberValue().longValue(); } - if (field.dataType instanceof StringType) { - assert (jsonValue.isTextual()) : - String.format("RootNode at %s isn't a string", field.name); + if (dataType instanceof StringType) { + if (!jsonValue.isTextual()) { + throw new RuntimeException( + String.format("Couldn't decode %s, expected a string", jsonValue) + ); + } return jsonValue.textValue(); } + if (dataType instanceof StructType) { + if (!jsonValue.isObject()) { + throw new RuntimeException( + String.format("Couldn't decode %s, expected a struct", jsonValue) + ); + } + return new JsonRow((ObjectNode) jsonValue, (StructType) dataType); + } + + if (dataType instanceof ArrayType) { + if (!jsonValue.isArray()) { + throw new RuntimeException( + String.format("Couldn't decode %s, expected an array", jsonValue) + ); + } + final ArrayType arrayType = ((ArrayType) dataType); + final ArrayNode jsonArray = (ArrayNode) jsonValue; + final List output = new ArrayList<>(); + + + for (Iterator it = jsonArray.elements(); it.hasNext();) { + final JsonNode element = it.next(); + final Object parsedElement = + decodeElement(element, arrayType.getElementType()); + output.add(parsedElement); + } + return output; + } + + if (dataType instanceof MapType) { + if (!jsonValue.isObject()) { + throw new RuntimeException( + String.format("Couldn't decode %s, expected a map", jsonValue) + ); + } + final MapType mapType = (MapType) dataType; + final Iterator> iter = ((ObjectNode) jsonValue).fields(); + final Map output = new HashMap<>(); + + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String keyParsed = entry.getKey(); // TODO: handle non-String keys + Object valueParsed = decodeElement(entry.getValue(), mapType.getValueType()); + output.put(keyParsed, valueParsed); + } + + return output; + } + throw new UnsupportedOperationException( - String.format("Unsupported DataType %s", field.dataType.typeName()) + String.format("Unsupported DataType %s for RootNode %s", dataType.typeName(), jsonValue) ); } + private static Object decodeField(ObjectNode rootNode, StructField field) { + if (rootNode.get(field.name) == null) { + if (field.nullable) { + return null; + } + + throw new RuntimeException( + String.format( + "Root node at key %s is null but field isn't nullable. Root node: %s", + field.name, + rootNode + ) + ); + } + + return decodeElement(rootNode.get(field.name), field.dataType); + } + + //////////////////////////////////////////////////////////////////////////////// + // Instance Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + private final ObjectNode rootNode; - private final Map ordinalToValueMap; + private final Object[] parsedValues; private final StructType readSchema; public JsonRow(ObjectNode rootNode, StructType readSchema) { this.rootNode = rootNode; this.readSchema = readSchema; - this.ordinalToValueMap = new HashMap<>(); + this.parsedValues = new Object[readSchema.length()]; for (int i = 0; i < readSchema.length(); i++) { final StructField field = readSchema.at(i); - Object val = parse(rootNode, field); - ordinalToValueMap.put(i, val); + final Object parsedValue = decodeField(rootNode, field); + parsedValues[i] = parsedValue; } } + //////////////////////////////////////// + // Public APIs + //////////////////////////////////////// + + @Override + public boolean isNullAt(int ordinal) { + return parsedValues[ordinal] == null; + } + + @Override + public boolean getBoolean(int ordinal) { + assertType(ordinal, BooleanType.INSTANCE); + return (boolean) parsedValues[ordinal]; + } + + @Override + public int getInt(int ordinal) { + assertType(ordinal, IntegerType.INSTANCE); + return (int) parsedValues[ordinal]; + } + @Override public long getLong(int ordinal) { - assert (readSchema.at(ordinal).dataType instanceof LongType); - return (long) ordinalToValueMap.get(ordinal); + assertType(ordinal, LongType.INSTANCE); + return (long) parsedValues[ordinal]; } @Override public String getString(int ordinal) { - assert (readSchema.at(ordinal).dataType instanceof StringType); - - return (String) ordinalToValueMap.get(ordinal); + assertType(ordinal, StringType.INSTANCE); + return (String) parsedValues[ordinal]; } @Override public Row getRecord(int ordinal) { - assert (readSchema.at(ordinal).dataType instanceof StructType); + assertType(ordinal, StructType.EMPTY_INSTANCE); + return (JsonRow) parsedValues[ordinal]; + } - if (ordinalToValueMap.get(ordinal) == null) return null; + @Override + public List getList(int ordinal) { + assertType(ordinal, ArrayType.EMPTY_INSTANCE); + return (List) parsedValues[ordinal]; + } - return new JsonRow( - (ObjectNode) ordinalToValueMap.get(ordinal), - (StructType) readSchema.at(ordinal).dataType - ); + @Override + public Map getMap(int ordinal) { + assertType(ordinal, MapType.EMPTY_INSTANCE); + return (Map) parsedValues[ordinal]; } @Override public String toString() { return "JsonRow{" + "rootNode=" + rootNode + - ", ordinalToValueMap=" + ordinalToValueMap + + ", parsedValues=" + parsedValues + ", readSchema=" + readSchema + '}'; } + + //////////////////////////////////////// + // Private Helper Methods + //////////////////////////////////////// + + private void assertType(int ordinal, DataType expectedType) { + final String actualTypeName = readSchema.at(ordinal).dataType.typeName(); + if (!actualTypeName.equals(expectedType.typeName()) && + !actualTypeName.equals(UnresolvedDataType.INSTANCE.typeName())) { + throw new RuntimeException( + String.format( + "Tried to read %s at ordinal %s but actual data type is %s", + expectedType.typeName(), + ordinal, + actualTypeName + ) + ); + } + } } diff --git a/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java b/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java index 18fd37586..cd6782836 100644 --- a/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java +++ b/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java @@ -108,13 +108,7 @@ public boolean hasNext() { @Override public Row next() { - final String json = iter.next(); - try { - final JsonNode jsonNode = objectMapper.readTree(json); - return new JsonRow((ObjectNode) jsonNode, readSchema); - } catch (JsonProcessingException ex) { - throw new RuntimeException(String.format("Could not parse JSON: %s", json), ex); - } + return parseJson(iter.next(), readSchema); } }; } @@ -130,8 +124,13 @@ public CloseableIterator readParquetFile(String path, StructType readSchema } @Override - public Row parseStats(String statsJson) { - return null; + public Row parseJson(String json, StructType readSchema) { + try { + final JsonNode jsonNode = objectMapper.readTree(json); + return new JsonRow((ObjectNode) jsonNode, readSchema); + } catch (JsonProcessingException ex) { + throw new RuntimeException(String.format("Could not parse JSON: %s", json), ex); + } } @Override diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000000.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 000000000..a771bd1bb Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000000.json.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000001.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000001.json.crc new file mode 100644 index 000000000..90931c186 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000001.json.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000002.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000002.json.crc new file mode 100644 index 000000000..d8cfcd768 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000002.json.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000003.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000003.json.crc new file mode 100644 index 000000000..8999cc2a1 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000003.json.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000004.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000004.json.crc new file mode 100644 index 000000000..0be59fee6 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000004.json.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000005.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000005.json.crc new file mode 100644 index 000000000..24adaf5b0 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000005.json.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000006.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000006.json.crc new file mode 100644 index 000000000..f2b4583ce Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000006.json.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000007.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000007.json.crc new file mode 100644 index 000000000..912ab8773 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000007.json.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000008.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000008.json.crc new file mode 100644 index 000000000..da9133971 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000008.json.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000000.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000000.json new file mode 100644 index 000000000..3677534c6 --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000000.json @@ -0,0 +1,13 @@ +{"commitInfo":{"timestamp":1680202522862,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4780"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"a4769250-e8ad-44a7-8a90-b384ac907381"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part_a\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part_b\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part_a","part_b"],"configuration":{},"createdTime":1680202519496}} +{"add":{"path":"part_a=0/part_b=0/part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":0},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":2},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3},\"maxValues\":{\"id\":3},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-322e51ba-0754-4929-9eb8-6c714a86f901.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":6},\"maxValues\":{\"id\":6},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-b4c08c7f-fd3e-48a7-95dd-829435d27ba6.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":8},\"maxValues\":{\"id\":8},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":5},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-a5f8c5df-41b0-48b5-b3ac-499d27a94c54.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":7},\"maxValues\":{\"id\":7},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":9},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000001.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000001.json new file mode 100644 index 000000000..d84612a84 --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000001.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202527500,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4779"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"8aceb033-273d-4ed7-95cf-a5ef87c355e3"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":477,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":10},\"maxValues\":{\"id\":10},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":12},\"maxValues\":{\"id\":12},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":14},\"maxValues\":{\"id\":14},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":11},\"maxValues\":{\"id\":11},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-39d574e1-c089-4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":13},\"maxValues\":{\"id\":13},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":16},\"maxValues\":{\"id\":16},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-8b04e895-6377-41f7-89a7-a763d8b9f2be.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":18},\"maxValues\":{\"id\":18},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-5a0c01cf-07af-47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":15},\"maxValues\":{\"id\":15},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-07da6a7d-231d-4c66-9c96-752defe5436b.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":17},\"maxValues\":{\"id\":17},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":19},\"maxValues\":{\"id\":19},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000002.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000002.json new file mode 100644 index 000000000..b06f60ab9 --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000002.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202528841,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4780"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"408658ef-25b8-4747-b59c-553b7143abda"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":20},\"maxValues\":{\"id\":20},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":22},\"maxValues\":{\"id\":22},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-76cb0626-0e86-42f6-bf48-809045bfc174.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":24},\"maxValues\":{\"id\":24},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-a05e7f3b-1548-47d1-95d2-b802d9f602c0.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":21},\"maxValues\":{\"id\":21},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-4ee45e83-7063-4c4b-a008-c7068640b8a5.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":23},\"maxValues\":{\"id\":23},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":26},\"maxValues\":{\"id\":26},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-43e6640a-5f8b-4205-b274-7a324fc3f950.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":28},\"maxValues\":{\"id\":28},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":25},\"maxValues\":{\"id\":25},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-89424aa4-1fc4-4aa9-819c-3c164d7e926e.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":27},\"maxValues\":{\"id\":27},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":29},\"maxValues\":{\"id\":29},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000003.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000003.json new file mode 100644 index 000000000..1c8785eda --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000003.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202530099,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4779"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"c17af0ad-f0a1-41a7-841d-598c2e07d0bf"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-94c8da2e-b2e8-4aed-9579-611eb8dc52e4.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":477,"modificationTime":1680202529000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":30},\"maxValues\":{\"id\":30},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202529000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":32},\"maxValues\":{\"id\":32},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-b8aa7c7e-d3b4-410e-8419-4095f4e69b16.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202530000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":34},\"maxValues\":{\"id\":34},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-815f63f2-21c2-4e16-b8c2-68e7d6414be8.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202530000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":31},\"maxValues\":{\"id\":31},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202530000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":33},\"maxValues\":{\"id\":33},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202529000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":36},\"maxValues\":{\"id\":36},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-28fb675f-e269-4174-ba7d-4e8cb41a65d0.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202529000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":38},\"maxValues\":{\"id\":38},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202530000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":35},\"maxValues\":{\"id\":35},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-35842ca7-408d-4023-bf03-1195979e33a8.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202530000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":37},\"maxValues\":{\"id\":37},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-f847a367-cc91-4d50-85bb-83aeaa5c63a1.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202530000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":39},\"maxValues\":{\"id\":39},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000004.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000004.json new file mode 100644 index 000000000..88e28d1e3 --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000004.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202531634,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4779"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"79a877c7-781a-478f-bf55-55ddc9dd7972"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":40},\"maxValues\":{\"id\":40},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-6b5109e8-5533-4165-b61c-3a83d0033ae1.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":477,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":42},\"maxValues\":{\"id\":42},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":44},\"maxValues\":{\"id\":44},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-f86461d3-6066-4b87-ab45-48c06bddee6c.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":41},\"maxValues\":{\"id\":41},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-e6b34e43-1eda-454b-be03-5442a09982de.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":43},\"maxValues\":{\"id\":43},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":46},\"maxValues\":{\"id\":46},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-71d28824-3adc-406d-ae8a-2bf56033861e.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":48},\"maxValues\":{\"id\":48},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":45},\"maxValues\":{\"id\":45},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-60f06dc7-46d8-4ec9-9735-81db2fd05163.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":47},\"maxValues\":{\"id\":47},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":49},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000005.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000005.json new file mode 100644 index 000000000..dc0d721be --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000005.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202532814,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":4,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4780"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"e158ea6f-74b0-4f52-8db1-6dd8c2302e55"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":50},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":52},\"maxValues\":{\"id\":52},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":54},\"maxValues\":{\"id\":54},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-e7557908-e149-4724-9c15-72b4d5df3c78.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":51},\"maxValues\":{\"id\":51},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-cc14211d-2fd6-4fc9-baae-b62451ecf8a1.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":53},\"maxValues\":{\"id\":53},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":56},\"maxValues\":{\"id\":56},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-57289bbc-8e73-416a-a973-0b824d398b97.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":58},\"maxValues\":{\"id\":58},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":55},\"maxValues\":{\"id\":55},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-d4895419-3e8e-4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":57},\"maxValues\":{\"id\":57},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":59},\"maxValues\":{\"id\":59},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000006.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000006.json new file mode 100644 index 000000000..2c931c5c7 --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000006.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202534045,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":5,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4778"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"4ef8295a-eb45-4a17-8448-bbbd6375869f"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-8429619b-3453-4ed9-b8cc-d95960b905bb.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202533000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":60},\"maxValues\":{\"id\":60},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":477,"modificationTime":1680202533000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":62},\"maxValues\":{\"id\":62},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202533000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":64},\"maxValues\":{\"id\":64},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202534000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":61},\"maxValues\":{\"id\":61},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-0e36beae-e5e5-4c33-8a3e-b68a446f2e2c.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202534000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":63},\"maxValues\":{\"id\":63},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":477,"modificationTime":1680202533000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":66},\"maxValues\":{\"id\":66},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-ad757130-9a3e-430a-add7-a0fc2ddaadde.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202533000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":68},\"maxValues\":{\"id\":68},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202533000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":65},\"maxValues\":{\"id\":65},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-5e20ed9a-4496-4a62-9dff-bb8de1862fc0.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202534000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":67},\"maxValues\":{\"id\":67},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202534000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":69},\"maxValues\":{\"id\":69},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000007.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000007.json new file mode 100644 index 000000000..fd5a758e1 --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000007.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202535263,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":6,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4780"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"04ebb104-1868-465a-b295-c181ed227b51"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":70},\"maxValues\":{\"id\":70},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":72},\"maxValues\":{\"id\":72},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":74},\"maxValues\":{\"id\":74},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-a2b765af-31d9-469b-93ed-511fa3d5ab9c.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":71},\"maxValues\":{\"id\":71},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-341a1715-9b26-46b4-886c-f5a71cd8b706.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":73},\"maxValues\":{\"id\":73},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":76},\"maxValues\":{\"id\":76},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-3b7a4158-7b4c-40d5-afa0-575aa17fe039.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":78},\"maxValues\":{\"id\":78},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":75},\"maxValues\":{\"id\":75},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-a640cc87-30fc-40fc-b737-6574970bc706.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":77},\"maxValues\":{\"id\":77},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":79},\"maxValues\":{\"id\":79},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000008.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000008.json new file mode 100644 index 000000000..e483f13bd --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000008.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202536324,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":7,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4779"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"237bb77f-95a4-4b25-ac89-9f971b538caf"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":80},\"maxValues\":{\"id\":80},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-a981598a-c92f-4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":82},\"maxValues\":{\"id\":82},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-830b2bc8-0ab3-4c2a-8be0-722e03e1c526.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":84},\"maxValues\":{\"id\":84},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":81},\"maxValues\":{\"id\":81},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-db27f739-561b-4e74-9215-6237b5f549f2.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":83},\"maxValues\":{\"id\":83},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":477,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":86},\"maxValues\":{\"id\":86},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-74334a72-5808-4647-ad0b-a30972d7b989.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":88},\"maxValues\":{\"id\":88},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-0d65d9b6-38a0-488e-b0c9-38de2f894b1e.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":85},\"maxValues\":{\"id\":85},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-908e73f4-4be3-4ba4-883b-2f1725f1c028.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":87},\"maxValues\":{\"id\":87},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-57ec9caa-1b45-4058-9e8b-d5b26d20227a.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":89},\"maxValues\":{\"id\":89},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet.crc new file mode 100644 index 000000000..81db3c02b Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet.crc new file mode 100644 index 000000000..e1af56995 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet.crc new file mode 100644 index 000000000..aa26ee472 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-8429619b-3453-4ed9-b8cc-d95960b905bb.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-8429619b-3453-4ed9-b8cc-d95960b905bb.c000.snappy.parquet.crc new file mode 100644 index 000000000..ca17d05ea Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-8429619b-3453-4ed9-b8cc-d95960b905bb.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet.crc new file mode 100644 index 000000000..9b6771d38 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet.crc new file mode 100644 index 000000000..45ffeca94 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-94c8da2e-b2e8-4aed-9579-611eb8dc52e4.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-94c8da2e-b2e8-4aed-9579-611eb8dc52e4.c000.snappy.parquet.crc new file mode 100644 index 000000000..630c12fae Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-94c8da2e-b2e8-4aed-9579-611eb8dc52e4.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet.crc new file mode 100644 index 000000000..e62558163 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet.crc new file mode 100644 index 000000000..a104a6a09 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet new file mode 100644 index 000000000..59572a1de Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet new file mode 100644 index 000000000..3dab41351 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet new file mode 100644 index 000000000..682edf9c0 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-8429619b-3453-4ed9-b8cc-d95960b905bb.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-8429619b-3453-4ed9-b8cc-d95960b905bb.c000.snappy.parquet new file mode 100644 index 000000000..41026bf82 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-8429619b-3453-4ed9-b8cc-d95960b905bb.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet new file mode 100644 index 000000000..678654414 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet new file mode 100644 index 000000000..612aa5856 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-94c8da2e-b2e8-4aed-9579-611eb8dc52e4.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-94c8da2e-b2e8-4aed-9579-611eb8dc52e4.c000.snappy.parquet new file mode 100644 index 000000000..001ff8c13 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-94c8da2e-b2e8-4aed-9579-611eb8dc52e4.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet new file mode 100644 index 000000000..2dd64b3f0 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet new file mode 100644 index 000000000..59e09c647 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet.crc new file mode 100644 index 000000000..7c2d0c30b Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet.crc new file mode 100644 index 000000000..b523c9143 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet.crc new file mode 100644 index 000000000..d714bcb29 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-322e51ba-0754-4929-9eb8-6c714a86f901.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-322e51ba-0754-4929-9eb8-6c714a86f901.c000.snappy.parquet.crc new file mode 100644 index 000000000..f683eb8ef Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-322e51ba-0754-4929-9eb8-6c714a86f901.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet.crc new file mode 100644 index 000000000..2ad0dc751 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet.crc new file mode 100644 index 000000000..d247646b9 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet.crc new file mode 100644 index 000000000..4f5424539 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet.crc new file mode 100644 index 000000000..452aa67c0 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet.crc new file mode 100644 index 000000000..dff80c2de Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet new file mode 100644 index 000000000..9996855df Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet new file mode 100644 index 000000000..a3b363c85 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet new file mode 100644 index 000000000..4bac5b08f Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-322e51ba-0754-4929-9eb8-6c714a86f901.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-322e51ba-0754-4929-9eb8-6c714a86f901.c000.snappy.parquet new file mode 100644 index 000000000..14b78faaa Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-322e51ba-0754-4929-9eb8-6c714a86f901.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet new file mode 100644 index 000000000..9ca2c3522 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet new file mode 100644 index 000000000..db6488a19 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet new file mode 100644 index 000000000..59f02e696 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet new file mode 100644 index 000000000..2f6c77d6d Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet new file mode 100644 index 000000000..49ee6c13c Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet.crc new file mode 100644 index 000000000..003ede4f4 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet.crc new file mode 100644 index 000000000..f7dc46ad2 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-6b5109e8-5533-4165-b61c-3a83d0033ae1.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-6b5109e8-5533-4165-b61c-3a83d0033ae1.c000.snappy.parquet.crc new file mode 100644 index 000000000..6bca36f54 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-6b5109e8-5533-4165-b61c-3a83d0033ae1.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet.crc new file mode 100644 index 000000000..cd843f61b Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet.crc new file mode 100644 index 000000000..16b92c304 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-a981598a-c92f-4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-a981598a-c92f-4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet.crc new file mode 100644 index 000000000..e07e49621 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-a981598a-c92f-4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet.crc new file mode 100644 index 000000000..ece8be75e Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet.crc new file mode 100644 index 000000000..e0ffa2b6b Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet.crc new file mode 100644 index 000000000..1d85b7ed9 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet new file mode 100644 index 000000000..5cee94ad6 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet new file mode 100644 index 000000000..0419b626f Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-6b5109e8-5533-4165-b61c-3a83d0033ae1.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-6b5109e8-5533-4165-b61c-3a83d0033ae1.c000.snappy.parquet new file mode 100644 index 000000000..39c0954ba Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-6b5109e8-5533-4165-b61c-3a83d0033ae1.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet new file mode 100644 index 000000000..bcd62522e Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet new file mode 100644 index 000000000..4f57fab70 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-a981598a-c92f-4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-a981598a-c92f-4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet new file mode 100644 index 000000000..6602df0e3 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-a981598a-c92f-4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet new file mode 100644 index 000000000..758bcd8a6 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet new file mode 100644 index 000000000..7b8bc3ddd Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet new file mode 100644 index 000000000..5f7af33f8 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-28fb675f-e269-4174-ba7d-4e8cb41a65d0.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-28fb675f-e269-4174-ba7d-4e8cb41a65d0.c000.snappy.parquet.crc new file mode 100644 index 000000000..01a77e5e6 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-28fb675f-e269-4174-ba7d-4e8cb41a65d0.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-3b7a4158-7b4c-40d5-afa0-575aa17fe039.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-3b7a4158-7b4c-40d5-afa0-575aa17fe039.c000.snappy.parquet.crc new file mode 100644 index 000000000..d78490c45 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-3b7a4158-7b4c-40d5-afa0-575aa17fe039.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-43e6640a-5f8b-4205-b274-7a324fc3f950.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-43e6640a-5f8b-4205-b274-7a324fc3f950.c000.snappy.parquet.crc new file mode 100644 index 000000000..fa0aaa6f9 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-43e6640a-5f8b-4205-b274-7a324fc3f950.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-57289bbc-8e73-416a-a973-0b824d398b97.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-57289bbc-8e73-416a-a973-0b824d398b97.c000.snappy.parquet.crc new file mode 100644 index 000000000..71919b97b Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-57289bbc-8e73-416a-a973-0b824d398b97.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-71d28824-3adc-406d-ae8a-2bf56033861e.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-71d28824-3adc-406d-ae8a-2bf56033861e.c000.snappy.parquet.crc new file mode 100644 index 000000000..aad44d7dc Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-71d28824-3adc-406d-ae8a-2bf56033861e.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-74334a72-5808-4647-ad0b-a30972d7b989.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-74334a72-5808-4647-ad0b-a30972d7b989.c000.snappy.parquet.crc new file mode 100644 index 000000000..fd896358c Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-74334a72-5808-4647-ad0b-a30972d7b989.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-8b04e895-6377-41f7-89a7-a763d8b9f2be.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-8b04e895-6377-41f7-89a7-a763d8b9f2be.c000.snappy.parquet.crc new file mode 100644 index 000000000..b1c18b33a Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-8b04e895-6377-41f7-89a7-a763d8b9f2be.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-ad757130-9a3e-430a-add7-a0fc2ddaadde.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-ad757130-9a3e-430a-add7-a0fc2ddaadde.c000.snappy.parquet.crc new file mode 100644 index 000000000..21e35e3ff Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-ad757130-9a3e-430a-add7-a0fc2ddaadde.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-b4c08c7f-fd3e-48a7-95dd-829435d27ba6.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-b4c08c7f-fd3e-48a7-95dd-829435d27ba6.c000.snappy.parquet.crc new file mode 100644 index 000000000..8d3202f7d Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-b4c08c7f-fd3e-48a7-95dd-829435d27ba6.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-28fb675f-e269-4174-ba7d-4e8cb41a65d0.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-28fb675f-e269-4174-ba7d-4e8cb41a65d0.c000.snappy.parquet new file mode 100644 index 000000000..14471b915 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-28fb675f-e269-4174-ba7d-4e8cb41a65d0.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-3b7a4158-7b4c-40d5-afa0-575aa17fe039.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-3b7a4158-7b4c-40d5-afa0-575aa17fe039.c000.snappy.parquet new file mode 100644 index 000000000..e794b3294 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-3b7a4158-7b4c-40d5-afa0-575aa17fe039.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-43e6640a-5f8b-4205-b274-7a324fc3f950.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-43e6640a-5f8b-4205-b274-7a324fc3f950.c000.snappy.parquet new file mode 100644 index 000000000..e7b6e141d Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-43e6640a-5f8b-4205-b274-7a324fc3f950.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-57289bbc-8e73-416a-a973-0b824d398b97.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-57289bbc-8e73-416a-a973-0b824d398b97.c000.snappy.parquet new file mode 100644 index 000000000..cc0eb7d45 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-57289bbc-8e73-416a-a973-0b824d398b97.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-71d28824-3adc-406d-ae8a-2bf56033861e.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-71d28824-3adc-406d-ae8a-2bf56033861e.c000.snappy.parquet new file mode 100644 index 000000000..7beaf06ee Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-71d28824-3adc-406d-ae8a-2bf56033861e.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-74334a72-5808-4647-ad0b-a30972d7b989.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-74334a72-5808-4647-ad0b-a30972d7b989.c000.snappy.parquet new file mode 100644 index 000000000..e18ab7b9c Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-74334a72-5808-4647-ad0b-a30972d7b989.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-8b04e895-6377-41f7-89a7-a763d8b9f2be.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-8b04e895-6377-41f7-89a7-a763d8b9f2be.c000.snappy.parquet new file mode 100644 index 000000000..89e161410 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-8b04e895-6377-41f7-89a7-a763d8b9f2be.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-ad757130-9a3e-430a-add7-a0fc2ddaadde.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-ad757130-9a3e-430a-add7-a0fc2ddaadde.c000.snappy.parquet new file mode 100644 index 000000000..2a2a3b685 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-ad757130-9a3e-430a-add7-a0fc2ddaadde.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-b4c08c7f-fd3e-48a7-95dd-829435d27ba6.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-b4c08c7f-fd3e-48a7-95dd-829435d27ba6.c000.snappy.parquet new file mode 100644 index 000000000..c8161cc4f Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-b4c08c7f-fd3e-48a7-95dd-829435d27ba6.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet.crc new file mode 100644 index 000000000..aac8c6275 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet.crc new file mode 100644 index 000000000..dd9a72d7a Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet.crc new file mode 100644 index 000000000..5566aef45 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-76cb0626-0e86-42f6-bf48-809045bfc174.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-76cb0626-0e86-42f6-bf48-809045bfc174.c000.snappy.parquet.crc new file mode 100644 index 000000000..9a7991e8d Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-76cb0626-0e86-42f6-bf48-809045bfc174.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-830b2bc8-0ab3-4c2a-8be0-722e03e1c526.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-830b2bc8-0ab3-4c2a-8be0-722e03e1c526.c000.snappy.parquet.crc new file mode 100644 index 000000000..1c72a44ee Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-830b2bc8-0ab3-4c2a-8be0-722e03e1c526.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-b8aa7c7e-d3b4-410e-8419-4095f4e69b16.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-b8aa7c7e-d3b4-410e-8419-4095f4e69b16.c000.snappy.parquet.crc new file mode 100644 index 000000000..42c5f3020 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-b8aa7c7e-d3b4-410e-8419-4095f4e69b16.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet.crc new file mode 100644 index 000000000..cdd380cad Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet.crc new file mode 100644 index 000000000..f34c94e74 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet.crc new file mode 100644 index 000000000..af559374b Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet new file mode 100644 index 000000000..a552c4995 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet new file mode 100644 index 000000000..6e1e96ea0 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet new file mode 100644 index 000000000..fba348aa4 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-76cb0626-0e86-42f6-bf48-809045bfc174.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-76cb0626-0e86-42f6-bf48-809045bfc174.c000.snappy.parquet new file mode 100644 index 000000000..9e3ebfc9a Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-76cb0626-0e86-42f6-bf48-809045bfc174.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-830b2bc8-0ab3-4c2a-8be0-722e03e1c526.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-830b2bc8-0ab3-4c2a-8be0-722e03e1c526.c000.snappy.parquet new file mode 100644 index 000000000..929fa8d70 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-830b2bc8-0ab3-4c2a-8be0-722e03e1c526.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-b8aa7c7e-d3b4-410e-8419-4095f4e69b16.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-b8aa7c7e-d3b4-410e-8419-4095f4e69b16.c000.snappy.parquet new file mode 100644 index 000000000..fd5dd7be8 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-b8aa7c7e-d3b4-410e-8419-4095f4e69b16.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet new file mode 100644 index 000000000..383f4992b Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet new file mode 100644 index 000000000..54f2c0bab Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet new file mode 100644 index 000000000..f71bcb77f Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet.crc new file mode 100644 index 000000000..92e97cef2 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-0d65d9b6-38a0-488e-b0c9-38de2f894b1e.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-0d65d9b6-38a0-488e-b0c9-38de2f894b1e.c000.snappy.parquet.crc new file mode 100644 index 000000000..993230c23 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-0d65d9b6-38a0-488e-b0c9-38de2f894b1e.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet.crc new file mode 100644 index 000000000..1287221ea Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-5a0c01cf-07af-47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-5a0c01cf-07af-47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet.crc new file mode 100644 index 000000000..36fa13a54 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-5a0c01cf-07af-47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet.crc new file mode 100644 index 000000000..c5bc67f65 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet.crc new file mode 100644 index 000000000..750654f01 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet.crc new file mode 100644 index 000000000..d316f76fd Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet.crc new file mode 100644 index 000000000..501fd4ead Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet.crc new file mode 100644 index 000000000..a1951c580 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet new file mode 100644 index 000000000..9b0ea7ea6 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-0d65d9b6-38a0-488e-b0c9-38de2f894b1e.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-0d65d9b6-38a0-488e-b0c9-38de2f894b1e.c000.snappy.parquet new file mode 100644 index 000000000..a4219cb8c Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-0d65d9b6-38a0-488e-b0c9-38de2f894b1e.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet new file mode 100644 index 000000000..e4412d13f Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-5a0c01cf-07af-47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-5a0c01cf-07af-47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet new file mode 100644 index 000000000..723700c99 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-5a0c01cf-07af-47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet new file mode 100644 index 000000000..cf26d1f52 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet new file mode 100644 index 000000000..3950f6024 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet new file mode 100644 index 000000000..7f05f7f1a Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet new file mode 100644 index 000000000..57e0c1087 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet new file mode 100644 index 000000000..6bc54dd22 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet.crc new file mode 100644 index 000000000..be0b429a0 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-815f63f2-21c2-4e16-b8c2-68e7d6414be8.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-815f63f2-21c2-4e16-b8c2-68e7d6414be8.c000.snappy.parquet.crc new file mode 100644 index 000000000..a4683a501 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-815f63f2-21c2-4e16-b8c2-68e7d6414be8.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet.crc new file mode 100644 index 000000000..1bc4211ea Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-a05e7f3b-1548-47d1-95d2-b802d9f602c0.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-a05e7f3b-1548-47d1-95d2-b802d9f602c0.c000.snappy.parquet.crc new file mode 100644 index 000000000..2b1eaadb4 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-a05e7f3b-1548-47d1-95d2-b802d9f602c0.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-a2b765af-31d9-469b-93ed-511fa3d5ab9c.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-a2b765af-31d9-469b-93ed-511fa3d5ab9c.c000.snappy.parquet.crc new file mode 100644 index 000000000..fedff1688 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-a2b765af-31d9-469b-93ed-511fa3d5ab9c.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet.crc new file mode 100644 index 000000000..938d643c5 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-e7557908-e149-4724-9c15-72b4d5df3c78.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-e7557908-e149-4724-9c15-72b4d5df3c78.c000.snappy.parquet.crc new file mode 100644 index 000000000..b3acd7567 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-e7557908-e149-4724-9c15-72b4d5df3c78.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet.crc new file mode 100644 index 000000000..f300d0c69 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-f86461d3-6066-4b87-ab45-48c06bddee6c.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-f86461d3-6066-4b87-ab45-48c06bddee6c.c000.snappy.parquet.crc new file mode 100644 index 000000000..1111f628e Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-f86461d3-6066-4b87-ab45-48c06bddee6c.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet new file mode 100644 index 000000000..ea9e80477 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-815f63f2-21c2-4e16-b8c2-68e7d6414be8.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-815f63f2-21c2-4e16-b8c2-68e7d6414be8.c000.snappy.parquet new file mode 100644 index 000000000..f912221f3 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-815f63f2-21c2-4e16-b8c2-68e7d6414be8.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet new file mode 100644 index 000000000..7f971e798 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-a05e7f3b-1548-47d1-95d2-b802d9f602c0.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-a05e7f3b-1548-47d1-95d2-b802d9f602c0.c000.snappy.parquet new file mode 100644 index 000000000..892729262 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-a05e7f3b-1548-47d1-95d2-b802d9f602c0.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-a2b765af-31d9-469b-93ed-511fa3d5ab9c.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-a2b765af-31d9-469b-93ed-511fa3d5ab9c.c000.snappy.parquet new file mode 100644 index 000000000..79a37781c Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-a2b765af-31d9-469b-93ed-511fa3d5ab9c.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet new file mode 100644 index 000000000..4ce8f45b4 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-e7557908-e149-4724-9c15-72b4d5df3c78.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-e7557908-e149-4724-9c15-72b4d5df3c78.c000.snappy.parquet new file mode 100644 index 000000000..292d12419 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-e7557908-e149-4724-9c15-72b4d5df3c78.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet new file mode 100644 index 000000000..504f53e83 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-f86461d3-6066-4b87-ab45-48c06bddee6c.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-f86461d3-6066-4b87-ab45-48c06bddee6c.c000.snappy.parquet new file mode 100644 index 000000000..2c2664877 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-f86461d3-6066-4b87-ab45-48c06bddee6c.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-07da6a7d-231d-4c66-9c96-752defe5436b.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-07da6a7d-231d-4c66-9c96-752defe5436b.c000.snappy.parquet.crc new file mode 100644 index 000000000..904d19ee2 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-07da6a7d-231d-4c66-9c96-752defe5436b.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-35842ca7-408d-4023-bf03-1195979e33a8.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-35842ca7-408d-4023-bf03-1195979e33a8.c000.snappy.parquet.crc new file mode 100644 index 000000000..fdb41b63f Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-35842ca7-408d-4023-bf03-1195979e33a8.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-5e20ed9a-4496-4a62-9dff-bb8de1862fc0.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-5e20ed9a-4496-4a62-9dff-bb8de1862fc0.c000.snappy.parquet.crc new file mode 100644 index 000000000..d39e1a579 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-5e20ed9a-4496-4a62-9dff-bb8de1862fc0.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-60f06dc7-46d8-4ec9-9735-81db2fd05163.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-60f06dc7-46d8-4ec9-9735-81db2fd05163.c000.snappy.parquet.crc new file mode 100644 index 000000000..7d2832554 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-60f06dc7-46d8-4ec9-9735-81db2fd05163.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-89424aa4-1fc4-4aa9-819c-3c164d7e926e.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-89424aa4-1fc4-4aa9-819c-3c164d7e926e.c000.snappy.parquet.crc new file mode 100644 index 000000000..2086d3fd6 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-89424aa4-1fc4-4aa9-819c-3c164d7e926e.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-908e73f4-4be3-4ba4-883b-2f1725f1c028.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-908e73f4-4be3-4ba4-883b-2f1725f1c028.c000.snappy.parquet.crc new file mode 100644 index 000000000..7e574cac1 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-908e73f4-4be3-4ba4-883b-2f1725f1c028.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-a5f8c5df-41b0-48b5-b3ac-499d27a94c54.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-a5f8c5df-41b0-48b5-b3ac-499d27a94c54.c000.snappy.parquet.crc new file mode 100644 index 000000000..7607d06fc Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-a5f8c5df-41b0-48b5-b3ac-499d27a94c54.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-a640cc87-30fc-40fc-b737-6574970bc706.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-a640cc87-30fc-40fc-b737-6574970bc706.c000.snappy.parquet.crc new file mode 100644 index 000000000..1f7586b9e Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-a640cc87-30fc-40fc-b737-6574970bc706.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-d4895419-3e8e-4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-d4895419-3e8e-4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet.crc new file mode 100644 index 000000000..bc872a035 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-d4895419-3e8e-4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-07da6a7d-231d-4c66-9c96-752defe5436b.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-07da6a7d-231d-4c66-9c96-752defe5436b.c000.snappy.parquet new file mode 100644 index 000000000..3011e3f56 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-07da6a7d-231d-4c66-9c96-752defe5436b.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-35842ca7-408d-4023-bf03-1195979e33a8.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-35842ca7-408d-4023-bf03-1195979e33a8.c000.snappy.parquet new file mode 100644 index 000000000..7ce90b22d Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-35842ca7-408d-4023-bf03-1195979e33a8.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-5e20ed9a-4496-4a62-9dff-bb8de1862fc0.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-5e20ed9a-4496-4a62-9dff-bb8de1862fc0.c000.snappy.parquet new file mode 100644 index 000000000..5ae651dd9 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-5e20ed9a-4496-4a62-9dff-bb8de1862fc0.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-60f06dc7-46d8-4ec9-9735-81db2fd05163.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-60f06dc7-46d8-4ec9-9735-81db2fd05163.c000.snappy.parquet new file mode 100644 index 000000000..0a8754606 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-60f06dc7-46d8-4ec9-9735-81db2fd05163.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-89424aa4-1fc4-4aa9-819c-3c164d7e926e.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-89424aa4-1fc4-4aa9-819c-3c164d7e926e.c000.snappy.parquet new file mode 100644 index 000000000..825b51108 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-89424aa4-1fc4-4aa9-819c-3c164d7e926e.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-908e73f4-4be3-4ba4-883b-2f1725f1c028.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-908e73f4-4be3-4ba4-883b-2f1725f1c028.c000.snappy.parquet new file mode 100644 index 000000000..6e14edc57 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-908e73f4-4be3-4ba4-883b-2f1725f1c028.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-a5f8c5df-41b0-48b5-b3ac-499d27a94c54.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-a5f8c5df-41b0-48b5-b3ac-499d27a94c54.c000.snappy.parquet new file mode 100644 index 000000000..fe5d23e3a Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-a5f8c5df-41b0-48b5-b3ac-499d27a94c54.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-a640cc87-30fc-40fc-b737-6574970bc706.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-a640cc87-30fc-40fc-b737-6574970bc706.c000.snappy.parquet new file mode 100644 index 000000000..c66bfe473 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-a640cc87-30fc-40fc-b737-6574970bc706.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-d4895419-3e8e-4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-d4895419-3e8e-4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet new file mode 100644 index 000000000..b8f681650 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-d4895419-3e8e-4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-0e36beae-e5e5-4c33-8a3e-b68a446f2e2c.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-0e36beae-e5e5-4c33-8a3e-b68a446f2e2c.c000.snappy.parquet.crc new file mode 100644 index 000000000..b754f5fe7 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-0e36beae-e5e5-4c33-8a3e-b68a446f2e2c.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-341a1715-9b26-46b4-886c-f5a71cd8b706.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-341a1715-9b26-46b4-886c-f5a71cd8b706.c000.snappy.parquet.crc new file mode 100644 index 000000000..06219e2cd Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-341a1715-9b26-46b4-886c-f5a71cd8b706.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-39d574e1-c089-4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-39d574e1-c089-4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet.crc new file mode 100644 index 000000000..22e7ddf68 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-39d574e1-c089-4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-4ee45e83-7063-4c4b-a008-c7068640b8a5.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-4ee45e83-7063-4c4b-a008-c7068640b8a5.c000.snappy.parquet.crc new file mode 100644 index 000000000..a7a2ca55f Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-4ee45e83-7063-4c4b-a008-c7068640b8a5.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet.crc new file mode 100644 index 000000000..cfd927cf4 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet.crc new file mode 100644 index 000000000..4291c0678 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-cc14211d-2fd6-4fc9-baae-b62451ecf8a1.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-cc14211d-2fd6-4fc9-baae-b62451ecf8a1.c000.snappy.parquet.crc new file mode 100644 index 000000000..1070a8f2b Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-cc14211d-2fd6-4fc9-baae-b62451ecf8a1.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-db27f739-561b-4e74-9215-6237b5f549f2.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-db27f739-561b-4e74-9215-6237b5f549f2.c000.snappy.parquet.crc new file mode 100644 index 000000000..f333eeb66 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-db27f739-561b-4e74-9215-6237b5f549f2.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-e6b34e43-1eda-454b-be03-5442a09982de.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-e6b34e43-1eda-454b-be03-5442a09982de.c000.snappy.parquet.crc new file mode 100644 index 000000000..c465494c5 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-e6b34e43-1eda-454b-be03-5442a09982de.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-0e36beae-e5e5-4c33-8a3e-b68a446f2e2c.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-0e36beae-e5e5-4c33-8a3e-b68a446f2e2c.c000.snappy.parquet new file mode 100644 index 000000000..acdfc7577 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-0e36beae-e5e5-4c33-8a3e-b68a446f2e2c.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-341a1715-9b26-46b4-886c-f5a71cd8b706.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-341a1715-9b26-46b4-886c-f5a71cd8b706.c000.snappy.parquet new file mode 100644 index 000000000..2f5d96564 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-341a1715-9b26-46b4-886c-f5a71cd8b706.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-39d574e1-c089-4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-39d574e1-c089-4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet new file mode 100644 index 000000000..b23ff4883 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-39d574e1-c089-4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-4ee45e83-7063-4c4b-a008-c7068640b8a5.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-4ee45e83-7063-4c4b-a008-c7068640b8a5.c000.snappy.parquet new file mode 100644 index 000000000..08e15d5e5 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-4ee45e83-7063-4c4b-a008-c7068640b8a5.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet new file mode 100644 index 000000000..25ce726d0 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet new file mode 100644 index 000000000..b6b6ee594 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-cc14211d-2fd6-4fc9-baae-b62451ecf8a1.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-cc14211d-2fd6-4fc9-baae-b62451ecf8a1.c000.snappy.parquet new file mode 100644 index 000000000..a121748c1 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-cc14211d-2fd6-4fc9-baae-b62451ecf8a1.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-db27f739-561b-4e74-9215-6237b5f549f2.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-db27f739-561b-4e74-9215-6237b5f549f2.c000.snappy.parquet new file mode 100644 index 000000000..a12d928cc Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-db27f739-561b-4e74-9215-6237b5f549f2.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-e6b34e43-1eda-454b-be03-5442a09982de.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-e6b34e43-1eda-454b-be03-5442a09982de.c000.snappy.parquet new file mode 100644 index 000000000..09099cc1b Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-e6b34e43-1eda-454b-be03-5442a09982de.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet.crc new file mode 100644 index 000000000..9ec5e06d9 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet.crc new file mode 100644 index 000000000..9cc1780aa Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-57ec9caa-1b45-4058-9e8b-d5b26d20227a.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-57ec9caa-1b45-4058-9e8b-d5b26d20227a.c000.snappy.parquet.crc new file mode 100644 index 000000000..77029aff4 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-57ec9caa-1b45-4058-9e8b-d5b26d20227a.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet.crc new file mode 100644 index 000000000..2b5666ec3 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet.crc new file mode 100644 index 000000000..9d4bb1cf9 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet.crc new file mode 100644 index 000000000..dfe9d9afa Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet.crc new file mode 100644 index 000000000..85806a487 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet.crc new file mode 100644 index 000000000..a0e7303a7 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-f847a367-cc91-4d50-85bb-83aeaa5c63a1.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-f847a367-cc91-4d50-85bb-83aeaa5c63a1.c000.snappy.parquet.crc new file mode 100644 index 000000000..e893eb8b2 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-f847a367-cc91-4d50-85bb-83aeaa5c63a1.c000.snappy.parquet.crc differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet new file mode 100644 index 000000000..564c878dd Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet new file mode 100644 index 000000000..5f7c0e0a2 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-57ec9caa-1b45-4058-9e8b-d5b26d20227a.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-57ec9caa-1b45-4058-9e8b-d5b26d20227a.c000.snappy.parquet new file mode 100644 index 000000000..0e431c905 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-57ec9caa-1b45-4058-9e8b-d5b26d20227a.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet new file mode 100644 index 000000000..62247906e Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet new file mode 100644 index 000000000..7e7e745cc Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet new file mode 100644 index 000000000..f4077d888 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet new file mode 100644 index 000000000..295411366 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet new file mode 100644 index 000000000..c7fafbc3d Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet differ diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-f847a367-cc91-4d50-85bb-83aeaa5c63a1.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-f847a367-cc91-4d50-85bb-83aeaa5c63a1.c000.snappy.parquet new file mode 100644 index 000000000..ddb066dd7 Binary files /dev/null and b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-f847a367-cc91-4d50-85bb-83aeaa5c63a1.c000.snappy.parquet differ diff --git a/kernel-default/src/test/scala/io/delta/core/TableSuite.scala b/kernel-default/src/test/scala/io/delta/core/internal/TableSuite.scala similarity index 69% rename from kernel-default/src/test/scala/io/delta/core/TableSuite.scala rename to kernel-default/src/test/scala/io/delta/core/internal/TableSuite.scala index ecaa275a4..e2f9f7431 100644 --- a/kernel-default/src/test/scala/io/delta/core/TableSuite.scala +++ b/kernel-default/src/test/scala/io/delta/core/internal/TableSuite.scala @@ -14,12 +14,16 @@ * limitations under the License. */ -package io.delta.core +package io.delta.core.internal +import scala.collection.JavaConverters._ + +import io.delta.core.Table import io.delta.core.helpers.DefaultTableHelper import io.delta.core.util.GoldenTableUtils import org.scalatest.funsuite.AnyFunSuite +// scalastyle:off println class TableSuite extends AnyFunSuite with GoldenTableUtils { test("can load latest table version - with a checkpoint") { @@ -54,4 +58,21 @@ class TableSuite extends AnyFunSuite with GoldenTableUtils { iter.close() } } + + test("can load table protocol & schema - table without a checkpoint") { + withGoldenTable("basic-no-checkpoint") { path => + val table = Table.forPath(path, new DefaultTableHelper()) + val snapshot = table.getLatestSnapshot.asInstanceOf[SnapshotImpl] + println(snapshot.getSchema) + println(snapshot.getProtocol) + } + } + + test("can parse add file partition values - basic - no checkpoint") { + withGoldenTable("basic-partitioned-no-checkpoint") { path => + val table = Table.forPath(path, new DefaultTableHelper()) + val snapshot = table.getLatestSnapshot.asInstanceOf[SnapshotImpl] + snapshot.getAddFiles.forEachRemaining(x => println(x)) + } + } } diff --git a/kernel-default/src/test/scala/io/delta/core/util/GoldenTablesGenerator.scala b/kernel-default/src/test/scala/io/delta/core/util/GoldenTablesGenerator.scala index d1f4040b3..cb5726dca 100644 --- a/kernel-default/src/test/scala/io/delta/core/util/GoldenTablesGenerator.scala +++ b/kernel-default/src/test/scala/io/delta/core/util/GoldenTablesGenerator.scala @@ -21,10 +21,13 @@ import java.io.File import org.apache.spark.SparkConf import org.apache.spark.network.util.JavaUtils import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession /** * GENERATE_GOLDEN_TABLES=1 build/sbt 'kernelDefault/testOnly *GoldenTablesGenerator' + * or + * GENERATE_GOLDEN_TABLES=1 build/sbt 'kernelDefault/testOnly *GoldenTablesGenerator -- -z ""' */ class GoldenTablesGenerator extends QueryTest with SharedSparkSession { @@ -66,4 +69,18 @@ class GoldenTablesGenerator extends QueryTest with SharedSparkSession { spark.range(low, high).write.format("delta").mode("append").save(path) } } + + generate("basic-partitioned-no-checkpoint") { path => + for (i <- 0 to 8) { + val low = i * 10 + val high = (i + 1) * 10 + spark.range(low, high) + .withColumn("part_a", col("id") % 2) + .withColumn("part_b", col("id") % 5) + .write + .format("delta") + .partitionBy("part_a", "part_b") + .mode("append").save(path) + } + } } diff --git a/kernel/src/main/java/io/delta/core/data/Row.java b/kernel/src/main/java/io/delta/core/data/Row.java index df460d008..351759ec7 100644 --- a/kernel/src/main/java/io/delta/core/data/Row.java +++ b/kernel/src/main/java/io/delta/core/data/Row.java @@ -8,15 +8,15 @@ public interface Row { -// boolean isNullAt(int ordinal); + boolean isNullAt(int ordinal); // -// boolean getBoolean(int ordinal); + boolean getBoolean(int ordinal); // // byte getByte(int ordinal); // // short getShort(int ordinal); // -// int getInt(int ordinal); + int getInt(int ordinal); long getLong(int ordinal); @@ -36,7 +36,7 @@ public interface Row { // Row getRecord(int ordinal); // -// List getList(int ordinal); + List getList(int ordinal); // -// Map getMap(int ordinal); + Map getMap(int ordinal); } diff --git a/kernel/src/main/java/io/delta/core/helpers/TableHelper.java b/kernel/src/main/java/io/delta/core/helpers/TableHelper.java index 767f3a05b..00bf6d66e 100644 --- a/kernel/src/main/java/io/delta/core/helpers/TableHelper.java +++ b/kernel/src/main/java/io/delta/core/helpers/TableHelper.java @@ -24,7 +24,7 @@ CloseableIterator readParquetFile( StructType readSchema, Expression skippingFilter); - Row parseStats(String statsJson); + Row parseJson(String json, StructType schema); ScanHelper getScanHelper(); } diff --git a/kernel/src/main/java/io/delta/core/internal/SnapshotImpl.java b/kernel/src/main/java/io/delta/core/internal/SnapshotImpl.java index 2014bb029..513eb559c 100644 --- a/kernel/src/main/java/io/delta/core/internal/SnapshotImpl.java +++ b/kernel/src/main/java/io/delta/core/internal/SnapshotImpl.java @@ -3,6 +3,7 @@ import io.delta.core.ScanBuilder; import io.delta.core.Snapshot; import io.delta.core.fs.Path; +import io.delta.core.internal.actions.AddFile; import io.delta.core.internal.actions.Metadata; import io.delta.core.internal.actions.Protocol; import io.delta.core.internal.lang.Lazy; @@ -10,6 +11,7 @@ import io.delta.core.internal.replay.LogReplay; import io.delta.core.internal.snapshot.LogSegment; import io.delta.core.types.StructType; +import io.delta.core.utils.CloseableIterator; public class SnapshotImpl implements Snapshot { private final Path logPath; @@ -37,6 +39,10 @@ public SnapshotImpl( this.protocolAndMetadata = logReplay.lazyLoadProtocolAndMetadata(); } + //////////////////////////////////////// + // Public APIs + //////////////////////////////////////// + @Override public long getVersion() { return version; @@ -51,4 +57,16 @@ public StructType getSchema() { public ScanBuilder getScanBuilder() { return new ScanBuilderImpl(logReplay); } + + //////////////////////////////////////// + // Internal APIs + //////////////////////////////////////// + + public CloseableIterator getAddFiles() { + return logReplay.getAddFiles(); + } + + public Protocol getProtocol() { + return protocolAndMetadata.get()._1; + } } diff --git a/kernel/src/main/java/io/delta/core/internal/actions/AddFile.java b/kernel/src/main/java/io/delta/core/internal/actions/AddFile.java index d82968806..d6fe86619 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/AddFile.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/AddFile.java @@ -1,24 +1,55 @@ package io.delta.core.internal.actions; +import java.util.Map; import java.util.Optional; import io.delta.core.data.Row; -import io.delta.core.types.StringType; -import io.delta.core.types.StructType; +import io.delta.core.types.*; public class AddFile extends FileAction { + //////////////////////////////////////////////////////////////////////////////// + // Static Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + public static AddFile fromRow(Row row) { if (row == null) return null; - return new AddFile(row.getString(0), true); + final String path = row.getString(0); + final Map partitionValues = row.getMap(1); + final long size = row.getLong(2); + final long modificationTime = row.getLong(3); + final boolean dataChange = row.getBoolean(4); + + return new AddFile(path, partitionValues, size, modificationTime, dataChange); } public static final StructType READ_SCHEMA = new StructType() - .add("path", new StringType()); + .add("path", StringType.INSTANCE) + .add("partitionValues", new MapType(StringType.INSTANCE, StringType.INSTANCE, false)) + .add("size", LongType.INSTANCE) + .add("modificationTime", LongType.INSTANCE) + .add("dataChange", BooleanType.INSTANCE); + + //////////////////////////////////////////////////////////////////////////////// + // Instance Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + + private final Map partitionValues; + private final long size; + private final long modificationTime; - public AddFile(String path, boolean dataChange) { + public AddFile( + String path, + Map partitionValues, + long size, + long modificationTime, + boolean dataChange) { super(path, dataChange); + + this.partitionValues = partitionValues; + this.size = size; + this.modificationTime = modificationTime; } public Optional getDeletionVectorUniqueId() { @@ -29,4 +60,15 @@ public Optional getDeletionVectorUniqueId() { public AddFile copyWithDataChange(boolean dataChange) { return this; // TODO } + + @Override + public String toString() { + return "AddFile{" + + "path='" + path + '\'' + + ", partitionValues=" + partitionValues + + ", size=" + size + + ", modificationTime=" + modificationTime + + ", dataChange=" + dataChange + + '}'; + } } diff --git a/kernel/src/main/java/io/delta/core/internal/actions/FileAction.java b/kernel/src/main/java/io/delta/core/internal/actions/FileAction.java index fc8e8163b..7e9ddb5a5 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/FileAction.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/FileAction.java @@ -7,8 +7,8 @@ import io.delta.core.internal.lang.Lazy; public abstract class FileAction implements Action { - private final String path; - private final boolean dataChange; + protected final String path; + protected final boolean dataChange; private final Lazy pathAsUri; private final Lazy pathAsPath; diff --git a/kernel/src/main/java/io/delta/core/internal/actions/Format.java b/kernel/src/main/java/io/delta/core/internal/actions/Format.java new file mode 100644 index 000000000..bdac86b80 --- /dev/null +++ b/kernel/src/main/java/io/delta/core/internal/actions/Format.java @@ -0,0 +1,25 @@ +package io.delta.core.internal.actions; + +import io.delta.core.data.Row; +import io.delta.core.types.StringType; +import io.delta.core.types.StructType; + +public class Format { + public static Format fromRow(Row row) { + if (row == null) return null; + + final String provider = row.getString(0); + + return new Format(provider); + } + + public static final StructType READ_SCHEMA = new StructType() + .add("provider", StringType.INSTANCE); + // TODO: options + + private final String provider; + + public Format(String provider) { + this.provider = provider; + } +} diff --git a/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java b/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java index c5661dbe9..06c46b78b 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java @@ -1,19 +1,58 @@ package io.delta.core.internal.actions; import io.delta.core.data.Row; -import io.delta.core.types.StructType; +import io.delta.core.helpers.TableHelper; +import io.delta.core.types.*; public class Metadata implements Action { - public static Metadata fromRow(Row row) { + //////////////////////////////////////////////////////////////////////////////// + // Static Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + + public static Metadata fromRow(Row row, TableHelper tableHelper) { if (row == null) return null; + final String id = row.getString(0); + final String name = row.getString(1); + final String description = row.getString(2); + final Format format = Format.fromRow(row.getRecord(3)); + final String schemaJson = row.getString(4); + Row schemaRow = tableHelper.parseJson(schemaJson, StructType.READ_SCHEMA); + StructType schema = StructType.fromRow(schemaRow); - return new Metadata(); + return new Metadata(schema); } - public static final StructType READ_SCHEMA = new StructType(); + public static final StructType READ_SCHEMA = new StructType() + .add("id", StringType.INSTANCE) + .add("name", StringType.INSTANCE) + .add("description", StringType.INSTANCE) + .add("format", Format.READ_SCHEMA) + .add("schemaString", StringType.INSTANCE) + .add("partitionColumns", new ArrayType(StringType.INSTANCE, false /* contains null */)) + .add("configuration", new MapType(StringType.INSTANCE, StringType.INSTANCE, false)) + .add("createdTime", LongType.INSTANCE); + + //////////////////////////////////////////////////////////////////////////////// + // Instance Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + + // id + // name + // description + // format + // schemaString + // partitionColumns + // configuration + // createdTime + + private final StructType schema; + + public Metadata(StructType schema) { + this.schema = schema; + } public StructType getSchema() { - return null; + return schema; } } diff --git a/kernel/src/main/java/io/delta/core/internal/actions/Protocol.java b/kernel/src/main/java/io/delta/core/internal/actions/Protocol.java index f718e90ea..fba102fba 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/Protocol.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/Protocol.java @@ -1,15 +1,32 @@ package io.delta.core.internal.actions; import io.delta.core.data.Row; +import io.delta.core.types.IntegerType; import io.delta.core.types.StructType; public class Protocol implements Action { public static Protocol fromRow(Row row) { if (row == null) return null; + final int minReaderVersion = row.getInt(0); + final int minWriterVersion = row.getInt(1); + return new Protocol(minReaderVersion, minWriterVersion); + } + + public static final StructType READ_SCHEMA = new StructType() + .add("minReaderVersion", IntegerType.INSTANCE) + .add("minWriterVersion", IntegerType.INSTANCE); - return new Protocol(); + private final int minReaderVersion; + private final int minWriterVersion; + + public Protocol(int minReaderVersion, int minWriterVersion) { + this.minReaderVersion = minReaderVersion; + this.minWriterVersion = minWriterVersion; } - public static final StructType READ_SCHEMA = new StructType(); + @Override + public String toString() { + return String.format("Protocol(%s,%s)", minReaderVersion, minWriterVersion); + } } diff --git a/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java b/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java index de8676232..fdfe09ad2 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java @@ -1,11 +1,16 @@ package io.delta.core.internal.actions; import io.delta.core.data.Row; +import io.delta.core.helpers.TableHelper; import io.delta.core.types.StructType; public class SingleAction { - public static SingleAction fromRow(Row row) { + //////////////////////////////////////////////////////////////////////////////// + // Static Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + + public static SingleAction fromRow(Row row, TableHelper tableHelper) { final SetTransaction txn = SetTransaction.fromRow(row.getRecord(0)); if (txn != null) { return new SingleAction(txn, null, null, null, null, null, null); @@ -21,7 +26,7 @@ public static SingleAction fromRow(Row row) { return new SingleAction(null, null, remove, null, null, null, null); } - final Metadata metadata = Metadata.fromRow(row.getRecord(3)); + final Metadata metadata = Metadata.fromRow(row.getRecord(3), tableHelper); if (metadata != null) { return new SingleAction(null, null, null, metadata, null, null, null); } @@ -53,6 +58,10 @@ public static SingleAction fromRow(Row row) { .add("cdc", AddCDCFile.READ_SCHEMA) .add("commitInfo", CommitInfo.READ_SCHEMA); + //////////////////////////////////////////////////////////////////////////////// + // Instance Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + private final SetTransaction txn; private final AddFile add; private final RemoveFile remove; @@ -86,6 +95,7 @@ public Action unwrap() { if (protocol != null) return protocol; if (cdc != null) return cdc; if (commitInfo != null) return commitInfo; - return null; + + throw new IllegalStateException("SingleAction row contained no non-null actions"); } } diff --git a/kernel/src/main/java/io/delta/core/internal/checkpoint/CheckpointMetaData.java b/kernel/src/main/java/io/delta/core/internal/checkpoint/CheckpointMetaData.java index 4d1cb6491..fc12c6d0d 100644 --- a/kernel/src/main/java/io/delta/core/internal/checkpoint/CheckpointMetaData.java +++ b/kernel/src/main/java/io/delta/core/internal/checkpoint/CheckpointMetaData.java @@ -14,8 +14,8 @@ public static CheckpointMetaData fromRow(Row row) { } public static StructType READ_SCHEMA = new StructType() - .add("version", new LongType()) - .add("size", new LongType()); + .add("version", LongType.INSTANCE) + .add("size", LongType.INSTANCE); public final long version; public final long size; diff --git a/kernel/src/main/java/io/delta/core/internal/replay/ReverseFilesToActionsIterable.java b/kernel/src/main/java/io/delta/core/internal/replay/ReverseFilesToActionsIterable.java index d23988366..780fc3250 100644 --- a/kernel/src/main/java/io/delta/core/internal/replay/ReverseFilesToActionsIterable.java +++ b/kernel/src/main/java/io/delta/core/internal/replay/ReverseFilesToActionsIterable.java @@ -108,13 +108,11 @@ private CloseableIterator> getNextActionsIter() { try { if (nextPath.getName().endsWith(".json")) { - System.out.println("Scott > ReverseFilesToActionsIterable > reading " + nextPath.getName()); return new RowToActionIterator( tableHelper.readJsonFile(nextPath.toString(), SingleAction.READ_SCHEMA), false // isFromCheckpoint ); } else if (nextPath.getName().endsWith(".parquet")) { - System.out.println("Scott > ReverseFilesToActionsIterable > reading " + nextPath.getName()); return new RowToActionIterator( tableHelper.readParquetFile(nextPath.toString(), SingleAction.READ_SCHEMA), true // isFromCheckpoint @@ -131,7 +129,7 @@ private CloseableIterator> getNextActionsIter() { }; } - private static class RowToActionIterator implements CloseableIterator> { + private class RowToActionIterator implements CloseableIterator> { private final CloseableIterator impl; private final boolean isFromCheckpoint; @@ -148,7 +146,10 @@ public boolean hasNext() { @Override public Tuple2 next() { - return new Tuple2<>(SingleAction.fromRow(impl.next()).unwrap(), isFromCheckpoint); + return new Tuple2<>( + SingleAction.fromRow(impl.next(), tableHelper).unwrap(), + isFromCheckpoint + ); } @Override diff --git a/kernel/src/main/java/io/delta/core/types/ArrayType.java b/kernel/src/main/java/io/delta/core/types/ArrayType.java new file mode 100644 index 000000000..10abc3dd4 --- /dev/null +++ b/kernel/src/main/java/io/delta/core/types/ArrayType.java @@ -0,0 +1,21 @@ +package io.delta.core.types; + +public class ArrayType extends DataType { + public static ArrayType EMPTY_INSTANCE = new ArrayType(null, false); + + private final DataType elementType; + private final boolean containsNull; + + public ArrayType(DataType elementType, boolean containsNull) { + this.elementType = elementType; + this.containsNull = containsNull; + } + + public DataType getElementType() { + return elementType; + } + + public boolean containsNull() { + return containsNull; + } +} diff --git a/kernel/src/main/java/io/delta/core/types/BooleanType.java b/kernel/src/main/java/io/delta/core/types/BooleanType.java new file mode 100644 index 000000000..5cdb70431 --- /dev/null +++ b/kernel/src/main/java/io/delta/core/types/BooleanType.java @@ -0,0 +1,7 @@ +package io.delta.core.types; + +public class BooleanType extends DataType { + public static final BooleanType INSTANCE = new BooleanType(); + + private BooleanType() { } +} diff --git a/kernel/src/main/java/io/delta/core/types/DataType.java b/kernel/src/main/java/io/delta/core/types/DataType.java index 368093d6f..920eb92b3 100644 --- a/kernel/src/main/java/io/delta/core/types/DataType.java +++ b/kernel/src/main/java/io/delta/core/types/DataType.java @@ -3,6 +3,16 @@ import java.util.Locale; public abstract class DataType { + + public static DataType createPrimitive(String typeName) { + if (typeName.equals(LongType.INSTANCE.typeName())) return LongType.INSTANCE; + if (typeName.equals(StringType.INSTANCE.typeName())) return StringType.INSTANCE; + + throw new IllegalArgumentException( + String.format("Can't create primitive for type type %s", typeName) + ); + } + public String typeName() { String name = this.getClass().getSimpleName(); if (name.endsWith("Type")) { diff --git a/kernel/src/main/java/io/delta/core/types/IntegerType.java b/kernel/src/main/java/io/delta/core/types/IntegerType.java new file mode 100644 index 000000000..a42af3280 --- /dev/null +++ b/kernel/src/main/java/io/delta/core/types/IntegerType.java @@ -0,0 +1,7 @@ +package io.delta.core.types; + +public class IntegerType extends DataType { + public static final IntegerType INSTANCE = new IntegerType(); + + private IntegerType() { } +} diff --git a/kernel/src/main/java/io/delta/core/types/LongType.java b/kernel/src/main/java/io/delta/core/types/LongType.java index 2e222bd48..c8655e7f2 100644 --- a/kernel/src/main/java/io/delta/core/types/LongType.java +++ b/kernel/src/main/java/io/delta/core/types/LongType.java @@ -1,4 +1,7 @@ package io.delta.core.types; public class LongType extends DataType { + public static final LongType INSTANCE = new LongType(); + + private LongType() { } } diff --git a/kernel/src/main/java/io/delta/core/types/MapType.java b/kernel/src/main/java/io/delta/core/types/MapType.java new file mode 100644 index 000000000..7598fd43d --- /dev/null +++ b/kernel/src/main/java/io/delta/core/types/MapType.java @@ -0,0 +1,28 @@ +package io.delta.core.types; + +public class MapType extends DataType { + + public static final MapType EMPTY_INSTANCE = new MapType(null, null, false); + + private final DataType keyType; + private final DataType valueType; + private final boolean valueContainsNull; + + public MapType(DataType keyType, DataType valueType, boolean valueContainsNull) { + this.keyType = keyType; + this.valueType = valueType; + this.valueContainsNull = valueContainsNull; + } + + public DataType getKeyType() { + return keyType; + } + + public DataType getValueType() { + return valueType; + } + + public boolean isValueContainsNull() { + return valueContainsNull; + } +} diff --git a/kernel/src/main/java/io/delta/core/types/StringType.java b/kernel/src/main/java/io/delta/core/types/StringType.java index 951cc513a..0fe8598d8 100644 --- a/kernel/src/main/java/io/delta/core/types/StringType.java +++ b/kernel/src/main/java/io/delta/core/types/StringType.java @@ -1,4 +1,7 @@ package io.delta.core.types; public class StringType extends DataType { + public static final StringType INSTANCE = new StringType(); + + private StringType() { } } diff --git a/kernel/src/main/java/io/delta/core/types/StructField.java b/kernel/src/main/java/io/delta/core/types/StructField.java index 4d089f8fd..9c7dd37c8 100644 --- a/kernel/src/main/java/io/delta/core/types/StructField.java +++ b/kernel/src/main/java/io/delta/core/types/StructField.java @@ -1,6 +1,29 @@ package io.delta.core.types; +import io.delta.core.data.Row; + public class StructField { + + //////////////////////////////////////////////////////////////////////////////// + // Static Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + + public static StructField fromRow(Row row) { + final String name = row.getString(0); + final DataType type = UnresolvedDataType.fromRow(row, 1); + final boolean nullable = row.getBoolean(2); + return new StructField(name, type, nullable); + } + + public static final StructType READ_SCHEMA = new StructType() + .add("name", StringType.INSTANCE) + .add("type", UnresolvedDataType.INSTANCE) + .add("nullable", BooleanType.INSTANCE); + + //////////////////////////////////////////////////////////////////////////////// + // Instance Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + public final String name; public final DataType dataType; public final boolean nullable; @@ -11,4 +34,9 @@ public StructField(String name, DataType dataType, boolean nullable) { this.dataType = dataType; this.nullable = nullable; } + + @Override + public String toString() { + return String.format("StructField(%s,%s,%s)", name, dataType, nullable); + } } diff --git a/kernel/src/main/java/io/delta/core/types/StructType.java b/kernel/src/main/java/io/delta/core/types/StructType.java index 60604c89c..0b8b2caf3 100644 --- a/kernel/src/main/java/io/delta/core/types/StructType.java +++ b/kernel/src/main/java/io/delta/core/types/StructType.java @@ -5,7 +5,33 @@ import java.util.List; import java.util.stream.Collectors; +import io.delta.core.data.Row; + public final class StructType extends DataType { + + //////////////////////////////////////////////////////////////////////////////// + // Static Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + + public static StructType EMPTY_INSTANCE = new StructType(); + + public static StructType fromRow(Row row) { + final List fields = row.getList(0); + return new StructType( + fields + .stream() + .map(StructField::fromRow) + .collect(Collectors.toList()) + ); + } + + public static StructType READ_SCHEMA = new StructType() + .add("fields", new ArrayType(StructField.READ_SCHEMA, false /* contains null */ )); + + //////////////////////////////////////////////////////////////////////////////// + // Instance Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + private final List fields; public StructType() { @@ -43,6 +69,15 @@ public StructField at(int index) { return fields.get(index); } + @Override + public String toString() { + return String.format( + "%s(%s)", + getClass().getSimpleName(), + fields.stream().map(StructField::toString).collect(Collectors.joining(", ")) + ); + } + public String treeString() { return "TODO"; } diff --git a/kernel/src/main/java/io/delta/core/types/UnresolvedDataType.java b/kernel/src/main/java/io/delta/core/types/UnresolvedDataType.java new file mode 100644 index 000000000..b808668c7 --- /dev/null +++ b/kernel/src/main/java/io/delta/core/types/UnresolvedDataType.java @@ -0,0 +1,27 @@ +package io.delta.core.types; + +import io.delta.core.data.Row; + +/** + * e.g. IntegerType -> {"name":"as_int","type":"integer","nullable":true,"metadata":{}} + * e.g. LongType -> {"name":"as_long","type":"long","nullable":true,"metadata":{}} + * e.g. ArrayType(IntegerType) -> {"name":"as_array_int","type":{"type":"array","elementType":"integer","containsNull":true},"nullable":true,"metadata":{}} + * e.g. MapType(IntegerType) -> {"name":"a","type":{"type":"map","keyType":"integer","valueType":"integer","valueContainsNull":true},"nullable":true,"metadata":{}} + */ +public class UnresolvedDataType extends DataType { + + public static final UnresolvedDataType INSTANCE = new UnresolvedDataType(); + + public static DataType fromRow(Row row, int ordinal) { + try { + // e.g. IntegerType -> {"name":"as_int","type":"integer","nullable":true,"metadata":{} + // e.g. LongType -> {"name":"as_long","type":"long","nullable":true,"metadata":{}} + final String typeName = row.getString(ordinal); + return DataType.createPrimitive(typeName); + } catch (RuntimeException ex) { + throw new RuntimeException("Failed to parse UnresolvedDataType"); + } + } + + private UnresolvedDataType() { } +}