From bf7cf34754cadae3c13aab80a60f2140e8d658bd Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Wed, 29 Mar 2023 15:58:29 -0700 Subject: [PATCH 1/6] WIP; i think it's impossible or bad to parse it this way; JSON won't have a strict schema? Is it fundamentally wrong to assume the JSON schema? --- .../main/java/io/delta/core/data/JsonRow.java | 17 +++++--- .../core/helpers/DefaultTableHelper.java | 7 +++- .../test/scala/io/delta/core/TableSuite.scala | 9 ++++ .../core/util/GoldenTablesGenerator.scala | 4 ++ .../io/delta/core/helpers/TableHelper.java | 4 +- .../delta/core/internal/actions/AddFile.java | 2 +- .../delta/core/internal/actions/Format.java | 14 +++++++ .../delta/core/internal/actions/Metadata.java | 41 ++++++++++++++++--- .../core/internal/actions/SingleAction.java | 5 ++- .../checkpoint/CheckpointMetaData.java | 4 +- .../replay/ReverseFilesToActionsIterable.java | 7 +++- .../java/io/delta/core/types/ArrayType.java | 11 +++++ .../java/io/delta/core/types/BooleanType.java | 7 ++++ .../java/io/delta/core/types/DataType.java | 8 ++++ .../java/io/delta/core/types/LongType.java | 3 ++ .../java/io/delta/core/types/MapType.java | 4 ++ .../java/io/delta/core/types/StringType.java | 3 ++ .../java/io/delta/core/types/StructField.java | 14 +++++++ .../java/io/delta/core/types/StructType.java | 4 ++ .../delta/core/types/UnresolvedDataType.java | 23 +++++++++++ 20 files changed, 171 insertions(+), 20 deletions(-) create mode 100644 kernel/src/main/java/io/delta/core/internal/actions/Format.java create mode 100644 kernel/src/main/java/io/delta/core/types/ArrayType.java create mode 100644 kernel/src/main/java/io/delta/core/types/BooleanType.java create mode 100644 kernel/src/main/java/io/delta/core/types/MapType.java create mode 100644 kernel/src/main/java/io/delta/core/types/UnresolvedDataType.java diff --git a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java index 47cf1af74..4defbd282 100644 --- a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java +++ b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java @@ -45,7 +45,7 @@ private static Object parse(ObjectNode rootNode, StructField field) { } throw new UnsupportedOperationException( - String.format("Unsupported DataType %s", field.dataType.typeName()) + String.format("Unsupported DataType %s for RootNode %s", field.dataType.typeName(), jsonValue) ); } @@ -67,23 +67,28 @@ public JsonRow(ObjectNode rootNode, StructType readSchema) { @Override public long getLong(int ordinal) { - assert (readSchema.at(ordinal).dataType instanceof LongType); + if (!(readSchema.at(ordinal).dataType instanceof LongType)) { + throw new RuntimeException(); + } return (long) ordinalToValueMap.get(ordinal); } @Override public String getString(int ordinal) { - assert (readSchema.at(ordinal).dataType instanceof StringType); - + if (!(readSchema.at(ordinal).dataType instanceof StringType)) { + throw new RuntimeException(); + } return (String) ordinalToValueMap.get(ordinal); } @Override public Row getRecord(int ordinal) { - assert (readSchema.at(ordinal).dataType instanceof StructType); + if (!(readSchema.at(ordinal).dataType instanceof StructType)) { + throw new RuntimeException(); + } if (ordinalToValueMap.get(ordinal) == null) return null; - + System.out.println("Trying to parse inner record: " + ordinalToValueMap.get(ordinal)); return new JsonRow( (ObjectNode) ordinalToValueMap.get(ordinal), (StructType) readSchema.at(ordinal).dataType diff --git a/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java b/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java index 18fd37586..815a71c46 100644 --- a/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java +++ b/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java @@ -130,7 +130,12 @@ public CloseableIterator readParquetFile(String path, StructType readSchema } @Override - public Row parseStats(String statsJson) { + public Row parseStats(String statsJson, StructType statsSchema) { + return null; + } + + @Override + public StructType parseSchema(String schemaJson) { return null; } diff --git a/kernel-default/src/test/scala/io/delta/core/TableSuite.scala b/kernel-default/src/test/scala/io/delta/core/TableSuite.scala index ecaa275a4..4263adb6c 100644 --- a/kernel-default/src/test/scala/io/delta/core/TableSuite.scala +++ b/kernel-default/src/test/scala/io/delta/core/TableSuite.scala @@ -20,6 +20,7 @@ import io.delta.core.helpers.DefaultTableHelper import io.delta.core.util.GoldenTableUtils import org.scalatest.funsuite.AnyFunSuite +// scalastyle:off println class TableSuite extends AnyFunSuite with GoldenTableUtils { test("can load latest table version - with a checkpoint") { @@ -54,4 +55,12 @@ class TableSuite extends AnyFunSuite with GoldenTableUtils { iter.close() } } + + test("can load table schema - table without a checkpoint") { + withGoldenTable("basic-no-checkpoint") { path => + val table = Table.forPath(path, new DefaultTableHelper()) + val snapshot = table.getLatestSnapshot + println(snapshot.getSchema) + } + } } diff --git a/kernel-default/src/test/scala/io/delta/core/util/GoldenTablesGenerator.scala b/kernel-default/src/test/scala/io/delta/core/util/GoldenTablesGenerator.scala index d1f4040b3..638d39dc3 100644 --- a/kernel-default/src/test/scala/io/delta/core/util/GoldenTablesGenerator.scala +++ b/kernel-default/src/test/scala/io/delta/core/util/GoldenTablesGenerator.scala @@ -66,4 +66,8 @@ class GoldenTablesGenerator extends QueryTest with SharedSparkSession { spark.range(low, high).write.format("delta").mode("append").save(path) } } + + generate("primitive-data-types") { path => + + } } diff --git a/kernel/src/main/java/io/delta/core/helpers/TableHelper.java b/kernel/src/main/java/io/delta/core/helpers/TableHelper.java index 767f3a05b..e914c05fa 100644 --- a/kernel/src/main/java/io/delta/core/helpers/TableHelper.java +++ b/kernel/src/main/java/io/delta/core/helpers/TableHelper.java @@ -24,7 +24,9 @@ CloseableIterator readParquetFile( StructType readSchema, Expression skippingFilter); - Row parseStats(String statsJson); + Row parseStats(String statsJson, StructType statsSchema); + + StructType parseSchema(String schemaJson); ScanHelper getScanHelper(); } diff --git a/kernel/src/main/java/io/delta/core/internal/actions/AddFile.java b/kernel/src/main/java/io/delta/core/internal/actions/AddFile.java index d82968806..d6c17b06f 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/AddFile.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/AddFile.java @@ -15,7 +15,7 @@ public static AddFile fromRow(Row row) { } public static final StructType READ_SCHEMA = new StructType() - .add("path", new StringType()); + .add("path", StringType.INSTANCE); public AddFile(String path, boolean dataChange) { super(path, dataChange); diff --git a/kernel/src/main/java/io/delta/core/internal/actions/Format.java b/kernel/src/main/java/io/delta/core/internal/actions/Format.java new file mode 100644 index 000000000..e969919da --- /dev/null +++ b/kernel/src/main/java/io/delta/core/internal/actions/Format.java @@ -0,0 +1,14 @@ +package io.delta.core.internal.actions; + +import io.delta.core.data.Row; +import io.delta.core.types.StructType; + +public class Format { + public static Format fromRow(Row row) { + if (row == null) return null; + + return new Format(); + } + + public static final StructType READ_SCHEMA = new StructType(); +} diff --git a/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java b/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java index c5661dbe9..e04cee46e 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java @@ -1,19 +1,50 @@ package io.delta.core.internal.actions; import io.delta.core.data.Row; -import io.delta.core.types.StructType; +import io.delta.core.helpers.TableHelper; +import io.delta.core.types.*; public class Metadata implements Action { - public static Metadata fromRow(Row row) { + public static Metadata fromRow(Row row, TableHelper tableHelper) { if (row == null) return null; - return new Metadata(); + final String schemaJson = row.getString(4); + StructType schema = tableHelper.parseSchema(schemaJson); + + return new Metadata(schema); } - public static final StructType READ_SCHEMA = new StructType(); + /* + {"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1679943453303} + + */ + public static final StructType READ_SCHEMA = new StructType() + .add("id", StringType.INSTANCE) + .add("name", StringType.INSTANCE) + .add("description", StringType.INSTANCE) + .add("format", Format.READ_SCHEMA) + .add("schemaString", StringType.INSTANCE) + .add("partitionColumns", new ArrayType(StringType.INSTANCE, false /* contains null */)) + .add("configuration", new MapType()) + .add("createdTime", LongType.INSTANCE); + + // id + // name + // description + // format + // schemaString + // partitionColumns + // configuration + // createdTime + + private final StructType schema; + + public Metadata(StructType schema) { + this.schema = schema; + } public StructType getSchema() { - return null; + return schema; } } diff --git a/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java b/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java index de8676232..c075ccee1 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java @@ -1,11 +1,12 @@ package io.delta.core.internal.actions; import io.delta.core.data.Row; +import io.delta.core.helpers.TableHelper; import io.delta.core.types.StructType; public class SingleAction { - public static SingleAction fromRow(Row row) { + public static SingleAction fromRow(Row row, TableHelper tableHelper) { final SetTransaction txn = SetTransaction.fromRow(row.getRecord(0)); if (txn != null) { return new SingleAction(txn, null, null, null, null, null, null); @@ -21,7 +22,7 @@ public static SingleAction fromRow(Row row) { return new SingleAction(null, null, remove, null, null, null, null); } - final Metadata metadata = Metadata.fromRow(row.getRecord(3)); + final Metadata metadata = Metadata.fromRow(row.getRecord(3), tableHelper); if (metadata != null) { return new SingleAction(null, null, null, metadata, null, null, null); } diff --git a/kernel/src/main/java/io/delta/core/internal/checkpoint/CheckpointMetaData.java b/kernel/src/main/java/io/delta/core/internal/checkpoint/CheckpointMetaData.java index 4d1cb6491..fc12c6d0d 100644 --- a/kernel/src/main/java/io/delta/core/internal/checkpoint/CheckpointMetaData.java +++ b/kernel/src/main/java/io/delta/core/internal/checkpoint/CheckpointMetaData.java @@ -14,8 +14,8 @@ public static CheckpointMetaData fromRow(Row row) { } public static StructType READ_SCHEMA = new StructType() - .add("version", new LongType()) - .add("size", new LongType()); + .add("version", LongType.INSTANCE) + .add("size", LongType.INSTANCE); public final long version; public final long size; diff --git a/kernel/src/main/java/io/delta/core/internal/replay/ReverseFilesToActionsIterable.java b/kernel/src/main/java/io/delta/core/internal/replay/ReverseFilesToActionsIterable.java index d23988366..988de1ecc 100644 --- a/kernel/src/main/java/io/delta/core/internal/replay/ReverseFilesToActionsIterable.java +++ b/kernel/src/main/java/io/delta/core/internal/replay/ReverseFilesToActionsIterable.java @@ -131,7 +131,7 @@ private CloseableIterator> getNextActionsIter() { }; } - private static class RowToActionIterator implements CloseableIterator> { + private class RowToActionIterator implements CloseableIterator> { private final CloseableIterator impl; private final boolean isFromCheckpoint; @@ -148,7 +148,10 @@ public boolean hasNext() { @Override public Tuple2 next() { - return new Tuple2<>(SingleAction.fromRow(impl.next()).unwrap(), isFromCheckpoint); + return new Tuple2<>( + SingleAction.fromRow(impl.next(), tableHelper).unwrap(), + isFromCheckpoint + ); } @Override diff --git a/kernel/src/main/java/io/delta/core/types/ArrayType.java b/kernel/src/main/java/io/delta/core/types/ArrayType.java new file mode 100644 index 000000000..4b5f9ffef --- /dev/null +++ b/kernel/src/main/java/io/delta/core/types/ArrayType.java @@ -0,0 +1,11 @@ +package io.delta.core.types; + +public class ArrayType extends DataType { + private final DataType elementType; + private final boolean containsNull; + + public ArrayType(DataType elementType, boolean containsNull) { + this.elementType = elementType; + this.containsNull = containsNull; + } +} diff --git a/kernel/src/main/java/io/delta/core/types/BooleanType.java b/kernel/src/main/java/io/delta/core/types/BooleanType.java new file mode 100644 index 000000000..5cdb70431 --- /dev/null +++ b/kernel/src/main/java/io/delta/core/types/BooleanType.java @@ -0,0 +1,7 @@ +package io.delta.core.types; + +public class BooleanType extends DataType { + public static final BooleanType INSTANCE = new BooleanType(); + + private BooleanType() { } +} diff --git a/kernel/src/main/java/io/delta/core/types/DataType.java b/kernel/src/main/java/io/delta/core/types/DataType.java index 368093d6f..307e08eb3 100644 --- a/kernel/src/main/java/io/delta/core/types/DataType.java +++ b/kernel/src/main/java/io/delta/core/types/DataType.java @@ -3,6 +3,14 @@ import java.util.Locale; public abstract class DataType { + + public static DataType fromTypeName(String primitiveTypeName) { + if (primitiveTypeName.equals(LongType.INSTANCE.typeName())) return LongType.INSTANCE; + if (primitiveTypeName.equals(StringType.INSTANCE.typeName())) return StringType.INSTANCE; + + throw new IllegalArgumentException(String.format("Can't parse type %s", primitiveTypeName)); + } + public String typeName() { String name = this.getClass().getSimpleName(); if (name.endsWith("Type")) { diff --git a/kernel/src/main/java/io/delta/core/types/LongType.java b/kernel/src/main/java/io/delta/core/types/LongType.java index 2e222bd48..c8655e7f2 100644 --- a/kernel/src/main/java/io/delta/core/types/LongType.java +++ b/kernel/src/main/java/io/delta/core/types/LongType.java @@ -1,4 +1,7 @@ package io.delta.core.types; public class LongType extends DataType { + public static final LongType INSTANCE = new LongType(); + + private LongType() { } } diff --git a/kernel/src/main/java/io/delta/core/types/MapType.java b/kernel/src/main/java/io/delta/core/types/MapType.java new file mode 100644 index 000000000..634fcb110 --- /dev/null +++ b/kernel/src/main/java/io/delta/core/types/MapType.java @@ -0,0 +1,4 @@ +package io.delta.core.types; + +public class MapType extends DataType { +} diff --git a/kernel/src/main/java/io/delta/core/types/StringType.java b/kernel/src/main/java/io/delta/core/types/StringType.java index 951cc513a..0fe8598d8 100644 --- a/kernel/src/main/java/io/delta/core/types/StringType.java +++ b/kernel/src/main/java/io/delta/core/types/StringType.java @@ -1,4 +1,7 @@ package io.delta.core.types; public class StringType extends DataType { + public static final StringType INSTANCE = new StringType(); + + private StringType() { } } diff --git a/kernel/src/main/java/io/delta/core/types/StructField.java b/kernel/src/main/java/io/delta/core/types/StructField.java index 4d089f8fd..459027be8 100644 --- a/kernel/src/main/java/io/delta/core/types/StructField.java +++ b/kernel/src/main/java/io/delta/core/types/StructField.java @@ -1,6 +1,20 @@ package io.delta.core.types; +import io.delta.core.data.Row; + public class StructField { + public static final StructType READ_SCHEMA = new StructType() + .add("name", StringType.INSTANCE) + .add("type", new UnresolvedDataType()) + .add("nullable", BooleanType.INSTANCE); + + public static StructField fromRow(Row row) { + final String name = row.getString(0); + final DataType type = UnresolvedDataType.fromRow(row.getRecord(1)); + final boolean nullable = true; // TODO row.getBoolean(2); + return new StructField(name, type, nullable); + } + public final String name; public final DataType dataType; public final boolean nullable; diff --git a/kernel/src/main/java/io/delta/core/types/StructType.java b/kernel/src/main/java/io/delta/core/types/StructType.java index 60604c89c..2d458199b 100644 --- a/kernel/src/main/java/io/delta/core/types/StructType.java +++ b/kernel/src/main/java/io/delta/core/types/StructType.java @@ -6,6 +6,10 @@ import java.util.stream.Collectors; public final class StructType extends DataType { + + public static StructType READ_SCHEMA = new StructType() + .add("fields", new ArrayType(StructField.READ_SCHEMA, false /* contains null */ )); + private final List fields; public StructType() { diff --git a/kernel/src/main/java/io/delta/core/types/UnresolvedDataType.java b/kernel/src/main/java/io/delta/core/types/UnresolvedDataType.java new file mode 100644 index 000000000..593723a16 --- /dev/null +++ b/kernel/src/main/java/io/delta/core/types/UnresolvedDataType.java @@ -0,0 +1,23 @@ +package io.delta.core.types; + +import io.delta.core.data.Row; + +/** + * e.g. IntegerType -> {"name":"as_int","type":"integer","nullable":true,"metadata":{}} + * e.g. LongType -> {"name":"as_long","type":"long","nullable":true,"metadata":{}} + * e.g. ArrayType(IntegerType) -> {"name":"as_array_int","type":{"type":"array","elementType":"integer","containsNull":true},"nullable":true,"metadata":{}} + * e.g. MapType(IntegerType) -> {"name":"a","type":{"type":"map","keyType":"integer","valueType":"integer","valueContainsNull":true},"nullable":true,"metadata":{}} + */ +public class UnresolvedDataType extends DataType { + + public static DataType fromRow(Row row) { + try { + final String typeName = row.getString(0); + return DataType.fromTypeName(typeName); + } catch (RuntimeException ex) { + // TODO: parse ArrayType, etc. + throw new RuntimeException("TODO"); + } + } + +} From c03bbec5f7cf224f8b738d7432c85bc84dec5e01 Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Thu, 30 Mar 2023 10:44:14 -0700 Subject: [PATCH 2/6] Can read the schema, containing fields of UnresolvedDataTypes --- .../main/java/io/delta/core/data/JsonRow.java | 193 +++++++++++++----- .../core/helpers/DefaultTableHelper.java | 22 +- .../src/main/java/io/delta/core/data/Row.java | 8 +- .../io/delta/core/helpers/TableHelper.java | 4 +- .../delta/core/internal/actions/Format.java | 15 +- .../delta/core/internal/actions/Metadata.java | 20 +- .../delta/core/internal/actions/Protocol.java | 21 +- .../core/internal/actions/SingleAction.java | 25 ++- .../replay/ReverseFilesToActionsIterable.java | 2 - .../java/io/delta/core/types/ArrayType.java | 10 + .../java/io/delta/core/types/DataType.java | 10 +- .../java/io/delta/core/types/IntegerType.java | 7 + .../java/io/delta/core/types/StructField.java | 26 ++- .../java/io/delta/core/types/StructType.java | 31 +++ .../delta/core/types/UnresolvedDataType.java | 14 +- 15 files changed, 305 insertions(+), 103 deletions(-) create mode 100644 kernel/src/main/java/io/delta/core/types/IntegerType.java diff --git a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java index 4defbd282..74ce8b615 100644 --- a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java +++ b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java @@ -1,105 +1,204 @@ package io.delta.core.data; -import java.util.HashMap; -import java.util.Map; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.delta.core.types.LongType; -import io.delta.core.types.StringType; -import io.delta.core.types.StructField; -import io.delta.core.types.StructType; +import io.delta.core.types.*; public class JsonRow implements Row { - // TODO: we can do this cleaner / smarter / better - private static Object parse(ObjectNode rootNode, StructField field) { - if (rootNode.get(field.name) == null) { - if (field.nullable) { + //////////////////////////////////////////////////////////////////////////////// + // Static Methods + //////////////////////////////////////////////////////////////////////////////// + + private static Object decode( + ObjectNode rootNode, + String fieldName, + DataType dataType, + boolean isNullable) { + if (rootNode.get(fieldName) == null) { + if (isNullable) { return null; } - throw new NullPointerException( - String.format("Root node at key %s is null but field %s isn't nullable", rootNode, field) + throw new RuntimeException( + String.format( + "Root node at key %s is null but field isn't nullable. Root node: %s", + fieldName, + rootNode + ) ); } - JsonNode jsonValue = rootNode.get(field.name); + JsonNode jsonValue = rootNode.get(fieldName); - if (field.dataType instanceof LongType) { - assert (jsonValue.isLong()) : - String.format("RootNode at %s isn't a long", field.name); - return jsonValue.longValue(); + if (dataType instanceof UnresolvedDataType) { + if (jsonValue.isTextual()) { + return jsonValue.textValue(); + } else if (jsonValue instanceof ObjectNode) { + // TODO + } + } + + if (dataType instanceof BooleanType) { + if (!jsonValue.isBoolean()) { + throw new RuntimeException( + String.format("RootNode at %s isn't a boolean", fieldName) + ); + } + return jsonValue.booleanValue(); } - if (field.dataType instanceof StructType) { - assert (jsonValue.isObject()) : - String.format("RootNode at %s isn't a object", field.name); - return (ObjectNode) jsonValue; + if (dataType instanceof IntegerType) { + if (!jsonValue.isInt()) { + throw new RuntimeException( + String.format("RootNode at %s isn't an int", fieldName) + ); + } + return jsonValue.intValue(); + } + + if (dataType instanceof LongType) { + if (!jsonValue.isLong()) { + throw new RuntimeException( + String.format("RootNode at %s isn't a long", fieldName) + ); + } + return jsonValue.longValue(); } - if (field.dataType instanceof StringType) { - assert (jsonValue.isTextual()) : - String.format("RootNode at %s isn't a string", field.name); + if (dataType instanceof StringType) { + if (!jsonValue.isTextual()) { + throw new RuntimeException( + String.format("RootNode at %s isn't a string", fieldName) + ); + } return jsonValue.textValue(); } + if (dataType instanceof StructType) { + if (!jsonValue.isObject()) { + throw new RuntimeException( + String.format("RootNode at %s isn't an object", fieldName) + ); + } + return new JsonRow((ObjectNode) jsonValue, (StructType) dataType); + } + + if (dataType instanceof ArrayType) { + if (!jsonValue.isArray()) { + throw new RuntimeException( + String.format("RootNode at %s isn't an array", fieldName) + ); + } + final ArrayType arrayType = ((ArrayType) dataType); + final List output = new ArrayList<>(); + final ArrayNode array = (ArrayNode) jsonValue; + + for (Iterator it = array.elements(); it.hasNext();) { + JsonNode element = it.next(); + if (element.isObject()) { + output.add( + new JsonRow((ObjectNode) element, (StructType) arrayType.getElementType()) + ); + } + } + return output; + } + + if (dataType instanceof MapType) { + // TODO: + return null; + } + throw new UnsupportedOperationException( - String.format("Unsupported DataType %s for RootNode %s", field.dataType.typeName(), jsonValue) + String.format("Unsupported DataType %s for RootNode %s", dataType.typeName(), jsonValue) ); } + //////////////////////////////////////////////////////////////////////////////// + // Instance Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + private final ObjectNode rootNode; - private final Map ordinalToValueMap; + private final Object[] parsedValues; private final StructType readSchema; public JsonRow(ObjectNode rootNode, StructType readSchema) { this.rootNode = rootNode; this.readSchema = readSchema; - this.ordinalToValueMap = new HashMap<>(); + this.parsedValues = new Object[readSchema.length()]; for (int i = 0; i < readSchema.length(); i++) { final StructField field = readSchema.at(i); - Object val = parse(rootNode, field); - ordinalToValueMap.put(i, val); + final Object parsedValue = decode(rootNode, field.name, field.dataType, field.nullable); + parsedValues[i] = parsedValue; + } + } + + private void assertType(int ordinal, DataType expectedType) { + final String actualTypeName = readSchema.at(ordinal).dataType.typeName(); + if (!actualTypeName.equals(expectedType.typeName()) && + !actualTypeName.equals(UnresolvedDataType.INSTANCE.typeName())) { + throw new RuntimeException( + String.format( + "Tried to read long at ordinal %s but actual data type is", + ordinal, readSchema.at(ordinal).dataType.typeName() + ) + ); } } + @Override + public boolean isNullAt(int ordinal) { + return parsedValues[ordinal] == null; + } + + @Override + public boolean getBoolean(int ordinal) { + assertType(ordinal, BooleanType.INSTANCE); + return (boolean) parsedValues[ordinal]; + } + + @Override + public int getInt(int ordinal) { + assertType(ordinal, IntegerType.INSTANCE); + return (int) parsedValues[ordinal]; + } + @Override public long getLong(int ordinal) { - if (!(readSchema.at(ordinal).dataType instanceof LongType)) { - throw new RuntimeException(); - } - return (long) ordinalToValueMap.get(ordinal); + assertType(ordinal, LongType.INSTANCE); + return (long) parsedValues[ordinal]; } @Override public String getString(int ordinal) { - if (!(readSchema.at(ordinal).dataType instanceof StringType)) { - throw new RuntimeException(); - } - return (String) ordinalToValueMap.get(ordinal); + assertType(ordinal, StringType.INSTANCE); + return (String) parsedValues[ordinal]; } @Override public Row getRecord(int ordinal) { - if (!(readSchema.at(ordinal).dataType instanceof StructType)) { - throw new RuntimeException(); - } + assertType(ordinal, StructType.EMPTY_INSTANCE); + return (JsonRow) parsedValues[ordinal]; + } - if (ordinalToValueMap.get(ordinal) == null) return null; - System.out.println("Trying to parse inner record: " + ordinalToValueMap.get(ordinal)); - return new JsonRow( - (ObjectNode) ordinalToValueMap.get(ordinal), - (StructType) readSchema.at(ordinal).dataType - ); + @Override + public List getList(int ordinal) { + assertType(ordinal, ArrayType.EMPTY_INSTANCE); + return (List) parsedValues[ordinal]; } @Override public String toString() { return "JsonRow{" + "rootNode=" + rootNode + - ", ordinalToValueMap=" + ordinalToValueMap + + ", parsedValues=" + parsedValues + ", readSchema=" + readSchema + '}'; } diff --git a/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java b/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java index 815a71c46..cd6782836 100644 --- a/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java +++ b/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java @@ -108,13 +108,7 @@ public boolean hasNext() { @Override public Row next() { - final String json = iter.next(); - try { - final JsonNode jsonNode = objectMapper.readTree(json); - return new JsonRow((ObjectNode) jsonNode, readSchema); - } catch (JsonProcessingException ex) { - throw new RuntimeException(String.format("Could not parse JSON: %s", json), ex); - } + return parseJson(iter.next(), readSchema); } }; } @@ -130,13 +124,13 @@ public CloseableIterator readParquetFile(String path, StructType readSchema } @Override - public Row parseStats(String statsJson, StructType statsSchema) { - return null; - } - - @Override - public StructType parseSchema(String schemaJson) { - return null; + public Row parseJson(String json, StructType readSchema) { + try { + final JsonNode jsonNode = objectMapper.readTree(json); + return new JsonRow((ObjectNode) jsonNode, readSchema); + } catch (JsonProcessingException ex) { + throw new RuntimeException(String.format("Could not parse JSON: %s", json), ex); + } } @Override diff --git a/kernel/src/main/java/io/delta/core/data/Row.java b/kernel/src/main/java/io/delta/core/data/Row.java index df460d008..fb1c0f68c 100644 --- a/kernel/src/main/java/io/delta/core/data/Row.java +++ b/kernel/src/main/java/io/delta/core/data/Row.java @@ -8,15 +8,15 @@ public interface Row { -// boolean isNullAt(int ordinal); + boolean isNullAt(int ordinal); // -// boolean getBoolean(int ordinal); + boolean getBoolean(int ordinal); // // byte getByte(int ordinal); // // short getShort(int ordinal); // -// int getInt(int ordinal); + int getInt(int ordinal); long getLong(int ordinal); @@ -36,7 +36,7 @@ public interface Row { // Row getRecord(int ordinal); // -// List getList(int ordinal); + List getList(int ordinal); // // Map getMap(int ordinal); } diff --git a/kernel/src/main/java/io/delta/core/helpers/TableHelper.java b/kernel/src/main/java/io/delta/core/helpers/TableHelper.java index e914c05fa..00bf6d66e 100644 --- a/kernel/src/main/java/io/delta/core/helpers/TableHelper.java +++ b/kernel/src/main/java/io/delta/core/helpers/TableHelper.java @@ -24,9 +24,7 @@ CloseableIterator readParquetFile( StructType readSchema, Expression skippingFilter); - Row parseStats(String statsJson, StructType statsSchema); - - StructType parseSchema(String schemaJson); + Row parseJson(String json, StructType schema); ScanHelper getScanHelper(); } diff --git a/kernel/src/main/java/io/delta/core/internal/actions/Format.java b/kernel/src/main/java/io/delta/core/internal/actions/Format.java index e969919da..bdac86b80 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/Format.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/Format.java @@ -1,14 +1,25 @@ package io.delta.core.internal.actions; import io.delta.core.data.Row; +import io.delta.core.types.StringType; import io.delta.core.types.StructType; public class Format { public static Format fromRow(Row row) { if (row == null) return null; - return new Format(); + final String provider = row.getString(0); + + return new Format(provider); } - public static final StructType READ_SCHEMA = new StructType(); + public static final StructType READ_SCHEMA = new StructType() + .add("provider", StringType.INSTANCE); + // TODO: options + + private final String provider; + + public Format(String provider) { + this.provider = provider; + } } diff --git a/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java b/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java index e04cee46e..e51b9ac61 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java @@ -6,19 +6,23 @@ public class Metadata implements Action { + //////////////////////////////////////////////////////////////////////////////// + // Static Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + public static Metadata fromRow(Row row, TableHelper tableHelper) { if (row == null) return null; - + final String id = row.getString(0); + final String name = row.getString(1); + final String description = row.getString(2); + final Format format = Format.fromRow(row.getRecord(3)); final String schemaJson = row.getString(4); - StructType schema = tableHelper.parseSchema(schemaJson); + Row schemaRow = tableHelper.parseJson(schemaJson, StructType.READ_SCHEMA); + StructType schema = StructType.fromRow(schemaRow); return new Metadata(schema); } - /* - {"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1679943453303} - - */ public static final StructType READ_SCHEMA = new StructType() .add("id", StringType.INSTANCE) .add("name", StringType.INSTANCE) @@ -29,6 +33,10 @@ public static Metadata fromRow(Row row, TableHelper tableHelper) { .add("configuration", new MapType()) .add("createdTime", LongType.INSTANCE); + //////////////////////////////////////////////////////////////////////////////// + // Instance Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + // id // name // description diff --git a/kernel/src/main/java/io/delta/core/internal/actions/Protocol.java b/kernel/src/main/java/io/delta/core/internal/actions/Protocol.java index f718e90ea..fba102fba 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/Protocol.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/Protocol.java @@ -1,15 +1,32 @@ package io.delta.core.internal.actions; import io.delta.core.data.Row; +import io.delta.core.types.IntegerType; import io.delta.core.types.StructType; public class Protocol implements Action { public static Protocol fromRow(Row row) { if (row == null) return null; + final int minReaderVersion = row.getInt(0); + final int minWriterVersion = row.getInt(1); + return new Protocol(minReaderVersion, minWriterVersion); + } + + public static final StructType READ_SCHEMA = new StructType() + .add("minReaderVersion", IntegerType.INSTANCE) + .add("minWriterVersion", IntegerType.INSTANCE); - return new Protocol(); + private final int minReaderVersion; + private final int minWriterVersion; + + public Protocol(int minReaderVersion, int minWriterVersion) { + this.minReaderVersion = minReaderVersion; + this.minWriterVersion = minWriterVersion; } - public static final StructType READ_SCHEMA = new StructType(); + @Override + public String toString() { + return String.format("Protocol(%s,%s)", minReaderVersion, minWriterVersion); + } } diff --git a/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java b/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java index c075ccee1..d3378eb72 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java @@ -6,38 +6,42 @@ public class SingleAction { + //////////////////////////////////////////////////////////////////////////////// + // Static Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + public static SingleAction fromRow(Row row, TableHelper tableHelper) { - final SetTransaction txn = SetTransaction.fromRow(row.getRecord(0)); + final SetTransaction txn = row.isNullAt(0) ? null : SetTransaction.fromRow(row.getRecord(0)); if (txn != null) { return new SingleAction(txn, null, null, null, null, null, null); } - final AddFile add = AddFile.fromRow(row.getRecord(1)); + final AddFile add = row.isNullAt(1) ? null : AddFile.fromRow(row.getRecord(1)); if (add != null) { return new SingleAction(null, add, null, null, null, null, null); } - final RemoveFile remove = RemoveFile.fromRow(row.getRecord(2)); + final RemoveFile remove = row.isNullAt(2) ? null : RemoveFile.fromRow(row.getRecord(2)); if (remove != null) { return new SingleAction(null, null, remove, null, null, null, null); } - final Metadata metadata = Metadata.fromRow(row.getRecord(3), tableHelper); + final Metadata metadata = row.isNullAt(3) ? null : Metadata.fromRow(row.getRecord(3), tableHelper); if (metadata != null) { return new SingleAction(null, null, null, metadata, null, null, null); } - final Protocol protocol = Protocol.fromRow(row.getRecord(4)); + final Protocol protocol = row.isNullAt(4) ? null : Protocol.fromRow(row.getRecord(4)); if (protocol != null) { return new SingleAction(null, null, null, null, protocol, null, null); } - final AddCDCFile cdc = AddCDCFile.fromRow(row.getRecord(5)); + final AddCDCFile cdc = row.isNullAt(5) ? null : AddCDCFile.fromRow(row.getRecord(5)); if (cdc != null) { return new SingleAction(null, null, null, null, null, cdc, null); } - final CommitInfo commitInfo = CommitInfo.fromRow(row.getRecord(6)); + final CommitInfo commitInfo = row.isNullAt(6) ? null : CommitInfo.fromRow(row.getRecord(6)); if (commitInfo != null) { return new SingleAction(null, null, null, null, null, null, commitInfo); } @@ -54,6 +58,10 @@ public static SingleAction fromRow(Row row, TableHelper tableHelper) { .add("cdc", AddCDCFile.READ_SCHEMA) .add("commitInfo", CommitInfo.READ_SCHEMA); + //////////////////////////////////////////////////////////////////////////////// + // Instance Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + private final SetTransaction txn; private final AddFile add; private final RemoveFile remove; @@ -87,6 +95,7 @@ public Action unwrap() { if (protocol != null) return protocol; if (cdc != null) return cdc; if (commitInfo != null) return commitInfo; - return null; + + throw new IllegalStateException("SingleAction row contained no non-null actions"); } } diff --git a/kernel/src/main/java/io/delta/core/internal/replay/ReverseFilesToActionsIterable.java b/kernel/src/main/java/io/delta/core/internal/replay/ReverseFilesToActionsIterable.java index 988de1ecc..780fc3250 100644 --- a/kernel/src/main/java/io/delta/core/internal/replay/ReverseFilesToActionsIterable.java +++ b/kernel/src/main/java/io/delta/core/internal/replay/ReverseFilesToActionsIterable.java @@ -108,13 +108,11 @@ private CloseableIterator> getNextActionsIter() { try { if (nextPath.getName().endsWith(".json")) { - System.out.println("Scott > ReverseFilesToActionsIterable > reading " + nextPath.getName()); return new RowToActionIterator( tableHelper.readJsonFile(nextPath.toString(), SingleAction.READ_SCHEMA), false // isFromCheckpoint ); } else if (nextPath.getName().endsWith(".parquet")) { - System.out.println("Scott > ReverseFilesToActionsIterable > reading " + nextPath.getName()); return new RowToActionIterator( tableHelper.readParquetFile(nextPath.toString(), SingleAction.READ_SCHEMA), true // isFromCheckpoint diff --git a/kernel/src/main/java/io/delta/core/types/ArrayType.java b/kernel/src/main/java/io/delta/core/types/ArrayType.java index 4b5f9ffef..10abc3dd4 100644 --- a/kernel/src/main/java/io/delta/core/types/ArrayType.java +++ b/kernel/src/main/java/io/delta/core/types/ArrayType.java @@ -1,6 +1,8 @@ package io.delta.core.types; public class ArrayType extends DataType { + public static ArrayType EMPTY_INSTANCE = new ArrayType(null, false); + private final DataType elementType; private final boolean containsNull; @@ -8,4 +10,12 @@ public ArrayType(DataType elementType, boolean containsNull) { this.elementType = elementType; this.containsNull = containsNull; } + + public DataType getElementType() { + return elementType; + } + + public boolean containsNull() { + return containsNull; + } } diff --git a/kernel/src/main/java/io/delta/core/types/DataType.java b/kernel/src/main/java/io/delta/core/types/DataType.java index 307e08eb3..920eb92b3 100644 --- a/kernel/src/main/java/io/delta/core/types/DataType.java +++ b/kernel/src/main/java/io/delta/core/types/DataType.java @@ -4,11 +4,13 @@ public abstract class DataType { - public static DataType fromTypeName(String primitiveTypeName) { - if (primitiveTypeName.equals(LongType.INSTANCE.typeName())) return LongType.INSTANCE; - if (primitiveTypeName.equals(StringType.INSTANCE.typeName())) return StringType.INSTANCE; + public static DataType createPrimitive(String typeName) { + if (typeName.equals(LongType.INSTANCE.typeName())) return LongType.INSTANCE; + if (typeName.equals(StringType.INSTANCE.typeName())) return StringType.INSTANCE; - throw new IllegalArgumentException(String.format("Can't parse type %s", primitiveTypeName)); + throw new IllegalArgumentException( + String.format("Can't create primitive for type type %s", typeName) + ); } public String typeName() { diff --git a/kernel/src/main/java/io/delta/core/types/IntegerType.java b/kernel/src/main/java/io/delta/core/types/IntegerType.java new file mode 100644 index 000000000..a42af3280 --- /dev/null +++ b/kernel/src/main/java/io/delta/core/types/IntegerType.java @@ -0,0 +1,7 @@ +package io.delta.core.types; + +public class IntegerType extends DataType { + public static final IntegerType INSTANCE = new IntegerType(); + + private IntegerType() { } +} diff --git a/kernel/src/main/java/io/delta/core/types/StructField.java b/kernel/src/main/java/io/delta/core/types/StructField.java index 459027be8..a9069f48a 100644 --- a/kernel/src/main/java/io/delta/core/types/StructField.java +++ b/kernel/src/main/java/io/delta/core/types/StructField.java @@ -3,18 +3,27 @@ import io.delta.core.data.Row; public class StructField { - public static final StructType READ_SCHEMA = new StructType() - .add("name", StringType.INSTANCE) - .add("type", new UnresolvedDataType()) - .add("nullable", BooleanType.INSTANCE); + + //////////////////////////////////////////////////////////////////////////////// + // Static Fields / Methods + //////////////////////////////////////////////////////////////////////////////// public static StructField fromRow(Row row) { final String name = row.getString(0); - final DataType type = UnresolvedDataType.fromRow(row.getRecord(1)); - final boolean nullable = true; // TODO row.getBoolean(2); + final DataType type = UnresolvedDataType.fromRow(row, 1); + final boolean nullable = row.getBoolean(2); return new StructField(name, type, nullable); } + public static final StructType READ_SCHEMA = new StructType() + .add("name", StringType.INSTANCE) + .add("type", StringType.INSTANCE) + .add("nullable", BooleanType.INSTANCE); + + //////////////////////////////////////////////////////////////////////////////// + // Instance Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + public final String name; public final DataType dataType; public final boolean nullable; @@ -25,4 +34,9 @@ public StructField(String name, DataType dataType, boolean nullable) { this.dataType = dataType; this.nullable = nullable; } + + @Override + public String toString() { + return String.format("StructField(%s,%s,%s)", name, dataType, nullable); + } } diff --git a/kernel/src/main/java/io/delta/core/types/StructType.java b/kernel/src/main/java/io/delta/core/types/StructType.java index 2d458199b..0b8b2caf3 100644 --- a/kernel/src/main/java/io/delta/core/types/StructType.java +++ b/kernel/src/main/java/io/delta/core/types/StructType.java @@ -5,11 +5,33 @@ import java.util.List; import java.util.stream.Collectors; +import io.delta.core.data.Row; + public final class StructType extends DataType { + //////////////////////////////////////////////////////////////////////////////// + // Static Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + + public static StructType EMPTY_INSTANCE = new StructType(); + + public static StructType fromRow(Row row) { + final List fields = row.getList(0); + return new StructType( + fields + .stream() + .map(StructField::fromRow) + .collect(Collectors.toList()) + ); + } + public static StructType READ_SCHEMA = new StructType() .add("fields", new ArrayType(StructField.READ_SCHEMA, false /* contains null */ )); + //////////////////////////////////////////////////////////////////////////////// + // Instance Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + private final List fields; public StructType() { @@ -47,6 +69,15 @@ public StructField at(int index) { return fields.get(index); } + @Override + public String toString() { + return String.format( + "%s(%s)", + getClass().getSimpleName(), + fields.stream().map(StructField::toString).collect(Collectors.joining(", ")) + ); + } + public String treeString() { return "TODO"; } diff --git a/kernel/src/main/java/io/delta/core/types/UnresolvedDataType.java b/kernel/src/main/java/io/delta/core/types/UnresolvedDataType.java index 593723a16..b808668c7 100644 --- a/kernel/src/main/java/io/delta/core/types/UnresolvedDataType.java +++ b/kernel/src/main/java/io/delta/core/types/UnresolvedDataType.java @@ -10,14 +10,18 @@ */ public class UnresolvedDataType extends DataType { - public static DataType fromRow(Row row) { + public static final UnresolvedDataType INSTANCE = new UnresolvedDataType(); + + public static DataType fromRow(Row row, int ordinal) { try { - final String typeName = row.getString(0); - return DataType.fromTypeName(typeName); + // e.g. IntegerType -> {"name":"as_int","type":"integer","nullable":true,"metadata":{} + // e.g. LongType -> {"name":"as_long","type":"long","nullable":true,"metadata":{}} + final String typeName = row.getString(ordinal); + return DataType.createPrimitive(typeName); } catch (RuntimeException ex) { - // TODO: parse ArrayType, etc. - throw new RuntimeException("TODO"); + throw new RuntimeException("Failed to parse UnresolvedDataType"); } } + private UnresolvedDataType() { } } From e73360630a2b7fdfedbee422d1afad9bf7ecab77 Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Thu, 30 Mar 2023 10:49:05 -0700 Subject: [PATCH 3/6] minor comments --- .../main/java/io/delta/core/data/JsonRow.java | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java index 74ce8b615..866b12bd8 100644 --- a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java +++ b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java @@ -140,18 +140,9 @@ public JsonRow(ObjectNode rootNode, StructType readSchema) { } } - private void assertType(int ordinal, DataType expectedType) { - final String actualTypeName = readSchema.at(ordinal).dataType.typeName(); - if (!actualTypeName.equals(expectedType.typeName()) && - !actualTypeName.equals(UnresolvedDataType.INSTANCE.typeName())) { - throw new RuntimeException( - String.format( - "Tried to read long at ordinal %s but actual data type is", - ordinal, readSchema.at(ordinal).dataType.typeName() - ) - ); - } - } + //////////////////////////////////////// + // Public APIs + //////////////////////////////////////// @Override public boolean isNullAt(int ordinal) { @@ -202,4 +193,21 @@ public String toString() { ", readSchema=" + readSchema + '}'; } + + //////////////////////////////////////// + // Private Helper Methods + //////////////////////////////////////// + + private void assertType(int ordinal, DataType expectedType) { + final String actualTypeName = readSchema.at(ordinal).dataType.typeName(); + if (!actualTypeName.equals(expectedType.typeName()) && + !actualTypeName.equals(UnresolvedDataType.INSTANCE.typeName())) { + throw new RuntimeException( + String.format( + "Tried to read long at ordinal %s but actual data type is", + ordinal, readSchema.at(ordinal).dataType.typeName() + ) + ); + } + } } From 2caed421187f9f7eb34ed131d4ac4ca528d18cb9 Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Thu, 30 Mar 2023 10:56:43 -0700 Subject: [PATCH 4/6] Fix error msg; Fix StructField READ_SCHEMA to use UnresolvedDataType --- .../src/main/java/io/delta/core/data/JsonRow.java | 6 ++++-- .../io/delta/core/{ => internal}/TableSuite.scala | 7 +++++-- .../java/io/delta/core/internal/SnapshotImpl.java | 12 ++++++++++++ .../main/java/io/delta/core/types/StructField.java | 2 +- 4 files changed, 22 insertions(+), 5 deletions(-) rename kernel-default/src/test/scala/io/delta/core/{ => internal}/TableSuite.scala (90%) diff --git a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java index 866b12bd8..bb53a4d50 100644 --- a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java +++ b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java @@ -204,8 +204,10 @@ private void assertType(int ordinal, DataType expectedType) { !actualTypeName.equals(UnresolvedDataType.INSTANCE.typeName())) { throw new RuntimeException( String.format( - "Tried to read long at ordinal %s but actual data type is", - ordinal, readSchema.at(ordinal).dataType.typeName() + "Tried to read %s at ordinal %s but actual data type is %s", + expectedType.typeName(), + ordinal, + actualTypeName ) ); } diff --git a/kernel-default/src/test/scala/io/delta/core/TableSuite.scala b/kernel-default/src/test/scala/io/delta/core/internal/TableSuite.scala similarity index 90% rename from kernel-default/src/test/scala/io/delta/core/TableSuite.scala rename to kernel-default/src/test/scala/io/delta/core/internal/TableSuite.scala index 4263adb6c..bc78919b5 100644 --- a/kernel-default/src/test/scala/io/delta/core/TableSuite.scala +++ b/kernel-default/src/test/scala/io/delta/core/internal/TableSuite.scala @@ -14,9 +14,11 @@ * limitations under the License. */ -package io.delta.core +package io.delta.core.internal +import io.delta.core.Table import io.delta.core.helpers.DefaultTableHelper +import io.delta.core.internal.SnapshotImpl import io.delta.core.util.GoldenTableUtils import org.scalatest.funsuite.AnyFunSuite @@ -59,8 +61,9 @@ class TableSuite extends AnyFunSuite with GoldenTableUtils { test("can load table schema - table without a checkpoint") { withGoldenTable("basic-no-checkpoint") { path => val table = Table.forPath(path, new DefaultTableHelper()) - val snapshot = table.getLatestSnapshot + val snapshot = table.getLatestSnapshot.asInstanceOf[SnapshotImpl] println(snapshot.getSchema) + println(snapshot.getProtocol) } } } diff --git a/kernel/src/main/java/io/delta/core/internal/SnapshotImpl.java b/kernel/src/main/java/io/delta/core/internal/SnapshotImpl.java index 2014bb029..8eb1290d8 100644 --- a/kernel/src/main/java/io/delta/core/internal/SnapshotImpl.java +++ b/kernel/src/main/java/io/delta/core/internal/SnapshotImpl.java @@ -37,6 +37,10 @@ public SnapshotImpl( this.protocolAndMetadata = logReplay.lazyLoadProtocolAndMetadata(); } + //////////////////////////////////////// + // Public APIs + //////////////////////////////////////// + @Override public long getVersion() { return version; @@ -51,4 +55,12 @@ public StructType getSchema() { public ScanBuilder getScanBuilder() { return new ScanBuilderImpl(logReplay); } + + //////////////////////////////////////// + // Internal APIs + //////////////////////////////////////// + + public Protocol getProtocol() { + return protocolAndMetadata.get()._1; + } } diff --git a/kernel/src/main/java/io/delta/core/types/StructField.java b/kernel/src/main/java/io/delta/core/types/StructField.java index a9069f48a..9c7dd37c8 100644 --- a/kernel/src/main/java/io/delta/core/types/StructField.java +++ b/kernel/src/main/java/io/delta/core/types/StructField.java @@ -17,7 +17,7 @@ public static StructField fromRow(Row row) { public static final StructType READ_SCHEMA = new StructType() .add("name", StringType.INSTANCE) - .add("type", StringType.INSTANCE) + .add("type", UnresolvedDataType.INSTANCE) .add("nullable", BooleanType.INSTANCE); //////////////////////////////////////////////////////////////////////////////// From d9635a22a4450da12854d98f91eca0da09738b10 Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Thu, 30 Mar 2023 15:10:19 -0700 Subject: [PATCH 5/6] Can parse add file partition values --- .../main/java/io/delta/core/data/JsonRow.java | 58 +++++++++++++----- .../core/helpers/DefaultTableHelper.java | 2 +- .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 40 bytes .../_delta_log/.00000000000000000001.json.crc | Bin 0 -> 36 bytes .../_delta_log/.00000000000000000002.json.crc | Bin 0 -> 36 bytes .../_delta_log/.00000000000000000003.json.crc | Bin 0 -> 36 bytes .../_delta_log/.00000000000000000004.json.crc | Bin 0 -> 36 bytes .../_delta_log/.00000000000000000005.json.crc | Bin 0 -> 36 bytes .../_delta_log/.00000000000000000006.json.crc | Bin 0 -> 36 bytes .../_delta_log/.00000000000000000007.json.crc | Bin 0 -> 36 bytes .../_delta_log/.00000000000000000008.json.crc | Bin 0 -> 36 bytes .../_delta_log/00000000000000000000.json | 13 ++++ .../_delta_log/00000000000000000001.json | 11 ++++ .../_delta_log/00000000000000000002.json | 11 ++++ .../_delta_log/00000000000000000003.json | 11 ++++ .../_delta_log/00000000000000000004.json | 11 ++++ .../_delta_log/00000000000000000005.json | 11 ++++ .../_delta_log/00000000000000000006.json | 11 ++++ .../_delta_log/00000000000000000007.json | 11 ++++ .../_delta_log/00000000000000000008.json | 11 ++++ ...-8bd3-8fc6847677c8.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-886f-13de0ebc22bd.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9ef8-565385546d18.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b8cc-d95960b905bb.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a041-5f99111f6fea.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b0e9-41141cc04b8e.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9579-611eb8dc52e4.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b9c8-35b7dcdca2e7.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b133-be5982448972.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...4b71-8bd3-8fc6847677c8.c000.snappy.parquet | Bin 0 -> 478 bytes ...47d9-886f-13de0ebc22bd.c000.snappy.parquet | Bin 0 -> 478 bytes ...4e9a-9ef8-565385546d18.c000.snappy.parquet | Bin 0 -> 478 bytes ...4ed9-b8cc-d95960b905bb.c000.snappy.parquet | Bin 0 -> 478 bytes ...49c3-a041-5f99111f6fea.c000.snappy.parquet | Bin 0 -> 478 bytes ...40c5-b0e9-41141cc04b8e.c000.snappy.parquet | Bin 0 -> 478 bytes ...4aed-9579-611eb8dc52e4.c000.snappy.parquet | Bin 0 -> 477 bytes ...46de-b9c8-35b7dcdca2e7.c000.snappy.parquet | Bin 0 -> 478 bytes ...4ed2-b133-be5982448972.c000.snappy.parquet | Bin 0 -> 477 bytes ...-8cd3-9b0aadb81ae6.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-990a-aba1179b5842.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a4a6-2a4eb588aacf.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9eb8-6c714a86f901.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a578-b6a8c9d85a61.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-8b19-70cc07858d48.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-ba0e-3a188ced91d7.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9368-8d675b55f6b2.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9873-9a733ab777a4.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet | Bin 0 -> 477 bytes ...4d62-990a-aba1179b5842.c000.snappy.parquet | Bin 0 -> 478 bytes ...4f11-a4a6-2a4eb588aacf.c000.snappy.parquet | Bin 0 -> 478 bytes ...4929-9eb8-6c714a86f901.c000.snappy.parquet | Bin 0 -> 478 bytes ...4c2f-a578-b6a8c9d85a61.c000.snappy.parquet | Bin 0 -> 477 bytes ...4787-8b19-70cc07858d48.c000.snappy.parquet | Bin 0 -> 478 bytes ...409d-ba0e-3a188ced91d7.c000.snappy.parquet | Bin 0 -> 478 bytes ...4bef-9368-8d675b55f6b2.c000.snappy.parquet | Bin 0 -> 478 bytes ...4b83-9873-9a733ab777a4.c000.snappy.parquet | Bin 0 -> 478 bytes ...-a3fc-c033a86d7023.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a068-cc9558fdb9e6.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b61c-3a83d0033ae1.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9465-16a9c37e796a.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b010-68804566ce23.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-ad9d-7c4fed9e3a1a.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-901d-c7738beeb46c.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a6c4-4289d293a3b6.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a15c-23dbc9bf0a58.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...4f57-a3fc-c033a86d7023.c000.snappy.parquet | Bin 0 -> 478 bytes ...4393-a068-cc9558fdb9e6.c000.snappy.parquet | Bin 0 -> 478 bytes ...4165-b61c-3a83d0033ae1.c000.snappy.parquet | Bin 0 -> 477 bytes ...4773-9465-16a9c37e796a.c000.snappy.parquet | Bin 0 -> 478 bytes ...49ad-b010-68804566ce23.c000.snappy.parquet | Bin 0 -> 478 bytes ...4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet | Bin 0 -> 478 bytes ...43d4-901d-c7738beeb46c.c000.snappy.parquet | Bin 0 -> 477 bytes ...4fa4-a6c4-4289d293a3b6.c000.snappy.parquet | Bin 0 -> 478 bytes ...4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet | Bin 0 -> 478 bytes ...-ba7d-4e8cb41a65d0.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-afa0-575aa17fe039.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b274-7a324fc3f950.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a973-0b824d398b97.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-ae8a-2bf56033861e.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-ad0b-a30972d7b989.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-89a7-a763d8b9f2be.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-add7-a0fc2ddaadde.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-95dd-829435d27ba6.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...4174-ba7d-4e8cb41a65d0.c000.snappy.parquet | Bin 0 -> 478 bytes ...40d5-afa0-575aa17fe039.c000.snappy.parquet | Bin 0 -> 478 bytes ...4205-b274-7a324fc3f950.c000.snappy.parquet | Bin 0 -> 478 bytes ...416a-a973-0b824d398b97.c000.snappy.parquet | Bin 0 -> 478 bytes ...406d-ae8a-2bf56033861e.c000.snappy.parquet | Bin 0 -> 478 bytes ...4647-ad0b-a30972d7b989.c000.snappy.parquet | Bin 0 -> 478 bytes ...41f7-89a7-a763d8b9f2be.c000.snappy.parquet | Bin 0 -> 478 bytes ...430a-add7-a0fc2ddaadde.c000.snappy.parquet | Bin 0 -> 478 bytes ...48a7-95dd-829435d27ba6.c000.snappy.parquet | Bin 0 -> 478 bytes ...-9cf7-c713d05d5324.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-8fde-54c60a8a86ed.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a1af-750d98a7bfb5.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-bf48-809045bfc174.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-8be0-722e03e1c526.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-8419-4095f4e69b16.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-bc93-89e7f9e82c67.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a0db-848cbdfb8ee5.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-800a-3df7387e8d7b.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...41be-9cf7-c713d05d5324.c000.snappy.parquet | Bin 0 -> 478 bytes ...40e1-8fde-54c60a8a86ed.c000.snappy.parquet | Bin 0 -> 478 bytes ...4a41-a1af-750d98a7bfb5.c000.snappy.parquet | Bin 0 -> 478 bytes ...42f6-bf48-809045bfc174.c000.snappy.parquet | Bin 0 -> 478 bytes ...4c2a-8be0-722e03e1c526.c000.snappy.parquet | Bin 0 -> 478 bytes ...410e-8419-4095f4e69b16.c000.snappy.parquet | Bin 0 -> 478 bytes ...48c3-bc93-89e7f9e82c67.c000.snappy.parquet | Bin 0 -> 478 bytes ...4c31-a0db-848cbdfb8ee5.c000.snappy.parquet | Bin 0 -> 478 bytes ...4e3d-800a-3df7387e8d7b.c000.snappy.parquet | Bin 0 -> 478 bytes ...-a794-f01a7e102e9c.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b0c9-38de2f894b1e.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-8600-256b7207272e.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-bdbc-b7bb2a577f2f.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9ae2-525d65e64317.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-97bf-af12ad8b073e.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-878e-0812e8e0e16d.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-8c92-540aef5935a9.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b82c-bb75409ebf32.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...48e9-a794-f01a7e102e9c.c000.snappy.parquet | Bin 0 -> 478 bytes ...488e-b0c9-38de2f894b1e.c000.snappy.parquet | Bin 0 -> 478 bytes ...4b6a-8600-256b7207272e.c000.snappy.parquet | Bin 0 -> 478 bytes ...47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet | Bin 0 -> 478 bytes ...48ee-9ae2-525d65e64317.c000.snappy.parquet | Bin 0 -> 478 bytes ...4679-97bf-af12ad8b073e.c000.snappy.parquet | Bin 0 -> 478 bytes ...4de8-878e-0812e8e0e16d.c000.snappy.parquet | Bin 0 -> 478 bytes ...4532-8c92-540aef5935a9.c000.snappy.parquet | Bin 0 -> 478 bytes ...4af2-b82c-bb75409ebf32.c000.snappy.parquet | Bin 0 -> 478 bytes ...-9a39-7c8203afca7f.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b8c2-68e7d6414be8.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-ae6f-bb0ca971e91a.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-95d2-b802d9f602c0.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-93ed-511fa3d5ab9c.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-be2e-cd4fc74d3733.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9c15-72b4d5df3c78.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-96c9-1f26ce36ec56.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-ab45-48c06bddee6c.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...41e6-9a39-7c8203afca7f.c000.snappy.parquet | Bin 0 -> 478 bytes ...4e16-b8c2-68e7d6414be8.c000.snappy.parquet | Bin 0 -> 478 bytes ...4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet | Bin 0 -> 478 bytes ...47d1-95d2-b802d9f602c0.c000.snappy.parquet | Bin 0 -> 478 bytes ...469b-93ed-511fa3d5ab9c.c000.snappy.parquet | Bin 0 -> 478 bytes ...4050-be2e-cd4fc74d3733.c000.snappy.parquet | Bin 0 -> 478 bytes ...4724-9c15-72b4d5df3c78.c000.snappy.parquet | Bin 0 -> 478 bytes ...4ce5-96c9-1f26ce36ec56.c000.snappy.parquet | Bin 0 -> 478 bytes ...4b87-ab45-48c06bddee6c.c000.snappy.parquet | Bin 0 -> 478 bytes ...-9c96-752defe5436b.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-bf03-1195979e33a8.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9dff-bb8de1862fc0.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9735-81db2fd05163.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-819c-3c164d7e926e.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-883b-2f1725f1c028.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b3ac-499d27a94c54.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b737-6574970bc706.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b55e-a1c2fa4d87be.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...4c66-9c96-752defe5436b.c000.snappy.parquet | Bin 0 -> 478 bytes ...4023-bf03-1195979e33a8.c000.snappy.parquet | Bin 0 -> 478 bytes ...4a62-9dff-bb8de1862fc0.c000.snappy.parquet | Bin 0 -> 478 bytes ...4ec9-9735-81db2fd05163.c000.snappy.parquet | Bin 0 -> 478 bytes ...4aa9-819c-3c164d7e926e.c000.snappy.parquet | Bin 0 -> 478 bytes ...4ba4-883b-2f1725f1c028.c000.snappy.parquet | Bin 0 -> 478 bytes ...48b5-b3ac-499d27a94c54.c000.snappy.parquet | Bin 0 -> 478 bytes ...40fc-b737-6574970bc706.c000.snappy.parquet | Bin 0 -> 478 bytes ...4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet | Bin 0 -> 478 bytes ...-8a3e-b68a446f2e2c.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-886c-f5a71cd8b706.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-bcce-7b52c1e6fc0d.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a008-c7068640b8a5.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a716-9505ee9274a0.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a413-5606dee7b638.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-baae-b62451ecf8a1.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9215-6237b5f549f2.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-be03-5442a09982de.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...4c33-8a3e-b68a446f2e2c.c000.snappy.parquet | Bin 0 -> 478 bytes ...46b4-886c-f5a71cd8b706.c000.snappy.parquet | Bin 0 -> 478 bytes ...4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet | Bin 0 -> 478 bytes ...4c4b-a008-c7068640b8a5.c000.snappy.parquet | Bin 0 -> 478 bytes ...4109-a716-9505ee9274a0.c000.snappy.parquet | Bin 0 -> 478 bytes ...4c30-a413-5606dee7b638.c000.snappy.parquet | Bin 0 -> 478 bytes ...4fc9-baae-b62451ecf8a1.c000.snappy.parquet | Bin 0 -> 478 bytes ...4e74-9215-6237b5f549f2.c000.snappy.parquet | Bin 0 -> 478 bytes ...454b-be03-5442a09982de.c000.snappy.parquet | Bin 0 -> 478 bytes ...-b8f3-fad035f29300.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-bb01-c527e1978199.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9e8b-d5b26d20227a.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9bd8-6a1ff65e9835.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-b848-e223015a10d4.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-87a7-da3c10a235de.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-8b69-abb1f4aa86f9.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-ae0c-09d2db85ed99.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-85bb-83aeaa5c63a1.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...4d08-b8f3-fad035f29300.c000.snappy.parquet | Bin 0 -> 478 bytes ...4f95-bb01-c527e1978199.c000.snappy.parquet | Bin 0 -> 478 bytes ...4058-9e8b-d5b26d20227a.c000.snappy.parquet | Bin 0 -> 478 bytes ...4c11-9bd8-6a1ff65e9835.c000.snappy.parquet | Bin 0 -> 478 bytes ...4940-b848-e223015a10d4.c000.snappy.parquet | Bin 0 -> 478 bytes ...46e2-87a7-da3c10a235de.c000.snappy.parquet | Bin 0 -> 478 bytes ...46d7-8b69-abb1f4aa86f9.c000.snappy.parquet | Bin 0 -> 478 bytes ...45d3-ae0c-09d2db85ed99.c000.snappy.parquet | Bin 0 -> 478 bytes ...4d50-85bb-83aeaa5c63a1.c000.snappy.parquet | Bin 0 -> 478 bytes .../io/delta/core/internal/TableSuite.scala | 13 +++- .../core/util/GoldenTablesGenerator.scala | 17 ++++- .../src/main/java/io/delta/core/data/Row.java | 2 +- .../io/delta/core/internal/SnapshotImpl.java | 6 ++ .../delta/core/internal/actions/AddFile.java | 52 ++++++++++++++-- .../core/internal/actions/FileAction.java | 4 +- .../delta/core/internal/actions/Metadata.java | 2 +- .../core/internal/actions/SingleAction.java | 14 ++--- .../java/io/delta/core/types/MapType.java | 12 ++++ 209 files changed, 246 insertions(+), 37 deletions(-) create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000000.json.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000001.json.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000002.json.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000003.json.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000004.json.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000005.json.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000006.json.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000007.json.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000008.json.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000000.json create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000001.json create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000002.json create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000003.json create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000004.json create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000005.json create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000006.json create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000007.json create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000008.json create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-8429619b-3453-4ed9-b8cc-d95960b905bb.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-94c8da2e-b2e8-4aed-9579-611eb8dc52e4.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-8429619b-3453-4ed9-b8cc-d95960b905bb.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-94c8da2e-b2e8-4aed-9579-611eb8dc52e4.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-322e51ba-0754-4929-9eb8-6c714a86f901.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-322e51ba-0754-4929-9eb8-6c714a86f901.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-6b5109e8-5533-4165-b61c-3a83d0033ae1.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-a981598a-c92f-4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-6b5109e8-5533-4165-b61c-3a83d0033ae1.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-a981598a-c92f-4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-28fb675f-e269-4174-ba7d-4e8cb41a65d0.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-3b7a4158-7b4c-40d5-afa0-575aa17fe039.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-43e6640a-5f8b-4205-b274-7a324fc3f950.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-57289bbc-8e73-416a-a973-0b824d398b97.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-71d28824-3adc-406d-ae8a-2bf56033861e.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-74334a72-5808-4647-ad0b-a30972d7b989.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-8b04e895-6377-41f7-89a7-a763d8b9f2be.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-ad757130-9a3e-430a-add7-a0fc2ddaadde.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/.part-00001-b4c08c7f-fd3e-48a7-95dd-829435d27ba6.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-28fb675f-e269-4174-ba7d-4e8cb41a65d0.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-3b7a4158-7b4c-40d5-afa0-575aa17fe039.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-43e6640a-5f8b-4205-b274-7a324fc3f950.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-57289bbc-8e73-416a-a973-0b824d398b97.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-71d28824-3adc-406d-ae8a-2bf56033861e.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-74334a72-5808-4647-ad0b-a30972d7b989.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-8b04e895-6377-41f7-89a7-a763d8b9f2be.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-ad757130-9a3e-430a-add7-a0fc2ddaadde.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-b4c08c7f-fd3e-48a7-95dd-829435d27ba6.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-76cb0626-0e86-42f6-bf48-809045bfc174.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-830b2bc8-0ab3-4c2a-8be0-722e03e1c526.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-b8aa7c7e-d3b4-410e-8419-4095f4e69b16.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-76cb0626-0e86-42f6-bf48-809045bfc174.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-830b2bc8-0ab3-4c2a-8be0-722e03e1c526.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-b8aa7c7e-d3b4-410e-8419-4095f4e69b16.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-0d65d9b6-38a0-488e-b0c9-38de2f894b1e.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-5a0c01cf-07af-47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-0d65d9b6-38a0-488e-b0c9-38de2f894b1e.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-5a0c01cf-07af-47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-815f63f2-21c2-4e16-b8c2-68e7d6414be8.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-a05e7f3b-1548-47d1-95d2-b802d9f602c0.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-a2b765af-31d9-469b-93ed-511fa3d5ab9c.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-e7557908-e149-4724-9c15-72b4d5df3c78.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-f86461d3-6066-4b87-ab45-48c06bddee6c.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-815f63f2-21c2-4e16-b8c2-68e7d6414be8.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-a05e7f3b-1548-47d1-95d2-b802d9f602c0.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-a2b765af-31d9-469b-93ed-511fa3d5ab9c.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-e7557908-e149-4724-9c15-72b4d5df3c78.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-f86461d3-6066-4b87-ab45-48c06bddee6c.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-07da6a7d-231d-4c66-9c96-752defe5436b.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-35842ca7-408d-4023-bf03-1195979e33a8.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-5e20ed9a-4496-4a62-9dff-bb8de1862fc0.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-60f06dc7-46d8-4ec9-9735-81db2fd05163.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-89424aa4-1fc4-4aa9-819c-3c164d7e926e.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-908e73f4-4be3-4ba4-883b-2f1725f1c028.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-a5f8c5df-41b0-48b5-b3ac-499d27a94c54.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-a640cc87-30fc-40fc-b737-6574970bc706.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-d4895419-3e8e-4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-07da6a7d-231d-4c66-9c96-752defe5436b.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-35842ca7-408d-4023-bf03-1195979e33a8.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-5e20ed9a-4496-4a62-9dff-bb8de1862fc0.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-60f06dc7-46d8-4ec9-9735-81db2fd05163.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-89424aa4-1fc4-4aa9-819c-3c164d7e926e.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-908e73f4-4be3-4ba4-883b-2f1725f1c028.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-a5f8c5df-41b0-48b5-b3ac-499d27a94c54.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-a640cc87-30fc-40fc-b737-6574970bc706.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-d4895419-3e8e-4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-0e36beae-e5e5-4c33-8a3e-b68a446f2e2c.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-341a1715-9b26-46b4-886c-f5a71cd8b706.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-39d574e1-c089-4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-4ee45e83-7063-4c4b-a008-c7068640b8a5.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-cc14211d-2fd6-4fc9-baae-b62451ecf8a1.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-db27f739-561b-4e74-9215-6237b5f549f2.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-e6b34e43-1eda-454b-be03-5442a09982de.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-0e36beae-e5e5-4c33-8a3e-b68a446f2e2c.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-341a1715-9b26-46b4-886c-f5a71cd8b706.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-39d574e1-c089-4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-4ee45e83-7063-4c4b-a008-c7068640b8a5.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-cc14211d-2fd6-4fc9-baae-b62451ecf8a1.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-db27f739-561b-4e74-9215-6237b5f549f2.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-e6b34e43-1eda-454b-be03-5442a09982de.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-57ec9caa-1b45-4058-9e8b-d5b26d20227a.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-f847a367-cc91-4d50-85bb-83aeaa5c63a1.c000.snappy.parquet.crc create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-57ec9caa-1b45-4058-9e8b-d5b26d20227a.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet create mode 100644 kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-f847a367-cc91-4d50-85bb-83aeaa5c63a1.c000.snappy.parquet diff --git a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java index bb53a4d50..bf57c43a7 100644 --- a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java +++ b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java @@ -1,10 +1,10 @@ package io.delta.core.data; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; +import java.util.*; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.delta.core.types.*; @@ -19,7 +19,8 @@ private static Object decode( ObjectNode rootNode, String fieldName, DataType dataType, - boolean isNullable) { + boolean isNullable, + ObjectMapper objectMapper) { if (rootNode.get(fieldName) == null) { if (isNullable) { return null; @@ -54,6 +55,7 @@ private static Object decode( } if (dataType instanceof IntegerType) { + // TODO: handle other number cases (e.g. short) and throw on invalid cases (e.g. long) if (!jsonValue.isInt()) { throw new RuntimeException( String.format("RootNode at %s isn't an int", fieldName) @@ -63,12 +65,15 @@ private static Object decode( } if (dataType instanceof LongType) { - if (!jsonValue.isLong()) { + if (!jsonValue.isNumber()) { throw new RuntimeException( - String.format("RootNode at %s isn't a long", fieldName) + String.format("RootNode at %s isn't a long.\nRootNode: %s\nElement: %s", fieldName, rootNode, jsonValue) ); } - return jsonValue.longValue(); + if (!jsonValue.isLong()) { + System.out.println("WARN: expected a long, but jsonValue is " + jsonValue.numberValue().getClass().getSimpleName()); + } + return jsonValue.numberValue().longValue(); } if (dataType instanceof StringType) { @@ -86,7 +91,7 @@ private static Object decode( String.format("RootNode at %s isn't an object", fieldName) ); } - return new JsonRow((ObjectNode) jsonValue, (StructType) dataType); + return new JsonRow((ObjectNode) jsonValue, (StructType) dataType, objectMapper); } if (dataType instanceof ArrayType) { @@ -97,13 +102,21 @@ private static Object decode( } final ArrayType arrayType = ((ArrayType) dataType); final List output = new ArrayList<>(); - final ArrayNode array = (ArrayNode) jsonValue; + final ArrayNode jsonArray = (ArrayNode) jsonValue; - for (Iterator it = array.elements(); it.hasNext();) { - JsonNode element = it.next(); + for (Iterator it = jsonArray.elements(); it.hasNext();) { + final JsonNode element = it.next(); + // TODO: parse using the arrayType.getElementType() while verifying that the element + // matches that if (element.isObject()) { output.add( - new JsonRow((ObjectNode) element, (StructType) arrayType.getElementType()) + new JsonRow((ObjectNode) element, (StructType) arrayType.getElementType(), objectMapper) + ); + } else if (element.isTextual()) { + output.add(element.textValue()); + } else { + throw new RuntimeException( + String.format("TODO implement this.\nparent%s\nthis%s\nchild%s", rootNode, jsonArray, element) ); } } @@ -111,8 +124,14 @@ private static Object decode( } if (dataType instanceof MapType) { - // TODO: - return null; + if (!jsonValue.isObject()) { + throw new RuntimeException( + String.format("RootNode at %s isn't an map", fieldName) + ); + } + + return objectMapper + .convertValue(jsonValue, new TypeReference>() {}); } throw new UnsupportedOperationException( @@ -128,14 +147,15 @@ private static Object decode( private final Object[] parsedValues; private final StructType readSchema; - public JsonRow(ObjectNode rootNode, StructType readSchema) { + public JsonRow(ObjectNode rootNode, StructType readSchema, ObjectMapper objectMapper) { this.rootNode = rootNode; this.readSchema = readSchema; this.parsedValues = new Object[readSchema.length()]; for (int i = 0; i < readSchema.length(); i++) { final StructField field = readSchema.at(i); - final Object parsedValue = decode(rootNode, field.name, field.dataType, field.nullable); + final Object parsedValue = + decode(rootNode, field.name, field.dataType, field.nullable, objectMapper); parsedValues[i] = parsedValue; } } @@ -185,6 +205,12 @@ public List getList(int ordinal) { return (List) parsedValues[ordinal]; } + @Override + public Map getMap(int ordinal) { + assertType(ordinal, MapType.EMPTY_INSTANCE); + return (Map) parsedValues[ordinal]; + } + @Override public String toString() { return "JsonRow{" + diff --git a/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java b/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java index cd6782836..67fa2bbde 100644 --- a/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java +++ b/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java @@ -127,7 +127,7 @@ public CloseableIterator readParquetFile(String path, StructType readSchema public Row parseJson(String json, StructType readSchema) { try { final JsonNode jsonNode = objectMapper.readTree(json); - return new JsonRow((ObjectNode) jsonNode, readSchema); + return new JsonRow((ObjectNode) jsonNode, readSchema, objectMapper); } catch (JsonProcessingException ex) { throw new RuntimeException(String.format("Could not parse JSON: %s", json), ex); } diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000000.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..a771bd1bbd708c1940170093f24fd271a82ce42d GIT binary patch literal 40 wcmYc;N@ieSU}Bj2KBY9EFRJu~;F(&c@^S~~<3WysQdVI`QyL%KJC)}S03lQln*aa+ literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000001.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000001.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..90931c1865bbc8476557bd78df9b87fb29095959 GIT binary patch literal 36 scmYc;N@ieSU}C6zDZ`rDazo;>=v3xwXRHkDr&*o}pUug>``dw;0Nb$+2><{9 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000002.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000002.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..d8cfcd7688fd965ad9a9bffe8cdda9fb45345530 GIT binary patch literal 36 scmYc;N@ieSU}E53x6t;f246+20O!%y_sb5)tZd3Y7Rvr&uF~UO0OGL@{{R30 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000003.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000003.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..8999cc2a1b3c20567875cc9e7e49c4a109dc4916 GIT binary patch literal 36 scmYc;N@ieSU}Dfa{`|~Vm+cQ9rj`aPx# literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000004.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000004.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..0be59fee67206ac28119574754befc92c8f9e59f GIT binary patch literal 36 scmYc;N@ieSU}A9BQdsfx0xO$BvDdN#taBNHOI7PGM6F|(Cc^s{0L6L?Bme*a literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000005.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000005.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..24adaf5b07896c2be24276c4a7aad7f393f22361 GIT binary patch literal 36 scmYc;N@ieSU}6w)_}TL8&$-Bd0%zi6Uz?tmpC2{lGt--Tp;Z=p0SWIA5&!@I literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000006.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000006.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..f2b4583cefeb2c6e5573d3d8bf8de0d82a621bd7 GIT binary patch literal 36 scmYc;N@ieSU}CU-_p_(?{Pf$~>N13it-KpPEtbsOpvtghV_y~z00}w|2mk;8 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000007.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000007.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..912ab877308d9f2fd73a395df3b3f1e975bdd26b GIT binary patch literal 36 scmYc;N@ieSU}ETAER}Uxrb@#;*tsn_;!%0Wg{f0NrNlC9SjQ6u0N#KN-T(jq literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000008.json.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/.00000000000000000008.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..da91339714032ea66ece366b5a2a7d32bb6fed3b GIT binary patch literal 36 scmYc;N@ieSU}E@?t#;+=&Rs&M7!IlAINfnMcfvSVKJIo5Z)MnX0RGhv6#xJL literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000000.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000000.json new file mode 100644 index 000000000..3677534c6 --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000000.json @@ -0,0 +1,13 @@ +{"commitInfo":{"timestamp":1680202522862,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4780"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"a4769250-e8ad-44a7-8a90-b384ac907381"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"testId","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part_a\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"part_b\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["part_a","part_b"],"configuration":{},"createdTime":1680202519496}} +{"add":{"path":"part_a=0/part_b=0/part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":0},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":2},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3},\"maxValues\":{\"id\":3},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-322e51ba-0754-4929-9eb8-6c714a86f901.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":6},\"maxValues\":{\"id\":6},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-b4c08c7f-fd3e-48a7-95dd-829435d27ba6.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":8},\"maxValues\":{\"id\":8},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":5},\"maxValues\":{\"id\":5},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-a5f8c5df-41b0-48b5-b3ac-499d27a94c54.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":7},\"maxValues\":{\"id\":7},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202522000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":9},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000001.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000001.json new file mode 100644 index 000000000..d84612a84 --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000001.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202527500,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":0,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4779"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"8aceb033-273d-4ed7-95cf-a5ef87c355e3"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":477,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":10},\"maxValues\":{\"id\":10},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":12},\"maxValues\":{\"id\":12},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":14},\"maxValues\":{\"id\":14},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":11},\"maxValues\":{\"id\":11},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-39d574e1-c089-4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":13},\"maxValues\":{\"id\":13},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":16},\"maxValues\":{\"id\":16},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-8b04e895-6377-41f7-89a7-a763d8b9f2be.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":18},\"maxValues\":{\"id\":18},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-5a0c01cf-07af-47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":15},\"maxValues\":{\"id\":15},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-07da6a7d-231d-4c66-9c96-752defe5436b.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":17},\"maxValues\":{\"id\":17},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202527000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":19},\"maxValues\":{\"id\":19},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000002.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000002.json new file mode 100644 index 000000000..b06f60ab9 --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000002.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202528841,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4780"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"408658ef-25b8-4747-b59c-553b7143abda"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":20},\"maxValues\":{\"id\":20},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":22},\"maxValues\":{\"id\":22},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-76cb0626-0e86-42f6-bf48-809045bfc174.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":24},\"maxValues\":{\"id\":24},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-a05e7f3b-1548-47d1-95d2-b802d9f602c0.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":21},\"maxValues\":{\"id\":21},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-4ee45e83-7063-4c4b-a008-c7068640b8a5.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":23},\"maxValues\":{\"id\":23},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":26},\"maxValues\":{\"id\":26},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-43e6640a-5f8b-4205-b274-7a324fc3f950.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":28},\"maxValues\":{\"id\":28},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":25},\"maxValues\":{\"id\":25},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-89424aa4-1fc4-4aa9-819c-3c164d7e926e.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":27},\"maxValues\":{\"id\":27},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202528000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":29},\"maxValues\":{\"id\":29},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000003.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000003.json new file mode 100644 index 000000000..1c8785eda --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000003.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202530099,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":2,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4779"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"c17af0ad-f0a1-41a7-841d-598c2e07d0bf"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-94c8da2e-b2e8-4aed-9579-611eb8dc52e4.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":477,"modificationTime":1680202529000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":30},\"maxValues\":{\"id\":30},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202529000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":32},\"maxValues\":{\"id\":32},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-b8aa7c7e-d3b4-410e-8419-4095f4e69b16.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202530000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":34},\"maxValues\":{\"id\":34},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-815f63f2-21c2-4e16-b8c2-68e7d6414be8.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202530000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":31},\"maxValues\":{\"id\":31},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202530000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":33},\"maxValues\":{\"id\":33},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202529000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":36},\"maxValues\":{\"id\":36},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-28fb675f-e269-4174-ba7d-4e8cb41a65d0.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202529000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":38},\"maxValues\":{\"id\":38},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202530000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":35},\"maxValues\":{\"id\":35},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-35842ca7-408d-4023-bf03-1195979e33a8.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202530000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":37},\"maxValues\":{\"id\":37},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-f847a367-cc91-4d50-85bb-83aeaa5c63a1.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202530000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":39},\"maxValues\":{\"id\":39},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000004.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000004.json new file mode 100644 index 000000000..88e28d1e3 --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000004.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202531634,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":3,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4779"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"79a877c7-781a-478f-bf55-55ddc9dd7972"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":40},\"maxValues\":{\"id\":40},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-6b5109e8-5533-4165-b61c-3a83d0033ae1.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":477,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":42},\"maxValues\":{\"id\":42},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":44},\"maxValues\":{\"id\":44},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-f86461d3-6066-4b87-ab45-48c06bddee6c.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":41},\"maxValues\":{\"id\":41},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-e6b34e43-1eda-454b-be03-5442a09982de.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":43},\"maxValues\":{\"id\":43},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":46},\"maxValues\":{\"id\":46},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-71d28824-3adc-406d-ae8a-2bf56033861e.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":48},\"maxValues\":{\"id\":48},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":45},\"maxValues\":{\"id\":45},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-60f06dc7-46d8-4ec9-9735-81db2fd05163.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":47},\"maxValues\":{\"id\":47},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202531000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":49},\"maxValues\":{\"id\":49},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000005.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000005.json new file mode 100644 index 000000000..dc0d721be --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000005.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202532814,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":4,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4780"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"e158ea6f-74b0-4f52-8db1-6dd8c2302e55"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":50},\"maxValues\":{\"id\":50},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":52},\"maxValues\":{\"id\":52},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":54},\"maxValues\":{\"id\":54},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-e7557908-e149-4724-9c15-72b4d5df3c78.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":51},\"maxValues\":{\"id\":51},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-cc14211d-2fd6-4fc9-baae-b62451ecf8a1.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":53},\"maxValues\":{\"id\":53},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":56},\"maxValues\":{\"id\":56},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-57289bbc-8e73-416a-a973-0b824d398b97.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":58},\"maxValues\":{\"id\":58},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":55},\"maxValues\":{\"id\":55},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-d4895419-3e8e-4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":57},\"maxValues\":{\"id\":57},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202532000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":59},\"maxValues\":{\"id\":59},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000006.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000006.json new file mode 100644 index 000000000..2c931c5c7 --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000006.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202534045,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":5,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4778"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"4ef8295a-eb45-4a17-8448-bbbd6375869f"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-8429619b-3453-4ed9-b8cc-d95960b905bb.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202533000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":60},\"maxValues\":{\"id\":60},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":477,"modificationTime":1680202533000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":62},\"maxValues\":{\"id\":62},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202533000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":64},\"maxValues\":{\"id\":64},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202534000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":61},\"maxValues\":{\"id\":61},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-0e36beae-e5e5-4c33-8a3e-b68a446f2e2c.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202534000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":63},\"maxValues\":{\"id\":63},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":477,"modificationTime":1680202533000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":66},\"maxValues\":{\"id\":66},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-ad757130-9a3e-430a-add7-a0fc2ddaadde.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202533000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":68},\"maxValues\":{\"id\":68},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202533000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":65},\"maxValues\":{\"id\":65},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-5e20ed9a-4496-4a62-9dff-bb8de1862fc0.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202534000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":67},\"maxValues\":{\"id\":67},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202534000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":69},\"maxValues\":{\"id\":69},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000007.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000007.json new file mode 100644 index 000000000..fd5a758e1 --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000007.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202535263,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":6,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4780"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"04ebb104-1868-465a-b295-c181ed227b51"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":70},\"maxValues\":{\"id\":70},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":72},\"maxValues\":{\"id\":72},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":74},\"maxValues\":{\"id\":74},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-a2b765af-31d9-469b-93ed-511fa3d5ab9c.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":71},\"maxValues\":{\"id\":71},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-341a1715-9b26-46b4-886c-f5a71cd8b706.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":73},\"maxValues\":{\"id\":73},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":76},\"maxValues\":{\"id\":76},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-3b7a4158-7b4c-40d5-afa0-575aa17fe039.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":78},\"maxValues\":{\"id\":78},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":75},\"maxValues\":{\"id\":75},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-a640cc87-30fc-40fc-b737-6574970bc706.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":77},\"maxValues\":{\"id\":77},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202535000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":79},\"maxValues\":{\"id\":79},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000008.json b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000008.json new file mode 100644 index 000000000..e483f13bd --- /dev/null +++ b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/_delta_log/00000000000000000008.json @@ -0,0 +1,11 @@ +{"commitInfo":{"timestamp":1680202536324,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[\"part_a\",\"part_b\"]"},"readVersion":7,"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"10","numOutputRows":"10","numOutputBytes":"4779"},"engineInfo":"Apache-Spark/3.3.2 Delta-Lake/2.2.0","txnId":"237bb77f-95a4-4b25-ac89-9f971b538caf"}} +{"add":{"path":"part_a=0/part_b=0/part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"0"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":80},\"maxValues\":{\"id\":80},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=2/part-00000-a981598a-c92f-4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"2"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":82},\"maxValues\":{\"id\":82},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=4/part-00000-830b2bc8-0ab3-4c2a-8be0-722e03e1c526.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"4"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":84},\"maxValues\":{\"id\":84},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=1/part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"1"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":81},\"maxValues\":{\"id\":81},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=3/part-00000-db27f739-561b-4e74-9215-6237b5f549f2.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"3"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":83},\"maxValues\":{\"id\":83},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=1/part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"1"},"size":477,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":86},\"maxValues\":{\"id\":86},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=0/part_b=3/part-00001-74334a72-5808-4647-ad0b-a30972d7b989.c000.snappy.parquet","partitionValues":{"part_a":"0","part_b":"3"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":88},\"maxValues\":{\"id\":88},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=0/part-00001-0d65d9b6-38a0-488e-b0c9-38de2f894b1e.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"0"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":85},\"maxValues\":{\"id\":85},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=2/part-00001-908e73f4-4be3-4ba4-883b-2f1725f1c028.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"2"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":87},\"maxValues\":{\"id\":87},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"part_a=1/part_b=4/part-00001-57ec9caa-1b45-4058-9e8b-d5b26d20227a.c000.snappy.parquet","partitionValues":{"part_a":"1","part_b":"4"},"size":478,"modificationTime":1680202536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":89},\"maxValues\":{\"id\":89},\"nullCount\":{\"id\":0}}"}} diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..81db3c02b99bc6487f95f3e563099f3d13fac0ad GIT binary patch literal 12 TcmYc;N@ieSU}6yRK9LLn5S#+T literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..e1af56995daeedc4c31c309f1a05b4bc7ca2077f GIT binary patch literal 12 TcmYc;N@ieSU}E6vTm2dU5t9RH literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..aa26ee47253e02bf623160ca761c9243efe4d317 GIT binary patch literal 12 TcmYc;N@ieSU}D%F{K*Ue6K?}; literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-8429619b-3453-4ed9-b8cc-d95960b905bb.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-8429619b-3453-4ed9-b8cc-d95960b905bb.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..ca17d05ea1ac43e0e9063b5338c5ff8b0f4496a7 GIT binary patch literal 12 TcmYc;N@ieSU}8A>*vu6G6Yc|E literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..9b6771d384d2d816efa691da73aac8ffd180e5c1 GIT binary patch literal 12 TcmYc;N@ieSU}C7S%bx)N5s(7- literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..45ffeca9444ce4289cb7196e6d1da3afc90755a4 GIT binary patch literal 12 TcmYc;N@ieSU}ESHi;V&R5cUF) literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-94c8da2e-b2e8-4aed-9579-611eb8dc52e4.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-94c8da2e-b2e8-4aed-9579-611eb8dc52e4.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..630c12fae8b9e79338f75e030ca693878a29ae9a GIT binary patch literal 12 TcmYc;N@ieSU}A`f%<}>O5fK8U literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..e625581637190b3672e605f61e86c4a3f9079b27 GIT binary patch literal 12 TcmYc;N@ieSU}ETK5qAXu5uXC1 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/.part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..a104a6a09c8ced1f14f9becb70aa46f25dfcc134 GIT binary patch literal 12 TcmYc;N@ieSU}EqR`}Pz75xWC+ literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-07a90d8f-7a65-4b71-8bd3-8fc6847677c8.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..59572a1de90d07f5421220ebdcd55d50ed24ff94 GIT binary patch literal 478 zcmZWmO-sW-5S^~I7YY^e;&N zqJP7w)aKwGmU-{Z%zH3{oBJ9CYEy@9zqg-fHYErP)BrfG6972M-QvG{);Y0E#3pyX#RqjBn*HN%eL&2Vbi?TFrZ-VSV|aSJ=QN7 zSJ~dIVo*L9XDnJ`R8zlo_%RiIL34%a+=UufRtNgA*Gl)HQ!>L^Hf_hHRvYPyrc z0O@^OG@nfn%~nyQMiHtwrd|HGkayRc&Ep2l-rnwV8aw-ZPEFK2;W_2eDvScB)zY%+ px2qi~+fD5^JYV`_)%0b3)Y4-qy_)nqzf-SRwv89~$N>!SvOf(cXJ7yT literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-5e7a1ebb-8449-47d9-886f-13de0ebc22bd.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..3dab4135189155b8a687d243390ba1e340324cbd GIT binary patch literal 478 zcmZWm%Sr=55bTUG>_G(~Gwi?~mH|ToaY%OZMg(u-Ap{S(h{$Aj5*IhGeJB!g@#fuM z@qhdWf548!lJ0;7ZdET_3z&9)9x&iqtV#W*xkkci&hO3d_h zBZUFd+qP&vn;@EHQKUx^syJq<;%}kot=8-NHP|;>yYp%6Zu7ZyS@)#pmPc6_1#Y`- rRL$?yx=M9g#&3GQ^2fU6tH!8p#!7j0<#~R$QQ?k*7kJA74Dhl)_k?D9 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-745ae599-b333-4e9a-9ef8-565385546d18.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..682edf9c0c46fdb2cbfe0ef360951a9cbf9bd0e5 GIT binary patch literal 478 zcmZWm%SyvQ6ulj5$wCC7GbE5j7_hVuhc?qUBDj+-N^#MRh)k1d4W_R=6e->K4St26 z<2U#hZoHM+ES$w~&V8J7;Et|u8x*Kb9s2$G^!Sq6BF1oR}mGfDy~K>S{V4PRc60@Qm0DCXcG8@w?KowFk*szR=T+>{}Goe-rQ;6R{;#tQ@xLlaW zQb^%}S^GDUtRsQh*O^X5ynGRJovJ{k${cTa`nuA*&yzIH{FJ+V8tN!Wc>iw07izw1 zg#niLW6^RkL$=7GNKGQ_;+l5(pCIpVw%hwHn1j9D#XNTQ`J9Gmc*1k4lPrt^r_<50 q?sw}wDZ6d$w>)3^Q`PombJEdMDZPgDJiphhS+K3F0^S zK`#9oZ>2U1XEB^}ALksnqwCu`1!_}=etx}w8#X0KPpAQK)*t|IROsWsd)7IzOyj_E zNQuR;Z8L)o5dd=c<}_KU_~{|>AN5>O0VJ;W?E_7Dj>7ZfjZd pJGHKqotE~So-h5WYWcD;Y3r$!UR`>g-)&Sa+r|sL6#zzf*&ixwXJG&U literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-8ddb3168-a63d-49c3-a041-5f99111f6fea.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..67865441441f1b3693d0502c1907abee30ac4ef3 GIT binary patch literal 478 zcmZWm%Sr=55bTUG>_G(~Gwi?~mH|ToaY%OZMg(u-Ap{S(h{$Aj5*IhGeJB!g(O>Xa zJo;_g3DF#!!_ZaTT~*M7tJ^vS>QI+{eSW_mJCq*+&hu`+`Q^IZJcW7Ty z!WjT}m__?o0OIQPQL@nS(?jAvnyID)X#R}YBn*HN?r^7MIW*6k78Lk~rGyc7!}=xb zVz0Ze7?l5v_bggsRV?J>avMlKn>Y#QGy70U zB|We!|0a@UBr*FkGpUT{FLG*99q3fs!<9&17e@3%lE#^zimDigCJGYKyIYBwo^D!U zfb?!Gn$ISPW?2;JQG{JwGga{`D0-{)`hE@e&Cc$88oRrEZe7+r>AB@m7Dj>FZW~qe pJGHJ-otE*Np0E6|ZuzP)YMZfAUR`;f-)&U5<|13Wzqlu literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-92af219c-d5c3-40c5-b0e9-41141cc04b8e.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..612aa58565941b494911c5240579b53aecdeb77c GIT binary patch literal 478 zcmZWmO>4qH5FNL+gi;CxcS#_JuuxI3i)Q_%6nZNjM0)U2N?DWDhDM|LXer{w-_TQk zj=!v9vF6Y{Ec4!*nfG8u_m2$<)TR#IeebsCHYEtpr~z=xx*WaYT7Z ziN#R1nL&qG0OH!iMY2@!+e_lT>bar>Xz`kuB=CU|%eL&YVbh}3GT_YGvy?Ex25eY1 zZu7l)#h`LD&RMj?tY`q&2-+MaBVJ9;9e+mNp7^_o@7w($b*1%MGA_6N+dW#a$< literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-e0c4cd67-c913-46de-b9c8-35b7dcdca2e7.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..2dd64b3f060e3e5bdc44edc03958e39728e249c2 GIT binary patch literal 478 zcmZWmO-sW-5S^~I<{*O5T@uJ4ELd8oOPlRCB6t%IrFiH?M7GJc29vbSha#mHe}jL- zf8o`?m&{4 zv3V$k6fT&xe;w&E6qtRP>r5o`7ZK@9`6^T9c*V2Vh2{gEW=Za4yw0bA4*ir5?p8cj z(MAeZz1&Q@XC#Z`CZ@#tXa^07iJ(AD%mCOaK4? literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=0/part-00000-f4e998f4-27e5-4ed2-b133-be5982448972.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..59e09c647386224421bacce20105ad79db7c94cf GIT binary patch literal 477 zcmZWm+e*Vg5S^}7f`}k=mjv<<7A!5)rOoz^2)-5RLlN;sM7PPd2Ggd^MUhevKf%B8 zH~b=}Qkw_&VVQHz%$x%=y1Z^spf+{r_t*QQO$nj{Y5*KJ2>|RB4)NbS>x5XQu|s)D ziN#R1nL&qG0OH! zQn+AN|8*qGP+<0Drc)6|uVSWC<*QVgw=15$Ei~`*B#kpKTZd-dT*OT5vwLRG!cl1O`w;^5E>ou#EZQ}(#3IHQK><>DRWpn@l literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..7c2d0c30bc70083a748f02dd23a7562ac3e1f68c GIT binary patch literal 12 TcmYc;N@ieSU}AXYs4fHm5<3E$ literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..b523c9143841b45baf1f4d51440d3191c343b7ea GIT binary patch literal 12 TcmYc;N@ieSU}Bh7c<(j<6T$=g literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..d714bcb29b54789c4594f941a30a87d00681cb8d GIT binary patch literal 12 TcmYc;N@ieSU}A8uKJWzq5`hD; literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-322e51ba-0754-4929-9eb8-6c714a86f901.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-322e51ba-0754-4929-9eb8-6c714a86f901.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..f683eb8ef3a1b126cfbb5bf3e07287866d9d4f0b GIT binary patch literal 12 TcmYc;N@ieSU}A`raV`Y_5HkX3 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..2ad0dc7510d098e38f2a900a09f40f3e2ef77771 GIT binary patch literal 12 TcmYc;N@ieSU}E^fw|_4H6SV`m literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..d247646b98fe0052133ac5dd83dfc7e6b015efbc GIT binary patch literal 12 TcmYc;N@ieSU}89Yc}_3@6p#bJ literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..4f5424539d972f9466f791631f27c82f7e6a26de GIT binary patch literal 12 TcmYc;N@ieSU}E6e$M_cj5a|O0 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..452aa67c0a3a4d4b1a404de0a471f4b467a96cd5 GIT binary patch literal 12 TcmYc;N@ieSU}6xu*;E4n5z_-C literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/.part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..dff80c2de90ffa85f0a693eac49a0943c336f51e GIT binary patch literal 12 TcmYc;N@ieSU}Bi|c*+p~6m0|y literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-2c21e300-fc96-4eaa-8cd3-9b0aadb81ae6.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9996855df482000dd73411dcc81185c6915e5cfd GIT binary patch literal 477 zcmZWm%SyvQ6ulj5$wCC7GbE5j7_hWZhc?qUBDgEkMG@&nM5f8K2J>i>RFTq+pWrX} zBmRNEd>#x@1aczq7pR#PFn;3jtUd}x6e8wmTBzK zp`^rODBH}SBP;-MeRz^BRQmjwc~3f0lmIQB6O#l!Fk;!3T`_E0wAu!gtSw6kBW%b< z731=-_fRpY?u`#DT4GkT17IHoT4s0j3UCXl7;IF=M0L|l(z#Fzg(<{uAn~HpEJ$MW zSPCgzFl+xhvSlbR`*NjoktVMq(z)_guFT1r=Wh$m2RzHum6!86p9MPfGd{Rq^H@b& zDe#frwS`GMM-;EZP)$QraZK0w-$FiEZ#EAbFmHBt7g6f$@;Ob>bcO3wr>h|Joo-jl rhSzKKrR;UI*LFSW%~Z#et!Y=!q;#9ob-jM8X4y7g;JpAa#>4&qNup<$ literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-2cfcfa4e-9351-4d62-990a-aba1179b5842.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a3b363c852b4761e60978fbbd5c3b220aab30e02 GIT binary patch literal 478 zcmZWmO-sW-5S^~I=r2PNlX7_pr=+Z)V%^u6VTl?5$1MT?2L*xu=2@r2GL0SD zmy}oxWt$mvhy@_7Umv9ll{`J9-lL8bB|wX3#3X?aj99j1R}7mLt+oLrYr|5)2)kj! zigB^uyRR5jcgA}bEio(F0kF>kEweGb1l&R@1{;$xNt)!W7~+ka*Tf8q8z! zPzotrFl+xh(q$+x`!d&=Naimh(wXvArp)n*XRiy*2RzM^+{<{KPXitLDIeUec&wt0 z6!=JQ+roJ~Lloy>s3sw*IHv3TZy_J7*6aH1NFU literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-30c7e5c3-13d7-4f11-a4a6-2a4eb588aacf.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4bac5b08f5ea6575e562a3f3c5e7a9e60fd7b3c1 GIT binary patch literal 478 zcmZWmO-sW-5S^~I4W?7YY#Gl~5 z@E7@G1gBD)gL_!!y*D%O!3?i&>lCO>9s2(L^?71bg0Mghfa3-MfP>r_{@Z7r63aC9 zC@U$k7|J#?=l}~qT)jC;7Ak&vNW4cqQv2_vk} z1|{P%+nZGk%6sFCMN74@lEDT=OjI?^c{~+rp)iH`4J4j*oCNd8 zJd{ET7tG4Pj$|1M%)VUdRK)WaG1IB?RjSO3Dy=x2S(G*d%3PUvxQN=M`<$nu#Z@t;vZ@}#D>@H@pv&-kyMcoyyQy#B^(0AHx sEo)w<)|Il;(q7Z`q&HD5Pd3JFJ(1F_OV{|MM^h*gDbzu zPw@j>dMmYAIE&$)bMC$8z#U%QHYiY=I`s4F`{T%_1YwC904GfX00)H<{@Z7r63aC9 zC@(3o7|J#?=l}~q>|P%yOBFvoB>tnGDN2AA&xlFF02r}s%dQwUEm|!DO4gR8gb{Yb z1{LEX-gUf`_&FvQFL0M5*2PXGV_ literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-4edd37c8-ddea-4c2f-a578-b6a8c9d85a61.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9ca2c35222053898ea491da1106f89ed4436249e GIT binary patch literal 477 zcmZWmO-sW-5S^~I4W><-q>7YY{0ZJY z`78W)PNg;n_pr=+Z)Vo*SIStY9gy&Spt1t?jPDjhS q->vtg?6$Sv@_gw}RNI%$aYs+2^cvFh{9dzW**0F_y#O%6!~OvN7G}Z# literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-70e9e10a-6f29-4787-8b19-70cc07858d48.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..db6488a1978ddde1cdb5ed5280f21c5b3d39d882 GIT binary patch literal 478 zcmZWm%SyvQ6ulj5$wEY-GbE5j7_hWZhc?qUBDfP5rMT!uM5f8K29vbSLy^*r-{3d+ zE&huu|G-b6ON+SH+6-yfgErUYS$8UV*F0ssev6Z|*NIwh8A z?9jfX#9}Dh%%DRo0CD~LC|#)J=^^zVb)+Z(T0A2r34CD0vMsw}*tBT14JcU~mJ&wT z4I5UBi~Zhx#h|(~-m_?lS;nj#oT;U1&bwX_n+(#_N0<=+IC3;BLiZ z6>X%zM|#^9&f^)PI1fWL2~ou{UFUxb`CzqP-><>E+1i~)iL=e;G)2=Du2Y@lLFhZ( tu9gk2*XT>x>u9g-deWP!jwf4_uAWNiHl^!&{Z`GgZM?u+0bqoe{Q)EFW{m&< literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-72c85022-eb37-409d-ba0e-3a188ced91d7.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..59f02e696523b1b0e755483d583fafdf292ee09f GIT binary patch literal 478 zcmZWm%SyvQ6ulj5%|ZmBGbE5j7_hWZhc?qUBDfP5rMT!uM5f8K29vbSLy^*r-yr^l zpWw&%C*DeJ7tUh1=bU@*IdDf;w@nJvrVjo5`1(GwDM47G2Ea*+0Kh?^hyUhTr^GUi z9omFCx;J@>QnH@rq}!3(W^S&63>9c%4rJ9r`IB+^u-5 zqKy>zNN?N1c|1cD=V7QOA*wj0>-=vaAFS5v`!$$1Tf6fpaklxKrf9mtb*hs*2z{s9 s)w1FB8ht5y9qqMUPkK|;@nmb#)l(_mrgUAe->O-*jTd+;0F3amKl>hMK>z>% literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-a742dd5c-9a50-4bef-9368-8d675b55f6b2.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..2f6c77d6d8ae962e2691c13c03b5508fadd54a1a GIT binary patch literal 478 zcmZWmO-sW-5S^}7au7l2E(zoi7A-B*rOoym5xhwcrFiH?M7GJc2Ggd^ha#mHFa80K z{wV*1|HG-&=HMQddGF24doZKx+Xe+{Q-^-Pe|{005`;(8061w905~X|;lF*>DX~mr zkMfcdi=k{YgATC(#I>8_WU1n(hs1l-b43Zz;yE!%-~%I;ZP{hRrbVk|z>&3ODPe>S z*syF|=6mysL1k~8vuKG~(GGxp6ljT!=@sA>Qa0GIgo$dV8O1Z9mI_md-$3G7$4L+^ z%tI-paKWtp>qu6i!0f9`ry`DC#9XJ!SE(|`Yo5L?HShBzjWaLhH9ie==qJ2?x8@5q z-%5dx^sX(87BfVPEDY5oL>0$$jsGp={q<&ZzX5Zwv%8qb&Mu$R5DizjPGyn>q3?7$ sTGqX8y(eY2t-Y4(NpGs!o@`D!dMc&ckgn_XnpMlT@d9rJfDvBy2OC6YrvLx| literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=1/part-00001-e45e2661-7cc7-4b83-9873-9a733ab777a4.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..49ee6c13ccf4e868f37b690de91eb983250948b4 GIT binary patch literal 478 zcmZWmO-sW-5S^}7au7l2E(zoi7A!5)rOoym5xhx}Qatn`BHLtJgZXHaRFTq)zri2q zRs03soJwsE?qQkt-psrQGrGEMP@pz-==ay>2eBzZctj0=lO_RxgTf8|n`fO8%QSXq zUs7T*lx=3vAr^qRc72@9Rr>Uhd5=0$lmIQB5t9TyFk;!3T{di5v|0unSsRuTM%aK2 z%f`ii@4jMC*%|Lyw8X4v2f*G7w8X~r5^xJC8*Es@L^ab)(wR_mg(<{uAn~lzEJ$MW zPzotrFsuJMvPCE``(mkcktQ!9(z)_guFUa@=dW|k`#j6jrI+&>p9VVgGv2>j@mNJ0 zDe#frwuMPNLliH=P)$NqaZK0v-$LGBt=IQ!Fb7+^^C)$;`J9GmxWaWRlVuS4PN$<~ r-RstSQg++gYq_5ErmF49=A@&iQo0T4x?ZnYwQL(N@KyjA;bngSETU+X literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..003ede4f4edd90836493aa0247601bbbf7802a34 GIT binary patch literal 12 TcmYc;N@ieSU}BiySaKEs5;y}* literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..f7dc46ad2748407a0ec3e9c8e9bfe151be6ad8d2 GIT binary patch literal 12 TcmYc;N@ieSU}AXmal<|U75M~w literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-6b5109e8-5533-4165-b61c-3a83d0033ae1.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-6b5109e8-5533-4165-b61c-3a83d0033ae1.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..6bca36f54d158a311c053af1d5a2b47abcc0a315 GIT binary patch literal 12 TcmYc;N@ieSU}BiZk)a0w5e@=( literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..cd843f61b4522854d1c8a33031632d3adc44f01a GIT binary patch literal 12 TcmYc;N@ieSU}9KlQn(!e5?KR4 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..16b92c304c67e4d7e1b21a0dd21758183ee38d8e GIT binary patch literal 12 TcmYc;N@ieSU}A{hx3~rX68Hmm literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-a981598a-c92f-4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-a981598a-c92f-4175-ad9d-7c4fed9e3a1a.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..e07e496218629b0c7d904873eb1123982891b394 GIT binary patch literal 12 TcmYc;N@ieSU}8x4%_9H+5*Y%r literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..ece8be75e312ff46be8c9ad776d6a20d0fc3a20f GIT binary patch literal 12 TcmYc;N@ieSU}ETT|Ii2k66OPc literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..e0ffa2b6b8ef6d9ddbc220ce21e30aff759eb2ea GIT binary patch literal 12 TcmYc;N@ieSU}Cu4vh@f66s80d literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/.part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..1d85b7ed91a1237bca8fa8b087abe72611d28f08 GIT binary patch literal 12 TcmYc;N@ieSU}CttMg9^16d(h= literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-2ee5d4ee-89e6-4f57-a3fc-c033a86d7023.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5cee94ad66561d9b91c16f8bc3b5893033991ff7 GIT binary patch literal 478 zcmZWm%SyvQ6uli=$wCC7GbE5j7_hVuhc?qUBDj+-N^#MRh)k1d4W_R=6e->K4gP>T z|HM!5cf66>E}X@1&pG$rbKs7yZyOY-O&$96`TbF{DM5Hl4S>@o0f3`I{^!p+BbI6G z(V?WoVkq0pphGMGv3qlptW^B;kob>!t|$RoJSQdz17O6mExTgav}m;qIJR<@5=Pj7 z4J*dwVeg@0P~96JShU2fXa~UF3$)C}^a^kbsTgcn#zd}ZF5{U{D}^biy5Lt7DZ|jp^9VL<$nu#f3w}*Z^0bw>@Mc9v&-i+M8gxFQ=MdC6gZuZ smUX{d?@8HhYrp0B(x0lfFPoE&o=WL8r04m)X3er~yue!lV1$?b0qMeJ%K!iX literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-3faf749d-1770-4393-a068-cc9558fdb9e6.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0419b626fd3bbe3e12900db17ed450fdac4cf593 GIT binary patch literal 478 zcmZWmJxc>Y5S^@vTp@yxU2fruW5JL>T$0;-BZ8fz2*Dzah-~gQ@$m9>ABu!D{st>M zJO7UV$ce%<%fQPJueqr5|6P-vx&fgvZnZIBgIBILa~nw~wC@Zd-eF zC@JB{a)()Tux%r*-kc;$9X~xJ{-c>`N`U6iiA}-)7~u|gN|r?z=&mAb_Jb^R@;D*wXIUZ2ph6d#kk0I zbHt#!H_oeQiCNJOfPEHdncdP$KowFk*r<$&>ZZAfXF@F%rVzh@#EXuTU=f*zQb^%~ zS^GDUtU`g=SD8*lym%FJoho0Y%Dh|i^lhp6fG263c`2{+X`n+t;e-1%kJNnI3VbZ@ z#==E3Ll$LWs3swHaZT6xpCBKsH=BnInAbbI^Lgy-@;Ob>bcO3wCs`2sPPeON!|OHr nQuaF9YrCHGrmEw~)}*VaQo2p)x?aCkvuqnL@KFF5<6(aQ_v>a2 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-9d53f304-e11c-4773-9465-16a9c37e796a.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..bcd62522e2488816ba8fc7e6893384c116082ad3 GIT binary patch literal 478 zcmZWmO-sW-5S^~IfShK(Gs(w9RPbT&@vm+OTaCpVz7Q06S<~2PbNYw6s8crfyA>;(r7+2 z52cX81GDz8BVEPN@lVaj`VYd%xc ztrUew@7m(|Y=UUEieoj3QN=Os^1p?=x87{-H(=iE?9Qi&v&-i+M8gxFQys0MICR== sE$cz2-j%Y`(m~S;WH45(K>DM$9!u#pq~`@)zh>DsUf`_&Fu=?H0LrvyAOHXW literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-a40544ec-ee09-49ad-b010-68804566ce23.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4f57fab70e471aad4045babedf3389b5d573edd2 GIT binary patch literal 478 zcmZWm%SyvQ6ulj5$wCF8GbE5j7_hVuhc?qUBDgDEl;WZr5t$~_8cbhFsz~X^Z}30- z3xC4D@C&?^+AN&KaL+mS-gDp%Z*FT8s7)RE{qgyAYEy!+Kn;MiIst&ATo3=lvoU9n;CR~1t4~NC&^O9PY;RzsAq~2p!o}8k}v>9EZee6hE4NU!+?UdV<};T z_1U0gTxEN+ib45coUv$$So5wO uR!hsO->!C~Y&W&v@O;a z$NDAXD%+b?49W-Nj73Y#igp0(Jx7adM6Ut2kdndrMNH(H<|3X7wN#iw{00)wI!?mH z+&q>-3J=W6zm8-T3CzA)>r}*x7ctYR3RJ4h(T1n5OU=7HN#nJja+gm+9R&&R-fj3? z&2~~4AiZyk7V{~h`8tZ!I6@W2w9Ee%^6qB4z2Ab_+uL2vVrQSvsfn5=Jf}Qfhf(0P tT3S~9cC{mAyQ%$#=SzR0n!c=$TY4g;SCgLScj^_(w($aQIe-CP_6Ih$XrllC literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-b2490b57-55c4-43d4-901d-c7738beeb46c.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..758bcd8a6e627f493c3246ab7072592b23b5c098 GIT binary patch literal 477 zcmZWm%SyvQ6ulj*WFdmk84}1M3|LyIL!0RvDYz@rMGd^16&sUogL`T#BIB609I4BJ8-#+Vrj!Y)(3QDy7?yuIu%hRm-;V0`CQYF&_2@4wh$9 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-c37bb2f2-f67b-4fa4-a6c4-4289d293a3b6.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7b8bc3ddd7de550d3f9e25e53ca2bdc72b00952a GIT binary patch literal 478 zcmZWm%SyvQ6ulj5$wCF8GbE5j7_hWZhc?qUBDfP5rMT!uM5f8K29vbSLy^*r-{7D4 z34V^?H+WmBT{w&3oO2)Nz#U!PHYre>I`sSVbiDeqj z79-n3N-Tzg%?vt30LbgtN9jT(PY6Z9vJ|sFX0m zZrHG5TUgp>>FTMJZd1Ch*KgG<+r|sL6#zzf*&pA#XAJ-V literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=2/part-00000-c4a17e64-ac6c-4dbb-a15c-23dbc9bf0a58.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5f7af33f817d89bfee5d6d3b66fa7935aaf9f5ef GIT binary patch literal 478 zcmZWmO-sW-5S^~I=1>HoyCjf9Sg^DZmp0pPMDQj(l;WWm5!oi&8ce^ERFTq)7yp3& zz(3~S@a9x%b8rvKy!U42J(%J3ZJh$OsY5@%-oH<5N)Q&P0dU$N0C1Em;=g;=8L>>` zfU=Sji=k{YgAT9&#O_ToS*rNyA@LveOi= z8TgJTR;OI+9f+F#Bq)QxPv-#7w6uP^mIU8=k%{HSh5xjn{t4T|NnQ6ePTNx8ZX& z+eu-7^u8@x%%_Ov>nKv=2vr=@F8^D|dzxqd?=RukRz95`-mc0Gu=l02~x5_-~$dN-WdZ zp?yh-#Zb1HK?hg>;`;S*x=_i}L+U^3SWyDBct%VT2Ed4ATXw~;Y0+vKP_i~GC5*5e zHmDdE`@Q>$L3L-mXVDU~q8$KxFVHd@(M!NBq++l^857k_bDm6vS}05*eglbTouuJ> zW*$l*g$HKsUq`x(1ZH36IuptKMZ`K&fy$IQTJh|4p?RODS(5u1uk%T$qafw|yA_|Q zcq4@Y(%ZIZKAR$%-=va@2}SD`!$$1Tf6f(aklv!SGb<=oa#6aqrmBO sv~2j@Mo-FaTl+1~m;OYxec2p$^h8R}m7eGKnl;O|@d9rJfFWM?2T8nU!~g&Q literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-3b7a4158-7b4c-40d5-afa0-575aa17fe039.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-3b7a4158-7b4c-40d5-afa0-575aa17fe039.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e794b329406d274e927f46e6a2436b836115bbcf GIT binary patch literal 478 zcmZWmO-sW-5S^~I1Q9{#E(zoi7A!5)rOEai5xj|qQatn`BHLtJgGoQ~p-AaP|AK!= zf0SpZQrm-jSmwPqGw;C+uWnrm)TR!7|Ni_svME7Wq6WZ8lK{X$VSxYUS*OG@jUC#T zlvoU9n;CR~1t6|pAEyhIJUyiTqmC6NK#OO@Bw+xIShi(X44W3MmH{Pe!&1TsyJ3Th zak1aKuNYK!#(NemF)P{uu#W;Qvk|=n+(Ie_8@xY#T4|Rsa~{Wq$w+t7rKD literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-43e6640a-5f8b-4205-b274-7a324fc3f950.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-43e6640a-5f8b-4205-b274-7a324fc3f950.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e7b6e141de2f8dba8d076325b85bd105df715c2b GIT binary patch literal 478 zcmZWm%SyvQ6ulj5$wCC7GbE5j7_hVuhc?qUBDj+-N^#MRh)k1d4W_R=6e->K4Ss@O z;0O5^?!A%PE}X@1&V8J7;Et|u8x*Kb9s2$8^=a6YAT3b?;Iv5q;HY5ZpFiu2Sf;UO z2S|yWj3Z)fGVV7uwfY!xu&^{XF{zMrVzh@#Iuf*aJev# zrI5k{v-WQySw{l1uQQ#Bc=;mcI#q#6l{wz<^mV0qpC@UY`6+k#G}KX$@c!L~FVsA5 zg#ng#W6^RkL$=7GNKGQ_;+l5(pCIpVw%hwHn1h|&#XNR)`J9Gmc*1k4lPrt^r_<50 q?sw}wDZ6d$w>)3^Q`PombJEdMDZPgDJiphhS$2M)w*tTjFZ%;(3TDOt literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-57289bbc-8e73-416a-a973-0b824d398b97.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-57289bbc-8e73-416a-a973-0b824d398b97.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cc0eb7d4590a80f81fd8237cf6a2f32a48a091b7 GIT binary patch literal 478 zcmZWmO-sW-5S^~IR z`6K*0PNg;n_pr=+Z)V%^u6VTl?5$4vqN2L&Jh?XymaWg2^w zmy}oxWt$naj|CuhuaDA&N}e9l;89N%B|wX3#3WG&j99j1R}7mLt(E~LYs*r?2)kkZ zigA(e%_|1gy>ZT>C1yoC0QOO!Wj3UjfLln#VEr;Ca!qrdOoUn}Od);)iD#Xp(R^ke zN+E>@X6;``x{L*8U#@f}lKG36>P&?yQ|55Zv)6^@K2Nh`6=dAy<4DJ0%Kf`FpQ-6q zibAA!ZSj0IK{Q*%u^Ppw;+S^%-$L%MH=Fwnm^VAS^J(Jj@;MFB@Py}7N2@3folZx~ rdeE)+r0lkJ(DDKqj8!|3%~3~>rSuxo^MYQpX4y7g;H>~Kz{~ys*9vDa literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-71d28824-3adc-406d-ae8a-2bf56033861e.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-71d28824-3adc-406d-ae8a-2bf56033861e.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7beaf06ee31eabe171a0dc0a860387bb63a8ea1c GIT binary patch literal 478 zcmZWm%SyvQ6ulj5$wCC7GbE5j7_hVuhc?qUBDj+-N^#MRh)k1d4W_R=6e-=f@dx|> zKf`bGE4-E3ES$w~&pG$rbKs7yZyOY-O&$96{{2a8N)VQ)0dU$R0B}^O8L>>` zfbxWvQqKWL*hT`xuOJU@tl|>41f{Kw(N>w)1uWfpk(b>N*G}S zHmn$z`QE%@P(2vuELvh#v;$xt1zKifdIh+JR17vOVU z=CKq~cwpB4btLOZVD@#UQxPv;#9XH;P^mJ<8=k(dH1G2yjWa*xE}w=v3KHJG+wg^& z@1!t5dfyf;7c)eQEQ-`5LKVlf%l{Vg{${(q--0>V+g;3KXP?h$h=wOTr#i{PC~!I* sE$e=_-jlN1)_%+Lr9V||Up6NlJ(bdHNYC?o&6;J~c!9S9zz8q<19vlK>;M1& literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-74334a72-5808-4647-ad0b-a30972d7b989.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-74334a72-5808-4647-ad0b-a30972d7b989.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e18ab7b9c928a65e5873317864f18103e7ca9501 GIT binary patch literal 478 zcmZWmO-sW-5S^}7au7l2E(zoi7A!5)rOoym5xhwcrFbZUh-{N>4d$awQbkHH{sIr) zz4-6^6HcYJ2lueddv9jmgBe}jG$>G;I`r%N^PSj~AUvW5z)6z;z(L^_|IM>biDeo) zv@a>K7|J#?=nxA)T)RHb<|=)9$h=1#DN2AA&xlC^9~iN0%Pt!>Em|!Dj;sw!2_tO4 zhGpYozjt3TsO*gQELvh#v;$xt1zKWbdI`9Nlnpj4VWOI8Ch1J5xxy6UH;{PNX%-~0 zc_@VxE|}GS9oZrjn0>L-xk!^25$Rm{Dp%%s#q-y>=6#;!>C(%2jZXs|`Wf%vt$3`W zjTHDuZ`;Boo*{~tVW=h{syL=={BI%euh#4PHJF2~-FcKc+k8$#G+g02mB}&)eW%mW rvhH>3Jt@0w?X_G_dQ;W*WOLHdQz_ksbX~94tXj5>7kDcGjPSBQ@ndKC literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-8b04e895-6377-41f7-89a7-a763d8b9f2be.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-8b04e895-6377-41f7-89a7-a763d8b9f2be.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..89e161410794aeb243643b3518d7f19484798bb9 GIT binary patch literal 478 zcmZWmO-sW-5S^~I1Q9{$E(zoi7A!5)rOoym5xhwcrFiH?M7GJc2Gg`nQbkHH{s2Mn zSBgKvsnq7+9+r9U&CGi+!|U5R1!_}=etf@w9odv1EKmdBq(K1SAa{oU_F1RIGL1dT zN=ht-vds)Szyc6gZ;q3Nik}`5?@`YbB|!7%#3X?aj99j1mkgWct)>A5Ys*r?2FYxC9#7JE<)ysJCxH(Ag!k^&JW{i* z6!=K*+QNA>MHH>VP>n-WaZFeF-$LG7Z#MTEF#9{Zi&^aK@;P-;cZKVe$EzUpopxKx rn%AjyrR=n{*K|GUO;pR1jd5E~q;%`jb-iw*V%auc;4KF*#LNBws`+M4 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-ad757130-9a3e-430a-add7-a0fc2ddaadde.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-ad757130-9a3e-430a-add7-a0fc2ddaadde.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..2a2a3b685bfb8894920cd5c67625c71ee76de4c3 GIT binary patch literal 478 zcmZWmO-sW-5S^~I<{*O5T@uJ4ELd8oOPlRCA_yL&hf+NBA|l&lTZ3uSCaEH&7k`6) zz(3+I@)vk=Dz!Pdhh^S-GxHwI@ancsf!freU*De}M>Zu03)BEOX%GN7$X(;Vebyo8x4m;-`nid(<;U3DEo*F-hP9BbII1CBvq9t7$;N+Om`| z!uo7bGA^>cS;e5dH_lkJ#H?rsz}|DT$VT)Ma0@9JY*55RRnv^(sZa}rDa3Ce@vP${ zi00;@6jHcgR{nJ)%TQqURO1j;9Me_)w~+VNo6Y?O%>K^qd=@*qd`?}|UEwu$W8BsgDc!nsU9a1yShkH9c*_9{@v=WDT4(tH literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-b4c08c7f-fd3e-48a7-95dd-829435d27ba6.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=3/part-00001-b4c08c7f-fd3e-48a7-95dd-829435d27ba6.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c8161cc4f9dc1dfbe03a98f995c61f9e0ab4cc10 GIT binary patch literal 478 zcmZWm%Sr=55bTUm_8@|g8FpX~%YY$)I3zoHBZ4={Ap{S(h{$Aj5*IhGeJB!g@f-XE z5BUzi%ua~O!8uh`_f!@1;O?PDfjZQsoA1q!2j4?I;XT z-mgXT*#y}viy}RWaEjkdMf@3x?rOb$T7%u&$6Zfjci+#g$(koUw=~MaC~#XXqpE(p o+EJ?AG=9VLl|R-^U)4t~Ggiv0DbMpe^)h!HyufD;V1SqX1<`k9#{d8T literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..aac8c627536d331e2e1625678395bb22677cd47f GIT binary patch literal 12 TcmYc;N@ieSU}7*m&C(745kvyU literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..dd9a72d7a543255256b53e0961c47bd4b03b3130 GIT binary patch literal 12 TcmYc;N@ieSU}D&QaEB%U6gUHv literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..5566aef455fa994e3e5c62300cbfe8d179d0ee27 GIT binary patch literal 12 TcmYc;N@ieSU}D&q9H9UJ521^6Dk9! literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..cdd380cad044bcf3650cfbe8a0e42ed017a1119e GIT binary patch literal 12 TcmYc;N@ieSU}7-Ne9{L15#0kx literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..f34c94e74c29dd6e2c282cad1c21d53efd6a283d GIT binary patch literal 12 TcmYc;N@ieSU}C7_JK6^T5xN5! literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/.part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..af559374b68119be8d5bec089c0239905544ad15 GIT binary patch literal 12 TcmYc;N@ieSU}A9e&Uyp@5gP*& literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-39e1b015-006f-41be-9cf7-c713d05d5324.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a552c4995012f02e1a3e80fb2a89dc9b495f2aeb GIT binary patch literal 478 zcmZWmO-sW-5S^~Iv-Eio(F0kHQREwK^30^CB%1{;(xk!zZZcq-IVVG8jZNIdH}2^Vwo zSPCgTFsuJMl2s%y`)aLI5iefEOs6VPsWL|!p1v+M@9`v!*M7=fJ_&UcB)oUG;d3?H zNnwEWzAakJr-YnhN%6J_{fzxhl rS@S!!u9TgY_M4tB{fTP%vN3M!iIiSldY<2HR4v=a3%un3hIrW@PgZ84 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-5e68b610-422f-40e1-8fde-54c60a8a86ed.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6e1e96ea0e9894cf0c4a97321b3b9e4a16114cdd GIT binary patch literal 478 zcmZWmO-sW-5S^~A4W?iDP^9$Y$sgb^ z@rQZ$=2U8Pa1YD8_h#ljn9=oZg95dwL*GBYzDqVG2#=`&aM~mQa8wxJzkAjhu}tHD z@{$sZp=>jQ4zU2l?#)TEQt{J6;y>!Sq6BF1oR}mGfDy~K?22L2qSZ3s*xIp_Fv13G zSTQd1y?Mo;dN9sew8X4v2f#iGw9Llz3UCXl7;IR^M6PKrwdT1ld{{^e#`TvKUHmCHYXiDmC|cS&+~iDnq}K~fwuy{2rv5s9-L?W literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-75d9907b-1b87-4a41-a1af-750d98a7bfb5.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..fba348aa49895911ead606344964b3d637bd47b2 GIT binary patch literal 478 zcmZWm%SyvQ6ulj5$wCC7GbE5j7_hWZhbGfEBDj+-N^#MRh)k1d4JLi$p-Ab*Z}3~h z&+u3L18=3a3uiIhbI!f@9Jr(FTbBa0sY5@$zdngg3BnRJ08X0(0FDYp{I}0KBbI6G z(V?WoVkq0pphGMGasB2bU8>~iA@v`1tSA9mJSQdz17O6mExTgav}m;qC|O&U5=Pj7 z4J*dwVeg@0P~96JShU2fXa~UF3$)C}^a^kbsTgcn#zb|~TqHB0mI_md-$3G7Cuz8t zo5xZ};elEE*O9Iwf!SBN&P1|!5wXrxpfY8S*F1Y&YToB*mgIiM>wFsOC`ft#Zq4T^ z-b!JB^sX&h%x8$^c@(Khges2dI{#b9`|Hi-ego!UXLk`N&Mu$h3fB{!Q=Q~t6gZuZ smJPq#=tgUf`_&Fv83J0DB;29smFU literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-76cb0626-0e86-42f6-bf48-809045bfc174.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-76cb0626-0e86-42f6-bf48-809045bfc174.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9e3ebfc9a8e6aee641d789383cdf15edb2b5e41c GIT binary patch literal 478 zcmZWm%Sr=55bTUG?m+}0Gt9ysmH|ToaY%OZMg(u-Ap{S(h{$Aj5*IhGeJB!g@#ftR z^cVaAu_G}#IESIDs=KS82UoXs3e=%4{r>!VKXNERSfCcbNrM2uL5}0UdG;x>xwS+4 zk`kMt>@bV=u>i!?>*Hjh77~3DEo*;Uo-z5!G&_42$QL qd)<0Z%5GbGE!UIYRJA?XoOJY5O1B|h*XuQ_mTlt&-UK4c+-g z{)FG+t<+}WEQWi|x%Zv}cXWH-pg?Wv(B1dvSIMRX;VCr$&YJ`PP6{0V{j)BJWg176 zmy}oxWt$mvhy@^a@6M8yieH`*|5?u!B|wXp#3W$=j99j1R}7mLt(F0&)}Ez=5jJ4M zigBIq%_|1gqjAonC1yoC0Jba8G8@wyz%8U=uwfY!xu&^{XF{zMrVzh@#H)^zaJeu` zQb^%}S^L+KtRsQh*O^X5ynGXLovJ{k${cTa`o7Y<&yzIH{FJ+V8tN!Wc>iI;7izwj z!T{+*TeMuv5G}GOQj-W(9MdlUTgdyH?e=jC=HOs=HIJP`KBplXp75ONBnzX!>2$QL q``vm^%5GcxEzg(!RJDECoOJY5O0OY3&+j#BmTlt&J_-OMyzCDM=VqD! literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-c5a2be0f-6ecf-48c3-bc93-89e7f9e82c67.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..383f4992b43aefca1da6d715200bd28911e74271 GIT binary patch literal 478 zcmZWm%SyvQ6ulj5$wCC7GbE5j7_hVuhc?qUB6TNSl;WZr5t$~_8cbhFsz~X^KM-8` zG5(If;ccmA;Vg!8&V8H%cX)kUqd;xy(C@GJPs64JX@MF5r*#4VN4XyUyJwvd%QSj> zj4XqcSPTW58FYXEkh?c0$x_8n4~hS%XNnS_`Ez2DFaSm@+p4@lEDT=Oyrv8BAyDhRG32S1BquHC*fjl z9!nvG2WI78N3x0pW?!v!D&ob9nCVmnDplra!_(KL=3Sno@!C(h%O|0Zf`oVPHhivT zyHXgSd}xan^C`0VI*Qad!YYnwm;Wy0-OYA;zXh{@u)CPW&LN*u6E#nGPIpX#fBK literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-d94cd827-9a8d-4c31-a0db-848cbdfb8ee5.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..54f2c0bab53122a49ee307376815c2a832a3a5e6 GIT binary patch literal 478 zcmZWmO-sW-5S^~I<{*O5T@uJ4ELd8IOOx$4B6t%IrFiH?M7GJc2Gg%3RiyOdKM=$} z<)85`;kov!V)zAP8tLN4hk;*+h?5;%QW^V zFDbDY$~H4-9}7V2ULU6ml{`J9!K0okN`My6h)JRl7_n^2t{65gT1^8=)|RD&5q87+ z72_h`n^z2~d*hr%OU#OP0PLec%WOz50k@Ef!TM!PbQufGzFg@{B=Z+B)tL%arp)1*XRiy*dpymORgiI)k0Tw2Dev8_`Aki> zQWPS+Ym4Wz38L95j@2ke700y8{}%Gzdb7FTfO)gCJD(=bE}!EG&lkQ^9j&4`blPn# r>p`d9m9o>)LDLUpFjlQVHb!kdmeTj6?+4vR&9ZI0z*_-efS3INhHYl< literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=0/part_b=4/part-00000-fbe61411-32cd-4e3d-800a-3df7387e8d7b.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f71bcb77fb06c11c52d5f340c8666e064deafd3d GIT binary patch literal 478 zcmZWmOG*Pl5UsR{%t8bq-ORu&h6Y0dW0Oqhj|lE0ix6C7BO;xdP8^*4=A%f+#xr;s zkKiS|fp@SXF|)82O}+Q3>OH8@^=*R!wW&itzrH_8HYEs;sR3}>Bmi(!Xydbar>Xz`qwBn*HN%eL%_Vbh}3GT_+Sv6L{v z25eX{F7v&4#h`jH&RMj?tY`)Yd(E0<+jxPu0>B6_`va(dXPp25 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..92e97cef2170a98f2aca0e9658448d9d08d1f47b GIT binary patch literal 12 TcmYc;N@ieSU}CsAd&^}26z~K# literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-0d65d9b6-38a0-488e-b0c9-38de2f894b1e.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-0d65d9b6-38a0-488e-b0c9-38de2f894b1e.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..993230c2353b462bf73a3f18fdcf70bd63ee364b GIT binary patch literal 12 TcmYc;N@ieSU}9Lk(`r8e6N&?v literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..1287221ea7b8d761a65d8ee71e3c7a5d1a01010a GIT binary patch literal 12 TcmYc;N@ieSU}A7hIrta=5+wtO literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-5a0c01cf-07af-47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-5a0c01cf-07af-47f4-bdbc-b7bb2a577f2f.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..36fa13a54412cebe7bfea1f53a7f387189cb94ef GIT binary patch literal 12 TcmYc;N@ieSU}CtxTJ9(S6aE9J literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..c5bc67f65d47d936171ce79217a4563ef69a3168 GIT binary patch literal 12 TcmYc;N@ieSU}Bh(cmE6k6Ql#^ literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..750654f01d71087ee00283ea650340d1a6908546 GIT binary patch literal 12 TcmYc;N@ieSU}9LUU9bQE5-kHE literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..d316f76fdee71fe0d680743762807a2ad6f1509e GIT binary patch literal 12 TcmYc;N@ieSU}A`7c1i*O58DDg literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..501fd4eadf97c169f32a29f3e8e5523f99df8548 GIT binary patch literal 12 TcmYc;N@ieSU}A_}xF{O{5{3gt literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/.part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..a1951c5808837c2490d68fa389393f53520c0174 GIT binary patch literal 12 TcmYc;N@ieSU}Ct*`R@z>6UPJR literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-096aaf3f-1146-48e9-a794-f01a7e102e9c.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9b0ea7ea6fe9fed77cd698e3770d702e6759fe8a GIT binary patch literal 478 zcmZWm%SyvQ6uq6+l7$FDXGkE6Fk)#T4sE7yL~tivl;WZr5t$~_8cbhFsz~X^Z}1QN z1^>mLapA4hX5lP`d(OG{o&$GyeOsqMZR*gkkI(PCO$ovxH2_W;1OW26GyHeYIwh8A z98gwLVlk9$X3zl^fY`k`PL?WudPw|7JyVnbEu0gRgaI&O*_K^4Y+A6I1{AFwO9>;a z&jw}VGTWO~3@QiXj73Y#igp0(Jx5DyM6Uq1kg~xBB~0X+<|3X7wN#iw{00)wI!?mH z+&q#(3J=Wczm8-T3CzA)>r}*x7ctYR3RJ4h(T1n5OU-*cN#nJja+gm+9R&&R-EH_> z&2~~4AiZyk7V{~h`8tZ!I6@W2w9Ee%^4?~4W?g7sz~X@-{3#; z;=k~BcylVXIk<;q-g`6i9?bCSrcQy{)S=%WpI=3r5`+b602CVp0FH9E`0t)|N-Wbj zpsb|CVkq0ppaU!bv3q@zELHsUkob>!rYHfLKO-gy17O6mExT;kG;cKxC|Em|5=K~` z4a&wvwl}L7R1U@&i+H zc`St#9+=gC9my&Zn0>X@sfZUZVy06Ss8pGw4NqT}n)i5;#%n+2E}w)t3KHJC+wi%X z?W8b3dfyf;=2JxTbrh*_ges0{m;Wu~z0G!ezXh|uw>zK3&OV=07j;i~PG!6fqrhpm swXFG_T35h($ literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-26b88480-df5d-4b6a-8600-256b7207272e.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e4412d13f0c34e6c5861d22fee84d4ecea3a7b4c GIT binary patch literal 478 zcmZWm%Sr=55bTUG>_G$}Gwi?~mH|ToaY%OZMg(t?LkJ#n5s}I6Bra}V`%on0;x~Bp zPxM#(7CRD?gL4?Vs=B)hdU$m94Q^?;y^4Y{mxR~3= zN-F7rUHR9MtRjioSD8s=ym*l_lj=aH+8(V%`noitCz3SI{8UuMBs5Wwh~C{=%=K(5 zg#psLwrDY*BARDWq{k7eIA*HiZ=vX|H=Fwn*!`W|#VmGr`P{m!d(v~u<1CB zlvoU9n;CR~1t6~8ohAzvzdR-0vz{qRfaWiWNdg}jv24pO88*#ZO#=$nj-`YV)@Or~ zah>hWDhB0)amJ!0W<@&y_L-waHljCxTS&=ZgCZuXn&v#73bjy}Li`32uR2bGd1M|- zA%zQOEm`J0&ORQW1Z=4j2+_l4#?o}}^0OL>(~0v-AZ?>($}q-HxQ z@R8oPh4W~NC|ZS~8i%Ojn6C1_g}k@kY#ujY_V;#Iv)I|^bLyh*3fC!*S3&4I?Y5RR quT$$v*=cF7>3Y(esFo)iDHy|dfi6FvTeM;M-E_!m;C|6>SfCS literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-5c5ae31e-4a15-48ee-9ae2-525d65e64317.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cf26d1f524f0f368100335526c78bc636de662f7 GIT binary patch literal 478 zcmZWm%SyvQ6ulX%WFdmk84}1M3|Lx7hc?qUBDj+-N^#MRh)k1d4W_R=6e->K4Ss@u z;4iuJC%l!~E}X@1&pG$rbKnlIZ|fAOLtXm${qzm9Yj3(UUCbtaO#e9lrp2x8o$Ef0%_W0jI-dk@r_ZzVKJG<&Ead-LLx~TiYcgy2EibJ>E s*0L6KYF#NiEgdxdKn4@l3S?v4))OiHy7c{^+o;%%gBN%+0ET$k9{}KIGXMYp literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-74eda70a-a1ff-4679-97bf-af12ad8b073e.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..3950f6024b2c62dec90d5274f6832d940a843dff GIT binary patch literal 478 zcmZWmO-sW-5S?zT#b?E!o_easD1mQ6?08X0(0Ez`4|LwERh-Dgk zbSNpY7|J#?=l}~q?B1NDOO-r5q`{+}DN2Br&WTB)5E!v+%dQwUEm6UVD?q6Gm$J_#7t)@RGBhIYo5I>HSh5>OY$J&E}uj?4pZK{Tl2Y^ zZKWtgde;^&=2JxTJdV{kMis}j%l{Vg-g>jS-+3Ko7S+i^#FYs0X7~*Ar05Ha8&;S4c literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-b51d842e-ab5d-4de8-878e-0812e8e0e16d.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7f05f7f1a19dd135deedece085bac2ea8dbcfc54 GIT binary patch literal 478 zcmZWmO-sW-5S^~A4W?iDP^9$YS#SPT z&pMUb9Nfb)@4cCM4`y_I+n_*g>d>$E?~kHQ3BqG)0Gu`n02~!c`0t)|Ml91fpuD8S zVkq0pphGMGv3qlptW^B;kob>!t|$RoJSQdz17O6mExTgav}m;qIJR~yC5*5E8&-_V zd~aScs2+@S7A-L=+5xc70xh#Ky#m}qDh3;tF_CMU%XlW#N?{7|8%R9sI0=^vvm}KS z9+Po?x4()0XYvu4>gUf`_&Fv83J0827v1poj5 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-bac4eaf8-ce48-4532-8c92-540aef5935a9.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..57e0c1087b30f628e720340cd6e78295a56d0634 GIT binary patch literal 478 zcmZWmO-sW-5S?yo$w366yCjf9Sg^EEmnPe9MDQjRO7YN(h-{N>4JQ3aQbkHH{s#Y$ zX9fR~Q>pF2JuLIyo0<1uhF7;P1!_}=etv&^r4eIQ|4&Rv)6^@J)UOC%FlR>PeL69Dev8_`Ao%I zDGZR_wMFyU6wz!IMQR+OietLQ{}%Gzdb7FTfZ5;KoyUo@%jdYl^@QhC#;Y(2oOWBw ry5FgHrR=n{-}HRxPgKj7jd5E~r1V_rd49K1wQL(N@RkD@;$?pTFs5h- literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=0/part-00001-d607f307-1cf0-4af2-b82c-bb75409ebf32.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6bc54dd223a7594b071547ceea834d9da822d465 GIT binary patch literal 478 zcmZWmO>4qH5FNKxawtN{;`h5>Y;mB=DjyF@4<`*w+#x^rVjo1{`xH1lprin1EAO>0C18U;Jfle3dD4vf0PE z(M}3{r1x#%BAz3P*I}rpA*wj0Yy58^?{Bu-`z@Hmz1?M$IQx7~Lo{6BI+f`<2z{s1 s(X#G!>pdyEZSA#OPkJ-e_GEL~(K9LChIC!8*Q{E$jTd;!0gUmoKS~H^dH?_b literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..be0b429a0bab50f79e5a094cad00a83c537a3f98 GIT binary patch literal 12 TcmYc;N@ieSU}8vm!pjZ-5!C{f literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-815f63f2-21c2-4e16-b8c2-68e7d6414be8.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-815f63f2-21c2-4e16-b8c2-68e7d6414be8.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..a4683a5016a7b2f6a65ffe9be847f3687b11882b GIT binary patch literal 12 TcmYc;N@ieSU}8vdd&>s^5y}GK literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..1bc4211ea67d66a8e7d2bed70bcf5a30ef112e3f GIT binary patch literal 12 TcmYc;N@ieSU}6yardbOB5s3or literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-a05e7f3b-1548-47d1-95d2-b802d9f602c0.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-a05e7f3b-1548-47d1-95d2-b802d9f602c0.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..2b1eaadb4f8a6e8d596aa7c0f5698489b08987cb GIT binary patch literal 12 TcmYc;N@ieSU}AXiZsSS-71ab| literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-a2b765af-31d9-469b-93ed-511fa3d5ab9c.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-a2b765af-31d9-469b-93ed-511fa3d5ab9c.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..fedff168845cc214d5c49746bfbb123789ddf33c GIT binary patch literal 12 TcmYc;N@ieSU}E^8eMu1j6Q%=E literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..938d643c527db54c3ecaed0f13d79dffef9d2d76 GIT binary patch literal 12 TcmYc;N@ieSU}Dg3)?o(#5F`R1 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-e7557908-e149-4724-9c15-72b4d5df3c78.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-e7557908-e149-4724-9c15-72b4d5df3c78.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..b3acd7567db98523161bf26df1448048bcfd3db1 GIT binary patch literal 12 TcmYc;N@ieSU}8urbC&=B5fB1$ literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..f300d0c69f609749024007cc97cd483af552b8ab GIT binary patch literal 12 TcmYc;N@ieSU}BgQS9%Ko5~c%s literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-f86461d3-6066-4b87-ab45-48c06bddee6c.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/.part-00000-f86461d3-6066-4b87-ab45-48c06bddee6c.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..1111f628e43ab95accea5a9c956c1a45d946a16b GIT binary patch literal 12 TcmYc;N@ieSU}A_heo+Dd5zzxD literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-6562cad8-cb61-41e6-9a39-7c8203afca7f.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ea9e80477abb55054aa35b3e639e3fc392d4d709 GIT binary patch literal 478 zcmZWmO-sW-5S^}7au7l2E(zoi7A!5)rOEai5xhwcrF!T^M7GJc29u^uQbkHHUi<_8 zM8Uhiz^T;s;2xHF@6F76FvF``mjbn^L%-j@J_|M_2#=@%aMBiV~p3Gh&i307fj^vde}|i&oQsBWuG_!U*fL zLD{(2@7-4nDm&vniC zJd{ET56tSnj%*PL%)VIaT%^g1h;^<4l`C_!;`!@b^B&Lgbm`~3#wVeUf{gd>R(z)7 zjT8n*Z`-0|HbpdBMv)pvsN$He@xO(GZ1Xv;a6REUmGLr+0;k>9 rvhH{4T`4;)?KeGN`V-aiWno^Ib zg?T826fT&xe;vsx6qtRL=~Tqgiw z=36Q7k>0h1(PD;Zk%gg}gs9?}uJgZzyuaRT?l)i#c6R6U*xBWC8lvF}*QrjjAoQJ1 tN6QPZd(o4!+tyyo^`tjdZBI5Q9X*xOZAjPkdd-?;+jxPu0>B6_`vX%FX1)Lb literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-81d3661a-87ef-4b8d-ae6f-bb0ca971e91a.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7f971e798de8b0e4631855a3bc49750264c0a2ca GIT binary patch literal 478 zcmZWm%SyvQ6ulj52`GZl84}1M3|Lx-L!0Rv5!{K3Qe1Q+BGY7AgXt>|MM^jRf#84m z6aIq#d^PE&+n2=3BnRJ08W|&01gUA_-~(eN-WdZ zqr9ZVVkq0ppaU!bv3q@-ELHsUkob>!rYHegJR>Fv17O6mExTgav}m;qC|O&U5=Ph! z8&r&od~aScsP2t(7A-L=+5xbS0xh!OJa6Q3AAhMobd;z=&mAcEzx1(P|q|vNkLwjIbLv ztQZ&jz59wmb!WV1(Gs(w9RPbT&@vm-OTaCpVz6Nu6V**~p3H<=C`=)K1Bqvyq`^Ek z52cX81+(_ABVC39voCXpYp-oipMJ2 zNP&;^wk@2;GemJ7hH4U`ietLY{}%GWYQ4T+gL$*HJC71)o6l*ArYl^hI?033ce-6I r8(y!`m$KKy*c(X_-Zlhw^HeTQ@2Qb9T{s4~!XQ==H literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-bcecdda2-f59e-4050-be2e-cd4fc74d3733.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4ce8f45b48377c8207418bc7bf1a861871fe88c8 GIT binary patch literal 478 zcmZWm%SyvQ6ulj5$wCT3XGkE6FkopR4sE7yRB$ILZX5lP`d(OG{o&$GyeOsqMZR*gkkI(O-O$ovRH2{hY0su$3OZ<1wIwh8A z98gwLVlk9$X3zl^fY`k`NtP;pdPw|7JyVnb&7TpIgaI&O*_K^4Y?`;41{ACvO9>;a z&jw}VBHNo)3@QiXj73Y#igp0(Jx5DyM6Uq1kg~xBB~0X+<|3X7wN#iw{00)wI!?mH z+&q>-3J=Wczm8-T3CzA)>r}*x7ctYR3RJ4h(T1n5OU-*cN#nJja+gm+9R&&R-EH_> z&2~~4AiZyk7V{~h`8tZ!I6@W2w9Ee%^4?~$Sq%4_bMHL|?&$iqL4n%Tp>JQ`pCy|TgeTMhIBgODI4Vf|ch5Q_mT4SN zUQ%K)lx=3vAr^qxy(uLt6+b;B{-d5NN`My6iAll$7_n^2t{65gS}g-ktQ|`UBW%Ei z72`7Bn^z2~2jiSYOU#OP0PLec%WO=q0Jo5e!G>i_WE~02zRq+i;^m8&>r@3QRpxlZ)7O>eeV(Lo=BM1{(@;l2!uxj{zEJa> z6b4A|+oI)ShG>yRk(xxP;+S^%-$LHsY`6DYFb8|Pi+Swq^EnOC@Py}7Cs`N;PN$<~ r-S5_WQg++gZ+X7-r>gDC=A@&iQhE*Pd48{1vuqnL@Lm8I;bngSAiQTz literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=1/part-00000-eb0581fa-b5c3-4ce5-96c9-1f26ce36ec56.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..504f53e83d4df99c878e644a4631b99b8a5e48c2 GIT binary patch literal 478 zcmZWm%SyvQ6rGN>1QbE&3<+cr1}rVap~>`(2=2s1DK5GZk!dom!93bL6e->K4StK? z;L2U1XEEG!?&BOdldD^YAYu}We1Cp@9GL_ItPu^NlMY4*9aJjtZ=Z3B4PD!# zvLx7`&^DA)ngvO$ z9||tGi}cpNiEI^e@V+Wk&ePK4etE} zxBi1a<*n3a;Vg!G&bjxV19xd>#x?~jsA3BnU<0F;^p0FDYZ{CCefBbI3# zP+n4EF_dj)&>Po?x4()0XYvu4>gUf`_&Fv83J09E>C&j0`b literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-07da6a7d-231d-4c66-9c96-752defe5436b.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-07da6a7d-231d-4c66-9c96-752defe5436b.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..904d19ee224ee5a2d4935979b2211b134a61e8b8 GIT binary patch literal 12 TcmYc;N@ieSU}DhR+$IPB5n2MS literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-35842ca7-408d-4023-bf03-1195979e33a8.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/.part-00001-35842ca7-408d-4023-bf03-1195979e33a8.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..fdb41b63fc345efc553ec36062493782b39c7d9a GIT binary patch literal 12 TcmYc;N@ieSU}E^ObA}xN6!`4W?iDP^9$YP4MWC z@J|X(r8Womu*`dJX5NDtUEek+P@6jR>*M{qXj6jlgc<;6O#%Q%g%bX|XO)O$8V8h@ zlvoU9n;CS71t4~BPLq|2pB@tbQO^}6K#S+ZBw+xIShi(X44W3MmH{W$j-`YVHekbw zahdPUD+bkran7P8W<@&y_F151Hl|mATS&!V!!jmvO>-H~gjy*~A$|jiXB{Wua$z1z zA%zEK?O#W-js#|3XF3(}@q_%JPtrK^Q||IK4Ss_^ z;5YdbuDq4nES$w~&pG$rbKs7yZyOY-O&$92{rOe2DM5Hb4S=&I0f3`I8UNk0&WU9j z2b7nTSPW&G8FYvRAa-v~la-2}9uogi&lM#=i)CVxFaSm@+p;T$O^a5`fD>!SQo;xu zuwlix%=hLMgX+OJXVDU~q8$KxFVGSj(<{I&q++mP2@|=dxr}E*trVsZzk$TFj+1b? zFps5>!UMDRuOnGU0<*6(or-w*BIY_(fl8G*-thEwrFowxX`J~fclk8bQIPQd-G(pJ zd?$qg()+e(xtJkZWKpCh5vn+*UH-R__cz<^{T9r@-tJ-^JNtZ2Lo__$In_xPMuF4m sXj%8W^`4a7w)R_|Fa4=%`?5Ld=&6)mLwcUyYt}5=#tXa^07iJ(AJ-IT5C8xG literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-5e20ed9a-4496-4a62-9dff-bb8de1862fc0.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-5e20ed9a-4496-4a62-9dff-bb8de1862fc0.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5ae651dd95cae39b1e64a68b8fe4d7c6ff442fc9 GIT binary patch literal 478 zcmZWmO-sW-5S^~Id%)AFPy1r{rpf+{rcKiKVvME7Wq6WZelK{X`;RgTRv(AWR8V8h@ zlvoU9n;CS71t4~BPtt`-ULI5bSq@$-&dJXA$ey>@xY#T4|Q2-d>Wq$xAS_S=;G{tS;2?K_|Mpp@#4?RN z%1TNshO*5JI=})DS8tA!g^HgZ67NyZ6eU3OXT&6d4~$s0WtR+_=B=gy1#8Pv!U*fL zLCLtx_GT4>^4>UO(Gs(w9RT~t(IOksE5I$JWUxUI6ID%f9#4f@C`=)K1BquHC&4^2 z52cX81+((6BUy$5voBXV74iH<%yg=Jl`3b^8!-DjyYpG>?D9EvQFn#wl*g+e^qqEF s%bM4zb*1dIwAXY!=}lD2lZ|m(Po#9~(sjLVqhi@MUf?YUFvQFL0R3HOGynhq literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-89424aa4-1fc4-4aa9-819c-3c164d7e926e.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-89424aa4-1fc4-4aa9-819c-3c164d7e926e.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..825b5110892be3a3908b30ece8e8a08f187d801f GIT binary patch literal 478 zcmZWmO-sW-5S^~IL=ZvfE(zuk7A!5)rOoym5xhwcrFiH?M7GJc2Gg`nQbkHZ{0*M{ zJN_fjPNg;n_pr=+Z)V zO2%cjH>((w_r@8EmY5ao0N6*47TJhi0d65BgAIz9sA`(?cq-IFVG8jZNIdH}3FeV` zD1{U*n3aDW$ubm}eYw)9i03b2rc>psRGFhSPhS_B_jr=VD=+0$J_&T_C%kvJ=8>9h zrNBpe*A~vBDWYfeA_BBfiGuIqIh70b5q0`ED1Azt7YY^e_1P z#6Kc9mD(QM!!qx^nRyRpe0klZKyB*K_s_5Il1&N15;XvhTLb_O3OD#~o^?Vj)7YVX zNr}Z!wwXakSODVs)loKA>C=7YJ?cnN0g;1*Ib*r<$&>ZX~b4?@iqrVzh@#IsJbAc@UG zDWq`0to`fA7NNlGi>1y*n!Jce=gL>PGAApZzs@xu@GMW4Ue4=$7UUg(rrrD_4=)vW!rdxw*tTzFZ%=5fM|9A literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-a5f8c5df-41b0-48b5-b3ac-499d27a94c54.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-a5f8c5df-41b0-48b5-b3ac-499d27a94c54.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..fe5d23e3a8d3ea08409b791645ab1f27a00f66c4 GIT binary patch literal 478 zcmZWmO-sW-5S^~I=1>HoyCjf9Sg^DZmp0pPMDQj(l;WWm5!oi&8ce^ERFTq)e?SEP zhd;-^<5X&Ma1YD8_h#ljnBnzpodUI~L*GBXzlt^`2n*BzIBgIBILe*izkAjhu}tHD zvXT;up=>jQ4zK{k?oBaSs`%+4@gMa}Q35o7PD~O8z=&mAcG<9L-f9|9uy!mZjIcf% zl#R=5Z&op=9E>v-Eio(F0kF>;EwK^30^CB%1{;(xk!zZZcq-IVVG8jZNIdH}2^Vwo zSPCgTFsuJMl2s%y`)aLI5iefEOs6VPsWL|!p1v+M@9`v!*M7=fJ_&UcB)oUG;d3?H zNnwEWzAakJr-YnhN%6J_{fzxhl rS@S!!u9TgY_M4tB{fTP%vN3M!iIiSldY<2HR4v=a3%un3hIrW@v43T> literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-a640cc87-30fc-40fc-b737-6574970bc706.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-a640cc87-30fc-40fc-b737-6574970bc706.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c66bfe473aa982028b01342b0294c724934781a1 GIT binary patch literal 478 zcmZWm%SyvQ6ulj*WFdmk84}1M3|LyIL!0Rv5!^|^Qe1Q+BGY7AgL$+`sz~X^ZxBDm z5AYBCA8)0$3uiIhbI!f@9Jr&a+Xe+{Q-^+je0>*fN)R4V1K^}d0N|i-ga77Pr^GUi z9omXLFT4J!Iaajua(8i)X|nfe(yWwq=(Mn-;B>0Y}z`rGybS zV8gO;vERF|7*uw~dloG*E7}3D_W~`kF}(!bLdpgkmM~GxG?R2D)LdZ-@f%1y>og0J z*gTX%3Kz`kzm9AX3e3J(>RhDBi->fte3dJ6yyE%mT=PE9@^tCtyvC=24*iVx?^Zlk z(MAe@v5LF!0HU77d_gCxn{Tj@{*6ut?oozm+AsVi5oyuexguc`1 sXj%8V^`4a7w)R@CC%vg^d$Kv{=&6)$L%OclYgR4W#tXa^07iJ(AK2<=+W-In literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-d4895419-3e8e-4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=2/part-00001-d4895419-3e8e-4b4b-b55e-a1c2fa4d87be.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..b8f681650f5af2f2dbfe945fdb968058ffdabe94 GIT binary patch literal 478 zcmZWm%SyvQ6ulX1$wCC7GbE5j7_hVuhc?qUBDj+-N^#MRh)k1d4W_R=6e->K3F4>u z7lPm5t<-kmEQWi|x%Zv}cX)kUr$8O*((kYLPs5=EVUd~urwsxCM@AR_?X%B_ZJB#? zC@HZS$__K>01H6u-JGOLl{`J9!K0ojN`My5iAACi7_lAODVq*0*i93P_LilD5!Ppe zvUz#fd#D&x_QnSmEio(F0k98)me`110d65>lMPCk$g`|PG8Jm6FopOHB%XDWMvJ+1 zEQJ(4Sk-?W=_(eOeUB#Rd@)0qlYrmWGLXRk}mdpymOJjl4mCy|cBl=tq|e6D6& zDGHI^wZ)716wy47V>OOZ#WC&izlFTF-fZqSVD)!)7qi6O<#X$z?hD_ojPocC-F92c rTF|L=rR=nH(DVZtOjIk7jd5E~r1b03_k(VuYC8^I;LQLS;$?pTrRQg$ literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-0e36beae-e5e5-4c33-8a3e-b68a446f2e2c.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-0e36beae-e5e5-4c33-8a3e-b68a446f2e2c.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..b754f5fe7838cd3bfdf45c2f6a00bc16f0ae09b4 GIT binary patch literal 12 TcmYc;N@ieSU}8A_$2Jpq6rKac literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-39d574e1-c089-4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-39d574e1-c089-4ed8-bcce-7b52c1e6fc0d.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..22e7ddf68454edfd250009683c4d0d5dcb213f15 GIT binary patch literal 12 TcmYc;N@ieSU}7-8!a4x}5pn|C literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-4ee45e83-7063-4c4b-a008-c7068640b8a5.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-4ee45e83-7063-4c4b-a008-c7068640b8a5.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..a7a2ca55fa040402256693e8084ffc9367056044 GIT binary patch literal 12 TcmYc;N@ieSU}E5yxato85u*co literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..cfd927cf4cbbfc2b0fdacc0f35b6f945a898167b GIT binary patch literal 12 TcmYc;N@ieSU}7+Qvc3%f64V2Q literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/.part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..4291c0678520e4f62c64ce77e59407a63be0ff93 GIT binary patch literal 12 TcmYc;N@ieSU}C6J%|LwC*h-Dgk zl$VrP3}u@cbch8YcCU}pg-V|8Q~y!V6eU25r^F;-0E}3+WmgQF7Oj>6C2Px4!U!9% zVZ}Jl_vRIY>fShK(Gs(w9RPbT&@vm-3&1U;Vz6Nu6S<}tCl5j`6s8crfyA>;(lDNz zhf+x4fm!?4kuD>F*_SJwi6njzGo7hGWy&0{dG@-{ywB4tS@{`v`83o~kn;ZRn$Oj2 zD}@2lyS6BvKOmZ~qDW05RB=qZ{BI%euQ!{!4VZ(S-PtU0cKMu!Xn4YNs*_b11x}}< sW!>-Ads24W+HZNj^rx!r%jTq`r&4+i>3M#yS+i^#FYs0X7~y4q0PAySLjV8( literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-341a1715-9b26-46b4-886c-f5a71cd8b706.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-341a1715-9b26-46b4-886c-f5a71cd8b706.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..2f5d96564ff9271889f6b67060e21d5b69cd04e3 GIT binary patch literal 478 zcmZWmO-sW-5S^~I1QbE&E(zoi7A!5)rOoym5xi*+rFiH?M7GJc2GgcZQbkHH{s#Yu zf5m^{#oypmYIAT8%e?ny<~^9<)oq;uwW&itKEJ+?Y)TLor~z=&AOLWXyTO0^tW#o{ z#vWxQB^E>3W(FN#0f?*D$H_v)PY;RrsAq~2p!qXmlE4Q>EZee6hE4NU(}04tWhr5V z_1U0gTx5H*ia~jAoUv$$SoBIuz{hi(UEOvJJoVuvH!gb2yRS^14 uyRBu->(shZc3RqNx}NkVs^!VXxUDBrx^?NgUbj)PY#T4|mID~#Wq$yNifAw)1uWfpk!@WN*G}` zY)~;S_IvjggX+$B&!Qz}MLPiYUZ7<*qL+YMNX1}-GA62<<~*4SwNRKs{00)wI!VL% z%siAr3J=WMzm9Yn3CzCCbtaPei->im0+lIqwBp(8Li0XPvn2O3UgwigM?uQ_cPlXU5(b}<;Fv9w5 zP&O{Jy;;Seaxl(Vw8X4v2f#jZw8TdA3UCW48*EU*M6PKr;;B$eg(<{uAn~l@BwWnR zBPpctz^wl3NLG=+?5nj-MZ9223yvLI?Ui&F``6Sd)knrB!hR@Y( zCxrph`?hE?pCX#CqezV-RB=qZ{BI%eZMNI{Etvhi-Nh_+_W7K;sC&Y5D&ut+1x~xI sWzFx@x>9yp+HZQk^e3w2%f`5^CsKNK>3M#)QMGIvFYuNF7~*Ar03Neul>h($ literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-5fff390f-c1ce-4109-a716-9505ee9274a0.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..25ce726d0af19434e0294c9e3730284d36acbad3 GIT binary patch literal 478 zcmZWmO-sW-5S^~I1QbE&E(zoi7c4E*rOoym5xhwcrFiH?M7GJc2Gg`nQbkHH{sxc! z1^3W(FN#0f?(N$H_v)PY;RrsAq~2p!suRlE4Q>EZee6hE4NU(}04tWhr5V z_1U0gTxNT-ia~jAoUv$$S4db)k8WCuzL$QeNegK!<+9dv|Ld zso7Qve57}6;XIlmidJE$#v!UWrmOsKA@8j>oBIuz{hi&#EOvJJoVuvH!gb2yRS^14 uyRBu->(shZc3RqNx}NkVs^!VXxUDBrx^?NgUbj)PY#T4|mID~#Wq$xA3udSQ literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-6ff44301-7a6c-4c30-a413-5606dee7b638.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..b6b6ee5945c7328ce49f7cb3eb2230f07970ceaf GIT binary patch literal 478 zcmZWmO-sW-5S?zT4W?iDP^9!C{)GND z|BFYbQk#Q&SmwPqGw;ESZtoius7)RE`Srak+LRzXr3S!xlK?=mP{e=#tP5h9#u4Qu zB^E>3W(FN%0f^nZvt*^>m#4&k)^kM((9$I_Nf-blmTlP;!=@#xWx%PmXDMNX4cM?^ zT<3f9ib3^goU>?&S1=PX)cRQzL@tUWvOU(y7N#o2*d7V!K9r_6$+^zXS z&9_qEBfV=2qs0u-A`3$`2~ou{UFUxb`Cz@-+;70V+1Z`XV`rDoX^N&RT&Ft8g3x!m tT`e14uhEyX*U?_v^`tjd9Z$9B6_`vaWmXO#c| literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-db27f739-561b-4e74-9215-6237b5f549f2.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=3/part-00000-db27f739-561b-4e74-9215-6237b5f549f2.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a12d928cc3f87880c5f25333d675a62883a14ca1 GIT binary patch literal 478 zcmZWmO-sW-5S^~IL=ZvfE(zoi7A!5)rOoym5xhwcr6}|wBHLtJgK5(ysUoEpPyPT; z{vfaZC8tuGgL_!!y*D%O!3?i%>J+F=9s2(H{dHthg0MghfRhFRfP>sE{@Z7r63aC9 zC@U$k7|J#?=l}~qT)jR{7Ak&vNW4cqQv2_vk} z1|{Pn+nZGk%6sFCMN7S z*sx+;=6mysL3MAOvuKG~(GGyU7igJ{=@sA>QZd-DjEU-|8O1Z9mI_md-$3G7$4L+^ z%tI-paKWto>qu6i!0f9`ry`DC#9XJ!SE(|`Yo5L?HShBzjWaLhbv_Mr=qJ2?x8@5q z-%5dx^sX(87BfVPEDY5oL>0$$o&PQ5{q<&ZzX5Zwvpb*1&Mu$R5DizjPIZz6q3?7$ sT3&eFi=LF-w)R@CC%vg^d$Kv{=&6)$L%OclYt}5=#tXa^07iJ(AJr0P7ytkO literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..9ec5e06d92c4d0b5181ed246c2c7171864c714a6 GIT binary patch literal 12 TcmYc;N@ieSU}EUvjPV5k5Xb^| literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..9cc1780aaec85ba6f10f0135f1e138f6eb3ed69f GIT binary patch literal 12 TcmYc;N@ieSU}6vmIavY#5W)i7 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-57ec9caa-1b45-4058-9e8b-d5b26d20227a.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-57ec9caa-1b45-4058-9e8b-d5b26d20227a.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..77029aff4597a221d3f552ee20484e8aa389ccf7 GIT binary patch literal 12 TcmYc;N@ieSU}E4i_WB9{5848| literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..2b5666ec3a223c53eaf15a842bf6a2dee7cefad7 GIT binary patch literal 12 TcmYc;N@ieSU}E4~$nXjP5QPHN literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..9d4bb1cf9b69f54c40eeb163c6ce582770da7d75 GIT binary patch literal 12 TcmYc;N@ieSU}Bi^XX<4D6u|^D literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..dfe9d9afa150a80fdd59a54355927d712acee1be GIT binary patch literal 12 TcmYc;N@ieSU}89UX08hW6lMdd literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..85806a4874525f30763613d727f51c718c6bc3d8 GIT binary patch literal 12 TcmYc;N@ieSU}EU8{1y!W5~2f9 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..a0e7303a727ba786931110fd173d4ec1008330ce GIT binary patch literal 12 TcmYc;N@ieSU}E^!X`}%F6Q2Vd literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-f847a367-cc91-4d50-85bb-83aeaa5c63a1.c000.snappy.parquet.crc b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/.part-00001-f847a367-cc91-4d50-85bb-83aeaa5c63a1.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..e893eb8b2f3ac285b466eb682e70319343b09753 GIT binary patch literal 12 TcmYc;N@ieSU}6wB)|?Fh5rYE% literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-0775b09e-a772-4d08-b8f3-fad035f29300.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..564c878dda6e8613f0a5aa587670404a6040b7a1 GIT binary patch literal 478 zcmZWmO-sW-5S^~I4W>z(q>7YY{0;h( z{8M^yDz!bhhh^S-GxHwI==#>BKyB*K@2}5~qD=|H5;Xu$n*;!k3NHTJXPpttH1_CF zQerWbZD!CR7J#^ZbCNAo`t*?bk9w{s0a`pKCJ6&z#Ih~BV%W53wG1d(Tb2?=*nkZy z#^qt}p<+3weLN+1ziy9PI2a=BcyG=eWZ4gy&Qzt1t?jPDjgz q-);1y?6$Sv@_gw}Roj=%Nk>nm^jztAey>@xY#T4|Rsa~`Wq$y9JZF0V literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-40011326-2dde-4f95-bb01-c527e1978199.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..5f7c0e0a272925fa4ff8e457b059f5538d984227 GIT binary patch literal 478 zcmZWmO-sW-5S^}7au7l2E(zoi7A!5qrOoym5j+UeLn$755s_`Ot--WulT?w?ix4W><-q>7YY{0-ti z@b16zZ#b3O9^At+@4cCM4`y(6<58eCb?EoU*JshD1mO`i08Z)z01gVb_-~$dN-WdZ zp?yh-#Zb1HLHk$$V)yzuo2&HcAqyV$R8a!7ct%VTg}{hqTXxy7Y0+vJaAa**N*G~1 z)-M|u`@Q>$L1kyWXVDU~q8$KxFVGSj(o4WCq-?N$2@|=dnWPh;<_c4Y-$3G7r&*ND z%tI-p@WHJ9>&O1|t_%qECt%Q#k}7*!n8F8^D|yQ}s3ehp@CYj-|PooznH6P_=8r!rbbap<&K sTGoPgts`Z-se^_e$Y89Rfvk^OdMu^yN#74T^{Qptc!9S9zyL4%10!;1761SM literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-7e8491b3-26b5-4c11-9bd8-6a1ff65e9835.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..62247906ee210d65250065d6e02ba41458d87f50 GIT binary patch literal 478 zcmZWmO-sW-5S^~IL=ZvfE(zoi7A!5qrOozR5xhwcrFiH?M7GJc2Gg%3RiyOdZ|JY_ zhxP1KYIAT8%e?ny<~^9f{bP*+wW&k5yYJnZO$ovRH2^N^1OQHQH~88rL3RJ4h;hLwP3(dPcN#m8Da+i-o9R&&RKCSsoP4`k5 zAbn_y=CcW+*(!?EC_)v-w9Ee%^6q-GdES88JJ?-MW9N|1sfn5=Jf}Qbg;C(NT3S~9 pcC{mAyQ%$#=SzRAn!c=$T6!#{SCgLScj^_(w($aAIe-CP_6NRWXR!bP literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-91498ada-09ad-4940-b848-e223015a10d4.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7e7e745cc51c94a00ac01409a903e8ac26e22286 GIT binary patch literal 478 zcmZWmO-sW-5S^~IA5YsXT;2FZMS9#7JE?Wf%3lTb%N!h3fcK3B7y z6b4A|+oHvMifF!$A~lXs#WC&jzlFTF*>3N*VD|TR=d;+^=X2_!?g`JSjMrflIPJEU rHNR8qO4(^?zv=nXpQx5E8{@W~Na@w3=lR`6)v|58z*`Prh?o5Vc!_8> literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-97532fd0-b288-46e2-87a7-da3c10a235de.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f4077d888edbd689453c593ed61dbb9334ff86ec GIT binary patch literal 478 zcmZWm%SyvQ6ulj5$)X5iXGkE6FkopR4sE7yL~tivl;WZr5t$~_8cbhFsz~X^KM?!@ zf5{*4SG<+lES$w~&pG$rbKnlIZ|fAOO&$96{`pOfl8G*+VJ#ssdeBQ4Zlh}1HeTQ@2Qb9T{s5Q>W<>x1 literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-bb9e662a-099f-46d7-8b69-abb1f4aa86f9.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..2954113668b50c19a8e3f3661be3300e2ff29a63 GIT binary patch literal 478 zcmZWm%SyvQ6ulj5$wCT3XGkDI7_hVuhc?qUBDj+-N^#MRh)k1d4W_RoRit#|#vkxo z{1-pNKk!y+vv3x}J?Gqe&w)Fq!T=btY|AbiHqBd20}9rTrGydI zXM?hFneELg29<+x#-b%=MLPiYk)tIxqE~=hNZDY65+-s@a}iI4S}IH-eglbT9Vg*p zZXQb^g$HK!Uq`Zv1ZH2Ybt>Y;iut<9HEM1+U0)>d2h4b-fzL|@9i#Tv9r(T)J5GBo>LjG!zggt tZ7pklr`DCS)6#y^^QAvgEnhaqZ9S3Ft4q)GyN#-4+jxPu9KaAS`vXAsX0ZSO literal 0 HcmV?d00001 diff --git a/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet b/kernel-default/src/test/resources/basic-partitioned-no-checkpoint/part_a=1/part_b=4/part-00001-f1a5a71a-07ca-45d3-ae0c-09d2db85ed99.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c7fafbc3d9bf5a07ae387480ec58102ef08b6e50 GIT binary patch literal 478 zcmZWmO-sW-5S^~IM392eT@uJ4ELd8IOPlRCB6yP?O7YN(h-{N>4W?g7sz~X@i+{jB z=Wp-_cy%hZIk<;q-g`6i9?bCiwoZZC)S+LW@83n65`+b602CVp0FH7e`0t)|Ml91f zpsb|CVkq0ppaU!bv3qlxELHsUkob>!rYHfLKPM&$17O6mExT;kG;cKxC|Em|5=K~` z4a&x4wl}L7R1U@&i+H zc`St#9+=gC9my&Zn0>X@sfZUZVy06Ss8pGw4NqT}n)i5;#%n+2E}w)t3KHJC+wi%X z?W8b3dfyf;=2JxTbrh*_ges0{m;Wu~z0G!ezXh|ux4W3d&OV=07j;i~PG!6fqrhpm swXFG_T35DX~oB zfU=Sji=k{YgAT9&#O_ToS*rNyA@LveOi= z8TgJTR;OI+9f+F#Bq)QxPv-#7w6uP^mIU8=k%{HSh5xjn{t4T|NnQ6ePTNx8ZX& z+eu-7^u8@x%%_Ov>nKv=2vr=@F8^D|dzxq val table = Table.forPath(path, new DefaultTableHelper()) val snapshot = table.getLatestSnapshot.asInstanceOf[SnapshotImpl] @@ -66,4 +67,12 @@ class TableSuite extends AnyFunSuite with GoldenTableUtils { println(snapshot.getProtocol) } } + + test("can parse add file partition values - basic - no checkpoint") { + withGoldenTable("basic-partitioned-no-checkpoint") { path => + val table = Table.forPath(path, new DefaultTableHelper()) + val snapshot = table.getLatestSnapshot.asInstanceOf[SnapshotImpl] + snapshot.getAddFiles.forEachRemaining(x => println(x)) + } + } } diff --git a/kernel-default/src/test/scala/io/delta/core/util/GoldenTablesGenerator.scala b/kernel-default/src/test/scala/io/delta/core/util/GoldenTablesGenerator.scala index 638d39dc3..cb5726dca 100644 --- a/kernel-default/src/test/scala/io/delta/core/util/GoldenTablesGenerator.scala +++ b/kernel-default/src/test/scala/io/delta/core/util/GoldenTablesGenerator.scala @@ -21,10 +21,13 @@ import java.io.File import org.apache.spark.SparkConf import org.apache.spark.network.util.JavaUtils import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.functions.col import org.apache.spark.sql.test.SharedSparkSession /** * GENERATE_GOLDEN_TABLES=1 build/sbt 'kernelDefault/testOnly *GoldenTablesGenerator' + * or + * GENERATE_GOLDEN_TABLES=1 build/sbt 'kernelDefault/testOnly *GoldenTablesGenerator -- -z ""' */ class GoldenTablesGenerator extends QueryTest with SharedSparkSession { @@ -67,7 +70,17 @@ class GoldenTablesGenerator extends QueryTest with SharedSparkSession { } } - generate("primitive-data-types") { path => - + generate("basic-partitioned-no-checkpoint") { path => + for (i <- 0 to 8) { + val low = i * 10 + val high = (i + 1) * 10 + spark.range(low, high) + .withColumn("part_a", col("id") % 2) + .withColumn("part_b", col("id") % 5) + .write + .format("delta") + .partitionBy("part_a", "part_b") + .mode("append").save(path) + } } } diff --git a/kernel/src/main/java/io/delta/core/data/Row.java b/kernel/src/main/java/io/delta/core/data/Row.java index fb1c0f68c..351759ec7 100644 --- a/kernel/src/main/java/io/delta/core/data/Row.java +++ b/kernel/src/main/java/io/delta/core/data/Row.java @@ -38,5 +38,5 @@ public interface Row { // List getList(int ordinal); // -// Map getMap(int ordinal); + Map getMap(int ordinal); } diff --git a/kernel/src/main/java/io/delta/core/internal/SnapshotImpl.java b/kernel/src/main/java/io/delta/core/internal/SnapshotImpl.java index 8eb1290d8..513eb559c 100644 --- a/kernel/src/main/java/io/delta/core/internal/SnapshotImpl.java +++ b/kernel/src/main/java/io/delta/core/internal/SnapshotImpl.java @@ -3,6 +3,7 @@ import io.delta.core.ScanBuilder; import io.delta.core.Snapshot; import io.delta.core.fs.Path; +import io.delta.core.internal.actions.AddFile; import io.delta.core.internal.actions.Metadata; import io.delta.core.internal.actions.Protocol; import io.delta.core.internal.lang.Lazy; @@ -10,6 +11,7 @@ import io.delta.core.internal.replay.LogReplay; import io.delta.core.internal.snapshot.LogSegment; import io.delta.core.types.StructType; +import io.delta.core.utils.CloseableIterator; public class SnapshotImpl implements Snapshot { private final Path logPath; @@ -60,6 +62,10 @@ public ScanBuilder getScanBuilder() { // Internal APIs //////////////////////////////////////// + public CloseableIterator getAddFiles() { + return logReplay.getAddFiles(); + } + public Protocol getProtocol() { return protocolAndMetadata.get()._1; } diff --git a/kernel/src/main/java/io/delta/core/internal/actions/AddFile.java b/kernel/src/main/java/io/delta/core/internal/actions/AddFile.java index d6c17b06f..d6fe86619 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/AddFile.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/AddFile.java @@ -1,24 +1,55 @@ package io.delta.core.internal.actions; +import java.util.Map; import java.util.Optional; import io.delta.core.data.Row; -import io.delta.core.types.StringType; -import io.delta.core.types.StructType; +import io.delta.core.types.*; public class AddFile extends FileAction { + //////////////////////////////////////////////////////////////////////////////// + // Static Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + public static AddFile fromRow(Row row) { if (row == null) return null; - return new AddFile(row.getString(0), true); + final String path = row.getString(0); + final Map partitionValues = row.getMap(1); + final long size = row.getLong(2); + final long modificationTime = row.getLong(3); + final boolean dataChange = row.getBoolean(4); + + return new AddFile(path, partitionValues, size, modificationTime, dataChange); } public static final StructType READ_SCHEMA = new StructType() - .add("path", StringType.INSTANCE); + .add("path", StringType.INSTANCE) + .add("partitionValues", new MapType(StringType.INSTANCE, StringType.INSTANCE, false)) + .add("size", LongType.INSTANCE) + .add("modificationTime", LongType.INSTANCE) + .add("dataChange", BooleanType.INSTANCE); + + //////////////////////////////////////////////////////////////////////////////// + // Instance Fields / Methods + //////////////////////////////////////////////////////////////////////////////// + + private final Map partitionValues; + private final long size; + private final long modificationTime; - public AddFile(String path, boolean dataChange) { + public AddFile( + String path, + Map partitionValues, + long size, + long modificationTime, + boolean dataChange) { super(path, dataChange); + + this.partitionValues = partitionValues; + this.size = size; + this.modificationTime = modificationTime; } public Optional getDeletionVectorUniqueId() { @@ -29,4 +60,15 @@ public Optional getDeletionVectorUniqueId() { public AddFile copyWithDataChange(boolean dataChange) { return this; // TODO } + + @Override + public String toString() { + return "AddFile{" + + "path='" + path + '\'' + + ", partitionValues=" + partitionValues + + ", size=" + size + + ", modificationTime=" + modificationTime + + ", dataChange=" + dataChange + + '}'; + } } diff --git a/kernel/src/main/java/io/delta/core/internal/actions/FileAction.java b/kernel/src/main/java/io/delta/core/internal/actions/FileAction.java index fc8e8163b..7e9ddb5a5 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/FileAction.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/FileAction.java @@ -7,8 +7,8 @@ import io.delta.core.internal.lang.Lazy; public abstract class FileAction implements Action { - private final String path; - private final boolean dataChange; + protected final String path; + protected final boolean dataChange; private final Lazy pathAsUri; private final Lazy pathAsPath; diff --git a/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java b/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java index e51b9ac61..06c46b78b 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/Metadata.java @@ -30,7 +30,7 @@ public static Metadata fromRow(Row row, TableHelper tableHelper) { .add("format", Format.READ_SCHEMA) .add("schemaString", StringType.INSTANCE) .add("partitionColumns", new ArrayType(StringType.INSTANCE, false /* contains null */)) - .add("configuration", new MapType()) + .add("configuration", new MapType(StringType.INSTANCE, StringType.INSTANCE, false)) .add("createdTime", LongType.INSTANCE); //////////////////////////////////////////////////////////////////////////////// diff --git a/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java b/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java index d3378eb72..fdfe09ad2 100644 --- a/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java +++ b/kernel/src/main/java/io/delta/core/internal/actions/SingleAction.java @@ -11,37 +11,37 @@ public class SingleAction { //////////////////////////////////////////////////////////////////////////////// public static SingleAction fromRow(Row row, TableHelper tableHelper) { - final SetTransaction txn = row.isNullAt(0) ? null : SetTransaction.fromRow(row.getRecord(0)); + final SetTransaction txn = SetTransaction.fromRow(row.getRecord(0)); if (txn != null) { return new SingleAction(txn, null, null, null, null, null, null); } - final AddFile add = row.isNullAt(1) ? null : AddFile.fromRow(row.getRecord(1)); + final AddFile add = AddFile.fromRow(row.getRecord(1)); if (add != null) { return new SingleAction(null, add, null, null, null, null, null); } - final RemoveFile remove = row.isNullAt(2) ? null : RemoveFile.fromRow(row.getRecord(2)); + final RemoveFile remove = RemoveFile.fromRow(row.getRecord(2)); if (remove != null) { return new SingleAction(null, null, remove, null, null, null, null); } - final Metadata metadata = row.isNullAt(3) ? null : Metadata.fromRow(row.getRecord(3), tableHelper); + final Metadata metadata = Metadata.fromRow(row.getRecord(3), tableHelper); if (metadata != null) { return new SingleAction(null, null, null, metadata, null, null, null); } - final Protocol protocol = row.isNullAt(4) ? null : Protocol.fromRow(row.getRecord(4)); + final Protocol protocol = Protocol.fromRow(row.getRecord(4)); if (protocol != null) { return new SingleAction(null, null, null, null, protocol, null, null); } - final AddCDCFile cdc = row.isNullAt(5) ? null : AddCDCFile.fromRow(row.getRecord(5)); + final AddCDCFile cdc = AddCDCFile.fromRow(row.getRecord(5)); if (cdc != null) { return new SingleAction(null, null, null, null, null, cdc, null); } - final CommitInfo commitInfo = row.isNullAt(6) ? null : CommitInfo.fromRow(row.getRecord(6)); + final CommitInfo commitInfo = CommitInfo.fromRow(row.getRecord(6)); if (commitInfo != null) { return new SingleAction(null, null, null, null, null, null, commitInfo); } diff --git a/kernel/src/main/java/io/delta/core/types/MapType.java b/kernel/src/main/java/io/delta/core/types/MapType.java index 634fcb110..d9791454c 100644 --- a/kernel/src/main/java/io/delta/core/types/MapType.java +++ b/kernel/src/main/java/io/delta/core/types/MapType.java @@ -1,4 +1,16 @@ package io.delta.core.types; public class MapType extends DataType { + + public static final MapType EMPTY_INSTANCE = new MapType(null, null, false); + + private final DataType keyType; + private final DataType valueType; + private final boolean valueContainsNull; + + public MapType(DataType keyType, DataType valueType, boolean valueContainsNull) { + this.keyType = keyType; + this.valueType = valueType; + this.valueContainsNull = valueContainsNull; + } } From ad555f422b01425f63ff831b64aba7a08d638344 Mon Sep 17 00:00:00 2001 From: Scott Sandre Date: Thu, 30 Mar 2023 15:34:48 -0700 Subject: [PATCH 6/6] Clean up some decoding code; more robust map parsing --- .../main/java/io/delta/core/data/JsonRow.java | 101 ++++++++---------- .../core/helpers/DefaultTableHelper.java | 2 +- .../java/io/delta/core/types/MapType.java | 12 +++ 3 files changed, 59 insertions(+), 56 deletions(-) diff --git a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java index bf57c43a7..117ee7489 100644 --- a/kernel-default/src/main/java/io/delta/core/data/JsonRow.java +++ b/kernel-default/src/main/java/io/delta/core/data/JsonRow.java @@ -2,9 +2,7 @@ import java.util.*; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.delta.core.types.*; @@ -15,40 +13,19 @@ public class JsonRow implements Row { // Static Methods //////////////////////////////////////////////////////////////////////////////// - private static Object decode( - ObjectNode rootNode, - String fieldName, - DataType dataType, - boolean isNullable, - ObjectMapper objectMapper) { - if (rootNode.get(fieldName) == null) { - if (isNullable) { - return null; - } - - throw new RuntimeException( - String.format( - "Root node at key %s is null but field isn't nullable. Root node: %s", - fieldName, - rootNode - ) - ); - } - - JsonNode jsonValue = rootNode.get(fieldName); - + private static Object decodeElement(JsonNode jsonValue, DataType dataType) { if (dataType instanceof UnresolvedDataType) { if (jsonValue.isTextual()) { return jsonValue.textValue(); } else if (jsonValue instanceof ObjectNode) { - // TODO + throw new RuntimeException("TODO handle UnresolvedDataType of type object"); } } if (dataType instanceof BooleanType) { if (!jsonValue.isBoolean()) { throw new RuntimeException( - String.format("RootNode at %s isn't a boolean", fieldName) + String.format("Couldn't decode %s, expected a boolean", jsonValue) ); } return jsonValue.booleanValue(); @@ -58,7 +35,7 @@ private static Object decode( // TODO: handle other number cases (e.g. short) and throw on invalid cases (e.g. long) if (!jsonValue.isInt()) { throw new RuntimeException( - String.format("RootNode at %s isn't an int", fieldName) + String.format("Couldn't decode %s, expected an int", jsonValue) ); } return jsonValue.intValue(); @@ -67,19 +44,16 @@ private static Object decode( if (dataType instanceof LongType) { if (!jsonValue.isNumber()) { throw new RuntimeException( - String.format("RootNode at %s isn't a long.\nRootNode: %s\nElement: %s", fieldName, rootNode, jsonValue) + String.format("Couldn't decode %s, expected a long", jsonValue) ); } - if (!jsonValue.isLong()) { - System.out.println("WARN: expected a long, but jsonValue is " + jsonValue.numberValue().getClass().getSimpleName()); - } return jsonValue.numberValue().longValue(); } if (dataType instanceof StringType) { if (!jsonValue.isTextual()) { throw new RuntimeException( - String.format("RootNode at %s isn't a string", fieldName) + String.format("Couldn't decode %s, expected a string", jsonValue) ); } return jsonValue.textValue(); @@ -88,37 +62,28 @@ private static Object decode( if (dataType instanceof StructType) { if (!jsonValue.isObject()) { throw new RuntimeException( - String.format("RootNode at %s isn't an object", fieldName) + String.format("Couldn't decode %s, expected a struct", jsonValue) ); } - return new JsonRow((ObjectNode) jsonValue, (StructType) dataType, objectMapper); + return new JsonRow((ObjectNode) jsonValue, (StructType) dataType); } if (dataType instanceof ArrayType) { if (!jsonValue.isArray()) { throw new RuntimeException( - String.format("RootNode at %s isn't an array", fieldName) + String.format("Couldn't decode %s, expected an array", jsonValue) ); } final ArrayType arrayType = ((ArrayType) dataType); - final List output = new ArrayList<>(); final ArrayNode jsonArray = (ArrayNode) jsonValue; + final List output = new ArrayList<>(); + for (Iterator it = jsonArray.elements(); it.hasNext();) { final JsonNode element = it.next(); - // TODO: parse using the arrayType.getElementType() while verifying that the element - // matches that - if (element.isObject()) { - output.add( - new JsonRow((ObjectNode) element, (StructType) arrayType.getElementType(), objectMapper) - ); - } else if (element.isTextual()) { - output.add(element.textValue()); - } else { - throw new RuntimeException( - String.format("TODO implement this.\nparent%s\nthis%s\nchild%s", rootNode, jsonArray, element) - ); - } + final Object parsedElement = + decodeElement(element, arrayType.getElementType()); + output.add(parsedElement); } return output; } @@ -126,12 +91,21 @@ private static Object decode( if (dataType instanceof MapType) { if (!jsonValue.isObject()) { throw new RuntimeException( - String.format("RootNode at %s isn't an map", fieldName) + String.format("Couldn't decode %s, expected a map", jsonValue) ); } + final MapType mapType = (MapType) dataType; + final Iterator> iter = ((ObjectNode) jsonValue).fields(); + final Map output = new HashMap<>(); + + while (iter.hasNext()) { + Map.Entry entry = iter.next(); + String keyParsed = entry.getKey(); // TODO: handle non-String keys + Object valueParsed = decodeElement(entry.getValue(), mapType.getValueType()); + output.put(keyParsed, valueParsed); + } - return objectMapper - .convertValue(jsonValue, new TypeReference>() {}); + return output; } throw new UnsupportedOperationException( @@ -139,6 +113,24 @@ private static Object decode( ); } + private static Object decodeField(ObjectNode rootNode, StructField field) { + if (rootNode.get(field.name) == null) { + if (field.nullable) { + return null; + } + + throw new RuntimeException( + String.format( + "Root node at key %s is null but field isn't nullable. Root node: %s", + field.name, + rootNode + ) + ); + } + + return decodeElement(rootNode.get(field.name), field.dataType); + } + //////////////////////////////////////////////////////////////////////////////// // Instance Fields / Methods //////////////////////////////////////////////////////////////////////////////// @@ -147,15 +139,14 @@ private static Object decode( private final Object[] parsedValues; private final StructType readSchema; - public JsonRow(ObjectNode rootNode, StructType readSchema, ObjectMapper objectMapper) { + public JsonRow(ObjectNode rootNode, StructType readSchema) { this.rootNode = rootNode; this.readSchema = readSchema; this.parsedValues = new Object[readSchema.length()]; for (int i = 0; i < readSchema.length(); i++) { final StructField field = readSchema.at(i); - final Object parsedValue = - decode(rootNode, field.name, field.dataType, field.nullable, objectMapper); + final Object parsedValue = decodeField(rootNode, field); parsedValues[i] = parsedValue; } } diff --git a/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java b/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java index 67fa2bbde..cd6782836 100644 --- a/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java +++ b/kernel-default/src/main/java/io/delta/core/helpers/DefaultTableHelper.java @@ -127,7 +127,7 @@ public CloseableIterator readParquetFile(String path, StructType readSchema public Row parseJson(String json, StructType readSchema) { try { final JsonNode jsonNode = objectMapper.readTree(json); - return new JsonRow((ObjectNode) jsonNode, readSchema, objectMapper); + return new JsonRow((ObjectNode) jsonNode, readSchema); } catch (JsonProcessingException ex) { throw new RuntimeException(String.format("Could not parse JSON: %s", json), ex); } diff --git a/kernel/src/main/java/io/delta/core/types/MapType.java b/kernel/src/main/java/io/delta/core/types/MapType.java index d9791454c..7598fd43d 100644 --- a/kernel/src/main/java/io/delta/core/types/MapType.java +++ b/kernel/src/main/java/io/delta/core/types/MapType.java @@ -13,4 +13,16 @@ public MapType(DataType keyType, DataType valueType, boolean valueContainsNull) this.valueType = valueType; this.valueContainsNull = valueContainsNull; } + + public DataType getKeyType() { + return keyType; + } + + public DataType getValueType() { + return valueType; + } + + public boolean isValueContainsNull() { + return valueContainsNull; + } }