use defaultvariantvalue

Author: richardc-db

kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/value/DefaultVariantValue.java

@@ -0,0 +1,61 @@
+/*
+ * Copyright (2023) The Delta Lake Project Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.delta.kernel.defaults.internal.data.value;
+
+import java.util.Arrays;
+
+import io.delta.kernel.data.VariantValue;
+
+/**
+ * Default implementation of a Delta kernel VariantValue.
+ */
+public class DefaultVariantValue implements VariantValue {
+    private final byte[] value;
+    private final byte[] metadata;
+
+    public DefaultVariantValue(byte[] value, byte[] metadata) {
+        this.value = value;
+        this.metadata = metadata;
+    }
+
+    @Override
+    public byte[] getValue() {
+        return value;
+    }
+
+    @Override
+    public byte[] getMetadata() {
+        return metadata;
+    }
+
+    public String debugString() {
+        return "VariantValue{value=" + Arrays.toString(value) +
+            ", metadata=" + Arrays.toString(metadata) + '}';
+    }
+    /**
+     * Compare two variants in bytes. The variant equality is more complex than it, and we haven't
+     * supported it in the user surface yet. This method is only intended for tests.
+     */
+    @Override
+    public boolean equals(Object other) {
+        if (other instanceof DefaultVariantValue) {
+            return Arrays.equals(value, ((DefaultVariantValue) other).getValue()) &&
+                Arrays.equals(metadata, ((DefaultVariantValue) other).getMetadata());
+        } else {
+            return false;
+        }
+    }
+}

kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DefaultVariantVector.java

@@ -23,6 +23,7 @@
 import io.delta.kernel.types.DataType;
 
 import static io.delta.kernel.internal.util.Preconditions.checkArgument;
+import io.delta.kernel.defaults.internal.data.value.DefaultVariantValue;
 
 /**
  * {@link io.delta.kernel.data.ColumnVector} implementation for variant type data.
@@ -68,20 +69,22 @@ public VariantValue getVariant(int rowId) {
         if (isNullAt(rowId)) {
             return null;
         }
-        return new VariantValue() {
-            private final byte[] value = valueVector.getBinary(rowId);
-            private final byte[] metadata = metadataVector.getBinary(rowId);
+        // return new VariantValue() {
+        //     private final byte[] value = valueVector.getBinary(rowId);
+        //     private final byte[] metadata = metadataVector.getBinary(rowId);
 
-            @Override
-            public byte[] getValue() {
-                return value;
-            }
+        //     @Override
+        //     public byte[] getValue() {
+        //         return value;
+        //     }
 
-            @Override
-            public byte[] getMetadata() {
-                return metadata;
-            }
-        };
+        //     @Override
+        //     public byte[] getMetadata() {
+        //         return metadata;
+        //     }
+        // };
+        return new DefaultVariantValue(
+            valueVector.getBinary(rowId), metadataVector.getBinary(rowId));
     }
 
     /**

kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestRow.scala

@@ -20,6 +20,7 @@ import org.apache.spark.sql.{types => sparktypes}
 import org.apache.spark.sql.{Row => SparkRow}
 import org.apache.spark.unsafe.types.VariantVal
 import io.delta.kernel.data.{ArrayValue, ColumnVector, MapValue, Row}
+import io.delta.kernel.defaults.internal.data.value.DefaultVariantValue
 import io.delta.kernel.types._
 
 import java.time.LocalDate
@@ -104,9 +105,7 @@ object TestRow {
         case _: ArrayType => arrayValueToScalaSeq(row.getArray(i))
         case _: MapType => mapValueToScalaMap(row.getMap(i))
         case _: StructType => TestRow(row.getStruct(i))
-        case _: VariantType =>
-          val kernelVariant = row.getVariant(i)
-          new VariantVal(kernelVariant.getValue(), kernelVariant.getMetadata())
+        case _: VariantType => row.getVariant(i)
         case _ => throw new UnsupportedOperationException("unrecognized data type")
       }
     }.toSeq)
@@ -168,7 +167,9 @@ object TestRow {
             decodeCellValue(mapType.keyType, k) -> decodeCellValue(mapType.valueType, v)
           }
         case _: sparktypes.StructType => TestRow(row.getStruct(i))
-        case _: sparktypes.VariantType => row.getAs[VariantVal](i)
+        case _: sparktypes.VariantType =>
+          val sparkVariant = row.getAs[VariantVal](i)
+          new DefaultVariantValue(sparkVariant.getValue(), sparkVariant.getMetadata())
         case _ => throw new UnsupportedOperationException("unrecognized data type")
       }
     })

kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestUtils.scala

@@ -41,7 +41,6 @@ import org.apache.hadoop.shaded.org.apache.commons.io.FileUtils
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.{types => sparktypes}
 import org.apache.spark.sql.catalyst.plans.SQLHelper
-import org.apache.spark.unsafe.types.VariantVal
 import org.scalatest.Assertions
 
 trait TestUtils extends Assertions with SQLHelper {
@@ -415,7 +414,6 @@ trait TestUtils extends Assertions with SQLHelper {
         java.lang.Double.doubleToRawLongBits(a) == java.lang.Double.doubleToRawLongBits(b)
       case (a: Float, b: Float) =>
         java.lang.Float.floatToRawIntBits(a) == java.lang.Float.floatToRawIntBits(b)
-      case (a: VariantVal, b: VariantVal) => a.debugString() == b.debugString()
       case (a, b) =>
         if (!a.equals(b)) {
           val sds = 200;