Test Variant read from files

Author: aihuaxu

parquet-avro/pom.xml

@@ -133,6 +133,18 @@
       <version>${powermock.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.assertj</groupId>
+      <artifactId>assertj-core</artifactId>
+      <version>3.25.3</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <version>5.12.2</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

parquet-avro/src/main/java/org/apache/parquet/avro/AvroRecordConverter.java

@@ -395,7 +395,7 @@ private static Converter newConverter(
         }
         return newStringConverter(schema, model, parent, validator);
       case RECORD:
-        if (type.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.VariantLogicalTypeAnnotation) {
+        if (type.getName().equals("var") || type.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.VariantLogicalTypeAnnotation) {
           return new AvroVariantConverter(parent, type.asGroupType(), schema, model);
         } else {
           return new AvroRecordConverter(parent, type.asGroupType(), schema, model, validator);

parquet-avro/src/main/java/org/apache/parquet/avro/AvroVariantConverter.java

@@ -27,6 +27,7 @@
 import org.apache.parquet.io.api.GroupConverter;
 import org.apache.parquet.schema.GroupType;
 import org.apache.parquet.variant.ImmutableMetadata;
+import org.apache.parquet.variant.Variant;
 import org.apache.parquet.variant.VariantBuilder;
 import org.apache.parquet.variant.VariantConverters;
 
@@ -35,21 +36,21 @@
  */
 class AvroVariantConverter extends GroupConverter implements VariantConverters.ParentConverter<VariantBuilder> {
   private final ParentValueContainer parent;
-  private final Schema avroSchema;
-  private final GenericData model;
-  private final int metadataPos;
-  private final int valuePos;
+//   private final Schema avroSchema;
+//   private final GenericData model;
+//   private final int metadataPos;
+//   private final int valuePos;
   private final GroupConverter wrappedConverter;
 
   private VariantBuilder builder = null;
   private ImmutableMetadata metadata = null;
 
   AvroVariantConverter(ParentValueContainer parent, GroupType variantGroup, Schema avroSchema, GenericData model) {
     this.parent = parent;
-    this.avroSchema = avroSchema;
-    this.metadataPos = avroSchema.getField("metadata").pos();
-    this.valuePos = avroSchema.getField("value").pos();
-    this.model = model;
+//     this.avroSchema = avroSchema;
+//     this.metadataPos = avroSchema.getField("metadata").pos();
+//     this.valuePos = avroSchema.getField("value").pos();
+//     this.model = model;
     this.wrappedConverter = VariantConverters.newVariantConverter(variantGroup, this::setMetadata, this);
   }
 
@@ -77,10 +78,12 @@ public void end() {
 
     builder.appendNullIfEmpty();
 
-    Object record = model.newRecord(null, avroSchema);
-    model.setField(record, "metadata", metadataPos, metadata.getEncodedBuffer());
-    model.setField(record, "value", valuePos, builder.encodedValue());
-    parent.add(record);
+    Variant variant = builder.build();
+    parent.add(variant);
+//     Object record = model.newRecord(null, avroSchema);
+//     model.setField(record, "metadata", metadataPos, metadata.getEncodedBuffer());
+//     model.setField(record, "value", valuePos, builder.encodedValue());
+//     parent.add(record);
 
     this.builder = null;
   }

parquet-avro/src/test/java/org/apache/parquet/variant/JsonUtil.java

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.parquet.variant;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonFactoryBuilder;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.parquet.Preconditions;
+
+
+public class JsonUtil {
+
+  private JsonUtil() {}
+
+  private static final JsonFactory FACTORY =
+      new JsonFactoryBuilder()
+          .configure(JsonFactory.Feature.INTERN_FIELD_NAMES, false)
+          .configure(JsonFactory.Feature.FAIL_ON_SYMBOL_HASH_OVERFLOW, false)
+          .build();
+  private static final ObjectMapper MAPPER = new ObjectMapper(FACTORY);
+
+  public static ObjectMapper mapper() {
+    return MAPPER;
+  }
+
+  public static int getInt(String property, JsonNode node) {
+    Preconditions.checkArgument(node.has(property), "Cannot parse missing int: %s", property);
+    JsonNode pNode = node.get(property);
+    Preconditions.checkArgument(
+        pNode != null && !pNode.isNull() && pNode.isIntegralNumber() && pNode.canConvertToInt(),
+        "Cannot parse to an integer value: %s: %s",
+        property,
+        pNode);
+    return pNode.asInt();
+  }
+
+  public static String getString(String property, JsonNode node) {
+    Preconditions.checkArgument(node.has(property), "Cannot parse missing string: %s", property);
+    JsonNode pNode = node.get(property);
+    Preconditions.checkArgument(
+        pNode != null && !pNode.isNull() && pNode.isTextual(),
+        "Cannot parse to a string value: %s: %s",
+        property,
+        pNode);
+    return pNode.asText();
+  }
+
+  public static String getStringOrNull(String property, JsonNode node) {
+    if (!node.has(property)) {
+      return null;
+    }
+    JsonNode pNode = node.get(property);
+    if (pNode != null && pNode.isNull()) {
+      return null;
+    }
+    return getString(property, node);
+  }
+
+}