Skip to content

Commit 2f2e107

Browse files
mnorburymnorbury
authored
chore: adding type transformer (#35)
Implementing tests for a type converter to convert types in a similar way to the current [Go implementation](https://github.com/cloudquery/plugin-sdk/blob/main/transformers/struct_test.go#L22-L52). * Some simple stubbed Arrow extension types needed to be added for `InetType` and `JSONType` - implementation to follow in future PRs. * A wrapper around the arrow `List` type was created to reproduce the `ListOf` functionality present in the Go arrow SDK but not in the Java. fixes: #31
1 parent 060a628 commit 2f2e107

File tree

8 files changed

+336
-3
lines changed

8 files changed

+336
-3
lines changed

‎lib/src/main/java/io/cloudquery/transformers/NameTransformer.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import com.fasterxml.jackson.annotation.JsonProperty;
44
import io.cloudquery.caser.Caser;
55

6-
import javax.xml.transform.TransformerException;
76
import java.lang.reflect.Field;
87

98
public interface NameTransformer {
@@ -16,10 +15,9 @@ class DefaultNameTransformer implements NameTransformer {
1615
*
1716
* @param field Field to transform
1817
* @return Transformed field name
19-
* @throws TransformerException If the field name cannot be transformed
2018
*/
2119
@Override
22-
public String transform(Field field) throws TransformerException {
20+
public String transform(Field field) {
2321
JsonProperty annotation = field.getAnnotation(JsonProperty.class);
2422
if (annotation != null) {
2523
return annotation.value();

‎lib/src/main/java/io/cloudquery/transformers/TransformerException.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package io.cloudquery.transformers;
2+
3+
public class TransformerException extends Exception{
4+
public TransformerException(String message) {
5+
super(message);
6+
}
7+
}

‎lib/src/main/java/io/cloudquery/transformers/TypeTransformer.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package io.cloudquery.transformers;
2+
3+
import io.cloudquery.types.InetType;
4+
import io.cloudquery.types.JSONType;
5+
import io.cloudquery.types.ListType;
6+
import org.apache.arrow.vector.types.FloatingPointPrecision;
7+
import org.apache.arrow.vector.types.TimeUnit;
8+
import org.apache.arrow.vector.types.pojo.ArrowType;
9+
10+
import java.lang.reflect.Field;
11+
12+
public interface TypeTransformer {
13+
class DefaultTypeTransformer implements TypeTransformer {
14+
@Override
15+
public ArrowType transform(Field field) throws TransformerException {
16+
return transformArrowType(field.getName(), field.getType());
17+
}
18+
19+
private static ArrowType transformArrowType(String name, Class<?> type) throws TransformerException {
20+
switch (type.getName()) {
21+
case "java.lang.String" -> {
22+
return ArrowType.Utf8.INSTANCE;
23+
}
24+
case "java.lang.Boolean", "boolean" -> {
25+
return ArrowType.Bool.INSTANCE;
26+
}
27+
case "java.lang.Integer", "int", "java.lang.Long", "long" -> {
28+
return new ArrowType.Int(64, true);
29+
}
30+
case "float", "double", "java.lang.Float", "java.lang.Double" -> {
31+
return new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE);
32+
}
33+
case "java.util.Map" -> {
34+
return JSONType.INSTANCE;
35+
}
36+
case "java.net.InetAddress" -> {
37+
return InetType.INSTANCE;
38+
}
39+
case "java.time.LocalDateTime" -> {
40+
return new ArrowType.Timestamp(TimeUnit.MICROSECOND, null);
41+
}
42+
default -> {
43+
if (type.isArray()) {
44+
Class<?> componentType = type.getComponentType();
45+
if (componentType.getName().equals("byte")) {
46+
return ArrowType.Binary.INSTANCE;
47+
}
48+
return ListType.listOf(transformArrowType(name, componentType));
49+
}
50+
if (!type.isPrimitive()) {
51+
return JSONType.INSTANCE;
52+
}
53+
}
54+
}
55+
throw new TransformerException("Unsupported type: " + type.getName() + " for field: " + name);
56+
}
57+
}
58+
59+
ArrowType transform(Field field) throws TransformerException;
60+
}

‎lib/src/main/java/io/cloudquery/types/InetType.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package io.cloudquery.types;
2+
3+
import org.apache.arrow.memory.BufferAllocator;
4+
import org.apache.arrow.vector.FieldVector;
5+
import org.apache.arrow.vector.types.pojo.ArrowType;
6+
import org.apache.arrow.vector.types.pojo.FieldType;
7+
8+
public class InetType extends ArrowType.ExtensionType {
9+
public static final InetType INSTANCE = new InetType();
10+
11+
@Override
12+
public ArrowType storageType() {
13+
return Binary.INSTANCE;
14+
}
15+
16+
@Override
17+
public String extensionName() {
18+
return "inet";
19+
}
20+
21+
@Override
22+
public boolean extensionEquals(ExtensionType other) {
23+
if (!(other instanceof InetType))
24+
return false;
25+
return true;
26+
}
27+
28+
@Override
29+
public String serialize() {
30+
return null;
31+
}
32+
33+
@Override
34+
public ArrowType deserialize(ArrowType storageType, String serializedData) {
35+
return null;
36+
}
37+
38+
@Override
39+
public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) {
40+
return null;
41+
}
42+
}

‎lib/src/main/java/io/cloudquery/types/JSONType.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package io.cloudquery.types;
2+
3+
import org.apache.arrow.memory.BufferAllocator;
4+
import org.apache.arrow.vector.FieldVector;
5+
import org.apache.arrow.vector.types.pojo.ArrowType;
6+
import org.apache.arrow.vector.types.pojo.ArrowType.ExtensionType;
7+
import org.apache.arrow.vector.types.pojo.FieldType;
8+
9+
public class JSONType extends ExtensionType {
10+
public static final JSONType INSTANCE = new JSONType();
11+
12+
@Override
13+
public ArrowType storageType() {
14+
return ArrowType.Binary.INSTANCE;
15+
}
16+
17+
@Override
18+
public String extensionName() {
19+
return "json";
20+
}
21+
22+
@Override
23+
public boolean extensionEquals(ExtensionType other) {
24+
return false;
25+
}
26+
27+
@Override
28+
public String serialize() {
29+
return null;
30+
}
31+
32+
@Override
33+
public ArrowType deserialize(ArrowType storageType, String serializedData) {
34+
return null;
35+
}
36+
37+
@Override
38+
public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator) {
39+
return null;
40+
}
41+
42+
@Override
43+
public int hashCode() {
44+
return java.util.Arrays.deepHashCode(new Object[]{});
45+
}
46+
47+
@Override
48+
public boolean equals(Object obj) {
49+
if (!(obj instanceof JSONType)) {
50+
return false;
51+
}
52+
return true;
53+
}
54+
}

‎lib/src/main/java/io/cloudquery/types/ListType.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package io.cloudquery.types;
2+
3+
import org.apache.arrow.vector.types.pojo.ArrowType;
4+
5+
import java.util.Objects;
6+
7+
public class ListType extends ArrowType.List {
8+
9+
public static ListType listOf(ArrowType elementType) {
10+
return new ListType(elementType);
11+
}
12+
13+
private final ArrowType elementType;
14+
15+
public ListType(ArrowType elementType) {
16+
this.elementType = elementType;
17+
}
18+
19+
public ArrowType getElementType() {
20+
return elementType;
21+
}
22+
23+
@Override
24+
public boolean equals(Object o) {
25+
if (this == o) return true;
26+
if (o == null || getClass() != o.getClass()) return false;
27+
if (!super.equals(o)) return false;
28+
ListType listType = (ListType) o;
29+
return Objects.equals(elementType, listType.elementType);
30+
}
31+
32+
@Override
33+
public int hashCode() {
34+
return Objects.hash(super.hashCode(), elementType);
35+
}
36+
37+
@Override
38+
public String toString() {
39+
return "ListType{" +
40+
"elementType=" + elementType +
41+
'}';
42+
}
43+
}

‎lib/src/test/java/io/cloudquery/transformers/TypeTransformerTest.java‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
package io.cloudquery.transformers;
2+
3+
import io.cloudquery.transformers.TypeTransformer.DefaultTypeTransformer;
4+
import io.cloudquery.types.InetType;
5+
import io.cloudquery.types.JSONType;
6+
import io.cloudquery.types.ListType;
7+
import org.apache.arrow.vector.types.FloatingPointPrecision;
8+
import org.apache.arrow.vector.types.TimeUnit;
9+
import org.apache.arrow.vector.types.pojo.ArrowType;
10+
import org.junit.jupiter.params.ParameterizedTest;
11+
import org.junit.jupiter.params.provider.Arguments;
12+
import org.junit.jupiter.params.provider.MethodSource;
13+
14+
import java.net.InetAddress;
15+
import java.time.LocalDateTime;
16+
import java.util.Map;
17+
import java.util.stream.Stream;
18+
19+
import static org.junit.jupiter.api.Assertions.assertEquals;
20+
21+
class TypeTransformerTest {
22+
23+
@SuppressWarnings("unused")
24+
private static class InnerClass {
25+
private String innerClassStringField;
26+
}
27+
28+
@SuppressWarnings("unused")
29+
private static class SimpleClass {
30+
private String stringField;
31+
32+
private boolean booleanField;
33+
private Boolean booleanObjectField;
34+
35+
private int intField;
36+
private Integer integerObjectField;
37+
private long longField;
38+
private Long longObjectField;
39+
40+
private float floatField;
41+
private Float floatObjectField;
42+
private double doubleField;
43+
private Double doubleObjectField;
44+
45+
private Map<String, String> mapField;
46+
47+
private InnerClass innerClassObjectField;
48+
49+
private int[] intArrayField;
50+
private String[] stringArrayField;
51+
52+
private LocalDateTime timeField;
53+
54+
private InetAddress inetField;
55+
56+
private byte[] byteArrayField;
57+
}
58+
59+
@ParameterizedTest
60+
@MethodSource("testArgumentsSource")
61+
public void shouldTransformFields(String fieldName, ArrowType expectedArrowType) throws NoSuchFieldException, TransformerException {
62+
DefaultTypeTransformer transfomer = new DefaultTypeTransformer();
63+
64+
ArrowType arrowType = transfomer.transform(SimpleClass.class.getDeclaredField(fieldName));
65+
66+
assertEquals(expectedArrowType, arrowType);
67+
}
68+
69+
public static Stream<Arguments> testArgumentsSource() {
70+
return Stream.of(
71+
// Integer arguments
72+
Arguments.of("intField", new ArrowType.Int(64, true)),
73+
Arguments.of("integerObjectField", new ArrowType.Int(64, true)),
74+
Arguments.of("longField", new ArrowType.Int(64, true)),
75+
Arguments.of("longObjectField", new ArrowType.Int(64, true)),
76+
77+
// String arguments
78+
Arguments.of("stringField", ArrowType.Utf8.INSTANCE),
79+
80+
// Boolean arguments
81+
Arguments.of("booleanField", ArrowType.Bool.INSTANCE),
82+
Arguments.of("booleanObjectField", ArrowType.Bool.INSTANCE),
83+
84+
// Float field
85+
Arguments.of("floatField", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)),
86+
Arguments.of("floatObjectField", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)),
87+
Arguments.of("doubleField", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)),
88+
Arguments.of("doubleObjectField", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)),
89+
90+
// Map field
91+
Arguments.of("mapField", JSONType.INSTANCE),
92+
93+
// Inner class
94+
Arguments.of("innerClassObjectField", JSONType.INSTANCE),
95+
96+
// Array field
97+
Arguments.of("intArrayField", ListType.listOf(new ArrowType.Int(64, true))),
98+
Arguments.of("stringArrayField", ListType.listOf(ArrowType.Utf8.INSTANCE)),
99+
100+
// Time
101+
Arguments.of("timeField", new ArrowType.Timestamp(TimeUnit.MICROSECOND, null)),
102+
103+
// Byte
104+
Arguments.of("byteArrayField", ArrowType.Binary.INSTANCE),
105+
106+
// Inet
107+
Arguments.of("inetField", InetType.INSTANCE)
108+
);
109+
}
110+
}

‎lib/src/test/java/io/cloudquery/types/ListTypeTest.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package io.cloudquery.types;
2+
3+
import org.apache.arrow.vector.types.pojo.ArrowType;
4+
import org.junit.jupiter.api.Test;
5+
6+
import static org.junit.jupiter.api.Assertions.assertEquals;
7+
import static org.junit.jupiter.api.Assertions.assertNotEquals;
8+
9+
class ListTypeTest {
10+
@Test
11+
public void testEquality() {
12+
ListType listType1 = ListType.listOf(new ArrowType.Int(64, true));
13+
ListType listType2 = ListType.listOf(new ArrowType.Int(64, true));
14+
ListType listType3 = ListType.listOf(new ArrowType.Int(32, true));
15+
16+
assertEquals(listType1, listType2);
17+
assertNotEquals(listType1, listType3);
18+
}
19+
}

0 commit comments

Comments
 (0)