diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroAnnotationIntrospector.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroAnnotationIntrospector.java index c314a84d1..796140072 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroAnnotationIntrospector.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroAnnotationIntrospector.java @@ -1,14 +1,20 @@ package com.fasterxml.jackson.dataformat.avro; +import java.io.File; + +import org.apache.avro.reflect.AvroDefault; +import org.apache.avro.reflect.AvroIgnore; +import org.apache.avro.reflect.AvroName; +import org.apache.avro.reflect.Stringable; + import com.fasterxml.jackson.core.Version; import com.fasterxml.jackson.databind.AnnotationIntrospector; import com.fasterxml.jackson.databind.PropertyName; import com.fasterxml.jackson.databind.introspect.Annotated; +import com.fasterxml.jackson.databind.introspect.AnnotatedClass; +import com.fasterxml.jackson.databind.introspect.AnnotatedConstructor; import com.fasterxml.jackson.databind.introspect.AnnotatedMember; - -import org.apache.avro.reflect.AvroDefault; -import org.apache.avro.reflect.AvroIgnore; -import org.apache.avro.reflect.AvroName; +import com.fasterxml.jackson.databind.ser.std.ToStringSerializer; /** * Adds support for the following annotations from the Apache Avro implementation: @@ -18,6 +24,8 @@ *
  • {@link AvroDefault @AvroDefault("default value")} - Alias for JsonProperty.defaultValue, to * define default value for generated Schemas *
    • + *
  • {@link Stringable @Stringable} - Alias for JsonCreator on the constructor and JsonValue on + * the {@link #toString()} method.
    • * * * @since 2.9 @@ -57,4 +65,23 @@ protected PropertyName _findName(Annotated a) AvroName ann = _findAnnotation(a, AvroName.class); return (ann == null) ? null : PropertyName.construct(ann.value()); } + + @Override + public boolean hasCreatorAnnotation(Annotated a) { + AnnotatedConstructor constructor = a instanceof AnnotatedConstructor ? (AnnotatedConstructor) a : null; + AnnotatedClass parentClass = + a instanceof AnnotatedConstructor && ((AnnotatedConstructor) a).getTypeContext() instanceof AnnotatedClass + ? (AnnotatedClass) ((AnnotatedConstructor) a).getTypeContext() + : null; + return constructor != null && parentClass != null && parentClass.hasAnnotation(Stringable.class) + && constructor.getParameterCount() == 1 && String.class.equals(constructor.getRawParameterType(0)); + } + + @Override + public Object findSerializer(Annotated a) { + if (a instanceof AnnotatedClass && a.hasAnnotation(Stringable.class) || a.getRawType() == File.class) { + return ToStringSerializer.class; + } + return null; + } } diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroParser.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroParser.java index 5368b25a0..79a772a78 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroParser.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/AvroParser.java @@ -1,6 +1,9 @@ package com.fasterxml.jackson.dataformat.avro; -import java.io.*; +import java.io.IOException; +import java.io.InputStream; +import java.io.Writer; +import java.math.BigDecimal; import com.fasterxml.jackson.core.*; import com.fasterxml.jackson.core.base.ParserBase; @@ -281,6 +284,17 @@ public JsonLocation getCurrentLocation() @Override public abstract JsonToken nextToken() throws IOException; + @Override + protected void convertNumberToBigDecimal() throws IOException { + // ParserBase uses _textValue instead of _numberDouble for some reason when NR_DOUBLE is set, but _textValue is not set by setNumber() + // Catch and use _numberDouble instead + if ((_numTypesValid & NR_DOUBLE) != 0 && _textValue == null) { + _numberBigDecimal = BigDecimal.valueOf(_numberDouble); + return; + } + super.convertNumberToBigDecimal(); + } + /* /********************************************************** /* String value handling diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java index d4321e0a7..8d6d7e9d8 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/AvroSchemaHelper.java @@ -145,14 +145,15 @@ public static Schema numericAvroSchema(JsonParser.NumberType type) { switch (type) { case INT: return Schema.create(Schema.Type.INT); - case BIG_INTEGER: case LONG: return Schema.create(Schema.Type.LONG); case FLOAT: return Schema.create(Schema.Type.FLOAT); - case BIG_DECIMAL: case DOUBLE: return Schema.create(Schema.Type.DOUBLE); + case BIG_INTEGER: + case BIG_DECIMAL: + return Schema.create(Schema.Type.STRING); default: } throw new IllegalStateException("Unrecognized number type: "+type); @@ -211,6 +212,17 @@ public static Schema parseJsonSchema(String json) { return parser.parse(json); } + /** + * Constructs a new enum schema + * + * @param bean Enum type to use for name / description / namespace + * @param values List of enum names + * @return An {@link org.apache.avro.Schema.Type#ENUM ENUM} schema. + */ + public static Schema createEnumSchema(BeanDescription bean, List values) { + return Schema.createEnum(getName(bean.getType()), bean.findClassDescription(), getNamespace(bean.getType()), values); + } + /** * Returns the Avro type ID for a given type */ diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/DoubleVisitor.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/DoubleVisitor.java index 8374a72bd..3daec32dc 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/DoubleVisitor.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/DoubleVisitor.java @@ -3,15 +3,19 @@ import org.apache.avro.Schema; import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonNumberFormatVisitor; public class DoubleVisitor extends JsonNumberFormatVisitor.Base implements SchemaBuilder { + protected final JavaType _hint; protected JsonParser.NumberType _type; - public DoubleVisitor() { } + public DoubleVisitor(JavaType typeHint) { + _hint = typeHint; + } @Override public void numberType(JsonParser.NumberType type) { @@ -25,6 +29,6 @@ public Schema builtAvroSchema() { // would require union most likely return AvroSchemaHelper.anyNumberSchema(); } - return AvroSchemaHelper.numericAvroSchema(_type); + return AvroSchemaHelper.numericAvroSchema(_type, _hint); } } diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/MapVisitor.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/MapVisitor.java index e7528f0c1..2cc3a5ecb 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/MapVisitor.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/MapVisitor.java @@ -12,11 +12,13 @@ public class MapVisitor extends JsonMapFormatVisitor.Base implements SchemaBuilder { protected final JavaType _type; - + protected final DefinedSchemas _schemas; protected Schema _valueSchema; - + + protected JavaType _keyType; + public MapVisitor(SerializerProvider p, JavaType type, DefinedSchemas schemas) { super(p); @@ -30,7 +32,23 @@ public Schema builtAvroSchema() { if (_valueSchema == null) { throw new IllegalStateException("Missing value type for "+_type); } - return Schema.createMap(_valueSchema); + + Schema schema = Schema.createMap(_valueSchema); + + // add the key type if there is one + if (_keyType != null && AvroSchemaHelper.isStringable(getProvider() + .getConfig() + .introspectClassAnnotations(_keyType) + .getClassInfo())) { + schema.addProp(AvroSchemaHelper.AVRO_SCHEMA_PROP_KEY_CLASS, AvroSchemaHelper.getTypeId(_keyType)); + } else if (_keyType != null && !_keyType.isEnumType()) { + // Avro handles non-stringable keys by converting the map to an array of key/value records + // TODO add support for these in the schema, and custom serializers / deserializers to handle map restructuring + throw new UnsupportedOperationException( + "Key " + _keyType + " is not stringable and non-stringable map keys are not supported yet."); + } + + return schema; } /* @@ -43,12 +61,7 @@ public Schema builtAvroSchema() { public void keyFormat(JsonFormatVisitable handler, JavaType keyType) throws JsonMappingException { - /* We actually don't care here, since Avro only has String-keyed - * Maps like JSON: meaning that anything Jackson can regularly - * serialize must convert to Strings anyway. - * If we do find problem cases, we can start verifying them here, - * but for now assume it all "just works". - */ + _keyType = keyType; } @Override diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java index b89136932..cfb88439d 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/StringVisitor.java @@ -6,7 +6,9 @@ import org.apache.avro.Schema; import com.fasterxml.jackson.core.JsonParser.NumberType; +import com.fasterxml.jackson.databind.BeanDescription; import com.fasterxml.jackson.databind.JavaType; +import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonStringFormatVisitor; import com.fasterxml.jackson.databind.jsonFormatVisitors.JsonValueFormat; import com.fasterxml.jackson.databind.type.TypeFactory; @@ -14,14 +16,16 @@ public class StringVisitor extends JsonStringFormatVisitor.Base implements SchemaBuilder { + protected final SerializerProvider _provider; protected final JavaType _type; protected final DefinedSchemas _schemas; protected Set _enums; - public StringVisitor(DefinedSchemas schemas, JavaType t) { + public StringVisitor(SerializerProvider provider, DefinedSchemas schemas, JavaType t) { _schemas = schemas; _type = t; + _provider = provider; } @Override @@ -40,13 +44,17 @@ public Schema builtAvroSchema() { if (_type.hasRawClass(char.class) || _type.hasRawClass(Character.class)) { return AvroSchemaHelper.numericAvroSchema(NumberType.INT, TypeFactory.defaultInstance().constructType(Character.class)); } - if (_enums == null) { - return Schema.create(Schema.Type.STRING); + BeanDescription bean = _provider.getConfig().introspectClassAnnotations(_type); + if (_enums != null) { + Schema s = AvroSchemaHelper.createEnumSchema(bean, new ArrayList<>(_enums)); + _schemas.addSchema(_type, s); + return s; } - Schema s = Schema.createEnum(AvroSchemaHelper.getName(_type), "", - AvroSchemaHelper.getNamespace(_type), - new ArrayList(_enums)); - _schemas.addSchema(_type, s); - return s; + Schema schema = Schema.create(Schema.Type.STRING); + // Stringable classes need to include the type + if (AvroSchemaHelper.isStringable(bean.getClassInfo())) { + schema.addProp(AvroSchemaHelper.AVRO_SCHEMA_PROP_CLASS, AvroSchemaHelper.getTypeId(_type)); + } + return schema; } } diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java index a423d256a..4c8aac1e1 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/schema/VisitorFormatWrapperImpl.java @@ -118,14 +118,14 @@ public JsonStringFormatVisitor expectStringFormat(JavaType type) _valueSchema = s; return null; } - StringVisitor v = new StringVisitor(_schemas, type); + StringVisitor v = new StringVisitor(_provider, _schemas, type); _builder = v; return v; } @Override public JsonNumberFormatVisitor expectNumberFormat(JavaType convertedType) { - DoubleVisitor v = new DoubleVisitor(); + DoubleVisitor v = new DoubleVisitor(convertedType); _builder = v; return v; } diff --git a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java index 9db1559c0..25693eed0 100644 --- a/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java +++ b/avro/src/main/java/com/fasterxml/jackson/dataformat/avro/ser/NonBSGenericDatumWriter.java @@ -9,6 +9,7 @@ import org.apache.avro.Schema.Type; import org.apache.avro.generic.GenericDatumWriter; import org.apache.avro.io.Encoder; +import org.apache.avro.reflect.Stringable; import com.fasterxml.jackson.dataformat.avro.schema.AvroSchemaHelper; @@ -58,11 +59,17 @@ public int resolveUnion(Schema union, Object datum) { } } } else if (datum instanceof BigDecimal) { + int subOptimal = -1; for (int i = 0, len = schemas.size(); i < len; i++) { - if (schemas.get(i).getType() == Type.DOUBLE) { + if (schemas.get(i).getType() == Type.STRING) { return i; + } else if (schemas.get(i).getType() == Type.DOUBLE) { + subOptimal = i; } } + if (subOptimal > -1) { + return subOptimal; + } } // otherwise just default to base impl, stupid as it is... @@ -71,8 +78,29 @@ public int resolveUnion(Schema union, Object datum) { @Override protected void write(Schema schema, Object datum, Encoder out) throws IOException { - if ((schema.getType() == Type.DOUBLE) && datum instanceof BigDecimal) { - out.writeDouble(((BigDecimal)datum).doubleValue()); + // Cocerce numerical types, like BigDecimal -> double and BigInteger -> long + if (datum instanceof Number) { + switch (schema.getType()) { + case LONG: + super.write(schema, (((Number) datum).longValue()), out); + return; + case INT: + super.write(schema, (((Number) datum).intValue()), out); + return; + case FLOAT: + super.write(schema, (((Number) datum).floatValue()), out); + return; + case DOUBLE: + super.write(schema, (((Number) datum).doubleValue()), out); + return; + case STRING: + super.write(schema, datum.toString(), out); + return; + } + } + // Handle stringable classes + if (schema.getType() == Type.STRING && datum != null && datum.getClass().getAnnotation(Stringable.class) != null) { + super.write(schema, datum.toString(), out); return; } if (datum instanceof String) { diff --git a/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/interop/annotations/StringableTest.java b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/interop/annotations/StringableTest.java new file mode 100644 index 000000000..1d1a1b9fd --- /dev/null +++ b/avro/src/test/java/com/fasterxml/jackson/dataformat/avro/interop/annotations/StringableTest.java @@ -0,0 +1,280 @@ +package com.fasterxml.jackson.dataformat.avro.interop.annotations; + +import java.io.File; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.avro.reflect.AvroSchema; +import org.apache.avro.reflect.Stringable; +import org.apache.avro.specific.SpecificData; +import org.junit.Test; + +import com.fasterxml.jackson.dataformat.avro.interop.ApacheAvroInteropUtil; +import com.fasterxml.jackson.dataformat.avro.interop.InteropTestBase; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assume.assumeTrue; + +/** + * Tests support for using classes marked {@link Stringable @Stringable} as map keys. These classes must have a constructor which accepts a + * single string as an argument, and their {@link #toString()} must return a serialized version of the object that can be passed back into + * the constructor to recreate it. In addition, Avro considers the following classes {@link SpecificData#stringableClasses stringable by + * default}: + *
      + *
    • {@link File}
      • + *
    • {@link BigInteger}
      • + *
    • {@link BigDecimal}
      • + *
    • {@link URI}
      • + *
    • {@link URL}
      • + *
    + */ +public class StringableTest extends InteropTestBase { + @Stringable + @Data + public static class CustomStringableKey { + private final String test; + + public CustomStringableKey(String test) { + this.test = test; + } + + @Override + public String toString() { + return test; + } + } + + @Data + @NoArgsConstructor + @AllArgsConstructor + static class BigNumberWrapper { + @AvroSchema("\"double\"") + private BigDecimal bigDecimal; + @AvroSchema("\"long\"") + private BigInteger bigInteger; + } + + @Test + public void testBigDecimalWithDoubleSchema() { + // Apache impl can't do coercion + assumeTrue(serializeFunctor != ApacheAvroInteropUtil.apacheSerializer); + assumeTrue(deserializeFunctor != ApacheAvroInteropUtil.apacheDeserializer); + + double value = 0.32198154657; + BigNumberWrapper wrapper = new BigNumberWrapper(new BigDecimal(value), BigInteger.ONE); + // + BigNumberWrapper result = roundTrip(wrapper); + // + assertThat(result.bigDecimal.doubleValue()).isEqualTo(value); + } + + @Test + public void testBigIntegerWithDoubleSchema() { + // Apache impl can't do coercion + assumeTrue(serializeFunctor != ApacheAvroInteropUtil.apacheSerializer); + assumeTrue(deserializeFunctor != ApacheAvroInteropUtil.apacheDeserializer); + + long value = 948241716844286248L; + BigNumberWrapper wrapper = new BigNumberWrapper(BigDecimal.ZERO, BigInteger.valueOf(value)); + // + BigNumberWrapper result = roundTrip(wrapper); + // + assertThat(result.bigInteger.longValue()).isEqualTo(value); + } + + @Test + public void testBigDecimal() { + BigDecimal original = new BigDecimal("0.7193789624775822761924891294139324921"); + // + BigDecimal result = roundTrip(original); + // + assertThat(result).isEqualTo(original); + } + + @Test + public void testBigDecimalArray() { + ArrayList array = new ArrayList<>(); + array.add(new BigDecimal("32165498701061140.034501381101601018405251061")); + array.add(new BigDecimal("0.7193789624775822761924891294139324921")); + // + ArrayList result = roundTrip(type(ArrayList.class, BigDecimal.class), array); + // + assertThat(result).isEqualTo(array); + } + + @Test + public void testBigDecimalKeys() { + Map map = new HashMap<>(); + map.put(new BigDecimal("32165498701061140.034501381101601018405251061"), "one"); + map.put(new BigDecimal("0.7193789624775822761924891294139324921"), "two"); + // + Map result = roundTrip(type(Map.class, BigDecimal.class, String.class), map); + // + assertThat(result).isEqualTo(map); + } + + @Test + public void testBigInteger() { + BigInteger original = new BigInteger("1236549816934246813682843621431493681279364198"); + // + BigInteger result = roundTrip(original); + // + assertThat(result).isEqualTo(original); + } + + @Test + public void testBigIntegerArray() { + ArrayList array = new ArrayList<>(); + array.add(new BigInteger("32165498701061140034501381101601018405251061")); + array.add(new BigInteger("7193789624775822761924891294139324921")); + // + ArrayList result = roundTrip(type(ArrayList.class, BigInteger.class), array); + // + assertThat(result).isEqualTo(array); + } + + @Test + public void testBigIntegerKeys() { + Map map = new HashMap<>(); + map.put(new BigInteger("32165498701061140034501381101601018405251061"), "one"); + map.put(new BigInteger("7193789624775822761924891294139324921"), "two"); + // + Map result = roundTrip(type(Map.class, BigInteger.class, String.class), map); + // + assertThat(result).isEqualTo(map); + } + + @Test + public void testCustomStringable() { + CustomStringableKey original = new CustomStringableKey("one"); + // + CustomStringableKey result = roundTrip(original); + // + assertThat(result).isEqualTo(original); + } + + @Test + public void testCustomStringableArray() { + ArrayList array = new ArrayList<>(); + array.add(new CustomStringableKey("one")); + array.add(new CustomStringableKey("two")); + // + ArrayList result = roundTrip(type(ArrayList.class, CustomStringableKey.class), array); + // + assertThat(result).isEqualTo(array); + } + + @Test + public void testCustomStringableKeyWithScalarValue() { + Map object = new HashMap<>(); + object.put(new CustomStringableKey("one"), "two"); + object.put(new CustomStringableKey("three"), "four"); + // + Map result = roundTrip(type(Map.class, CustomStringableKey.class, String.class), object); + // + assertThat(result).isEqualTo(object); + } + + @Test + public void testFile() { + File original = new File("/a/cool/file"); + // + File result = roundTrip(original); + // + assertThat(result).isEqualTo(original); + } + + @Test + public void testFileArray() { + ArrayList array = new ArrayList<>(); + array.add(new File("/some/path")); + array.add(new File("/some/other/path")); + // + ArrayList result = roundTrip(type(ArrayList.class, File.class), array); + // + assertThat(result).isEqualTo(array); + } + + @Test + public void testFileKeys() { + Map object = new HashMap<>(); + object.put(new File("/some/path"), "one"); + object.put(new File("/some/other/path"), "two"); + // + Map result = roundTrip(type(Map.class, File.class, String.class), object); + // + assertThat(result).isEqualTo(object); + } + + @Test + public void testURI() throws URISyntaxException { + URI original = new URI("https://github.com"); + // + URI result = roundTrip(original); + // + assertThat(result).isEqualTo(original); + } + + @Test + public void testURIArray() throws URISyntaxException { + ArrayList array = new ArrayList<>(); + array.add(new URI("http://fasterxml.com")); + array.add(new URI("https://github.com")); + // + ArrayList result = roundTrip(type(ArrayList.class, URI.class), array); + // + assertThat(result).isEqualTo(array); + } + + @Test + public void testURIKeys() throws URISyntaxException { + Map object = new HashMap<>(); + object.put(new URI("http://fasterxml.com"), "one"); + object.put(new URI("https://github.com"), "two"); + // + Map result = roundTrip(type(Map.class, URI.class, String.class), object); + // + assertThat(result).isEqualTo(object); + } + + @Test + public void testURL() throws MalformedURLException { + URL original = new URL("https://github.com"); + // + URL result = roundTrip(original); + // + assertThat(result).isEqualTo(original); + } + + @Test + public void testURLArray() throws MalformedURLException { + ArrayList array = new ArrayList<>(); + array.add(new URL("http://fasterxml.com")); + array.add(new URL("https://github.com")); + // + ArrayList result = roundTrip(type(ArrayList.class, URL.class), array); + // + assertThat(result).isEqualTo(array); + } + + @Test + public void testURLKeys() throws MalformedURLException { + Map map = new HashMap<>(); + map.put(new URL("http://fasterxml.com"), "one"); + map.put(new URL("https://github.com"), "two"); + // + Map result = roundTrip(type(Map.class, URL.class, String.class), map); + // + assertThat(result).isEqualTo(map); + } +}