diff --git a/common/src/main/java/com/datastax/oss/common/sink/state/LifeCycleManager.java b/common/src/main/java/com/datastax/oss/common/sink/state/LifeCycleManager.java index 876e78a..fc716e5 100644 --- a/common/src/main/java/com/datastax/oss/common/sink/state/LifeCycleManager.java +++ b/common/src/main/java/com/datastax/oss/common/sink/state/LifeCycleManager.java @@ -53,15 +53,10 @@ import com.datastax.oss.driver.api.core.metadata.schema.ColumnMetadata; import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; -import com.datastax.oss.driver.api.core.type.CqlVectorType; -import com.datastax.oss.driver.api.core.type.DataType; import com.datastax.oss.driver.api.core.type.DataTypes; -import com.datastax.oss.driver.api.core.type.codec.TypeCodec; -import com.datastax.oss.driver.api.core.type.reflect.GenericType; import com.datastax.oss.driver.internal.core.auth.PlainTextAuthProvider; import com.datastax.oss.driver.internal.core.config.typesafe.DefaultDriverConfigLoader; import com.datastax.oss.driver.internal.core.config.typesafe.DefaultProgrammaticDriverConfigLoaderBuilder; -import com.datastax.oss.driver.internal.core.type.codec.CqlVectorCodec; import com.datastax.oss.driver.internal.core.type.codec.registry.DefaultCodecRegistry; import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; @@ -69,7 +64,6 @@ import com.typesafe.config.Config; import com.typesafe.config.ConfigFactory; import edu.umd.cs.findbugs.annotations.NonNull; -import edu.umd.cs.findbugs.annotations.Nullable; import java.net.InetSocketAddress; import java.nio.file.Path; import java.util.Collection; @@ -97,20 +91,7 @@ public class LifeCycleManager { new ConcurrentHashMap<>(); private static MetricRegistry metricRegistry = new MetricRegistry(); private static final DefaultCodecRegistry CODEC_REGISTRY = - new DefaultCodecRegistry("default-registry") { - - protected TypeCodec createCodec( - @Nullable DataType cqlType, - @Nullable GenericType javaType, - boolean isJavaCovariant) { - if (cqlType instanceof CqlVectorType) { - log.info("Automatically Registering codec for CqlVectorType {}", cqlType); - CqlVectorType vectorType = (CqlVectorType) cqlType; - return new CqlVectorCodec<>(vectorType, codecFor(vectorType.getSubtype())); - } - return super.createCodec(cqlType, javaType, isJavaCovariant); - } - }; + new DefaultCodecRegistry("default-registry"); /** This is a utility class that no one should instantiate. */ private LifeCycleManager() {} diff --git a/pom.xml b/pom.xml index 199d45a..5e1e54e 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ 8 2.4.0 2.6.2 - 4.16.0 + 4.16.1-SNAPSHOT 1.10.0 1.0.3 25.1-jre diff --git a/text/pom.xml b/text/pom.xml index 09462ad..ab40941 100644 --- a/text/pom.xml +++ b/text/pom.xml @@ -22,7 +22,7 @@ dsbulk-codecs com.datastax.oss 1.10.0 - + com.datastax.oss 1.0.16-SNAPSHOT @@ -44,7 +44,7 @@ com.datastax.oss java-driver-core - 4.16.0 + 4.16.1-SNAPSHOT com.datastax.oss diff --git a/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeConvertingCodecProvider.java b/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeConvertingCodecProvider.java index 442b06f..2566247 100644 --- a/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeConvertingCodecProvider.java +++ b/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeConvertingCodecProvider.java @@ -64,7 +64,6 @@ import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.data.TupleValue; import com.datastax.oss.driver.api.core.data.UdtValue; -import com.datastax.oss.driver.api.core.type.CqlVectorType; import com.datastax.oss.driver.api.core.type.CustomType; import com.datastax.oss.driver.api.core.type.DataType; import com.datastax.oss.driver.api.core.type.DataTypes; @@ -73,12 +72,13 @@ import com.datastax.oss.driver.api.core.type.SetType; import com.datastax.oss.driver.api.core.type.TupleType; import com.datastax.oss.driver.api.core.type.UserDefinedType; +import com.datastax.oss.driver.api.core.type.VectorType; import com.datastax.oss.driver.api.core.type.codec.CodecNotFoundException; import com.datastax.oss.driver.api.core.type.codec.TypeCodec; import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; import com.datastax.oss.driver.api.core.type.codec.registry.CodecRegistry; import com.datastax.oss.driver.api.core.type.reflect.GenericType; -import com.datastax.oss.driver.internal.core.type.codec.CqlVectorCodec; +import com.datastax.oss.driver.internal.core.type.codec.VectorCodec; import com.datastax.oss.driver.internal.core.type.codec.registry.DefaultCodecRegistry; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; @@ -117,6 +117,8 @@ public class JsonNodeConvertingCodecProvider implements ConvertingCodecProvider private static final String DATE_RANGE_CLASS_NAME = "org.apache.cassandra.db.marshal.DateRangeType"; + private static final String VECTOR_CLASS_NAME = "org.apache.cassandra.db.marshal.VectorType"; + @NonNull @Override public Optional> maybeProvide( @@ -393,22 +395,25 @@ public class JsonNodeConvertingCodecProvider implements ConvertingCodecProvider nullStrings); case DATE_RANGE_CLASS_NAME: return new JsonNodeToDateRangeCodec(nullStrings); - case CqlVectorType.CQLVECTOR_CLASS_NAME: - CqlVectorType cqlVectorType = (CqlVectorType) cqlType; + case VECTOR_CLASS_NAME: + VectorType cqlVectorType = (VectorType) cqlType; // Step 1: create a JSON codec which will take the input JSON nodes and generate // something matching the expected data type ConvertingCodec jsonCodec = - createJsonNodeConvertingCodec(cqlVectorType.getSubtype(), codecFactory, false); + createJsonNodeConvertingCodec( + cqlVectorType.getElementType(), codecFactory, false); // Step 2: create a conventional codec which will take instances of the Java type // generated by the JSON codec above and perform standard serde on them. ConvertingCodec standardCodec = codecFactory.createConvertingCodec( - cqlVectorType.getSubtype(), jsonCodec.getInternalJavaType(), false); + cqlVectorType.getElementType(), jsonCodec.getInternalJavaType(), false); return new JsonNodeToVectorCodec( - new CqlVectorCodec(cqlVectorType, standardCodec), + new VectorCodec(cqlVectorType, standardCodec), jsonCodec, context.getAttribute(OBJECT_MAPPER), nullStrings); + default: + LOGGER.error("Unsupported custom type {}", customType.getClassName()); } } // fall through diff --git a/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeToVectorCodec.java b/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeToVectorCodec.java index b0a7f8c..3431850 100644 --- a/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeToVectorCodec.java +++ b/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeToVectorCodec.java @@ -16,21 +16,23 @@ package com.datastax.oss.dsbulk.codecs.text.json; import com.datastax.oss.driver.api.core.data.CqlVector; -import com.datastax.oss.driver.internal.core.type.codec.CqlVectorCodec; +import com.datastax.oss.driver.internal.core.type.codec.VectorCodec; import com.datastax.oss.dsbulk.codecs.api.ConvertingCodec; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import java.util.ArrayList; import java.util.Iterator; import java.util.List; -public class JsonNodeToVectorCodec extends JsonNodeConvertingCodec> { +public class JsonNodeToVectorCodec + extends JsonNodeConvertingCodec> { private final ConvertingCodec subtypeCodec; private final ObjectMapper objectMapper; public JsonNodeToVectorCodec( - CqlVectorCodec targetCodec, + VectorCodec targetCodec, ConvertingCodec subtypeCodec, ObjectMapper objectMapper, List nullStrings) { @@ -42,17 +44,17 @@ public JsonNodeToVectorCodec( @Override public CqlVector externalToInternal(JsonNode jsonNode) { if (jsonNode == null || !jsonNode.isArray()) return null; - CqlVector.Builder builder = CqlVector.builder(); + List result = new ArrayList<>(); for (Iterator it = jsonNode.elements(); it.hasNext(); ) - builder.add(subtypeCodec.externalToInternal(it.next())); - return builder.build(); + result.add(subtypeCodec.externalToInternal(it.next())); + return CqlVector.newInstance(result); } @Override public JsonNode internalToExternal(CqlVector value) { if (value == null) return null; ArrayNode root = objectMapper.createArrayNode(); - for (SubtypeT element : value.getValues()) { + for (SubtypeT element : value) { root.add(subtypeCodec.internalToExternal(element)); } return root; diff --git a/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringConvertingCodecProvider.java b/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringConvertingCodecProvider.java index 3c89984..6b77ccb 100644 --- a/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringConvertingCodecProvider.java +++ b/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringConvertingCodecProvider.java @@ -62,15 +62,15 @@ import com.datastax.oss.driver.api.core.data.TupleValue; import com.datastax.oss.driver.api.core.data.UdtValue; -import com.datastax.oss.driver.api.core.type.CqlVectorType; import com.datastax.oss.driver.api.core.type.CustomType; import com.datastax.oss.driver.api.core.type.DataType; import com.datastax.oss.driver.api.core.type.DataTypes; +import com.datastax.oss.driver.api.core.type.VectorType; import com.datastax.oss.driver.api.core.type.codec.CodecNotFoundException; import com.datastax.oss.driver.api.core.type.codec.TypeCodec; import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; import com.datastax.oss.driver.api.core.type.reflect.GenericType; -import com.datastax.oss.driver.internal.core.type.codec.CqlVectorCodec; +import com.datastax.oss.driver.internal.core.type.codec.VectorCodec; import com.datastax.oss.driver.internal.core.type.codec.registry.DefaultCodecRegistry; import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableList; import com.datastax.oss.dsbulk.codecs.api.ConversionContext; @@ -105,6 +105,7 @@ public class StringConvertingCodecProvider implements ConvertingCodecProvider { private static final String POLYGON_CLASS_NAME = "org.apache.cassandra.db.marshal.PolygonType"; private static final String DATE_RANGE_CLASS_NAME = "org.apache.cassandra.db.marshal.DateRangeType"; + private static final String VECTOR_CLASS_NAME = "org.apache.cassandra.db.marshal.VectorType"; @NonNull @Override @@ -332,13 +333,15 @@ public class StringConvertingCodecProvider implements ConvertingCodecProvider { return new StringToPolygonCodec(context.getAttribute(GEO_FORMAT), nullStrings); case DATE_RANGE_CLASS_NAME: return new StringToDateRangeCodec(nullStrings); - case CqlVectorType.CQLVECTOR_CLASS_NAME: - CqlVectorType cqlVectorType = (CqlVectorType) cqlType; + case VECTOR_CLASS_NAME: + VectorType cqlVectorType = (VectorType) cqlType; ConvertingCodec subtypeCodec = codecFactory.createConvertingCodec( - cqlVectorType.getSubtype(), GenericType.STRING, false); + cqlVectorType.getElementType(), GenericType.STRING, false); return new StringToVectorCodec( - new CqlVectorCodec(cqlVectorType, subtypeCodec), nullStrings); + new VectorCodec(cqlVectorType, subtypeCodec), nullStrings); + default: + LOGGER.error("Unsupported custom type {}", customType.getClassName()); } } // fall through diff --git a/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringToVectorCodec.java b/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringToVectorCodec.java index 69334a4..c958aee 100644 --- a/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringToVectorCodec.java +++ b/text/src/main/java/com/datastax/oss/dsbulk/codecs/text/string/StringToVectorCodec.java @@ -16,12 +16,13 @@ package com.datastax.oss.dsbulk.codecs.text.string; import com.datastax.oss.driver.api.core.data.CqlVector; -import com.datastax.oss.driver.internal.core.type.codec.CqlVectorCodec; +import com.datastax.oss.driver.internal.core.type.codec.VectorCodec; import java.util.List; -public class StringToVectorCodec extends StringConvertingCodec> { +public class StringToVectorCodec + extends StringConvertingCodec> { - public StringToVectorCodec(CqlVectorCodec subcodec, List nullStrings) { + public StringToVectorCodec(VectorCodec subcodec, List nullStrings) { super(subcodec, nullStrings); } diff --git a/text/src/test/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeToVectorCodecTest.java b/text/src/test/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeToVectorCodecTest.java index a7eea57..c7c72d5 100644 --- a/text/src/test/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeToVectorCodecTest.java +++ b/text/src/test/java/com/datastax/oss/dsbulk/codecs/text/json/JsonNodeToVectorCodecTest.java @@ -19,11 +19,10 @@ import static com.datastax.oss.dsbulk.tests.assertions.TestAssertions.assertThat; import com.datastax.oss.driver.api.core.data.CqlVector; -import com.datastax.oss.driver.api.core.type.CqlVectorType; import com.datastax.oss.driver.api.core.type.DataTypes; import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; import com.datastax.oss.driver.api.core.type.reflect.GenericType; -import com.datastax.oss.driver.internal.core.type.codec.CqlVectorCodec; +import com.datastax.oss.driver.internal.core.type.codec.VectorCodec; import com.datastax.oss.driver.shaded.guava.common.collect.Lists; import com.datastax.oss.dsbulk.codecs.api.ConvertingCodecFactory; import com.fasterxml.jackson.databind.JsonNode; @@ -34,9 +33,9 @@ public class JsonNodeToVectorCodecTest { private final ArrayList values = Lists.newArrayList(1.1f, 2.2f, 3.3f, 4.4f, 5.5f); - private final CqlVector vector = CqlVector.builder().addAll(values).build(); - private final CqlVectorCodec vectorCodec = - new CqlVectorCodec(new CqlVectorType(DataTypes.FLOAT, 5), TypeCodecs.FLOAT); + private final CqlVector vector = CqlVector.newInstance(values); + private final VectorCodec vectorCodec = + new VectorCodec(DataTypes.vectorOf(DataTypes.FLOAT, 5), TypeCodecs.FLOAT); private final ArrayNode vectorDoc; private final ConvertingCodecFactory factory = new ConvertingCodecFactory(); @@ -87,8 +86,8 @@ void should_not_convert_from_invalid_internal() { tooFew.remove(0); assertThat(dsbulkCodec) - .cannotConvertFromInternal(CqlVector.builder().addAll(tooMany).build()) - .cannotConvertFromInternal(CqlVector.builder().addAll(tooFew).build()) + .cannotConvertFromInternal(CqlVector.newInstance((tooMany))) + .cannotConvertFromInternal(CqlVector.newInstance((tooFew))) .cannotConvertFromInternal("not a valid vector"); } } diff --git a/text/src/test/java/com/datastax/oss/dsbulk/codecs/text/string/StringToVectorCodecTest.java b/text/src/test/java/com/datastax/oss/dsbulk/codecs/text/string/StringToVectorCodecTest.java index 097e836..25665a9 100644 --- a/text/src/test/java/com/datastax/oss/dsbulk/codecs/text/string/StringToVectorCodecTest.java +++ b/text/src/test/java/com/datastax/oss/dsbulk/codecs/text/string/StringToVectorCodecTest.java @@ -18,10 +18,9 @@ import static com.datastax.oss.dsbulk.tests.assertions.TestAssertions.assertThat; import com.datastax.oss.driver.api.core.data.CqlVector; -import com.datastax.oss.driver.api.core.type.CqlVectorType; import com.datastax.oss.driver.api.core.type.DataTypes; import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; -import com.datastax.oss.driver.internal.core.type.codec.CqlVectorCodec; +import com.datastax.oss.driver.internal.core.type.codec.VectorCodec; import com.datastax.oss.driver.shaded.guava.common.collect.Lists; import java.util.ArrayList; import org.junit.jupiter.api.Disabled; @@ -30,9 +29,9 @@ public class StringToVectorCodecTest { private final ArrayList values = Lists.newArrayList(1.1f, 2.2f, 3.3f, 4.4f, 5.5f); - private final CqlVector vector = CqlVector.builder().addAll(values).build(); - private final CqlVectorCodec vectorCodec = - new CqlVectorCodec(new CqlVectorType(DataTypes.FLOAT, 5), TypeCodecs.FLOAT); + private final CqlVector vector = CqlVector.newInstance(values); + private final VectorCodec vectorCodec = + new VectorCodec(DataTypes.vectorOf(DataTypes.FLOAT, 5), TypeCodecs.FLOAT); private final StringToVectorCodec dsbulkCodec = new StringToVectorCodec(vectorCodec, Lists.newArrayList("NULL")); @@ -69,8 +68,8 @@ void should_not_convert_from_invalid_internal() { tooFew.remove(0); assertThat(dsbulkCodec) - .cannotConvertFromInternal(CqlVector.builder().addAll(tooMany).build()) - .cannotConvertFromInternal(CqlVector.builder().addAll(tooFew).build()) + .cannotConvertFromInternal(CqlVector.newInstance((tooMany))) + .cannotConvertFromInternal(CqlVector.newInstance((tooFew))) .cannotConvertFromInternal("not a valid vector"); } }