Skip to content

Commit 476e5fc

Browse files
mieslepmieslep
committed
CDM-37 adding bidi codecs, and enabling DiffJob to do conversion as necessary
1 parent 9683b10 commit 476e5fc

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1633
-500
lines changed

‎SIT/features/03_codec/expected.out

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11

2-
key | val_bigint | val_decimal | val_int | val_timestamp
3-
------+---------------------+-------------+---------+---------------------------------
4-
key1 | 9223372036854775807 | 3.14 | 1234 | 2004-06-16 11:00:00.000000+0000
5-
key3 | 3141592653589793 | 5.14 | 123456 | 2099-06-16 11:00:00.000000+0000
6-
key2 | 2147483648 | 4.14 | 12345 | 2099-06-16 11:00:00.000000+0000
2+
key | val_bigint | val_decimal | val_double | val_int | val_timestamp
3+
------+---------------------+-------------+------------+---------+---------------------------------
4+
key1 | 9223372036854775807 | 3.14 | 2.1475e+10 | 1234 | 2004-06-16 11:00:00.000000+0000
5+
key3 | 3141592653589793 | 5.14 | 2.1475e+10 | 123456 | 2099-06-16 11:00:00.000000+0000
6+
key2 | 2147483648 | 4.14 | 2.1475e+10 | 12345 | 2099-06-16 11:00:00.000000+0000
77

88
(3 rows)

‎SIT/features/03_codec/fix.properties

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,13 @@ spark.cdm.schema.origin.keyspaceTable origin.codec
55
spark.cdm.schema.target.keyspaceTable target.codec
66
spark.cdm.perfops.numParts 1
77

8-
spark.cdm.schema.origin.column.names key,val_int,val_bigint,val_timestamp,val_decimal
8+
spark.cdm.schema.origin.column.names key,val_int,val_bigint,val_timestamp,val_decimal,val_double
99
spark.cdm.schema.origin.column.partition.names key
1010
spark.cdm.schema.target.column.id.names key
11-
spark.cdm.schema.origin.column.types 0,0,0,0,0
11+
spark.cdm.schema.origin.column.types 0,0,0,0,0,0
12+
spark.cdm.schema.target.column.types 0,1,2,4,14,3
1213

13-
spark.cdm.transform.codecs CQL_INT_TO_STRING,CQL_BIGINT_TO_STRING,CQL_DECIMAL_TO_STRING,CQL_TIMESTAMP_TO_STRING_FORMAT
14+
spark.cdm.transform.codecs INT_STRING,DOUBLE_STRING,BIGINT_STRING,DECIMAL_STRING,TIMESTAMP_STRING_FORMAT
1415
spark.cdm.transform.codecs.timestamp.string.format yyMMddHHmmss
1516

1617
spark.cdm.autocorrect.missing true

‎SIT/features/03_codec/migrate.properties

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,12 @@ spark.cdm.schema.origin.keyspaceTable origin.codec
55
spark.cdm.schema.target.keyspaceTable target.codec
66
spark.cdm.perfops.numParts 1
77

8-
spark.cdm.schema.origin.column.names key,val_int,val_bigint,val_timestamp,val_decimal
8+
spark.cdm.schema.origin.column.names key,val_int,val_bigint,val_timestamp,val_decimal,val_double
99
spark.cdm.schema.origin.column.partition.names key
1010
spark.cdm.schema.target.column.id.names key
11-
spark.cdm.schema.origin.column.types 0,0,0,0,0
11+
spark.cdm.schema.origin.column.types 0,0,0,0,0,0
12+
spark.cdm.schema.target.column.types 0,1,2,4,14,3
1213

13-
spark.cdm.transform.codecs CQL_INT_TO_STRING,CQL_BIGINT_TO_STRING,CQL_DECIMAL_TO_STRING,CQL_TIMESTAMP_TO_STRING_FORMAT
14+
spark.cdm.transform.codecs INT_STRING,DOUBLE_STRING,BIGINT_STRING,DECIMAL_STRING,TIMESTAMP_STRING_FORMAT
1415
spark.cdm.transform.codecs.timestamp.string.format yyMMddHHmmss
1516

‎SIT/features/03_codec/setup.cql

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
DROP TABLE IF EXISTS origin.codec;
2-
CREATE TABLE origin.codec(key text, val_int text, val_bigint text, val_timestamp text, val_decimal text, PRIMARY KEY (key));
3-
INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal)
4-
VALUES ('key1','1234' ,'9223372036854775807','040616110000', '3.14');
5-
INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal)
6-
VALUES ('key2','12345' ,'2147483648' ,'990616110000', '4.14');
7-
INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal)
8-
VALUES ('key3','123456','3141592653589793' ,'990616110000', '5.14');
2+
CREATE TABLE origin.codec(key text, val_int text, val_bigint text, val_timestamp text, val_decimal text, val_double text, PRIMARY KEY (key));
3+
INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal,val_double)
4+
VALUES ('key1','1234' ,'9223372036854775807','040616110000', '3.14', '21474836470.7');
5+
INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal,val_double)
6+
VALUES ('key2','12345' ,'2147483648' ,'990616110000', '4.14', '21474836470.7');
7+
INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal,val_double)
8+
VALUES ('key3','123456','3141592653589793' ,'990616110000', '5.14', '21474836470.7');
99
SELECT * FROM origin.codec;
1010

1111
DROP TABLE IF EXISTS target.codec;
12-
CREATE TABLE target.codec(key text, val_int int, val_bigint bigint, val_timestamp timestamp, val_decimal decimal, PRIMARY KEY (key));
12+
CREATE TABLE target.codec(key text, val_int int, val_bigint bigint, val_timestamp timestamp, val_decimal decimal, val_double double, PRIMARY KEY (key));

‎SIT/regression/01_explode_map_with_constants/fix.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ spark.cdm.feature.constantColumns.names const1,const2
1818
spark.cdm.feature.constantColumns.types 0,1
1919
spark.cdm.feature.constantColumns.values 'abcd',1234
2020

21-
spark.cdm.transform.codecs CQL_TIMESTAMP_TO_STRING_MILLIS
21+
spark.cdm.transform.codecs TIMESTAMP_STRING_MILLIS
2222

2323
spark.cdm.autocorrect.missing true
2424
spark.cdm.autocorrect.mismatch true

‎SIT/regression/01_explode_map_with_constants/migrate.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,6 @@ spark.cdm.feature.constantColumns.names const1,const2
1818
spark.cdm.feature.constantColumns.types 0,1
1919
spark.cdm.feature.constantColumns.values 'abcd',1234
2020

21-
spark.cdm.transform.codecs CQL_TIMESTAMP_TO_STRING_MILLIS
21+
spark.cdm.transform.codecs TIMESTAMP_STRING_MILLIS
2222

2323

‎src/main/java/datastax/cdm/cql/CqlHelper.java

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import com.datastax.oss.driver.api.core.cql.Row;
66
import com.datastax.oss.driver.api.core.type.codec.TypeCodec;
77
import com.datastax.oss.driver.api.core.type.codec.registry.MutableCodecRegistry;
8-
import datastax.cdm.cql.codec.AbstractBaseCodec;
98
import datastax.cdm.cql.codec.CodecFactory;
109
import datastax.cdm.cql.codec.Codecset;
1110
import datastax.cdm.job.MigrateDataType;
@@ -122,11 +121,13 @@ private void registerTargetCodecs() {
122121
(MutableCodecRegistry) targetSession.getContext().getCodecRegistry();
123122

124123
StringBuilder sb = new StringBuilder("PARAM -- Codecs Enabled: ");
125-
for (String codec : codecList) {
126-
TypeCodec<?> typeCodec = CodecFactory.getCodec(propertyHelper, this, Codecset.valueOf(codec));
127-
registry.register(typeCodec);
128-
codecMap.put(Codecset.valueOf(codec), typeCodec);
129-
sb.append(codec).append(" ");
124+
for (String codecString : codecList) {
125+
Codecset codecEnum = Codecset.valueOf(codecString);
126+
for (TypeCodec<?> codec : CodecFactory.getCodecs(propertyHelper, this, codecEnum)) {
127+
registry.register(codec);
128+
codecMap.put(codecEnum, codec);
129+
}
130+
sb.append(codecString).append(" ");
130131
}
131132
logger.info(sb.toString());
132133
}

‎src/main/java/datastax/cdm/cql/codec/AbstractBaseCodec.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,5 @@ public AbstractBaseCodec(PropertyHelper propertyHelper, CqlHelper cqlHelper) {
1212
this.propertyHelper = propertyHelper;
1313
this.cqlHelper = cqlHelper;
1414
}
15+
1516
}

‎src/main/java/datastax/cdm/cql/codec/CqlBigintToStringCodec.java renamed to ‎src/main/java/datastax/cdm/cql/codec/BIGINT_StringCodec.java

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,9 @@
1111

1212
import java.nio.ByteBuffer;
1313

14-
/**
15-
* This codec converts a CQL BigInt to a Java String.
16-
*/
17-
public class CqlBigintToStringCodec extends AbstractBaseCodec<String> {
14+
public class BIGINT_StringCodec extends AbstractBaseCodec<String> {
1815

19-
public CqlBigintToStringCodec(PropertyHelper propertyHelper, CqlHelper cqlHelper) {
16+
public BIGINT_StringCodec(PropertyHelper propertyHelper, CqlHelper cqlHelper) {
2017
super(propertyHelper, cqlHelper);
2118
}
2219

@@ -57,5 +54,6 @@ public String parse(String value) {
5754
Long longValue = TypeCodecs.BIGINT.parse(value);
5855
return longValue == null ? null : longValue.toString();
5956
}
57+
6058
}
6159

‎src/main/java/datastax/cdm/cql/codec/CodecFactory.java

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,18 @@
44
import datastax.cdm.cql.CqlHelper;
55
import datastax.cdm.properties.PropertyHelper;
66

7+
import java.util.Arrays;
8+
import java.util.List;
9+
710
public class CodecFactory {
8-
public static TypeCodec<?> getCodec(PropertyHelper propertyHelper, CqlHelper cqlHelper, Codecset codec) {
11+
public static List<TypeCodec<?>> getCodecs(PropertyHelper propertyHelper, CqlHelper cqlHelper, Codecset codec) {
912
switch (codec) {
10-
case CQL_INT_TO_STRING: return new CqlIntToStringCodec(propertyHelper, cqlHelper);
11-
case CQL_BIGINT_TO_STRING: return new CqlBigintToStringCodec(propertyHelper, cqlHelper);
12-
case CQL_DECIMAL_TO_STRING: return new CqlDecimalToStringCodec(propertyHelper, cqlHelper);
13-
case CQL_TIMESTAMP_TO_STRING_MILLIS: return new CqlTimestampToString_Millis_Codec(propertyHelper, cqlHelper);
14-
case CQL_TIMESTAMP_TO_STRING_FORMAT: return new CqlTimestampToString_Format_Codec(propertyHelper, cqlHelper);
13+
case INT_STRING: return Arrays.asList(new INT_StringCodec(propertyHelper, cqlHelper), new TEXT_IntegerCodec(propertyHelper, cqlHelper));
14+
case DOUBLE_STRING: return Arrays.asList(new DOUBLE_StringCodec(propertyHelper, cqlHelper), new TEXT_DoubleCodec(propertyHelper, cqlHelper));
15+
case BIGINT_STRING: return Arrays.asList(new BIGINT_StringCodec(propertyHelper, cqlHelper), new TEXT_LongCodec(propertyHelper, cqlHelper));
16+
case DECIMAL_STRING: return Arrays.asList(new DECIMAL_StringCodec(propertyHelper, cqlHelper), new TEXT_BigDecimalCodec(propertyHelper, cqlHelper));
17+
case TIMESTAMP_STRING_MILLIS: return Arrays.asList(new TIMESTAMP_StringMillisCodec(propertyHelper, cqlHelper), new TEXTMillis_InstantCodec(propertyHelper, cqlHelper));
18+
case TIMESTAMP_STRING_FORMAT: return Arrays.asList(new TIMESTAMP_StringFormatCodec(propertyHelper, cqlHelper), new TEXTFormat_InstantCodec(propertyHelper, cqlHelper));
1519
default:
1620
throw new IllegalArgumentException("Unknown codec: " + codec);
1721
}

0 commit comments

Comments
 (0)