Skip to content

Commit 031a5d7

Browse files
pravinbhatpravinbhat
authored
Merge pull request #155 from datastax/feature/no-ttl-and-writetime
CDM-62 handle no TTL & Writetime graciously
2 parents ec6041c + 6f3a390 commit 031a5d7

File tree

5 files changed

+76
-67
lines changed

5 files changed

+76
-67
lines changed

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
<properties>
1010
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
11-
<revision>3.4.4</revision>
11+
<revision>3.4.5</revision>
1212
<scala.version>2.12.17</scala.version>
1313
<scala.main.version>2.12</scala.main.version>
1414
<spark.version>3.3.1</spark.version>

‎src/main/java/datastax/astra/migrate/CopyJobSession.java

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ public class CopyJobSession extends AbstractJobSession {
2323

2424
protected CopyJobSession(CqlSession sourceSession, CqlSession astraSession, SparkConf sc) {
2525
super(sourceSession, astraSession, sc);
26+
handleNoTtlWriteTime(Boolean.parseBoolean(Util.getSparkPropOr(sc, "spark.migrate.disableTtlAndWriteTime", "false")));
27+
2628
filterData = Boolean.parseBoolean(sc.get("spark.origin.FilterData", "false"));
2729
filterColName = Util.getSparkPropOrEmpty(sc, "spark.origin.FilterColumn");
2830
filterColType = Util.getSparkPropOrEmpty(sc, "spark.origin.FilterColumnType");
@@ -42,6 +44,18 @@ public static CopyJobSession getInstance(CqlSession sourceSession, CqlSession as
4244
return copyJobSession;
4345
}
4446

47+
private void handleNoTtlWriteTime(boolean disableTtlAndWriteTime) {
48+
if (ttlWTCols.isEmpty()) {
49+
if (disableTtlAndWriteTime) {
50+
logger.warn("***** NO NON-FROZEN NON-PRIMARY-KEY COLUMNS FOUND TO DERIVE TTL AND WRITETIME! PROCEEDING WITHOUT MIGRATING TTL AND WRITETIME!! *****");
51+
} else {
52+
logger.error("***** NO NON-FROZEN NON-PRIMARY-KEY COLUMNS FOUND TO DERIVE TTL AND WRITETIME! EXITING JOB!! *****");
53+
logger.error("***** IF YOU WANT TO PROCEED WITH MIGRATION WITHOUT USING TTL AND WRITETIMES FROM ORIGIN, RERUN THE JOB USING OPTION --conf spark.migrate.disableTtlAndWriteTime=true !! *****");
54+
System.exit(-1);
55+
}
56+
}
57+
}
58+
4559
public void getDataAndInsert(BigInteger min, BigInteger max) {
4660
logger.info("ThreadID: {} Processing min: {} max: {}", Thread.currentThread().getId(), min, max);
4761
boolean done = false;
@@ -69,7 +83,8 @@ public void getDataAndInsert(BigInteger min, BigInteger max) {
6983
}
7084

7185
if (filterData) {
72-
String col = (String) getData(new TypeInfo(filterColType), filterColIndex, sourceRow);
86+
TypeInfo typeInfo = tableInfo.getColumns().get(filterColIndex).getTypeInfo();
87+
String col = (String) getData(typeInfo, filterColIndex, sourceRow);
7388
if (col.trim().equalsIgnoreCase(filterColValue)) {
7489
logger.warn("Skipping row and filtering out: {}", getKey(sourceRow, tableInfo));
7590
skipCnt++;
@@ -118,7 +133,8 @@ public void getDataAndInsert(BigInteger min, BigInteger max) {
118133
}
119134

120135
if (filterData) {
121-
String colValue = (String) getData(new TypeInfo(filterColType), filterColIndex, sourceRow);
136+
TypeInfo typeInfo = tableInfo.getColumns().get(filterColIndex).getTypeInfo();
137+
String colValue = (String) getData(typeInfo, filterColIndex, sourceRow);
122138
if (colValue.trim().equalsIgnoreCase(filterColValue)) {
123139
logger.warn("Skipping row and filtering out: {}", getKey(sourceRow, tableInfo));
124140
skipCnt++;

‎src/main/java/datastax/astra/migrate/DiffJobSession.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
import java.util.stream.IntStream;
2222
import java.util.stream.StreamSupport;
2323

24-
public class DiffJobSession extends CopyJobSession {
24+
public class DiffJobSession extends AbstractJobSession {
2525

2626
private static DiffJobSession diffJobSession;
2727
private final AtomicLong mismatchCounter = new AtomicLong(0);
@@ -111,9 +111,6 @@ private void diffAndClear(Map<Row, CompletionStage<AsyncResultSet>> srcToTargetR
111111
} catch (ExecutionException | InterruptedException e) {
112112
logger.error("Could not perform diff for Key: {}", getKey(srcRow, tableInfo), e);
113113
throw new RuntimeException(e);
114-
} catch (Exception ee) {
115-
logger.error("Could not perform diff for Key: {}", getKey(srcRow, tableInfo), ee);
116-
throw new RuntimeException(ee);
117114
}
118115
}
119116
srcToTargetRowMap.clear();

‎src/main/java/datastax/astra/migrate/schema/ColumnInfo.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import lombok.Data;
66

77
import java.util.List;
8-
import java.util.Locale;
98
import java.util.Map;
109
import java.util.Set;
1110

@@ -33,8 +32,8 @@ public class ColumnInfo {
3332
if (typeInfo.getTypeClass().equals(UdtValue.class)) {
3433
isUDT = true;
3534
}
36-
if (cm.getType().toString().toLowerCase(Locale.ROOT).contains(", frozen")) {
37-
isFrozen = true;
38-
}
35+
36+
isFrozen = typeInfo.isFrozen();
3937
}
38+
4039
}

‎src/main/java/datastax/astra/migrate/schema/TypeInfo.java

Lines changed: 53 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import com.datastax.oss.driver.api.core.data.TupleValue;
44
import com.datastax.oss.driver.api.core.data.UdtValue;
5-
import com.datastax.oss.driver.api.core.type.DataType;
5+
import com.datastax.oss.driver.api.core.type.*;
66
import lombok.Data;
77

88
import java.math.BigDecimal;
@@ -15,75 +15,72 @@
1515

1616
@Data
1717
public class TypeInfo {
18-
private static Map<String, Class> typeMap = loadTypeMap();
18+
private static Map<DataType, Class> cqlToJavaTypeMap = loadCqlToJavaTypeMap();
1919
private Class typeClass = Object.class;
2020
private List<Class> subTypes = new ArrayList<Class>();
2121
private boolean isCounter = false;
22+
private boolean isFrozen = false;
2223

2324
public TypeInfo(DataType dataType) {
24-
this(dataType.toString());
25-
}
26-
27-
public TypeInfo(String dataTypeStr) {
28-
int sIdx = dataTypeStr.indexOf('(');
29-
int eIdx = -1;
30-
if (sIdx != -1) {
31-
eIdx = dataTypeStr.substring(sIdx + 1).indexOf('(');
32-
if (eIdx == -1) {
33-
eIdx = dataTypeStr.substring(sIdx + 1).indexOf(',');
34-
}
35-
eIdx += sIdx + 1;
36-
}
25+
typeClass = getDataTypeToType(dataType);
3726

38-
if (dataTypeStr.toLowerCase(Locale.ROOT).startsWith("list")) {
39-
typeClass = List.class;
40-
subTypes.add(typeMap.get(dataTypeStr.substring(sIdx + 1, eIdx).toLowerCase(Locale.ROOT)));
41-
} else if (dataTypeStr.toLowerCase(Locale.ROOT).startsWith("set")) {
42-
typeClass = Set.class;
43-
subTypes.add(typeMap.get(dataTypeStr.substring(sIdx + 1, eIdx).toLowerCase(Locale.ROOT)));
44-
} else if (dataTypeStr.toLowerCase(Locale.ROOT).startsWith("map")) {
45-
typeClass = Map.class;
46-
subTypes.add(typeMap.get(dataTypeStr.substring(sIdx + 1, dataTypeStr.indexOf("=>")).trim().toLowerCase(Locale.ROOT)));
47-
subTypes.add(typeMap.get(dataTypeStr.substring(dataTypeStr.indexOf("=>") + 2, dataTypeStr.indexOf(',')).trim().toLowerCase(Locale.ROOT)));
48-
} else if (dataTypeStr.toLowerCase(Locale.ROOT).startsWith("udt")) {
49-
typeClass = UdtValue.class;
50-
} else if (dataTypeStr.toLowerCase(Locale.ROOT).startsWith("tuple")) {
51-
typeClass = TupleValue.class;
52-
} else if (dataTypeStr.toLowerCase(Locale.ROOT).startsWith("counter")) {
53-
typeClass = Long.class;
27+
if (dataType instanceof UserDefinedType) {
28+
isFrozen = ((UserDefinedType) dataType).isFrozen();
29+
} else if (dataType instanceof ListType) {
30+
subTypes.add(getDataTypeToType(((ListType) dataType).getElementType()));
31+
isFrozen = ((ListType) dataType).isFrozen();
32+
} else if (dataType instanceof SetType) {
33+
subTypes.add(getDataTypeToType(((SetType) dataType).getElementType()));
34+
isFrozen = ((SetType) dataType).isFrozen();
35+
} else if (dataType instanceof MapType) {
36+
subTypes.add(getDataTypeToType(((MapType) dataType).getKeyType()));
37+
subTypes.add(getDataTypeToType(((MapType) dataType).getValueType()));
38+
isFrozen = ((MapType) dataType).isFrozen();
39+
} else if (DataTypes.COUNTER.equals(dataType)) {
5440
isCounter = true;
55-
} else {
56-
typeClass = typeMap.get(dataTypeStr.toLowerCase(Locale.ROOT));
5741
}
5842
}
5943

60-
private static Map loadTypeMap() {
61-
Map typeMap = new HashMap<>();
62-
typeMap.put("ascii", String.class);
63-
typeMap.put("bigint", Long.class);
64-
typeMap.put("blob", ByteBuffer.class);
65-
typeMap.put("boolean", Boolean.class);
66-
typeMap.put("counter", Long.class);
67-
typeMap.put("date", LocalDate.class);
68-
typeMap.put("decimal", BigDecimal.class);
69-
typeMap.put("double", Double.class);
70-
typeMap.put("float", Float.class);
71-
typeMap.put("int", Integer.class);
72-
typeMap.put("inet", String.class);
73-
typeMap.put("smallint", Short.class);
74-
typeMap.put("text", String.class);
75-
typeMap.put("time", LocalTime.class);
76-
typeMap.put("timestamp", Instant.class);
77-
typeMap.put("timeuuid", UUID.class);
78-
typeMap.put("tinyint", Byte.class);
79-
typeMap.put("udt", UdtValue.class);
80-
typeMap.put("uuid", UUID.class);
81-
typeMap.put("varchar", String.class);
82-
typeMap.put("varint", BigInteger.class);
44+
private static Map loadCqlToJavaTypeMap() {
45+
Map<DataType, Class> typeMap = new HashMap<>();
46+
typeMap.put(DataTypes.ASCII, String.class);
47+
typeMap.put(DataTypes.BIGINT, Long.class);
48+
typeMap.put(DataTypes.BLOB, ByteBuffer.class);
49+
typeMap.put(DataTypes.BOOLEAN, Boolean.class);
50+
typeMap.put(DataTypes.COUNTER, Long.class);
51+
typeMap.put(DataTypes.DATE, LocalDate.class);
52+
typeMap.put(DataTypes.DECIMAL, BigDecimal.class);
53+
typeMap.put(DataTypes.DOUBLE, Double.class);
54+
typeMap.put(DataTypes.FLOAT, Float.class);
55+
typeMap.put(DataTypes.INT, Integer.class);
56+
typeMap.put(DataTypes.INET, String.class);
57+
typeMap.put(DataTypes.SMALLINT, Short.class);
58+
typeMap.put(DataTypes.TEXT, String.class);
59+
typeMap.put(DataTypes.TIME, LocalTime.class);
60+
typeMap.put(DataTypes.TIMESTAMP, Instant.class);
61+
typeMap.put(DataTypes.TIMEUUID, UUID.class);
62+
typeMap.put(DataTypes.TINYINT, Byte.class);
63+
typeMap.put(DataTypes.UUID, UUID.class);
64+
typeMap.put(DataTypes.VARINT, BigInteger.class);
8365

8466
return typeMap;
8567
}
8668

69+
private Class getDataTypeToType(DataType dataType) {
70+
if (dataType instanceof UserDefinedType) return UdtValue.class;
71+
else if (dataType instanceof ListType) {
72+
return List.class;
73+
} else if (dataType instanceof SetType) {
74+
return Set.class;
75+
} else if (dataType instanceof MapType) {
76+
return Map.class;
77+
} else if (dataType instanceof TupleType) {
78+
return TupleValue.class;
79+
}
80+
81+
return cqlToJavaTypeMap.get(dataType);
82+
}
83+
8784
public String toString() {
8885
return "Type: " + typeClass.toString() + " SubTypes: " + subTypes.toString();
8986
}

0 commit comments

Comments
 (0)