Skip to content

Commit 883c0ec

Browse files
mieslepmieslep
authored
Merge pull request #115 from datastax/issue/CDM-20
Issue/cdm 20
2 parents 18c0b6b + a0e786f commit 883c0ec

File tree

11 files changed

+641
-29
lines changed

11 files changed

+641
-29
lines changed

‎src/main/java/datastax/astra/migrate/cql/CqlHelper.java

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import com.datastax.oss.driver.api.core.cql.Row;
88
import datastax.astra.migrate.MigrateDataType;
99
import datastax.astra.migrate.Util;
10+
import datastax.astra.migrate.cql.features.*;
1011
import datastax.astra.migrate.properties.KnownProperties;
1112
import datastax.astra.migrate.properties.PropertyHelper;
1213
import org.slf4j.Logger;
@@ -30,6 +31,7 @@ public enum CQL {
3031
// Values with public Getters
3132
private final Map<CQL,String> cqlMap = new HashMap<>(CQL.values().length);
3233
private final Map<CQL,PreparedStatement> preparedStatementMap = new HashMap<>(CQL.values().length);
34+
private final Map<Featureset, Feature> featureMap = new HashMap<>(Featureset.values().length);
3335

3436
private CqlSession originSession;
3537
private CqlSession targetSession;
@@ -46,19 +48,23 @@ public CqlHelper() {
4648
this.propertyHelper = PropertyHelper.getInstance();
4749
}
4850

49-
public void initialize() {
51+
public boolean initialize() {
52+
boolean validInit = true;
5053

5154
readConsistencyLevel = Util.mapToConsistencyLevel(propertyHelper.getString(KnownProperties.READ_CL));
5255
writeConsistencyLevel = Util.mapToConsistencyLevel(propertyHelper.getString(KnownProperties.WRITE_CL));;
5356

54-
if (hasWriteTimestampFilter()) {
55-
propertyHelper.setProperty(KnownProperties.SPARK_BATCH_SIZE, 1);
57+
for (Featureset f : Featureset.values()) {
58+
if (f.toString().startsWith("TEST_")) continue; // Skip test features
59+
Feature feature = FeatureFactory.getFeature(f); // FeatureFactory throws an RTE if the feature is not implemented
60+
if (!feature.initialize(this.propertyHelper))
61+
validInit = false;
62+
else
63+
featureMap.put(f, feature);
5664
}
5765

58-
String originSelectCondition = getOriginFilterCondition();
59-
if (null != originSelectCondition && !originSelectCondition.isEmpty() && !originSelectCondition.trim().toUpperCase().startsWith("AND")) {
60-
originSelectCondition = " AND " + originSelectCondition;
61-
propertyHelper.setProperty(KnownProperties.ORIGIN_FILTER_CONDITION, originSelectCondition);
66+
if (hasWriteTimestampFilter()) {
67+
propertyHelper.setProperty(KnownProperties.SPARK_BATCH_SIZE, 1);
6268
}
6369

6470
logger.info("PARAM -- Read Consistency: {}", readConsistencyLevel);
@@ -79,6 +85,8 @@ public void initialize() {
7985
logger.info("PARAM -- ORIGIN SELECT Query used: {}", getCql(CQL.ORIGIN_SELECT));
8086
logger.info("PARAM -- TARGET INSERT Query used: {}", getCql(CQL.TARGET_INSERT));
8187
logger.info("PARAM -- TARGET SELECT Query used: {}", getCql(CQL.TARGET_SELECT_ORIGIN_BY_PK));
88+
89+
return validInit;
8290
}
8391

8492
public BoundStatement bindInsert(PreparedStatement insertStatement, Row originRow, Row targetRow) {
@@ -147,7 +155,8 @@ private String cqlOriginSelect() {
147155
String partitionKey = propertyHelper.getAsString(KnownProperties.ORIGIN_PARTITION_KEY).trim();
148156
fullSelectQuery = "SELECT " + propertyHelper.getAsString(KnownProperties.ORIGIN_COLUMN_NAMES) + selectTTLWriteTimeCols + " FROM " + getOriginKeyspaceTable() +
149157
" WHERE TOKEN(" + partitionKey + ") >= ? AND TOKEN(" + partitionKey + ") <= ? " +
150-
getOriginFilterCondition() + " ALLOW FILTERING";
158+
getFeature(Featureset.ORIGIN_FILTER).getAsString(OriginFilterCondition.Property.CONDITION) +
159+
" ALLOW FILTERING";
151160
} else {
152161
String keyBinds = "";
153162
for (String key : propertyHelper.getStringList(KnownProperties.TARGET_PRIMARY_KEY)) {
@@ -439,13 +448,20 @@ private String getTargetKeyspaceTable() {
439448
return propertyHelper.getString(KnownProperties.TARGET_KEYSPACE_TABLE);
440449
}
441450

442-
private String getOriginFilterCondition() {
443-
return propertyHelper.getString(KnownProperties.ORIGIN_FILTER_CONDITION);
444-
}
445-
446451
private void abendIfSessionsNotSet() {
447452
if (null == originSession || originSession.isClosed() || null == targetSession || targetSession.isClosed()) {
448453
throw new RuntimeException("Origin and/or Target sessions are either not set, or are closed");
449454
}
450455
}
456+
457+
public Feature getFeature(Featureset featureEnum) {
458+
return featureMap.get(featureEnum);
459+
}
460+
461+
public Boolean isFeatureEnabled(Featureset featureEnum) {
462+
if (!featureMap.containsKey(featureEnum)) {
463+
return false;
464+
}
465+
return featureMap.get(featureEnum).isEnabled();
466+
}
451467
}

‎src/main/java/datastax/astra/migrate/cql/features/AbstractFeature.java

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
package datastax.astra.migrate.cql.features;
2+
3+
import datastax.astra.migrate.MigrateDataType;
4+
import datastax.astra.migrate.properties.KnownProperties;
5+
import datastax.astra.migrate.properties.PropertyHelper;
6+
7+
import java.util.HashMap;
8+
import java.util.Map;
9+
import java.util.List;
10+
11+
public abstract class AbstractFeature implements Feature {
12+
13+
protected boolean isEnabled = false;
14+
protected boolean isInitialized = false;
15+
16+
private final Map<Enum<?>,String> strings;
17+
private final Map<Enum<?>,Boolean> booleans;
18+
private final Map<Enum<?>,Number> numbers;
19+
private final Map<Enum<?>,MigrateDataType> migrateDataTypes;
20+
private final Map<Enum<?>,List<String>> stringLists;
21+
private final Map<Enum<?>,List<Number>> numberLists;
22+
private final Map<Enum<?>,List<MigrateDataType>> migrateDataTypeLists;
23+
private final Map<Enum<?>, KnownProperties.PropertyType> propertyTypes;
24+
25+
public AbstractFeature() {
26+
this.strings = new HashMap<>();
27+
this.booleans = new HashMap<>();
28+
this.numbers = new HashMap<>();
29+
this.migrateDataTypes = new HashMap<>();
30+
this.stringLists = new HashMap<>();
31+
this.numberLists = new HashMap<>();
32+
this.migrateDataTypeLists = new HashMap<>();
33+
this.propertyTypes = new HashMap<>();
34+
}
35+
36+
@Override
37+
public boolean isEnabled() {
38+
if (!isInitialized) throw new RuntimeException("Feature not initialized");
39+
return isEnabled;
40+
}
41+
42+
@Override
43+
public String getAsString(Enum<?> key) {
44+
if (!isEnabled) return "";
45+
KnownProperties.PropertyType type = propertyTypes.get(key);
46+
if (null==type) return "";
47+
String rtn = "";
48+
switch(type) {
49+
case STRING: rtn=PropertyHelper.asString(getString(key), KnownProperties.PropertyType.STRING); break;
50+
case BOOLEAN: rtn=PropertyHelper.asString(getBoolean(key), KnownProperties.PropertyType.BOOLEAN); break;
51+
case NUMBER: rtn=PropertyHelper.asString(getNumber(key), KnownProperties.PropertyType.NUMBER); break;
52+
case MIGRATION_TYPE: rtn=PropertyHelper.asString(getMigrateDataType(key), KnownProperties.PropertyType.MIGRATION_TYPE); break;
53+
case STRING_LIST: rtn=PropertyHelper.asString(getStringList(key), KnownProperties.PropertyType.STRING_LIST); break;
54+
case NUMBER_LIST: rtn=PropertyHelper.asString(getNumberList(key), KnownProperties.PropertyType.NUMBER_LIST); break;
55+
case MIGRATION_TYPE_LIST: rtn=PropertyHelper.asString(getMigrateDataTypeList(key), KnownProperties.PropertyType.MIGRATION_TYPE_LIST); break;
56+
}
57+
return rtn;
58+
}
59+
60+
@Override
61+
public String getString(Enum<?> key) {
62+
return !isEnabled ? null : getRawString(key);
63+
}
64+
65+
protected String getRawString(Enum<?> key) {
66+
return strings.get(key);
67+
}
68+
69+
@Override
70+
public Boolean getBoolean(Enum<?> key) {
71+
return !isEnabled ? null : getRawBoolean(key);
72+
}
73+
74+
protected Boolean getRawBoolean(Enum<?> key) {
75+
return booleans.get(key);
76+
}
77+
78+
@Override
79+
public Number getNumber(Enum<?> key) {
80+
return !isEnabled ? null : getRawNumber(key);
81+
}
82+
83+
protected Number getRawNumber(Enum<?> key) {
84+
return numbers.get(key);
85+
}
86+
87+
@Override
88+
public Integer getInteger(Enum<?> key) {
89+
return !isEnabled ? null : getRawInteger(key);
90+
}
91+
92+
protected Integer getRawInteger(Enum<?> key) {
93+
return PropertyHelper.toInteger(getRawNumber(key));
94+
}
95+
96+
@Override
97+
public MigrateDataType getMigrateDataType(Enum<?> key) {
98+
return !isEnabled ? null : getRawMigrateDataType(key);
99+
}
100+
101+
protected MigrateDataType getRawMigrateDataType(Enum<?> key) {
102+
return migrateDataTypes.get(key);
103+
}
104+
105+
@Override
106+
public List<String> getStringList(Enum<?> key) {
107+
return !isEnabled ? null : getRawStringList(key);
108+
}
109+
110+
protected List<String> getRawStringList(Enum<?> key) {
111+
return stringLists.get(key);
112+
}
113+
114+
@Override
115+
public List<Number> getNumberList(Enum<?> key) {
116+
return !isEnabled ? null : getRawNumberList(key);
117+
}
118+
119+
protected List<Number> getRawNumberList(Enum<?> key) {
120+
return numberLists.get(key);
121+
}
122+
123+
@Override
124+
public List<Integer> getIntegerList(Enum<?> key) {
125+
return !isEnabled ? null : getRawIntegerList(key);
126+
}
127+
128+
protected List<Integer> getRawIntegerList(Enum<?> key) {
129+
return PropertyHelper.toIntegerList(getRawNumberList(key));
130+
}
131+
132+
@Override
133+
public List<MigrateDataType> getMigrateDataTypeList(Enum key) {
134+
return !isEnabled ? null : getRawMigrateDataTypeList(key);
135+
}
136+
137+
protected List<MigrateDataType> getRawMigrateDataTypeList(Enum key) {
138+
return migrateDataTypeLists.get(key);
139+
}
140+
141+
protected void putString(Enum<?> key, String value) {
142+
strings.put(key, value);
143+
propertyTypes.put(key, KnownProperties.PropertyType.STRING);
144+
}
145+
146+
protected void putBoolean(Enum<?> key, Boolean value) {
147+
booleans.put(key, value);
148+
propertyTypes.put(key, KnownProperties.PropertyType.BOOLEAN);
149+
}
150+
151+
protected void putNumber(Enum<?> key, Number value) {
152+
numbers.put(key, value);
153+
propertyTypes.put(key, KnownProperties.PropertyType.NUMBER);
154+
}
155+
156+
protected void putMigrateDataType(Enum<?> key, MigrateDataType value) {
157+
migrateDataTypes.put(key, value);
158+
propertyTypes.put(key, KnownProperties.PropertyType.MIGRATION_TYPE);
159+
}
160+
161+
protected void putStringList(Enum<?> key, List<String> value) {
162+
stringLists.put(key, value);
163+
propertyTypes.put(key, KnownProperties.PropertyType.STRING_LIST);
164+
}
165+
166+
protected void putNumberList(Enum<?> key, List<Number> value) {
167+
numberLists.put(key, value);
168+
propertyTypes.put(key, KnownProperties.PropertyType.NUMBER_LIST);
169+
}
170+
171+
protected void putMigrateDataTypeList(Enum<?> key, List<MigrateDataType> value) {
172+
migrateDataTypeLists.put(key, value);
173+
propertyTypes.put(key, KnownProperties.PropertyType.MIGRATION_TYPE_LIST);
174+
}
175+
176+
}

‎src/main/java/datastax/astra/migrate/cql/features/Feature.java

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
package datastax.astra.migrate.cql.features;
2+
3+
import datastax.astra.migrate.MigrateDataType;
4+
import datastax.astra.migrate.properties.PropertyHelper;
5+
import java.util.List;
6+
7+
public interface Feature {
8+
9+
/**
10+
* Initializes the feature based on properties
11+
* @param propertyHelper propertyHelper containing initialized properties
12+
* @return true if the feature is initialized and valid, false otherwise
13+
*/
14+
public boolean initialize(PropertyHelper propertyHelper);
15+
16+
/**
17+
* Indicates if feature is enabled.
18+
* @return true if the feature is enabled, false otherwise
19+
* @throws RuntimeException if the feature is not initialized
20+
*/
21+
public boolean isEnabled();
22+
23+
/**
24+
* Convenience method to get a property as a String no matter the underlying type.
25+
* @param prop property to get as a String
26+
* @return the property value as a String, but an empty string if the configured String is null, or if the feature is not enabled
27+
* @throws RuntimeException if the feature is not initialized
28+
*/
29+
public String getAsString(Enum<?> prop);
30+
31+
/**
32+
* String property getter.
33+
* @param prop property to get, it should have been set as a String
34+
* @return the property value as a String, but null if
35+
* 1. the configured String is null, or
36+
* 2. if the feature is not enabled, or
37+
* 3. if the property is not a String
38+
*/
39+
public String getString(Enum<?> prop);
40+
41+
/**
42+
* StringList property getter.
43+
* @param prop property to get, it should have been set as a StringList
44+
* @return the property value as a StringList, but null if
45+
* 1. the configured StringList is null, or
46+
* 2. if the feature is not enabled, or
47+
* 3. if the property is not a StringList
48+
*/
49+
public List<String> getStringList(Enum<?> prop);
50+
51+
/**
52+
* MigrateDataTypeList property getter.
53+
* @param prop property to get, it should have been set as a MigrateDataTypeList
54+
* @return the property value as a MigrateDataTypeList, but null if
55+
* 1. the configured MigrateDataTypeList is null, or
56+
* 2. if the feature is not enabled, or
57+
* 3. if the property is not a MigrateDataTypeList
58+
*/
59+
public List<MigrateDataType> getMigrateDataTypeList(Enum<?> prop);
60+
61+
/**
62+
* NumberList property getter.
63+
* @param prop property to get, it should have been set as a NumberList
64+
* @return the property value as a NumberList, but null if
65+
* 1. the configured NumberList is null, or
66+
* 2. if the feature is not enabled, or
67+
* 3. if the property is not a NumberList
68+
*/
69+
public List<Number> getNumberList(Enum<?> prop);
70+
71+
/**
72+
* IntegerList property getter.
73+
* @param prop property to get, it should have been set as a NumberList
74+
* @return the property value as a NumberList, but null if
75+
* 1. the configured NumberList is null, or
76+
* 2. if the feature is not enabled, or
77+
* 3. if the property is not a NumberList
78+
* 4. if the any value within the NumberList cannot be represented as an Integer
79+
*/
80+
public List<Integer> getIntegerList(Enum<?> prop);
81+
82+
/**
83+
* Boolean property getter.
84+
* @param prop property to get, it should have been set as a Boolean
85+
* @return the property value as a Boolean, but null if
86+
* 1. the configured Boolean is null, or
87+
* 2. if the feature is not enabled, or
88+
* 3. if the property is not a Boolean
89+
*/
90+
public Boolean getBoolean(Enum<?> prop);
91+
92+
/**
93+
* Integer property getter.
94+
* @param prop property to get, it should have been set as a Number
95+
* @return the property value as a Number, but null if
96+
* 1. the configured Number is null, or
97+
* 2. if the feature is not enabled, or
98+
* 3. if the property is not a Number
99+
* 4. if the value cannot be represented as an Integer
100+
*/
101+
public Integer getInteger(Enum<?> prop);
102+
103+
/**
104+
* Number property getter.
105+
* @param prop property to get, it should have been set as a Number
106+
* @return the property value as a Number, but null if
107+
* 1. the configured Number is null, or
108+
* 2. if the feature is not enabled, or
109+
* 3. if the property is not a Number
110+
*/
111+
public Number getNumber(Enum<?> prop);
112+
113+
/**
114+
* MigrateDataType property getter.
115+
* @param prop property to get, it should have been set as a MigrateDataType
116+
* @return the property value as a MigrateDataType, but null if
117+
* 1. the configured MigrateDataType is null, or
118+
* 2. if the feature is not enabled, or
119+
* 3. if the property is not a MigrateDataType
120+
*/
121+
public MigrateDataType getMigrateDataType(Enum<?> prop);
122+
}

0 commit comments

Comments
 (0)