Skip to content

Commit 9010922

Browse files
mieslepmieslep
committed
CDM-22 simplifying further to make CDM-23 easier to implement
1 parent 691e744 commit 9010922

File tree

9 files changed

+166
-119
lines changed

9 files changed

+166
-119
lines changed

‎src/main/java/datastax/astra/migrate/MigrateDataType.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ public class MigrateDataType {
1919
private boolean isValid = false;
2020
private static int minType = 0;
2121
private static int maxType = 19;
22+
public static final int UNKNOWN_TYPE = 99;
2223

2324
public MigrateDataType(String dataType) {
2425
dataTypeString = dataType;
@@ -38,7 +39,7 @@ public MigrateDataType(String dataType) {
3839
}
3940
this.typeClass = getType(this.type);
4041

41-
if (this.type >= minType && this.type <= maxType) {
42+
if ((this.type >= minType && this.type <= maxType) || this.type == UNKNOWN_TYPE) {
4243
isValid = true;
4344
for (Object o : subTypes) {
4445
if (null == o || Object.class == o) {
@@ -51,6 +52,13 @@ public MigrateDataType(String dataType) {
5152
}
5253
}
5354

55+
public MigrateDataType() {
56+
this.dataTypeString = "UNKNOWN";
57+
this.type = UNKNOWN_TYPE;
58+
this.typeClass = getType(this.type);
59+
isValid = true;
60+
}
61+
5462
private int typeAsInt(String dataType) {
5563
int rtn = -1;
5664
try {

‎src/main/java/datastax/astra/migrate/cql/CqlHelper.java

Lines changed: 17 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,13 @@ public boolean initialize() {
6363
featureMap.put(f, feature);
6464
}
6565

66+
for (Featureset f : Featureset.values()) {
67+
if (f.toString().startsWith("TEST_")) continue; // Skip test features
68+
Feature feature = getFeature(f);
69+
if (null!=feature && feature.isEnabled())
70+
feature.alterProperties(this.propertyHelper);
71+
}
72+
6673
if (hasWriteTimestampFilter()) {
6774
propertyHelper.setProperty(KnownProperties.SPARK_BATCH_SIZE, 1);
6875
}
@@ -208,27 +215,17 @@ private String cqlTargetInsert() {
208215

209216
private String cqlTargetSelectOriginByPK() {
210217
String keyBinds = "";
211-
if (!featureMap.get(Featureset.CONSTANT_COLUMNS).isEnabled()){
212-
for (String key : propertyHelper.getStringList(KnownProperties.TARGET_PRIMARY_KEY)) {
213-
if (keyBinds.isEmpty()) {
214-
keyBinds = key + "=?";
215-
} else {
216-
keyBinds += " AND " + key + "=?";
217-
}
218-
}
219-
}
220-
else {
221-
List<String> constantColumnNames = featureMap.get(Featureset.CONSTANT_COLUMNS).getStringList(ConstantColumns.Property.COLUMN_NAMES);
222-
List<String> constantColumnValues = featureMap.get(Featureset.CONSTANT_COLUMNS).getStringList(ConstantColumns.Property.COLUMN_VALUES);
223-
for (String key : propertyHelper.getStringList(KnownProperties.TARGET_PRIMARY_KEY)) {
224-
String whatToEqual = constantColumnNames.contains(key) ? constantColumnValues.get(constantColumnNames.indexOf(key)) : "?";
225-
if (keyBinds.isEmpty()) {
226-
keyBinds = key + "=" + whatToEqual;
227-
} else {
228-
keyBinds += " AND " + key + "=" + whatToEqual;
229-
}
218+
for (String key : propertyHelper.getStringList(KnownProperties.TARGET_PRIMARY_KEY)) {
219+
if (keyBinds.isEmpty()) {
220+
keyBinds = key + "=?";
221+
} else {
222+
keyBinds += " AND " + key + "=?";
230223
}
231224
}
225+
226+
// This will be empty string if feature is disabled
227+
keyBinds += featureMap.get(Featureset.CONSTANT_COLUMNS).getAsString(ConstantColumns.Property.WHERE_CLAUSE);
228+
232229
return "SELECT " + propertyHelper.getAsString(KnownProperties.TARGET_COLUMN_NAMES) + " FROM " + getTargetKeyspaceTable() + " WHERE " + keyBinds;
233230
}
234231

@@ -434,12 +431,7 @@ public List<MigrateDataType> getSelectColTypes() {
434431
}
435432

436433
public List<MigrateDataType> getIdColTypes() {
437-
List<MigrateDataType> rtn = propertyHelper.getMigrationTypeList(KnownProperties.TARGET_PRIMARY_KEY_TYPES);
438-
rtn = (List<MigrateDataType>) featureMap.get(Featureset.CONSTANT_COLUMNS).
439-
featureFunction(ConstantColumns.Function.TARGET_PK_WITHOUT_CONSTANTS,
440-
rtn,
441-
propertyHelper.getStringList(KnownProperties.TARGET_PRIMARY_KEY));
442-
return rtn;
434+
return propertyHelper.getMigrationTypeList(KnownProperties.TARGET_PRIMARY_KEY_TYPES);
443435
}
444436

445437
// These getters have no usage outside this class

‎src/main/java/datastax/astra/migrate/cql/features/AbstractFeature.java

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ public AbstractFeature() {
3333
this.propertyTypes = new HashMap<>();
3434
}
3535

36+
@Override
37+
public PropertyHelper alterProperties(PropertyHelper helper) {
38+
// Not implemented by default
39+
return helper;
40+
}
41+
3642
@Override
3743
public boolean isEnabled() {
3844
if (!isInitialized) throw new RuntimeException("Feature not initialized");
@@ -138,11 +144,6 @@ protected List<MigrateDataType> getRawMigrateDataTypeList(Enum key) {
138144
return migrateDataTypeLists.get(key);
139145
}
140146

141-
@Override
142-
public Object featureFunction(Enum<?> function, Object... args) {
143-
return null;
144-
}
145-
146147
protected void putString(Enum<?> key, String value) {
147148
strings.put(key, value);
148149
propertyTypes.put(key, KnownProperties.PropertyType.STRING);

‎src/main/java/datastax/astra/migrate/cql/features/ConstantColumns.java

Lines changed: 59 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,9 @@ public class ConstantColumns extends AbstractFeature {
1616
public enum Property {
1717
COLUMN_NAMES,
1818
COLUMN_TYPES,
19-
COLUMN_VALUES
20-
}
21-
22-
public enum Function {
23-
TARGET_PK_WITHOUT_CONSTANTS,
24-
TEST_FUNCTION
19+
COLUMN_VALUES,
20+
SPLIT_REGEX,
21+
WHERE_CLAUSE
2522
}
2623

2724
private boolean valid = true;
@@ -34,31 +31,30 @@ public boolean initialize(PropertyHelper propertyHelper) {
3431
List<MigrateDataType> columnTypes = propertyHelper.getMigrationTypeList(KnownProperties.CONSTANT_COLUMN_TYPES);
3532
putMigrateDataTypeList(Property.COLUMN_TYPES, columnTypes);
3633

34+
String splitRegex = propertyHelper.getString(KnownProperties.CONSTANT_COLUMN_SPLIT_REGEX);
35+
putString(Property.SPLIT_REGEX, splitRegex);
36+
3737
List<String> columnValues = columnValues(
3838
propertyHelper.getString(KnownProperties.CONSTANT_COLUMN_VALUES),
39-
propertyHelper.getString(KnownProperties.CONSTANT_COLUMN_SPLIT_REGEX));
39+
splitRegex);
4040
putStringList(Property.COLUMN_VALUES, columnValues);
4141

42+
setWhereClause(propertyHelper);
43+
44+
valid = isValid();
4245
isInitialized = true;
43-
valid = isValid(propertyHelper);
4446
isEnabled = valid && null!=columnNames && !columnNames.isEmpty();
4547
return valid;
4648
}
4749

4850
@Override
49-
public Object featureFunction(Enum<?> function, Object... args) {
50-
switch ((Function) function) {
51-
case TARGET_PK_WITHOUT_CONSTANTS:
52-
// args[] should be List<MigrateDataType> targetPrimaryKeyTypes, List<String> targetPrimaryKeyNames
53-
if (null==args || args.length!=2 || null==args[0] || null==args[1])
54-
throw new IllegalArgumentException("Expected 2 not-null arguments, got " + (null==args ? "1" : args.length));
55-
if (!(args[0] instanceof List<?>) || ((List<?>) args[0]).isEmpty() || !(((List<?>) args[0]).get(0) instanceof MigrateDataType))
56-
throw new IllegalArgumentException("First argument should be a non-empty List<MigrateDataType>, got " + args[0]);
57-
if (!(args[1] instanceof List<?>) || ((List<?>) args[1]).isEmpty() || !(((List<?>) args[1]).get(0) instanceof String))
58-
throw new IllegalArgumentException("Second argument should be a non-empty List<String>, got " + args[1]);
59-
return targetPrimaryKeyTypesWithoutConstantColumns((List<MigrateDataType>)args[0], (List<String>)args[1]);
60-
}
61-
return null;
51+
public PropertyHelper alterProperties(PropertyHelper helper) {
52+
if (!valid) return null;
53+
if (!isEnabled) return helper;
54+
55+
clean_targetPK(helper);
56+
57+
return helper;
6258
}
6359

6460
private List<String> columnValues(String columnValueString, String regexString) {
@@ -75,26 +71,56 @@ private List<String> columnValues(String columnValueString, String regexString)
7571
return columnValues;
7672
}
7773

78-
private List<MigrateDataType> targetPrimaryKeyTypesWithoutConstantColumns(List<MigrateDataType> targetPrimaryKeyTypes, List<String> targetPrimaryKeyNames) {
79-
if (!isEnabled) return targetPrimaryKeyTypes;
80-
if (!valid) return null;
74+
private void setWhereClause(PropertyHelper helper) {
75+
if (!isValid()) return;
8176

82-
// As this is valid, we know that the column names, types, and values are all the same size
8377
List<String> columnNames = getRawStringList(Property.COLUMN_NAMES);
78+
List<String> columnValues = getRawStringList(Property.COLUMN_VALUES);
8479

85-
List<MigrateDataType> rtn = new ArrayList<>();
86-
for (String keyName : targetPrimaryKeyNames) {
87-
if (!columnNames.contains(keyName)) {
88-
rtn.add(targetPrimaryKeyTypes.get(targetPrimaryKeyNames.indexOf(keyName)));
80+
if (null != columnNames && !columnNames.isEmpty()
81+
&& null != columnValues && !columnValues.isEmpty()) {
82+
List<String> targetPKNames = helper.getStringList(KnownProperties.TARGET_PRIMARY_KEY);
83+
84+
String whereClause = "";
85+
for (String columnName : columnNames) {
86+
if (null!=targetPKNames && targetPKNames.contains(columnName)) {
87+
if (!whereClause.isEmpty())
88+
whereClause += " AND ";
89+
whereClause += columnName + "=" + columnValues.get(columnNames.indexOf(columnName));
90+
}
8991
}
92+
putString(Property.WHERE_CLAUSE, " AND " + whereClause);
9093
}
91-
return rtn;
9294
}
9395

94-
private boolean isValid(PropertyHelper propertyHelper) {
96+
// Constant columns do not belong on the PK, as they are hard-coded to WHERE_CLAUSE
97+
private void clean_targetPK(PropertyHelper helper) {
98+
List<String> constantColumnNames = getRawStringList(Property.COLUMN_NAMES);
99+
List<String> currentPKNames = helper.getStringList(KnownProperties.TARGET_PRIMARY_KEY);
100+
List<MigrateDataType> currentPKTypes = helper.getMigrationTypeList(KnownProperties.TARGET_PRIMARY_KEY_TYPES);
101+
102+
List<String> newPKNames = new ArrayList<>();
103+
List<MigrateDataType> newPKTypes = new ArrayList<>();
104+
105+
for (String keyName : currentPKNames) {
106+
if (!constantColumnNames.contains(keyName)) {
107+
newPKNames.add(keyName);
108+
newPKTypes.add(currentPKTypes.get(currentPKNames.indexOf(keyName)));
109+
}
110+
else {
111+
logger.info("Removing constant column {} from target PK", keyName);
112+
}
113+
}
114+
115+
helper.setProperty(KnownProperties.TARGET_PRIMARY_KEY, newPKNames);
116+
helper.setProperty(KnownProperties.TARGET_PRIMARY_KEY_TYPES, newPKTypes);
117+
}
118+
119+
private boolean isValid() {
95120
List<String> columnNames = getRawStringList(Property.COLUMN_NAMES);
96121
List<MigrateDataType> columnTypes = getRawMigrateDataTypeList(Property.COLUMN_TYPES);
97122
List<String> columnValues = getRawStringList(Property.COLUMN_VALUES);
123+
String regexString = getRawString(Property.SPLIT_REGEX);
98124

99125
boolean haveColumnNames = null!=columnNames && !columnNames.isEmpty();
100126
boolean haveColumnTypes = null!=columnTypes && !columnTypes.isEmpty();
@@ -110,7 +136,7 @@ private boolean isValid(PropertyHelper propertyHelper) {
110136
KnownProperties.CONSTANT_COLUMN_NAMES, columnNames,
111137
KnownProperties.CONSTANT_COLUMN_TYPES, columnTypes,
112138
KnownProperties.CONSTANT_COLUMN_VALUES, columnValues,
113-
KnownProperties.CONSTANT_COLUMN_SPLIT_REGEX, propertyHelper.getAsString(KnownProperties.CONSTANT_COLUMN_SPLIT_REGEX));
139+
KnownProperties.CONSTANT_COLUMN_SPLIT_REGEX, regexString);
114140
valid = false;
115141
}
116142

@@ -121,7 +147,7 @@ private boolean isValid(PropertyHelper propertyHelper) {
121147
KnownProperties.CONSTANT_COLUMN_NAMES, columnNames,
122148
KnownProperties.CONSTANT_COLUMN_TYPES, columnTypes,
123149
KnownProperties.CONSTANT_COLUMN_VALUES, columnValues,
124-
KnownProperties.CONSTANT_COLUMN_SPLIT_REGEX, propertyHelper.getAsString(KnownProperties.CONSTANT_COLUMN_SPLIT_REGEX));
150+
KnownProperties.CONSTANT_COLUMN_SPLIT_REGEX, regexString);
125151
valid = false;
126152
}
127153

‎src/main/java/datastax/astra/migrate/cql/features/Feature.java

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,13 @@ public interface Feature {
1313
*/
1414
public boolean initialize(PropertyHelper propertyHelper);
1515

16+
/**
17+
* Modifies properties based on feature-specific logic
18+
* @param helper PropertyHelper to modify
19+
* @return the modified PropertyHelper
20+
*/
21+
public PropertyHelper alterProperties(PropertyHelper helper);
22+
1623
/**
1724
* Indicates if feature is enabled.
1825
* @return true if the feature is enabled, false otherwise
@@ -120,11 +127,4 @@ public interface Feature {
120127
*/
121128
public MigrateDataType getMigrateDataType(Enum<?> prop);
122129

123-
/**
124-
* Generic feature function that can be used to implement any feature specific functionality.
125-
* @param function Feature-specific name of the feature to call
126-
* @param args Feature- and function-specific arguments
127-
* @return Function-specific
128-
*/
129-
public Object featureFunction(Enum<?> function, Object... args);
130130
}

‎src/main/java/datastax/astra/migrate/properties/KnownProperties.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ public enum PropertyType {
140140
//==========================================================================
141141
public static final String TARGET_KEYSPACE_TABLE = "spark.target.keyspaceTable"; // test.a1
142142
public static final String TARGET_PRIMARY_KEY = "spark.query.target.id"; // comma-separated-partition-key,comma-separated-clustering-key
143-
public static final String TARGET_PRIMARY_KEY_TYPES = "spark.query.target.id.types"; // 9,1,4,3
143+
public static final String TARGET_PRIMARY_KEY_TYPES = "spark.cdm.cql.target.id.types"; // Code-managed, not an external property
144144
public static final String TARGET_COLUMN_NAMES = "spark.query.target";
145145
public static final String TARGET_CUSTOM_WRITETIME = "spark.target.custom.writeTime"; // 0
146146
public static final String TARGET_AUTOCORRECT_MISSING = "spark.target.autocorrect.missing"; // false
@@ -152,6 +152,7 @@ public enum PropertyType {
152152
types.put(TARGET_PRIMARY_KEY, PropertyType.STRING_LIST);
153153
required.add(TARGET_PRIMARY_KEY);
154154
types.put(TARGET_PRIMARY_KEY_TYPES, PropertyType.MIGRATION_TYPE_LIST);
155+
required.add(TARGET_PRIMARY_KEY_TYPES);
155156
types.put(TARGET_COLUMN_NAMES, PropertyType.STRING_LIST);
156157
required.add(TARGET_COLUMN_NAMES); // we need this, though it should be defaulted with ORIGIN_COLUMN_NAMES value
157158
types.put(TARGET_CUSTOM_WRITETIME, PropertyType.NUMBER);

‎src/main/java/datastax/astra/migrate/properties/PropertyHelper.java

Lines changed: 38 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -242,14 +242,7 @@ protected void loadSparkConf() {
242242
setProperty(KnownProperties.TARGET_COLUMN_NAMES, get(KnownProperties.ORIGIN_COLUMN_NAMES));
243243
}
244244

245-
// Target column key list defaults to the first N columns of the source column list, where N is the size of the target primary key list
246-
if (null == get(KnownProperties.TARGET_PRIMARY_KEY_TYPES) || getAsString(KnownProperties.TARGET_PRIMARY_KEY_TYPES).isEmpty()) {
247-
if (null != getMigrationTypeList(KnownProperties.ORIGIN_COLUMN_TYPES) && !getMigrationTypeList(KnownProperties.ORIGIN_COLUMN_TYPES).isEmpty()) {
248-
List<MigrateDataType> targetPKTypes = getMigrationTypeList(KnownProperties.ORIGIN_COLUMN_TYPES).subList(0, getStringList(KnownProperties.TARGET_PRIMARY_KEY).size());
249-
logger.info("Setting known property [" + KnownProperties.TARGET_PRIMARY_KEY_TYPES + "] based on [" + KnownProperties.ORIGIN_COLUMN_TYPES + "], which is [" + getAsString(KnownProperties.ORIGIN_COLUMN_TYPES) + "]");
250-
setProperty(KnownProperties.TARGET_PRIMARY_KEY_TYPES, targetPKTypes);
251-
}
252-
}
245+
setTargetPKTypes();
253246

254247
if (fullyLoaded) {
255248
fullyLoaded = isValidConfig();
@@ -258,6 +251,43 @@ protected void loadSparkConf() {
258251
this.sparkConfFullyLoaded = fullyLoaded;
259252
}
260253

254+
// Previously, the target primary key types were assumed to be the first N types of ORIGIN_COLUMN_TYPES, where N = TARGET_PRIMARY_KEY.size()
255+
// This updated method references the ORIGIN_COLUMN_NAMES and looks up the ORIGIN_COLUMN_TYPE, based on the name
256+
// If the name is not found, the PK type will be set to UNKNOWN.
257+
private void setTargetPKTypes() {
258+
if (null == get(KnownProperties.TARGET_PRIMARY_KEY_TYPES) || getAsString(KnownProperties.TARGET_PRIMARY_KEY_TYPES).isEmpty()) {
259+
260+
List<String> targetPK = getStringList(KnownProperties.TARGET_PRIMARY_KEY);
261+
if (null==targetPK || targetPK.isEmpty()) {
262+
logger.error("Unable to set {}, because {} is not set",KnownProperties.TARGET_PRIMARY_KEY_TYPES,KnownProperties.TARGET_PRIMARY_KEY);
263+
return;
264+
}
265+
266+
List<String> originColumnNames = getStringList(KnownProperties.ORIGIN_COLUMN_NAMES);
267+
List<MigrateDataType> originColumnTypes = getMigrationTypeList(KnownProperties.ORIGIN_COLUMN_TYPES);
268+
if (null == originColumnNames || originColumnNames.isEmpty()
269+
|| null == originColumnTypes || originColumnTypes.isEmpty()
270+
|| originColumnNames.size() != originColumnTypes.size()) {
271+
logger.error("Unable to set {}, because {} and {} are not both set and of the same length",KnownProperties.TARGET_PRIMARY_KEY_TYPES,KnownProperties.ORIGIN_COLUMN_NAMES,KnownProperties.ORIGIN_COLUMN_TYPES);
272+
return;
273+
}
274+
275+
List<MigrateDataType> targetPKTypes = new ArrayList<>();
276+
277+
for (String targetPKColumn : targetPK) {
278+
int originIndex = originColumnNames.indexOf(targetPKColumn);
279+
if (originIndex < 0) {
280+
targetPKTypes.add(new MigrateDataType());
281+
}
282+
else {
283+
targetPKTypes.add(originColumnTypes.get(originIndex));
284+
}
285+
}
286+
287+
setProperty(KnownProperties.TARGET_PRIMARY_KEY_TYPES, targetPKTypes);
288+
}
289+
}
290+
261291
protected boolean isValidConfig() {
262292
boolean valid = true;
263293

‎src/test/java/datastax/astra/migrate/cql/features/AbstractFeatureTest.java

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -163,14 +163,10 @@ public void valueSet_butDisabled() {
163163
);
164164
}
165165

166-
enum TestEnum {
167-
TEST
168-
}
169-
170166
@Test
171-
public void featureFunction_Null() {
167+
public void alterProperties_Null() {
172168
TestFeature testFeature = new TestFeature();
173-
testFeature.initialize_withValues();
174-
assertNull(testFeature.featureFunction(TestEnum.TEST,""));
169+
testFeature.alterProperties(null);
170+
assertNull(testFeature.alterProperties(null));
175171
}
176172
}

0 commit comments

Comments
 (0)