Skip to content

Commit 4280a78

Browse files
pravinbhatpravinbhat
committed
Updated docs and properties based on PR review comments
1 parent 0d508ba commit 4280a78

File tree

6 files changed

+43
-35
lines changed

6 files changed

+43
-35
lines changed

‎RELEASE.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Release Notes
22
## [4.3.5] - 2024-08-23
3-
- Added feature `park.cdm.feature.extractJson` which allows you to extract a json value from a column with json content in an Origin table and map it to a column in the Target table.
3+
- Added feature `spark.cdm.feature.extractJson` which allows you to extract a json value from a column with json content in an Origin table and map it to a column in the Target table.
44

55
## [4.3.4] - 2024-07-31
66
- Use `spark.cdm.schema.origin.keyspaceTable` when `spark.cdm.schema.target.keyspaceTable` is missing. Fixes [bug introduced in prior version](https://github.com/datastax/cassandra-data-migrator/issues/284).

‎src/main/java/com/datastax/cdm/feature/ExtractJson.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -48,13 +48,13 @@ public boolean loadProperties(IPropertyHelper helper) {
4848
}
4949

5050
originColumnName = getColumnName(helper, KnownProperties.EXTRACT_JSON_ORIGIN_COLUMN_NAME);
51-
targetColumnName = getColumnName(helper, KnownProperties.EXTRACT_JSON_TARGET_COLUMN_NAME);
51+
targetColumnName = getColumnName(helper, KnownProperties.EXTRACT_JSON_TARGET_COLUMN_MAPPING);
5252
// Convert columnToFieldMapping to targetColumnName and originJsonFieldName
5353
if (!targetColumnName.isBlank()) {
5454
String[] parts = targetColumnName.split("\\:");
5555
if (parts.length == 2) {
56-
targetColumnName = parts[0];
57-
originJsonFieldName = parts[1];
56+
originJsonFieldName = parts[0];
57+
targetColumnName = parts[1];
5858
} else {
5959
originJsonFieldName = targetColumnName;
6060
}
@@ -89,13 +89,13 @@ protected boolean validateProperties() {
8989
@Override
9090
public boolean initializeAndValidate(CqlTable originTable, CqlTable targetTable) {
9191
if (null == originTable || null == targetTable) {
92-
throw new IllegalArgumentException("originTable and/or targetTable is null");
92+
throw new IllegalArgumentException("Origin table and/or Target table is null");
9393
}
9494
if (!originTable.isOrigin()) {
95-
throw new IllegalArgumentException("Origin table is not an origin table");
95+
throw new IllegalArgumentException(originTable.getKeyspaceTable() + " is not an origin table");
9696
}
9797
if (targetTable.isOrigin()) {
98-
throw new IllegalArgumentException("Target table is not a target table");
98+
throw new IllegalArgumentException(targetTable.getKeyspaceTable() + " is not a target table");
9999
}
100100

101101
if (!validateProperties()) {

‎src/main/java/com/datastax/cdm/properties/KnownProperties.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -241,14 +241,14 @@ public enum PropertyType {
241241
// Extract JsonFeature
242242
// ==========================================================================
243243
public static final String EXTRACT_JSON_EXCLUSIVE = "spark.cdm.feature.extractJson.exclusive";
244-
public static final String EXTRACT_JSON_ORIGIN_COLUMN_NAME = "spark.cdm.feature.extractJson.origin.name";
245-
public static final String EXTRACT_JSON_TARGET_COLUMN_NAME = "spark.cdm.feature.extractJson.target.name";
244+
public static final String EXTRACT_JSON_ORIGIN_COLUMN_NAME = "spark.cdm.feature.extractJson.originColumn";
245+
public static final String EXTRACT_JSON_TARGET_COLUMN_MAPPING = "spark.cdm.feature.extractJson.propertyMapping";
246246

247247
static {
248248
types.put(EXTRACT_JSON_EXCLUSIVE, PropertyType.BOOLEAN);
249249
defaults.put(EXTRACT_JSON_EXCLUSIVE, "false");
250250
types.put(EXTRACT_JSON_ORIGIN_COLUMN_NAME, PropertyType.STRING);
251-
types.put(EXTRACT_JSON_TARGET_COLUMN_NAME, PropertyType.STRING);
251+
types.put(EXTRACT_JSON_TARGET_COLUMN_MAPPING, PropertyType.STRING);
252252
}
253253

254254
// ==========================================================================

‎src/main/java/com/datastax/cdm/schema/CqlTable.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -400,10 +400,10 @@ private void setCqlMetadata(CqlSession cqlSession) {
400400
String originColumnName = unFormatName(
401401
propertyHelper.getString(KnownProperties.EXTRACT_JSON_ORIGIN_COLUMN_NAME));
402402
String targetColumnName = unFormatName(
403-
propertyHelper.getString(KnownProperties.EXTRACT_JSON_TARGET_COLUMN_NAME));
403+
propertyHelper.getString(KnownProperties.EXTRACT_JSON_TARGET_COLUMN_MAPPING));
404404
extractColumnName = isOrigin ? originColumnName : targetColumnName;
405405
if (!extractColumnName.isBlank() && extractColumnName.contains(":")) {
406-
extractColumnName = extractColumnName.split("\\:")[0];
406+
extractColumnName = extractColumnName.split("\\:")[1];
407407
}
408408
}
409409
String columnName = extractColumnName;

‎src/resources/cdm-detailed.properties

Lines changed: 20 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -368,24 +368,30 @@ spark.cdm.perfops.ratelimit.target 20000
368368

369369

370370
#===========================================================================================================
371-
# Extract Json Feature allows you to extract a json value from a column with json content in an Origin table
372-
# and map it to a column in the Target table
371+
# The Extract JSON Feature enables extraction of a JSON value from a column containing JSON content in an
372+
# Origin table and maps it to a specified column in the Target table.
373373
#
374374
# spark.cdm.feature.extractJson
375-
# .exclusive Default is false. When true, Migration and Validation jobs will exclusively use only
376-
# the below provided columns from the origin and target tables and skip the rest.
377-
# .target.name Name of the target column where the extracted json value will be added. The json
378-
# column in origin table must be of CQL type 'text' while the column in the target table
379-
# can be of any primitive JSON types (String, Number or Boolean) but must match the json
380-
# value that you want to extract. If the json key-name does not exist in the json content,
381-
# the target column will be set to null.
382-
# Note: The target column-name must be the same as the key-name within the json content
383-
# that you want to extract. If the json key-name is not the same as the target column-name,
384-
# use the format target_columnname:json_keyname
375+
# .exclusive Default is false. When set to true, only the columns specified below from the Origin
376+
# and Target tables will be used, while all others will be ignored during Migration and
377+
# Validation jobs.
378+
#
379+
# .originColumn Specifies the name of the Origin column that contains JSON content. This column must
380+
# be of CQL type 'text'.
381+
#
382+
# .propertyMapping Defines the mapping between a JSON property in the Origin column and the corresponding
383+
# Target column. The format should be `origin_json_propertyname:target_columnname`.
384+
# - `origin_json_propertyname`: The name of the JSON property within the Origin column.
385+
# - `target_columnname`: The name of the Target column where the extracted JSON value
386+
# will be stored. The Target column can be of any primitive JSON type (String, Number,
387+
# or Boolean) and must match the data type of the JSON value being extracted.
388+
# - If the specified JSON property does not exist in the JSON content, the Target column
389+
# will be set to null.
390+
# Note: This feature currently supports extraction of only one JSON property.
385391
#-----------------------------------------------------------------------------------------------------------
386392
#spark.cdm.feature.extractJson.exclusive false
387-
#spark.cdm.feature.extractJson.origin.name origin_columnname_with_json_content
388-
#spark.cdm.feature.extractJson.target.name target_columnname
393+
#spark.cdm.feature.extractJson.originColumn origin_columnname_with_json_content
394+
#spark.cdm.feature.extractJson.propertyMapping origin_json_propertyname:target_columnname
389395

390396
#===========================================================================================================
391397
# Guardrail feature manages records that exceed guardrail checks. The Guardrail job will generate a

‎src/test/java/com/datastax/cdm/feature/ExtractJsonTest.java

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,14 @@ public void setup() {
6666
MockitoAnnotations.openMocks(this);
6767

6868
when(propertyHelper.getString(KnownProperties.EXTRACT_JSON_ORIGIN_COLUMN_NAME)).thenReturn(standardOriginName);
69-
when(propertyHelper.getString(KnownProperties.EXTRACT_JSON_TARGET_COLUMN_NAME)).thenReturn(standardTargetName);
69+
when(propertyHelper.getString(KnownProperties.EXTRACT_JSON_TARGET_COLUMN_MAPPING)).thenReturn(standardTargetName);
7070

71+
when(originTable.getKeyspaceTable()).thenReturn("ORIGIN_TABLE");
7172
when(originTable.isOrigin()).thenReturn(true);
7273
when(originTable.extendColumns(Collections.singletonList(standardOriginName))).
7374
thenReturn(Collections.singletonList(CqlData.getBindClass(standardOriginTypes.get(1))));
7475

76+
when(targetTable.getKeyspaceTable()).thenReturn("TARGET_TABLE");
7577
when(targetTable.isOrigin()).thenReturn(false);
7678
when(targetTable.extendColumns(Collections.singletonList(standardTargetName))).
7779
thenReturn(Collections.singletonList(CqlData.getBindClass(standardTargetTypes.get(1))));
@@ -114,7 +116,7 @@ public void loadPropertiesOriginError() {
114116

115117
@Test
116118
public void loadPropertiesTargetError() {
117-
when(propertyHelper.getString(KnownProperties.EXTRACT_JSON_TARGET_COLUMN_NAME)).thenReturn(null);
119+
when(propertyHelper.getString(KnownProperties.EXTRACT_JSON_TARGET_COLUMN_MAPPING)).thenReturn(null);
118120
assertFalse(feature.loadProperties(propertyHelper), "Target column name is not set");
119121
}
120122

@@ -143,7 +145,7 @@ public void extractNull() throws JsonMappingException, JsonProcessingException{
143145
@Test
144146
public void disabledFeature() {
145147
when(propertyHelper.getString(KnownProperties.EXTRACT_JSON_ORIGIN_COLUMN_NAME)).thenReturn("");
146-
when(propertyHelper.getString(KnownProperties.EXTRACT_JSON_TARGET_COLUMN_NAME)).thenReturn("");
148+
when(propertyHelper.getString(KnownProperties.EXTRACT_JSON_TARGET_COLUMN_MAPPING)).thenReturn("");
147149

148150
assertAll(
149151
() -> assertTrue(feature.loadProperties(propertyHelper), "loadProperties"),
@@ -155,20 +157,20 @@ public void disabledFeature() {
155157
() -> assertEquals(-1, feature.getOriginColumnIndex(), "origin index")
156158
);
157159

158-
when(propertyHelper.getString(KnownProperties.EXTRACT_JSON_TARGET_COLUMN_NAME)).thenReturn(null);
160+
when(propertyHelper.getString(KnownProperties.EXTRACT_JSON_TARGET_COLUMN_MAPPING)).thenReturn(null);
159161
assertEquals("", feature.getTargetColumnName(), "target name");
160162
}
161163

162164
@Test
163165
public void initializeAndValidateExceptionOriginNull() {
164166
IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> feature.initializeAndValidate(null, targetTable));
165-
assertTrue(thrown.getMessage().contains("originTable and/or targetTable is null"));
167+
assertTrue(thrown.getMessage().contains("Origin table and/or Target table is null"));
166168
}
167169

168170
@Test
169171
public void initializeAndValidateExceptionTargetNull() {
170172
IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> feature.initializeAndValidate(originTable, null));
171-
assertTrue(thrown.getMessage().contains("originTable and/or targetTable is null"));
173+
assertTrue(thrown.getMessage().contains("Origin table and/or Target table is null"));
172174
}
173175

174176
@Test
@@ -182,7 +184,7 @@ public void initializeAndValidateExceptionOriginColumn() {
182184

183185
@Test
184186
public void initializeAndValidateExceptionTargetColumn() {
185-
when(propertyHelper.getString(KnownProperties.EXTRACT_JSON_TARGET_COLUMN_NAME)).thenReturn("incorrect_column");
187+
when(propertyHelper.getString(KnownProperties.EXTRACT_JSON_TARGET_COLUMN_MAPPING)).thenReturn("incorrect_column");
186188

187189
feature.loadProperties(propertyHelper);
188190

@@ -194,15 +196,15 @@ public void initializeAndValidateExceptionOriginIncorrect() {
194196
when(originTable.isOrigin()).thenReturn(false);
195197

196198
IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> feature.initializeAndValidate(originTable, targetTable));
197-
assertTrue(thrown.getMessage().contains("Origin table is not an origin table"));
199+
assertTrue(thrown.getMessage().contains("ORIGIN_TABLE is not an origin table"));
198200
}
199201

200202
@Test
201203
public void initializeAndValidateExceptionTargetIncorrect() {
202204
when(targetTable.isOrigin()).thenReturn(true);
203205

204206
IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> feature.initializeAndValidate(originTable, targetTable));
205-
assertTrue(thrown.getMessage().contains("Target table is not a target table"));
207+
assertTrue(thrown.getMessage().contains("TARGET_TABLE is not a target table"));
206208
}
207209

208210
@Test

0 commit comments

Comments
 (0)