Skip to content

Commit 9361b79

Browse files
mfmaher2mfmaher2
authored
Merge pull request #226 from arvy/fix-auto-ttlWritime-issue220
issue-220: Fixes ttl-writetime auto flag handling
2 parents 1e803b2 + 7ba9aac commit 9361b79

File tree

3 files changed

+112
-1
lines changed

3 files changed

+112
-1
lines changed

‎src/main/java/com/datastax/cdm/feature/WritetimeTTL.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,12 @@ public boolean loadProperties(IPropertyHelper propertyHelper) {
7575
logger.info("PARAM -- {}: {} datetime is {} ", KnownProperties.FILTER_WRITETS_MAX, filterMax, Instant.ofEpochMilli(filterMax / 1000));
7676
}
7777

78+
7879
isValid = validateProperties();
7980
isEnabled = isValid &&
8081
((null != ttlNames && !ttlNames.isEmpty())
8182
|| (null != writetimeNames && !writetimeNames.isEmpty())
83+
|| autoTTLNames || autoWritetimeNames
8284
|| customWritetime > 0);
8385

8486
isLoaded = true;
@@ -309,4 +311,12 @@ private void validateFilterRangeProperties() {
309311
isValid = false;
310312
}
311313
}
314+
315+
public List<String> getTtlNames() {
316+
return ttlNames;
317+
}
318+
319+
public List<String> getWritetimeNames() {
320+
return writetimeNames;
321+
}
312322
}

‎src/resources/cdm-detailed.properties

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,17 @@ spark.cdm.connect.target.password cassandra
101101
# columns need to be listed.
102102
#-----------------------------------------------------------------------------------------------------------
103103
spark.cdm.schema.origin.keyspaceTable keyspace_name.table_name
104+
105+
# Max TTL value of all non-PK columns will be used for insert on target
104106
#spark.cdm.schema.origin.column.ttl.automatic true
107+
108+
# Max TTL value of specified non-PK columns will be used for insert on target (overrides automatic setting)
105109
#spark.cdm.schema.origin.column.ttl.names data_col1,data_col2,...
110+
111+
# Max WRITETIME value of all non-PK columns will be used for insert on target
106112
#spark.cdm.schema.origin.column.writetime.automatic true
113+
114+
# Max WRITETIME value of specified non-PK columns will be used for insert on target (overrides automatic setting)
107115
#spark.cdm.schema.origin.column.writetime.names data_col1,data_col2,...
108116
#spark.cdm.schema.origin.column.names.to.target partition_col1:partition_col_1,partition_col2:partition_col_2,...
109117

‎src/test/java/com/datastax/cdm/feature/TTLAndWritetimeTest.java

Lines changed: 94 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ public void smokeTest_disabledFeature() {
114114
when(propertyHelper.getStringList(KnownProperties.ORIGIN_TTL_NAMES)).thenReturn(null);
115115
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME)).thenReturn(null);
116116
when(propertyHelper.getStringList(KnownProperties.ORIGIN_WRITETIME_NAMES)).thenReturn(null);
117+
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_WRITETIME_AUTO)).thenReturn(Boolean.FALSE);
118+
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_TTL_AUTO)).thenReturn(Boolean.FALSE);
117119

118120
assertAll(
119121
() -> assertTrue(feature.loadProperties(propertyHelper), "loadProperties"),
@@ -126,6 +128,49 @@ public void smokeTest_disabledFeature() {
126128
);
127129
}
128130

131+
132+
@Test
133+
public void smokeTest_enabledFeature_withOnlyWritetimeAuto() {
134+
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MIN)).thenReturn(null);
135+
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MAX)).thenReturn(null);
136+
when(propertyHelper.getStringList(KnownProperties.ORIGIN_TTL_NAMES)).thenReturn(null);
137+
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME)).thenReturn(null);
138+
when(propertyHelper.getStringList(KnownProperties.ORIGIN_WRITETIME_NAMES)).thenReturn(null);
139+
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_WRITETIME_AUTO)).thenReturn(Boolean.TRUE);
140+
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_TTL_AUTO)).thenReturn(Boolean.FALSE);
141+
142+
assertAll(
143+
() -> assertTrue(feature.loadProperties(propertyHelper), "loadProperties"),
144+
() -> assertTrue(feature.initializeAndValidate(originTable, targetTable), "initializeAndValidate"),
145+
() -> assertTrue(feature.isEnabled(), "feature should be enabled"),
146+
() -> assertEquals(0L, feature.getCustomWritetime(), "customWritetime is not set"),
147+
() -> assertFalse(feature.hasWriteTimestampFilter(), "hasWriteTimestampFilter"),
148+
() -> assertTrue(feature.hasWritetimeColumns(), "hasWritetimeColumns with Auto set"),
149+
() -> assertFalse(feature.hasTTLColumns(), "hasTTLColumns")
150+
);
151+
}
152+
153+
@Test
154+
public void smokeTest_enabledFeature_withOnlyTTLAuto() {
155+
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MIN)).thenReturn(null);
156+
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MAX)).thenReturn(null);
157+
when(propertyHelper.getStringList(KnownProperties.ORIGIN_TTL_NAMES)).thenReturn(null);
158+
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME)).thenReturn(null);
159+
when(propertyHelper.getStringList(KnownProperties.ORIGIN_WRITETIME_NAMES)).thenReturn(null);
160+
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_WRITETIME_AUTO)).thenReturn(Boolean.FALSE);
161+
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_TTL_AUTO)).thenReturn(Boolean.TRUE);
162+
163+
assertAll(
164+
() -> assertTrue(feature.loadProperties(propertyHelper), "loadProperties"),
165+
() -> assertTrue(feature.initializeAndValidate(originTable, targetTable), "initializeAndValidate"),
166+
() -> assertTrue(feature.isEnabled(), "feature should be enabled"),
167+
() -> assertEquals(0L, feature.getCustomWritetime(), "customWritetime is not set"),
168+
() -> assertFalse(feature.hasWriteTimestampFilter(), "hasWriteTimestampFilter"),
169+
() -> assertFalse(feature.hasWritetimeColumns(), "hasWritetimeColumns"),
170+
() -> assertTrue(feature.hasTTLColumns(), "hasTTLColumns with ttlAuto set")
171+
);
172+
}
173+
129174
@Test
130175
public void smoke_writetimeWithoutTTL() {
131176
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME)).thenReturn(0L);
@@ -250,7 +295,7 @@ public void counter_unconfigured() {
250295

251296
assertAll(
252297
() -> assertFalse(feature.isEnabled(), "isEnabled"),
253-
() -> assertTrue(feature.isValid, "isValid")
298+
() -> assertFalse(feature.isValid, "isValid")
254299
);
255300
}
256301

@@ -497,4 +542,52 @@ public void customWriteTime_withAutoWritetime() {
497542

498543
verify(originTable, times(0)).extendColumns(any(),any());
499544
}
545+
546+
@Test
547+
public void specifiedColumnsOverrideAuto(){
548+
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME)).thenReturn(0L);
549+
when(propertyHelper.getStringList(KnownProperties.ORIGIN_WRITETIME_NAMES)).thenReturn(Arrays.<String>asList("writetime_ttl_col"));
550+
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_WRITETIME_AUTO)).thenReturn(true);
551+
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MIN)).thenReturn(null);
552+
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MAX)).thenReturn(null);
553+
when(propertyHelper.getStringList(KnownProperties.ORIGIN_TTL_NAMES)).thenReturn(Arrays.<String>asList("writetime_ttl_col"));
554+
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_TTL_AUTO)).thenReturn(true);
555+
556+
feature.loadProperties(propertyHelper);
557+
feature.initializeAndValidate(originTable, targetTable);
558+
assertAll(
559+
() -> assertTrue(feature.hasTTLColumns(), "has TTL columns"),
560+
() -> assertTrue(feature.hasWritetimeColumns(), "has Writetime columns"),
561+
() -> assertEquals(1, feature.getWritetimeNames().size(), "has exactly 1 Writetime column"),
562+
() -> assertEquals(1, feature.getTtlNames().size(), "has exactly 1 TTL column"),
563+
() -> assertTrue(feature.isEnabled, "feature is enabled")
564+
);
565+
566+
//gets called once for adding TTL(..) columns and once for adding WRITETIME(..) columns
567+
verify(originTable, times(2)).extendColumns(any(),any());
568+
}
569+
570+
@Test
571+
public void autoFlagResultsInAllValueColumnsBeingUsed(){
572+
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME)).thenReturn(0L);
573+
when(propertyHelper.getStringList(KnownProperties.ORIGIN_WRITETIME_NAMES)).thenReturn(null);
574+
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_WRITETIME_AUTO)).thenReturn(true);
575+
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MIN)).thenReturn(null);
576+
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MAX)).thenReturn(null);
577+
when(propertyHelper.getStringList(KnownProperties.ORIGIN_TTL_NAMES)).thenReturn(null);
578+
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_TTL_AUTO)).thenReturn(true);
579+
580+
feature.loadProperties(propertyHelper);
581+
feature.initializeAndValidate(originTable, targetTable);
582+
assertAll(
583+
() -> assertTrue(feature.hasTTLColumns(), "has TTL columns"),
584+
() -> assertTrue(feature.hasWritetimeColumns(), "has Writetime columns"),
585+
() -> assertEquals(3, feature.getWritetimeNames().size(), "has exactly 1 Writetime column"),
586+
() -> assertEquals(3, feature.getTtlNames().size(), "has exactly 1 TTL column"),
587+
() -> assertTrue(feature.isEnabled, "feature is enabled")
588+
);
589+
590+
//gets called once for adding TTL(..) columns and once for adding WRITETIME(..) columns
591+
verify(originTable, times(2)).extendColumns(any(),any());
592+
}
500593
}

0 commit comments

Comments
 (0)