Skip to content

Commit 4073b87

Browse files
pravinbhatpravinbhat
authored
Implemented customTTL feature (#295)
1 parent 2f04303 commit 4073b87

File tree

5 files changed

+45
-4
lines changed

5 files changed

+45
-4
lines changed

‎RELEASE.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
# Release Notes
2+
## [4.3.7] - 2024-09-03
3+
- Added property `spark.cdm.transform.custom.ttl` to allow a custom constant value to be set for TTL instead of using the values from `origin` rows.
4+
25
## [4.3.6] - 2024-08-29
36
- Added `overwrite` option to conditionally check or skip `Validation` when it has a non-null value in `target` for the `spark.cdm.feature.extractJson` feature.
47

‎src/main/java/com/datastax/cdm/feature/WritetimeTTL.java

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ public class WritetimeTTL extends AbstractFeature {
3737
private List<String> writetimeNames;
3838
private boolean autoWritetimeNames;
3939
private Long customWritetime = 0L;
40+
private Long customTTL = 0L;
4041
private List<Integer> ttlSelectColumnIndexes = null;
4142
private List<Integer> writetimeSelectColumnIndexes = null;
4243
private Long filterMin;
@@ -67,6 +68,11 @@ public boolean loadProperties(IPropertyHelper propertyHelper) {
6768

6869
this.writetimeIncrement = propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME_INCREMENT);
6970

71+
this.customTTL = getCustomTTL(propertyHelper);
72+
if (this.customTTL > 0) {
73+
logger.info("PARAM -- {} is set to TTL {} ", KnownProperties.TRANSFORM_CUSTOM_TTL, customTTL);
74+
}
75+
7076
this.filterMin = getMinFilter(propertyHelper);
7177
this.filterMax = getMaxFilter(propertyHelper);
7278
this.hasWriteTimestampFilter = (null != filterMin && null != filterMax && filterMin > 0 && filterMax > 0 && filterMax > filterMin);
@@ -81,7 +87,7 @@ public boolean loadProperties(IPropertyHelper propertyHelper) {
8187
((null != ttlNames && !ttlNames.isEmpty())
8288
|| (null != writetimeNames && !writetimeNames.isEmpty())
8389
|| autoTTLNames || autoWritetimeNames
84-
|| customWritetime > 0);
90+
|| customWritetime > 0 || customTTL > 0);
8591

8692
isLoaded = true;
8793
return isValid;
@@ -167,11 +173,16 @@ public static List<String> getWritetimeNames(IPropertyHelper propertyHelper) {
167173
return CqlTable.unFormatNames(propertyHelper.getStringList(KnownProperties.ORIGIN_WRITETIME_NAMES));
168174
}
169175

170-
public static Long getCustomWritetime(IPropertyHelper propertyHelper) {
176+
protected static Long getCustomWritetime(IPropertyHelper propertyHelper) {
171177
Long cwt = propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME);
172178
return null==cwt ? 0L : cwt;
173179
}
174180

181+
protected static Long getCustomTTL(IPropertyHelper propertyHelper) {
182+
Long cttl = propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_TTL);
183+
return null == cttl ? 0L : cttl;
184+
}
185+
175186
public static Long getMinFilter(IPropertyHelper propertyHelper) {
176187
return propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MIN);
177188
}
@@ -181,11 +192,15 @@ public static Long getMaxFilter(IPropertyHelper propertyHelper) {
181192
}
182193

183194
public Long getCustomWritetime() { return customWritetime; }
195+
public Long getCustomTTL() { return customTTL; }
184196
public boolean hasWriteTimestampFilter() { return isEnabled && hasWriteTimestampFilter; }
185197
public Long getMinWriteTimeStampFilter() { return (this.hasWriteTimestampFilter && null!=this.filterMin) ? this.filterMin : Long.MIN_VALUE; }
186198
public Long getMaxWriteTimeStampFilter() { return (this.hasWriteTimestampFilter && null!=this.filterMax) ? this.filterMax : Long.MAX_VALUE; }
187199

188-
public boolean hasTTLColumns() { return null!=this.ttlSelectColumnIndexes && !this.ttlSelectColumnIndexes.isEmpty(); }
200+
public boolean hasTTLColumns() {
201+
return customTTL > 0 || null != this.ttlSelectColumnIndexes && !this.ttlSelectColumnIndexes.isEmpty();
202+
}
203+
189204
public boolean hasWritetimeColumns() { return customWritetime>0 || null!=this.writetimeSelectColumnIndexes && !this.writetimeSelectColumnIndexes.isEmpty(); }
190205

191206
public Long getLargestWriteTimeStamp(Row row) {
@@ -200,7 +215,8 @@ public Long getLargestWriteTimeStamp(Row row) {
200215
}
201216

202217
public Integer getLargestTTL(Row row) {
203-
if (logDebug) logger.debug("getLargestTTL: ttlSelectColumnIndexes={}", ttlSelectColumnIndexes);
218+
if (logDebug) logger.debug("getLargestTTL: customTTL={}, ttlSelectColumnIndexes={}", customTTL, ttlSelectColumnIndexes);
219+
if (this.customTTL > 0) return this.customTTL.intValue();
204220
if (null==this.ttlSelectColumnIndexes || this.ttlSelectColumnIndexes.isEmpty()) return null;
205221
OptionalInt max = this.ttlSelectColumnIndexes.stream()
206222
.mapToInt(row::getInt)

‎src/main/java/com/datastax/cdm/properties/KnownProperties.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,7 @@ public enum PropertyType {
161161
public static final String TRANSFORM_REPLACE_MISSING_TS = "spark.cdm.transform.missing.key.ts.replace.value";
162162
public static final String TRANSFORM_CUSTOM_WRITETIME = "spark.cdm.transform.custom.writetime";
163163
public static final String TRANSFORM_CUSTOM_WRITETIME_INCREMENT = "spark.cdm.transform.custom.writetime.incrementBy";
164+
public static final String TRANSFORM_CUSTOM_TTL = "spark.cdm.transform.custom.ttl";
164165
public static final String TRANSFORM_CODECS = "spark.cdm.transform.codecs";
165166
public static final String TRANSFORM_CODECS_TIMESTAMP_STRING_FORMAT = "spark.cdm.transform.codecs.timestamp.string.format";
166167
public static final String TRANSFORM_CODECS_TIMESTAMP_STRING_FORMAT_ZONE = "spark.cdm.transform.codecs.timestamp.string.zone";
@@ -170,6 +171,8 @@ public enum PropertyType {
170171
types.put(TRANSFORM_REPLACE_MISSING_TS, PropertyType.NUMBER);
171172
types.put(TRANSFORM_CUSTOM_WRITETIME, PropertyType.NUMBER);
172173
defaults.put(TRANSFORM_CUSTOM_WRITETIME, "0");
174+
types.put(TRANSFORM_CUSTOM_TTL, PropertyType.NUMBER);
175+
defaults.put(TRANSFORM_CUSTOM_TTL, "0");
173176
types.put(TRANSFORM_CUSTOM_WRITETIME_INCREMENT, PropertyType.NUMBER);
174177
defaults.put(TRANSFORM_CUSTOM_WRITETIME_INCREMENT, "0");
175178
types.put(TRANSFORM_CODECS, PropertyType.STRING_LIST);

‎src/resources/cdm-detailed.properties

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ spark.cdm.connect.target.password cassandra
7474
# all origin columns that can have TTL set (which excludes partition key,
7575
# clustering key, collections and UDTs). When false, and .names is not set, the target
7676
# record will have the TTL determined by the target table configuration.
77+
#
78+
# *** Note spark.cdm.transform.custom.ttl overrides this setting ***
79+
#
7780
# .names : Default is empty, meaning they will be determined automatically if that is set
7881
# (see above). Specify a subset of eligible columns that are used to calculate
7982
# the TTL of the target record.
@@ -247,6 +250,12 @@ spark.cdm.perfops.ratelimit.target 20000
247250
# and subsequent UPSERTs would add duplicates to the list. Future versions
248251
# of CDM may tombstone the previous list, but for now this solution is
249252
# viable and, crucially, more performant.
253+
# .ttl Default is 0 (no expiry). Time-to-live value in seconds to use as the
254+
# TTL for the target record. This is useful when the TTL of the record in
255+
# Origin cannot be determined (such as the only non-key columns are
256+
# collections) OR is a new TTL needs to be set during migration. This
257+
# parameter allows a crude constant value to be used in its place, and
258+
# overrides .schema.origin.column.ttl.names
250259
# .codecs Default is empty. A comma-separated list of additional codecs to
251260
# enable. Current codecs are:
252261
# INT_STRING : int stored in a String
@@ -273,6 +282,7 @@ spark.cdm.perfops.ratelimit.target 20000
273282
#spark.cdm.transform.missing.key.ts.replace.value 1685577600000
274283
#spark.cdm.transform.custom.writetime 0
275284
#spark.cdm.transform.custom.writetime.incrementBy 0
285+
#spark.cdm.transform.custom.ttl 0
276286
#spark.cdm.transform.codecs
277287
#spark.cdm.transform.codecs.timestamp.string.format yyyyMMddHHmmss
278288
#spark.cdm.transform.codecs.timestamp.string.zone UTC

‎src/test/java/com/datastax/cdm/feature/TTLAndWritetimeTest.java

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ public class TTLAndWritetimeTest extends CommonMocks {
3434

3535
WritetimeTTL feature;
3636
Long customWritetime = 123456789L;
37+
Long customTTL = 1000L;
3738
Long filterMin = 100000000L;
3839
Long filterMax = 200000000L;
3940
String writetimeColumnName = "writetime_col";
@@ -69,6 +70,7 @@ private void setTestWhens(){
6970
String argument = invocation.getArgument(0);
7071
return originValueColumns.contains(argument);
7172
});
73+
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_TTL)).thenReturn(customTTL);
7274
}
7375

7476

@@ -83,6 +85,7 @@ public void smoke_loadProperties() {
8385
assertAll(
8486
() -> assertTrue(feature.isEnabled(), "enabled"),
8587
() -> assertEquals(customWritetime, feature.getCustomWritetime(), "customWritetime"),
88+
() -> assertEquals(customTTL, feature.getCustomTTL(), "customTTL"),
8689
() -> assertTrue(feature.hasWriteTimestampFilter(), "hasWriteTimestampFilter"),
8790
() -> assertTrue(feature.hasWritetimeColumns(), "hasWritetimeColumns with custom writetime"),
8891
() -> assertEquals(customWritetime, feature.getCustomWritetime(), "customWritetime is set"),
@@ -113,6 +116,7 @@ public void smokeTest_disabledFeature() {
113116
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MAX)).thenReturn(null);
114117
when(propertyHelper.getStringList(KnownProperties.ORIGIN_TTL_NAMES)).thenReturn(null);
115118
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME)).thenReturn(null);
119+
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_TTL)).thenReturn(null);
116120
when(propertyHelper.getStringList(KnownProperties.ORIGIN_WRITETIME_NAMES)).thenReturn(null);
117121
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_WRITETIME_AUTO)).thenReturn(Boolean.FALSE);
118122
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_TTL_AUTO)).thenReturn(Boolean.FALSE);
@@ -135,6 +139,7 @@ public void smokeTest_enabledFeature_withOnlyWritetimeAuto() {
135139
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MAX)).thenReturn(null);
136140
when(propertyHelper.getStringList(KnownProperties.ORIGIN_TTL_NAMES)).thenReturn(null);
137141
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME)).thenReturn(null);
142+
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_TTL)).thenReturn(null);
138143
when(propertyHelper.getStringList(KnownProperties.ORIGIN_WRITETIME_NAMES)).thenReturn(null);
139144
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_WRITETIME_AUTO)).thenReturn(Boolean.TRUE);
140145
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_TTL_AUTO)).thenReturn(Boolean.FALSE);
@@ -174,6 +179,7 @@ public void smokeTest_enabledFeature_withOnlyTTLAuto() {
174179
@Test
175180
public void smoke_writetimeWithoutTTL() {
176181
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME)).thenReturn(0L);
182+
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_TTL)).thenReturn(null);
177183
when(propertyHelper.getStringList(KnownProperties.ORIGIN_TTL_NAMES)).thenReturn(null);
178184
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MIN)).thenReturn(filterMin);
179185
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MAX)).thenReturn(filterMax);
@@ -225,6 +231,7 @@ public void smoke_ttlWithoutWritetime_withCustomWritetime() {
225231
@Test
226232
public void smoke_autoWritetime_noCustomWritetime() {
227233
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME)).thenReturn(0L);
234+
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_TTL)).thenReturn(null);
228235
when(propertyHelper.getStringList(KnownProperties.ORIGIN_WRITETIME_NAMES)).thenReturn(null);
229236
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_WRITETIME_AUTO)).thenReturn(true);
230237
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MIN)).thenReturn(null);
@@ -244,6 +251,7 @@ public void smoke_autoWritetime_noCustomWritetime() {
244251
@Test
245252
public void smoke_autoWritetime_CustomWritetime() {
246253
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_WRITETIME)).thenReturn(100L);
254+
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_TTL)).thenReturn(null);
247255
when(propertyHelper.getStringList(KnownProperties.ORIGIN_WRITETIME_NAMES)).thenReturn(null);
248256
when(propertyHelper.getBoolean(KnownProperties.ORIGIN_WRITETIME_AUTO)).thenReturn(true);
249257
when(propertyHelper.getLong(KnownProperties.FILTER_WRITETS_MIN)).thenReturn(null);
@@ -373,6 +381,7 @@ public void getLargestWriteTimeStampWithCustomTimeTest() {
373381

374382
@Test
375383
public void getLargestTTLTest() {
384+
when(propertyHelper.getLong(KnownProperties.TRANSFORM_CUSTOM_TTL)).thenReturn(null);
376385
when(originTable.indexOf("TTL("+ttlColumnName+")")).thenReturn(100);
377386
when(originRow.getInt(eq(100))).thenReturn(30);
378387
when(originTable.indexOf("TTL("+writetimeTTLColumnName+")")).thenReturn(101);

0 commit comments

Comments
 (0)