Fixed tests(smoke, regression & feature) scripts & code related to writetime/ttl

Author: pravinbhat

SIT/features/01_constant_column/breakData.cql

@@ -14,5 +14,11 @@
 
 DELETE FROM target.feature_constant_column WHERE key='key2' AND const1='abcd';
 UPDATE target.feature_constant_column SET value='value999' WHERE key='key3' AND const1='abcd';
+
+# This upsert to origin will update the writetime on origin to be newer than target
+INSERT INTO origin.feature_constant_column(key,value) VALUES ('key1','valueA');
+INSERT INTO origin.feature_constant_column(key,value) VALUES ('key2','valueB');
+INSERT INTO origin.feature_constant_column(key,value) VALUES ('key3','valueC');
+
 SELECT * FROM target.feature_constant_column;
 

SIT/features/02_explode_map/breakData.cql

@@ -14,5 +14,11 @@ DELETE FROM target.feature_explode_map WHERE key='key2';
 UPDATE target.feature_explode_map SET value='value999' WHERE key='key3' AND fruit='apples';
 UPDATE target.feature_explode_map SET fruit_qty=999 WHERE key='key3' AND fruit='oranges';
 DELETE FROM target.feature_explode_map WHERE key='key3' AND fruit='kiwi';
+
+# This upsert to origin will update the writetime on origin to be newer than target
+INSERT INTO origin.feature_explode_map(key,value,fruits) VALUES ('key1','valueA', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11});
+INSERT INTO origin.feature_explode_map(key,value,fruits) VALUES ('key2','valueB', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7});
+INSERT INTO origin.feature_explode_map(key,value,fruits) VALUES ('key3','valueC', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42});
+
 SELECT * FROM target.feature_explode_map;
 

SIT/features/03_codec/breakData.cql

@@ -18,4 +18,13 @@ UPDATE target.codec SET
        val_timestamp='2024-04-16 10:30:00+0000', 
        val_decimal=999.1234
 WHERE key='key3';
+
+# This upsert to origin will update the writetime on origin to be newer than target
+INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal,val_double)
+      VALUES ('key1','1234'  ,'9223372036854775807','040616110000', '3.14', '21474836470.7');
+INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal,val_double)
+      VALUES ('key2','12345' ,'2147483648'         ,'990616110000', '4.14', '21474836470.7');
+INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal,val_double)
+      VALUES ('key3','123456','3141592653589793'   ,'990616110000', '5.14', '21474836470.7');
+
 SELECT * FROM target.codec;

SIT/features/06_partition_range/breakData.cql

@@ -14,5 +14,11 @@
 
 DELETE FROM target.feature_partition_range WHERE key='key1';
 UPDATE target.feature_partition_range SET value='value999' WHERE key='key2';
+
+# This upsert to origin will update the writetime on origin to be newer than target
+INSERT INTO origin.feature_partition_range(key,value) VALUES ('key1','valueA');
+INSERT INTO origin.feature_partition_range(key,value) VALUES ('key2','valueB');
+INSERT INTO origin.feature_partition_range(key,value) VALUES ('key3','valueC');
+
 SELECT * FROM target.feature_partition_range;
 

SIT/regression/01_explode_map_with_constants/breakData.cql

@@ -16,4 +16,10 @@ DELETE FROM target.feature_explode_map_with_constants WHERE const1='abcd' AND co
 UPDATE target.feature_explode_map_with_constants SET time=7398730800000 WHERE const1='abcd' AND const2=1234 AND key='key3' AND fruit='apples';
 UPDATE target.feature_explode_map_with_constants SET fruit_qty=999      WHERE const1='abcd' AND const2=1234 AND key='key3' AND fruit='oranges';
 DELETE FROM target.feature_explode_map_with_constants WHERE const1='abcd' AND const2=1234 AND key='key3' AND fruit='kiwi';
+
+# This upsert to origin will update the writetime on origin to be newer than target
+INSERT INTO origin.feature_explode_map_with_constants(key,time,fruits) VALUES ('key1','1087383600000', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11});
+INSERT INTO origin.feature_explode_map_with_constants(key,time,fruits) VALUES ('key2','1087383600000', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7});
+INSERT INTO origin.feature_explode_map_with_constants(key,time,fruits) VALUES ('key3','1087383600000', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42});
+
 SELECT * FROM target.feature_explode_map_with_constants;

SIT/regression/02_ColumnRenameWithConstantsAndExplode/breakData.cql

@@ -12,8 +12,8 @@
  limitations under the License.
 */
 
-  DELETE FROM target.column_rename_with_constants_and_explode                   WHERE customer='CUSTOMER' AND year=2023 AND key_name='key2';
-       UPDATE target.column_rename_with_constants_and_explode SET fruit_qty=999 WHERE customer='CUSTOMER' AND year=2023 AND key_name='key3' AND fruit='oranges';
-  DELETE FROM target.column_rename_with_constants_and_explode                   WHERE customer='CUSTOMER' AND year=2023 AND key_name='key3' AND fruit='kiwi';
+DELETE FROM target.column_rename_with_constants_and_explode                   WHERE customer='CUSTOMER' AND year=2023 AND key_name='key2';
+UPDATE target.column_rename_with_constants_and_explode SET fruit_qty=999 WHERE customer='CUSTOMER' AND year=2023 AND key_name='key3' AND fruit='oranges';
+DELETE FROM target.column_rename_with_constants_and_explode                   WHERE customer='CUSTOMER' AND year=2023 AND key_name='key3' AND fruit='kiwi';
 SELECT * FROM target.column_rename_with_constants_and_explode;
 

SIT/regression/03_performance/breakData.cql

@@ -12,5 +12,5 @@
  limitations under the License.
 */
 
-DELETE   FROM target.regression_performance WHERE pk_id = 600;
+DELETE FROM target.regression_performance WHERE pk_id = 600;
 SELECT * FROM target.regression_performance WHERE pk_id >= 400 AND pk_id < 700 ALLOW FILTERING;

SIT/regression/03_performance/execute.sh

@@ -28,5 +28,8 @@ dsbulk load -url $workingDir/data.csv -h $CASS_CLUSTER -u $CASS_USERNAME -p $CAS
 dsbulk load -url $workingDir/data_break.csv -h $CASS_CLUSTER -u $CASS_USERNAME -p $CASS_PASSWORD -k target -t regression_performance
 cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err
 
+# This upsert to origin will update the writetime on origin to be newer than target
+dsbulk load -url $workingDir/data.csv -h $CASS_CLUSTER -u $CASS_USERNAME -p $CASS_PASSWORD -k origin -t regression_performance
+
 /local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err
 /local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir"

SIT/regression/03_performance/migrate.properties

@@ -12,7 +12,6 @@
 # limitations under the License.
 #
 
-
 spark.cdm.connect.origin.host                     cdm-sit-cass
 spark.cdm.connect.target.host                     cdm-sit-cass
 

SIT/smoke/02_autocorrect_kvp/breakData.cql

@@ -12,5 +12,11 @@
 
 DELETE FROM target.smoke_autocorrect_kvp WHERE key='key2';
 UPDATE target.smoke_autocorrect_kvp SET value='value999' WHERE key='key3';
+
+# This upsert to origin will update the writetime on origin to be newer than target
+INSERT INTO origin.smoke_autocorrect_kvp(key,value) VALUES ('key1','valueA');
+INSERT INTO origin.smoke_autocorrect_kvp(key,value) VALUES ('key2','valueB');
+INSERT INTO origin.smoke_autocorrect_kvp(key,value) VALUES ('key3','valueC');
+
 SELECT * FROM target.smoke_autocorrect_kvp;