Skip to content

Commit 9683b10

Browse files
mieslepmieslep
committed
CDM-37 adding second timestamp codec, as well as a decimal one
1 parent 84bf244 commit 9683b10

26 files changed

+526
-65
lines changed

‎SIT/features/03_codec/breakData.cql

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11

22
DELETE FROM target.codec WHERE key='key2';
3-
UPDATE target.codec SET val_int=654321, val_bigint=314159200000000, val_timestamp='2024-04-16 10:30:00+0000' WHERE key='key3';
3+
UPDATE target.codec SET
4+
val_int=654321,
5+
val_bigint=314159200000000,
6+
val_timestamp='2024-04-16 10:30:00+0000',
7+
val_decimal=999.1234
8+
WHERE key='key3';
49
SELECT * FROM target.codec;

‎SIT/features/03_codec/expected.out

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11

2-
key | val_bigint | val_int | val_timestamp
3-
------+---------------------+---------+---------------------------------
4-
key1 | 9223372036854775807 | 1234 | 2004-06-16 11:00:00.000000+0000
5-
key3 | 3141592653589793 | 123456 | 2004-06-16 11:30:00.000000+0000
6-
key2 | 2147483648 | 12345 | 2004-06-16 11:15:00.000000+0000
2+
key | val_bigint | val_decimal | val_int | val_timestamp
3+
------+---------------------+-------------+---------+---------------------------------
4+
key1 | 9223372036854775807 | 3.14 | 1234 | 2004-06-16 11:00:00.000000+0000
5+
key3 | 3141592653589793 | 5.14 | 123456 | 2099-06-16 11:00:00.000000+0000
6+
key2 | 2147483648 | 4.14 | 12345 | 2099-06-16 11:00:00.000000+0000
77

88
(3 rows)

‎SIT/features/03_codec/fix.properties

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,13 @@ spark.cdm.schema.origin.keyspaceTable origin.codec
55
spark.cdm.schema.target.keyspaceTable target.codec
66
spark.cdm.perfops.numParts 1
77

8-
spark.cdm.schema.origin.column.names key,val_int,val_bigint,val_timestamp
8+
spark.cdm.schema.origin.column.names key,val_int,val_bigint,val_timestamp,val_decimal
99
spark.cdm.schema.origin.column.partition.names key
1010
spark.cdm.schema.target.column.id.names key
11-
spark.cdm.schema.origin.column.types 0,0,0,0
11+
spark.cdm.schema.origin.column.types 0,0,0,0,0
1212

13-
spark.cdm.transform.codecs CQL_INT_TO_STRING,CQL_BIGINT_TO_STRING,CQL_TIMESTAMP_TO_STRING
13+
spark.cdm.transform.codecs CQL_INT_TO_STRING,CQL_BIGINT_TO_STRING,CQL_DECIMAL_TO_STRING,CQL_TIMESTAMP_TO_STRING_FORMAT
14+
spark.cdm.transform.codecs.timestamp.string.format yyMMddHHmmss
1415

1516
spark.cdm.autocorrect.missing true
1617
spark.cdm.autocorrect.mismatch true

‎SIT/features/03_codec/migrate.properties

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,11 @@ spark.cdm.schema.origin.keyspaceTable origin.codec
55
spark.cdm.schema.target.keyspaceTable target.codec
66
spark.cdm.perfops.numParts 1
77

8-
spark.cdm.schema.origin.column.names key,val_int,val_bigint,val_timestamp
8+
spark.cdm.schema.origin.column.names key,val_int,val_bigint,val_timestamp,val_decimal
99
spark.cdm.schema.origin.column.partition.names key
1010
spark.cdm.schema.target.column.id.names key
11-
spark.cdm.schema.origin.column.types 0,0,0,0
12-
13-
spark.cdm.transform.codecs CQL_INT_TO_STRING,CQL_BIGINT_TO_STRING,CQL_TIMESTAMP_TO_STRING
11+
spark.cdm.schema.origin.column.types 0,0,0,0,0
1412

13+
spark.cdm.transform.codecs CQL_INT_TO_STRING,CQL_BIGINT_TO_STRING,CQL_DECIMAL_TO_STRING,CQL_TIMESTAMP_TO_STRING_FORMAT
14+
spark.cdm.transform.codecs.timestamp.string.format yyMMddHHmmss
1515

‎SIT/features/03_codec/setup.cql

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
DROP TABLE IF EXISTS origin.codec;
2-
CREATE TABLE origin.codec(key text, val_int text, val_bigint text, val_timestamp text, PRIMARY KEY (key));
3-
INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp) VALUES ('key1','1234' ,'9223372036854775807','1087383600000');
4-
INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp) VALUES ('key2','12345' ,'2147483648' ,'1087384500000');
5-
INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp) VALUES ('key3','123456','3141592653589793' ,'1087385400000');
2+
CREATE TABLE origin.codec(key text, val_int text, val_bigint text, val_timestamp text, val_decimal text, PRIMARY KEY (key));
3+
INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal)
4+
VALUES ('key1','1234' ,'9223372036854775807','040616110000', '3.14');
5+
INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal)
6+
VALUES ('key2','12345' ,'2147483648' ,'990616110000', '4.14');
7+
INSERT INTO origin.codec(key,val_int,val_bigint,val_timestamp,val_decimal)
8+
VALUES ('key3','123456','3141592653589793' ,'990616110000', '5.14');
69
SELECT * FROM origin.codec;
710

811
DROP TABLE IF EXISTS target.codec;
9-
CREATE TABLE target.codec(key text, val_int int, val_bigint bigint, val_timestamp timestamp, PRIMARY KEY (key));
12+
CREATE TABLE target.codec(key text, val_int int, val_bigint bigint, val_timestamp timestamp, val_decimal decimal, PRIMARY KEY (key));

‎SIT/regression/01_explode_map_with_constants/breakData.cql

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11

22
DELETE FROM target.feature_explode_map_with_constants WHERE const1='abcd' AND const2=1234 AND key='key2';
3-
UPDATE target.feature_explode_map_with_constants SET value='value999' WHERE const1='abcd' AND const2=1234 AND key='key3' AND fruit='apples';
4-
UPDATE target.feature_explode_map_with_constants SET fruit_qty=999 WHERE const1='abcd' AND const2=1234 AND key='key3' AND fruit='oranges';
3+
UPDATE target.feature_explode_map_with_constants SET time=7398730800000 WHERE const1='abcd' AND const2=1234 AND key='key3' AND fruit='apples';
4+
UPDATE target.feature_explode_map_with_constants SET fruit_qty=999 WHERE const1='abcd' AND const2=1234 AND key='key3' AND fruit='oranges';
55
DELETE FROM target.feature_explode_map_with_constants WHERE const1='abcd' AND const2=1234 AND key='key3' AND fruit='kiwi';
66
SELECT * FROM target.feature_explode_map_with_constants;
7-

‎SIT/regression/01_explode_map_with_constants/expected.out

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11

2-
const1 | const2 | key | fruit | fruit_qty | value
3-
--------+--------+------+---------+-----------+--------
4-
abcd | 1234 | key1 | apples | 3 | valueA
5-
abcd | 1234 | key1 | bananas | 2 | valueA
6-
abcd | 1234 | key1 | grapes | 11 | valueA
7-
abcd | 1234 | key1 | oranges | 5 | valueA
8-
abcd | 1234 | key3 | apples | 5 | valueC
9-
abcd | 1234 | key3 | bananas | 4 | valueC
10-
abcd | 1234 | key3 | kiwi | 42 | valueC
11-
abcd | 1234 | key3 | oranges | 7 | valueC
12-
abcd | 1234 | key2 | apples | 4 | valueB
13-
abcd | 1234 | key2 | bananas | 3 | valueB
14-
abcd | 1234 | key2 | oranges | 6 | valueB
15-
abcd | 1234 | key2 | pears | 7 | valueB
2+
const1 | const2 | key | fruit | fruit_qty | time
3+
--------+--------+------+---------+-----------+---------------------------------
4+
abcd | 1234 | key1 | apples | 3 | 2004-06-16 11:00:00.000000+0000
5+
abcd | 1234 | key1 | bananas | 2 | 2004-06-16 11:00:00.000000+0000
6+
abcd | 1234 | key1 | grapes | 11 | 2004-06-16 11:00:00.000000+0000
7+
abcd | 1234 | key1 | oranges | 5 | 2004-06-16 11:00:00.000000+0000
8+
abcd | 1234 | key3 | apples | 5 | 2004-06-16 11:00:00.000000+0000
9+
abcd | 1234 | key3 | bananas | 4 | 2004-06-16 11:00:00.000000+0000
10+
abcd | 1234 | key3 | kiwi | 42 | 2004-06-16 11:00:00.000000+0000
11+
abcd | 1234 | key3 | oranges | 7 | 2004-06-16 11:00:00.000000+0000
12+
abcd | 1234 | key2 | apples | 4 | 2004-06-16 11:00:00.000000+0000
13+
abcd | 1234 | key2 | bananas | 3 | 2004-06-16 11:00:00.000000+0000
14+
abcd | 1234 | key2 | oranges | 6 | 2004-06-16 11:00:00.000000+0000
15+
abcd | 1234 | key2 | pears | 7 | 2004-06-16 11:00:00.000000+0000
1616

1717
(12 rows)

‎SIT/regression/01_explode_map_with_constants/fix.properties

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ spark.cdm.schema.origin.keyspaceTable origin.feature_explode_map_wit
55
spark.cdm.schema.target.keyspaceTable target.feature_explode_map_with_constants
66
spark.cdm.perfops.numParts 1
77

8-
spark.cdm.schema.origin.column.names key,value,fruits
8+
spark.cdm.schema.origin.column.names key,time,fruits
99
spark.cdm.schema.origin.column.partition.names key
1010
spark.cdm.schema.target.column.id.names const1,const2,key,fruit
1111
spark.cdm.schema.origin.column.types 0,0,5%0%1
@@ -18,6 +18,8 @@ spark.cdm.feature.constantColumns.names const1,const2
1818
spark.cdm.feature.constantColumns.types 0,1
1919
spark.cdm.feature.constantColumns.values 'abcd',1234
2020

21+
spark.cdm.transform.codecs CQL_TIMESTAMP_TO_STRING_MILLIS
22+
2123
spark.cdm.autocorrect.missing true
2224
spark.cdm.autocorrect.mismatch true
2325

‎SIT/regression/01_explode_map_with_constants/migrate.properties

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ spark.cdm.schema.origin.keyspaceTable origin.feature_explode_map_wit
55
spark.cdm.schema.target.keyspaceTable target.feature_explode_map_with_constants
66
spark.cdm.perfops.numParts 1
77

8-
spark.cdm.schema.origin.column.names key,value,fruits
8+
spark.cdm.schema.origin.column.names key,time,fruits
99
spark.cdm.schema.origin.column.partition.names key
1010
spark.cdm.schema.target.column.id.names const1,const2,key,fruit
1111
spark.cdm.schema.origin.column.types 0,0,5%0%1
@@ -18,3 +18,6 @@ spark.cdm.feature.constantColumns.names const1,const2
1818
spark.cdm.feature.constantColumns.types 0,1
1919
spark.cdm.feature.constantColumns.values 'abcd',1234
2020

21+
spark.cdm.transform.codecs CQL_TIMESTAMP_TO_STRING_MILLIS
22+
23+

‎SIT/regression/01_explode_map_with_constants/setup.cql

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
DROP TABLE IF EXISTS origin.feature_explode_map_with_constants;
2-
CREATE TABLE origin.feature_explode_map_with_constants(key text, value text, fruits map<text,int>, PRIMARY KEY (key));
3-
INSERT INTO origin.feature_explode_map_with_constants(key,value,fruits) VALUES ('key1','valueA', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11});
4-
INSERT INTO origin.feature_explode_map_with_constants(key,value,fruits) VALUES ('key2','valueB', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7});
5-
INSERT INTO origin.feature_explode_map_with_constants(key,value,fruits) VALUES ('key3','valueC', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42});
2+
CREATE TABLE origin.feature_explode_map_with_constants(key text, time text, fruits map<text,int>, PRIMARY KEY (key));
3+
INSERT INTO origin.feature_explode_map_with_constants(key,time,fruits) VALUES ('key1','1087383600000', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11});
4+
INSERT INTO origin.feature_explode_map_with_constants(key,time,fruits) VALUES ('key2','1087383600000', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7});
5+
INSERT INTO origin.feature_explode_map_with_constants(key,time,fruits) VALUES ('key3','1087383600000', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42});
66

77
DROP TABLE IF EXISTS target.feature_explode_map_with_constants;
8-
CREATE TABLE target.feature_explode_map_with_constants(const1 text, const2 int, key text, fruit text, value text, fruit_qty int, PRIMARY KEY ((const1,const2,key), fruit));
8+
CREATE TABLE target.feature_explode_map_with_constants(const1 text, const2 int, key text, fruit text, time timestamp, fruit_qty int, PRIMARY KEY ((const1,const2,key), fruit));

0 commit comments

Comments
 (0)