Skip to content

Commit 6fbdb4b

Browse files
mieslepmieslep
committed
CDM-38 adding SIT test and fixing issues with the code
1 parent 5cb7b76 commit 6fbdb4b

16 files changed

+263
-53
lines changed

‎SIT/features/04_udt_mapper/breakData.cql

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
DELETE FROM target.udt_mapper WHERE key='key2';
2+
UPDATE target.udt_mapper SET
3+
val_udt = {text_value: 'value9990', long_value: 9990000000, double_value: 999.00}
4+
,val_list = [ {text_value: 'value9991', long_value: 9990000001, double_value: 999.01}
5+
,{text_value: 'value9992', long_value: 9990000002, double_value: 999.02}]
6+
,val_set = { {text_value: 'value9993', long_value: 9990000003, double_value: 999.03}
7+
,{text_value: 'value9994', long_value: 9990000004, double_value: 999.04}}
8+
,val_map = {'map_key1': {text_value: 'value9995', long_value: 9990000005, double_value: 999.05}
9+
,'map_key2': {text_value: 'value9996', long_value: 9990000006, double_value: 999.06}}
10+
WHERE key='key3';
11+
SELECT * FROM target.udt_mapper;

‎SIT/features/04_udt_mapper/cdm.fixData.assert

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Read Record Count: 3
2+
Mismatch Record Count: 1
3+
Corrected Mismatch Record Count: 1
4+
Missing Record Count: 1
5+
Corrected Missing Record Count: 1
6+
Valid Record Count: 1
7+
Skipped Record Count: 0

‎SIT/features/04_udt_mapper/cdm.migrateData.assert

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Read Record Count: 3
2+
Skipped Record Count: 0
3+
Write Record Count: 3
4+
Error Record Count: 0

‎SIT/features/04_udt_mapper/cdm.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
migrateData datastax.cdm.job.Migrate migrate.properties
2+
validateData datastax.cdm.job.DiffData migrate.properties
3+
fixData datastax.cdm.job.DiffData fix.properties

‎SIT/features/04_udt_mapper/cdm.validateData.assert

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Read Record Count: 3
2+
Mismatch Record Count: 0
3+
Corrected Mismatch Record Count: 0
4+
Missing Record Count: 0
5+
Corrected Missing Record Count: 0
6+
Valid Record Count: 3
7+
Skipped Record Count: 0

‎SIT/features/04_udt_mapper/execute.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash -e
2+
3+
workingDir="$1"
4+
cd "$workingDir"
5+
6+
/local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err
7+
/local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir"
8+
9+
/local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err
10+
/local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir"
11+
12+
cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err
13+
14+
/local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err
15+
/local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir"

‎SIT/features/04_udt_mapper/expected.cql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
SELECT * FROM target.udt_mapper;

‎SIT/features/04_udt_mapper/expected.out

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2+
key | val_list | val_map | val_set | val_udt
3+
------+--------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------
4+
key1 | [{text_value: 'value11', long_value: 1000000001, double_value: 1.01}, {text_value: 'value12', long_value: 1000000002, double_value: 1.02}] | {'map_key1': {text_value: 'value15', long_value: 1000000005, double_value: 1.05}, 'map_key2': {text_value: 'value16', long_value: 1000000006, double_value: 1.06}} | {{text_value: 'value13', long_value: 1000000003, double_value: 1.03}, {text_value: 'value14', long_value: 1000000004, double_value: 1.04}} | {text_value: 'value10', long_value: 1000000000, double_value: 1}
5+
key3 | [{text_value: 'value31', long_value: 3000000001, double_value: 3.01}, {text_value: 'value32', long_value: 3000000002, double_value: 3.02}] | {'map_key1': {text_value: 'value35', long_value: 3000000005, double_value: 3.05}, 'map_key2': {text_value: 'value36', long_value: 3000000006, double_value: 3.06}} | {{text_value: 'value33', long_value: 3000000003, double_value: 3.03}, {text_value: 'value34', long_value: 3000000004, double_value: 3.04}} | {text_value: 'value30', long_value: 3000000000, double_value: 3}
6+
key2 | [{text_value: 'value21', long_value: 2000000001, double_value: 2.01}, {text_value: 'value22', long_value: 2000000002, double_value: 2.02}] | {'map_key1': {text_value: 'value25', long_value: 2000000005, double_value: 2.05}, 'map_key2': {text_value: 'value26', long_value: 2000000006, double_value: 2.06}} | {{text_value: 'value23', long_value: 2000000003, double_value: 2.03}, {text_value: 'value24', long_value: 2000000004, double_value: 2.04}} | {text_value: 'value20', long_value: 2000000000, double_value: 2}
7+
8+
(3 rows)

‎SIT/features/04_udt_mapper/fix.properties

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
spark.cdm.origin.connect.host cdm-sit-cass
2+
spark.cdm.target.connect.host cdm-sit-cass
3+
4+
spark.cdm.schema.origin.keyspaceTable origin.udt_mapper
5+
spark.cdm.schema.target.keyspaceTable target.udt_mapper
6+
spark.cdm.perfops.numParts 1
7+
8+
spark.cdm.schema.origin.column.names key,val_udt,val_list,val_set,val_map
9+
spark.cdm.schema.origin.column.partition.names key
10+
spark.cdm.schema.target.column.id.names key
11+
spark.cdm.schema.origin.column.types 0,16,6%16,8%16,5%0%16
12+
13+
spark.cdm.autocorrect.missing true
14+
spark.cdm.autocorrect.mismatch true
15+

‎SIT/features/04_udt_mapper/migrate.properties

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
spark.cdm.origin.connect.host cdm-sit-cass
2+
spark.cdm.target.connect.host cdm-sit-cass
3+
4+
spark.cdm.schema.origin.keyspaceTable origin.udt_mapper
5+
spark.cdm.schema.target.keyspaceTable target.udt_mapper
6+
spark.cdm.perfops.numParts 1
7+
8+
spark.cdm.schema.origin.column.names key,val_udt,val_list,val_set,val_map
9+
spark.cdm.schema.origin.column.partition.names key
10+
spark.cdm.schema.target.column.id.names key
11+
spark.cdm.schema.origin.column.types 0,16,6%16,8%16,5%0%16
12+
13+

0 commit comments

Comments
 (0)