Skip to content

Commit addbca5

Browse files
mfmaher2mfmaher2
authored
Merge pull request #124 from datastax/issue/CDM-37
Issue/cdm 37
2 parents acd0506 + 70b5e75 commit addbca5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+2224
-34
lines changed

‎SIT/features/03_codec/breakData.cql

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
2+
DELETE FROM target.codec WHERE key='key2';
3+
UPDATE target.codec SET
4+
val_int=654321,
5+
val_bigint=314159200000000,
6+
val_timestamp='2024-04-16 10:30:00+0000',
7+
val_decimal=999.1234
8+
WHERE key='key3';
9+
SELECT * FROM target.codec;

‎SIT/features/03_codec/cdm.fixData.assert

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Read Record Count: 3
2+
Mismatch Record Count: 1
3+
Corrected Mismatch Record Count: 1
4+
Missing Record Count: 1
5+
Corrected Missing Record Count: 1
6+
Valid Record Count: 1
7+
Skipped Record Count: 0

‎SIT/features/03_codec/cdm.migrateData.assert

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Read Record Count: 3
2+
Skipped Record Count: 0
3+
Write Record Count: 3
4+
Error Record Count: 0

‎SIT/features/03_codec/cdm.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
migrateData datastax.cdm.job.Migrate migrate.properties
2+
validateData datastax.cdm.job.DiffData migrate.properties
3+
fixData datastax.cdm.job.DiffData fix.properties

‎SIT/features/03_codec/cdm.validateData.assert

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Read Record Count: 3
2+
Mismatch Record Count: 0
3+
Corrected Mismatch Record Count: 0
4+
Missing Record Count: 0
5+
Corrected Missing Record Count: 0
6+
Valid Record Count: 3
7+
Skipped Record Count: 0

‎SIT/features/03_codec/execute.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash -e
2+
3+
workingDir="$1"
4+
cd "$workingDir"
5+
6+
/local/cdm.sh -f cdm.txt -s migrateData -d "$workingDir" > cdm.migrateData.out 2>cdm.migrateData.err
7+
/local/cdm-assert.sh -f cdm.migrateData.out -a cdm.migrateData.assert -d "$workingDir"
8+
9+
/local/cdm.sh -f cdm.txt -s validateData -d "$workingDir" > cdm.validateData.out 2>cdm.validateData.err
10+
/local/cdm-assert.sh -f cdm.validateData.out -a cdm.validateData.assert -d "$workingDir"
11+
12+
cqlsh -u $CASS_USERNAME -p $CASS_PASSWORD $CASS_CLUSTER -f $workingDir/breakData.cql > $workingDir/breakData.out 2> $workingDir/breakData.err
13+
14+
/local/cdm.sh -f cdm.txt -s fixData -d "$workingDir" > cdm.fixData.out 2>cdm.fixData.err
15+
/local/cdm-assert.sh -f cdm.fixData.out -a cdm.fixData.assert -d "$workingDir"

‎SIT/features/03_codec/expected.cql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
SELECT * FROM target.codec;

‎SIT/features/03_codec/expected.out

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
2+
key | val_bigint | val_decimal | val_double | val_int | val_timestamp
3+
------+---------------------+-------------+------------+---------+---------------------------------
4+
key1 | 9223372036854775807 | 3.14 | 2.1475e+10 | 1234 | 2004-06-16 11:00:00.000000+0000
5+
key3 | 3141592653589793 | 5.14 | 2.1475e+10 | 123456 | 2099-06-16 11:00:00.000000+0000
6+
key2 | 2147483648 | 4.14 | 2.1475e+10 | 12345 | 2099-06-16 11:00:00.000000+0000
7+
8+
(3 rows)

‎SIT/features/03_codec/fix.properties

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
spark.cdm.origin.connect.host cdm-sit-cass
2+
spark.cdm.target.connect.host cdm-sit-cass
3+
4+
spark.cdm.schema.origin.keyspaceTable origin.codec
5+
spark.cdm.schema.target.keyspaceTable target.codec
6+
spark.cdm.perfops.numParts 1
7+
8+
spark.cdm.schema.origin.column.names key,val_int,val_bigint,val_timestamp,val_decimal,val_double
9+
spark.cdm.schema.origin.column.partition.names key
10+
spark.cdm.schema.target.column.id.names key
11+
spark.cdm.schema.origin.column.types 0,0,0,0,0,0
12+
spark.cdm.schema.target.column.types 0,1,2,4,14,3
13+
14+
spark.cdm.transform.codecs INT_STRING,DOUBLE_STRING,BIGINT_STRING,DECIMAL_STRING,TIMESTAMP_STRING_FORMAT
15+
spark.cdm.transform.codecs.timestamp.string.format yyMMddHHmmss
16+
17+
spark.cdm.autocorrect.missing true
18+
spark.cdm.autocorrect.mismatch true
19+

‎SIT/features/03_codec/migrate.properties

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
spark.cdm.origin.connect.host cdm-sit-cass
2+
spark.cdm.target.connect.host cdm-sit-cass
3+
4+
spark.cdm.schema.origin.keyspaceTable origin.codec
5+
spark.cdm.schema.target.keyspaceTable target.codec
6+
spark.cdm.perfops.numParts 1
7+
8+
spark.cdm.schema.origin.column.names key,val_int,val_bigint,val_timestamp,val_decimal,val_double
9+
spark.cdm.schema.origin.column.partition.names key
10+
spark.cdm.schema.target.column.id.names key
11+
spark.cdm.schema.origin.column.types 0,0,0,0,0,0
12+
spark.cdm.schema.target.column.types 0,1,2,4,14,3
13+
14+
spark.cdm.transform.codecs INT_STRING,DOUBLE_STRING,BIGINT_STRING,DECIMAL_STRING,TIMESTAMP_STRING_FORMAT
15+
spark.cdm.transform.codecs.timestamp.string.format yyMMddHHmmss
16+

0 commit comments

Comments
 (0)