Skip to content

Commit fc9c8e2

Browse files
faizalrub-datastaxfaizalrub-datastax
committed
Merge branch 'main' into feature/exception_handler_refactor
# Conflicts: # src/main/java/com/datastax/cdm/job/AbstractJobSession.java # src/main/java/datastax/cdm/job/BaseJobSession.java # src/main/java/datastax/cdm/job/DiffJobSession.java
2 parents e5d0832 + 7923d7a commit fc9c8e2

File tree

170 files changed

+11033
-5418
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

170 files changed

+11033
-5418
lines changed

‎CODEOWNERS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
* @datastax/cdm-core

‎SIT/cdm.sh

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,4 +91,14 @@ if [ $argErrors -ne 0 ]; then
9191
_usage
9292
fi
9393

94-
spark-submit --properties-file "${PROPERTIES}" --master "local[*]" --class ${CLASS} /local/cassandra-data-migrator.jar
94+
if [ ! -f /local/log4j.xml ]; then
95+
cd /local && jar xvf /local/cassandra-data-migrator.jar log4j.xml && cd -
96+
fi
97+
98+
spark-submit --properties-file "${PROPERTIES}" \
99+
--master "local[*]" \
100+
--conf "spark.driver.extraJavaOptions=-Dlog4j.configurationFile=file:///local/log4j.xml -Dcom.datastax.cdm.log.level=TRACE" \
101+
--conf "spark.executor.extraJavaOptions=-Dlog4j.configurationFile=file:///local/log4j.xml -Dcom.datastax.cdm.log.level=TRACE" \
102+
--class ${CLASS} \
103+
/local/cassandra-data-migrator.jar
104+

‎SIT/features/01_constant_column/cdm.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
migrateData datastax.cdm.job.Migrate migrate.properties
2-
validateData datastax.cdm.job.DiffData migrate.properties
3-
fixData datastax.cdm.job.DiffData fix.properties
1+
migrateData com.datastax.cdm.job.Migrate migrate.properties
2+
validateData com.datastax.cdm.job.DiffData migrate.properties
3+
fixData com.datastax.cdm.job.DiffData fix.properties

‎SIT/features/01_constant_column/fix.properties

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,7 @@ spark.cdm.schema.origin.keyspaceTable origin.feature_constant_column
55
spark.cdm.schema.target.keyspaceTable target.feature_constant_column
66
spark.cdm.perfops.numParts 1
77

8-
spark.cdm.schema.origin.column.names key,value
9-
spark.cdm.schema.origin.column.partition.names key
10-
spark.cdm.schema.target.column.id.names const1,key
11-
spark.cdm.schema.origin.column.types 0,0
12-
138
spark.cdm.feature.constantColumns.names const1,const2
14-
spark.cdm.feature.constantColumns.types 0,1
159
spark.cdm.feature.constantColumns.values 'abcd',1234
1610

1711
spark.cdm.autocorrect.missing true

‎SIT/features/01_constant_column/migrate.properties

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,6 @@ spark.cdm.schema.origin.keyspaceTable origin.feature_constant_column
55
spark.cdm.schema.target.keyspaceTable target.feature_constant_column
66
spark.cdm.perfops.numParts 1
77

8-
spark.cdm.schema.origin.column.names key,value
9-
spark.cdm.schema.origin.column.partition.names key
10-
spark.cdm.schema.target.column.id.names const1,key
11-
spark.cdm.schema.origin.column.types 0,0
12-
138
spark.cdm.feature.constantColumns.names const1,const2
14-
spark.cdm.feature.constantColumns.types 0,1
159
spark.cdm.feature.constantColumns.values 'abcd',1234
1610

‎SIT/features/02_explode_map/cdm.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
migrateData datastax.cdm.job.Migrate migrate.properties
2-
validateData datastax.cdm.job.DiffData migrate.properties
3-
fixData datastax.cdm.job.DiffData fix.properties
1+
migrateData com.datastax.cdm.job.Migrate migrate.properties
2+
validateData com.datastax.cdm.job.DiffData migrate.properties
3+
fixData com.datastax.cdm.job.DiffData fix.properties

‎SIT/features/02_explode_map/fix.properties

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,6 @@ spark.cdm.schema.origin.keyspaceTable origin.feature_explode_map
55
spark.cdm.schema.target.keyspaceTable target.feature_explode_map
66
spark.cdm.perfops.numParts 1
77

8-
spark.cdm.schema.origin.column.names key,value,fruits
9-
spark.cdm.schema.origin.column.partition.names key
10-
spark.cdm.schema.target.column.id.names key,fruit
11-
spark.cdm.schema.origin.column.types 0,0,5%0%1
12-
138
spark.cdm.feature.explodeMap.origin.name fruits
149
spark.cdm.feature.explodeMap.target.name.key fruit
1510
spark.cdm.feature.explodeMap.target.name.value fruit_qty

‎SIT/features/02_explode_map/migrate.properties

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,6 @@ spark.cdm.schema.origin.keyspaceTable origin.feature_explode_map
55
spark.cdm.schema.target.keyspaceTable target.feature_explode_map
66
spark.cdm.perfops.numParts 1
77

8-
spark.cdm.schema.origin.column.names key,value,fruits
9-
spark.cdm.schema.origin.column.partition.names key
10-
spark.cdm.schema.target.column.id.names key,fruit
11-
spark.cdm.schema.origin.column.types 0,0,5%0%1
12-
138
spark.cdm.feature.explodeMap.origin.name fruits
149
spark.cdm.feature.explodeMap.target.name.key fruit
1510
spark.cdm.feature.explodeMap.target.name.value fruit_qty

‎SIT/features/03_codec/cdm.txt

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
migrateData datastax.cdm.job.Migrate migrate.properties
2-
validateData datastax.cdm.job.DiffData migrate.properties
3-
fixData datastax.cdm.job.DiffData fix.properties
1+
migrateData com.datastax.cdm.job.Migrate migrate.properties
2+
validateData com.datastax.cdm.job.DiffData migrate.properties
3+
fixData com.datastax.cdm.job.DiffData fix.properties

‎SIT/features/03_codec/fix.properties

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,6 @@ spark.cdm.schema.origin.keyspaceTable origin.codec
55
spark.cdm.schema.target.keyspaceTable target.codec
66
spark.cdm.perfops.numParts 1
77

8-
spark.cdm.schema.origin.column.names key,val_int,val_bigint,val_timestamp,val_decimal,val_double
9-
spark.cdm.schema.origin.column.partition.names key
10-
spark.cdm.schema.target.column.id.names key
11-
spark.cdm.schema.origin.column.types 0,0,0,0,0,0
12-
spark.cdm.schema.target.column.types 0,1,2,4,14,3
13-
148
spark.cdm.transform.codecs INT_STRING,DOUBLE_STRING,BIGINT_STRING,DECIMAL_STRING,TIMESTAMP_STRING_FORMAT
159
spark.cdm.transform.codecs.timestamp.string.format yyMMddHHmmss
1610

0 commit comments

Comments
 (0)