Skip to content

Commit 779c7ec

Browse files
mieslepmieslep
committed
CDM-30 renaming property names, consolidating filters
1 parent c3b62a8 commit 779c7ec

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+902
-778
lines changed

‎SIT/common.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ export DOCKER_CASS=cdm-sit-cass
22
export DOCKER_CDM=cdm-sit-cdm
33
export CASS_USERNAME=cassandra
44
export CASS_PASSWORD=cassandra
5-
export KEYSPACES="source target"
5+
export KEYSPACES="origin target"
66
export CDM_DIRECTORY=/local
77
export CDM_JARFILE=cassandra-data-migrator.jar
88

‎SIT/features/01_constant_column/fix.properties

Lines changed: 14 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,18 @@
1-
spark.origin.host cdm-sit-cass
2-
spark.origin.port 9042
3-
spark.origin.username cassandra
4-
spark.origin.password cassandra
5-
spark.origin.keyspaceTable source.feature_constant_column
1+
spark.cdm.origin.connect.host cdm-sit-cass
2+
spark.cdm.target.connect.host cdm-sit-cass
63

7-
spark.target.host cdm-sit-cass
8-
spark.target.port 9042
9-
spark.target.username cassandra
10-
spark.target.password cassandra
11-
spark.target.keyspaceTable target.feature_constant_column
4+
spark.cdm.schema.origin.keyspaceTable origin.feature_constant_column
5+
spark.cdm.schema.target.keyspaceTable target.feature_constant_column
6+
spark.cdm.perfops.numParts 1
127

13-
spark.numSplits 1
8+
spark.cdm.schema.origin.column.names key,value
9+
spark.cdm.schema.origin.column.partition.names key
10+
spark.cdm.schema.target.column.id.names const1,key
11+
spark.cdm.schema.origin.column.types 0,0
1412

15-
spark.query.origin key,value
16-
spark.query.origin.partitionKey key
17-
spark.query.target.id const1,key
18-
spark.query.types 0,0
13+
spark.cdm.feature.constantColumns.names const1,const2
14+
spark.cdm.feature.constantColumns.types 0,1
15+
spark.cdm.feature.constantColumns.values 'abcd',1234
1916

20-
spark.cdm.cql.feature.constantColumns.names const1,const2
21-
spark.cdm.cql.feature.constantColumns.types 0,1
22-
spark.cdm.cql.feature.constantColumns.values 'abcd',1234
23-
24-
spark.target.autocorrect.missing true
25-
spark.target.autocorrect.mismatch true
17+
spark.cdm.autocorrect.missing true
18+
spark.cdm.autocorrect.mismatch true

‎SIT/features/01_constant_column/migrate.properties

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,16 @@
1-
spark.origin.host cdm-sit-cass
2-
spark.origin.port 9042
3-
spark.origin.username cassandra
4-
spark.origin.password cassandra
5-
spark.origin.keyspaceTable source.feature_constant_column
1+
spark.cdm.origin.connect.host cdm-sit-cass
2+
spark.cdm.target.connect.host cdm-sit-cass
63

7-
spark.target.host cdm-sit-cass
8-
spark.target.port 9042
9-
spark.target.username cassandra
10-
spark.target.password cassandra
11-
spark.target.keyspaceTable target.feature_constant_column
4+
spark.cdm.schema.origin.keyspaceTable origin.feature_constant_column
5+
spark.cdm.schema.target.keyspaceTable target.feature_constant_column
6+
spark.cdm.perfops.numParts 1
127

13-
spark.numSplits 1
8+
spark.cdm.schema.origin.column.names key,value
9+
spark.cdm.schema.origin.column.partition.names key
10+
spark.cdm.schema.target.column.id.names const1,key
11+
spark.cdm.schema.origin.column.types 0,0
1412

15-
spark.query.origin key,value
16-
spark.query.origin.partitionKey key
17-
spark.query.target.id const1,key
18-
spark.query.types 0,0
13+
spark.cdm.feature.constantColumns.names const1,const2
14+
spark.cdm.feature.constantColumns.types 0,1
15+
spark.cdm.feature.constantColumns.values 'abcd',1234
1916

20-
spark.cdm.cql.feature.constantColumns.names const1,const2
21-
spark.cdm.cql.feature.constantColumns.types 0,1
22-
spark.cdm.cql.feature.constantColumns.values 'abcd',1234

‎SIT/features/01_constant_column/setup.cql

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
DROP TABLE IF EXISTS source.feature_constant_column;
2-
CREATE TABLE source.feature_constant_column(key text, value text, PRIMARY KEY (key));
3-
INSERT INTO source.feature_constant_column(key,value) VALUES ('key1','valueA');
4-
INSERT INTO source.feature_constant_column(key,value) VALUES ('key2','valueB');
5-
INSERT INTO source.feature_constant_column(key,value) VALUES ('key3','valueC');
1+
DROP TABLE IF EXISTS origin.feature_constant_column;
2+
CREATE TABLE origin.feature_constant_column(key text, value text, PRIMARY KEY (key));
3+
INSERT INTO origin.feature_constant_column(key,value) VALUES ('key1','valueA');
4+
INSERT INTO origin.feature_constant_column(key,value) VALUES ('key2','valueB');
5+
INSERT INTO origin.feature_constant_column(key,value) VALUES ('key3','valueC');
66

77
DROP TABLE IF EXISTS target.feature_constant_column;
88
CREATE TABLE target.feature_constant_column(const1 text, key text, value text, const2 int, PRIMARY KEY (const1, key));

‎SIT/features/02_explode_map/fix.properties

Lines changed: 14 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,19 @@
1-
spark.origin.host cdm-sit-cass
2-
spark.origin.port 9042
3-
spark.origin.username cassandra
4-
spark.origin.password cassandra
5-
spark.origin.keyspaceTable source.feature_explode_map
1+
spark.cdm.origin.connect.host cdm-sit-cass
2+
spark.cdm.target.connect.host cdm-sit-cass
63

7-
spark.target.host cdm-sit-cass
8-
spark.target.port 9042
9-
spark.target.username cassandra
10-
spark.target.password cassandra
11-
spark.target.keyspaceTable target.feature_explode_map
4+
spark.cdm.schema.origin.keyspaceTable origin.feature_explode_map
5+
spark.cdm.schema.target.keyspaceTable target.feature_explode_map
6+
spark.cdm.perfops.numParts 1
127

13-
spark.numSplits 1
8+
spark.cdm.schema.origin.column.names key,value,fruits
9+
spark.cdm.schema.origin.column.partition.names key
10+
spark.cdm.schema.target.column.id.names key,fruit
11+
spark.cdm.schema.origin.column.types 0,0,5%0%1
1412

15-
spark.query.origin key,value,fruits
16-
spark.query.origin.partitionKey key
17-
spark.query.target.id key,fruit
18-
spark.query.types 0,0,5%0%1
13+
spark.cdm.feature.explodeMap.origin.name fruits
14+
spark.cdm.feature.explodeMap.target.name.key fruit
15+
spark.cdm.feature.explodeMap.target.name.value fruit_qty
1916

20-
spark.cdm.cql.feature.explodeMap.origin.name fruits
21-
spark.cdm.cql.feature.explodeMap.target.name.key fruit
22-
spark.cdm.cql.feature.explodeMap.target.name.value fruit_qty
17+
spark.cdm.autocorrect.missing true
18+
spark.cdm.autocorrect.mismatch true
2319

24-
spark.target.autocorrect.missing true
25-
spark.target.autocorrect.mismatch true

‎SIT/features/02_explode_map/migrate.properties

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,16 @@
1-
spark.origin.host cdm-sit-cass
2-
spark.origin.port 9042
3-
spark.origin.username cassandra
4-
spark.origin.password cassandra
5-
spark.origin.keyspaceTable source.feature_explode_map
1+
spark.cdm.origin.connect.host cdm-sit-cass
2+
spark.cdm.target.connect.host cdm-sit-cass
63

7-
spark.target.host cdm-sit-cass
8-
spark.target.port 9042
9-
spark.target.username cassandra
10-
spark.target.password cassandra
11-
spark.target.keyspaceTable target.feature_explode_map
4+
spark.cdm.schema.origin.keyspaceTable origin.feature_explode_map
5+
spark.cdm.schema.target.keyspaceTable target.feature_explode_map
6+
spark.cdm.perfops.numParts 1
127

13-
spark.numSplits 1
8+
spark.cdm.schema.origin.column.names key,value,fruits
9+
spark.cdm.schema.origin.column.partition.names key
10+
spark.cdm.schema.target.column.id.names key,fruit
11+
spark.cdm.schema.origin.column.types 0,0,5%0%1
1412

15-
spark.query.origin key,value,fruits
16-
spark.query.origin.partitionKey key
17-
spark.query.target.id key,fruit
18-
spark.query.types 0,0,5%0%1
13+
spark.cdm.feature.explodeMap.origin.name fruits
14+
spark.cdm.feature.explodeMap.target.name.key fruit
15+
spark.cdm.feature.explodeMap.target.name.value fruit_qty
1916

20-
spark.cdm.cql.feature.explodeMap.origin.name fruits
21-
spark.cdm.cql.feature.explodeMap.target.name.key fruit
22-
spark.cdm.cql.feature.explodeMap.target.name.value fruit_qty

‎SIT/features/02_explode_map/setup.cql

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
DROP TABLE IF EXISTS source.feature_explode_map;
2-
CREATE TABLE source.feature_explode_map(key text, value text, fruits map<text,int>, PRIMARY KEY (key));
3-
INSERT INTO source.feature_explode_map(key,value,fruits) VALUES ('key1','valueA', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11});
4-
INSERT INTO source.feature_explode_map(key,value,fruits) VALUES ('key2','valueB', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7});
5-
INSERT INTO source.feature_explode_map(key,value,fruits) VALUES ('key3','valueC', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42});
1+
DROP TABLE IF EXISTS origin.feature_explode_map;
2+
CREATE TABLE origin.feature_explode_map(key text, value text, fruits map<text,int>, PRIMARY KEY (key));
3+
INSERT INTO origin.feature_explode_map(key,value,fruits) VALUES ('key1','valueA', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11});
4+
INSERT INTO origin.feature_explode_map(key,value,fruits) VALUES ('key2','valueB', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7});
5+
INSERT INTO origin.feature_explode_map(key,value,fruits) VALUES ('key3','valueC', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42});
66

77
DROP TABLE IF EXISTS target.feature_explode_map;
88
CREATE TABLE target.feature_explode_map(key text, fruit text, value text, fruit_qty int, PRIMARY KEY ((key), fruit));

‎SIT/features/03_explode_map_with_constants/fix.properties

Lines changed: 17 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,23 @@
1-
spark.origin.host cdm-sit-cass
2-
spark.origin.port 9042
3-
spark.origin.username cassandra
4-
spark.origin.password cassandra
5-
spark.origin.keyspaceTable source.feature_explode_map_with_constants
1+
spark.cdm.origin.connect.host cdm-sit-cass
2+
spark.cdm.target.connect.host cdm-sit-cass
63

7-
spark.target.host cdm-sit-cass
8-
spark.target.port 9042
9-
spark.target.username cassandra
10-
spark.target.password cassandra
11-
spark.target.keyspaceTable target.feature_explode_map_with_constants
4+
spark.cdm.schema.origin.keyspaceTable origin.feature_explode_map_with_constants
5+
spark.cdm.schema.target.keyspaceTable target.feature_explode_map_with_constants
6+
spark.cdm.perfops.numParts 1
127

13-
spark.numSplits 1
8+
spark.cdm.schema.origin.column.names key,value,fruits
9+
spark.cdm.schema.origin.column.partition.names key
10+
spark.cdm.schema.target.column.id.names const1,const2,key,fruit
11+
spark.cdm.schema.origin.column.types 0,0,5%0%1
1412

15-
spark.query.origin key,value,fruits
16-
spark.query.types 0,0,5%0%1
17-
spark.query.origin.partitionKey key
18-
spark.query.target.id const1,const2,key,fruit
13+
spark.cdm.feature.explodeMap.origin.name fruits
14+
spark.cdm.feature.explodeMap.target.name.key fruit
15+
spark.cdm.feature.explodeMap.target.name.value fruit_qty
1916

20-
spark.cdm.cql.feature.constantColumns.names const1,const2
21-
spark.cdm.cql.feature.constantColumns.types 0,1
22-
spark.cdm.cql.feature.constantColumns.values 'abcd',1234
17+
spark.cdm.feature.constantColumns.names const1,const2
18+
spark.cdm.feature.constantColumns.types 0,1
19+
spark.cdm.feature.constantColumns.values 'abcd',1234
2320

24-
spark.cdm.cql.feature.explodeMap.origin.name fruits
25-
spark.cdm.cql.feature.explodeMap.target.name.key fruit
26-
spark.cdm.cql.feature.explodeMap.target.name.value fruit_qty
21+
spark.cdm.autocorrect.missing true
22+
spark.cdm.autocorrect.mismatch true
2723

28-
spark.target.autocorrect.missing true
29-
spark.target.autocorrect.mismatch true

‎SIT/features/03_explode_map_with_constants/migrate.properties

Lines changed: 15 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,20 @@
1-
spark.origin.host cdm-sit-cass
2-
spark.origin.port 9042
3-
spark.origin.username cassandra
4-
spark.origin.password cassandra
5-
spark.origin.keyspaceTable source.feature_explode_map_with_constants
1+
spark.cdm.origin.connect.host cdm-sit-cass
2+
spark.cdm.target.connect.host cdm-sit-cass
63

7-
spark.target.host cdm-sit-cass
8-
spark.target.port 9042
9-
spark.target.username cassandra
10-
spark.target.password cassandra
11-
spark.target.keyspaceTable target.feature_explode_map_with_constants
4+
spark.cdm.schema.origin.keyspaceTable origin.feature_explode_map_with_constants
5+
spark.cdm.schema.target.keyspaceTable target.feature_explode_map_with_constants
6+
spark.cdm.perfops.numParts 1
127

13-
spark.numSplits 1
8+
spark.cdm.schema.origin.column.names key,value,fruits
9+
spark.cdm.schema.origin.column.partition.names key
10+
spark.cdm.schema.target.column.id.names const1,const2,key,fruit
11+
spark.cdm.schema.origin.column.types 0,0,5%0%1
1412

15-
spark.query.origin key,value,fruits
16-
spark.query.types 0,0,5%0%1
17-
spark.query.origin.partitionKey key
18-
spark.query.target.id const1,const2,key,fruit
13+
spark.cdm.feature.explodeMap.origin.name fruits
14+
spark.cdm.feature.explodeMap.target.name.key fruit
15+
spark.cdm.feature.explodeMap.target.name.value fruit_qty
1916

20-
spark.cdm.cql.feature.constantColumns.names const1,const2
21-
spark.cdm.cql.feature.constantColumns.types 0,1
22-
spark.cdm.cql.feature.constantColumns.values 'abcd',1234
17+
spark.cdm.feature.constantColumns.names const1,const2
18+
spark.cdm.feature.constantColumns.types 0,1
19+
spark.cdm.feature.constantColumns.values 'abcd',1234
2320

24-
spark.cdm.cql.feature.explodeMap.origin.name fruits
25-
spark.cdm.cql.feature.explodeMap.target.name.key fruit
26-
spark.cdm.cql.feature.explodeMap.target.name.value fruit_qty

‎SIT/features/03_explode_map_with_constants/setup.cql

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
DROP TABLE IF EXISTS source.feature_explode_map_with_constants;
2-
CREATE TABLE source.feature_explode_map_with_constants(key text, value text, fruits map<text,int>, PRIMARY KEY (key));
3-
INSERT INTO source.feature_explode_map_with_constants(key,value,fruits) VALUES ('key1','valueA', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11});
4-
INSERT INTO source.feature_explode_map_with_constants(key,value,fruits) VALUES ('key2','valueB', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7});
5-
INSERT INTO source.feature_explode_map_with_constants(key,value,fruits) VALUES ('key3','valueC', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42});
1+
DROP TABLE IF EXISTS origin.feature_explode_map_with_constants;
2+
CREATE TABLE origin.feature_explode_map_with_constants(key text, value text, fruits map<text,int>, PRIMARY KEY (key));
3+
INSERT INTO origin.feature_explode_map_with_constants(key,value,fruits) VALUES ('key1','valueA', {'apples': 3, 'oranges': 5, 'bananas': 2, 'grapes': 11});
4+
INSERT INTO origin.feature_explode_map_with_constants(key,value,fruits) VALUES ('key2','valueB', {'apples': 4, 'oranges': 6, 'bananas': 3, 'pears': 7});
5+
INSERT INTO origin.feature_explode_map_with_constants(key,value,fruits) VALUES ('key3','valueC', {'apples': 5, 'oranges': 7, 'bananas': 4, 'kiwi': 42});
66

77
DROP TABLE IF EXISTS target.feature_explode_map_with_constants;
88
CREATE TABLE target.feature_explode_map_with_constants(const1 text, const2 int, key text, fruit text, value text, fruit_qty int, PRIMARY KEY ((const1,const2,key), fruit));

0 commit comments

Comments
 (0)