16
16
17
17
public abstract class AbstractJobSession {
18
18
19
- public Logger logger = LoggerFactory .getLogger (this .getClass ().getName ());
20
-
21
- protected PreparedStatement sourceSelectStatement ;
22
- protected String sourceSelectCondition ;
23
-
24
- protected PreparedStatement astraSelectStatement ;
25
-
26
19
// Read/Write Rate limiter
27
20
// Determine the total throughput for the entire cluster in terms of wries/sec,
28
21
// reads/sec
@@ -31,12 +24,18 @@ public abstract class AbstractJobSession {
31
24
// Rate = Total Throughput (write/read per sec) / Total Executors
32
25
protected final RateLimiter readLimiter ;
33
26
protected final RateLimiter writeLimiter ;
27
+ public Logger logger = LoggerFactory .getLogger (this .getClass ().getName ());
28
+ protected PreparedStatement sourceSelectStatement ;
29
+ protected String sourceSelectCondition ;
30
+ protected PreparedStatement astraSelectStatement ;
31
+ protected PreparedStatement astraInsertStatement ;
34
32
protected Integer maxRetries = 10 ;
35
33
36
34
protected CqlSession sourceSession ;
37
35
protected CqlSession astraSession ;
38
36
protected List <MigrateDataType > selectColTypes = new ArrayList <MigrateDataType >();
39
37
protected List <MigrateDataType > idColTypes = new ArrayList <MigrateDataType >();
38
+ protected List <Integer > updateSelectMapping = new ArrayList <Integer >();
40
39
41
40
protected Integer batchSize = 1 ;
42
41
protected Integer printStatsAfter = 100000 ;
@@ -106,45 +105,69 @@ protected AbstractJobSession(CqlSession sourceSession, CqlSession astraSession,
106
105
maxWriteTimeStampFilter = Long .parseLong (maxWriteTimeStampFilterStr );
107
106
}
108
107
109
- logger .info (" DEFAULT -- Write Batch Size: " + batchSize );
110
- logger .info (" DEFAULT -- Source Keyspace Table: " + sourceKeyspaceTable );
111
- logger .info (" DEFAULT -- Destination Keyspace Table: " + astraKeyspaceTable );
112
- logger .info (" DEFAULT -- ReadRateLimit: " + readLimiter .getRate ());
113
- logger .info (" DEFAULT -- WriteRateLimit: " + writeLimiter .getRate ());
114
- logger .info (" DEFAULT -- WriteTimestampFilter: " + writeTimeStampFilter );
115
- logger .info (" DEFAULT -- WriteTimestampFilterCols: " + writeTimeStampCols );
116
- logger .info (" DEFAULT -- isPreserveTTLWritetime: " + isPreserveTTLWritetime );
117
- logger .info (" DEFAULT -- TTLCols: " + ttlCols );
108
+ logger .info ("PARAM -- Write Batch Size: " + batchSize );
109
+ logger .info ("PARAM -- Source Keyspace Table: " + sourceKeyspaceTable );
110
+ logger .info ("PARAM -- Destination Keyspace Table: " + astraKeyspaceTable );
111
+ logger .info ("PARAM -- ReadRateLimit: " + readLimiter .getRate ());
112
+ logger .info ("PARAM -- WriteRateLimit: " + writeLimiter .getRate ());
113
+ logger .info ("PARAM -- WriteTimestampFilter: " + writeTimeStampFilter );
114
+ logger .info ("PARAM -- WriteTimestampFilterCols: " + writeTimeStampCols );
115
+ logger .info ("PARAM -- isPreserveTTLWritetime: " + isPreserveTTLWritetime );
116
+ logger .info ("PARAM -- isPreserveTTLWritetime: " + isPreserveTTLWritetime );
117
+ logger .info ("PARAM -- TTLCols: " + ttlCols );
118
+
119
+ String selectCols = sparkConf .get ("spark.query.source" );
120
+ String partionKey = sparkConf .get ("spark.query.source.partitionKey" );
121
+ selectColTypes = getTypes (sparkConf .get ("spark.query.types" ));
122
+ String idCols = sparkConf .get ("spark.query.destination.id" , "" );
123
+ idColTypes = selectColTypes .subList (0 , idCols .split ("," ).length );
124
+ sourceSelectCondition = sparkConf .get ("spark.query.condition" , "" );
125
+ sourceSelectStatement = sourceSession .prepare (
126
+ "select " + selectCols + " from " + sourceKeyspaceTable + " where token(" + partionKey .trim ()
127
+ + ") >= ? and token(" + partionKey .trim () + ") <= ? " + sourceSelectCondition + " ALLOW FILTERING" );
118
128
119
- hasRandomPartitioner = Boolean .parseBoolean (sparkConf .get ("spark.source.hasRandomPartitioner" , "false" ));
129
+ String insertCols = sparkConf .get ("spark.query.destination" , "" );
130
+ if (null == insertCols || insertCols .trim ().isEmpty ()) {
131
+ insertCols = selectCols ;
132
+ }
133
+ String insertBinds = "" ;
134
+ for (String str : idCols .split ("," )) {
135
+ if (insertBinds .isEmpty ()) {
136
+ insertBinds = str + "= ?" ;
137
+ } else {
138
+ insertBinds += " and " + str + "= ?" ;
139
+ }
140
+ }
141
+ astraSelectStatement = astraSession .prepare (
142
+ "select " + insertCols + " from " + astraKeyspaceTable
143
+ + " where " + insertBinds );
120
144
145
+ hasRandomPartitioner = Boolean .parseBoolean (sparkConf .get ("spark.source.hasRandomPartitioner" , "false" ));
121
146
isCounterTable = Boolean .parseBoolean (sparkConf .get ("spark.counterTable" , "false" ));
122
- selectColTypes = getTypes (sparkConf .get ("spark.diff.select.types" ));
123
- String partionKey = sparkConf .get ("spark.query.cols.partitionKey" );
124
- String idCols = sparkConf .get ("spark.query.cols.id" );
125
- idColTypes = getTypes (sparkConf .get ("spark.query.cols.id.types" ));
147
+ if (isCounterTable ) {
148
+ String updateSelectMappingStr = sparkConf .get ("spark.counterTable.cql.index" , "0" );
149
+ for (String updateSelectIndex : updateSelectMappingStr .split ("," )) {
150
+ updateSelectMapping .add (Integer .parseInt (updateSelectIndex ));
151
+ }
126
152
127
- String selectCols = sparkConf .get ("spark.query.cols.select" );
153
+ String counterTableUpdate = sparkConf .get ("spark.counterTable.cql" );
154
+ astraInsertStatement = astraSession .prepare (counterTableUpdate );
155
+ } else {
156
+ insertBinds = "" ;
157
+ for (String str : insertCols .split ("," )) {
158
+ if (insertBinds .isEmpty ()) {
159
+ insertBinds += "?" ;
160
+ } else {
161
+ insertBinds += ", ?" ;
162
+ }
163
+ }
128
164
129
- String idBinds = "" ;
130
- int count = 1 ;
131
- for (String str : idCols .split ("," )) {
132
- if (count > 1 ) {
133
- idBinds = idBinds + " and " + str + "= ?" ;
165
+ if (isPreserveTTLWritetime ) {
166
+ astraInsertStatement = astraSession .prepare ("insert into " + astraKeyspaceTable + " (" + insertCols + ") VALUES (" + insertBinds + ") using TTL ? and TIMESTAMP ?" );
134
167
} else {
135
- idBinds = str + "= ?" ;
168
+ astraInsertStatement = astraSession . prepare ( "insert into " + astraKeyspaceTable + " (" + insertCols + ") VALUES (" + insertBinds + ")" ) ;
136
169
}
137
- count ++;
138
170
}
139
-
140
- sourceSelectCondition = sparkConf .get ("spark.query.cols.select.condition" , "" );
141
- sourceSelectStatement = sourceSession .prepare (
142
- "select " + selectCols + " from " + sourceKeyspaceTable + " where token(" + partionKey .trim ()
143
- + ") >= ? and token(" + partionKey .trim () + ") <= ? " + sourceSelectCondition + " ALLOW FILTERING" );
144
-
145
- astraSelectStatement = astraSession .prepare (
146
- "select " + selectCols + " from " + astraKeyspaceTable
147
- + " where " + idBinds );
148
171
}
149
172
150
173
public List <MigrateDataType > getTypes (String types ) {
0 commit comments