Skip to content

Commit e6f3811

Browse files
pravinbhatmsmygit
pravinbhat
and
msmygit
authored
Implemented support for custom runId (#305)
* Implemented support for custom `runId` * Apply suggestions from code review * Review comments --------- Co-authored-by: Madhavan <[email protected]>
1 parent 8ca2666 commit e6f3811

File tree

14 files changed

+74
-59
lines changed

14 files changed

+74
-59
lines changed

‎RELEASE.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
# Release Notes
2+
## [4.3.10] - 2024-09-12
3+
- Added property `spark.cdm.trackRun.runId` to support a custom unique identifier for the current run. This can be used by wrapper scripts to pass a known `runId` and then use it to query the `cdm_run_info` and `cdm_run_details` tables.
4+
25
## [4.3.9] - 2024-09-11
36
- Added new `status` value of `DIFF_CORRECTED` on `cdm_run_details` table to specifically mark partitions that were corrected during the CDM validation run.
4-
- Upgraded Validation job skip partitions with `DIFF_CORRECTED` status on rerun with a previous `runId`.
7+
- Upgraded Validation job to skip partitions with `DIFF_CORRECTED` status on rerun with a previous `runId`.
58

69
## [4.3.8] - 2024-09-09
710
- Upgraded `spark.cdm.trackRun` feature to include `status` on `cdm_run_info` table. Also improved the code to handle rerun of previous run which may have exited before being correctly initialized.

‎src/main/java/com/datastax/cdm/cql/statement/TargetUpsertRunDetailsStatement.java

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,6 @@ public class TargetUpsertRunDetailsStatement {
3737
private CqlSession session;
3838
private String keyspaceName;
3939
private String tableName;
40-
private long runId;
41-
private long prevRunId;
4240
private BoundStatement boundInitInfoStatement;
4341
private BoundStatement boundInitStatement;
4442
private BoundStatement boundEndInfoStatement;
@@ -60,16 +58,18 @@ public TargetUpsertRunDetailsStatement(CqlSession session, String keyspaceTable)
6058
String cdmKsTabInfo = this.keyspaceName + ".cdm_run_info";
6159
String cdmKsTabDetails = this.keyspaceName + ".cdm_run_details";
6260

63-
this.session.execute("create table if not exists " + cdmKsTabInfo
64-
+ " (table_name text, run_id bigint, run_type text, prev_run_id bigint, start_time timestamp, end_time timestamp, run_info text, status text, primary key (table_name, run_id))");
61+
this.session.execute("CREATE TABLE IF NOT EXISTS " + cdmKsTabInfo
62+
+ " (table_name TEXT, run_id BIGINT, run_type TEXT, prev_run_id BIGINT, start_time TIMESTAMP, end_time TIMESTAMP, run_info TEXT, status TEXT, PRIMARY KEY (table_name, run_id))");
6563

6664
// TODO: Remove this code block after a few releases, its only added for backward compatibility
6765
try {
68-
this.session.execute("alter table " + cdmKsTabInfo + " add status text");
66+
this.session.execute("ALTER TABLE " + cdmKsTabInfo + " ADD status TEXT");
6967
} catch (Exception e) {
7068
// ignore if column already exists
7169
logger.trace("Column 'status' already exists in table {}", cdmKsTabInfo);
7270
}
71+
this.session.execute("CREATE TABLE IF NOT EXISTS " + cdmKsTabDetails
72+
+ " (table_name TEXT, run_id BIGINT, start_time TIMESTAMP, token_min BIGINT, token_max BIGINT, status TEXT, PRIMARY KEY ((table_name, run_id), token_min))");
7373

7474
boundInitInfoStatement = bindStatement("INSERT INTO " + cdmKsTabInfo
7575
+ " (table_name, run_id, run_type, prev_run_id, start_time, status) VALUES (?, ?, ?, ?, dateof(now()), ?)");
@@ -88,7 +88,6 @@ public TargetUpsertRunDetailsStatement(CqlSession session, String keyspaceTable)
8888
}
8989

9090
public Collection<SplitPartitions.Partition> getPendingPartitions(long prevRunId) throws RunNotStartedException {
91-
this.prevRunId = prevRunId;
9291
final Collection<SplitPartitions.Partition> pendingParts = new ArrayList<SplitPartitions.Partition>();
9392
if (prevRunId == 0) {
9493
return pendingParts;
@@ -117,31 +116,34 @@ public Collection<SplitPartitions.Partition> getPendingPartitions(long prevRunId
117116
return pendingParts;
118117
}
119118

120-
public long initCdmRun(Collection<SplitPartitions.Partition> parts, RUN_TYPE runType) {
121-
runId = System.currentTimeMillis();
119+
public void initCdmRun(long runId, long prevRunId, Collection<SplitPartitions.Partition> parts, RUN_TYPE runType) {
120+
ResultSet rsInfo = session
121+
.execute(boundSelectInfoStatement.setString("table_name", tableName).setLong("run_id", runId));
122+
if (null != rsInfo.one()) {
123+
throw new RuntimeException("Run id " + runId + " already exists for table " + tableName);
124+
}
122125
session.execute(boundInitInfoStatement.setString("table_name", tableName).setLong("run_id", runId)
123126
.setString("run_type", runType.toString()).setLong("prev_run_id", prevRunId)
124127
.setString("status", TrackRun.RUN_STATUS.NOT_STARTED.toString()));
125-
parts.forEach(part -> initCdmRun(part));
128+
parts.forEach(part -> initCdmRun(runId, part));
126129
session.execute(boundInitInfoStatement.setString("table_name", tableName).setLong("run_id", runId)
127130
.setString("run_type", runType.toString()).setLong("prev_run_id", prevRunId)
128131
.setString("status", TrackRun.RUN_STATUS.STARTED.toString()));
129-
return runId;
130132
}
131133

132-
private void initCdmRun(Partition partition) {
134+
private void initCdmRun(long runId, Partition partition) {
133135
session.execute(boundInitStatement.setString("table_name", tableName).setLong("run_id", runId)
134136
.setLong("token_min", partition.getMin().longValue())
135137
.setLong("token_max", partition.getMax().longValue())
136138
.setString("status", TrackRun.RUN_STATUS.NOT_STARTED.toString()));
137139
}
138140

139-
public void endCdmRun(String runInfo) {
141+
public void endCdmRun(long runId, String runInfo) {
140142
session.execute(boundEndInfoStatement.setString("table_name", tableName).setLong("run_id", runId)
141143
.setString("run_info", runInfo).setString("status", TrackRun.RUN_STATUS.ENDED.toString()));
142144
}
143145

144-
public void updateCdmRun(BigInteger min, TrackRun.RUN_STATUS status) {
146+
public void updateCdmRun(long runId, BigInteger min, TrackRun.RUN_STATUS status) {
145147
if (TrackRun.RUN_STATUS.STARTED.equals(status)) {
146148
session.execute(boundUpdateStartStatement.setString("table_name", tableName).setLong("run_id", runId)
147149
.setLong("token_min", min.longValue()).setString("status", status.toString()));

‎src/main/java/com/datastax/cdm/feature/TrackRun.java

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -49,18 +49,16 @@ public Collection<SplitPartitions.Partition> getPendingPartitions(long prevRunId
4949
return pendingParts;
5050
}
5151

52-
public long initCdmRun(Collection<SplitPartitions.Partition> parts, RUN_TYPE runType) {
53-
long runId = runStatement.initCdmRun(parts, runType);
52+
public void initCdmRun(long runId, long prevRunId, Collection<SplitPartitions.Partition> parts, RUN_TYPE runType) {
53+
runStatement.initCdmRun(runId, prevRunId, parts, runType);
5454
logger.info("###################### Run Id for this job is: {} ######################", runId);
55-
56-
return runId;
5755
}
5856

59-
public void updateCdmRun(BigInteger min, RUN_STATUS status) {
60-
runStatement.updateCdmRun(min, status);
57+
public void updateCdmRun(long runId, BigInteger min, RUN_STATUS status) {
58+
runStatement.updateCdmRun(runId, min, status);
6159
}
6260

63-
public void endCdmRun(String runInfo) {
64-
runStatement.endCdmRun(runInfo);
61+
public void endCdmRun(long runId, String runInfo) {
62+
runStatement.endCdmRun(runId, runInfo);
6563
}
6664
}

‎src/main/java/com/datastax/cdm/job/AbstractJobSession.java

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ public abstract class AbstractJobSession<T> extends BaseJobSession {
4141
protected JobCounter jobCounter;
4242
protected Long printStatsAfter;
4343
protected TrackRun trackRunFeature;
44+
protected long runId;
4445

4546
protected AbstractJobSession(CqlSession originSession, CqlSession targetSession, SparkConf sc) {
4647
this(originSession, targetSession, sc, false);
@@ -102,12 +103,17 @@ protected AbstractJobSession(CqlSession originSession, CqlSession targetSession,
102103

103104
public abstract void processSlice(T slice);
104105

105-
public synchronized void initCdmRun(Collection<SplitPartitions.Partition> parts, TrackRun trackRunFeature) {
106+
public synchronized void initCdmRun(long runId, long prevRunId, Collection<SplitPartitions.Partition> parts,
107+
TrackRun trackRunFeature, TrackRun.RUN_TYPE runType) {
108+
this.runId = runId;
109+
this.trackRunFeature = trackRunFeature;
110+
if (null != trackRunFeature)
111+
trackRunFeature.initCdmRun(runId, prevRunId, parts, runType);
106112
}
107113

108114
public synchronized void printCounts(boolean isFinal) {
109115
if (isFinal) {
110-
jobCounter.printFinal(trackRunFeature);
116+
jobCounter.printFinal(runId, trackRunFeature);
111117
} else {
112118
jobCounter.printProgress();
113119
}

‎src/main/java/com/datastax/cdm/job/CopyJobSession.java

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -70,17 +70,11 @@ public void processSlice(SplitPartitions.Partition slice) {
7070
this.getDataAndInsert(slice.getMin(), slice.getMax());
7171
}
7272

73-
public synchronized void initCdmRun(Collection<SplitPartitions.Partition> parts, TrackRun trackRunFeature) {
74-
this.trackRunFeature = trackRunFeature;
75-
if (null != trackRunFeature)
76-
trackRunFeature.initCdmRun(parts, TrackRun.RUN_TYPE.MIGRATE);
77-
}
78-
7973
private void getDataAndInsert(BigInteger min, BigInteger max) {
8074
ThreadContext.put(THREAD_CONTEXT_LABEL, getThreadLabel(min, max));
8175
logger.info("ThreadID: {} Processing min: {} max: {}", Thread.currentThread().getId(), min, max);
8276
if (null != trackRunFeature)
83-
trackRunFeature.updateCdmRun(min, TrackRun.RUN_STATUS.STARTED);
77+
trackRunFeature.updateCdmRun(runId, min, TrackRun.RUN_STATUS.STARTED);
8478

8579
BatchStatement batch = BatchStatement.newInstance(BatchType.UNLOGGED);
8680
String guardrailCheck;
@@ -139,13 +133,13 @@ private void getDataAndInsert(BigInteger min, BigInteger max) {
139133
jobCounter.getCount(JobCounter.CounterType.UNFLUSHED));
140134
jobCounter.threadReset(JobCounter.CounterType.UNFLUSHED);
141135
if (null != trackRunFeature)
142-
trackRunFeature.updateCdmRun(min, TrackRun.RUN_STATUS.PASS);
136+
trackRunFeature.updateCdmRun(runId, min, TrackRun.RUN_STATUS.PASS);
143137
} catch (Exception e) {
144138
jobCounter.threadIncrement(JobCounter.CounterType.ERROR,
145139
jobCounter.getCount(JobCounter.CounterType.READ) - jobCounter.getCount(JobCounter.CounterType.WRITE)
146140
- jobCounter.getCount(JobCounter.CounterType.SKIPPED));
147141
if (null != trackRunFeature)
148-
trackRunFeature.updateCdmRun(min, TrackRun.RUN_STATUS.FAIL);
142+
trackRunFeature.updateCdmRun(runId, min, TrackRun.RUN_STATUS.FAIL);
149143
logger.error("Error with PartitionRange -- ThreadID: {} Processing min: {} max: {}",
150144
Thread.currentThread().getId(), min, max, e);
151145
logger.error("Error stats " + jobCounter.getThreadCounters(false));

‎src/main/java/com/datastax/cdm/job/DiffJobSession.java

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
import java.math.BigInteger;
1919
import java.util.ArrayList;
20-
import java.util.Collection;
2120
import java.util.Collections;
2221
import java.util.List;
2322
import java.util.concurrent.CompletionStage;
@@ -124,18 +123,11 @@ public void processSlice(SplitPartitions.Partition slice) {
124123
this.getDataAndDiff(slice.getMin(), slice.getMax());
125124
}
126125

127-
@Override
128-
public synchronized void initCdmRun(Collection<SplitPartitions.Partition> parts, TrackRun trackRunFeature) {
129-
this.trackRunFeature = trackRunFeature;
130-
if (null != trackRunFeature)
131-
trackRunFeature.initCdmRun(parts, TrackRun.RUN_TYPE.DIFF_DATA);
132-
}
133-
134126
private void getDataAndDiff(BigInteger min, BigInteger max) {
135127
ThreadContext.put(THREAD_CONTEXT_LABEL, getThreadLabel(min, max));
136128
logger.info("ThreadID: {} Processing min: {} max: {}", Thread.currentThread().getId(), min, max);
137129
if (null != trackRunFeature)
138-
trackRunFeature.updateCdmRun(min, TrackRun.RUN_STATUS.STARTED);
130+
trackRunFeature.updateCdmRun(runId, min, TrackRun.RUN_STATUS.STARTED);
139131

140132
AtomicBoolean hasDiff = new AtomicBoolean(false);
141133
try {
@@ -196,12 +188,12 @@ private void getDataAndDiff(BigInteger min, BigInteger max) {
196188
.getCount(JobCounter.CounterType.CORRECTED_MISSING)
197189
&& jobCounter.getCount(JobCounter.CounterType.MISMATCH) == jobCounter
198190
.getCount(JobCounter.CounterType.CORRECTED_MISMATCH)) {
199-
trackRunFeature.updateCdmRun(min, TrackRun.RUN_STATUS.DIFF_CORRECTED);
191+
trackRunFeature.updateCdmRun(runId, min, TrackRun.RUN_STATUS.DIFF_CORRECTED);
200192
} else {
201-
trackRunFeature.updateCdmRun(min, TrackRun.RUN_STATUS.DIFF);
193+
trackRunFeature.updateCdmRun(runId, min, TrackRun.RUN_STATUS.DIFF);
202194
}
203195
} else if (null != trackRunFeature) {
204-
trackRunFeature.updateCdmRun(min, TrackRun.RUN_STATUS.PASS);
196+
trackRunFeature.updateCdmRun(runId, min, TrackRun.RUN_STATUS.PASS);
205197
}
206198
} catch (Exception e) {
207199
jobCounter.threadIncrement(JobCounter.CounterType.ERROR,
@@ -212,7 +204,7 @@ private void getDataAndDiff(BigInteger min, BigInteger max) {
212204
logger.error("Error with PartitionRange -- ThreadID: {} Processing min: {} max: {}",
213205
Thread.currentThread().getId(), min, max, e);
214206
if (null != trackRunFeature)
215-
trackRunFeature.updateCdmRun(min, TrackRun.RUN_STATUS.FAIL);
207+
trackRunFeature.updateCdmRun(runId, min, TrackRun.RUN_STATUS.FAIL);
216208
} finally {
217209
jobCounter.globalIncrement();
218210
printCounts(false);

‎src/main/java/com/datastax/cdm/job/JobCounter.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ protected void printAndLogProgress(String message, boolean global) {
178178
logger.info(fullMessage);
179179
}
180180

181-
public void printFinal(TrackRun trackRunFeature) {
181+
public void printFinal(long runId, TrackRun trackRunFeature) {
182182
if (null != trackRunFeature) {
183183
StringBuilder sb = new StringBuilder();
184184
if (counterMap.containsKey(CounterType.READ))
@@ -202,7 +202,7 @@ public void printFinal(TrackRun trackRunFeature) {
202202
if (counterMap.containsKey(CounterType.LARGE))
203203
sb.append("; Large: " + counterMap.get(CounterType.LARGE).getGlobalCounter());
204204

205-
trackRunFeature.endCdmRun(sb.toString());
205+
trackRunFeature.endCdmRun(runId, sb.toString());
206206
}
207207
logger.info("################################################################################################");
208208
if (counterMap.containsKey(CounterType.READ))

‎src/main/java/com/datastax/cdm/properties/KnownProperties.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ public enum PropertyType {
109109
public static final String AUTOCORRECT_MISMATCH = "spark.cdm.autocorrect.mismatch"; // false
110110
public static final String AUTOCORRECT_MISSING_COUNTER = "spark.cdm.autocorrect.missing.counter"; // false
111111
public static final String TRACK_RUN = "spark.cdm.trackRun";
112+
public static final String RUN_ID = "spark.cdm.trackRun.runId";
112113
public static final String PREV_RUN_ID = "spark.cdm.trackRun.previousRunId";
113114

114115
public static final String PERF_NUM_PARTS = "spark.cdm.perfops.numParts"; // 5000, was spark.splitSize
@@ -131,6 +132,8 @@ public enum PropertyType {
131132
defaults.put(AUTOCORRECT_MISSING_COUNTER, "false");
132133
types.put(TRACK_RUN, PropertyType.BOOLEAN);
133134
defaults.put(TRACK_RUN, "false");
135+
types.put(RUN_ID, PropertyType.NUMBER);
136+
defaults.put(RUN_ID, "0");
134137
types.put(PREV_RUN_ID, PropertyType.NUMBER);
135138
defaults.put(PREV_RUN_ID, "0");
136139

‎src/main/scala/com/datastax/cdm/job/BaseJob.scala

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ abstract class BaseJob[T: ClassTag] extends App {
4747
var coveragePercent: Int = _
4848
var numSplits: Int = _
4949
var trackRun: Boolean = _
50+
var runId: Long = _
5051
var prevRunId: Long = _
5152

5253
var parts: util.Collection[T] = _
@@ -80,14 +81,21 @@ abstract class BaseJob[T: ClassTag] extends App {
8081
maxPartition = getMaxPartition(propertyHelper.getString(KnownProperties.PARTITION_MAX), hasRandomPartitioner)
8182
coveragePercent = propertyHelper.getInteger(KnownProperties.TOKEN_COVERAGE_PERCENT)
8283
numSplits = propertyHelper.getInteger(KnownProperties.PERF_NUM_PARTS)
84+
runId = propertyHelper.getLong(KnownProperties.RUN_ID)
8385
prevRunId = propertyHelper.getLong(KnownProperties.PREV_RUN_ID)
84-
trackRun = if (0 != prevRunId) true else propertyHelper.getBoolean(KnownProperties.TRACK_RUN)
86+
trackRun = if (0 != prevRunId || 0 != runId) true else propertyHelper.getBoolean(KnownProperties.TRACK_RUN)
87+
if (trackRun == true && runId == 0) {
88+
runId = System.nanoTime();
89+
}
8590

8691
abstractLogger.info("PARAM -- Min Partition: " + minPartition)
8792
abstractLogger.info("PARAM -- Max Partition: " + maxPartition)
8893
abstractLogger.info("PARAM -- Number of Splits : " + numSplits)
8994
abstractLogger.info("PARAM -- Track Run : " + trackRun)
90-
abstractLogger.info("PARAM -- Previous RunId : " + prevRunId)
95+
if (trackRun == true) {
96+
abstractLogger.info("PARAM -- RunId : " + runId)
97+
abstractLogger.info("PARAM -- Previous RunId : " + prevRunId)
98+
}
9199
abstractLogger.info("PARAM -- Coverage Percent: " + coveragePercent)
92100
this.parts = getParts(numSplits)
93101
abstractLogger.info("PARAM Calculated -- Total Partitions: " + parts.size())

‎src/main/scala/com/datastax/cdm/job/DiffData.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
*/
1616
package com.datastax.cdm.job
1717

18+
import com.datastax.cdm.feature.TrackRun
19+
1820
object DiffData extends BasePartitionJob {
1921
setup("Data Validation Job", new DiffJobSessionFactory())
2022
execute()
@@ -24,7 +26,7 @@ object DiffData extends BasePartitionJob {
2426
if (!parts.isEmpty()) {
2527
originConnection.withSessionDo(originSession =>
2628
targetConnection.withSessionDo(targetSession =>
27-
jobFactory.getInstance(originSession, targetSession, sc).initCdmRun(parts, trackRunFeature)));
29+
jobFactory.getInstance(originSession, targetSession, sc).initCdmRun(runId, prevRunId, parts, trackRunFeature, TrackRun.RUN_TYPE.DIFF_DATA)));
2830

2931
slices.foreach(slice => {
3032
originConnection.withSessionDo(originSession =>

0 commit comments

Comments
 (0)