File tree Expand file tree Collapse file tree 2 files changed +7
-10
lines changed
src/main/scala/com/datastax/cdm/job Expand file tree Collapse file tree 2 files changed +7
-10
lines changed Original file line number Diff line number Diff line change @@ -32,6 +32,7 @@ abstract class BaseJob[T: ClassTag] extends App {
32
32
var maxPartition : BigInteger = _
33
33
var coveragePercent : Int = _
34
34
var numSplits : Int = _
35
+ var partitionFile : String = _
35
36
36
37
var parts : util.Collection [T ] = _
37
38
var slices : RDD [T ] = _
@@ -69,8 +70,9 @@ abstract class BaseJob[T: ClassTag] extends App {
69
70
maxPartition = getMaxPartition(propertyHelper.getString(KnownProperties .PARTITION_MAX ), hasRandomPartitioner)
70
71
coveragePercent = propertyHelper.getInteger(KnownProperties .TOKEN_COVERAGE_PERCENT )
71
72
numSplits = propertyHelper.getInteger(KnownProperties .PERF_NUM_PARTS )
72
- if (" " .equals(this .fileName)) {
73
- this .fileName = propertyHelper.getString(KnownProperties .PARTITIONS_TOKEN_RANGE_FILE )
73
+ partitionFile = propertyHelper.getString(KnownProperties .PARTITIONS_TOKEN_RANGE_FILE )
74
+ if (partitionFile.trim.nonEmpty && jobName.contains(" Partitions from File" )) {
75
+ this .fileName = partitionFile
74
76
}
75
77
abstractLogger.info(" PARAM -- Min Partition: " + minPartition)
76
78
abstractLogger.info(" PARAM -- Max Partition: " + maxPartition)
Original file line number Diff line number Diff line change 1
1
package com .datastax .cdm .job
2
2
3
3
object DiffPartitionsFromFile extends BasePartitionJob {
4
- setup(" Diff Partitions from File Job" , new DiffJobSessionFactory ())
5
- if (" " .equals(this .fileName)) {
6
- abstractLogger.error(" Please set conf for spark.input.partitionFile " )
7
- }
8
- else {
9
- execute()
10
- finish()
11
- }
4
+ setup(" Diff Partitions from File Job" , new DiffJobSessionFactory (), " ./partitions.csv" )
5
+ execute()
6
+ finish()
12
7
13
8
override def execute (): Unit = {
14
9
slices.foreach(slice => {
You can’t perform that action at this time.
0 commit comments