Skip to content

Commit a1b74cc

Browse files
andy-hf-kwokandy-hf-kwok
committed
Fix test - pt1
Signed-off-by: Andy HF Kwok <[email protected]>
1 parent 7bf0be6 commit a1b74cc

File tree

3 files changed

+41
-32
lines changed

3 files changed

+41
-32
lines changed

‎spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase {
5252
def numericScanBenchmark(values: Int, dataType: DataType): Unit = {
5353
// Benchmarks running through spark sql.
5454
val sqlBenchmark =
55-
new Benchmark(s"SQL Single ${dataType.sql} Column Scan", values, output = output)
55+
new Benchmark(s"SQL Single ${dataType.sql} Column Scan", values.toLong, output = output)
5656

5757
withTempPath { dir =>
5858
withTempTable("parquetV1Table") {
@@ -101,7 +101,10 @@ class CometReadBaseBenchmark extends CometBenchmarkBase {
101101
def encryptedScanBenchmark(values: Int, dataType: DataType): Unit = {
102102
// Benchmarks running through spark sql.
103103
val sqlBenchmark =
104-
new Benchmark(s"SQL Single ${dataType.sql} Encrypted Column Scan", values, output = output)
104+
new Benchmark(
105+
s"SQL Single ${dataType.sql} Encrypted Column Scan",
106+
values.toLong,
107+
output = output)
105108

106109
val encoder = Base64.getEncoder
107110
val footerKey =
@@ -189,7 +192,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase {
189192
def decimalScanBenchmark(values: Int, precision: Int, scale: Int): Unit = {
190193
val sqlBenchmark = new Benchmark(
191194
s"SQL Single Decimal(precision: $precision, scale: $scale) Column Scan",
192-
values,
195+
values.toLong,
193196
output = output)
194197

195198
withTempPath { dir =>
@@ -237,7 +240,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase {
237240

238241
def readerBenchmark(values: Int, dataType: DataType): Unit = {
239242
val sqlBenchmark =
240-
new Benchmark(s"Parquet reader benchmark for $dataType", values, output = output)
243+
new Benchmark(s"Parquet reader benchmark for $dataType", values.toLong, output = output)
241244

242245
withTempPath { dir =>
243246
withTempTable("parquetV1Table") {
@@ -318,7 +321,10 @@ class CometReadBaseBenchmark extends CometBenchmarkBase {
318321
def numericFilterScanBenchmark(values: Int, fractionOfZeros: Double): Unit = {
319322
val percentageOfZeros = fractionOfZeros * 100
320323
val benchmark =
321-
new Benchmark(s"Numeric Filter Scan ($percentageOfZeros% zeros)", values, output = output)
324+
new Benchmark(
325+
s"Numeric Filter Scan ($percentageOfZeros% zeros)",
326+
values.toLong,
327+
output = output)
322328

323329
withTempPath { dir =>
324330
withTempTable("parquetV1Table", "parquetV2Table") {
@@ -365,7 +371,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase {
365371

366372
def stringWithDictionaryScanBenchmark(values: Int): Unit = {
367373
val sqlBenchmark =
368-
new Benchmark("String Scan with Dictionary Encoding", values, output = output)
374+
new Benchmark("String Scan with Dictionary Encoding", values.toLong, output = output)
369375

370376
withTempPath { dir =>
371377
withTempTable("parquetV1Table", "parquetV2Table") {
@@ -424,7 +430,10 @@ class CometReadBaseBenchmark extends CometBenchmarkBase {
424430
def stringWithNullsScanBenchmark(values: Int, fractionOfNulls: Double): Unit = {
425431
val percentageOfNulls = fractionOfNulls * 100
426432
val benchmark =
427-
new Benchmark(s"String with Nulls Scan ($percentageOfNulls%)", values, output = output)
433+
new Benchmark(
434+
s"String with Nulls Scan ($percentageOfNulls%)",
435+
values.toLong,
436+
output = output)
428437

429438
withTempPath { dir =>
430439
withTempTable("parquetV1Table") {
@@ -483,7 +492,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase {
483492

484493
def columnsBenchmark(values: Int, width: Int): Unit = {
485494
val benchmark =
486-
new Benchmark(s"Single Column Scan from $width columns", values, output = output)
495+
new Benchmark(s"Single Column Scan from $width columns", values.toLong, output = output)
487496

488497
withTempPath { dir =>
489498
withTempTable("t1", "parquetV1Table") {
@@ -533,7 +542,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase {
533542
val benchmark =
534543
new Benchmark(
535544
s"Large String Filter Scan ($percentageOfZeros% zeros)",
536-
values,
545+
values.toLong,
537546
output = output)
538547

539548
withTempPath { dir =>
@@ -584,7 +593,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase {
584593
val benchmark =
585594
new Benchmark(
586595
s"Sorted Lg Str Filter Scan ($percentageOfZeros% zeros)",
587-
values,
596+
values.toLong,
588597
output = output)
589598

590599
withTempPath { dir =>

‎spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase {
7171
val benchmark =
7272
new Benchmark(
7373
s"SQL ${dataType.sql} shuffle on array ($partitionNum Partition)",
74-
values,
74+
values.toLong,
7575
output = output)
7676

7777
withTempPath { dir =>
@@ -123,7 +123,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase {
123123
val benchmark =
124124
new Benchmark(
125125
s"SQL ${dataType.sql} shuffle on struct ($partitionNum Partition)",
126-
values,
126+
values.toLong,
127127
output = output)
128128

129129
withTempPath { dir =>
@@ -182,7 +182,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase {
182182
val benchmark =
183183
new Benchmark(
184184
s"SQL ${dataType.sql} Dictionary Shuffle($partitionNum Partition)",
185-
values,
185+
values.toLong,
186186
output = output)
187187

188188
withTempPath { dir =>
@@ -273,7 +273,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase {
273273
val benchmark =
274274
new Benchmark(
275275
s"SQL Single ${dataType.sql} Shuffle($partitionNum Partition) $randomTitle",
276-
values,
276+
values.toLong,
277277
output = output)
278278

279279
withTempPath { dir =>
@@ -359,7 +359,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase {
359359
val benchmark =
360360
new Benchmark(
361361
s"SQL Wide ($width cols) ${dataType.sql} Shuffle($partitionNum Partition)",
362-
values,
362+
values.toLong,
363363
output = output)
364364

365365
val projection = (1 to width)
@@ -429,7 +429,7 @@ object CometShuffleBenchmark extends CometBenchmarkBase {
429429
val benchmark =
430430
new Benchmark(
431431
s"SQL Wide ($width cols) ${dataType.sql} Range Partition Shuffle($partitionNum Partition)",
432-
values,
432+
values.toLong,
433433
output = output)
434434

435435
val projection = (1 to width)

‎spark/src/test/scala/org/apache/spark/sql/benchmark/CometStringExpressionBenchmark.scala‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import org.apache.comet.CometConf
3232
object CometStringExpressionBenchmark extends CometBenchmarkBase {
3333

3434
def subStringExprBenchmark(values: Int): Unit = {
35-
val benchmark = new Benchmark("Substring Expr", values, output = output)
35+
val benchmark = new Benchmark("Substring Expr", values.toLong, output = output)
3636

3737
withTempPath { dir =>
3838
withTempTable("parquetV1Table") {
@@ -62,7 +62,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
6262
}
6363

6464
def stringSpaceExprBenchmark(values: Int): Unit = {
65-
val benchmark = new Benchmark("StringSpace Expr", values, output = output)
65+
val benchmark = new Benchmark("StringSpace Expr", values.toLong, output = output)
6666

6767
withTempPath { dir =>
6868
withTempTable("parquetV1Table") {
@@ -92,7 +92,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
9292
}
9393

9494
def asciiExprBenchmark(values: Int): Unit = {
95-
val benchmark = new Benchmark("Expr ascii", values, output = output)
95+
val benchmark = new Benchmark("Expr ascii", values.toLong, output = output)
9696

9797
withTempPath { dir =>
9898
withTempTable("parquetV1Table") {
@@ -122,7 +122,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
122122
}
123123

124124
def bitLengthExprBenchmark(values: Int): Unit = {
125-
val benchmark = new Benchmark("Expr bit_length", values, output = output)
125+
val benchmark = new Benchmark("Expr bit_length", values.toLong, output = output)
126126

127127
withTempPath { dir =>
128128
withTempTable("parquetV1Table") {
@@ -152,7 +152,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
152152
}
153153

154154
def octetLengthExprBenchmark(values: Int): Unit = {
155-
val benchmark = new Benchmark("Expr octet_length", values, output = output)
155+
val benchmark = new Benchmark("Expr octet_length", values.toLong, output = output)
156156

157157
withTempPath { dir =>
158158
withTempTable("parquetV1Table") {
@@ -182,7 +182,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
182182
}
183183

184184
def upperExprBenchmark(values: Int): Unit = {
185-
val benchmark = new Benchmark("Expr upper", values, output = output)
185+
val benchmark = new Benchmark("Expr upper", values.toLong, output = output)
186186

187187
withTempPath { dir =>
188188
withTempTable("parquetV1Table") {
@@ -213,7 +213,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
213213
}
214214

215215
def lowerExprBenchmark(values: Int): Unit = {
216-
val benchmark = new Benchmark("Expr lower", values, output = output)
216+
val benchmark = new Benchmark("Expr lower", values.toLong, output = output)
217217

218218
withTempPath { dir =>
219219
withTempTable("parquetV1Table") {
@@ -243,7 +243,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
243243
}
244244

245245
def chrExprBenchmark(values: Int): Unit = {
246-
val benchmark = new Benchmark("Expr chr", values, output = output)
246+
val benchmark = new Benchmark("Expr chr", values.toLong, output = output)
247247

248248
withTempPath { dir =>
249249
withTempTable("parquetV1Table") {
@@ -273,7 +273,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
273273
}
274274

275275
def initCapExprBenchmark(values: Int): Unit = {
276-
val benchmark = new Benchmark("Expr initCap", values, output = output)
276+
val benchmark = new Benchmark("Expr initCap", values.toLong, output = output)
277277

278278
withTempPath { dir =>
279279
withTempTable("parquetV1Table") {
@@ -303,7 +303,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
303303
}
304304

305305
def trimExprBenchmark(values: Int): Unit = {
306-
val benchmark = new Benchmark("Expr trim", values, output = output)
306+
val benchmark = new Benchmark("Expr trim", values.toLong, output = output)
307307

308308
withTempPath { dir =>
309309
withTempTable("parquetV1Table") {
@@ -333,7 +333,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
333333
}
334334

335335
def concatwsExprBenchmark(values: Int): Unit = {
336-
val benchmark = new Benchmark("Expr concatws", values, output = output)
336+
val benchmark = new Benchmark("Expr concatws", values.toLong, output = output)
337337

338338
withTempPath { dir =>
339339
withTempTable("parquetV1Table") {
@@ -363,7 +363,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
363363
}
364364

365365
def lengthExprBenchmark(values: Int): Unit = {
366-
val benchmark = new Benchmark("Expr length", values, output = output)
366+
val benchmark = new Benchmark("Expr length", values.toLong, output = output)
367367

368368
withTempPath { dir =>
369369
withTempTable("parquetV1Table") {
@@ -393,7 +393,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
393393
}
394394

395395
def repeatExprBenchmark(values: Int): Unit = {
396-
val benchmark = new Benchmark("Expr repeat", values, output = output)
396+
val benchmark = new Benchmark("Expr repeat", values.toLong, output = output)
397397

398398
withTempPath { dir =>
399399
withTempTable("parquetV1Table") {
@@ -423,7 +423,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
423423
}
424424

425425
def reverseExprBenchmark(values: Int): Unit = {
426-
val benchmark = new Benchmark("Expr reverse", values, output = output)
426+
val benchmark = new Benchmark("Expr reverse", values.toLong, output = output)
427427

428428
withTempPath { dir =>
429429
withTempTable("parquetV1Table") {
@@ -483,7 +483,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
483483
}
484484

485485
def replaceExprBenchmark(values: Int): Unit = {
486-
val benchmark = new Benchmark("Expr replace", values, output = output)
486+
val benchmark = new Benchmark("Expr replace", values.toLong, output = output)
487487

488488
withTempPath { dir =>
489489
withTempTable("parquetV1Table") {
@@ -513,7 +513,7 @@ object CometStringExpressionBenchmark extends CometBenchmarkBase {
513513
}
514514

515515
def translateExprBenchmark(values: Int): Unit = {
516-
val benchmark = new Benchmark("Expr translate", values, output = output)
516+
val benchmark = new Benchmark("Expr translate", values.toLong, output = output)
517517

518518
withTempPath { dir =>
519519
withTempTable("parquetV1Table") {

0 commit comments

Comments
 (0)