Skip to content

perf: Improve date/time microbenchmarks to avoid redundant/duplicate benchmarks #3020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andygrove merged 5 commits into from
Jan 6, 2026
Merged

perf: Improve date/time microbenchmarks to avoid redundant/duplicate benchmarks #3020

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bd2c075
improve date/time microbenchmarks
andygrove Jan 2, 2026
9af4184
skip some CI workflows for benchmark changes
andygrove Jan 2, 2026
6869f79
skip failing suite
andygrove Jan 2, 2026
15d8515
Merge branch 'skip-ci-bench' into improve-date-time-bench
andygrove Jan 2, 2026
8679f1e
Merge remote-tracking branch 'apache/main' into improve-date-time-bench
andygrove Jan 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 11 additions & 31 deletions spark/src/test/scala/org/apache/spark/sql/benchmark/CometDatetimeExpressionBenchmark.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,47 +30,33 @@ import org.apache.spark.sql.internal.SQLConf
*/
object CometDatetimeExpressionBenchmark extends CometBenchmarkBase {

def dateTruncExprBenchmark(values: Int, useDictionary: Boolean): Unit = {
def dateTruncExprBenchmark(values: Int): Unit = {
withTempPath { dir =>
withTempTable("parquetV1Table") {
prepareTable(
dir,
spark.sql(
s"select cast(timestamp_micros(cast(value/100000 as integer)) as date) as dt FROM $tbl"))
Seq("YEAR", "YYYY", "YY", "MON", "MONTH", "MM").foreach { level =>
val isDictionary = if (useDictionary) "(Dictionary)" else ""
val name = s"Date Truncate $isDictionary - $level"
Seq("YEAR", "MONTH").foreach { level =>
val name = s"Date Truncate - $level"
val query = s"select trunc(dt, '$level') from parquetV1Table"
runExpressionBenchmark(name, values, query)
}
}
}
}

def timestampTruncExprBenchmark(values: Int, useDictionary: Boolean): Unit = {
Copy link
Contributor

@mbutrovich mbutrovich Jan 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think there are a lot of tests we could start removing this logic from too.

def timestampTruncExprBenchmark(values: Int): Unit = {
withTempPath { dir =>
withTempTable("parquetV1Table") {
prepareTable(
dir,
spark.sql(s"select timestamp_micros(cast(value/100000 as integer)) as ts FROM $tbl"))
Seq(
"YEAR",
"YYYY",
"YY",
"MON",
"MONTH",
"MM",
"DAY",
"DD",
"HOUR",
"MINUTE",
"SECOND",
"WEEK",
"QUARTER").foreach { level =>
val isDictionary = if (useDictionary) "(Dictionary)" else ""
val name = s"Timestamp Truncate $isDictionary - $level"
val query = s"select date_trunc('$level', ts) from parquetV1Table"
runExpressionBenchmark(name, values, query)
Seq("YEAR", "MONTH", "DAY", "HOUR", "MINUTE", "SECOND", "WEEK", "QUARTER").foreach {
level =>
val name = s"Timestamp Truncate - $level"
val query = s"select date_trunc('$level', ts) from parquetV1Table"
runExpressionBenchmark(name, values, query)
}
}
}
Expand All @@ -85,16 +71,10 @@ object CometDatetimeExpressionBenchmark extends CometBenchmarkBase {
"spark.sql.parquet.datetimeRebaseModeInWrite" -> "CORRECTED") {

runBenchmarkWithTable("DateTrunc", values) { v =>
dateTruncExprBenchmark(v, useDictionary = false)
}
runBenchmarkWithTable("DateTrunc (Dictionary)", values, useDictionary = true) { v =>
dateTruncExprBenchmark(v, useDictionary = true)
dateTruncExprBenchmark(v)
}
runBenchmarkWithTable("TimestampTrunc", values) { v =>
timestampTruncExprBenchmark(v, useDictionary = false)
}
runBenchmarkWithTable("TimestampTrunc (Dictionary)", values, useDictionary = true) { v =>
timestampTruncExprBenchmark(v, useDictionary = true)
timestampTruncExprBenchmark(v)
}
}
}
Expand Down