diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala
index 7e3c91f87b..3207daa211 100644
--- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCDSMicroBenchmark.scala
@@ -84,7 +84,7 @@ object CometTPCDSMicroBenchmark extends CometTPCQueryBenchmarkBase {
       benchmarkName: String,
       nameSuffix: String = ""): Unit = {
     queries.foreach { name =>
-      val source = Source.fromFile(s"spark/src/test/resources/tpcds-micro-benchmarks/$name.sql")
+      val source = Source.fromFile(s"src/test/resources/tpcds-micro-benchmarks/$name.sql")
       val queryString = source
         .getLines()
         .filterNot(_.startsWith("--"))
@@ -108,34 +108,13 @@ object CometTPCDSMicroBenchmark extends CometTPCQueryBenchmarkBase {
       benchmark.addCase(s"$name$nameSuffix") { _ =>
         cometSpark.sql(queryString).noop()
       }
-      benchmark.addCase(s"$name$nameSuffix: Comet (Scan)") { _ =>
-        withSQLConf(CometConf.COMET_ENABLED.key -> "true") {
-          cometSpark.sql(queryString).noop()
-        }
-      }
-      benchmark.addCase(s"$name$nameSuffix: Comet (Scan, Exec)") { _ =>
-        withSQLConf(
-          CometConf.COMET_ENABLED.key -> "true",
-          CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true",
-          CometConf.COMET_SHUFFLE_MODE.key -> "auto",
-          CometConf.COMET_REGEXP_ALLOW_INCOMPATIBLE.key -> "true",
-          // enabling COMET_EXPLAIN_NATIVE_ENABLED may add overhead but is useful for debugging
-          CometConf.COMET_EXPLAIN_NATIVE_ENABLED.key -> "false",
-          CometConf.COMET_EXEC_ENABLED.key -> "true") {
-          cometSpark.sql(queryString).noop()
-        }
-      }
-      benchmark.addCase(s"$name$nameSuffix: Comet (Exec)") { _ =>
+      benchmark.addCase(s"$name$nameSuffix: Comet") { _ =>
         withSQLConf(
           CometConf.COMET_ENABLED.key -> "true",
-          CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "false",
-          CometConf.COMET_CONVERT_FROM_PARQUET_ENABLED.key -> "true",
-          CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true",
-          CometConf.COMET_SHUFFLE_MODE.key -> "auto",
+          CometConf.COMET_EXEC_ENABLED.key -> "true",
           CometConf.COMET_REGEXP_ALLOW_INCOMPATIBLE.key -> "true",
           // enabling COMET_EXPLAIN_NATIVE_ENABLED may add overhead but is useful for debugging
-          CometConf.COMET_EXPLAIN_NATIVE_ENABLED.key -> "false",
-          CometConf.COMET_EXEC_ENABLED.key -> "true") {
+          CometConf.COMET_EXPLAIN_NATIVE_ENABLED.key -> "false") {
           cometSpark.sql(queryString).noop()
         }
       }
diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala
index 50be485a84..da8d20d22e 100644
--- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometTPCQueryBenchmarkBase.scala
@@ -65,28 +65,10 @@ trait CometTPCQueryBenchmarkBase extends SqlBasedBenchmark with CometTPCQueryBas
       benchmark.addCase(s"$name$nameSuffix") { _ =>
         cometSpark.sql(queryString).noop()
       }
-      benchmark.addCase(s"$name$nameSuffix: Comet (Scan)") { _ =>
-        withSQLConf(CometConf.COMET_ENABLED.key -> "true") {
-          cometSpark.sql(queryString).noop()
-        }
-      }
-      benchmark.addCase(s"$name$nameSuffix: Comet (Scan, Exec)") { _ =>
-        withSQLConf(
-          CometConf.COMET_ENABLED.key -> "true",
-          CometConf.COMET_EXEC_ENABLED.key -> "true",
-          CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true",
-          CometConf.COMET_SHUFFLE_MODE.key -> "auto") {
-          cometSpark.sql(queryString).noop()
-        }
-      }
-      benchmark.addCase(s"$name$nameSuffix: Comet (Exec)") { _ =>
+      benchmark.addCase(s"$name$nameSuffix: Comet") { _ =>
         withSQLConf(
           CometConf.COMET_ENABLED.key -> "true",
-          CometConf.COMET_EXEC_ENABLED.key -> "true",
-          CometConf.COMET_NATIVE_SCAN_ENABLED.key -> "false",
-          CometConf.COMET_CONVERT_FROM_PARQUET_ENABLED.key -> "true",
-          CometConf.COMET_EXEC_SHUFFLE_ENABLED.key -> "true",
-          CometConf.COMET_SHUFFLE_MODE.key -> "auto") {
+          CometConf.COMET_EXEC_ENABLED.key -> "true") {
           cometSpark.sql(queryString).noop()
         }
       }