Skip to content

Commit 8a087af

Browse files
committed
revert unnecessary changes in DataSourceRDD.scala
1 parent fef4c22 commit 8a087af

File tree

1 file changed

+2
-20
lines changed

1 file changed

+2
-20
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala

Lines changed: 2 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -135,11 +135,6 @@ class DataSourceRDD(
135135
}
136136
}
137137

138-
/**
139-
* An iterator that wraps a PartitionReader and periodically updates custom metrics.
140-
* Metrics are updated every `CustomMetrics.NUM_ROWS_PER_UPDATE` rows to balance
141-
* between metric accuracy and performance overhead.
142-
*/
143138
private class PartitionIterator[T](
144139
reader: PartitionReader[T],
145140
customMetrics: Map[String, SQLMetric]) extends Iterator[T] {
@@ -169,12 +164,7 @@ private class PartitionIterator[T](
169164
}
170165
}
171166

172-
/**
173-
* Handles updates to Spark's built-in input metrics (records read and bytes read).
174-
* Bytes read are updated periodically based on Spark's update interval to avoid
175-
* excessive overhead from filesystem calls.
176-
*/
177-
private[spark] class MetricsHandler extends Logging with Serializable {
167+
private class MetricsHandler extends Logging with Serializable {
178168
private val inputMetrics = TaskContext.get().taskMetrics().inputMetrics
179169
private val startingBytesRead = inputMetrics.bytesRead
180170
private val getBytesRead = SparkHadoopUtil.get.getFSBytesReadOnThreadCallback()
@@ -189,15 +179,7 @@ private[spark] class MetricsHandler extends Logging with Serializable {
189179
}
190180
}
191181

192-
/**
193-
* Base class for iterators that track metrics and save the final custom metrics values
194-
* when iteration completes. The saved metrics are stored in a ThreadLocal so the task
195-
* completion listener can access them for final reporting.
196-
*
197-
* When `hasNext` returns false (iteration complete), this iterator saves the current
198-
* custom metrics to the ThreadLocal before forcing a final metrics update.
199-
*/
200-
private[spark] abstract class MetricsIterator[I](iter: Iterator[I]) extends Iterator[I] {
182+
private abstract class MetricsIterator[I](iter: Iterator[I]) extends Iterator[I] {
201183
protected val metricsHandler = new MetricsHandler
202184

203185
override def hasNext: Boolean = {

0 commit comments

Comments
 (0)