[SPARK-55480][PYTHON] Remove all unused noqa for ruff

gaogaotiantian · zhengruifeng · commit 52c43ca8c8d0 · 2026-02-11T20:25:13.000+08:00
### What changes were proposed in this pull request? Removed all unused `# noqa` comments for linter ### Why are the changes needed? We accumulated years of `# noqa` comments for linter. A lot of them are unused (because linter gets smarter). We removed all the unused ones and make `ruff` check for unused noqa in the future - just like `mypy` did. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? CI, but this should be a comment only change. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #54264 from gaogaotiantian/remove-unused-noqa. Authored-by: Tian Gao <gaogaotiantian@hotmail.com> Signed-off-by: Ruifeng Zheng <ruifengz@apache.org>
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
@@ -674,7 +674,7 @@ def __hash__(self):
         "pyspark.sql.tests.pandas.streaming.test_pandas_transform_with_state",
         "pyspark.sql.tests.pandas.streaming.test_pandas_transform_with_state_checkpoint_v2",
         "pyspark.sql.tests.pandas.streaming.test_pandas_transform_with_state_state_variable",
-        "pyspark.sql.tests.pandas.streaming.test_pandas_transform_with_state_state_variable_checkpoint_v2",  # noqa: E501
+        "pyspark.sql.tests.pandas.streaming.test_pandas_transform_with_state_state_variable_checkpoint_v2",
         "pyspark.sql.tests.pandas.streaming.test_transform_with_state",
         "pyspark.sql.tests.pandas.streaming.test_transform_with_state_checkpoint_v2",
         "pyspark.sql.tests.pandas.streaming.test_transform_with_state_state_variable",
@@ -1215,9 +1215,9 @@ def __hash__(self):
         "pyspark.sql.tests.connect.streaming.test_parity_foreach_batch",
         "pyspark.sql.tests.connect.pandas.streaming.test_parity_pandas_grouped_map_with_state",
         "pyspark.sql.tests.connect.pandas.streaming.test_parity_pandas_transform_with_state",
-        "pyspark.sql.tests.connect.pandas.streaming.test_parity_pandas_transform_with_state_state_variable",  # noqa: E501
+        "pyspark.sql.tests.connect.pandas.streaming.test_parity_pandas_transform_with_state_state_variable",
         "pyspark.sql.tests.connect.pandas.streaming.test_parity_transform_with_state",
-        "pyspark.sql.tests.connect.pandas.streaming.test_parity_transform_with_state_state_variable",  # noqa: E501
+        "pyspark.sql.tests.connect.pandas.streaming.test_parity_transform_with_state_state_variable",
     ],
     excluded_python_implementations=[
         "PyPy"  # Skip these tests under PyPy since they require numpy and it isn't available there
diff --git a/pyproject.toml b/pyproject.toml
@@ -38,7 +38,8 @@ exclude = [
 
 [tool.ruff.lint]
 extend-select = [
-    "G010"  # logging-warn
+    "G010",  # logging-warn
+    "RUF100",  # unused-noqa
 ]
 ignore = [
     "E402", # Module top level import is disabled for optional import check, etc.
diff --git a/python/pyspark/accumulators.py b/python/pyspark/accumulators.py
@@ -27,7 +27,7 @@
 from pyspark.errors import PySparkRuntimeError
 
 if TYPE_CHECKING:
-    from pyspark._typing import SupportsIAdd  # noqa: F401
+    from pyspark._typing import SupportsIAdd
     import socketserver.BaseRequestHandler  # type: ignore[import-not-found]
 
 
diff --git a/python/pyspark/errors/__init__.py b/python/pyspark/errors/__init__.py
@@ -18,7 +18,7 @@
 """
 PySpark exceptions.
 """
-from pyspark.errors.exceptions.base import (  # noqa: F401
+from pyspark.errors.exceptions.base import (
     PySparkException,
     AnalysisException,
     SessionNotSameException,
diff --git a/python/pyspark/errors/exceptions/tblib.py b/python/pyspark/errors/exceptions/tblib.py
@@ -206,9 +206,7 @@ def as_traceback(self) -> Optional[TracebackType]:
 
             # noinspection PyBroadException
             try:
-                exec(
-                    code, dict(current.tb_frame.f_globals), dict(current.tb_frame.f_locals)
-                )  # noqa: S102
+                exec(code, dict(current.tb_frame.f_globals), dict(current.tb_frame.f_locals))
             except Exception:
                 next_tb = sys.exc_info()[2].tb_next  # type: ignore
                 if top_tb is None:
diff --git a/python/pyspark/errors_doc_gen.py b/python/pyspark/errors_doc_gen.py
@@ -44,7 +44,7 @@ def generate_errors_doc(output_rst_file_path: str) -> None:
 This is a list of common, named error classes returned by PySpark which are defined at `error-conditions.json <https://github.com/apache/spark/blob/master/python/pyspark/errors/error-conditions.json>`_.
 
 When writing PySpark errors, developers must use an error class from the list. If an appropriate error class is not available, add a new one into the list. For more information, please refer to `Contributing Error and Exception <contributing.rst#contributing-error-and-exception>`_.
-"""  # noqa
+"""
     with open(output_rst_file_path, "w") as f:
         f.write(header + "\n\n")
         for error_key, error_details in ERROR_CLASSES_MAP.items():
diff --git a/python/pyspark/logger/__init__.py b/python/pyspark/logger/__init__.py
@@ -18,6 +18,6 @@
 """
 PySpark logging
 """
-from pyspark.logger.logger import PySparkLogger, SPARK_LOG_SCHEMA  # noqa: F401
+from pyspark.logger.logger import PySparkLogger, SPARK_LOG_SCHEMA
 
 __all__ = ["PySparkLogger", "SPARK_LOG_SCHEMA"]
diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py
@@ -2180,7 +2180,7 @@ def __init__(
             0.9,
             0.95,
             0.99,
-        ],  # noqa: B005
+        ],
         quantilesCol: Optional[str] = None,
         aggregationDepth: int = 2,
         maxBlockSizeInMB: float = 0.0,
@@ -2220,7 +2220,7 @@ def setParams(
             0.9,
             0.95,
             0.99,
-        ],  # noqa: B005
+        ],
         quantilesCol: Optional[str] = None,
         aggregationDepth: int = 2,
         maxBlockSizeInMB: float = 0.0,
diff --git a/python/pyspark/mllib/_typing.pyi b/python/pyspark/mllib/_typing.pyi
@@ -19,7 +19,7 @@
 from typing import List, Tuple, TYPE_CHECKING, TypeVar, Union
 
 from typing_extensions import Literal
-from numpy import ndarray  # noqa: F401
+from numpy import ndarray
 from py4j.java_gateway import JavaObject
 
 from pyspark.mllib.linalg import Vector
diff --git a/python/pyspark/pandas/plot/__init__.py b/python/pyspark/pandas/plot/__init__.py
@@ -14,4 +14,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from pyspark.pandas.plot.core import *  # noqa: F401,F403
+from pyspark.pandas.plot.core import *  # noqa: F403
diff --git a/python/pyspark/pandas/tests/computation/test_apply_func.py b/python/pyspark/pandas/tests/computation/test_apply_func.py
@@ -154,7 +154,7 @@ def identify1(x) -> ps.DataFrame[int, int]:
         self.assert_eq(sorted(actual["c0"].to_numpy()), sorted(expected["a"].to_numpy()))
         self.assert_eq(sorted(actual["c1"].to_numpy()), sorted(expected["b"].to_numpy()))
 
-        def identify2(x) -> ps.DataFrame[slice("a", int), slice("b", int)]:  # noqa: F405
+        def identify2(x) -> ps.DataFrame[slice("a", int), slice("b", int)]:
             return x
 
         actual = psdf.apply(identify2, axis=1)
@@ -229,7 +229,7 @@ def identify1(x) -> ps.DataFrame[int, int]:
         self.assert_eq(sorted(actual["c0"].to_numpy()), sorted(expected["a"].to_numpy()))
         self.assert_eq(sorted(actual["c1"].to_numpy()), sorted(expected["b"].to_numpy()))
 
-        def identify2(x) -> ps.DataFrame[slice("a", int), slice("b", int)]:  # noqa: F405
+        def identify2(x) -> ps.DataFrame[slice("a", int), slice("b", int)]:
             return x
 
         actual = psdf.pandas_on_spark.apply_batch(identify2)
@@ -425,7 +425,7 @@ def identify1(x) -> ps.DataFrame[int, int]:
         self.assert_eq(sorted(actual["c0"].to_numpy()), sorted(expected["a"].to_numpy()))
         self.assert_eq(sorted(actual["c1"].to_numpy()), sorted(expected["b"].to_numpy()))
 
-        def identify2(x) -> ps.DataFrame[slice("a", int), slice("b", int)]:  # noqa: F405
+        def identify2(x) -> ps.DataFrame[slice("a", int), slice("b", int)]:
             return x
 
         actual = psdf.pandas_on_spark.transform_batch(identify2)
diff --git a/python/pyspark/pandas/tests/connect/indexes/test_parity_indexing_adv.py b/python/pyspark/pandas/tests/connect/indexes/test_parity_indexing_adv.py
@@ -29,7 +29,7 @@ class IndexingAdvParityTests(
 
 
 if __name__ == "__main__":
-    from pyspark.pandas.tests.connect.indexes.test_parity_indexing import *  # noqa: F401
+    from pyspark.pandas.tests.connect.indexes.test_parity_indexing import *
 
     from pyspark.testing import main
 
diff --git a/python/pyspark/pandas/tests/groupby/test_apply_func.py b/python/pyspark/pandas/tests/groupby/test_apply_func.py
@@ -145,7 +145,7 @@ def add_max1(x) -> ps.DataFrame[int, int, int]:
 
         def add_max2(
             x,
-        ) -> ps.DataFrame[slice("a", int), slice("b", int), slice("c", int)]:  # noqa: F405
+        ) -> ps.DataFrame[slice("a", int), slice("b", int), slice("c", int)]:
             return x + x.min()
 
         actual = psdf.groupby("b").apply(add_max2).sort_index()
diff --git a/python/pyspark/pandas/typedef/__init__.py b/python/pyspark/pandas/typedef/__init__.py
@@ -15,4 +15,4 @@
 # limitations under the License.
 #
 
-from pyspark.pandas.typedef.typehints import *  # noqa: F401,F403,F405
+from pyspark.pandas.typedef.typehints import *  # noqa: F403
diff --git a/python/pyspark/pandas/utils.py b/python/pyspark/pandas/utils.py
@@ -47,7 +47,7 @@
 from pyspark.sql.types import DoubleType
 from pyspark.sql.utils import is_remote
 from pyspark.errors import PySparkTypeError, UnsupportedOperationException
-from pyspark import pandas as ps  # noqa: F401
+from pyspark import pandas as ps
 from pyspark.pandas._typing import (
     Axis,
     Label,
diff --git a/python/pyspark/sql/connect/client/__init__.py b/python/pyspark/sql/connect/client/__init__.py
@@ -19,5 +19,5 @@
 
 check_dependencies(__name__)
 
-from pyspark.sql.connect.client.core import *  # noqa: F401,F403
+from pyspark.sql.connect.client.core import *  # noqa: F403
 from pyspark.sql.connect.logging import getLogLevel  # noqa: F401
diff --git a/python/pyspark/sql/connect/functions/__init__.py b/python/pyspark/sql/connect/functions/__init__.py
@@ -19,5 +19,5 @@
 from pyspark.testing.utils import should_test_connect
 
 if should_test_connect:
-    from pyspark.sql.connect.functions.builtin import *  # noqa: F401,F403
-    from pyspark.sql.connect.functions import partitioning  # noqa: F401,F403
+    from pyspark.sql.connect.functions.builtin import *  # noqa: F403
+    from pyspark.sql.connect.functions import partitioning  # noqa: F401
diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py
@@ -1976,7 +1976,7 @@ def command(self, session: "SparkConnectClient") -> proto.Command:
                 tsm = self.table_save_method.lower()
                 if tsm == "save_as_table":
                     plan.write_operation.table.save_method = (
-                        proto.WriteOperation.SaveTable.TableSaveMethod.TABLE_SAVE_METHOD_SAVE_AS_TABLE  # noqa: E501
+                        proto.WriteOperation.SaveTable.TableSaveMethod.TABLE_SAVE_METHOD_SAVE_AS_TABLE
                     )
                 elif tsm == "insert_into":
                     plan.write_operation.table.save_method = (
diff --git a/python/pyspark/sql/connect/session.py b/python/pyspark/sql/connect/session.py
@@ -774,7 +774,7 @@ def createDataFrame(
             configs["spark.sql.session.localRelationChunkSizeBytes"]  # type: ignore[arg-type]
         )
         max_batch_of_chunks_size_bytes = int(
-            configs["spark.sql.session.localRelationBatchOfChunksSizeBytes"]  # type: ignore[arg-type] # noqa: E501
+            configs["spark.sql.session.localRelationBatchOfChunksSizeBytes"]  # type: ignore[arg-type]
         )
         plan: LogicalPlan = local_relation
         if cache_threshold <= _table.nbytes:
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
@@ -38,7 +38,7 @@
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.readwriter import DataFrameReader
 from pyspark.sql.streaming import DataStreamReader
-from pyspark.sql.udf import UDFRegistration  # noqa: F401
+from pyspark.sql.udf import UDFRegistration
 from pyspark.sql.udtf import UDTFRegistration
 from pyspark.errors.exceptions.captured import install_exception_handler
 from pyspark.sql.types import AtomicType, DataType, StructType
diff --git a/python/pyspark/sql/functions/__init__.py b/python/pyspark/sql/functions/__init__.py
@@ -17,8 +17,8 @@
 
 """PySpark Functions"""
 
-from pyspark.sql.functions.builtin import *  # noqa: F401,F403
-from pyspark.sql.functions import partitioning  # noqa: F401,F403
+from pyspark.sql.functions.builtin import *  # noqa: F403
+from pyspark.sql.functions import partitioning  # noqa: F401
 
 __all__ = [  # noqa: F405
     # Normal functions
diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py
@@ -61,11 +61,11 @@
 
 # Keep pandas_udf and PandasUDFType import for backwards compatible import; moved in SPARK-28264
 from pyspark.sql.pandas.functions import (  # noqa: F401
-    arrow_udf,  # noqa: F401
-    pandas_udf,  # noqa: F401
-    ArrowUDFType,  # noqa: F401
-    PandasUDFType,  # noqa: F401
-)  # noqa: F401
+    arrow_udf,
+    pandas_udf,
+    ArrowUDFType,
+    PandasUDFType,
+)
 
 from pyspark.sql.utils import (
     to_str as _to_str,
@@ -7130,7 +7130,7 @@ def count_min_sketch(
     +----------------------------------------------------------------------------------------------------------------------------------------+
     |0000000100000000000000640000000200000002000000005ADECCEE00000000153EBE090000000000000033000000000000003100000000000000320000000000000032|
     +----------------------------------------------------------------------------------------------------------------------------------------+
-    """  # noqa: E501
+    """
     _eps = lit(eps)
     _conf = lit(confidence)
     if seed is None:
@@ -20311,7 +20311,7 @@ def explode(col: "ColumnOrName") -> Column:
     |  1|  2|
     |  3|  4|
     +---+---+
-    """  # noqa: E501
+    """
     return _invoke_function_over_columns("explode", col)
 
 
@@ -20391,7 +20391,7 @@ def posexplode(col: "ColumnOrName") -> Column:
     |1  |{1 -> 2, 3 -> 4, 5 -> NULL}|1  |3  |4    |
     |1  |{1 -> 2, 3 -> 4, 5 -> NULL}|2  |5  |NULL |
     +---+---------------------------+---+---+-----+
-    """  # noqa: E501
+    """
     return _invoke_function_over_columns("posexplode", col)
 
 
@@ -20510,7 +20510,7 @@ def inline(col: "ColumnOrName") -> Column:
     |1  |[{1, 2}, NULL, {3, 4}]|NULL|NULL|
     |1  |[{1, 2}, NULL, {3, 4}]|3   |4   |
     +---+----------------------+----+----+
-    """  # noqa: E501
+    """
     return _invoke_function_over_columns("inline", col)
 
 
@@ -20577,7 +20577,7 @@ def explode_outer(col: "ColumnOrName") -> Column:
     |2  |{}                         |NULL|NULL |
     |3  |NULL                       |NULL|NULL |
     +---+---------------------------+----+-----+
-    """  # noqa: E501
+    """
     return _invoke_function_over_columns("explode_outer", col)
 
 
@@ -20644,7 +20644,7 @@ def posexplode_outer(col: "ColumnOrName") -> Column:
     |2  |{}                         |NULL|NULL|NULL |
     |3  |NULL                       |NULL|NULL|NULL |
     +---+---------------------------+----+----+-----+
-    """  # noqa: E501
+    """
     return _invoke_function_over_columns("posexplode_outer", col)
 
 
@@ -20700,7 +20700,7 @@ def inline_outer(col: "ColumnOrName") -> Column:
     |2  |[]                    |NULL|NULL|
     |3  |NULL                  |NULL|NULL|
     +---+----------------------+----+----+
-    """  # noqa: E501
+    """
     return _invoke_function_over_columns("inline_outer", col)
 
 
@@ -28210,7 +28210,7 @@ def aes_encrypt(
     +-------------------------------------------------------------------------------------------------------------+
     |Spark SQL                                                                                                    |
     +-------------------------------------------------------------------------------------------------------------+
-    """  # noqa: E501
+    """
     _mode = lit("GCM") if mode is None else mode
     _padding = lit("DEFAULT") if padding is None else padding
     _iv = lit("") if iv is None else iv
diff --git a/python/pyspark/sql/plot/__init__.py b/python/pyspark/sql/plot/__init__.py
@@ -18,4 +18,4 @@
 """
 This package includes the plotting APIs for PySpark DataFrame.
 """
-from pyspark.sql.plot.core import *  # noqa: F403, F401
+from pyspark.sql.plot.core import *  # noqa: F403
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
@@ -468,7 +468,7 @@ def json(
             return self._df(self._jreader.json(self._spark._sc._jvm.PythonUtils.toSeq(path)))
 
         if not is_remote_only():
-            from pyspark.core.rdd import RDD  # noqa: F401
+            from pyspark.core.rdd import RDD
 
         if not is_remote_only() and isinstance(path, RDD):
 
@@ -838,7 +838,7 @@ def csv(
             return self._df(self._jreader.csv(self._spark._sc._jvm.PythonUtils.toSeq(path)))
 
         if not is_remote_only():
-            from pyspark.core.rdd import RDD  # noqa: F401
+            from pyspark.core.rdd import RDD
 
         if not is_remote_only() and isinstance(path, RDD):
 
@@ -965,7 +965,7 @@ def xml(
             return self._df(self._jreader.xml(self._spark._sc._jvm.PythonUtils.toSeq(path)))
 
         if not is_remote_only():
-            from pyspark.core.rdd import RDD  # noqa: F401
+            from pyspark.core.rdd import RDD
 
         if not is_remote_only() and isinstance(path, RDD):
 
diff --git a/python/pyspark/sql/streaming/__init__.py b/python/pyspark/sql/streaming/__init__.py
@@ -20,8 +20,8 @@
 from pyspark.sql.streaming.readwriter import DataStreamReader, DataStreamWriter  # noqa: F401
 from pyspark.sql.streaming.listener import StreamingQueryListener  # noqa: F401
 from pyspark.sql.streaming.stateful_processor import (  # noqa: F401
-    StatefulProcessor,  # noqa: F401
-    StatefulProcessorHandle,  # noqa: F401
-)  # noqa: F401
+    StatefulProcessor,
+    StatefulProcessorHandle,
+)
 from pyspark.sql.streaming.tws_tester import TwsTester  # noqa: F401
 from pyspark.errors import StreamingQueryException  # noqa: F401
diff --git a/python/pyspark/sql/streaming/listener.py b/python/pyspark/sql/streaming/listener.py
@@ -1123,7 +1123,7 @@ def _test() -> None:
     globs = pyspark.sql.streaming.listener.__dict__.copy()
     try:
         spark = SparkSession._getActiveSessionOrCreate()
-    except Py4JError:  # noqa: F821
+    except Py4JError:
         spark = SparkSession(sc)  # type: ignore[name-defined] # noqa: F821
 
     globs["spark"] = spark
diff --git a/python/pyspark/sql/streaming/query.py b/python/pyspark/sql/streaming/query.py
@@ -790,7 +790,7 @@ def _test() -> None:
     globs = pyspark.sql.streaming.query.__dict__.copy()
     try:
         spark = SparkSession._getActiveSessionOrCreate()
-    except Py4JError:  # noqa: F821
+    except Py4JError:
         spark = SparkSession(sc)  # type: ignore[name-defined] # noqa: F821
 
     globs["spark"] = spark
diff --git a/python/pyspark/sql/tests/arrow/test_arrow.py b/python/pyspark/sql/tests/arrow/test_arrow.py
@@ -71,7 +71,7 @@
     from pandas.testing import assert_frame_equal
 
 if have_pyarrow:
-    import pyarrow as pa  # noqa: F401
+    import pyarrow as pa
 
 
 class ArrowTestsMixin:
diff --git a/python/pyspark/sql/tests/connect/client/test_reattach.py b/python/pyspark/sql/tests/connect/client/test_reattach.py
@@ -72,7 +72,7 @@ def wait_for_responses():
         def check_error(q):
             try:
                 list(q)  # Iterate all.
-            except Exception as e:  # noqa: F841
+            except Exception as e:
                 return e
 
         e = check_error(query1)
diff --git a/python/pyspark/sql/tests/connect/test_connect_collection.py b/python/pyspark/sql/tests/connect/test_connect_collection.py
diff --git a/python/pyspark/sql/tests/pandas/streaming/test_pandas_transform_with_state_state_variable.py b/python/pyspark/sql/tests/pandas/streaming/test_pandas_transform_with_state_state_variable.py
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf.py b/python/pyspark/sql/tests/pandas/test_pandas_udf.py
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py b/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py
diff --git a/python/pyspark/util.py b/python/pyspark/util.py
diff --git a/python/run-tests.py b/python/run-tests.py

Original file line number	Diff line number	Diff line change
`@@ -38,7 +38,8 @@ exclude = [`
`38`	`38`
`39`	`39`	`[tool.ruff.lint]`
`40`	`40`	`extend-select = [`
`41`		`- "G010" # logging-warn`
	`41`	`+ "G010", # logging-warn`
	`42`	`+ "RUF100", # unused-noqa`
`42`	`43`	`]`
`43`	`44`	`ignore = [`
`44`	`45`	`"E402", # Module top level import is disabled for optional import check, etc.`
Original file line number	Diff line number	Diff line change
`@@ -14,4 +14,4 @@`
`14`	`14`	`# See the License for the specific language governing permissions and`
`15`	`15`	`# limitations under the License.`
`16`	`16`	`#`
`17`		`-from pyspark.pandas.plot.core import * # noqa: F401,F403`
	`17`	`+from pyspark.pandas.plot.core import * # noqa: F403`