Working through more ruff checks and disabling a bunch for now

Author: timsaucer

docs/source/conf.py

@@ -73,7 +73,7 @@
 autoapi_python_class_content = "both"
 
 
-def autoapi_skip_member_fn(app, what, name, obj, skip, options):
+def autoapi_skip_member_fn(app, what, name, obj, skip, options):  # noqa: ARG001
     skip_contents = [
         # Re-exports
         ("class", "datafusion.DataFrame"),

examples/python-udwf.py

@@ -59,7 +59,7 @@ def __init__(self, alpha: float) -> None:
     def supports_bounded_execution(self) -> bool:
         return True
 
-    def get_range(self, idx: int, num_rows: int) -> tuple[int, int]:
+    def get_range(self, idx: int, num_rows: int) -> tuple[int, int]:  # noqa: ARG002
         # Override the default range of current row since uses_window_frame is False
         # So for the purpose of this test we just smooth from the previous row to
         # current.

pyproject.toml

@@ -79,8 +79,42 @@ ignore = [
     "SLF001",  # Allow accessing private members
     "TD002",
     "TD003",   # Allow TODO lines
-    "UP007"    # Disallowing Union is pedantic
+    "UP007",   # Disallowing Union is pedantic
     # TODO: Enable all of the following, but this PR is getting too large already
+    "PT001",
+    "ANN101",
+    "ANN204",
+    "B008",
+    "EM101",
+    "PLR0913",
+    "PLR1714",
+    "ANN201",
+    "C400",
+    "TRY003",
+    "B904",
+    "UP006",
+    "RUF012",
+    "FBT003",
+    "C416",
+    "SIM102",
+    "PGH003",
+    "PLR2004",
+    "PERF401",
+    "PD901",
+    "EM102",
+    "ERA001",
+    "SIM108",
+    "ICN001",
+    "ANN001",
+    "ANN202",
+    "PTH",
+    "N812",
+    "INP001",
+    "DTZ007",
+    "PLW2901",
+    "RET503",
+    "RUF015",
+    "TCH001",
 ]
 
 [tool.ruff.lint.pydocstyle]
@@ -103,11 +137,12 @@ max-doc-length = 88
     "PT011",
     "RUF015",
     "S608",
-    "PLR0913"
+    "PLR0913",
+    "PT004",
 ]
-"examples/*" = ["D", "W505"]
-"dev/*" = ["D"]
-"benchmarks/*" = ["D", "F"]
+"examples/*" = ["D", "W505", "E501", "T201", "S101"]
+"dev/*" = ["D", "E", "T", "S", "PLR", "C", "SIM", "UP", "EXE", "N817"]
+"benchmarks/*" = ["D", "F", "T", "BLE", "FURB", "PLR", "E", "TD", "TRY", "S", "SIM", "EXE", "UP"]
 "docs/*" = ["D"]
 
 [dependency-groups]

python/datafusion/dataframe.py

@@ -29,6 +29,7 @@
     List,
     Literal,
     Optional,
+    Type,
     Union,
     overload,
 )
@@ -49,10 +50,11 @@
     import polars as pl
     import pyarrow as pa
 
+    from datafusion._internal import DataFrame as DataFrameInternal
+    from datafusion._internal import expr as expr_internal
+
 from enum import Enum
 
-from datafusion._internal import DataFrame as DataFrameInternal
-from datafusion._internal import expr as expr_internal
 from datafusion.expr import Expr, SortExpr, sort_or_default
 
 
@@ -73,7 +75,7 @@ class Compression(Enum):
     LZ4_RAW = "lz4_raw"
 
     @classmethod
-    def from_str(cls, value: str) -> Compression:
+    def from_str(cls: Type[Compression], value: str) -> Compression:
         """Convert a string to a Compression enum value.
 
         Args:
@@ -88,8 +90,9 @@ def from_str(cls, value: str) -> Compression:
         try:
             return cls(value.lower())
         except ValueError:
+            valid_values = str([item.value for item in Compression])
             raise ValueError(
-                f"{value} is not a valid Compression. Valid values are: {[item.value for item in Compression]}"
+                f"{value} is not a valid Compression. Valid values are: {valid_values}"
             )
 
     def get_default_level(self) -> Optional[int]:

python/datafusion/input/location.py

@@ -84,6 +84,6 @@ def build_table(
             raise RuntimeError(msg)
 
         # Input could possibly be multiple files. Create a list if so
-        input_files = glob.glob(input_item)  # noqa: PTH207
+        input_files = glob.glob(input_item)
 
         return SqlTable(table_name, columns, num_rows, input_files)

python/datafusion/io.py

@@ -33,7 +33,7 @@
     from datafusion.expr import Expr
 
 
-def read_parquet(  # noqa: PLR0913
+def read_parquet(
     path: str | pathlib.Path,
     table_partition_cols: list[tuple[str, str]] | None = None,
     parquet_pruning: bool = True,
@@ -81,7 +81,7 @@ def read_parquet(  # noqa: PLR0913
     )
 
 
-def read_json(  # noqa: PLR0913
+def read_json(
     path: str | pathlib.Path,
     schema: pa.Schema | None = None,
     schema_infer_max_records: int = 1000,
@@ -122,7 +122,7 @@ def read_json(  # noqa: PLR0913
     )
 
 
-def read_csv(  # noqa: PLR0913
+def read_csv(
     path: str | pathlib.Path | list[str] | list[pathlib.Path],
     schema: pa.Schema | None = None,
     has_header: bool = True,

python/datafusion/udf.py

@@ -202,13 +202,13 @@ def _decorator(
             volatility: Volatility | str,
             name: Optional[str] = None,
         ) -> Callable:
-            def decorator(func: Callable):  # noqa: ANN202
+            def decorator(func: Callable):
                 udf_caller = ScalarUDF.udf(
                     func, input_types, return_type, volatility, name
                 )
 
                 @functools.wraps(func)
-                def wrapper(*args: Any, **kwargs: Any):  # noqa: ANN202
+                def wrapper(*args: Any, **kwargs: Any):
                     return udf_caller(*args, **kwargs)
 
                 return wrapper
@@ -249,7 +249,7 @@ class AggregateUDF:
     also :py:class:`ScalarUDF` for operating on a row by row basis.
     """
 
-    def __init__(  # noqa: PLR0913
+    def __init__(
         self,
         name: str,
         accumulator: Callable[[], Accumulator],
@@ -375,7 +375,7 @@ def udf4() -> Summarize:
             aggregation or window function calls.
         """
 
-        def _function(  # noqa: PLR0913
+        def _function(
             accum: Callable[[], Accumulator],
             input_types: pa.DataType | list[pa.DataType],
             return_type: pa.DataType,

python/tests/test_aggregation.py

@@ -88,7 +88,7 @@ def df_aggregate_100():
             f.covar_samp(column("b"), column("c")),
             lambda a, b, c, d: np.array(np.cov(b, c, ddof=1)[0][1]),
         ),
-        # f.grouping(col_a),  # No physical plan implemented yet  # noqa: ERA001
+        # f.grouping(col_a),  # No physical plan implemented yet
         (f.max(column("a")), lambda a, b, c, d: np.array(np.max(a))),
         (f.mean(column("b")), lambda a, b, c, d: np.array(np.mean(b))),
         (f.median(column("b")), lambda a, b, c, d: np.array(np.median(b))),

python/tests/test_functions.py

@@ -81,7 +81,7 @@ def test_literal(df):
         literal("1"),
         literal("OK"),
         literal(3.14),
-        literal(True),  # noqa: FBT003
+        literal(True),
         literal(b"hello world"),
     )
     result = df.collect()

python/tests/test_imports.py

@@ -169,9 +169,10 @@ def test_class_module_is_datafusion():
 
 
 def test_import_from_functions_submodule():
-    from datafusion.functions import abs, sin  # noqa: A004
+    from datafusion.functions import abs as df_abs
+    from datafusion.functions import sin
 
-    assert functions.abs is abs
+    assert functions.abs is df_abs
     assert functions.sin is sin
 
     msg = "cannot import name 'foobar' from 'datafusion.functions'"