Ruff UP006: dont use typing module for dict and list typehints

Author: evertlammerts

duckdb/experimental/spark/conf.py

@@ -12,20 +12,20 @@ def contains(self, key: str) -> bool:
     def get(self, key: str, defaultValue: Optional[str] = None) -> Optional[str]:
         raise ContributionsAcceptedError
 
-    def getAll(self) -> List[Tuple[str, str]]:
+    def getAll(self) -> list[tuple[str, str]]:
         raise ContributionsAcceptedError
 
     def set(self, key: str, value: str) -> "SparkConf":
         raise ContributionsAcceptedError
 
-    def setAll(self, pairs: List[Tuple[str, str]]) -> "SparkConf":
+    def setAll(self, pairs: list[tuple[str, str]]) -> "SparkConf":
         raise ContributionsAcceptedError
 
     def setAppName(self, value: str) -> "SparkConf":
         raise ContributionsAcceptedError
 
     def setExecutorEnv(
-        self, key: Optional[str] = None, value: Optional[str] = None, pairs: Optional[List[Tuple[str, str]]] = None
+        self, key: Optional[str] = None, value: Optional[str] = None, pairs: Optional[list[tuple[str, str]]] = None
     ) -> "SparkConf":
         raise ContributionsAcceptedError
 

duckdb/experimental/spark/errors/exceptions/base.py

@@ -13,7 +13,7 @@ def __init__(
         # The error class, decides the message format, must be one of the valid options listed in 'error_classes.py'
         error_class: Optional[str] = None,
         # The dictionary listing the arguments specified in the message (or the error_class)
-        message_parameters: Optional[Dict[str, str]] = None,
+        message_parameters: Optional[dict[str, str]] = None,
     ):
         # `message` vs `error_class` & `message_parameters` are mutually exclusive.
         assert (message is not None and (error_class is None and message_parameters is None)) or (
@@ -24,7 +24,7 @@ def __init__(
 
         if message is None:
             self.message = self.error_reader.get_error_message(
-                cast(str, error_class), cast(Dict[str, str], message_parameters)
+                cast(str, error_class), cast(dict[str, str], message_parameters)
             )
         else:
             self.message = message
@@ -45,7 +45,7 @@ def getErrorClass(self) -> Optional[str]:
         """
         return self.error_class
 
-    def getMessageParameters(self) -> Optional[Dict[str, str]]:
+    def getMessageParameters(self) -> Optional[dict[str, str]]:
         """
         Returns a message parameters as a dictionary.
 

duckdb/experimental/spark/errors/utils.py

@@ -29,7 +29,7 @@ class ErrorClassesReader:
     def __init__(self) -> None:
         self.error_info_map = ERROR_CLASSES_MAP
 
-    def get_error_message(self, error_class: str, message_parameters: Dict[str, str]) -> str:
+    def get_error_message(self, error_class: str, message_parameters: dict[str, str]) -> str:
         """
         Returns the completed error message by applying message parameters to the message template.
         """

duckdb/experimental/spark/sql/_typing.py

@@ -57,7 +57,7 @@
     float,
 )
 
-RowLike = TypeVar("RowLike", List[Any], Tuple[Any, ...], types.Row)
+RowLike = TypeVar("RowLike", list[Any], tuple[Any, ...], types.Row)
 
 SQLBatchedUDFType = Literal[100]
 

duckdb/experimental/spark/sql/catalog.py

@@ -36,7 +36,7 @@ class Catalog:
     def __init__(self, session: SparkSession):
         self._session = session
 
-    def listDatabases(self) -> List[Database]:
+    def listDatabases(self) -> list[Database]:
         res = self._session.conn.sql('select database_name from duckdb_databases()').fetchall()
 
         def transform_to_database(x) -> Database:
@@ -45,7 +45,7 @@ def transform_to_database(x) -> Database:
         databases = [transform_to_database(x) for x in res]
         return databases
 
-    def listTables(self) -> List[Table]:
+    def listTables(self) -> list[Table]:
         res = self._session.conn.sql('select table_name, database_name, sql, temporary from duckdb_tables()').fetchall()
 
         def transform_to_table(x) -> Table:
@@ -54,7 +54,7 @@ def transform_to_table(x) -> Table:
         tables = [transform_to_table(x) for x in res]
         return tables
 
-    def listColumns(self, tableName: str, dbName: Optional[str] = None) -> List[Column]:
+    def listColumns(self, tableName: str, dbName: Optional[str] = None) -> list[Column]:
         query = f"""
 			select column_name, data_type, is_nullable from duckdb_columns() where table_name = '{tableName}'
 		"""
@@ -68,7 +68,7 @@ def transform_to_column(x) -> Column:
         columns = [transform_to_column(x) for x in res]
         return columns
 
-    def listFunctions(self, dbName: Optional[str] = None) -> List[Function]:
+    def listFunctions(self, dbName: Optional[str] = None) -> list[Function]:
         raise NotImplementedError
 
     def setCurrentDatabase(self, dbName: str) -> None:

duckdb/experimental/spark/sql/column.py

@@ -234,10 +234,10 @@ def cast(self, dataType: Union[DataType, str]) -> "Column":
     def isin(self, *cols: Any) -> "Column":
         if len(cols) == 1 and isinstance(cols[0], (list, set)):
             # Only one argument supplied, it's a list
-            cols = cast(Tuple, cols[0])
+            cols = cast(tuple, cols[0])
 
         cols = cast(
-            Tuple,
+            tuple,
             [_get_expr(c) for c in cols],
         )
         return Column(self.expr.isin(*cols))

duckdb/experimental/spark/sql/dataframe.py

@@ -143,7 +143,7 @@ def withColumn(self, columnName: str, col: Column) -> "DataFrame":
         rel = self.relation.select(*cols)
         return DataFrame(rel, self.session)
 
-    def withColumns(self, *colsMap: Dict[str, Column]) -> "DataFrame":
+    def withColumns(self, *colsMap: dict[str, Column]) -> "DataFrame":
         """
         Returns a new :class:`DataFrame` by adding multiple columns or replacing the
         existing columns that have the same names.
@@ -218,7 +218,7 @@ def withColumns(self, *colsMap: Dict[str, Column]) -> "DataFrame":
         rel = self.relation.select(*cols)
         return DataFrame(rel, self.session)
 
-    def withColumnsRenamed(self, colsMap: Dict[str, str]) -> "DataFrame":
+    def withColumnsRenamed(self, colsMap: dict[str, str]) -> "DataFrame":
         """
         Returns a new :class:`DataFrame` by renaming multiple columns.
         This is a no-op if the schema doesn't contain the given column names.
@@ -356,7 +356,7 @@ def transform(
         return result
 
     def sort(
-        self, *cols: Union[str, Column, List[Union[str, Column]]], **kwargs: Any
+        self, *cols: Union[str, Column, list[Union[str, Column]]], **kwargs: Any
     ) -> "DataFrame":
         """Returns a new :class:`DataFrame` sorted by the specified column(s).
 
@@ -487,15 +487,15 @@ def sort(
 
     orderBy = sort
 
-    def head(self, n: Optional[int] = None) -> Union[Optional[Row], List[Row]]:
+    def head(self, n: Optional[int] = None) -> Union[Optional[Row], list[Row]]:
         if n is None:
             rs = self.head(1)
             return rs[0] if rs else None
         return self.take(n)
 
     first = head
 
-    def take(self, num: int) -> List[Row]:
+    def take(self, num: int) -> list[Row]:
         return self.limit(num).collect()
 
     def filter(self, condition: "ColumnOrName") -> "DataFrame":
@@ -579,7 +579,7 @@ def select(self, *cols) -> "DataFrame":
         return DataFrame(rel, self.session)
 
     @property
-    def columns(self) -> List[str]:
+    def columns(self) -> list[str]:
         """Returns all column names as a list.
 
         Examples
@@ -589,20 +589,20 @@ def columns(self) -> List[str]:
         """
         return [f.name for f in self.schema.fields]
 
-    def _ipython_key_completions_(self) -> List[str]:
+    def _ipython_key_completions_(self) -> list[str]:
         # Provides tab-completion for column names in PySpark DataFrame
         # when accessed in bracket notation, e.g. df['<TAB>]
         return self.columns
 
-    def __dir__(self) -> List[str]:
+    def __dir__(self) -> list[str]:
         out = set(super().__dir__())
         out.update(c for c in self.columns if c.isidentifier() and not iskeyword(c))
         return sorted(out)
 
     def join(
         self,
         other: "DataFrame",
-        on: Optional[Union[str, List[str], Column, List[Column]]] = None,
+        on: Optional[Union[str, list[str], Column, list[Column]]] = None,
         how: Optional[str] = None,
     ) -> "DataFrame":
         """Joins with another :class:`DataFrame`, using the given join expression.
@@ -704,7 +704,7 @@ def join(
             assert isinstance(
                 on[0], Expression
             ), "on should be Column or list of Column"
-            on = reduce(lambda x, y: x.__and__(y), cast(List[Expression], on))
+            on = reduce(lambda x, y: x.__and__(y), cast(list[Expression], on))
 
 
         if on is None and how is None:
@@ -893,11 +893,11 @@ def __getitem__(self, item: Union[int, str]) -> Column:
         ...
 
     @overload
-    def __getitem__(self, item: Union[Column, List, Tuple]) -> "DataFrame":
+    def __getitem__(self, item: Union[Column, list, tuple]) -> "DataFrame":
         ...
 
     def __getitem__(
-        self, item: Union[int, str, Column, List, Tuple]
+        self, item: Union[int, str, Column, list, tuple]
     ) -> Union[Column, "DataFrame"]:
         """Returns the column as a :class:`Column`.
 
@@ -942,7 +942,7 @@ def groupBy(self, *cols: "ColumnOrName") -> "GroupedData":
         ...
 
     @overload
-    def groupBy(self, __cols: Union[List[Column], List[str]]) -> "GroupedData":
+    def groupBy(self, __cols: Union[list[Column], list[str]]) -> "GroupedData":
         ...
 
     def groupBy(self, *cols: "ColumnOrName") -> "GroupedData":  # type: ignore[misc]
@@ -1259,7 +1259,7 @@ def exceptAll(self, other: "DataFrame") -> "DataFrame":
         """
         return DataFrame(self.relation.except_(other.relation), self.session)
 
-    def dropDuplicates(self, subset: Optional[List[str]] = None) -> "DataFrame":
+    def dropDuplicates(self, subset: Optional[list[str]] = None) -> "DataFrame":
         """Return a new :class:`DataFrame` with duplicate rows removed,
         optionally only considering certain columns.
 
@@ -1391,7 +1391,7 @@ def toDF(self, *cols) -> "DataFrame":
         new_rel = self.relation.project(*projections)
         return DataFrame(new_rel, self.session)
 
-    def collect(self) -> List[Row]:
+    def collect(self) -> list[Row]:
         columns = self.relation.columns
         result = self.relation.fetchall()
 

duckdb/experimental/spark/sql/functions.py

@@ -111,7 +111,7 @@ def struct(*cols: Column) -> Column:
 
 
 def array(
-    *cols: Union["ColumnOrName", Union[List["ColumnOrName"], Tuple["ColumnOrName", ...]]]
+    *cols: Union["ColumnOrName", Union[list["ColumnOrName"], tuple["ColumnOrName", ...]]]
 ) -> Column:
     """Creates a new array column.
 

duckdb/experimental/spark/sql/group.py

@@ -177,7 +177,7 @@ def avg(self, *cols: str) -> DataFrame:
         if len(columns) == 0:
             schema = self._df.schema
             # Take only the numeric types of the relation
-            columns: List[str] = [x.name for x in schema.fields if isinstance(x.dataType, NumericType)]
+            columns: list[str] = [x.name for x in schema.fields if isinstance(x.dataType, NumericType)]
         return _api_internal(self, "avg", *columns)
 
     @df_varargs_api
@@ -312,10 +312,10 @@ def agg(self, *exprs: Column) -> DataFrame:
         ...
 
     @overload
-    def agg(self, __exprs: Dict[str, str]) -> DataFrame:
+    def agg(self, __exprs: dict[str, str]) -> DataFrame:
         ...
 
-    def agg(self, *exprs: Union[Column, Dict[str, str]]) -> DataFrame:
+    def agg(self, *exprs: Union[Column, dict[str, str]]) -> DataFrame:
         """Compute aggregates and returns the result as a :class:`DataFrame`.
 
         The available aggregate functions can be:

duckdb/experimental/spark/sql/readwriter.py

@@ -26,7 +26,7 @@ def parquet(
         self,
         path: str,
         mode: Optional[str] = None,
-        partitionBy: Union[str, List[str], None] = None,
+        partitionBy: Union[str, list[str], None] = None,
         compression: Optional[str] = None,
     ) -> None:
         relation = self.dataframe.relation
@@ -94,7 +94,7 @@ def __init__(self, session: "SparkSession"):
 
     def load(
         self,
-        path: Optional[Union[str, List[str]]] = None,
+        path: Optional[Union[str, list[str]]] = None,
         format: Optional[str] = None,
         schema: Optional[Union[StructType, str]] = None,
         **options: OptionalPrimitiveType,
@@ -131,7 +131,7 @@ def load(
 
     def csv(
         self,
-        path: Union[str, List[str]],
+        path: Union[str, list[str]],
         schema: Optional[Union[StructType, str]] = None,
         sep: Optional[str] = None,
         encoding: Optional[str] = None,
@@ -263,7 +263,7 @@ def parquet(self, *paths: str, **options: "OptionalPrimitiveType") -> "DataFrame
 
     def json(
         self,
-        path: Union[str, List[str]],
+        path: Union[str, list[str]],
         schema: Optional[Union[StructType, str]] = None,
         primitivesAsString: Optional[Union[bool, str]] = None,
         prefersDecimal: Optional[Union[bool, str]] = None,