Clean usecol and date processing

Author: mroeschke

pandas/io/parsers/base_parser.py

@@ -324,6 +324,7 @@ def _clean_mapping(self, mapping):
     def _agg_index(self, index) -> Index:
         arrays = []
         converters = self._clean_mapping(self.converters)
+        clean_dtypes = self._clean_mapping(self.dtype)
 
         for i, arr in enumerate(index):
             if self._should_parse_dates(i):
@@ -352,8 +353,6 @@ def _agg_index(self, index) -> Index:
                 else:
                     col_na_values, col_na_fvalues = set(), set()
 
-            clean_dtypes = self._clean_mapping(self.dtype)
-
             cast_type = None
             index_converter = False
             if self.index_names is not None:
@@ -620,35 +619,6 @@ def _check_data_length(
                 stacklevel=find_stack_level(),
             )
 
-    @overload
-    def _evaluate_usecols(
-        self,
-        usecols: Callable[[Hashable], object],
-        names: Iterable[Hashable],
-    ) -> set[int]: ...
-
-    @overload
-    def _evaluate_usecols(
-        self, usecols: SequenceT, names: Iterable[Hashable]
-    ) -> SequenceT: ...
-
-    @final
-    def _evaluate_usecols(
-        self,
-        usecols: Callable[[Hashable], object] | SequenceT,
-        names: Iterable[Hashable],
-    ) -> SequenceT | set[int]:
-        """
-        Check whether or not the 'usecols' parameter
-        is a callable.  If so, enumerates the 'names'
-        parameter and returns a set of indices for
-        each entry in 'names' that evaluates to True.
-        If not a callable, returns 'usecols'.
-        """
-        if callable(usecols):
-            return {i for i, name in enumerate(names) if usecols(name)}
-        return usecols
-
     @final
     def _validate_usecols_names(self, usecols: SequenceT, names: Sequence) -> SequenceT:
         """
@@ -976,3 +946,32 @@ def _validate_usecols_arg(usecols):
 
         return usecols, usecols_dtype
     return usecols, None
+
+
+@overload
+def evaluate_callable_usecols(
+    usecols: Callable[[Hashable], object],
+    names: Iterable[Hashable],
+) -> set[int]: ...
+
+
+@overload
+def evaluate_callable_usecols(
+    usecols: SequenceT, names: Iterable[Hashable]
+) -> SequenceT: ...
+
+
+def evaluate_callable_usecols(
+    usecols: Callable[[Hashable], object] | SequenceT,
+    names: Iterable[Hashable],
+) -> SequenceT | set[int]:
+    """
+    Check whether or not the 'usecols' parameter
+    is a callable.  If so, enumerates the 'names'
+    parameter and returns a set of indices for
+    each entry in 'names' that evaluates to True.
+    If not a callable, returns 'usecols'.
+    """
+    if callable(usecols):
+        return {i for i, name in enumerate(names) if usecols(name)}
+    return usecols

pandas/io/parsers/c_parser_wrapper.py

@@ -31,6 +31,7 @@
     ParserBase,
     ParserError,
     date_converter,
+    evaluate_callable_usecols,
     is_index_col,
     validate_parse_dates_presence,
 )
@@ -133,7 +134,7 @@ def __init__(self, src: ReadCsvBuffer[str], **kwds) -> None:
         self.orig_names = self.names[:]  # type: ignore[has-type]
 
         if self.usecols:
-            usecols = self._evaluate_usecols(self.usecols, self.orig_names)
+            usecols = evaluate_callable_usecols(self.usecols, self.orig_names)
 
             # GH 14671
             # assert for mypy, orig_names is List or None, None would error in issubset
@@ -256,8 +257,7 @@ def read(
                     columns, self.col_names
                 )
 
-                if self.usecols is not None:
-                    columns = self._filter_usecols(columns)
+                columns = _filter_usecols(self.usecols, columns)
 
                 col_dict = {k: v for k, v in col_dict.items() if k in columns}
 
@@ -290,13 +290,21 @@ def read(
                 else:
                     values = data.pop(self.index_col[i])
 
-                values = self._maybe_parse_dates(values, i)
+                if self._should_parse_dates(i):
+                    values = date_converter(
+                        values,
+                        col=self.index_names[index]
+                        if self.index_names is not None
+                        else None,
+                        dayfirst=self.dayfirst,
+                        cache_dates=self.cache_dates,
+                        date_format=self.date_format,
+                    )
                 arrays.append(values)
 
             index = ensure_index_from_sequences(arrays)
 
-            if self.usecols is not None:
-                names = self._filter_usecols(names)
+            names = _filter_usecols(self.usecols, names)
 
             names = dedup_names(names, is_potential_multi_index(names, self.index_col))
 
@@ -320,8 +328,7 @@ def read(
             names = list(self.orig_names)
             names = dedup_names(names, is_potential_multi_index(names, self.index_col))
 
-            if self.usecols is not None:
-                names = self._filter_usecols(names)
+            names = _filter_usecols(self.usecols, names)
 
             # columns as list
             alldata = [x[1] for x in data_tups]
@@ -335,25 +342,13 @@ def read(
 
         return index, column_names, date_data
 
-    def _filter_usecols(self, names: SequenceT) -> SequenceT | list[Hashable]:
-        # hackish
-        usecols = self._evaluate_usecols(self.usecols, names)
-        if usecols is not None and len(names) != len(usecols):
-            return [
-                name for i, name in enumerate(names) if i in usecols or name in usecols
-            ]
-        return names
-
-    def _maybe_parse_dates(self, values, index: int):
-        if self._should_parse_dates(index):
-            values = date_converter(
-                values,
-                col=self.index_names[index] if self.index_names is not None else None,
-                dayfirst=self.dayfirst,
-                cache_dates=self.cache_dates,
-                date_format=self.date_format,
-            )
-        return values
+
+def _filter_usecols(usecols, names: SequenceT) -> SequenceT | list[Hashable]:
+    # hackish
+    usecols = evaluate_callable_usecols(usecols, names)
+    if usecols is not None and len(names) != len(usecols):
+        return [name for i, name in enumerate(names) if i in usecols or name in usecols]
+    return names
 
 
 def _concatenate_chunks(

pandas/io/parsers/python_parser.py

@@ -59,6 +59,7 @@
 )
 from pandas.io.parsers.base_parser import (
     ParserBase,
+    evaluate_callable_usecols,
     get_na_values,
     parser_defaults,
     validate_parse_dates_presence,
@@ -774,7 +775,7 @@ def _handle_usecols(
         col_indices: set[int] | list[int]
         if self.usecols is not None:
             if callable(self.usecols):
-                col_indices = self._evaluate_usecols(self.usecols, usecols_key)
+                col_indices = evaluate_callable_usecols(self.usecols, usecols_key)
             elif any(isinstance(u, str) for u in self.usecols):
                 if len(columns) > 1:
                     raise ValueError(