@@ -69,16 +69,6 @@ def _get_partition_bounds(schema_metadata):
69
69
return shapely .geometry .box (* bbox )
70
70
71
71
72
- def _extract_nullable_dtypes (** kwargs ):
73
- if DASK_2023_04_0 :
74
- use_nullable_dtypes = kwargs .get ("dtype_backend" , None ) == "numpy_nullable"
75
- elif DASK_2022_12_0_PLUS :
76
- use_nullable_dtypes = kwargs .get ("use_nullable_dtypes" , False )
77
- else :
78
- use_nullable_dtypes = False
79
- return use_nullable_dtypes
80
-
81
-
82
72
class ArrowDatasetEngine :
83
73
"""
84
74
Custom IO engine based on pyarrow.dataset.
@@ -141,25 +131,6 @@ def _arrow_table_to_pandas(
141
131
) -> pd .DataFrame :
142
132
_kwargs = kwargs .get ("arrow_to_pandas" , {})
143
133
_kwargs .update ({"use_threads" : False , "ignore_metadata" : False })
144
- use_nullable_dtypes = _extract_nullable_dtypes (** kwargs )
145
-
146
- if use_nullable_dtypes :
147
- from dask .dataframe .io .parquet .arrow import PYARROW_NULLABLE_DTYPE_MAPPING
148
-
149
- if "types_mapper" in _kwargs :
150
- # User-provided entries take priority over
151
- # PYARROW_NULLABLE_DTYPE_MAPPING
152
- types_mapper = _kwargs ["types_mapper" ]
153
-
154
- def _types_mapper (pa_type ):
155
- return types_mapper (pa_type ) or PYARROW_NULLABLE_DTYPE_MAPPING .get (
156
- pa_type
157
- )
158
-
159
- _kwargs ["types_mapper" ] = _types_mapper
160
-
161
- else :
162
- _kwargs ["types_mapper" ] = PYARROW_NULLABLE_DTYPE_MAPPING .get
163
134
164
135
return arrow_table .to_pandas (categories = categories , ** _kwargs )
165
136
@@ -199,25 +170,6 @@ def _arrow_table_to_pandas(
199
170
200
171
_kwargs = kwargs .get ("arrow_to_pandas" , {})
201
172
_kwargs .update ({"use_threads" : False , "ignore_metadata" : False })
202
- use_nullable_dtypes = _extract_nullable_dtypes (** kwargs )
203
-
204
- if use_nullable_dtypes :
205
- from dask .dataframe .io .parquet .arrow import PYARROW_NULLABLE_DTYPE_MAPPING
206
-
207
- if "types_mapper" in _kwargs :
208
- # User-provided entries take priority over
209
- # PYARROW_NULLABLE_DTYPE_MAPPING
210
- types_mapper = _kwargs ["types_mapper" ]
211
-
212
- def _types_mapper (pa_type ):
213
- return types_mapper (pa_type ) or PYARROW_NULLABLE_DTYPE_MAPPING .get (
214
- pa_type
215
- )
216
-
217
- _kwargs ["types_mapper" ] = _types_mapper
218
-
219
- else :
220
- _kwargs ["types_mapper" ] = PYARROW_NULLABLE_DTYPE_MAPPING .get
221
173
222
174
# TODO support additional keywords
223
175
try :
0 commit comments