POC: PDEP16 default to masked nullable dtypes

Author: jbrockmendel

pandas/_testing/__init__.py

@@ -89,6 +89,9 @@
         NpDtype,
     )
 
+# Alias so we can update old `assert obj.dtype == np_dtype` checks to PDEP16
+#  behavior.
+to_dtype = pd.core.dtypes.common.pandas_dtype
 
 UNSIGNED_INT_NUMPY_DTYPES: list[NpDtype] = ["uint8", "uint16", "uint32", "uint64"]
 UNSIGNED_INT_EA_DTYPES: list[Dtype] = ["UInt8", "UInt16", "UInt32", "UInt64"]

pandas/_testing/asserters.py

@@ -323,13 +323,19 @@ def _check_types(left, right, obj: str = "Index") -> None:
     elif check_exact and check_categorical:
         if not left.equals(right):
             mismatch = left._values != right._values
+            if isinstance(left, RangeIndex) and not mismatch.any():
+                # TODO: probably need to fix RangeIndex.equals?
+                pass
+            elif isinstance(right, RangeIndex) and not mismatch.any():
+                # TODO: probably need to fix some other equals method?
+                pass
+            else:
+                if not isinstance(mismatch, np.ndarray):
+                    mismatch = cast("ExtensionArray", mismatch).fillna(True)
 
-            if not isinstance(mismatch, np.ndarray):
-                mismatch = cast("ExtensionArray", mismatch).fillna(True)
-
-            diff = np.sum(mismatch.astype(int)) * 100.0 / len(left)
-            msg = f"{obj} values are different ({np.round(diff, 5)} %)"
-            raise_assert_detail(obj, msg, left, right)
+                diff = np.sum(mismatch.astype(int)) * 100.0 / len(left)
+                msg = f"{obj} values are different ({np.round(diff, 5)} %)"
+                raise_assert_detail(obj, msg, left, right)
     else:
         # if we have "equiv", this becomes True
         exact_bool = bool(exact)

pandas/core/arrays/_mixins.py

@@ -127,7 +127,7 @@ def view(self, dtype: Dtype | None = None) -> ArrayLike:
             #  pass those through to the underlying ndarray
             return self._ndarray.view(dtype)
 
-        dtype = pandas_dtype(dtype)
+        dtype = pandas_dtype(dtype, allow_numpy_dtypes=True)
         arr = self._ndarray
 
         if isinstance(dtype, PeriodDtype):

pandas/core/arrays/string_.py

@@ -837,7 +837,7 @@ def astype(self, dtype, copy: bool = True):
             arr_ea = self.copy()
             mask = self.isna()
             arr_ea[mask] = "0"
-            values = arr_ea.astype(dtype.numpy_dtype)
+            values = arr_ea.to_numpy(dtype=dtype.numpy_dtype)
             return FloatingArray(values, mask, copy=False)
         elif isinstance(dtype, ExtensionDtype):
             # Skip the NumpyExtensionArray.astype method

pandas/core/config_init.py

@@ -427,6 +427,15 @@ def is_terminal() -> bool:
         validator=is_one_of_factory([True, False, "warn"]),
     )
 
+with cf.config_prefix("mode"):
+    cf.register_option(
+        "pdep16_data_types",
+        True,
+        "Whether to default to numpy-nullable dtypes for integer, float, "
+        "and bool dtypes",
+        validator=is_one_of_factory([True, False]),
+    )
+
 
 # user warnings
 chained_assignment = """

pandas/core/construction.py

@@ -16,7 +16,10 @@
 import numpy as np
 from numpy import ma
 
-from pandas._config import using_string_dtype
+from pandas._config import (
+    get_option,
+    using_string_dtype,
+)
 
 from pandas._libs import lib
 from pandas._libs.tslibs import (
@@ -612,7 +615,9 @@ def sanitize_array(
         if dtype is None:
             subarr = data
             if data.dtype == object and infer_object:
-                subarr = maybe_infer_to_datetimelike(data)
+                subarr = maybe_infer_to_datetimelike(
+                    data, convert_to_nullable_dtype=get_option("mode.pdep16_data_types")
+                )
             elif data.dtype.kind == "U" and using_string_dtype():
                 from pandas.core.arrays.string_ import StringDtype
 
@@ -659,7 +664,10 @@ def sanitize_array(
             subarr = maybe_convert_platform(data)
             if subarr.dtype == object:
                 subarr = cast(np.ndarray, subarr)
-                subarr = maybe_infer_to_datetimelike(subarr)
+                subarr = maybe_infer_to_datetimelike(
+                    subarr,
+                    convert_to_nullable_dtype=get_option("mode.pdep16_data_types"),
+                )
 
     subarr = _sanitize_ndim(subarr, data, dtype, index, allow_2d=allow_2d)
 

pandas/core/dtypes/cast.py

@@ -18,7 +18,10 @@
 
 import numpy as np
 
-from pandas._config import using_string_dtype
+from pandas._config import (
+    get_option,
+    using_string_dtype,
+)
 
 from pandas._libs import (
     Interval,
@@ -135,7 +138,9 @@ def maybe_convert_platform(
 
     if arr.dtype == _dtype_obj:
         arr = cast(np.ndarray, arr)
-        arr = lib.maybe_convert_objects(arr)
+        arr = lib.maybe_convert_objects(
+            arr, convert_to_nullable_dtype=get_option("mode.pdep16_data_types")
+        )
 
     return arr
 

pandas/core/dtypes/common.py

@@ -12,7 +12,10 @@
 
 import numpy as np
 
-from pandas._config import using_string_dtype
+from pandas._config import (
+    get_option,
+    using_string_dtype,
+)
 
 from pandas._libs import (
     Interval,
@@ -1793,14 +1796,36 @@ def validate_all_hashable(*args, error_name: str | None = None) -> None:
         raise TypeError("All elements must be hashable")
 
 
-def pandas_dtype(dtype) -> DtypeObj:
+def _map_np_dtype(dtype: np.dtype) -> DtypeObj:
+    if dtype.kind in "iu":
+        from pandas.core.arrays.integer import NUMPY_INT_TO_DTYPE
+
+        return NUMPY_INT_TO_DTYPE[dtype]
+    elif dtype.kind == "f":
+        from pandas.core.arrays.floating import NUMPY_FLOAT_TO_DTYPE
+
+        if dtype.itemsize != 2:
+            # TODO: What do we do for float16?  float128?
+            return NUMPY_FLOAT_TO_DTYPE[dtype]
+
+    elif dtype.kind == "b":
+        from pandas import BooleanDtype
+
+        return BooleanDtype()
+
+    return dtype
+
+
+def pandas_dtype(dtype, allow_numpy_dtypes: bool = False) -> DtypeObj:
     """
     Convert input into a pandas only dtype object or a numpy dtype object.
 
     Parameters
     ----------
     dtype : object
         The object to be converted into a dtype.
+    allow_numpy_dtypes : bool, default False
+        Whether to return pre-PDEP16 numpy dtypes for ints, floats, and bools.
 
     Returns
     -------
@@ -1820,10 +1845,18 @@ def pandas_dtype(dtype) -> DtypeObj:
     >>> pd.api.types.pandas_dtype(int)
     dtype('int64')
     """
+    allow_numpy_dtypes = allow_numpy_dtypes or not get_option("mode.pdep16_data_types")
+
     # short-circuit
     if isinstance(dtype, np.ndarray):
-        return dtype.dtype
-    elif isinstance(dtype, (np.dtype, ExtensionDtype)):
+        if allow_numpy_dtypes:
+            return dtype.dtype
+        return _map_np_dtype(dtype.dtype)
+    elif isinstance(dtype, np.dtype):
+        if allow_numpy_dtypes:
+            return dtype
+        return _map_np_dtype(dtype)
+    elif isinstance(dtype, ExtensionDtype):
         return dtype
 
     # builtin aliases
@@ -1879,7 +1912,9 @@ def pandas_dtype(dtype) -> DtypeObj:
     elif npdtype.kind == "O":
         raise TypeError(f"dtype '{dtype}' not understood")
 
-    return npdtype
+    if allow_numpy_dtypes:
+        return npdtype
+    return _map_np_dtype(npdtype)
 
 
 def is_all_strings(value: ArrayLike) -> bool:

pandas/core/dtypes/dtypes.py

@@ -1777,7 +1777,7 @@ def __init__(self, dtype: Dtype = np.float64, fill_value: Any = None) -> None:
         )
         from pandas.core.dtypes.missing import na_value_for_dtype
 
-        dtype = pandas_dtype(dtype)
+        dtype = pandas_dtype(dtype, allow_numpy_dtypes=True)
         if is_string_dtype(dtype):
             dtype = np.dtype("object")
         if not isinstance(dtype, np.dtype):

pandas/core/indexes/base.py

@@ -172,6 +172,7 @@
 )
 import pandas.core.common as com
 from pandas.core.construction import (
+    array as pd_array,
     ensure_wrapped_if_datetimelike,
     extract_array,
     sanitize_array,
@@ -576,6 +577,8 @@ def __new__(
                 raise ValueError("Index data must be 1-dimensional") from err
             raise
         arr = ensure_wrapped_if_datetimelike(arr)
+        if arr.dtype.kind in "iufb" and get_option("mode.pdep16_data_types"):
+            arr = pd_array(arr, copy=False)
 
         klass = cls._dtype_to_subclass(arr.dtype)
 
@@ -5391,6 +5394,8 @@ def putmask(self, mask, value) -> Index:
 
             # See also: Block.coerce_to_target_dtype
             dtype = self._find_common_type_compat(value)
+            assert self.dtype != dtype, (self.dtype, value)
+            # FIXME: should raise with useful message to report a bug!
             return self.astype(dtype).putmask(mask, value)
 
         values = self._values.copy()