Better option name, fixture

Author: jbrockmendel

pandas/_config/__init__.py

@@ -35,6 +35,6 @@ def using_string_dtype() -> bool:
     return _mode_options["infer_string"]
 
 
-def using_pyarrow_strict_nans() -> bool:
+def is_nan_na() -> bool:
     _mode_options = _global_config["mode"]
-    return _mode_options["pyarrow_strict_nans"]
+    return _mode_options["nan_is_na"]

pandas/conftest.py

@@ -2116,3 +2116,10 @@ def temp_file(tmp_path):
 def monkeysession():
     with pytest.MonkeyPatch.context() as mp:
         yield mp
+
+
+@pytest.fixture(params=[True, False])
+def using_nan_is_na(request):
+    opt = request.param
+    with pd.option_context("mode.nan_is_na", opt):
+        yield opt

pandas/core/arrays/_utils.py

@@ -7,7 +7,7 @@
 
 import numpy as np
 
-from pandas._config import using_pyarrow_strict_nans
+from pandas._config import is_nan_na
 
 from pandas._libs import lib
 from pandas._libs.missing import NA
@@ -41,7 +41,7 @@ def to_numpy_dtype_inference(
                 else:
                     dtype = arr.dtype.numpy_dtype  # type: ignore[union-attr]
                 if na_value is lib.no_default:
-                    if is_pyarrow and using_pyarrow_strict_nans():
+                    if is_pyarrow and not is_nan_na():
                         na_value = NA
                         dtype = np.dtype(object)
                     else:

pandas/core/arrays/arrow/array.py

@@ -17,7 +17,7 @@
 
 import numpy as np
 
-from pandas._config import using_pyarrow_strict_nans
+from pandas._config import is_nan_na
 
 from pandas._libs import lib
 from pandas._libs.missing import is_pdna_or_none
@@ -36,6 +36,7 @@
 
 from pandas.core.dtypes.cast import (
     can_hold_element,
+    construct_1d_object_array_from_listlike,
     infer_dtype_from_scalar,
 )
 from pandas.core.dtypes.common import (
@@ -555,7 +556,22 @@ def _box_pa_array(
                 return pa_array
 
             mask = None
-            if getattr(value, "dtype", None) is None or value.dtype.kind not in "iumMf":
+            if is_nan_na():
+                try:
+                    arr_value = np.asarray(value)
+                    if arr_value.ndim > 1:
+                        # e.g. test_fixed_size_list we have list data.  ndim > 1
+                        #  means there were no scalar (NA) entries.
+                        mask = np.zeros(len(value), dtype=np.bool_)
+                    else:
+                        mask = isna(arr_value)
+                except ValueError:
+                    # Ragged data that numpy raises on
+                    arr_value = construct_1d_object_array_from_listlike(value)
+                    mask = isna(arr_value)
+            elif (
+                getattr(value, "dtype", None) is None or value.dtype.kind not in "iumMf"
+            ):
                 arr_value = np.asarray(value, dtype=object)
                 # similar to isna(value) but exclude NaN, NaT, nat-like, nan-like
                 mask = is_pdna_or_none(arr_value)
@@ -1490,7 +1506,9 @@ def to_numpy(
         na_value: object = lib.no_default,
     ) -> np.ndarray:
         original_na_value = na_value
-        dtype, na_value = to_numpy_dtype_inference(self, dtype, na_value, self._hasna)
+        dtype, na_value = to_numpy_dtype_inference(
+            self, dtype, na_value, self._hasna, is_pyarrow=True
+        )
         pa_type = self._pa_array.type
         if not self._hasna or isna(na_value) or pa.types.is_null(pa_type):
             data = self
@@ -1522,7 +1540,7 @@ def to_numpy(
                 or (
                     original_na_value is lib.no_default
                     and is_float_dtype(dtype)
-                    and not using_pyarrow_strict_nans()
+                    and is_nan_na()
                 )
             )
         ):

pandas/core/config_init.py

@@ -429,10 +429,11 @@ def is_terminal() -> bool:
 
 with cf.config_prefix("mode"):
     cf.register_option(
-        "pyarrow_strict_nans",
-        True,
+        "nan_is_na",
+        False,
         # TODO: Change this to False before merging
-        "Whether to make ArrowDtype arrays consistently treat NaN as distinct from NA",
+        "Whether to make ArrowDtype arrays consistently treat NaN as "
+        "interchangeable with pd.NA",
         validator=is_one_of_factory([True, False]),
     )
 

pandas/tests/extension/test_arrow.py

@@ -32,8 +32,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_pyarrow_strict_nans
-
 from pandas._libs import lib
 from pandas._libs.tslibs import timezones
 from pandas.compat import (
@@ -277,17 +275,14 @@ def test_compare_scalar(self, data, comparison_op):
         self._compare_other(ser, data, comparison_op, data[0])
 
     @pytest.mark.parametrize("na_action", [None, "ignore"])
-    def test_map(self, data_missing, na_action):
+    def test_map(self, data_missing, na_action, using_nan_is_na):
         if data_missing.dtype.kind in "mM":
             result = data_missing.map(lambda x: x, na_action=na_action)
             expected = data_missing.to_numpy(dtype=object)
             tm.assert_numpy_array_equal(result, expected)
         else:
             result = data_missing.map(lambda x: x, na_action=na_action)
-            if (
-                data_missing.dtype == "float32[pyarrow]"
-                and not using_pyarrow_strict_nans()
-            ):
+            if data_missing.dtype == "float32[pyarrow]" and using_nan_is_na:
                 # map roundtrips through objects, which converts to float64
                 expected = data_missing.to_numpy(dtype="float64", na_value=np.nan)
             else:
@@ -698,7 +693,7 @@ def test_setitem_preserves_views(self, data):
 
     @pytest.mark.parametrize("dtype_backend", ["pyarrow", no_default])
     @pytest.mark.parametrize("engine", ["c", "python"])
-    def test_EA_types(self, engine, data, dtype_backend, request):
+    def test_EA_types(self, engine, data, dtype_backend, request, using_nan_is_na):
         pa_dtype = data.dtype.pyarrow_dtype
         if pa.types.is_decimal(pa_dtype):
             request.applymarker(
@@ -719,7 +714,7 @@ def test_EA_types(self, engine, data, dtype_backend, request):
                 pytest.mark.xfail(reason="CSV parsers don't correctly handle binary")
             )
         df = pd.DataFrame({"with_dtype": pd.Series(data, dtype=str(data.dtype))})
-        if using_pyarrow_strict_nans():
+        if not using_nan_is_na:
             csv_output = df.to_csv(index=False, na_rep="NA")
         else:
             csv_output = df.to_csv(index=False, na_rep=np.nan)
@@ -1536,7 +1531,7 @@ def test_astype_errors_ignore():
     tm.assert_frame_equal(result, expected)
 
 
-def test_to_numpy_with_defaults(data):
+def test_to_numpy_with_defaults(data, using_nan_is_na):
     # GH49973
     result = data.to_numpy()
 
@@ -1548,21 +1543,19 @@ def test_to_numpy_with_defaults(data):
     else:
         expected = np.array(data._pa_array)
 
-    if data._hasna and (
-        not is_numeric_dtype(data.dtype) or using_pyarrow_strict_nans()
-    ):
+    if data._hasna and (not is_numeric_dtype(data.dtype) or not using_nan_is_na):
         expected = expected.astype(object)
         expected[pd.isna(data)] = pd.NA
 
     tm.assert_numpy_array_equal(result, expected)
 
 
-def test_to_numpy_int_with_na():
+def test_to_numpy_int_with_na(using_nan_is_na):
     # GH51227: ensure to_numpy does not convert int to float
     data = [1, None]
     arr = pd.array(data, dtype="int64[pyarrow]")
     result = arr.to_numpy()
-    if using_pyarrow_strict_nans():
+    if not using_nan_is_na:
         expected = np.array([1, pd.NA], dtype=object)
     else:
         expected = np.array([1, np.nan])
@@ -3528,10 +3521,10 @@ def test_cast_dictionary_different_value_dtype(arrow_type):
     assert result.dtypes.iloc[0] == data_type
 
 
-def test_map_numeric_na_action():
+def test_map_numeric_na_action(using_nan_is_na):
     ser = pd.Series([32, 40, None], dtype="int64[pyarrow]")
     result = ser.map(lambda x: 42, na_action="ignore")
-    if using_pyarrow_strict_nans():
+    if not using_nan_is_na:
         expected = pd.Series([42.0, 42.0, pd.NA], dtype="object")
     else:
         expected = pd.Series([42.0, 42.0, np.nan], dtype="float64")

pandas/tests/frame/methods/test_convert_dtypes.py

@@ -3,8 +3,6 @@
 import numpy as np
 import pytest
 
-from pandas._config import using_pyarrow_strict_nans
-
 import pandas.util._test_decorators as td
 
 import pandas as pd
@@ -61,7 +59,7 @@ def test_convert_dtypes_retain_column_names(self):
         tm.assert_index_equal(result.columns, df.columns)
         assert result.columns.name == "cols"
 
-    def test_pyarrow_dtype_backend(self):
+    def test_pyarrow_dtype_backend(self, using_nan_is_na):
         pa = pytest.importorskip("pyarrow")
         df = pd.DataFrame(
             {
@@ -76,7 +74,7 @@ def test_pyarrow_dtype_backend(self):
         )
         result = df.convert_dtypes(dtype_backend="pyarrow")
 
-        item = None if not using_pyarrow_strict_nans() else np.nan
+        item = None if using_nan_is_na else np.nan
         expected = pd.DataFrame(
             {
                 "a": pd.arrays.ArrowExtensionArray(

pandas/tests/series/methods/test_rank.py

@@ -279,6 +279,7 @@ def test_rank_tie_methods(self, ser, results, dtype, using_infer_string):
             dtype == "int64"
             or dtype == "int64[pyarrow]"
             or dtype == "uint64[pyarrow]"
+            or dtype == "float64[pyarrow]"
             or (not using_infer_string and dtype == "str")
         ):
             pytest.skip("int64/str does not support NaN")