ENH: Fix deprecation warnings about __array__ and quantile (xorbitsai#851)

Author: Dawnfz-Lenfeng

.github/workflows/python.yaml

@@ -156,7 +156,7 @@ jobs:
         else
           pip install -e "git+https://github.com/xorbitsai/xoscar.git@main#subdirectory=python&egg=xoscar"
           # TODO: pandas v2.2.3 now does not support well for numpy v2.2.0 and pyarrow v19.0.0. 
-          pip install -U "numpy<2.2.0" "pyarrow<19.0.0" scipy cython pyftpdlib coverage flaky numexpr openpyxl
+          pip install -U numpy pyarrow scipy cython pyftpdlib coverage flaky numexpr openpyxl
 
           if [[ "$MODULE" == "mars-core" ]]; then
             pip install oss2
@@ -291,7 +291,7 @@ jobs:
       run: |
         source activate ${{ env.CONDA_ENV }}
         pip install --extra-index-url=https://pypi.nvidia.com cudf-cu12==24.10.*
-        pip install ucxx-cu12 cython "numpy>=1.14.0,<2.0.0" cloudpickle scikit-learn \
+        pip install ucxx-cu12 cython numpy cloudpickle scikit-learn \
           pyyaml psutil tornado sqlalchemy defusedxml tqdm uvloop coverage \
           pytest pytest-cov pytest-timeout pytest-forked pytest-asyncio pytest-mock
         pip install -U xoscar

python/xorbits/_mars/dataframe/core.py

@@ -954,15 +954,15 @@ def __str__(self):
     def __repr__(self):
         return self._to_str(representation=True)
 
-    def _to_arr(self):
+    def _to_arr(self, dtype=None, copy=None):
         if len(self._executed_sessions) == 0:  # pragma: no cover
             raise NotImplementedError
 
         data = self.fetch(session=self._executed_sessions[-1])
-        return np.asarray(data)
+        return np.asarray(data, dtype=dtype, copy=copy)
 
-    def __array__(self):
-        return self._to_arr()
+    def __array__(self, dtype=None, copy=None):
+        return self._to_arr(dtype=dtype, copy=copy)
 
     def _to_mars_tensor(self, dtype=None, order="K", extract_multi_index=False):
         tensor = self.to_tensor(extract_multi_index=extract_multi_index)
@@ -1455,15 +1455,15 @@ def __str__(self):
     def __repr__(self):
         return self._to_str(representation=False)
 
-    def _to_arr(self):
+    def _to_arr(self, dtype=None, copy=None):
         if len(self._executed_sessions) == 0:  # pragma: no cover
             raise NotImplementedError
 
         data = self.fetch(session=self._executed_sessions[-1])
-        return np.asarray(data)
+        return np.asarray(data, dtype=dtype, copy=copy)
 
-    def __array__(self):
-        return self._to_arr()
+    def __array__(self, dtype=None, copy=None):
+        return self._to_arr(dtype=dtype, copy=copy)
 
     @property
     def dtype(self):
@@ -2054,9 +2054,9 @@ class BaseDataFrameData(HasShapeTileableData, _ToPandasMixin):
         "chunks",
         FieldTypes.reference(DataFrameChunkData),
         on_serialize=lambda x: [it.data for it in x] if x is not None else x,
-        on_deserialize=lambda x: [DataFrameChunk(it) for it in x]
-        if x is not None
-        else x,
+        on_deserialize=lambda x: (
+            [DataFrameChunk(it) for it in x] if x is not None else x
+        ),
     )
 
     def __init__(
@@ -2263,15 +2263,15 @@ def _to_str(self, representation=False):
     def __str__(self):
         return self._to_str(representation=False)
 
-    def _to_arr(self):
+    def _to_arr(self, dtype=None, copy=None):
         if len(self._executed_sessions) == 0:
             raise NotImplementedError
 
         data = self.fetch(session=self._executed_sessions[-1])
-        return np.asarray(data)
+        return np.asarray(data, dtype=dtype, copy=copy)
 
-    def __array__(self):
-        return self._to_arr()
+    def __array__(self, dtype=None, copy=None):
+        return self._to_arr(dtype=dtype, copy=copy)
 
     def __repr__(self):
         return self._to_str(representation=True)
@@ -2767,9 +2767,9 @@ class DataFrameGroupByData(BaseDataFrameData):
         "chunks",
         FieldTypes.reference(DataFrameGroupByChunkData),
         on_serialize=lambda x: [it.data for it in x] if x is not None else x,
-        on_deserialize=lambda x: [DataFrameGroupByChunk(it) for it in x]
-        if x is not None
-        else x,
+        on_deserialize=lambda x: (
+            [DataFrameGroupByChunk(it) for it in x] if x is not None else x
+        ),
     )
 
     @property
@@ -2816,9 +2816,9 @@ class SeriesGroupByData(BaseSeriesData):
         "chunks",
         FieldTypes.reference(SeriesGroupByChunkData),
         on_serialize=lambda x: [it.data for it in x] if x is not None else x,
-        on_deserialize=lambda x: [SeriesGroupByChunk(it) for it in x]
-        if x is not None
-        else x,
+        on_deserialize=lambda x: (
+            [SeriesGroupByChunk(it) for it in x] if x is not None else x
+        ),
     )
 
     @property
@@ -2995,9 +2995,9 @@ class CategoricalData(HasShapeTileableData, _ToPandasMixin):
         "chunks",
         FieldTypes.reference(CategoricalChunkData),
         on_serialize=lambda x: [it.data for it in x] if x is not None else x,
-        on_deserialize=lambda x: [CategoricalChunk(it) for it in x]
-        if x is not None
-        else x,
+        on_deserialize=lambda x: (
+            [CategoricalChunk(it) for it in x] if x is not None else x
+        ),
     )
 
     def __init__(
@@ -3195,9 +3195,9 @@ class DataFrameOrSeriesData(HasShapeTileableData, _ToPandasMixin):
         "chunks",
         FieldTypes.reference(DataFrameOrSeriesChunkData),
         on_serialize=lambda x: [it.data for it in x] if x is not None else x,
-        on_deserialize=lambda x: [DataFrameOrSeriesChunk(it) for it in x]
-        if x is not None
-        else x,
+        on_deserialize=lambda x: (
+            [DataFrameOrSeriesChunk(it) for it in x] if x is not None else x
+        ),
     )
 
     _data_type = StringField("data_type")

python/xorbits/_mars/dataframe/plotting/tests/test_plot.py

@@ -44,7 +44,7 @@ def assert_is_valid_plot_return_object(objs):  # pragma: no cover
     import matplotlib.pyplot as plt
 
     if isinstance(objs, (pd.Series, np.ndarray)):
-        for el in objs.ravel():
+        for el in objs.to_numpy():
             msg = (
                 "one of 'objs' is not a matplotlib Axes instance, "
                 f"type encountered {type(el).__name__}"

python/xorbits/_mars/dataframe/reduction/core.py

@@ -566,7 +566,7 @@ def _execute_map(cls, ctx, op):
             kwargs["skipna"] = op.skipna
         partial = getattr(in_data, getattr(cls, "_func_name"))(**kwargs)
         if op.skipna:
-            partial.fillna(method="ffill", axis=op.axis, inplace=True)
+            partial.ffill(axis=op.axis, inplace=True)
         ctx[op.outputs[0].key] = cls._get_last_slice(op, partial, -1)
 
     @classmethod
@@ -583,7 +583,7 @@ def _execute_combine(cls, ctx, op):
                 pd.concat(ref_datas, axis=op.axis), getattr(cls, "_func_name")
             )(**kwargs)
             if op.skipna:
-                concat_df.fillna(method="ffill", axis=op.axis, inplace=True)
+                concat_df.ffill(axis=op.axis, inplace=True)
 
             in_data = ctx[op.inputs[0].key]
             concat_df = pd.concat(

python/xorbits/_mars/dataframe/statistics/quantile.py

@@ -106,7 +106,7 @@ def _calc_dtype_on_axis_1(self, a, dtypes):
             dt = tensor_quantile(
                 tensor_from_series(a[name]),
                 self._q,
-                interpolation=self._interpolation,
+                method=self._interpolation,
                 handle_non_numeric=not self._numeric_only,
             ).dtype
             quantile_dtypes.append(dt)
@@ -152,7 +152,7 @@ def _call_dataframe(self, a, inputs):
             dt = tensor_quantile(
                 empty(a.shape[1], dtype=find_common_type(list(dtypes))),
                 self._q,
-                interpolation=self._interpolation,
+                method=self._interpolation,
                 handle_non_numeric=not self._numeric_only,
             ).dtype
             return self.new_series(
@@ -173,7 +173,7 @@ def _call_dataframe(self, a, inputs):
                     tensor_quantile(
                         tensor_from_series(a[name]),
                         self._q,
-                        interpolation=self._interpolation,
+                        method=self._interpolation,
                         handle_non_numeric=not self._numeric_only,
                     ).dtype
                 )
@@ -222,7 +222,7 @@ def _call_series(self, a, inputs):
         self._dtype = dtype = tensor_quantile(
             a_t,
             self._q,
-            interpolation=self._interpolation,
+            method=self._interpolation,
             handle_non_numeric=not self._numeric_only,
         ).dtype
 
@@ -266,7 +266,7 @@ def _tile_dataframe(cls, op):
                     t = tensor_quantile(
                         a,
                         op.q,
-                        interpolation=op.interpolation,
+                        method=op.interpolation,
                         handle_non_numeric=not op.numeric_only,
                     )
                     ts.append(t)
@@ -288,7 +288,7 @@ def _tile_dataframe(cls, op):
                     t,
                     op.q,
                     axis=1,
-                    interpolation=op.interpolation,
+                    method=op.interpolation,
                     handle_non_numeric=not op.numeric_only,
                 )
                 r = series_from_tensor(tr, index=op.input.index, name=tr.op.q.item())
@@ -301,7 +301,7 @@ def _tile_dataframe(cls, op):
                     t = tensor_quantile(
                         a,
                         op.q,
-                        interpolation=op.interpolation,
+                        method=op.interpolation,
                         handle_non_numeric=not op.numeric_only,
                     )
                     d[name] = t
@@ -315,7 +315,7 @@ def _tile_dataframe(cls, op):
                     t,
                     op.q,
                     axis=1,
-                    interpolation=op.interpolation,
+                    method=op.interpolation,
                     handle_non_numeric=not op.numeric_only,
                 )
                 if not op.input.index_value.has_value():
@@ -333,7 +333,7 @@ def _tile_series(cls, op):
         t = tensor_quantile(
             a,
             op.q,
-            interpolation=op.interpolation,
+            method=op.interpolation,
             handle_non_numeric=not op.numeric_only,
         )
         if isinstance(op.outputs[0], TENSOR_TYPE):

python/xorbits/_mars/lib/groupby_wrapper.py

@@ -40,7 +40,6 @@ def __init__(
         keys=None,
         axis=0,
         level=None,
-        grouper=None,
         exclusions=None,
         selection=None,
         as_index=True,
@@ -85,7 +84,6 @@ def _is_frame_groupby(data: Any) -> bool:
                 keys=keys,
                 axis=axis,
                 level=level,
-                grouper=grouper,
                 exclusions=exclusions,
                 as_index=as_index,
                 group_keys=group_keys,
@@ -115,7 +113,6 @@ def __getitem__(self, item):
             keys=self.keys,
             axis=self.axis,
             level=self.level,
-            grouper=self.groupby_obj.grouper,
             exclusions=self.exclusions,
             selection=item,
             as_index=self.as_index,

python/xorbits/_mars/tensor/statistics/percentile.py

@@ -28,7 +28,7 @@ def percentile(
     axis=None,
     out=None,
     overwrite_input=False,
-    interpolation="linear",
+    method="linear",
     keepdims=False,
 ):
     """
@@ -53,8 +53,8 @@ def percentile(
         but the type (of the output) will be cast if necessary.
     overwrite_input : bool, optional
         Just for compatibility with Numpy, would not take effect.
-    interpolation : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}
-        This optional parameter specifies the interpolation method to
+    method : {'linear', 'lower', 'higher', 'midpoint', 'nearest'}
+        This optional parameter specifies the method method to
         use when the desired percentile lies between two data points
         ``i < j``:
 
@@ -94,7 +94,7 @@ def percentile(
     Given a vector ``V`` of length ``N``, the q-th percentile of
     ``V`` is the value ``q/100`` of the way from the minimum to the
     maximum in a sorted copy of ``V``. The values and distances of
-    the two nearest neighbors as well as the `interpolation` parameter
+    the two nearest neighbors as well as the `method` parameter
     will determine the percentile if the normalized ranking does not
     match the location of ``q`` exactly. This function is the same as
     the median if ``q=50``, the same as the minimum if ``q=0`` and the
@@ -124,7 +124,7 @@ def percentile(
     >>> m.execute()
     array([6.5, 4.5, 2.5])
 
-    The different types of interpolation can be visualized graphically:
+    The different types of method can be visualized graphically:
 
     .. plot::
 
@@ -142,10 +142,10 @@ def percentile(
             ('nearest', '-.'),
             ('midpoint', '-.'),
         ]
-        for interpolation, style in lines:
+        for method, style in lines:
             ax.plot(
-                np.asarray(p), np.asarray(mt.percentile(a, p, interpolation=interpolation)),
-                label=interpolation, linestyle=style)
+                np.asarray(p), np.asarray(mt.percentile(a, p, method=method)),
+                label=method, linestyle=style)
         ax.set(
             title='Interpolation methods for list: ' + str(a),
             xlabel='Percentile',
@@ -170,7 +170,7 @@ def percentile(
         axis=axis,
         out=out,
         overwrite_input=overwrite_input,
-        interpolation=interpolation,
+        method=method,
         keepdims=keepdims,
         q_error_msg=q_error_msg,
     )