Remove should_be_monotonic property (#2949)

Author: hekaisheng

mars/dataframe/arithmetic/core.py

@@ -493,21 +493,17 @@ def _calc_properties(cls, x1, x2=None, axis="columns"):
                     index=x1.dtypes.index,
                 )
                 columns = copy.copy(x1.columns_value)
-                columns.value.should_be_monotonic = False
                 column_shape = len(dtypes)
             elif x1.dtypes is not None and x2.dtypes is not None:
                 dtypes = infer_dtypes(x1.dtypes, x2.dtypes, cls._operator)
                 columns = parse_index(dtypes.index, store_data=True)
-                columns.value.should_be_monotonic = True
                 column_shape = len(dtypes)
             if x1.index_value is not None and x2.index_value is not None:
                 if x1.index_value.key == x2.index_value.key:
                     index = copy.copy(x1.index_value)
-                    index.value.should_be_monotonic = False
                     index_shape = x1.shape[0]
                 else:
                     index = infer_index_value(x1.index_value, x2.index_value)
-                    index.value.should_be_monotonic = True
                     if index.key == x1.index_value.key == x2.index_value.key and (
                         not np.isnan(x1.shape[0]) or not np.isnan(x2.shape[0])
                     ):
@@ -539,12 +535,10 @@ def _calc_properties(cls, x1, x2=None, axis="columns"):
                             index=x1.dtypes.index,
                         )
                         columns = copy.copy(x1.columns_value)
-                        columns.value.should_be_monotonic = False
                         column_shape = len(dtypes)
                     else:  # pragma: no cover
                         dtypes = x1.dtypes  # FIXME
                         columns = infer_index_value(x1.columns_value, x2.index_value)
-                        columns.value.should_be_monotonic = True
                         column_shape = np.nan
             else:
                 assert axis == "index" or axis == 0
@@ -562,7 +556,6 @@ def _calc_properties(cls, x1, x2=None, axis="columns"):
                             index=x1.dtypes.index,
                         )
                         index = copy.copy(x1.index_value)
-                        index.value.should_be_monotonic = False
                         index_shape = x1.shape[0]
                     else:
                         if x1.dtypes is not None:
@@ -574,7 +567,6 @@ def _calc_properties(cls, x1, x2=None, axis="columns"):
                                 index=x1.dtypes.index,
                             )
                         index = infer_index_value(x1.index_value, x2.index_value)
-                        index.value.should_be_monotonic = True
                         index_shape = np.nan
             return {
                 "shape": (index_shape, column_shape),
@@ -592,11 +584,9 @@ def _calc_properties(cls, x1, x2=None, axis="columns"):
             if x1.index_value is not None and x2.index_value is not None:
                 if x1.index_value.key == x2.index_value.key:
                     index = copy.copy(x1.index_value)
-                    index.value.should_be_monotonic = False
                     index_shape = x1.shape[0]
                 else:
                     index = infer_index_value(x1.index_value, x2.index_value)
-                    index.value.should_be_monotonic = True
                     if index.key == x1.index_value.key == x2.index_value.key and (
                         not np.isnan(x1.shape[0]) or not np.isnan(x2.shape[0])
                     ):

mars/dataframe/arithmetic/tests/test_arithmetic.py

@@ -176,9 +176,7 @@ def test_without_shuffle(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert df3.columns_value.should_be_monotonic is True
     assert isinstance(df3.index_value.value, IndexValue.Int64Index)
-    assert df3.index_value.should_be_monotonic is True
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -192,9 +190,7 @@ def test_without_shuffle(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert df3.columns_value.should_be_monotonic is True
     assert isinstance(df3.index_value.value, IndexValue.Int64Index)
-    assert df3.index_value.should_be_monotonic is True
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -419,7 +415,6 @@ def test_dataframe_and_series_with_shuffle(func_name, func_opts):
         df2.columns_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
     assert df2.columns_value.key != df1.columns_value.key
-    assert df2.columns_value.should_be_monotonic is True
 
     df1, df2, s1 = tile(df1, df2, s1)
 
@@ -614,7 +609,6 @@ def test_series_and_series_with_shuffle(func_name, func_opts):
     pd.testing.assert_index_equal(
         s3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
-    assert s3.index_value.should_be_monotonic is True
 
     s1, s2, s3 = tile(s1, s2, s3)
 
@@ -673,9 +667,7 @@ def test_identical_index_and_columns(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert df3.columns_value.should_be_monotonic is False
     assert isinstance(df3.index_value.value, IndexValue.RangeIndex)
-    assert df3.index_value.should_be_monotonic is False
     pd.testing.assert_index_equal(df3.index_value.to_pandas(), pd.RangeIndex(0, 10))
     assert df3.index_value.key == df1.index_value.key
     assert df3.index_value.key == df2.index_value.key
@@ -734,9 +726,7 @@ def test_with_one_shuffle(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert df3.columns_value.should_be_monotonic is True
     assert isinstance(df3.index_value.value, IndexValue.Int64Index)
-    assert df3.index_value.should_be_monotonic is True
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -868,9 +858,7 @@ def test_with_all_shuffle(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert df3.columns_value.should_be_monotonic is True
     assert isinstance(df3.index_value.value, IndexValue.Int64Index)
-    assert df3.index_value.should_be_monotonic is True
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -970,9 +958,7 @@ def test_with_all_shuffle(func_name, func_opts):
     pd.testing.assert_index_equal(
         df6.columns_value.to_pandas(), func_opts.func(data4, data5).columns
     )
-    assert df6.columns_value.should_be_monotonic is True
     assert isinstance(df6.index_value.value, IndexValue.Int64Index)
-    assert df6.index_value.should_be_monotonic is True
     pd.testing.assert_index_equal(
         df6.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -1077,9 +1063,7 @@ def test_without_shuffle_and_with_one_chunk(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert df3.columns_value.should_be_monotonic is True
     assert isinstance(df3.index_value.value, IndexValue.Int64Index)
-    assert df3.index_value.should_be_monotonic is True
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -1191,9 +1175,7 @@ def test_both_one_chunk(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert df3.columns_value.should_be_monotonic is True
     assert isinstance(df3.index_value.value, IndexValue.Int64Index)
-    assert df3.index_value.should_be_monotonic is True
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -1237,9 +1219,7 @@ def test_with_shuffle_and_one_chunk(func_name, func_opts):
     pd.testing.assert_index_equal(
         df3.columns_value.to_pandas(), func_opts.func(data1, data2).columns
     )
-    assert df3.columns_value.should_be_monotonic is True
     assert isinstance(df3.index_value.value, IndexValue.Int64Index)
-    assert df3.index_value.should_be_monotonic is True
     pd.testing.assert_index_equal(
         df3.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )
@@ -1332,9 +1312,7 @@ def test_on_same_dataframe(func_name, func_opts):
     pd.testing.assert_index_equal(
         df2.columns_value.to_pandas(), func_opts.func(data, data).columns
     )
-    assert df2.columns_value.should_be_monotonic is False
     assert isinstance(df2.index_value.value, IndexValue.Int64Index)
-    assert df2.index_value.should_be_monotonic is False
     pd.testing.assert_index_equal(
         df2.index_value.to_pandas(), pd.Index([], dtype=np.int64)
     )

mars/dataframe/arithmetic/tests/test_arithmetic_execution.py

@@ -15,7 +15,7 @@
 import operator
 from dataclasses import dataclass
 from functools import partial
-from typing import Callable
+from typing import Callable, Union
 
 import numpy as np
 import pandas as pd
@@ -47,6 +47,16 @@ class FunctionOptions:
 )
 
 
+def sort_dataframe(
+    df: Union[pd.DataFrame, pd.Series], index: bool = True, columns: bool = True
+):
+    if index:
+        df.sort_index(inplace=True)
+    if columns and isinstance(df, pd.DataFrame):
+        df.sort_index(axis=1, inplace=True)
+    return df
+
+
 def to_boolean_if_needed(func_name, value, split_value=0.5):
     if func_name in ["__and__", "__or__", "__xor__"]:
         return value > split_value
@@ -81,7 +91,7 @@ def test_without_shuffle_execution(setup, func_name, func_opts):
     expected = func_opts.func(data1, data2)
     result = df3.execute().fetch()
 
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
 
 @pytest.mark.parametrize("func_name, func_opts", binary_functions.items())
@@ -113,7 +123,7 @@ def test_with_one_shuffle_execution(setup, func_name, func_opts):
     expected = func_opts.func(data1, data2)
     result = df3.execute().fetch()
 
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
     # only 1 axis is monotonic
     # data1 with columns split into [0...4], [5...9],
@@ -138,7 +148,7 @@ def test_with_one_shuffle_execution(setup, func_name, func_opts):
     expected = func_opts.func(data1, data2)
     result = df3.execute().fetch()
 
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
 
 @pytest.mark.parametrize("func_name, func_opts", binary_functions.items())
@@ -168,7 +178,7 @@ def test_with_all_shuffle_execution(setup, func_name, func_opts):
     expected = func_opts.func(data1, data2)
     result = df3.execute().fetch()
 
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
 
 @pytest.mark.parametrize("func_name, func_opts", binary_functions.items())
@@ -257,7 +267,7 @@ def test_without_shuffle_and_with_one_chunk(setup, func_name, func_opts):
     expected = func_opts.func(data1, data2)
     result = df3.execute().fetch()
 
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
     # only 1 axis is monotonic
     # data1 with columns split into [0...4], [5...9],
@@ -282,7 +292,7 @@ def test_without_shuffle_and_with_one_chunk(setup, func_name, func_opts):
     expected = func_opts.func(data1, data2)
     result = df3.execute().fetch()
 
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
 
 @pytest.mark.parametrize("func_name, func_opts", binary_functions.items())
@@ -312,7 +322,7 @@ def test_with_shuffle_and_with_one_chunk(setup, func_name, func_opts):
     expected = func_opts.func(data1, data2)
     result = df3.execute().fetch()
 
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
     # only 1 axis is monotonic
     # data1 with columns split into [0...4], [5...9],
@@ -337,7 +347,7 @@ def test_with_shuffle_and_with_one_chunk(setup, func_name, func_opts):
     expected = func_opts.func(data1, data2)
     result = df3.execute().fetch()
 
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
 
 @pytest.mark.parametrize("func_name, func_opts", binary_functions.items())
@@ -497,7 +507,7 @@ def test_with_shuffle_on_string_index(setup, func_name, func_opts):
     expected = func_opts.func(data1, data2)
     result = df3.execute().fetch()
 
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
 
 @pytest.mark.parametrize("func_name, func_opts", binary_functions.items())
@@ -527,23 +537,23 @@ def test_dataframe_and_series(setup, func_name, func_opts):
 
     expected = getattr(data1[[1]], func_opts.func_name)(data2[1], axis="index")
     result = r1.execute().fetch()
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
     # operate on dataframe and series without shuffle
     df2 = from_pandas(data1, chunk_size=(5, 5))
     r2 = getattr(df2, func_opts.func_name)(s1, axis="index")
 
     expected = getattr(data1, func_opts.func_name)(data2[1], axis="index")
     result = r2.execute().fetch()
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
     # operate on dataframe and series with shuffle
     df3 = from_pandas(data1, chunk_size=(5, 5))
     r3 = getattr(df3, func_opts.func_name)(s1, axis="columns")
 
     expected = getattr(data1, func_opts.func_name)(data2[1], axis="columns")
     result = r3.execute().fetch()
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
     # test both one chunk, axis=0
     pdf = pd.DataFrame({"ca": [1, 3, 2], "cb": [360, 180, 2]}, index=[1, 2, 3])
@@ -553,7 +563,7 @@ def test_dataframe_and_series(setup, func_name, func_opts):
     mars_series = from_pandas_series(series)
     result = getattr(df, func_opts.func_name)(mars_series, axis=0).execute().fetch()
     expected = getattr(pdf, func_opts.func_name)(series, axis=0)
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
     # test different number of chunks, axis=0
     pdf = pd.DataFrame({"ca": [1, 3, 2], "cb": [360, 180, 2]}, index=[1, 2, 3])
@@ -563,7 +573,7 @@ def test_dataframe_and_series(setup, func_name, func_opts):
     mars_series = from_pandas_series(series)
     result = getattr(df, func_opts.func_name)(mars_series, axis=0).execute().fetch()
     expected = getattr(pdf, func_opts.func_name)(series, axis=0)
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
     # test with row shuffle, axis=0
     pdf = pd.DataFrame({"ca": [1, 3, 2], "cb": [360, 180, 2]}, index=[2, 1, 3])
@@ -575,7 +585,7 @@ def test_dataframe_and_series(setup, func_name, func_opts):
     expected = getattr(pdf, func_opts.func_name)(series, axis=0).reindex([3, 1, 2])
     # modify the order of rows
     result = result.reindex(index=[3, 1, 2])
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
     # test both one chunk, axis=1
     pdf = pd.DataFrame(
@@ -587,7 +597,7 @@ def test_dataframe_and_series(setup, func_name, func_opts):
     mars_series = from_pandas_series(series)
     result = getattr(df, func_opts.func_name)(mars_series, axis=1).execute().fetch()
     expected = getattr(pdf, func_opts.func_name)(series, axis=1)
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
     # test different number of chunks, axis=1
     pdf = pd.DataFrame(
@@ -599,7 +609,7 @@ def test_dataframe_and_series(setup, func_name, func_opts):
     mars_series = from_pandas_series(series)
     result = getattr(df, func_opts.func_name)(mars_series, axis=1).execute().fetch()
     expected = getattr(pdf, func_opts.func_name)(series, axis=1)
-    pd.testing.assert_frame_equal(expected, result)
+    pd.testing.assert_frame_equal(sort_dataframe(expected), sort_dataframe(result))
 
     # test with row shuffle, axis=1
     pdf = pd.DataFrame(
@@ -665,7 +675,7 @@ def test_series(setup, func_name, func_opts):
     )
     result = r.execute().fetch()
     expected = func_opts.func(s1, s2)
-    pd.testing.assert_series_equal(expected, result)
+    pd.testing.assert_series_equal(sort_dataframe(expected), sort_dataframe(result))
 
     if func_opts.func_name in ["__and__", "__or__", "__xor__"]:
         # bitwise logical operators doesn\'t support floating point scalars