Fix CI & latest Numpy version (#3306)

Author: hekaisheng

.github/workflows/platform-ci.yml

@@ -18,11 +18,11 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: [3.8-kubernetes, 3.8-hadoop, 3.8-ray, 3.8-ray-deploy, 3.8-ray-dag, 3.8-vineyard, 3.8-dask]
+        python-version: [3.8-ray, 3.8-ray-deploy, 3.8-ray-dag, 3.8-vineyard, 3.8-dask]
         include:
-          - { os: ubuntu-latest, python-version: 3.8-kubernetes, no-common-tests: 1,
+          - { os: ubuntu-20.04, python-version: 3.8-kubernetes, no-common-tests: 1,
               no-deploy: 1, with-kubernetes: "with Kubernetes" }
-          - { os: ubuntu-latest, python-version: 3.8-hadoop, no-common-tests: 1,
+          - { os: ubuntu-20.04, python-version: 3.8-hadoop, no-common-tests: 1,
               no-deploy: 1, with-hadoop: "with hadoop" }
           - { os: ubuntu-latest, python-version: 3.8-vineyard, no-common-tests: 1,
               no-deploy: 1, with-vineyard: "with vineyard" }
@@ -72,7 +72,7 @@ jobs:
 
           source ./ci/rewrite-cov-config.sh
 
-          pip install numpy scipy cython
+          pip install numpy scipy cython "scikit-learn<1.2.0"
 
           pip install -e ".[dev,extra]"
 

azure-pipelines.yml

@@ -60,7 +60,7 @@ jobs:
         source ./ci/rewrite-cov-config.sh
 
         if [[ "$(mars.test.module)" == "learn" ]]; then
-          pip install numpy\<1.20.0 scipy cython
+          pip install numpy\<1.20.0 scipy cython scikit-learn\<1.2.0
         else
           pip install numpy scipy cython
         fi
@@ -94,6 +94,7 @@ jobs:
 
         # do compatibility test for earliest supported pandas release
         if [[ "$(mars.test.module)" == "dataframe" ]]; then
+          pip install numpy\<1.24.0
           pip install -i https://pkgs.dev.azure.com/mars-project/mars/_packaging/pandas/pypi/simple/ pandas==1.0.5
           pytest $PYTEST_CONFIG -m pd_compat mars/dataframe
           mv .coverage build/.coverage.pd_compat.file

mars/dataframe/reduction/tests/test_reduction_execution.py

@@ -42,25 +42,9 @@
 def check_ref_counts():
     yield
 
-    import functools
     import gc
 
-    # In https://github.com/pandas-dev/pandas/pull/48023, pandas cache current frame in this PR
-    # which leads to failure of decref mechanism.
     gc.collect()
-    wrappers = []
-    tp = functools._lru_cache_wrapper
-    for a in gc.get_objects():
-        try:
-            if isinstance(a, tp):
-                wrappers.append(a)
-        except ReferenceError:
-            # a may raises ReferenceError: weakly-referenced object no longer exists
-            # please refer to: https://github.com/mars-project/mars/issues/3290
-            pass
-
-    for wrapper in wrappers:
-        wrapper.cache_clear()
 
     sess = get_default_session()
     assert len(sess._get_ref_counts()) == 0

mars/tensor/__init__.py

@@ -377,6 +377,7 @@
     # suppress numpy warnings on types
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", DeprecationWarning)
+        warnings.simplefilter("ignore", FutureWarning)
         # noinspection PyUnresolvedReferences
         from numpy import object, int, bool, float
 except ImportError:  # pragma: no cover

mars/tensor/base/transpose.py

@@ -28,7 +28,7 @@
 def _reorder(x, axes):
     if x is None:
         return
-    return type(x)(np.array(x)[list(axes)].tolist())
+    return type(x)(x[ax] for ax in axes)
 
 
 class TensorTranspose(TensorHasInput, TensorOperandMixin):

mars/tensor/indexing/fill_diagonal.py

@@ -184,10 +184,8 @@ def _tile_nd(cls, op, val):
         # if more than 3d, we will rechunk the tensor into square chunk
         # on the diagonal direction
         in_tensor = op.input
-        nsplits = np.array(in_tensor.nsplits)
-        if not np.issubdtype(nsplits.dtype, np.integer) or not np.all(
-            np.diff(nsplits, axis=1) == 0
-        ):
+        nsplits = [tuple(np.array(split)) for split in in_tensor.nsplits]
+        if len(set(nsplits)) != 1:
             # need rechunk
             nsplit = decide_unify_split(*in_tensor.nsplits)
             in_tensor = yield from recursive_tile(