[Ray] Move some tests out of test_ray.py (#3109)

* move some tests to ray_cluster

* rename test_ray_cluster to test_ray_cluster_standalone

Author: chaokunyang

.github/workflows/platform-ci.yml

@@ -154,6 +154,8 @@ jobs:
           if [ -n "$WITH_RAY_DEPLOY" ]; then
             pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray.py -m ray
             mv .coverage build/.coverage.test_ray.file
+            pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_cluster_standalone.py -m ray
+            mv .coverage build/.coverage.test_ray_cluster_standalone.file
             pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_client.py -m ray
             mv .coverage build/.coverage.test_ray_client.file
             pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_fault_injection.py -m ray

mars/deploy/oscar/tests/test_ray.py

@@ -21,7 +21,6 @@
 import numpy as np
 import pytest
 
-import mars
 from .... import tensor as mt
 from .... import dataframe as md
 from ....oscar.errors import ReconstructWorkerError
@@ -32,8 +31,6 @@
     _load_config,
     ClusterStateActor,
     new_cluster,
-    new_cluster_in_ray,
-    new_ray_session,
 )
 from ..session import get_default_session, new_session
 from ..tests import test_local
@@ -190,47 +187,6 @@ def _sync_web_session_test(web_address):
     return True
 
 
-@require_ray
-def test_new_cluster_in_ray(stop_ray):
-    cluster = new_cluster_in_ray(worker_num=2)
-    mt.random.RandomState(0).rand(100, 5).sum().execute()
-    cluster.session.execute(mt.random.RandomState(0).rand(100, 5).sum())
-    mars.execute(mt.random.RandomState(0).rand(100, 5).sum())
-    session = new_ray_session(address=cluster.address, session_id="abcd", default=True)
-    session.execute(mt.random.RandomState(0).rand(100, 5).sum())
-    mars.execute(mt.random.RandomState(0).rand(100, 5).sum())
-    cluster.stop()
-
-
-@require_ray
-def test_new_ray_session(stop_ray):
-    new_ray_session_test()
-
-
-def new_ray_session_test():
-    session = new_ray_session(session_id="abc", worker_num=2)
-    mt.random.RandomState(0).rand(100, 5).sum().execute()
-    session.execute(mt.random.RandomState(0).rand(100, 5).sum())
-    mars.execute(mt.random.RandomState(0).rand(100, 5).sum())
-    session = new_ray_session(session_id="abcd", worker_num=2, default=True)
-    session.execute(mt.random.RandomState(0).rand(100, 5).sum())
-    mars.execute(mt.random.RandomState(0).rand(100, 5).sum())
-    df = md.DataFrame(mt.random.rand(100, 4), columns=list("abcd"))
-    # Convert mars dataframe to ray dataset
-    ds = md.to_ray_dataset(df)
-    print(ds.schema(), ds.count())
-    ds.filter(lambda row: row["a"] > 0.5).show(5)
-    # Convert ray dataset to mars dataframe
-    df2 = md.read_ray_dataset(ds)
-    print(df2.head(5).execute())
-    # Test ray cluster exists after session got gc.
-    del session
-    import gc
-
-    gc.collect()
-    mars.execute(mt.random.RandomState(0).rand(100, 5).sum())
-
-
 @require_ray
 @pytest.mark.parametrize(
     "test_option",

mars/deploy/oscar/tests/test_ray_client.py

@@ -17,7 +17,7 @@
 import tempfile
 import threading
 
-from .test_ray import new_ray_session_test
+from .test_ray_cluster_standalone import new_ray_session_test
 from ....tests.core import require_ray
 from ....utils import lazy_import
 

mars/deploy/oscar/tests/test_ray_cluster_standalone.py

@@ -0,0 +1,66 @@
+# Copyright 1999-2021 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mars
+from .... import tensor as mt
+from .... import dataframe as md
+from ....tests.core import require_ray
+from ....utils import lazy_import
+from ..ray import (
+    new_cluster_in_ray,
+    new_ray_session,
+)
+
+ray = lazy_import("ray")
+
+
+@require_ray
+def test_new_cluster_in_ray(stop_ray):
+    cluster = new_cluster_in_ray(worker_num=2)
+    mt.random.RandomState(0).rand(100, 5).sum().execute()
+    cluster.session.execute(mt.random.RandomState(0).rand(100, 5).sum())
+    mars.execute(mt.random.RandomState(0).rand(100, 5).sum())
+    session = new_ray_session(address=cluster.address, session_id="abcd", default=True)
+    session.execute(mt.random.RandomState(0).rand(100, 5).sum())
+    mars.execute(mt.random.RandomState(0).rand(100, 5).sum())
+    cluster.stop()
+
+
+@require_ray
+def test_new_ray_session(stop_ray):
+    new_ray_session_test()
+
+
+def new_ray_session_test():
+    session = new_ray_session(session_id="abc", worker_num=2)
+    mt.random.RandomState(0).rand(100, 5).sum().execute()
+    session.execute(mt.random.RandomState(0).rand(100, 5).sum())
+    mars.execute(mt.random.RandomState(0).rand(100, 5).sum())
+    session = new_ray_session(session_id="abcd", worker_num=2, default=True)
+    session.execute(mt.random.RandomState(0).rand(100, 5).sum())
+    mars.execute(mt.random.RandomState(0).rand(100, 5).sum())
+    df = md.DataFrame(mt.random.rand(100, 4), columns=list("abcd"))
+    # Convert mars dataframe to ray dataset
+    ds = md.to_ray_dataset(df)
+    print(ds.schema(), ds.count())
+    ds.filter(lambda row: row["a"] > 0.5).show(5)
+    # Convert ray dataset to mars dataframe
+    df2 = md.read_ray_dataset(ds)
+    print(df2.head(5).execute())
+    # Test ray cluster exists after session got gc.
+    del session
+    import gc
+
+    gc.collect()
+    mars.execute(mt.random.RandomState(0).rand(100, 5).sum())