[Dependency] Solve dependency compatibility issues (#3346)

* Pin ray<2.4.0

* Pin ray version in ci

* use 'pip install scikit-learn' rather than 'pip install sklearn'

* Pin mimesis<9.0.0 because iterations parameter for all methods of Schema has been removed

* CHORE: Compatible with Arrow 12.0.0 (#426)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

* Remove plasma storage setting

* Ignore FileNotFoundError of Python 3.7.x or below

* Ignore tests using shared memory of Python 3.7.x or below

* Remove two _system_config Ray does not support and change plasma to shared_memory in k8s

---------

Co-authored-by: aresnow1 <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: 3 people

.github/workflows/platform-ci.yml

@@ -104,12 +104,13 @@ jobs:
               rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test
             fi
             if [ -n "$WITH_RAY" ] || [ -n "$WITH_RAY_DAG" ] || [ -n "$WITH_RAY_DEPLOY" ]; then
+              pip install "ray>=1.8.0,<2.4.0"
               pip install "xgboost_ray<0.1.14" "protobuf<4"
               # Ray Datasets need pyarrow>=6.0.1
               pip install "pyarrow>=6.0.1"
             fi
             if [ -n "$RUN_DASK" ]; then
-              pip install dask[complete] mimesis sklearn
+              pip install "dask[complete]" "mimesis<9.0.0" scikit-learn
             fi
           fi
           conda list -n test

mars/deploy/kubernetes/config.yml

@@ -2,6 +2,4 @@
 cluster:
   backend: k8s
 storage:
-  backends: [plasma]
-  plasma:
-    store_memory: 20%
+  backends: [shared_memory]

mars/deploy/oscar/tests/fault_injection_config_with_rerun.yml

@@ -4,6 +4,3 @@ third_party_modules:
 scheduling:
   subtask_max_retries: 2
   subtask_max_reschedules: 2
-storage:
-  # shared-memory38 may lose object if the process crash after put success.
-  backends: [plasma]

mars/deploy/oscar/tests/test_fault_injection.py

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 import os
+import sys
+
 import pytest
 import traceback
 import numpy as np
@@ -197,6 +199,10 @@ async def test_rerun_subtask(fault_cluster, fault_config):
         await info
 
 
+@pytest.mark.skipif(
+    sys.version_info[:2] < (3, 8),
+    reason="Skip due to the incompatibilities of shared memory.",
+)
 @pytest.mark.parametrize(
     "fault_cluster", [{"config": RERUN_SUBTASK_CONFIG_FILE}], indirect=True
 )

mars/deploy/oscar/tests/test_local.py

@@ -1059,8 +1059,6 @@ async def speculative_cluster():
     config["scheduling"]["speculation"]["max_concurrent_run"] = 10
     config["scheduling"]["subtask_cancel_timeout"] = 0.1
     config["scheduling"]["enable_kill_slot"] = True
-    config["storage"]["backends"] = ["plasma"]
-    config["storage"]["plasma"]["store_memory"] = 10 * 1024 * 1024
     client = await new_cluster(
         config=config,
         n_worker=5,

mars/deploy/oscar/tests/test_ray_dag_failover.py

@@ -47,8 +47,6 @@ def test_basic_object_reconstruction(
     ray_large_cluster, reconstruction_enabled, stop_mars
 ):
     config = {
-        "num_heartbeats_timeout": 10,
-        "raylet_heartbeat_period_milliseconds": 200,
         "object_timeout_milliseconds": 200,
     }
     # Workaround to reset the config to the default value.

mars/services/storage/api/oscar.py

@@ -330,21 +330,7 @@ async def create(
 
         storage_configs = kwargs.get("storage_configs")
         if not storage_configs:
-            if sys.platform == "darwin":
-                plasma_dir = "/tmp"
-            else:
-                plasma_dir = "/dev/shm"
-            plasma_setup_params = dict(
-                store_memory=10 * 1024 * 1024,
-                plasma_directory=plasma_dir,
-                check_dir_size=False,
-            )
-            if _is_windows:
-                storage_configs = {"shared_memory": {}}
-            else:
-                storage_configs = {
-                    "plasma": plasma_setup_params,
-                }
+            storage_configs = {"shared_memory": {}}
 
         storage_handler_cls = kwargs.pop("storage_handler_cls", StorageHandlerActor)
         await mo.create_actor(

mars/services/storage/tests/test_api.py

@@ -17,6 +17,7 @@
 
 import numpy as np
 import pandas as pd
+import pyarrow as pa
 import pytest
 
 from .... import oscar as mo
@@ -47,7 +48,7 @@
 plasma_setup_params = dict(
     store_memory=plasma_storage_size, plasma_directory=plasma_dir, check_dir_size=False
 )
-if not sys.platform.lower().startswith("win"):
+if not (sys.platform.lower().startswith("win") or pa.__version__ >= "12.0.0"):
     storage_configs.append({"plasma": plasma_setup_params})
 
 # ray backend

mars/services/storage/tests/test_service.py

@@ -59,19 +59,10 @@ async def start_pool():
 async def test_storage_service(actor_pools):
     worker_pool = actor_pools
 
-    if sys.platform == "darwin":
-        plasma_dir = "/tmp"
-    else:
-        plasma_dir = "/dev/shm"
-    plasma_setup_params = dict(
-        store_memory=10 * 1024 * 1024, plasma_directory=plasma_dir, check_dir_size=False
-    )
-
     config = {
         "services": ["storage"],
         "storage": {
-            "backends": ["plasma" if not _is_windows else "shared_memory"],
-            "plasma": plasma_setup_params,
+            "backends": ["shared_memory"],
         },
     }
 
@@ -155,19 +146,11 @@ async def test_storage_service_with_cuda(actor_pools_with_gpu):
 
     worker_pool = actor_pools_with_gpu
 
-    if sys.platform == "darwin":
-        plasma_dir = "/tmp"
-    else:
-        plasma_dir = "/dev/shm"
-    plasma_setup_params = dict(
-        store_memory=10 * 1024 * 1024, plasma_directory=plasma_dir, check_dir_size=False
-    )
-
     config = {
         "services": ["storage"],
         "storage": {
-            "backends": ["plasma" if not _is_windows else "shared_memory", "cuda"],
-            "plasma": plasma_setup_params,
+            "backends": ["shared_memory", "cuda"],
+            "shared_memory": dict(),
             "cuda": dict(),
         },
     }

mars/services/storage/tests/test_spill.py

@@ -18,6 +18,7 @@
 import tempfile
 
 import numpy as np
+import pyarrow
 import pytest
 
 from .... import oscar as mo
@@ -88,6 +89,12 @@ async def create_actors(actor_pool):
     await mo.destroy_actor(manager_ref)
 
 
+@pytest.fixture(autouse=True)
+async def skip_wihtout_plasma():
+    if pyarrow.__version__ >= "12.0.0":
+        pytest.skip("Pyarrow.Plasma is deprecated since v12.0.0")
+
+
 @pytest.mark.asyncio
 async def test_spill(create_actors):
     worker_address, _, _ = create_actors