fix(python): refresh channel incase the service becomes available after initial heart beat (#4338)

also bump up rustup toolchain version to 1.76.0 to avoid some rust
packages compilation failure.

Author: siyuan0322

k8s/dockerfiles/graphscope-store.Dockerfile

@@ -14,6 +14,7 @@ COPY --chown=graphscope:graphscope . /home/graphscope/graphscope
 COPY --chown=graphscope:graphscope ./interactive_engine/assembly/src/conf/maven.settings.xml /home/graphscope/.m2/settings.xml
 
 USER graphscope
+RUN rustup toolchain install 1.76.0 && rustup default 1.76.0
 
 RUN cd /home/graphscope/graphscope \
     && . ~/.graphscope_env \

k8s/dockerfiles/interactive.Dockerfile

@@ -18,6 +18,7 @@ RUN cd /home/graphscope/GraphScope/ && \
     else \
         mkdir /home/graphscope/install; \
         . /home/graphscope/.graphscope_env; \
+        rustup toolchain install 1.76.0 && rustup default 1.76.0; \
         make interactive-install BUILD_TYPE="$profile" INSTALL_PREFIX=/home/graphscope/install; \
     fi
 

k8s/internal/Makefile

@@ -110,6 +110,7 @@ graphscope-manylinux2014-py3-nodocker:
 	sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* && \
 	sudo yum install java-11-openjdk-devel -y && \
 	sudo yum remove java-1.8.0-openjdk-devel java-1.8.0-openjdk java-1.8.0-openjdk-headless -y && \
+	rustup toolchain install 1.76.0 && rustup default 1.76.0 && \
 	cd $(WORKING_DIR)/../.. && \
 		if [[ "${PLATFORM}" == "aarch64" ]]; then \
 			export AUDITWHEEL_PLAT=manylinux2014_${PLATFORM}; \

python/graphscope/client/rpc.py

@@ -41,19 +41,23 @@ class GRPCClient(object):
     def __init__(self, launcher, endpoint, reconnect=False):
         """Connect to GRAPE engine at the given :code:`endpoint`."""
         # create the gRPC stub
-        options = [
+        self._options = [
             ("grpc.max_send_message_length", GS_GRPC_MAX_MESSAGE_LENGTH),
             ("grpc.max_receive_message_length", GS_GRPC_MAX_MESSAGE_LENGTH),
             ("grpc.max_metadata_size", GS_GRPC_MAX_MESSAGE_LENGTH),
         ]
+        self._endpoint = endpoint
         self._launcher = launcher
         self._grpc_utils = GRPCUtils()
-        self._channel = grpc.insecure_channel(endpoint, options=options)
-        self._stub = coordinator_service_pb2_grpc.CoordinatorServiceStub(self._channel)
+        self._stub = self._get_stub()
         self._session_id = None
         self._logs_fetching_thread = None
         self._reconnect = reconnect
 
+    def _get_stub(self):
+        channel = grpc.insecure_channel(self._endpoint, options=self._options)
+        return coordinator_service_pb2_grpc.CoordinatorServiceStub(channel)
+
     def waiting_service_ready(self, timeout_seconds=60):
         begin_time = time.time()
         request = message_pb2.HeartBeatRequest()
@@ -76,6 +80,9 @@ def waiting_service_ready(self, timeout_seconds=60):
                     logger.warning("Heart beat analytical engine failed, %s", msg)
                 if time.time() - begin_time >= timeout_seconds:
                     raise ConnectionError(f"Connect coordinator timeout, {msg}")
+                # refresh the channel incase the server became available
+                if e.code() == grpc.StatusCode.UNAVAILABLE:
+                    self._stub = self._get_stub()
                 time.sleep(1)
 
     def connect(self, cleanup_instance=True, dangling_timeout_seconds=60):