优化docker和k8s部署脚本

Author: hhhhsc701

Makefile

@@ -93,8 +93,9 @@ frontend-docker-uninstall:
 runtime-helm-install:
 	helm repo add kuberay https://ray-project.github.io/kuberay-helm/
 	helm repo update
-	helm install kuberay-operator kuberay/kuberay-operator --version 1.4.0
-	helm install raycluster deployment/helm/ray/ray-cluster/
+	helm upgrade kuberay-operator kuberay/kuberay-operator --version 1.4.0 --install
+	helm upgrade raycluster deployment/helm/ray/ray-cluster/ --install
+	kubectl apply -f deployment/helm/ray/service.yaml
 
 .PHONY: unstructured-k8s-install
 unstructured-k8s-install:

backend/services/main-application/pom.xml

@@ -139,6 +139,18 @@
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>${maven.compiler.source}</source>
+                    <target>${maven.compiler.target}</target>
+                    <compilerArgs>
+                        <arg>-parameters</arg>
+                    </compilerArgs>
+                </configuration>
+            </plugin>
+
             <plugin>
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>

deployment/helm/ray/ray-cluster/values.yaml

@@ -106,10 +106,22 @@ head:
       hostPath:
         path: /opt/data-platform/data/log
         type: DirectoryOrCreate
+    - name: dataset-volume
+      hostPath:
+        path: /opt/data-platform/data/dataset
+        type: DirectoryOrCreate
+    - name: flow-volume
+      hostPath:
+        path: /opt/data-platform/data/flow
+        type: DirectoryOrCreate
   volumeMounts:
     - mountPath: /tmp/ray
       name: log-volume
       subPath: ray
+    - mountPath: /dataset
+      name: dataset-volume
+    - mountPath: /flow
+      name: flow-volume
   # sidecarContainers specifies additional containers to attach to the Ray pod.
   # Follows standard K8s container spec.
   sidecarContainers:
@@ -127,6 +139,10 @@ head:
       volumeMounts:
         - mountPath: /var/log/data-platform
           name: log-volume
+        - mountPath: /dataset
+          name: dataset-volume
+        - mountPath: /flow
+          name: flow-volume
   # See docs/guidance/pod-command.md for more details about how to specify
   # container command for head Pod.
   command: []
@@ -210,10 +226,22 @@ worker:
       hostPath:
         path: /opt/data-platform/data/log
         type: DirectoryOrCreate
+    - name: dataset-volume
+      hostPath:
+        path: /opt/data-platform/data/dataset
+        type: DirectoryOrCreate
+    - name: flow-volume
+      hostPath:
+        path: /opt/data-platform/data/flow
+        type: DirectoryOrCreate
   volumeMounts:
     - mountPath: /tmp/ray
       name: log-volume
       subPath: ray
+    - mountPath: /dataset
+      name: dataset-volume
+    - mountPath: /flow
+      name: flow-volume
   # sidecarContainers specifies additional containers to attach to the Ray pod.
   # Follows standard K8s container spec.
   sidecarContainers: []
@@ -291,10 +319,22 @@ additionalWorkerGroups:
         hostPath:
           path: /opt/data-platform/data/log
           type: DirectoryOrCreate
+      - name: dataset-volume
+        hostPath:
+          path: /opt/data-platform/data/dataset
+          type: DirectoryOrCreate
+      - name: flow-volume
+        hostPath:
+          path: /opt/data-platform/data/flow
+          type: DirectoryOrCreate
     volumeMounts:
       - mountPath: /tmp/ray
         name: log-volume
         subPath: ray
+      - mountPath: /dataset
+        name: dataset-volume
+      - mountPath: /flow
+        name: flow-volume
     sidecarContainers: []
     # See docs/guidance/pod-command.md for more details about how to specify
     # container command for worker Pod.

deployment/helm/ray/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: runtime
+  labels:
+    ray.io/node-type: head
+spec:
+  type: ClusterIP
+  ports:
+    - port: 8081
+      targetPort: 8081
+      protocol: TCP
+  selector:
+    ray.io/node-type: head
+

runtime/python-executor/data_platform/operator_runtime.py

@@ -71,7 +71,7 @@ async def query_task_info(request: QueryTaskRequest):
 
 @app.post("/api/task/{task_id}/submit")
 async def submit_task(task_id):
-    config_path = f"/opt/runtime/flow/{task_id}/process.yaml"
+    config_path = f"/flow/{task_id}/process.yaml"
     logger.info("Start submitting job...")
 
     dataset_path = get_from_cfg(task_id, "dataset_path")
@@ -122,7 +122,7 @@ def check_valid_path(file_path):
 
 
 def get_from_cfg(task_id, key):
-    config_path = f"/opt/runtime/flow/{task_id}/process.yaml"
+    config_path = f"/flow/{task_id}/process.yaml"
     if not check_valid_path(config_path):
         logger.error(f"config_path is not existed! please check this path.")
         raise APIException(ErrorCode.FILE_NOT_FOUND_ERROR)