Merge remote-tracking branch 'origin/develop_930' into develop_930

Author: hefanli

Makefile

@@ -13,14 +13,12 @@ build: backend-docker-build frontend-docker-build runtime-docker-build
 install-%:
 	@echo "Choose a deployment method:"
 	@echo "1. Docker"
-	@echo "2. Kubernetes"
-	@echo "3. Helm"
+	@echo "2. Kubernetes/Helm"
 	@echo -n "Enter choice: "
 	@read choice; \
 	case $$choice in \
 		1) INSTALLER=docker ;; \
 		2) INSTALLER=k8s ;; \
-		3) INSTALLER=helm ;; \
 		*) echo "Invalid choice" && exit 1 ;; \
 	esac; \
 	$(MAKE) $*-$$INSTALLER-install
@@ -32,14 +30,12 @@ install: install-data-platform
 uninstall-%:
 	@echo "Choose a deployment method:"
 	@echo "1. Docker"
-	@echo "2. Kubernetes"
-	@echo "3. Helm"
+	@echo "2. Kubernetes/Helm"
 	@echo -n "Enter choice: "
 	@read choice; \
 	case $$choice in \
 		1) INSTALLER=docker ;; \
 		2) INSTALLER=k8s ;; \
-		3) INSTALLER=helm ;; \
 		*) echo "Invalid choice" && exit 1 ;; \
 	esac; \
     $(MAKE) $*-$$INSTALLER-uninstall
@@ -92,22 +88,33 @@ frontend-docker-install:
 frontend-docker-uninstall:
 	cd deployment/docker/data-platform && docker-compose down frontend
 
-.PHONY: runtime-helm-install
-runtime-helm-install:
-	helm repo add kuberay https://ray-project.github.io/kuberay-helm/
-	helm repo update
-	helm upgrade kuberay-operator kuberay/kuberay-operator --version 1.4.0 --install
+.PHONY: runtime-docker-install
+runtime-docker-install:
+	cd deployment/docker/data-platform && docker-compose up -d runtime
+
+.PHONY: runtime-docker-uninstall
+runtime-docker-uninstall:
+	cd deployment/docker/data-platform && docker-compose down runtime
+
+.PHONY: runtime-k8s-install
+runtime-k8s-install:
+	helm upgrade kuberay-operator deployment/helm/ray/kuberay-operator --install
 	helm upgrade raycluster deployment/helm/ray/ray-cluster/ --install
 	kubectl apply -f deployment/helm/ray/service.yaml
 
+.PHONY: runtime-k8s-uninstall
+runtime-k8s-uninstall:
+	helm uninstall raycluster
+	helm uninstall kuberay-operator
+	kubectl delete -f deployment/helm/ray/service.yaml
+
 .PHONY: unstructured-k8s-install
 unstructured-k8s-install:
 	kubectl apply -f deployment/kubernetes/unstructured/deploy.yaml
 
 .PHONY: mysql-k8s-install
 mysql-k8s-install:
-	kubectl create configmap init-sql --from-file=scripts/db/ --dry-run=client -o yaml > deployment/kubernetes/mysql/init-sql.yaml
-	kubectl apply -f deployment/kubernetes/mysql/init-sql.yaml
+	kubectl create configmap init-sql --from-file=scripts/db/ --dry-run=client -o yaml | kubectl apply -f -
 	kubectl apply -f deployment/kubernetes/mysql/deploy.yaml
 
 .PHONY: mysql-k8s-uninstall
@@ -140,7 +147,7 @@ data-platform-docker-uninstall:
 	cd deployment/docker/data-platform && docker-compose down
 
 .PHONY: data-platform-k8s-install
-data-platform-k8s-install: mysql-k8s-install backend-k8s-install frontend-k8s-install runtime-helm-install
+data-platform-k8s-install: mysql-k8s-install backend-k8s-install frontend-k8s-install runtime-k8s-install
 
 .PHONY: data-platform-k8s-uninstall
-data-platform-k8s-uninstall: mysql-k8s-uninstall backend-k8s-uninstall frontend-k8s-uninstall
+data-platform-k8s-uninstall: mysql-k8s-uninstall backend-k8s-uninstall frontend-k8s-uninstall runtime-k8s-uninstall

deployment/docker/data-platform/.env

@@ -1,4 +1,6 @@
-DATASET_VOLUME=/opt/data-platform/data/dataset
-FLOW_VOLUME=/opt/data-platform/data/flow
-MYSQL_VOLUME=/opt/data-platform/data/mysql
-LOG_VOLUME=/opt/data-platform/data/log
+BASE_VOLUME=../../../data
+DATASET_VOLUME=/dataset
+FLOW_VOLUME=/flow
+MYSQL_VOLUME=/mysql
+LOG_VOLUME=/log
+RAY_LOG_VOLUME=/log/ray

deployment/docker/data-platform/docker-compose.yml

@@ -6,9 +6,9 @@ services:
     ports:
       - "8080:8080"
     volumes:
-      - ${DATASET_VOLUME}:/dataset
-      - ${FLOW_VOLUME}:/flow
-      - ${LOG_VOLUME}:/var/log/data-platform
+      - ${BASE_VOLUME}${DATASET_VOLUME}:/dataset
+      - ${BASE_VOLUME}${FLOW_VOLUME}:/flow
+      - ${BASE_VOLUME}${LOG_VOLUME}:/var/log/data-platform
     networks: [ edatamate ]
 
   # 2) frontend（NodePort 30000）
@@ -18,7 +18,7 @@ services:
     ports:
       - "30000:80"            # nodePort → hostPort
     volumes:
-      - ${LOG_VOLUME}:/var/log/data-platform
+      - ${BASE_VOLUME}${LOG_VOLUME}:/var/log/data-platform
     networks: [ edatamate ]
 
   # 3) mysql
@@ -30,11 +30,35 @@ services:
     ports:
       - "3306:3306"
     volumes:
-      - ${MYSQL_VOLUME}:/var/lib/mysql
+      - ${BASE_VOLUME}${MYSQL_VOLUME}:/var/lib/mysql
       - ../../../scripts/db:/docker-entrypoint-initdb.d
-      - ${LOG_VOLUME}:/var/log/data-platform
+      - ${BASE_VOLUME}${LOG_VOLUME}:/var/log/data-platform
     networks: [ edatamate ]
 
+  runtime:
+    container_name: runtime
+    image: runtime
+    environment:
+      RAY_DEDUP_LOGS: "0"
+      RAY_TQDM_PATCH_PRINT: "0"
+      MYSQL_HOST: "mysql"
+      MYSQL_PORT: "3306"
+      MYSQL_USER: "root"
+      MYSQL_PASSWORD: "Huawei@123"
+      MYSQL_DATABASE: "dataengine"
+    ports:
+      - "8081:8081"
+    command:
+      - python
+      - /opt/runtime/data_platform/operator_runtime.py
+      - --port
+      - "8081"
+    volumes:
+      - ${BASE_VOLUME}${RAY_LOG_VOLUME}:/tmp/ray
+      - ${BASE_VOLUME}${LOG_VOLUME}:/var/log/data-platform
+      - ${BASE_VOLUME}${DATASET_VOLUME}:/dataset
+      - ${BASE_VOLUME}${FLOW_VOLUME}:/flow
+
 networks:
   edatamate:
     driver: bridge

deployment/helm/ray/kuberay-operator/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+
+name: kuberay-operator
+
+description: A Helm chart for deploying the Kuberay operator on Kubernetes.
+
+version: 1.4.2
+
+type: application
+
+keywords:
+- ray
+- ray operator
+- distributed computing
+- data processing
+- machine learning
+- deep learning
+- hyperparameter tuning
+- reinforcement learning
+- model serving
+
+home: https://github.com/ray-project/kuberay
+
+icon: https://github.com/ray-project/ray/raw/master/doc/source/images/ray_header_logo.png