Develop hsc (#58)

feature: 优化镜像构建/部署

Author: hhhhsc701

.github/workflows/docker-image-backend-python.yml

@@ -0,0 +1,27 @@
+name: Python Backend Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'backend/**'
+      - 'scripts/images/datamate-python/**'
+      - '.github/workflows/docker-image-backend-python.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'backend/**'
+      - 'scripts/images/datamate-python/**'
+      - '.github/workflows/docker-image-backend-python.yml'
+  workflow_dispatch:
+
+jobs:
+  call-docker-build:
+    name: Build and Push Backend Python Docker Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    permissions:
+      contents: read
+      packages: write
+    with:
+      service_name: backend-python
+      build_dir: .

Makefile

@@ -3,6 +3,7 @@ MAKEFLAGS += --no-print-directory
 WITH_MINERU ?= false  # 默认不构建mineru
 VERSION ?= latest
 NAMESPACE ?= datamate
+REGISTRY ?= "ghcr.io/modelengine-group/"
 
 ifdef COMSPEC
     # Windows 环境
@@ -17,7 +18,7 @@ build-%:
 	$(MAKE) $*-docker-build
 
 .PHONY: build
-build: backend-docker-build frontend-docker-build runtime-docker-build $(if $(WITH_MINERU),mineru-docker-build)
+build: backend-docker-build frontend-docker-build runtime-docker-build
 
 .PHONY: create-namespace
 create-namespace:
@@ -42,7 +43,24 @@ else
 endif
 
 .PHONY: install
-install: install-datamate
+install:
+ifeq ($(origin INSTALLER), undefined)
+	@echo "Choose a deployment method:"
+	@echo "1. Docker/Docker-Compose"
+	@echo "2. Kubernetes/Helm"
+	@echo -n "Enter choice: "
+	@read choice; \
+	case $$choice in \
+		1) INSTALLER=docker ;; \
+		2) INSTALLER=k8s ;; \
+		*) echo "Invalid choice" && exit 1 ;; \
+	esac; \
+	$(MAKE) datamate-$$INSTALLER-install; \
+	$(MAKE) milvus-$$INSTALLER-install
+else
+	$(MAKE) datamate-$(INSTALLER)-install; \
+	$(MAKE) milvus-$(INSTALLER)-install
+endif
 
 .PHONY: uninstall-%
 uninstall-%:
@@ -63,7 +81,24 @@ else
 endif
 
 .PHONY: uninstall
-uninstall: uninstall-datamate
+uninstall:
+ifeq ($(origin INSTALLER), undefined)
+	@echo "Choose a deployment method:"
+	@echo "1. Docker/Docker-Compose"
+	@echo "2. Kubernetes/Helm"
+	@echo -n "Enter choice: "
+	@read choice; \
+	case $$choice in \
+		1) INSTALLER=docker ;; \
+		2) INSTALLER=k8s ;; \
+		*) echo "Invalid choice" && exit 1 ;; \
+	esac; \
+    $(MAKE) milvus-$$INSTALLER-uninstall; \
+	$(MAKE) datamate-$$INSTALLER-uninstall
+else
+	$(MAKE) milvus-$(INSTALLER)-uninstall; \
+	$(MAKE) datamate-$(INSTALLER)-uninstall
+endif
 
 # build
 .PHONY: backend-docker-build
@@ -92,6 +127,11 @@ deer-flow-docker-build:
 .PHONY: mineru-docker-build
 mineru-docker-build:
 	docker build -t datamate-mineru:$(VERSION) . -f scripts/images/mineru/Dockerfile
+
+.PHONY: backend-python-docker-build
+backend-python-docker-build:
+	docker build -t datamate-backend-python:$(VERSION) . -f scripts/images/datamate-python/Dockerfile
+
 .PHONY: backend-docker-install
 backend-docker-install:
 	cd deployment/docker/datamate && docker compose up -d backend
@@ -118,7 +158,7 @@ runtime-docker-uninstall:
 
 .PHONY: mineru-docker-install
 mineru-docker-install:
-	cd deployment/docker/datamate && cp .env.example .env && docker compose up -d datamate-mineru
+	cd deployment/docker/datamate && export REGISTRY=$(REGISTRY) && docker compose up -d datamate-mineru
 
 .PHONY: mineru-docker-uninstall
 mineru-docker-uninstall:
@@ -134,31 +174,40 @@ mineru-k8s-uninstall:
 
 .PHONY: datamate-docker-install
 datamate-docker-install:
-	cd deployment/docker/datamate && cp .env.example .env && docker compose -f docker-compose.yml up -d
+	cd deployment/docker/datamate && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d
 
 .PHONY: datamate-docker-uninstall
 datamate-docker-uninstall:
-	cd deployment/docker/datamate && docker compose -f docker-compose.yml down -v
+	cd deployment/docker/datamate && docker compose -f docker-compose.yml --profile mineru down -v
 
 .PHONY: deer-flow-docker-install
 deer-flow-docker-install:
-	cd deployment/docker/datamate && cp .env.deer-flow.example .env && docker compose -f docker-compose.yml up -d
+	cd deployment/docker/datamate && export NGINX_CONF="./backend-with-deer-flow.conf" && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d
 	cp -n runtime/deer-flow/.env.example runtime/deer-flow/.env
 	cp -n runtime/deer-flow/conf.yaml.example runtime/deer-flow/conf.yaml
-	cp runtime/deer-flow/.env deployment/docker/deer-flow/.env && cp runtime/deer-flow/conf.yaml deployment/docker/deer-flow/conf.yaml
-	cd deployment/docker/deer-flow && docker compose -f docker-compose.yml up -d
+	cp runtime/deer-flow/.env deployment/docker/deer-flow/.env
+	cp runtime/deer-flow/conf.yaml deployment/docker/deer-flow/conf.yaml
+	cd deployment/docker/deer-flow && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d
 
 .PHONY: deer-flow-docker-uninstall
 deer-flow-docker-uninstall:
 	@if docker compose ls --filter name=datamate | grep -q datamate; then \
-		cd deployment/docker/datamate && docker compose -f docker-compose.yml up -d; \
+		cd deployment/docker/datamate && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d; \
 	fi
 	cd deployment/docker/deer-flow && docker compose -f docker-compose.yml down
 
+.PHONY: milvus-docker-install
+milvus-docker-install:
+	cd deployment/docker/milvus && docker compose -f docker-compose.yml up -d
+
+.PHONY: milvus-docker-uninstall
+milvus-docker-uninstall:
+	cd deployment/docker/milvus && docker compose -f docker-compose.yml down
+
 .PHONY: datamate-k8s-install
 datamate-k8s-install: create-namespace
 	kubectl create configmap datamate-init-sql --from-file=scripts/db/ --dry-run=client -o yaml | kubectl apply -f - -n $(NAMESPACE)
-	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install
+	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REPOSITORY)
 
 .PHONY: datamate-k8s-uninstall
 datamate-k8s-uninstall:
@@ -167,14 +216,17 @@ datamate-k8s-uninstall:
 
 .PHONY: deer-flow-k8s-install
 deer-flow-k8s-install:
-	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.deerFlow.enable=true
+	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.deerFlow.enable=true --set global.image.repository=$(REPOSITORY)
 	cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env
 	cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml
-	helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install
+	helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REPOSITORY)
 
 .PHONY: deer-flow-k8s-uninstall
 deer-flow-k8s-uninstall:
 	helm uninstall deer-flow -n $(NAMESPACE) --ignore-not-found
+	@if helm ls -n $(NAMESPACE) --filter datamate | grep -q datamate; then \
+		helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --set global.deerFlow.enable=false; \
+	fi
 
 .PHONY: milvus-k8s-install
 milvus-k8s-install:

deployment/docker/datamate/.env.deer-flow.example

@@ -9,6 +9,20 @@ server {
 
     add_header Set-Cookie "NEXT_LOCALE=zh";
 
+    location /api/synthesis/ {
+        proxy_pass http://datamate-backend-python:18000/api/synthesis/;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
+    location /api/annotation/ {
+        proxy_pass http://datamate-backend-python:18000/api/annotation/;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
     location /api/ {
         proxy_pass http://datamate-backend:8080/api/;
         proxy_set_header Host $host;

deployment/docker/datamate/.env.example

@@ -2,7 +2,7 @@ services:
   # 1) backend
   datamate-backend:
     container_name: datamate-backend
-    image: datamate-backend
+    image: ${REGISTRY:-}datamate-backend
     restart: on-failure
     privileged: true
     volumes:
@@ -18,7 +18,7 @@ services:
   # 1) backend (Python)
   datamate-backend-python:
     container_name: datamate-backend-python
-    image: datamate-backend-python
+    image: ${REGISTRY:-}datamate-backend-python
     restart: on-failure
     privileged: true
     environment:
@@ -34,13 +34,13 @@ services:
   # 2) frontend（NodePort 30000）
   datamate-frontend:
     container_name: datamate-frontend
-    image: datamate-frontend
+    image: ${REGISTRY:-}datamate-frontend
     restart: on-failure
     ports:
       - "30000:80"            # nodePort → hostPort
     volumes:
       - frontend_log_volume:/var/log/datamate/frontend
-      - $NGINX_CONF:/etc/nginx/conf.d/backend.conf
+      - ${NGINX_CONF:-./backend.conf}:/etc/nginx/conf.d/backend.conf
     networks: [ datamate ]
     depends_on:
       - datamate-backend
@@ -71,7 +71,7 @@ services:
   # 3) runtime
   datamate-runtime:
     container_name: datamate-runtime
-    image: datamate-runtime
+    image: ${REGISTRY:-}datamate-runtime
     restart: on-failure
     environment:
       RAY_DEDUP_LOGS: "0"
@@ -81,7 +81,6 @@ services:
       MYSQL_USER: "root"
       MYSQL_PASSWORD: "password"
       MYSQL_DATABASE: "datamate"
-      PDF_FORMATTER_BASE_URL: "http://datamate-mineru:9001"
     command:
       - python
       - /opt/runtime/datamate/operator_runtime.py
@@ -135,6 +134,8 @@ volumes:
     name: datamate-operator-upload-volume
   operator-runtime-volume:
     name: datamate-operator-runtime-volume
+  mineru_log_volume:
+    name: datamate-mineru_log_volume
 
 networks:
   datamate:

deployment/docker/datamate/backend-with-deer-flow.conf

@@ -1,6 +1,6 @@
 services:
   deer-flow-backend:
-    image: deer-flow-backend
+    image: ${REGISTRY:-}deer-flow-backend
     container_name: deer-flow-backend
     env_file:
       - .env
@@ -11,7 +11,7 @@ services:
       - datamate
 
   deer-flow-frontend:
-    image: deer-flow-frontend
+    image: ${REGISTRY:-}deer-flow-frontend
     container_name: deer-flow-frontend
     env_file:
       - .env

deployment/docker/datamate/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.5'
-
 services:
   etcd:
     container_name: milvus-etcd

deployment/docker/deer-flow/docker-compose.yml

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/docker/milvus/docker-compose.yml

@@ -0,0 +1,29 @@
+apiVersion: v2
+name: backend-python
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"
+
+dependencies:
+  - name: database
+    repository: file://../database
+    version: 0.0.1