feature: 数据库镜像制作 (#70)

* feature: 数据库镜像制作

* feature: 增加归档包流水线

Author: hhhhsc701

.github/workflows/docker-image-backend-python.yml

@@ -14,6 +14,7 @@ on:
       - 'scripts/images/datamate-python/**'
       - '.github/workflows/docker-image-backend-python.yml'
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   call-docker-build:

.github/workflows/docker-image-backend.yml

@@ -14,6 +14,7 @@ on:
       - 'scripts/images/backend/**'
       - '.github/workflows/docker-image-backend.yml'
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   call-docker-build:

.github/workflows/docker-image-database.yml

@@ -0,0 +1,28 @@
+name: Database Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'scripts/db/**'
+      - 'scripts/images/database/**'
+      - '.github/workflows/docker-image-database.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'scripts/db/**'
+      - 'scripts/images/database/**'
+      - '.github/workflows/docker-image-database.yml'
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  call-docker-build:
+    name: Build and Push Database Docker Image
+    uses: ./.github/workflows/docker-images-reusable.yml
+    permissions:
+      contents: read
+      packages: write
+    with:
+      service_name: database
+      build_dir: .

.github/workflows/docker-image-deer-flow.yml

@@ -16,6 +16,7 @@ on:
       - 'scripts/images/deer-flow-frontend/**'
       - '.github/workflows/docker-image-deer-flow.yml'
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   build-and-push:

.github/workflows/docker-image-frontend.yml

@@ -14,6 +14,7 @@ on:
       - 'scripts/images/frontend/**'
       - '.github/workflows/docker-image-frontend.yml'
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   call-docker-build:

.github/workflows/docker-image-runtime.yml

@@ -4,16 +4,19 @@ on:
   push:
     branches: [ "main" ]
     paths:
-      - 'runtime/**'
+      - 'runtime/ops/**'
+      - 'runtime/python-executor/**'
       - 'scripts/images/runtime/**'
       - '.github/workflows/docker-image-runtime.yml'
   pull_request:
     branches: [ "main" ]
     paths:
-      - 'runtime/**'
+      - 'runtime/ops/**'
+      - 'runtime/python-executor/**'
       - 'scripts/images/runtime/**'
       - '.github/workflows/docker-image-runtime.yml'
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   call-docker-build:

.github/workflows/docker-images-reusable.yml

@@ -51,7 +51,18 @@ jobs:
           docker tag datamate-${{ inputs.service_name }}:latest ${{ steps.set-tag.outputs.TAGS }}
 
       - name: Push Docker Image
-        if: github.event_name != 'pull_request'
+        if: github.event_name != 'pull_request' && !startsWith(github.workflow, 'Package')
         run: |
           docker push ${{ steps.set-tag.outputs.TAGS }}
 
+      - name: Save Docker Image
+        if: startsWith(github.workflow, 'Package')
+        run: |
+          docker save -o datamate-${{ inputs.service_name }}.tar ${{ steps.set-tag.outputs.TAGS }}
+
+      - name: Upload Docker Image
+        if: startsWith(github.workflow, 'Package')
+        uses: actions/upload-artifact@v4
+        with:
+          name: datamate-${{ inputs.service_name }}
+          path: datamate-${{ inputs.service_name }}.tar

.github/workflows/package.yml

@@ -0,0 +1,58 @@
+name: Package All
+
+on:
+  workflow_dispatch:
+
+jobs:
+  backend-docker-build:
+    name: Build and Push Backend Docker Image
+    uses: ./.github/workflows/docker-image-backend.yml
+
+  frontend-docker-build:
+    name: Build and Push Frontend Docker Image
+    uses: ./.github/workflows/docker-image-frontend.yml
+
+  database-docker-build:
+    name: Build and Push Database Docker Image
+    uses: ./.github/workflows/docker-image-database.yml
+
+  runtime-docker-build:
+    name: Build and Push Runtime Docker Image
+    uses: ./.github/workflows/docker-image-runtime.yml
+
+  backend-python-docker-build:
+    name: Build and Push Backend Python Docker Image
+    uses: ./.github/workflows/docker-image-backend-python.yml
+
+  package-all:
+    needs:
+      - backend-docker-build
+      - frontend-docker-build
+      - database-docker-build
+      - backend-python-docker-build
+      - runtime-docker-build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Save Image
+        uses: actions/download-artifact@v5
+        with:
+          path: images
+          pattern: datamate-*
+          merge-multiple: true
+
+      - name: Package
+        run: |
+          cp runtime/deer-flow/.env.example deployment/docker/deer-flow/.env
+          cp runtime/deer-flow/conf.yaml.example deployment/docker/deer-flow/conf.yaml
+          cp runtime/deer-flow/.env.example deployment/helm/deer-flow/charts/public/.env
+          cp runtime/deer-flow/conf.yaml.example deployment/helm/deer-flow/charts/public/conf.yaml
+          tar -cvf datamate.tar deployment/ images
+
+      - name: Upload Package
+        uses: actions/upload-artifact@v4
+        with:
+          name: package
+          path: datamate.tar

Makefile

@@ -18,7 +18,7 @@ build-%:
 	$(MAKE) $*-docker-build
 
 .PHONY: build
-build: backend-docker-build frontend-docker-build runtime-docker-build
+build: database-docker-build backend-docker-build frontend-docker-build runtime-docker-build
 
 .PHONY: create-namespace
 create-namespace:
@@ -105,6 +105,10 @@ endif
 backend-docker-build:
 	docker build -t datamate-backend:$(VERSION) . -f scripts/images/backend/Dockerfile
 
+.PHONY: database-docker-build
+database-docker-build:
+	docker build -t datamate-database:$(VERSION) . -f scripts/images/database/Dockerfile
+
 .PHONY: frontend-docker-build
 frontend-docker-build:
 	docker build -t datamate-frontend:$(VERSION) . -f scripts/images/frontend/Dockerfile
@@ -206,20 +210,18 @@ milvus-docker-uninstall:
 
 .PHONY: datamate-k8s-install
 datamate-k8s-install: create-namespace
-	kubectl create configmap datamate-init-sql --from-file=scripts/db/ --dry-run=client -o yaml | kubectl apply -f - -n $(NAMESPACE)
-	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REPOSITORY)
+	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY)
 
 .PHONY: datamate-k8s-uninstall
 datamate-k8s-uninstall:
 	helm uninstall datamate -n $(NAMESPACE) --ignore-not-found
-	kubectl delete configmap datamate-init-sql -n $(NAMESPACE) --ignore-not-found
 
 .PHONY: deer-flow-k8s-install
 deer-flow-k8s-install:
-	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.deerFlow.enable=true --set global.image.repository=$(REPOSITORY)
+	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.deerFlow.enable=true --set global.image.repository=$(REGISTRY)
 	cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env
 	cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml
-	helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REPOSITORY)
+	helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY)
 
 .PHONY: deer-flow-k8s-uninstall
 deer-flow-k8s-uninstall:

deployment/docker/datamate/docker-compose.yml

@@ -49,7 +49,7 @@ services:
   # 3) database
   datamate-database:
     container_name: datamate-database
-    image: mysql:8
+    image: ${REGISTRY:-}datamate-database
     restart: on-failure
     environment:
       MYSQL_ROOT_PASSWORD: password
@@ -61,8 +61,6 @@ services:
       "
     volumes:
       - mysql_volume:/var/lib/mysql
-      - ../../../scripts/db:/docker-entrypoint-initdb.d
-      - ./utf8.cnf:/etc/mysql/conf.d/utf8.cnf:ro
       - database_log_volume:/var/log/datamate/database
     ports:
       - "3306:3306"