Skip to content

Commit 22ea84c

Browse files
hhhhsc701hhhhsc701
committed
feature: 优化镜像构建/部署
1 parent c516dfb commit 22ea84c

File tree

18 files changed

+486
-40
lines changed

18 files changed

+486
-40
lines changed

‎Makefile‎

Lines changed: 44 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ build-%:
1818
$(MAKE) $*-docker-build
1919

2020
.PHONY: build
21-
build: backend-docker-build frontend-docker-build runtime-docker-build $(if $(WITH_MINERU),mineru-docker-build)
21+
build: backend-docker-build frontend-docker-build runtime-docker-build
2222

2323
.PHONY: create-namespace
2424
create-namespace:
@@ -43,7 +43,24 @@ else
4343
endif
4444

4545
.PHONY: install
46-
install: install-datamate install-milvus
46+
install:
47+
ifeq ($(origin INSTALLER), undefined)
48+
@echo "Choose a deployment method:"
49+
@echo "1. Docker/Docker-Compose"
50+
@echo "2. Kubernetes/Helm"
51+
@echo -n "Enter choice: "
52+
@read choice; \
53+
case $$choice in \
54+
1) INSTALLER=docker ;; \
55+
2) INSTALLER=k8s ;; \
56+
*) echo "Invalid choice" && exit 1 ;; \
57+
esac; \
58+
$(MAKE) datamate-$$INSTALLER-install; \
59+
$(MAKE) milvus-$$INSTALLER-install
60+
else
61+
$(MAKE) datamate-$(INSTALLER)-install; \
62+
$(MAKE) milvus-$(INSTALLER)-install
63+
endif
4764

4865
.PHONY: uninstall-%
4966
uninstall-%:
@@ -64,7 +81,24 @@ else
6481
endif
6582

6683
.PHONY: uninstall
67-
uninstall: uninstall-datamate
84+
uninstall:
85+
ifeq ($(origin INSTALLER), undefined)
86+
@echo "Choose a deployment method:"
87+
@echo "1. Docker/Docker-Compose"
88+
@echo "2. Kubernetes/Helm"
89+
@echo -n "Enter choice: "
90+
@read choice; \
91+
case $$choice in \
92+
1) INSTALLER=docker ;; \
93+
2) INSTALLER=k8s ;; \
94+
*) echo "Invalid choice" && exit 1 ;; \
95+
esac; \
96+
$(MAKE) milvus-$$INSTALLER-uninstall; \
97+
$(MAKE) datamate-$$INSTALLER-uninstall
98+
else
99+
$(MAKE) milvus-$(INSTALLER)-uninstall; \
100+
$(MAKE) datamate-$(INSTALLER)-uninstall
101+
endif
68102

69103
# build
70104
.PHONY: backend-docker-build
@@ -144,20 +178,21 @@ datamate-docker-install:
144178

145179
.PHONY: datamate-docker-uninstall
146180
datamate-docker-uninstall:
147-
cd deployment/docker/datamate && docker compose -f docker-compose.yml down -v
181+
cd deployment/docker/datamate && docker compose -f docker-compose.yml --profile mineru down -v
148182

149183
.PHONY: deer-flow-docker-install
150184
deer-flow-docker-install:
151185
cd deployment/docker/datamate && export NGINX_CONF="./backend-with-deer-flow.conf" && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d
152186
cp -n runtime/deer-flow/.env.example runtime/deer-flow/.env
153187
cp -n runtime/deer-flow/conf.yaml.example runtime/deer-flow/conf.yaml
154-
cp runtime/deer-flow/.env deployment/docker/deer-flow/.env && cp runtime/deer-flow/conf.yaml deployment/docker/deer-flow/conf.yaml
188+
cp runtime/deer-flow/.env deployment/docker/deer-flow/.env
189+
cp runtime/deer-flow/conf.yaml deployment/docker/deer-flow/conf.yaml
155190
cd deployment/docker/deer-flow && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d
156191

157192
.PHONY: deer-flow-docker-uninstall
158193
deer-flow-docker-uninstall:
159194
@if docker compose ls --filter name=datamate | grep -q datamate; then \
160-
cd deployment/docker/datamate && docker compose -f docker-compose.yml up -d; \
195+
cd deployment/docker/datamate && export REGISTRY=$(REGISTRY) && docker compose -f docker-compose.yml up -d; \
161196
fi
162197
cd deployment/docker/deer-flow && docker compose -f docker-compose.yml down
163198

@@ -189,6 +224,9 @@ deer-flow-k8s-install:
189224
.PHONY: deer-flow-k8s-uninstall
190225
deer-flow-k8s-uninstall:
191226
helm uninstall deer-flow -n $(NAMESPACE) --ignore-not-found
227+
@if helm ls -n $(NAMESPACE) --filter datamate | grep -q datamate; then \
228+
helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --set global.deerFlow.enable=false; \
229+
fi
192230

193231
.PHONY: milvus-k8s-install
194232
milvus-k8s-install:

‎deployment/docker/datamate/backend-with-deer-flow.conf‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,20 @@ server {
99

1010
add_header Set-Cookie "NEXT_LOCALE=zh";
1111

12+
location /api/synthesis/ {
13+
proxy_pass http://datamate-backend-python:18000/api/synthesis/;
14+
proxy_set_header Host $host;
15+
proxy_set_header X-Real-IP $remote_addr;
16+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
17+
}
18+
19+
location /api/annotation/ {
20+
proxy_pass http://datamate-backend-python:18000/api/annotation/;
21+
proxy_set_header Host $host;
22+
proxy_set_header X-Real-IP $remote_addr;
23+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
24+
}
25+
1226
location /api/ {
1327
proxy_pass http://datamate-backend:8080/api/;
1428
proxy_set_header Host $host;

‎deployment/docker/datamate/docker-compose.yml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ services:
22
# 1) backend
33
datamate-backend:
44
container_name: datamate-backend
5-
image: ${REGISTRY}datamate-backend
5+
image: ${REGISTRY:-""}datamate-backend
66
restart: on-failure
77
privileged: true
88
volumes:
@@ -18,7 +18,7 @@ services:
1818
# 1) backend (Python)
1919
datamate-backend-python:
2020
container_name: datamate-backend-python
21-
image: ${REGISTRY}datamate-backend-python
21+
image: ${REGISTRY:-""}datamate-backend-python
2222
restart: on-failure
2323
privileged: true
2424
environment:
@@ -34,7 +34,7 @@ services:
3434
# 2) frontend（NodePort 30000）
3535
datamate-frontend:
3636
container_name: datamate-frontend
37-
image: ${REGISTRY}datamate-frontend
37+
image: ${REGISTRY:-""}datamate-frontend
3838
restart: on-failure
3939
ports:
4040
- "30000:80" # nodePort → hostPort
@@ -71,7 +71,7 @@ services:
7171
# 3) runtime
7272
datamate-runtime:
7373
container_name: datamate-runtime
74-
image: ${REGISTRY}datamate-runtime
74+
image: ${REGISTRY:-""}datamate-runtime
7575
restart: on-failure
7676
environment:
7777
RAY_DEDUP_LOGS: "0"
@@ -81,7 +81,6 @@ services:
8181
MYSQL_USER: "root"
8282
MYSQL_PASSWORD: "password"
8383
MYSQL_DATABASE: "datamate"
84-
PDF_FORMATTER_BASE_URL: "http://datamate-mineru:9001"
8584
command:
8685
- python
8786
- /opt/runtime/datamate/operator_runtime.py
@@ -135,6 +134,8 @@ volumes:
135134
name: datamate-operator-upload-volume
136135
operator-runtime-volume:
137136
name: datamate-operator-runtime-volume
137+
mineru_log_volume:
138+
name: datamate-mineru_log_volume
138139

139140
networks:
140141
datamate:

‎deployment/docker/deer-flow/docker-compose.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
services:
22
deer-flow-backend:
3-
image: ${REGISTRY}deer-flow-backend
3+
image: ${REGISTRY:-""}deer-flow-backend
44
container_name: deer-flow-backend
55
env_file:
66
- .env
@@ -11,7 +11,7 @@ services:
1111
- datamate
1212

1313
deer-flow-frontend:
14-
image: ${REGISTRY}deer-flow-frontend
14+
image: ${REGISTRY:-""}deer-flow-frontend
1515
container_name: deer-flow-frontend
1616
env_file:
1717
- .env

‎deployment/docker/milvus/docker-compose.yml‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
version: '3.5'
2-
31
services:
42
etcd:
53
container_name: milvus-etcd

‎deployment/helm/datamate/charts/backend-python/.helmignore‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Patterns to ignore when building packages.
2+
# This supports shell glob matching, relative path matching, and
3+
# negation (prefixed with !). Only one pattern per line.
4+
.DS_Store
5+
# Common VCS dirs
6+
.git/
7+
.gitignore
8+
.bzr/
9+
.bzrignore
10+
.hg/
11+
.hgignore
12+
.svn/
13+
# Common backup files
14+
*.swp
15+
*.bak
16+
*.tmp
17+
*.orig
18+
*~
19+
# Various IDEs
20+
.project
21+
.idea/
22+
*.tmproj
23+
.vscode/

‎deployment/helm/datamate/charts/backend-python/Chart.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
apiVersion: v2
2+
name: backend-python
3+
description: A Helm chart for Kubernetes
4+
5+
# A chart can be either an 'application' or a 'library' chart.
6+
#
7+
# Application charts are a collection of templates that can be packaged into versioned archives
8+
# to be deployed.
9+
#
10+
# Library charts provide useful utilities or functions for the chart developer. They're included as
11+
# a dependency of application charts to inject those utilities and functions into the rendering
12+
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
13+
type: application
14+
15+
# This is the chart version. This version number should be incremented each time you make changes
16+
# to the chart and its templates, including the app version.
17+
# Versions are expected to follow Semantic Versioning (https://semver.org/)
18+
version: 0.0.1
19+
20+
# This is the version number of the application being deployed. This version number should be
21+
# incremented each time you make changes to the application. Versions are not expected to
22+
# follow Semantic Versioning. They should reflect the version the application is using.
23+
# It is recommended to use it with quotes.
24+
appVersion: "0.0.1"
25+
26+
dependencies:
27+
- name: database
28+
repository: file://../database
29+
version: 0.0.1

‎deployment/helm/datamate/charts/backend-python/templates/_helpers.tpl‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
{{/*
2+
Expand the name of the chart.
3+
*/}}
4+
{{- define "backend-python.name" -}}
5+
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
6+
{{- end }}
7+
8+
{{/*
9+
Create a default fully qualified app name.
10+
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
11+
If release name contains chart name it will be used as a full name.
12+
*/}}
13+
{{- define "backend-python.fullname" -}}
14+
{{- if .Values.fullnameOverride }}
15+
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
16+
{{- else }}
17+
{{- $name := default .Chart.Name .Values.nameOverride }}
18+
{{- if contains $name .Release.Name }}
19+
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
20+
{{- else }}
21+
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
22+
{{- end }}
23+
{{- end }}
24+
{{- end }}
25+
26+
{{/*
27+
Create chart name and version as used by the chart label.
28+
*/}}
29+
{{- define "backend-python.chart" -}}
30+
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
31+
{{- end }}
32+
33+
{{/*
34+
Common labels
35+
*/}}
36+
{{- define "backend-python.labels" -}}
37+
helm.sh/chart: {{ include "backend-python.chart" . }}
38+
{{ include "backend-python.selectorLabels" . }}
39+
{{- if .Chart.AppVersion }}
40+
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
41+
{{- end }}
42+
app.kubernetes.io/managed-by: {{ .Release.Service }}
43+
{{- end }}
44+
45+
{{/*
46+
Selector labels
47+
*/}}
48+
{{- define "backend-python.selectorLabels" -}}
49+
app.kubernetes.io/name: {{ include "backend-python.name" . }}
50+
app.kubernetes.io/instance: {{ .Release.Name }}
51+
{{- end }}
52+
53+
{{/*
54+
Create the name of the service account to use
55+
*/}}
56+
{{- define "backend-python.serviceAccountName" -}}
57+
{{- if .Values.serviceAccount.create }}
58+
{{- default (include "backend-python.fullname" .) .Values.serviceAccount.name -}}
59+
{{- else }}
60+
{{- default "default" .Values.serviceAccount.name -}}
61+
{{- end }}
62+
{{- end }}
63+
64+
{{/*
65+
Name of image
66+
*/}}
67+
{{- define "backend-python.image" -}}
68+
{{- $name := default .Values.image.repository .Values.global.image.backendPython.name }}
69+
{{- $tag := default .Values.image.tag .Values.global.image.backendPython.tag }}
70+
{{- if .Values.global.image.repository }}
71+
{{- .Values.global.image.repository | trimSuffix "/" }}/{{ $name }}:{{ $tag }}
72+
{{- else }}
73+
{{- $name }}:{{ $tag }}
74+
{{- end }}
75+
{{- end }}

0 commit comments

Comments
 (0)