Skip to content

Commit 937ae72

Browse files
Dallas98Dallas98
committed
Merge branch 'refs/heads/main' into dev
2 parents 61230fa + 4712f00 commit 937ae72

File tree

8 files changed

+287
-14
lines changed

8 files changed

+287
-14
lines changed

‎.github/workflows/docker-push-to-huaweicloud-reusable.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: docker-push-to-huaweicloud-reusable.yml
2+
on:
3+
workflow_call:
4+
inputs:
5+
version:
6+
description: 'image version (e.g. latest or v0.0.1-beta)'
7+
required: true
8+
default: 'latest'
9+
type: string
10+
service_name:
11+
description: 'image version (e.g. latest or v0.0.1-beta)'
12+
required: true
13+
default: 'latest'
14+
type: string
15+
secrets:
16+
HUAWEI_CLOUD_SWR_LOGIN_PWD:
17+
description: 'login pwd'
18+
required: true
19+
20+
jobs:
21+
retag-and-push:
22+
name: Retag and Push to Huawei Cloud SWR
23+
runs-on: ubuntu-latest
24+
steps:
25+
- name: Pull source image and retag (amd)
26+
shell: bash
27+
run: |
28+
SOURCE_IMAGE=ghcr.io/modelengine-group/${{ inputs.service_name}}:${{ inputs.version}}
29+
TARGET_IMAGE=swr.cn-east-3.myhuaweicloud.com/data-mate/${{ inputs.service_name}}:${{ inputs.version}}
30+
docker pull "${SOURCE_IMAGE}"
31+
docker tag "${SOURCE_IMAGE}" "${TARGET_IMAGE}"-amd
32+
- name: Pull source image and retag (arm)
33+
shell: bash
34+
run: |
35+
SOURCE_IMAGE=ghcr.io/modelengine-group/${{ inputs.service_name}}:${{ inputs.version}}
36+
TARGET_IMAGE=swr.cn-east-3.myhuaweicloud.com/data-mate/${{ inputs.service_name}}:${{ inputs.version}}
37+
docker pull "${SOURCE_IMAGE}" --platform=linux/arm64
38+
docker tag "${SOURCE_IMAGE}" "${TARGET_IMAGE}"-arm
39+
- name: Push image to Huawei SWR using
40+
shell: bash
41+
run: |
42+
TARGET_IMAGE=swr.cn-east-3.myhuaweicloud.com/data-mate/${{ inputs.service_name}}:${{ inputs.version}}
43+
echo "${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}" | docker login swr.cn-east-3.myhuaweicloud.com --username=cn-east-3@HPUA3ZGOTQFAFMM59DB9 --password-stdin
44+
docker push "${TARGET_IMAGE}"-amd
45+
docker push "${TARGET_IMAGE}"-arm
46+
docker manifest create "${TARGET_IMAGE}" "${TARGET_IMAGE}"-amd "${TARGET_IMAGE}"-arm
47+
docker manifest push "${TARGET_IMAGE}"

‎.github/workflows/docker-push-to-huaweicloud.yml‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
name: docker-push-to-huaweicloud.yml
2+
on:
3+
workflow_dispatch:
4+
inputs:
5+
version:
6+
description: 'image version (e.g. latest or v0.0.1-beta)'
7+
required: true
8+
default: 'latest'
9+
type: string
10+
11+
jobs:
12+
frontend:
13+
name: Push Frontend Image
14+
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
15+
with:
16+
service_name: datamate-frontend
17+
version: ${{ inputs.version}}
18+
secrets:
19+
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
20+
21+
backend:
22+
name: Push Backend Image
23+
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
24+
with:
25+
service_name: datamate-backend
26+
version: ${{ inputs.version}}
27+
secrets:
28+
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
29+
30+
backend-python:
31+
name: Push Backend-Python Image
32+
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
33+
with:
34+
service_name: datamate-backend-python
35+
version: ${{ inputs.version}}
36+
secrets:
37+
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
38+
39+
runtime:
40+
name: Push Runtime Image
41+
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
42+
with:
43+
service_name: datamate-runtime
44+
version: ${{ inputs.version}}
45+
secrets:
46+
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
47+
48+
database:
49+
name: Push Database Image
50+
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
51+
with:
52+
service_name: datamate-database
53+
version: ${{ inputs.version}}
54+
secrets:
55+
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
56+
57+
deer-flow-backend:
58+
name: Push deer-flow-backend Image
59+
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
60+
with:
61+
service_name: deer-flow-backend
62+
version: ${{ inputs.version}}
63+
secrets:
64+
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}
65+
66+
deer-flow-frontend:
67+
name: Push deer-flow-frontend Image
68+
uses: ./.github/workflows/docker-push-to-huaweicloud-reusable.yml
69+
with:
70+
service_name: deer-flow-frontend
71+
version: ${{ inputs.version}}
72+
secrets:
73+
HUAWEI_CLOUD_SWR_LOGIN_PWD: ${{ secrets.HUAWEI_CLOUD_SWR_LOGIN_PWD}}

‎.github/workflows/enterprise-package.yml‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
name: DataMate Package
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
milvus:
7+
type: boolean
8+
description: 是否打包milvus
9+
required: false
10+
default: true
11+
deer-flow:
12+
type: boolean
13+
description: 是否打包deer-flow
14+
required: false
15+
default: false
16+
version:
17+
type: string
18+
description: 版本号
19+
required: false
20+
default: 1.0.0
21+
22+
jobs:
23+
package-all:
24+
runs-on: ubuntu-latest
25+
steps:
26+
- name: Checkout
27+
uses: actions/checkout@v4
28+
29+
- name: Clone DataMate-Deploy
30+
uses: GuillaumeFalourd/clone-github-repo-action@v2.3
31+
with:
32+
owner: ModelEngine-Group
33+
repository: 'DataMate-Deploy'
34+
access-token: ${{ secrets.ACCESS_TOKEN }}
35+
36+
- name: Copy tools
37+
run: |
38+
cp -r DataMate-Deploy/tools tools
39+
40+
- name: DataMate Package
41+
run: |
42+
mkdir helm
43+
cp -r deployment/helm/datamate helm/datamate
44+
sed -i "s#^\(\s*repository:\s*\).*#\1\"\"#" helm/datamate/values.yaml
45+
sed -i "s#^\(\s*type:\s*\).*#\1ClusterIP#" helm/datamate/values.yaml
46+
47+
- name: DeerFlow Package
48+
if: inputs.deer-flow == true
49+
run: |
50+
cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env
51+
cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml
52+
cp -r deployment/helm/deer-flow helm/deer-flow
53+
54+
- name: Milvus Package
55+
if: inputs.milvus == true
56+
run: |
57+
cp -r deployment/helm/milvus helm/milvus
58+
sed -i "s#^\(\s*workload:\s*\).*#\1deployment#" helm/milvus/values.yaml
59+
60+
- name: Download DataMate Image
61+
run: |
62+
mkdir -p images/datamate
63+
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
64+
services=("runtime" "backend" "frontend" "database" "backend-python")
65+
for service in "${services[@]}"; do
66+
docker pull ghcr.io/$LOWERCASE_REPO/datamate-$service:latest --platform arm64
67+
docker tag ghcr.io/$LOWERCASE_REPO/datamate-$service:latest datamate-$service:latest
68+
docker save -o images/datamate/datamate-$service.tar datamate-$service:latest
69+
docker rmi ghcr.io/$LOWERCASE_REPO/datamate-$service:latest datamate-$service:latest
70+
done
71+
docker pull quay.io/kuberay/operator:v1.4.2 --platform arm64
72+
docker save -o images/datamate/kuberay-operator.tar quay.io/kuberay/operator:v1.4.2
73+
docker rmi quay.io/kuberay/operator:v1.4.2
74+
75+
- name: Download DeerFlow Image
76+
if: inputs.deer-flow == true
77+
run: |
78+
mkdir -p images/deer-flow
79+
LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
80+
services=("backend" "frontend")
81+
for service in "${services[@]}"; do
82+
docker pull ghcr.io/$LOWERCASE_REPO/deer-flow-$service:latest --platform arm64
83+
docker tag ghcr.io/$LOWERCASE_REPO/deer-flow-$service:latest deer-flow-$service:latest
84+
docker save -o images/deer-flow/deer-flow-$service.tar deer-flow-$service:latest
85+
docekr rmi ghcr.io/$LOWERCASE_REPO/deer-flow-$service:latest deer-flow-$service:latest
86+
done
87+
88+
- name: Download Milvus Image
89+
if: inputs.milvus == true
90+
run: |
91+
mkdir -p images/milvus
92+
docker pull milvusdb/milvus:v2.6.5 --platform arm64
93+
docker save -o images/milvus/milvus.tar milvusdb/milvus:v2.6.5
94+
docker rmi milvusdb/milvus:v2.6.5
95+
docker pull minio/minio:RELEASE.2024-12-18T13-15-44Z --platform arm64
96+
docker save -o images/milvus/minio.tar minio/minio:RELEASE.2024-12-18T13-15-44Z
97+
docker rmi minio/minio:RELEASE.2024-12-18T13-15-44Z
98+
docker pull milvusdb/etcd:3.5.18-r1 --platform arm64
99+
docker save -o images/milvus/etcd.tar milvusdb/etcd:3.5.18-r1
100+
docker rmi milvusdb/etcd:3.5.18-r1
101+
102+
- name: Upload Package
103+
uses: actions/upload-artifact@v4
104+
with:
105+
name: DataMate_${{ inputs.version }}_Aarch64
106+
include-hidden-files: true
107+
path: |
108+
helm/
109+
images/
110+
tools/

‎README-zh.md‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,6 @@ Choose a deployment method:
4242
Enter choice:
4343
```
4444

45-
在运行 `make uninstall` 时，卸载流程会只询问一次是否删除卷（数据），该选择会应用到所有组件。卸载顺序为：milvus -> label-studio -> datamate，确保在移除 datamate 网络前，所有使用该网络的服务已先停止。
46-
4745
### 拉取代码
4846

4947
```bash
@@ -56,6 +54,20 @@ cd DataMate
5654
```bash
5755
make install
5856
```
57+
58+
若您使用的机器没有make，请执行如下命令部署:
59+
```bash
60+
# Windows
61+
set REGISTRY=ghcr.io/modelengine-group/
62+
docker compose -f ./deployment/docker/datamate/docker-compose.yml up -d
63+
docker compose -f ./deployment/docker/milvus/docker-compose.yml up -d
64+
65+
# Linux/Mac
66+
export REGISTRY=ghcr.io/modelengine-group/
67+
docker compose -f ./deployment/docker/datamate/docker-compose.yml up -d
68+
docker compose -f ./deployment/docker/milvus/docker-compose.yml up -d
69+
```
70+
5971
当容器运行后，请在浏览器打开 http://localhost:30000 查看前端界面。
6072

6173
要查看所有可用的 Make 目标、选项和帮助信息，请运行：
@@ -82,6 +94,13 @@ make build
8294
make install dev=true
8395
```
8496

97+
### 卸载服务
98+
```bash
99+
make uninstall
100+
```
101+
102+
在运行 `make uninstall` 时，卸载流程会只询问一次是否删除卷（数据），该选择会应用到所有组件。卸载顺序为：milvus -> label-studio -> datamate，确保在移除 datamate 网络前，所有使用该网络的服务已先停止。
103+
85104
## 🤝 贡献指南
86105

87106
感谢您对本项目的关注！我们非常欢迎社区的贡献，无论是提交 Bug 报告、提出功能建议，还是直接参与代码开发，都能帮助项目变得更好。

‎README.md‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,6 @@ Choose a deployment method:
4545
Enter choice:
4646
```
4747

48-
When running make uninstall, the installer will prompt once whether to delete volumes; that single choice is applied to all components. The uninstall order is: milvus -> label-studio -> datamate, which ensures the datamate network is removed cleanly after services that use it have stopped.
49-
5048
### Clone the Code
5149

5250
```bash
@@ -59,6 +57,20 @@ cd DataMate
5957
```bash
6058
make install
6159
```
60+
61+
If the machine you are using does not have make installed, please run the following command to deploy it:
62+
```bash
63+
# Windows
64+
set REGISTRY=ghcr.io/modelengine-group/
65+
docker compose -f ./deployment/docker/datamate/docker-compose.yml up -d
66+
docker compose -f ./deployment/docker/milvus/docker-compose.yml up -d
67+
68+
# Linux/Mac
69+
export REGISTRY=ghcr.io/modelengine-group/
70+
docker compose -f ./deployment/docker/datamate/docker-compose.yml up -d
71+
docker compose -f ./deployment/docker/milvus/docker-compose.yml up -d
72+
```
73+
6274
Once the container is running, access http://localhost:30000 in a browser to view the front-end interface.
6375

6476
To list all available Make targets, flags and help text, run:
@@ -85,6 +97,13 @@ make build
8597
make install dev=true
8698
```
8799

100+
### Uninstall
101+
```bash
102+
make uninstall
103+
```
104+
105+
When running make uninstall, the installer will prompt once whether to delete volumes; that single choice is applied to all components. The uninstall order is: milvus -> label-studio -> datamate, which ensures the datamate network is removed cleanly after services that use it have stopped.
106+
88107
## 🤝 Contribution Guidelines
89108

90109
Thank you for your interest in this project! We warmly welcome contributions from the community. Whether it's submitting

‎deployment/docker/datamate/backend-with-deer-flow.conf‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,13 @@ server {
2323
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
2424
}
2525

26+
location /api/evaluation/ {
27+
proxy_pass http://datamate-backend-python:18000/api/evaluation/;
28+
proxy_set_header Host $host;
29+
proxy_set_header X-Real-IP $remote_addr;
30+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
31+
}
32+
2633
location /api/ {
2734
proxy_pass http://datamate-backend:8080/api/;
2835
proxy_set_header Host $host;

‎runtime/ops/README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ language: 'python'
1313
vendor: 'huawei'
1414
raw_id: 'TestMapper'
1515
version: '1.0.0'
16-
modal: 'text'
16+
modal: 'text' # text/image/audio/video/multimodal
1717
inputs: 'text'
1818
outputs: 'text'
1919
```
@@ -35,7 +35,7 @@ class TestMapper(Mapper):
3535
sample[self.text_key] += "\n新增的数据"
3636
return sample
3737

38-
38+
3939
# Filter用于过滤和选择性保留数据，使用时将需要过滤的数据的text或data置为空值
4040
from datamate.core.base_op import Filter
4141

@@ -86,4 +86,4 @@ from datamate.core.base_op import OPERATORS
8686
OPERATORS.register_module(module_name='TestMapper',
8787
module_path="ops.user.test_operator.process")
8888

89-
```
89+
```

‎runtime/python-executor/datamate/core/base_op.py‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -456,9 +456,8 @@ def execute(self, sample: Dict[str, Any]):
456456
new_file_name = base_name + '.' + file_type
457457
sample[self.filename_key] = new_file_name
458458

459-
base_name, _ = os.path.splitext(save_path)
460-
sample[self.filepath_key] = base_name
461-
file_size = os.path.getsize(base_name)
459+
sample[self.filepath_key] = save_path
460+
file_size = os.path.getsize(save_path)
462461
sample[self.filesize_key] = f"{file_size}"
463462

464463
logger.info(f"origin file named {file_name} has been save to {save_path}")
@@ -514,16 +513,15 @@ def get_medicalfile_handler(self, sample: Dict[str, Any]):
514513
return sample, save_path
515514

516515
def save_file(self, sample, save_path):
517-
file_name, _ = os.path.splitext(save_path)
518516
# 以二进制格式保存文件
519517
file_sample = sample[self.text_key].encode('utf-8') if sample[self.text_key] else sample[self.data_key]
520-
with open(file_name, 'wb') as f:
518+
with open(save_path, 'wb') as f:
521519
f.write(file_sample)
522520
# 获取父目录路径
523521

524-
parent_dir = os.path.dirname(file_name)
522+
parent_dir = os.path.dirname(save_path)
525523
os.chmod(parent_dir, 0o770)
526-
os.chmod(file_name, 0o640)
524+
os.chmod(save_path, 0o640)
527525

528526
def _get_from_data(self, sample: Dict[str, Any]) -> Dict[str, Any]:
529527
sample[self.data_key] = bytes(sample[self.data_key])

0 commit comments

Comments
 (0)