Skip to content

Commit 31ef8bc

Browse files
hhhhsc701hhhhsc701
authored
[Feature] Refactor project to use 'datamate' naming convention for services and configurations (#14)
* Enhance CleaningTaskService to track cleaning process progress and update ExecutorType to DATAMATE * Refactor project to use 'datamate' naming convention for services and configurations
1 parent 175d9de commit 31ef8bc

File tree

39 files changed

+310
-735
lines changed

39 files changed

+310
-735
lines changed

‎Makefile‎

Lines changed: 35 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ else
3333
endif
3434

3535
.PHONY: install
36-
install: install-data-mate
36+
install: install-datamate
3737

3838
.PHONY: uninstall-%
3939
uninstall-%:
@@ -54,115 +54,105 @@ else
5454
endif
5555

5656
.PHONY: uninstall
57-
uninstall: uninstall-data-mate
57+
uninstall: uninstall-datamate
5858

5959
# build
60-
.PHONY: mineru-docker-build
61-
mineru-docker-build:
62-
docker build -t mineru:$(VERSION) . -f scripts/images/mineru/Dockerfile
63-
64-
.PHONY: datax-docker-build
65-
datax-docker-build:
66-
docker build -t datax:$(VERSION) . -f scripts/images/datax/Dockerfile
67-
68-
.PHONY: unstructured-docker-build
69-
unstructured-docker-build:
70-
docker build -t unstructured:$(VERSION) . -f scripts/images/unstructured/Dockerfile
71-
7260
.PHONY: backend-docker-build
7361
backend-docker-build:
74-
docker build -t backend:$(VERSION) . -f scripts/images/backend/Dockerfile
62+
docker build -t datamate-backend:$(VERSION) . -f scripts/images/backend/Dockerfile
7563

7664
.PHONY: frontend-docker-build
7765
frontend-docker-build:
78-
docker build -t frontend:$(VERSION) . -f scripts/images/frontend/Dockerfile
66+
docker build -t datamate-frontend:$(VERSION) . -f scripts/images/frontend/Dockerfile
7967

8068
.PHONY: runtime-docker-build
8169
runtime-docker-build:
82-
docker build -t runtime:$(VERSION) . -f scripts/images/runtime/Dockerfile
70+
docker build -t datamate-runtime:$(VERSION) . -f scripts/images/runtime/Dockerfile
8371

8472
.PHONY: label-studio-adapter-docker-build
8573
label-studio-adapter-docker-build:
8674
docker build -t label-studio-adapter:$(VERSION) . -f scripts/images/label-studio-adapter/Dockerfile
8775

8876
.PHONY: backend-docker-install
8977
backend-docker-install:
90-
cd deployment/docker/data-mate && docker-compose up -d backend
78+
cd deployment/docker/datamate && docker-compose up -d backend
9179

9280
.PHONY: backend-docker-uninstall
9381
backend-docker-uninstall:
94-
cd deployment/docker/data-mate && docker-compose down backend
82+
cd deployment/docker/datamate && docker-compose down backend
9583

9684
.PHONY: frontend-docker-install
9785
frontend-docker-install:
98-
cd deployment/docker/data-mate && docker-compose up -d frontend
86+
cd deployment/docker/datamate && docker-compose up -d frontend
9987

10088
.PHONY: frontend-docker-uninstall
10189
frontend-docker-uninstall:
102-
cd deployment/docker/data-mate && docker-compose down frontend
90+
cd deployment/docker/datamate && docker-compose down frontend
10391

10492
.PHONY: runtime-docker-install
10593
runtime-docker-install:
106-
cd deployment/docker/data-mate && docker-compose up -d runtime
94+
cd deployment/docker/datamate && docker-compose up -d runtime
10795

10896
.PHONY: runtime-docker-uninstall
10997
runtime-docker-uninstall:
110-
cd deployment/docker/data-mate && docker-compose down runtime
98+
cd deployment/docker/datamate && docker-compose down runtime
11199

112100
.PHONY: runtime-k8s-install
113101
runtime-k8s-install: create-namespace
114-
helm upgrade kuberay-operator deployment/helm/ray/kuberay-operator --install -n $(NAMESPACE)
115-
helm upgrade raycluster deployment/helm/ray/ray-cluster/ --install -n $(NAMESPACE)
102+
helm upgrade datamate-kuberay-operator deployment/helm/ray/kuberay-operator --install -n $(NAMESPACE)
103+
helm upgrade datamate-raycluster deployment/helm/ray/ray-cluster/ --install -n $(NAMESPACE)
116104
kubectl apply -f deployment/helm/ray/service.yaml -n $(NAMESPACE)
117105

118106
.PHONY: runtime-k8s-uninstall
119107
runtime-k8s-uninstall:
120-
helm uninstall raycluster -n $(NAMESPACE)
121-
helm uninstall kuberay-operator -n $(NAMESPACE)
108+
helm uninstall datamate-raycluster -n $(NAMESPACE)
109+
helm uninstall datamate-kuberay-operator -n $(NAMESPACE)
122110
kubectl delete -f deployment/helm/ray/service.yaml -n $(NAMESPACE)
123111

124-
.PHONY: unstructured-k8s-install
125-
unstructured-k8s-install: create-namespace
126-
kubectl apply -f deployment/kubernetes/unstructured/deploy.yaml -n $(NAMESPACE)
127-
128112
.PHONY: mysql-k8s-install
129113
mysql-k8s-install: create-namespace
130-
kubectl create configmap init-sql --from-file=scripts/db/ --dry-run=client -o yaml | kubectl apply -f - -n $(NAMESPACE)
114+
kubectl create configmap datamate-init-sql --from-file=scripts/db/ --dry-run=client -o yaml | kubectl apply -f - -n $(NAMESPACE)
131115
kubectl apply -f deployment/kubernetes/mysql/configmap.yaml -n $(NAMESPACE)
132116
kubectl apply -f deployment/kubernetes/mysql/deploy.yaml -n $(NAMESPACE)
133117

134118
.PHONY: mysql-k8s-uninstall
135119
mysql-k8s-uninstall:
136-
kubectl delete configmap init-sql -n $(NAMESPACE)
137-
kubectl delete -f deployment/kubernetes/mysql/configmap.yaml -n $(NAMESPACE)
138-
kubectl delete -f deployment/kubernetes/mysql/deploy.yaml -n $(NAMESPACE)
120+
kubectl delete configmap datamate-init-sql -n $(NAMESPACE) --ignore-not-found
121+
kubectl delete -f deployment/kubernetes/mysql/configmap.yaml -n $(NAMESPACE) --ignore-not-found
122+
kubectl delete -f deployment/kubernetes/mysql/deploy.yaml -n $(NAMESPACE) --ignore-not-found
123+
124+
.PHONY: database-k8s-install
125+
database-k8s-install: mysql-k8s-install
126+
127+
.PHONY: database-k8s-uninstall
128+
database-k8s-uninstall: mysql-k8s-uninstall
139129

140130
.PHONY: backend-k8s-install
141131
backend-k8s-install: create-namespace
142132
kubectl apply -f deployment/kubernetes/backend/deploy.yaml -n $(NAMESPACE)
143133

144134
.PHONY: backend-k8s-uninstall
145135
backend-k8s-uninstall:
146-
kubectl delete -f deployment/kubernetes/backend/deploy.yaml -n $(NAMESPACE)
136+
kubectl delete -f deployment/kubernetes/backend/deploy.yaml -n $(NAMESPACE) --ignore-not-found
147137

148138
.PHONY: frontend-k8s-install
149139
frontend-k8s-install: create-namespace
150140
kubectl apply -f deployment/kubernetes/frontend/deploy.yaml -n $(NAMESPACE)
151141

152142
.PHONY: frontend-k8s-uninstall
153143
frontend-k8s-uninstall:
154-
kubectl delete -f deployment/kubernetes/frontend/deploy.yaml -n $(NAMESPACE)
144+
kubectl delete -f deployment/kubernetes/frontend/deploy.yaml -n $(NAMESPACE) --ignore-not-found
155145

156-
.PHONY: data-mate-docker-install
157-
data-mate-docker-install:
146+
.PHONY: datamate-docker-install
147+
datamate-docker-install:
158148
cd deployment/docker/datamate && docker-compose up -d
159149

160-
.PHONY: data-mate-docker-uninstall
161-
data-mate-docker-uninstall:
150+
.PHONY: datamate-docker-uninstall
151+
datamate-docker-uninstall:
162152
cd deployment/docker/datamate && docker-compose down
163153

164-
.PHONY: data-mate-k8s-install
165-
data-mate-k8s-install: create-namespace mysql-k8s-install backend-k8s-install frontend-k8s-install runtime-k8s-install
154+
.PHONY: datamate-k8s-install
155+
datamate-k8s-install: create-namespace database-k8s-install backend-k8s-install frontend-k8s-install runtime-k8s-install
166156

167-
.PHONY: data-mate-k8s-uninstall
168-
data-mate-k8s-uninstall: mysql-k8s-uninstall backend-k8s-uninstall frontend-k8s-uninstall runtime-k8s-uninstall
157+
.PHONY: datamate-k8s-uninstall
158+
datamate-k8s-uninstall: database-k8s-uninstall backend-k8s-uninstall frontend-k8s-uninstall runtime-k8s-uninstall

‎backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/httpclient/RuntimeClient.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
@Slf4j
1616
public class RuntimeClient {
17-
private static final String BASE_URL = "http://runtime:8081/api";
17+
private static final String BASE_URL = "http://datamate-runtime:8081/api";
1818

1919
private static final String CREATE_TASK_URL = BASE_URL + "/task/{0}/submit";
2020

‎backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/service/CleaningTaskService.java‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import com.datamate.cleaning.infrastructure.persistence.mapper.CleaningResultMapper;
1313
import com.datamate.cleaning.infrastructure.persistence.mapper.CleaningTaskMapper;
1414
import com.datamate.cleaning.infrastructure.persistence.mapper.OperatorInstanceMapper;
15+
import com.datamate.cleaning.interfaces.dto.CleaningProcess;
1516
import com.datamate.cleaning.interfaces.dto.CleaningTask;
1617
import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest;
1718
import com.datamate.cleaning.interfaces.dto.OperatorInstance;
@@ -55,7 +56,14 @@ public class CleaningTaskService {
5556

5657
public List<CleaningTask> getTasks(String status, String keywords, Integer page, Integer size) {
5758
Integer offset = page * size;
58-
return cleaningTaskMapper.findTasks(status, keywords, size, offset);
59+
List<CleaningTask> tasks = cleaningTaskMapper.findTasks(status, keywords, size, offset);
60+
tasks.forEach(this::setProcess);
61+
return tasks;
62+
}
63+
64+
private void setProcess(CleaningTask task) {
65+
int count = cleaningResultMapper.countByInstanceId(task.getId());
66+
task.setProgress(CleaningProcess.of(task.getFileCount(), count));
5967
}
6068

6169
public int countTasks(String status, String keywords) {
@@ -80,6 +88,7 @@ public CleaningTask createTask(CreateCleaningTaskRequest request) {
8088
task.setDestDatasetId(destDataset.getId());
8189
task.setDestDatasetName(destDataset.getName());
8290
task.setBeforeSize(srcDataset.getTotalSize());
91+
task.setFileCount(srcDataset.getFileCount());
8392
cleaningTaskMapper.insertTask(task);
8493

8594
List<OperatorInstancePo> instancePos = request.getInstance().stream()
@@ -93,7 +102,9 @@ public CleaningTask createTask(CreateCleaningTaskRequest request) {
93102
}
94103

95104
public CleaningTask getTask(String taskId) {
96-
return cleaningTaskMapper.findTaskById(taskId);
105+
CleaningTask task = cleaningTaskMapper.findTaskById(taskId);
106+
setProcess(task);
107+
return task;
97108
}
98109

99110
@Transactional
@@ -113,7 +124,7 @@ private void prepareTask(CleaningTask task, List<OperatorInstance> instances) {
113124
process.setDatasetId(task.getDestDatasetId());
114125
process.setDatasetPath(FLOW_PATH + "/" + task.getId() + "/dataset.jsonl");
115126
process.setExportPath(DATASET_PATH + "/" + task.getDestDatasetId());
116-
process.setExecutorType(ExecutorType.DATA_PLATFORM.getValue());
127+
process.setExecutorType(ExecutorType.DATAMATE.getValue());
117128
process.setProcess(instances.stream()
118129
.map(instance -> Map.of(instance.getId(), instance.getOverrides()))
119130
.toList());

‎backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/ExecutorType.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
@Getter
66
public enum ExecutorType {
7-
DATA_PLATFORM("data_platform"),
7+
DATAMATE("datamate"),
88
DATA_JUICER_RAY("ray"),
99
DATA_JUICER_DEFAULT("default");
1010

‎backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/mapper/CleaningResultMapper.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,6 @@
66
@Mapper
77
public interface CleaningResultMapper {
88
void deleteByInstanceId(@Param("instanceId") String instanceId);
9+
10+
int countByInstanceId(@Param("instanceId") String instanceId);
911
}

‎backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CleaningProcess.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@
44
import lombok.Getter;
55
import lombok.Setter;
66

7+
import java.math.BigDecimal;
8+
import java.math.RoundingMode;
9+
710
/**
811
* CleaningProcess
912
*/
@@ -16,5 +19,20 @@ public class CleaningProcess {
1619
private Integer totalFileNum;
1720

1821
private Integer finishedFileNum;
22+
23+
public CleaningProcess(int totalFileNum, int finishedFileNum) {
24+
this.totalFileNum = totalFileNum;
25+
this.finishedFileNum = finishedFileNum;
26+
if (totalFileNum == 0) {
27+
this.process = 0.0f;
28+
} else {
29+
this.process = BigDecimal.valueOf(finishedFileNum * 100L)
30+
.divide(BigDecimal.valueOf(totalFileNum), 2, RoundingMode.HALF_UP).floatValue();
31+
}
32+
}
33+
34+
public static CleaningProcess of(int totalFileNum, int finishedFileNum) {
35+
return new CleaningProcess(totalFileNum, finishedFileNum);
36+
}
1937
}
2038

‎backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CleaningTask.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ public class CleaningTask {
3636

3737
private long afterSize;
3838

39+
private int fileCount;
40+
3941
/**
4042
* 任务当前状态
4143
*/

‎backend/services/data-cleaning-service/src/main/resources/mappers/CleaningResultMapper.xml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,8 @@
55
DELETE FROM t_clean_result WHERE instance_id = #{instanceId}
66
</delete>
77

8+
<select id="countByInstanceId" resultType="java.lang.Integer">
9+
SELECT COUNT(1) FROM t_clean_result WHERE instance_id = #{instanceId}
10+
</select>
11+
812
</mapper>

‎backend/services/data-cleaning-service/src/main/resources/mappers/CleaningTaskMapper.xml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<mapper namespace="com.datamate.cleaning.infrastructure.persistence.mapper.CleaningTaskMapper">
44
<sql id="Base_Column_List">
55
id, name, description, src_dataset_id, src_dataset_name, dest_dataset_id, dest_dataset_name, before_size,
6-
after_size, status, created_at, started_at, finished_at
6+
after_size, file_count, status, created_at, started_at, finished_at
77
</sql>
88

99
<select id="findTasks" resultType="com.datamate.cleaning.interfaces.dto.CleaningTask">
@@ -28,9 +28,9 @@
2828

2929
<insert id="insertTask">
3030
INSERT INTO t_clean_task (id, name, description, status, src_dataset_id, src_dataset_name, dest_dataset_id,
31-
dest_dataset_name, before_size, after_size, created_at)
31+
dest_dataset_name, before_size, after_size, file_count, created_at)
3232
VALUES (#{id}, #{name}, #{description}, #{status}, #{srcDatasetId}, #{srcDatasetName}, #{destDatasetId},
33-
#{destDatasetName}, ${beforeSize}, ${afterSize}, NOW())
33+
#{destDatasetName}, #{beforeSize}, #{afterSize}, #{fileCount}, NOW())
3434
</insert>
3535

3636
<update id="updateTask">

‎backend/services/main-application/src/main/resources/application.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ spring:
1212
# 数据源配置
1313
datasource:
1414
driver-class-name: com.mysql.cj.jdbc.Driver
15-
url: jdbc:mysql://mysql:3306/datamate?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true
15+
url: jdbc:mysql://datamate-database:3306/datamate?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true
1616
username: ${DB_USERNAME:root}
1717
password: ${DB_PASSWORD:Huawei@123}
1818
hikari:

0 commit comments

Comments
 (0)