Merge branch 'dev' into dsip_94

SbloodyS · web-flow · commit c9f894f8efb3 · 2025-12-21T19:37:34.000+08:00
diff --git a/.github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh b/.github/workflows/cluster-test/mysql_with_mysql_registry/running_test.sh
@@ -46,22 +46,22 @@ do
 
   if [[ $i -eq $TIMEOUT ]];then
     if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
-      cat ~/ds_test/master-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/master-server/logs/*.log
       cat ~/ds_test/master-server/logs/*.out
       echo "master start health check failed"
     fi
     if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
-      cat ~/ds_test/worker-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/worker-server/logs/*.log
       cat ~/ds_test/worker-server/logs/*.out
       echo "worker start health check failed"
     fi
     if [[ $API_HTTP_STATUS -ne 200 ]]; then
-      cat ~/ds_test/api-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/api-server/logs/*.log
       cat ~/ds_test/api-server/logs/*.out
       echo "api start health check failed"
     fi
     if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
-      cat ~/ds_test/alert-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/alert-server/logs/*.log
       cat ~/ds_test/alert-server/logs/*.out
       echo "alert start health check failed"
     fi
diff --git a/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh b/.github/workflows/cluster-test/mysql_with_zookeeper_registry/running_test.sh
@@ -46,22 +46,22 @@ do
 
   if [[ $i -eq $TIMEOUT ]];then
     if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
-      cat ~/ds_test/master-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/master-server/logs/*.log
       cat ~/ds_test/master-server/logs/*.out
       echo "master start health check failed"
     fi
     if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
-      cat ~/ds_test/worker-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/worker-server/logs/*.log
       cat ~/ds_test/worker-server/logs/*.out
       echo "worker start health check failed"
     fi
     if [[ $API_HTTP_STATUS -ne 200 ]]; then
-      cat ~/ds_test/api-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/api-server/logs/*.log
       cat ~/ds_test/api-server/logs/*.out
       echo "api start health check failed"
     fi
     if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
-      cat ~/ds_test/alert-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/alert-server/logs/*.log
       cat ~/ds_test/alert-server/logs/*.out
       echo "alert start health check failed"
     fi
diff --git a/.github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh b/.github/workflows/cluster-test/postgresql_with_postgresql_registry/running_test.sh
@@ -46,22 +46,22 @@ do
 
   if [[ $i -eq $TIMEOUT ]];then
     if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
-      cat ~/ds_test/master-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/master-server/logs/*.log
       cat ~/ds_test/master-server/logs/*.out
       echo "master start health check failed"
     fi
     if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
-      cat ~/ds_test/worker-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/worker-server/logs/*.log
       cat ~/ds_test/worker-server/logs/*.out
       echo "worker start health check failed"
     fi
     if [[ $API_HTTP_STATUS -ne 200 ]]; then
-      cat ~/ds_test/api-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/api-server/logs/*.log
       cat ~/ds_test/api-server/logs/*.out
       echo "api start health check failed"
     fi
     if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
-      cat ~/ds_test/alert-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/alert-server/logs/*.log
       cat ~/ds_test/alert-server/logs/*.out
       echo "alert start health check failed"
     fi
diff --git a/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh b/.github/workflows/cluster-test/postgresql_with_zookeeper_registry/running_test.sh
@@ -46,22 +46,22 @@ do
 
   if [[ $i -eq $TIMEOUT ]];then
     if [[ $MASTER_HTTP_STATUS -ne 200 ]];then
-      cat ~/ds_test/master-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/master-server/logs/*.log
       cat ~/ds_test/master-server/logs/*.out
       echo "master start health check failed"
     fi
     if [[ $WORKER_HTTP_STATUS -ne 200 ]]; then
-      cat ~/ds_test/worker-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/worker-server/logs/*.log
       cat ~/ds_test/worker-server/logs/*.out
       echo "worker start health check failed"
     fi
     if [[ $API_HTTP_STATUS -ne 200 ]]; then
-      cat ~/ds_test/api-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/api-server/logs/*.log
       cat ~/ds_test/api-server/logs/*.out
       echo "api start health check failed"
     fi
     if [[ $ALERT_HTTP_STATUS -ne 200 ]]; then
-      cat ~/ds_test/alert-server/logs/dolphinscheduler-master.log
+      cat ~/ds_test/alert-server/logs/*.log
       cat ~/ds_test/alert-server/logs/*.out
       echo "alert start health check failed"
     fi
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -108,7 +108,6 @@ jobs:
     needs:
       - style
       - img-check
-      - dead-link
     if: always()
     steps:
       - name: Status
diff --git a/docs/docs/en/guide/installation/registry-plugins/jdbc.md b/docs/docs/en/guide/installation/registry-plugins/jdbc.md
@@ -38,19 +38,13 @@ registry:
   heartbeat-refresh-interval: 3s
   # Once the client's heartbeat is not refresh in this time, the server will consider the client is offline.
   session-timeout: 60s
-  # The hikari configuration, default will use the same datasource pool as DolphinScheduler.
-  hikari-config:
-    jdbc-url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
-    username: root
-    password: root
-    maximum-pool-size: 5
-    connection-timeout: 9000
-    idle-timeout: 600000
 ```
 
-## Use different database configuration for jdbc registry center
+## Set DataSource for worker
 
-You need to set the registry properties in master/worker/api's application.yml
+Since worker server doesn't contain datasource, so you need to config datasource for worker.
+
+You need to set the registry hikari-config properties at worker's application.yml
 
 ### Use Mysql as registry center
 
diff --git a/docs/docs/en/guide/upgrade/incompatible.md b/docs/docs/en/guide/upgrade/incompatible.md
@@ -36,5 +36,6 @@ This document records the incompatible updates between each version. You need to
 
 * Renamed the publicKey field to privateKey in the SSH connection parameters under the datasource configuration. ([#17666])(https://github.com/apache/dolphinscheduler/pull/17666)
 * Add table t_ds_serial_command. ([#17531])(https://github.com/apache/dolphinscheduler/pull/17531)
+* Remove the default value of `python-gateway.auth-token` at `api-server/application.yaml`. ([#17801])(https://github.com/apache/dolphinscheduler/pull/17801)
 * Remove the `Pytorch` from the `Task Plugin` ([#17808])(https://github.com/apache/dolphinscheduler/pull/17808)
 
diff --git a/docs/docs/zh/guide/installation/registry-plugins/jdbc.md b/docs/docs/zh/guide/installation/registry-plugins/jdbc.md
@@ -32,19 +32,11 @@ registry:
   heartbeat-refresh-interval: 3s
   # Once the client's heartbeat is not refresh in this time, the server will consider the client is offline.
   session-timeout: 60s
-  # The hikari configuration, default will use the same datasource pool as DolphinScheduler.
-  hikari-config:
-    jdbc-url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler
-    username: root
-    password: root
-    maximum-pool-size: 5
-    connection-timeout: 9000
-    idle-timeout: 600000
 ```
 
-## 对 JDBC 注册中心使用不同的数据库配置
+## 为 worker 配置数据源
 
-需要在 master/worker/api 的 application.yml 中设置属性
+由于Worker服务默认不包含数据源，因此你需要在 worker 的 application.yml 中为注册中心设置数据源属性
 
 ### 使用 MySQL 作为注册中心
 
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ApiConfig.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/ApiConfig.java
@@ -17,12 +17,15 @@
 
 package org.apache.dolphinscheduler.api.configuration;
 
+import org.apache.commons.lang3.StringUtils;
+
 import java.util.HashMap;
 import java.util.Map;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import lombok.ToString;
 import lombok.extern.slf4j.Slf4j;
 
 import org.springframework.boot.context.properties.ConfigurationProperties;
@@ -53,9 +56,16 @@ public boolean supports(Class<?> clazz) {
 
     @Override
     public void validate(Object target, Errors errors) {
+        validatePythonGateway(errors);
         printConfig();
     }
 
+    private void validatePythonGateway(Errors errors) {
+        if (pythonGateway.isEnabled() && StringUtils.isEmpty(pythonGateway.getAuthToken())) {
+            errors.rejectValue("pythonGateway.auth-token", null, "should not be empty when enabled is true");
+        }
+    }
+
     private void printConfig() {
         log.info("API config: baseUrl -> {} ", baseUrl);
         log.info("API config: uiUrl -> {} ", uiUrl);
@@ -76,12 +86,13 @@ public static class TrafficConfiguration {
         private Map<String, Integer> customizeTenantQpsRate = new HashMap<>();
     }
 
+    @ToString(exclude = "authToken")
     @Data
     @NoArgsConstructor
     @AllArgsConstructor
     public static class PythonGatewayConfiguration {
 
-        private boolean enabled = true;
+        private boolean enabled = false;
         private String gatewayServerAddress = "0.0.0.0";
         private int gatewayServerPort = 25333;
         private String pythonAddress = "127.0.0.1";
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProjectWorkerGroupRelationServiceImpl.java
@@ -94,7 +94,30 @@ public Result assignWorkerGroupsToProject(User loginUser, Long projectCode, List
             return result;
         }
 
+        Project project = projectMapper.queryByCode(projectCode);
+        if (Objects.isNull(project)) {
+            putMsg(result, Status.PROJECT_NOT_EXIST);
+            return result;
+        }
+
+        /*
+         * Todo : For modification operations on projects, we should acquire project row locks. All project-related
+         * operations and modification/creation actions for workflows/task definitions within the project require
+         * acquiring row locks first
+         */
         if (CollectionUtils.isEmpty(workerGroups)) {
+            Set<String> projectWorkerGroupNames =
+                    projectWorkerGroupDao.queryAssignedWorkerGroupNamesByProjectCode(projectCode);
+            if (CollectionUtils.isNotEmpty(projectWorkerGroupNames)) {
+                Set<String> usedWorkerGroups = getAllUsedWorkerGroups(project);
+                if (CollectionUtils.isNotEmpty(usedWorkerGroups)) {
+                    Set<String> usedInProject = SetUtils.intersection(usedWorkerGroups, projectWorkerGroupNames);
+                    if (!usedInProject.isEmpty()) {
+                        throw new ServiceException(Status.USED_WORKER_GROUP_EXISTS, usedInProject);
+                    }
+                }
+            }
+
             boolean deleted = projectWorkerGroupDao.deleteByProjectCode(projectCode);
             if (deleted) {
                 putMsg(result, Status.SUCCESS);
@@ -104,12 +127,6 @@ public Result assignWorkerGroupsToProject(User loginUser, Long projectCode, List
             return result;
         }
 
-        Project project = projectMapper.queryByCode(projectCode);
-        if (Objects.isNull(project)) {
-            putMsg(result, Status.PROJECT_NOT_EXIST);
-            return result;
-        }
-
         Set<String> allWorkerGroupNames = new HashSet<>(workerGroupDao.queryAllWorkerGroupNames());
         workerGroupService.getConfigWorkerGroupPageDetail().forEach(
                 workerGroupPageDetail -> allWorkerGroupNames.add(workerGroupPageDetail.getName()));
@@ -131,9 +148,11 @@ public Result assignWorkerGroupsToProject(User loginUser, Long projectCode, List
 
         if (CollectionUtils.isNotEmpty(needDeletedWorkerGroups)) {
             Set<String> usedWorkerGroups = getAllUsedWorkerGroups(project);
-            if (CollectionUtils.isNotEmpty(usedWorkerGroups) && usedWorkerGroups.containsAll(needDeletedWorkerGroups)) {
-                throw new ServiceException(Status.USED_WORKER_GROUP_EXISTS,
-                        SetUtils.intersection(usedWorkerGroups, needDeletedWorkerGroups).toSet());
+            if (CollectionUtils.isNotEmpty(usedWorkerGroups) && CollectionUtils.isNotEmpty(needDeletedWorkerGroups)) {
+                Set<String> shouldNotDelete = SetUtils.intersection(usedWorkerGroups, needDeletedWorkerGroups);
+                if (CollectionUtils.isNotEmpty(shouldNotDelete)) {
+                    throw new ServiceException(Status.USED_WORKER_GROUP_EXISTS, shouldNotDelete);
+                }
             }
             boolean deleted =
                     projectWorkerGroupDao.deleteByProjectCodeAndWorkerGroups(projectCode,
@@ -147,6 +166,7 @@ public Result assignWorkerGroupsToProject(User loginUser, Long projectCode, List
                 throw new ServiceException(Status.ASSIGN_WORKER_GROUP_TO_PROJECT_ERROR);
             }
         }
+
         Set<String> needAssignedWorkerGroups =
                 SetUtils.difference(unauthorizedWorkerGroupNames, projectWorkerGroupNames);
         if (CollectionUtils.isNotEmpty(needAssignedWorkerGroups)) {
diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml
@@ -154,9 +154,8 @@ api:
   python-gateway:
     # Weather enable python gateway server or not. The default value is false.
     enabled: false
-    # Authentication token for connection from python api to python gateway server. Should be changed the default value
-    # when you deploy in public network.
-    auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc
+    # Authentication token for connection from python api to python gateway server. Should not be empty when enabled
+    auth-token:
     # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
     # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
     gateway-server-address: 0.0.0.0
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectWorkerGroupRelationServiceTest.java
@@ -138,6 +138,13 @@ public void testAssignWorkerGroupsToProject() {
                 getWorkerGroups());
         Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode());
 
+        // success when no task referenced any wg
+        Mockito.when(projectWorkerGroupDao.deleteByProjectCode(projectCode))
+                .thenReturn(true);
+        result = projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode,
+                new ArrayList<>());
+        Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode());
+
         // db deletion fail
         Mockito.when(projectWorkerGroupDao.deleteByProjectCodeAndWorkerGroups(Mockito.any(), Mockito.any()))
                 .thenReturn(false);
@@ -154,6 +161,26 @@ public void testAssignWorkerGroupsToProject() {
         AssertionsHelper.assertThrowsServiceException(Status.USED_WORKER_GROUP_EXISTS,
                 () -> projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode,
                         getUnusedWorkerGroups()));
+
+        // test clear all wg and fail when wg is referenced by task definition
+        // test case: project all wg: test, task used wg: test, new wg: null
+        Mockito.when(taskDefinitionDao.queryAllTaskDefinitionWorkerGroups(Mockito.anyLong()))
+                .thenReturn(Collections.singletonList(getProjectWorkerGroup().getWorkerGroup()));
+        Mockito.when(projectWorkerGroupDao.queryAssignedWorkerGroupNamesByProjectCode(Mockito.any()))
+                .thenReturn(Sets.newHashSet(getProjectWorkerGroup().getWorkerGroup()));
+        AssertionsHelper.assertThrowsServiceException(Status.USED_WORKER_GROUP_EXISTS,
+                () -> projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode,
+                        new ArrayList<>()));
+
+        // test delete superset of the used wg collection and fail when wg is referenced by task definition
+        // test case: project all wg: test,test1,test2. task used wg: test. new wg: test1, delete test2 and test
+        Mockito.when(taskDefinitionDao.queryAllTaskDefinitionWorkerGroups(Mockito.anyLong()))
+                .thenReturn(Collections.singletonList(getProjectWorkerGroup().getWorkerGroup()));
+        Mockito.when(projectWorkerGroupDao.queryAssignedWorkerGroupNamesByProjectCode(Mockito.any()))
+                .thenReturn(Sets.newHashSet("test", "test1", "test2"));
+        AssertionsHelper.assertThrowsServiceException(Status.USED_WORKER_GROUP_EXISTS,
+                () -> projectWorkerGroupRelationService.assignWorkerGroupsToProject(loginUser, projectCode,
+                        getUnusedWorkerGroups()));
     }
 
     @Test
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryAutoConfiguration.java
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryProperties.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/JdbcRegistryProperties.java
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/JdbcRegistryDataManager.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/JdbcRegistryDataManager.java
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/JdbcRegistryLockManager.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/JdbcRegistryLockManager.java
diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/JdbcRegistryServer.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/main/java/org/apache/dolphinscheduler/plugin/registry/jdbc/server/JdbcRegistryServer.java
diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
