Skip to content

Commit 09d12f3

Browse files
fenyffenyf
authored
Merge pull request #4495 from fenyf/ai-sensitive
Implement AI-based automatic identification of sensitive columns in databases
2 parents d517c0f + ecb352e commit 09d12f3

File tree

62 files changed

+4396
-207
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+4396
-207
lines changed

‎.gitattributes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
* text=auto eol=lf

‎client‎

Submodule client updated 122 files

‎pom.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,17 @@
196196
<version>${springboot.version}</version>
197197
</dependency>
198198

199+
<!-- OpenAI SDK -->
200+
<dependency>
201+
<groupId>com.openai</groupId>
202+
<artifactId>openai-java</artifactId>
203+
<version>2.16.0</version>
204+
</dependency>
205+
<dependency>
206+
<groupId>org.jetbrains.kotlin</groupId>
207+
<artifactId>kotlin-stdlib-jdk7</artifactId>
208+
<version>1.9.24</version>
209+
</dependency>
199210

200211
<dependency>
201212
<groupId>org.codehaus.groovy</groupId>

‎server/integration-test/src/test/java/com/oceanbase/odc/service/datasecurity/SensitiveColumnScanningTaskManagerTest.java‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import com.oceanbase.odc.service.collaboration.project.model.Project;
4848
import com.oceanbase.odc.service.connection.database.model.Database;
4949
import com.oceanbase.odc.service.connection.model.ConnectionConfig;
50+
import com.oceanbase.odc.service.datasecurity.model.ScanningModeType;
5051
import com.oceanbase.odc.service.datasecurity.model.SensitiveColumnScanningTaskInfo;
5152
import com.oceanbase.odc.service.datasecurity.model.SensitiveColumnScanningTaskInfo.ScanningTaskStatus;
5253
import com.oceanbase.odc.service.datasecurity.model.SensitiveLevel;
@@ -107,7 +108,8 @@ public static void tearDown() {
107108
public void test_start_groovyRule_OBMySQL() {
108109
List<Database> databases = createDatabases(ConnectType.OB_MYSQL);
109110
List<SensitiveRule> rules = Arrays.asList(createGroovySensitiveRules());
110-
SensitiveColumnScanningTaskInfo taskInfo = manager.start(databases, rules, mysqlConnectionConfig, null);
111+
SensitiveColumnScanningTaskInfo taskInfo =
112+
manager.start(databases, rules, ScanningModeType.RULES_ONLY, mysqlConnectionConfig, null);
111113
await().atMost(20, SECONDS)
112114
.until(() -> manager.get(taskInfo.getTaskId()).getStatus() == ScanningTaskStatus.SUCCESS);
113115
Assert.assertEquals(2, manager.get(taskInfo.getTaskId()).getSensitiveColumns().size());
@@ -117,7 +119,8 @@ public void test_start_groovyRule_OBMySQL() {
117119
public void test_start_groovyRule_OBOracle() {
118120
List<Database> databases = createDatabases(ConnectType.OB_ORACLE);
119121
List<SensitiveRule> rules = Arrays.asList(createGroovySensitiveRules());
120-
SensitiveColumnScanningTaskInfo taskInfo = manager.start(databases, rules, oracleConnectionConfig, null);
122+
SensitiveColumnScanningTaskInfo taskInfo =
123+
manager.start(databases, rules, ScanningModeType.RULES_ONLY, oracleConnectionConfig, null);
121124
await().atMost(20, SECONDS)
122125
.until(() -> manager.get(taskInfo.getTaskId()).getStatus() == ScanningTaskStatus.SUCCESS);
123126
Assert.assertEquals(2, manager.get(taskInfo.getTaskId()).getSensitiveColumns().size());
@@ -127,7 +130,8 @@ public void test_start_groovyRule_OBOracle() {
127130
public void test_start_pathRule_OBMySQL() {
128131
List<Database> databases = createDatabases(ConnectType.OB_MYSQL);
129132
List<SensitiveRule> rules = Arrays.asList(createPathSensitiveRules());
130-
SensitiveColumnScanningTaskInfo taskInfo = manager.start(databases, rules, mysqlConnectionConfig, null);
133+
SensitiveColumnScanningTaskInfo taskInfo =
134+
manager.start(databases, rules, ScanningModeType.RULES_ONLY, mysqlConnectionConfig, null);
131135
await().atMost(20, SECONDS)
132136
.until(() -> manager.get(taskInfo.getTaskId()).getStatus() == ScanningTaskStatus.SUCCESS);
133137
Assert.assertEquals(20, manager.get(taskInfo.getTaskId()).getSensitiveColumns().size());
@@ -137,7 +141,8 @@ public void test_start_pathRule_OBMySQL() {
137141
public void test_start_pathRule_OBMOracle() {
138142
List<Database> databases = createDatabases(ConnectType.OB_ORACLE);
139143
List<SensitiveRule> rules = Arrays.asList(createPathSensitiveRules());
140-
SensitiveColumnScanningTaskInfo taskInfo = manager.start(databases, rules, oracleConnectionConfig, null);
144+
SensitiveColumnScanningTaskInfo taskInfo =
145+
manager.start(databases, rules, ScanningModeType.RULES_ONLY, oracleConnectionConfig, null);
141146
await().atMost(20, SECONDS)
142147
.until(() -> manager.get(taskInfo.getTaskId()).getStatus() == ScanningTaskStatus.SUCCESS);
143148
Assert.assertEquals(20, manager.get(taskInfo.getTaskId()).getSensitiveColumns().size());
@@ -147,7 +152,8 @@ public void test_start_pathRule_OBMOracle() {
147152
public void test_start_RegexRule_OBMySQL() {
148153
List<Database> databases = createDatabases(ConnectType.OB_MYSQL);
149154
List<SensitiveRule> rules = Arrays.asList(createRegexSensitiveRules(ConnectType.OB_MYSQL));
150-
SensitiveColumnScanningTaskInfo taskInfo = manager.start(databases, rules, mysqlConnectionConfig, null);
155+
SensitiveColumnScanningTaskInfo taskInfo =
156+
manager.start(databases, rules, ScanningModeType.RULES_ONLY, mysqlConnectionConfig, null);
151157
await().atMost(20, SECONDS)
152158
.until(() -> manager.get(taskInfo.getTaskId()).getStatus() == ScanningTaskStatus.SUCCESS);
153159
Assert.assertEquals(6, manager.get(taskInfo.getTaskId()).getSensitiveColumns().size());
@@ -157,7 +163,8 @@ public void test_start_RegexRule_OBMySQL() {
157163
public void test_start_RegexRule_OBOracle() {
158164
List<Database> databases = createDatabases(ConnectType.OB_ORACLE);
159165
List<SensitiveRule> rules = Arrays.asList(createRegexSensitiveRules(ConnectType.OB_ORACLE));
160-
SensitiveColumnScanningTaskInfo taskInfo = manager.start(databases, rules, oracleConnectionConfig, null);
166+
SensitiveColumnScanningTaskInfo taskInfo =
167+
manager.start(databases, rules, ScanningModeType.RULES_ONLY, oracleConnectionConfig, null);
161168
await().atMost(20, SECONDS)
162169
.until(() -> manager.get(taskInfo.getTaskId()).getStatus() == ScanningTaskStatus.SUCCESS);
163170
Assert.assertEquals(6, manager.get(taskInfo.getTaskId()).getSensitiveColumns().size());

‎server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/ErrorCodes.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,16 @@ public enum ErrorCodes implements ErrorCode {
334334
ExtractFileFailed,
335335
InvalidSignature,
336336

337+
/**
338+
* AI Service
339+
*/
340+
AIServiceNotAvailable,
341+
AIConfigurationIncomplete,
342+
AIClientNotInitialized,
343+
AIInferenceServiceError,
344+
AIResponseFormatError,
345+
AIResponseCountMismatch,
346+
337347

338348
;
339349

‎server/odc-core/src/main/resources/i18n/ErrorMessages.properties‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -212,4 +212,10 @@ com.oceanbase.odc.ErrorCodes.UpdateNotAllowed=Editing is not allowed in the curr
212212
com.oceanbase.odc.ErrorCodes.PauseNotAllowed=Disabling is not allowed in the current state, please check if there are any records in execution.
213213
com.oceanbase.odc.ErrorCodes.DeleteNotAllowed=Deletion is not allowed in the current state.
214214
com.oceanbase.odc.ErrorCodes.ExtractFileFailed=Failed to extract the file. Please check whether the file is correct. Details: {0}
215-
com.oceanbase.odc.ErrorCodes.InvalidSignature=File verification failed. Do not modify exported files or check if the correct key was used.
215+
com.oceanbase.odc.ErrorCodes.InvalidSignature=File verification failed. Do not modify exported files or check if the correct key was used.
216+
com.oceanbase.odc.ErrorCodes.AIServiceNotAvailable=AI service is not available. Please contact administrator to enable AI service.
217+
com.oceanbase.odc.ErrorCodes.AIConfigurationIncomplete=AI configuration is incomplete. Please contact administrator to configure AI parameters.
218+
com.oceanbase.odc.ErrorCodes.AIClientNotInitialized=AI client is not initialized. Please check AI configuration and restart service.
219+
com.oceanbase.odc.ErrorCodes.AIInferenceServiceError=Failed to call AI inference service. Details: {0}
220+
com.oceanbase.odc.ErrorCodes.AIResponseFormatError=AI response format is invalid. Details: {0}
221+
com.oceanbase.odc.ErrorCodes.AIResponseCountMismatch=AI response count does not match expected count. Expected: {0}, Actual: {1}

‎server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,4 +214,11 @@ com.oceanbase.odc.ErrorCodes.UnsupportedSyncTableStructure=结构同步暂不支
214214
com.oceanbase.odc.ErrorCodes.ScheduleIntervalTooShort=执行间隔配置过短，请重新配置。最小间隔为：{0} 秒
215215

216216
com.oceanbase.odc.ErrorCodes.ExtractFileFailed=提取文件失败，请确认文件是否正确，错误详情 {0}
217-
com.oceanbase.odc.ErrorCodes.InvalidSignature=文件验签不通过，请勿修改导出文件或检查密钥是否正确
217+
com.oceanbase.odc.ErrorCodes.InvalidSignature=文件验签不通过，请勿修改导出文件或检查密钥是否正确
218+
219+
com.oceanbase.odc.ErrorCodes.AIServiceNotAvailable=AI 服务不可用，请联系管理员启用 AI 服务
220+
com.oceanbase.odc.ErrorCodes.AIConfigurationIncomplete=AI 配置不完整，请联系管理员配置 AI 参数
221+
com.oceanbase.odc.ErrorCodes.AIClientNotInitialized=AI 客户端未初始化，请检查 AI 配置并重启服务
222+
com.oceanbase.odc.ErrorCodes.AIInferenceServiceError=调用 AI 推理服务失败，错误详情：{0}
223+
com.oceanbase.odc.ErrorCodes.AIResponseFormatError=AI 响应格式无效，错误详情：{0}
224+
com.oceanbase.odc.ErrorCodes.AIResponseCountMismatch=AI 响应数量与预期不符，预期：{0}，实际：{1}

‎server/odc-migrate/src/main/resources/migrate/common/V_4_3_4_20__alter_sensitive_rule.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
-- Add AI-related columns to table `data_security_sensitive_rule`
2+
alter table `data_security_sensitive_rule`
3+
add column `ai_sensitive_types` text default null comment 'A list of sensitive data types for AI rules, stored as a JSON array string.';
4+
5+
alter table `data_security_sensitive_rule`
6+
add column `ai_custom_prompt` text default null comment 'User-defined custom prompt for AI rules.';
7+

‎server/odc-migrate/src/main/resources/migrate/common/V_4_3_4_21__add_ai_system_config.sql‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
2+
INSERT INTO config_system_configuration(`key`, `value`, `description`)
3+
VALUES('odc.ai.enabled', 'false', 'Whether AI feature is enabled, disabled by default')
4+
ON DUPLICATE KEY UPDATE `id`=`id`;
5+
6+
INSERT INTO config_system_configuration(`key`, `value`, `description`)
7+
VALUES('odc.ai.api-key', '', 'AI API key, required when AI feature is enabled')
8+
ON DUPLICATE KEY UPDATE `id`=`id`;
9+
10+
INSERT INTO config_system_configuration(`key`, `value`, `description`)
11+
VALUES('odc.ai.base-url', 'https://api.openai.com', 'AI API base URL, defaults to OpenAI official API endpoint')
12+
ON DUPLICATE KEY UPDATE `id`=`id`;
13+
14+
INSERT INTO config_system_configuration(`key`, `value`, `description`)
15+
VALUES('odc.ai.model', 'gpt-3.5-turbo', 'AI model to use, defaults to gpt-3.5-turbo')
16+
ON DUPLICATE KEY UPDATE `id`=`id`;

‎server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/AIController.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Copyright (c) 2023 OceanBase.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.oceanbase.odc.server.web.controller.v2;
17+
18+
import org.springframework.beans.factory.annotation.Autowired;
19+
import org.springframework.web.bind.annotation.GetMapping;
20+
import org.springframework.web.bind.annotation.RequestMapping;
21+
import org.springframework.web.bind.annotation.RestController;
22+
23+
import com.oceanbase.odc.core.authority.util.SkipAuthorize;
24+
import com.oceanbase.odc.service.common.response.Responses;
25+
import com.oceanbase.odc.service.common.response.SuccessResponse;
26+
import com.oceanbase.odc.service.datasecurity.ai.AIConfig;
27+
import com.oceanbase.odc.service.datasecurity.ai.AIInferenceService;
28+
import com.oceanbase.odc.service.datasecurity.ai.AIStatusResponse;
29+
30+
import io.swagger.annotations.Api;
31+
import io.swagger.annotations.ApiOperation;
32+
33+
/**
34+
* @author fenyf
35+
* @date 2025/8/10 12:41
36+
*/
37+
@Api(tags = "AI")
38+
@RestController
39+
@RequestMapping("/api/v2/ai")
40+
public class AIController {
41+
42+
@Autowired
43+
private AIConfig aiConfig;
44+
45+
@Autowired
46+
private AIInferenceService aiInferenceService;
47+
48+
49+
@ApiOperation(value = "Query the status of the AI function",
50+
notes = "Return the status of whether the AI function is enabled and its configuration status")
51+
@SkipAuthorize("AI status is safe to query for authenticated users")
52+
@GetMapping("/status")
53+
public SuccessResponse<AIStatusResponse> getAIStatus() {
54+
AIStatusResponse response = new AIStatusResponse();
55+
response.setEnabled(aiConfig.isEnabled());
56+
response.setAvailable(aiInferenceService.isAIAvailable());
57+
response.setModel(aiConfig.getModel());
58+
response.setBaseUrl(aiConfig.getBaseUrl());
59+
response.setApiKeyConfigured(aiConfig.getApiKey() != null && !aiConfig.getApiKey().trim().isEmpty());
60+
61+
return Responses.success(response);
62+
}
63+
}

0 commit comments

Comments
 (0)