Skip to content

Commit 91f4333

Browse files
zijiacjzijiacj
committed
add external resource management feature
1 parent f66a330 commit 91f4333

File tree

8 files changed

+134
-17
lines changed

8 files changed

+134
-17
lines changed

‎server/odc-core/src/main/java/com/oceanbase/odc/core/shared/constant/AuditEventAction.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,8 @@ public enum AuditEventAction implements Translatable {
128128

129129
CREATE,
130130

131+
CALL,
132+
131133
OTHERS,
132134

133135
/**

‎server/odc-migrate/src/main/resources/init-config/default-audit-event-meta.yml‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -587,4 +587,27 @@
587587
sid_extract_expression: ""
588588
in_connection: 0
589589
enabled: 1
590-
id: 104
590+
id: 104
591+
592+
# 外部资源管理
593+
- type: "DATABASE_OPERATION"
594+
action: "CALL"
595+
method_signature: "com.oceanbase.odc.server.web.controller.v2.DBExternalResourceController.deleteExternalResource"
596+
sid_extract_expression: "{#sessionId}"
597+
in_connection: 1
598+
enabled: 1
599+
id: 105
600+
- type: "DATABASE_OPERATION"
601+
action: "CALL"
602+
method_signature: "com.oceanbase.odc.server.web.controller.v2.DBExternalResourceController.uploadExternalResource"
603+
sid_extract_expression: "{#sessionId}"
604+
in_connection: 1
605+
enabled: 1
606+
id: 106
607+
- type: "DATABASE_OPERATION"
608+
action: "SELECT"
609+
method_signature: "com.oceanbase.odc.server.web.controller.v2.DBExternalResourceController.downloadExternalResource"
610+
sid_extract_expression: "{#sessionId}"
611+
in_connection: 1
612+
enabled: 1
613+
id: 107

‎server/odc-server/src/main/java/com/oceanbase/odc/server/web/controller/v2/DBExternalResourceController.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import com.oceanbase.odc.service.common.response.ListResponse;
3535
import com.oceanbase.odc.service.common.response.Responses;
3636
import com.oceanbase.odc.service.common.response.SuccessResponse;
37+
import com.oceanbase.odc.service.common.util.SidUtils;
3738
import com.oceanbase.odc.service.db.DBExternalResourceService;
3839
import com.oceanbase.odc.service.db.model.DBExternalResourceReq;
3940
import com.oceanbase.odc.service.db.model.DBExternalResourceUploadReq;
@@ -66,7 +67,7 @@ public class DBExternalResourceController {
6667
@StatefulRoute(stateName = StateName.DB_SESSION, stateIdExpression = "#sessionId")
6768
public ListResponse<DBObjectIdentity> list(@PathVariable String sessionId,
6869
@PathVariable String databaseName) {
69-
ConnectionSession session = sessionService.nullSafeGet(sessionId, true);
70+
ConnectionSession session = sessionService.nullSafeGet(SidUtils.getSessionId(sessionId), true);
7071
return Responses.list(externalResourceService.list(session, databaseName));
7172
}
7273

@@ -79,7 +80,7 @@ public SuccessResponse<DBExternalResource> detail(@PathVariable String sessionId
7980
@RequestBody DBExternalResourceReq req) throws IOException {
8081
req.setName(resourceName);
8182
req.setSchemaName(databaseName);
82-
ConnectionSession session = sessionService.nullSafeGet(sessionId, true);
83+
ConnectionSession session = sessionService.nullSafeGet(SidUtils.getSessionId(sessionId), true);
8384
return Responses.success(externalResourceService.detail(session, req));
8485
}
8586

@@ -92,7 +93,7 @@ public SuccessResponse<Boolean> uploadExternalResource(@PathVariable String sess
9293
@PathVariable String resourceName,
9394
@RequestPart("req") DBExternalResourceUploadReq req,
9495
@RequestPart("file") MultipartFile file) {
95-
ConnectionSession session = sessionService.nullSafeGet(sessionId, true);
96+
ConnectionSession session = sessionService.nullSafeGet(SidUtils.getSessionId(sessionId), true);
9697
req.setSchemaName(databaseName);
9798
req.setName(resourceName);
9899
return Responses.success(externalResourceService.upload(session, req, file));
@@ -105,7 +106,7 @@ public SuccessResponse<Boolean> uploadExternalResource(@PathVariable String sess
105106
public ResponseEntity<InputStreamResource> downloadExternalResource(@PathVariable String sessionId,
106107
@PathVariable String databaseName,
107108
@PathVariable String resourceName) throws IOException {
108-
ConnectionSession session = sessionService.nullSafeGet(sessionId, true);
109+
ConnectionSession session = sessionService.nullSafeGet(SidUtils.getSessionId(sessionId), true);
109110
return externalResourceService.download(session, databaseName, resourceName);
110111
}
111112

@@ -116,7 +117,7 @@ public ResponseEntity<InputStreamResource> downloadExternalResource(@PathVariabl
116117
public SuccessResponse<Boolean> deleteExternalResource(@PathVariable String sessionId,
117118
@PathVariable String databaseName,
118119
@PathVariable String resourceName) {
119-
ConnectionSession session = sessionService.nullSafeGet(sessionId, true);
120+
ConnectionSession session = sessionService.nullSafeGet(SidUtils.getSessionId(sessionId), true);
120121
return Responses.success(externalResourceService.drop(session, databaseName, resourceName));
121122
}
122123

‎server/odc-service/src/main/java/com/oceanbase/odc/service/audit/AuditEventAspect.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -353,9 +353,14 @@ private String parseDetailFromApiParams(Map<String, Object> apiParams) {
353353
// 脚本上传，记录上传的文件名
354354
if (Objects.nonNull(apiParams.get("files"))) {
355355
List<MultipartFile> files = (List<MultipartFile>) apiParams.get("files");
356-
jsonStr = JsonUtils
357-
.toJson(files.stream().map(MultipartFile::getOriginalFilename).collect(Collectors.toList()));
356+
apiParams.put("files",
357+
files.stream().map(MultipartFile::getOriginalFilename).collect(Collectors.toList()));
358358
}
359+
if (Objects.nonNull(apiParams.get("file"))) {
360+
MultipartFile file = (MultipartFile) apiParams.get("file");
361+
apiParams.put("file", file.getOriginalFilename());
362+
}
363+
jsonStr = JsonUtils.toJson(apiParams);
359364
// audit_event 表的 detail 字段不能为空，除了脚本上传的场景，现在都是可序列化的。
360365
// 稳妥起见，这里返回一个指定字符串防止插入失败
361366
if (StringUtils.isEmpty(jsonStr)) {

‎server/odc-service/src/main/java/com/oceanbase/odc/service/db/DBExternalResourceService.java‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -133,11 +133,14 @@ public ResponseEntity<InputStreamResource> download(@NotNull ConnectionSession c
133133
.allMatch(o -> !StringUtils.equals(o.getName(), resourceName))) {
134134
throw new IllegalArgumentException(String.format("Resource %s does not exist", resourceName));
135135
}
136-
DBExternalResource resource = connectionSession.getSyncJdbcExecutor(
137-
ConnectionSessionConstants.BACKEND_DS_KEY)
138-
.execute((ConnectionCallback<DBExternalResource>) con -> getExternalResourceExtensionPoint(
139-
connectionSession)
140-
.getDetail(con, schemaName, resourceName));
136+
DBExternalResource resource = objectStorageExecutor.concurrentSafeExecute(() -> connectionSession
137+
.getSyncJdbcExecutor(
138+
ConnectionSessionConstants.BACKEND_DS_KEY)
139+
.execute(
140+
(ConnectionCallback<DBExternalResource>) con -> DBExternalResourceService.this
141+
.getExternalResourceExtensionPoint(
142+
connectionSession)
143+
.getDetail(con, schemaName, resourceName)));
141144
if (resource.getSize() > properties.getDownloadLimitBytes()) {
142145
resource.getInputStream().close();
143146
throw new IllegalStateException(String.format(
@@ -213,7 +216,7 @@ private String generateFileName(String schemaName, String resourceName, DBExtern
213216
} else if (DBExternalResourceType.PYTHON_PY == type) {
214217
return schemaName + "_" + resourceName + ".py";
215218
} else {
216-
throw new UnsupportedException(String.format("unsupported resource type %s", type));
219+
throw new UnsupportedException(String.format("unsupported external resource type %s", type));
217220
}
218221
}
219222

‎server/odc-service/src/main/java/com/oceanbase/odc/service/db/model/DBExternalResourceUploadReq.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,4 +42,6 @@ public class DBExternalResourceUploadReq {
4242
@NotNull(message = "type cannot be null")
4343
private DBExternalResourceType type;
4444

45+
private String preparedStatement = "call dbms_java.loadjava(?, ?, ?)";
46+
4547
}

‎server/odc-service/src/main/java/com/oceanbase/odc/service/db/schema/syncer/object/DBExternalResourceSyncer.java‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Copyright (c) 2023 OceanBase.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.oceanbase.odc.service.db.schema.syncer.object;
17+
18+
import java.sql.Connection;
19+
import java.util.Set;
20+
import java.util.stream.Collectors;
21+
22+
import org.springframework.beans.factory.annotation.Autowired;
23+
import org.springframework.stereotype.Component;
24+
25+
import com.oceanbase.odc.core.shared.constant.DialectType;
26+
import com.oceanbase.odc.plugin.connect.api.InformationExtensionPoint;
27+
import com.oceanbase.odc.plugin.schema.api.ExternalResourceExtensionPoint;
28+
import com.oceanbase.odc.service.connection.database.model.Database;
29+
import com.oceanbase.odc.service.feature.VersionDiffConfigService;
30+
import com.oceanbase.odc.service.plugin.ConnectionPluginUtil;
31+
import com.oceanbase.tools.dbbrowser.model.DBObjectIdentity;
32+
import com.oceanbase.tools.dbbrowser.model.DBObjectType;
33+
34+
import lombok.NonNull;
35+
import lombok.extern.slf4j.Slf4j;
36+
37+
/**
38+
* @description:
39+
* @author: zijia.cj
40+
* @date: 2025/9/1 11:02
41+
* @since: 4.4.1
42+
*/
43+
@Component
44+
@Slf4j
45+
public class DBExternalResourceSyncer extends AbstractDBObjectSyncer<ExternalResourceExtensionPoint> {
46+
47+
@Autowired
48+
private VersionDiffConfigService versionDiffConfigService;
49+
50+
@Override
51+
Set<String> getLatestObjectNames(@NonNull ExternalResourceExtensionPoint extensionPoint,
52+
@NonNull Connection connection, @NonNull Database database) {
53+
return extensionPoint.list(connection, database.getName()).stream().map(DBObjectIdentity::getName)
54+
.collect(Collectors.toSet());
55+
}
56+
57+
@Override
58+
Class<ExternalResourceExtensionPoint> getExtensionPointClass() {
59+
return ExternalResourceExtensionPoint.class;
60+
}
61+
62+
@Override
63+
public DBObjectType getObjectType() {
64+
return DBObjectType.EXTERNAL_RESOURCE;
65+
}
66+
67+
@Override
68+
public boolean supports(@NonNull DialectType dialectType, @NonNull Connection connection) {
69+
try {
70+
InformationExtensionPoint point =
71+
ConnectionPluginUtil.getInformationExtension(dialectType);
72+
String databaseProductVersion = point.getDBVersion(connection);
73+
return versionDiffConfigService.isExternalTableSupported(dialectType, databaseProductVersion)
74+
&& getExtensionPoint(dialectType) != null;
75+
} catch (Exception e) {
76+
log.warn("check external resource support failed", e);
77+
return false;
78+
}
79+
}
80+
81+
}

‎server/odc-service/src/main/java/com/oceanbase/odc/service/feature/VersionDiffConfigService.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ public class VersionDiffConfigService {
4949
public static final String SUPPORT_EXTERNAL_TABLE = "support_external_table";
5050
public static final String SUPPORT_MATERIALIZED_VIEW = "support_materialized_view";
5151
public static final String SUPPORT_MATERIALIZED_VIEW_LOG = "support_materialized_view_log";
52-
public static final String SUPPORT_JAVA_UDF = "support_java_udf";
52+
public static final String SUPPORT_EXTERNAL_RESOURCE = "support_external_resource";
5353
private static final String SUPPORT_PREFIX = "support";
5454
private static final String MAX_SUPPORT_KILL_OB_VERSION =
5555
"odc.session.kill-query-or-session.max-supported-ob-version";
@@ -175,8 +175,8 @@ public boolean isMViewLogSupported(@NonNull DialectType dialectType, @NonNull St
175175
return isFeatureSupported(dialectType, SUPPORT_MATERIALIZED_VIEW_LOG, versionNumber);
176176
}
177177

178-
public boolean isJavaUdfSupported(@NonNull DialectType dialectType, @NonNull String versionNumber) {
179-
return isFeatureSupported(dialectType, SUPPORT_JAVA_UDF, versionNumber);
178+
public boolean isExternalResourceSupported(@NonNull DialectType dialectType, @NonNull String versionNumber) {
179+
return isFeatureSupported(dialectType, SUPPORT_EXTERNAL_RESOURCE, versionNumber);
180180
}
181181

182182
private boolean isFeatureSupported(DialectType dialectType, String configKey, String versionNumber) {

0 commit comments

Comments
 (0)