diff --git a/document/docs/02-administration/deployment/configuration.md b/document/docs/02-administration/deployment/configuration.md index e89b3389..5228bc52 100644 --- a/document/docs/02-administration/deployment/configuration.md +++ b/document/docs/02-administration/deployment/configuration.md @@ -44,6 +44,10 @@ SkillHub 通过环境变量进行配置，主要配置项如下： | `SKILLHUB_STORAGE_S3_BUCKET` | S3 桶名 | - | | `SKILLHUB_STORAGE_S3_ACCESS_KEY` | S3 Access Key | - | | `SKILLHUB_STORAGE_S3_SECRET_KEY` | S3 Secret Key | - | +| `SKILLHUB_STORAGE_S3_MAX_CONNECTIONS` | S3 HTTP 连接池大小 | `100` | +| `SKILLHUB_STORAGE_S3_CONNECTION_ACQUISITION_TIMEOUT` | 等待连接池连接超时，ISO-8601 Duration | `PT2S` | +| `SKILLHUB_STORAGE_S3_API_CALL_ATTEMPT_TIMEOUT` | 单次 S3 请求尝试超时，ISO-8601 Duration | `PT10S` | +| `SKILLHUB_STORAGE_S3_API_CALL_TIMEOUT` | 整个 S3 请求总超时，ISO-8601 Duration | `PT30S` | ### OAuth 配置 @@ -64,6 +68,13 @@ SkillHub 通过环境变量进行配置，主要配置项如下： Spring Boot 配置文件位于 `server/skillhub-app/src/main/resources/`。 +## 下载与对象存储建议 + +- 生产环境建议显式配置 S3 连接池和超时参数，避免对象存储抖动时大量线程阻塞在连接池等待。 +- 版本列表接口现在使用数据库中的下载状态元数据，不再实时探测对象存储；部署时请确保数据库迁移已经执行。 +- 下载接口按用户/IP + namespace + slug + version/tag 进行更细粒度限流。如有批量分发场景，建议在入口层或专用账号策略中单独放宽，不建议直接关闭应用层限流。 + ## 下一步 - [认证配置](../security/authentication) - 配置身份认证 +- [下载与对象存储排查](../../05-reference/download-troubleshooting) - 排查下载、限流和对象存储故障 diff --git a/document/docs/05-reference/download-troubleshooting.md b/document/docs/05-reference/download-troubleshooting.md new file mode 100644 index 00000000..6de07313 --- /dev/null +++ b/document/docs/05-reference/download-troubleshooting.md @@ -0,0 +1,74 @@ +--- +title: 下载与对象存储排查 +sidebar_position: 6 +description: 排查下载接口、版本列表下载可用性和对象存储连接池问题 +--- + +# 下载与对象存储排查 + +本页用于排查下载接口、版本列表中的下载可用性，以及对象存储连接池耗尽等问题。 + +## 当前行为 + +- 版本列表接口不再实时访问对象存储，而是读取数据库里的 `skill_version.download_ready` +- 下载接口优先返回预签名 URL；只有需要代理流式下载时才真正打开对象流 +- 当 bundle 缺失时，服务会记录告警，并回退到逐文件重新打包 + +## 重点指标 + +- `skillhub.skill.download.delivery` + - 标签：`mode=redirect|stream` + - 标签：`fallback_bundle=true|false` +- `skillhub.skill.download.bundle_missing_fallback` + - bundle 缺失并触发逐文件回退打包的次数 +- `skillhub.storage.failure` + - 标签：`operation=exists|getMetadata|getObject|putObject|deleteObject|deleteObjects|generatePresignedUrl` +- `skillhub.ratelimit.exceeded` + - 标签：`category=download|search|publish|...` + +## 常见症状 + +### 下载返回 503 + +接口现在会把对象存储访问失败映射为 `503 error.storage.unavailable`，不再统一返回 500。 + +优先检查： + +- `skillhub.storage.failure` 是否持续上升 +- 对象存储端点、凭证、桶名是否正确 +- `SKILLHUB_STORAGE_S3_MAX_CONNECTIONS` +- `SKILLHUB_STORAGE_S3_CONNECTION_ACQUISITION_TIMEOUT` +- `SKILLHUB_STORAGE_S3_API_CALL_ATTEMPT_TIMEOUT` +- `SKILLHUB_STORAGE_S3_API_CALL_TIMEOUT` + +### 日志出现 bundle missing fallback + +说明数据库里该版本仍被认为可下载，但对象存储中的 `packages/{skillId}/{versionId}/bundle.zip` 已缺失。 + +影响： + +- 下载仍可通过逐文件打包完成 +- 性能会明显差于直接下载 bundle + +建议： + +- 优先补齐缺失 bundle +- 观察 `skillhub.skill.download.bundle_missing_fallback` +- 检查对象存储生命周期规则、人工清理脚本或历史数据不一致问题 + +### 用户频繁点击下载后被限流 + +下载限流键现在包含： + +- 用户或 IP +- namespace +- slug +- version 或 tag + +这样同一用户反复下载同一版本时，不会把所有下载请求混进一个粗粒度桶里。 + +如果确实需要更高吞吐： + +- 在入口层做白名单 +- 为自动化分发账号单独放宽策略 +- 不建议直接移除应用层下载限流 diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/SkillController.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/SkillController.java index c70948fc..8c7964d3 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/SkillController.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/controller/portal/SkillController.java @@ -14,6 +14,7 @@ import com.iflytek.skillhub.dto.SkillFileResponse; import com.iflytek.skillhub.dto.SkillVersionDetailResponse; import com.iflytek.skillhub.dto.SkillVersionResponse; +import com.iflytek.skillhub.metrics.SkillHubMetrics; import com.iflytek.skillhub.ratelimit.RateLimit; import org.springframework.core.io.InputStreamResource; import org.springframework.data.domain.Page; @@ -37,14 +38,17 @@ public class SkillController extends BaseApiController { private final SkillQueryService skillQueryService; private final SkillDownloadService skillDownloadService; + private final SkillHubMetrics metrics; public SkillController( SkillQueryService skillQueryService, SkillDownloadService skillDownloadService, + SkillHubMetrics metrics, ApiResponseFactory responseFactory) { super(responseFactory); this.skillQueryService = skillQueryService; this.skillDownloadService = skillDownloadService; + this.metrics = metrics; } @GetMapping("/{namespace}/{slug}") @@ -281,7 +285,7 @@ public ApiResponse resolveVersion( } @GetMapping("/{namespace}/{slug}/download") - @RateLimit(category = "download", authenticated = 120, anonymous = 30) + @RateLimit(category = "download", authenticated = 30, anonymous = 10) public ResponseEntity downloadLatest( @PathVariable String namespace, @PathVariable String slug, @@ -296,7 +300,7 @@ public ResponseEntity downloadLatest( } @GetMapping("/{namespace}/{slug}/versions/{version}/download") - @RateLimit(category = "download", authenticated = 120, anonymous = 30) + @RateLimit(category = "download", authenticated = 30, anonymous = 10) public ResponseEntity downloadVersion( @PathVariable String namespace, @PathVariable String slug, @@ -312,7 +316,7 @@ public ResponseEntity downloadVersion( } @GetMapping("/{namespace}/{slug}/tags/{tagName}/download") - @RateLimit(category = "download", authenticated = 120, anonymous = 30) + @RateLimit(category = "download", authenticated = 30, anonymous = 10) public ResponseEntity downloadByTag( @PathVariable String namespace, @PathVariable String slug, @@ -329,16 +333,24 @@ public ResponseEntity downloadByTag( private ResponseEntity buildDownloadResponse(HttpServletRequest request, SkillDownloadService.DownloadResult result) { if (shouldRedirectToPresignedUrl(request, result.presignedUrl())) { + metrics.recordDownloadDelivery("redirect", result.fallbackBundle()); + if (result.fallbackBundle()) { + metrics.incrementBundleMissingFallback(); + } return ResponseEntity.status(HttpStatus.FOUND) .header(HttpHeaders.LOCATION, result.presignedUrl()) .build(); } + metrics.recordDownloadDelivery("stream", result.fallbackBundle()); + if (result.fallbackBundle()) { + metrics.incrementBundleMissingFallback(); + } return ResponseEntity.ok() .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + result.filename() + "\"") .contentType(MediaType.parseMediaType(result.contentType())) .contentLength(result.contentLength()) - .body(new InputStreamResource(result.content())); + .body(new InputStreamResource(result.openContent())); } private boolean shouldRedirectToPresignedUrl(HttpServletRequest request, String presignedUrl) { diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/exception/GlobalExceptionHandler.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/exception/GlobalExceptionHandler.java index a1694498..ebf0f43c 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/exception/GlobalExceptionHandler.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/exception/GlobalExceptionHandler.java @@ -7,7 +7,9 @@ import com.iflytek.skillhub.domain.shared.exception.DomainBadRequestException; import com.iflytek.skillhub.domain.shared.exception.DomainForbiddenException; import com.iflytek.skillhub.domain.shared.exception.DomainNotFoundException; +import com.iflytek.skillhub.metrics.SkillHubMetrics; import com.iflytek.skillhub.security.SensitiveLogSanitizer; +import com.iflytek.skillhub.storage.StorageAccessException; import jakarta.servlet.http.HttpServletRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,11 +29,14 @@ public class GlobalExceptionHandler { private static final Logger logger = LoggerFactory.getLogger(GlobalExceptionHandler.class); private final ApiResponseFactory apiResponseFactory; private final SensitiveLogSanitizer sensitiveLogSanitizer; + private final SkillHubMetrics metrics; public GlobalExceptionHandler(ApiResponseFactory apiResponseFactory, - SensitiveLogSanitizer sensitiveLogSanitizer) { + SensitiveLogSanitizer sensitiveLogSanitizer, + SkillHubMetrics metrics) { this.apiResponseFactory = apiResponseFactory; this.sensitiveLogSanitizer = sensitiveLogSanitizer; + this.metrics = metrics; } @ExceptionHandler(LocalizedException.class) @@ -108,6 +113,23 @@ public ResponseEntity> handleAccessDenied(AccessDeniedExceptio apiResponseFactory.error(403, "error.forbidden")); } + @ExceptionHandler(StorageAccessException.class) + public ResponseEntity> handleStorageAccess(StorageAccessException ex, HttpServletRequest request) { + metrics.incrementStorageAccessFailure(ex.getOperation()); + logger.warn( + "Object storage unavailable [requestId={}, method={}, path={}, userId={}, operation={}, key={}]", + MDC.get("requestId"), + request.getMethod(), + sensitiveLogSanitizer.sanitizeRequestTarget(request), + resolveUserId(request), + ex.getOperation(), + ex.getKey(), + ex + ); + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE).body( + apiResponseFactory.error(503, "error.storage.unavailable")); + } + @ExceptionHandler(Exception.class) public ResponseEntity> handleGlobalException(Exception ex, HttpServletRequest request) { logger.error( diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/SkillHubMetrics.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/SkillHubMetrics.java index 75e24c35..ab9714fc 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/SkillHubMetrics.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/metrics/SkillHubMetrics.java @@ -31,4 +31,30 @@ public void incrementSkillPublish(String namespace, String status) { "status", status ).increment(); } + + public void recordDownloadDelivery(String mode, boolean fallbackBundle) { + meterRegistry.counter( + "skillhub.skill.download.delivery", + "mode", mode, + "fallback_bundle", Boolean.toString(fallbackBundle) + ).increment(); + } + + public void incrementBundleMissingFallback() { + meterRegistry.counter("skillhub.skill.download.bundle_missing_fallback").increment(); + } + + public void incrementRateLimitExceeded(String category) { + meterRegistry.counter( + "skillhub.ratelimit.exceeded", + "category", category + ).increment(); + } + + public void incrementStorageAccessFailure(String operation) { + meterRegistry.counter( + "skillhub.storage.failure", + "operation", operation + ).increment(); + } } diff --git a/server/skillhub-app/src/main/java/com/iflytek/skillhub/ratelimit/RateLimitInterceptor.java b/server/skillhub-app/src/main/java/com/iflytek/skillhub/ratelimit/RateLimitInterceptor.java index f1cf5ef7..017d573c 100644 --- a/server/skillhub-app/src/main/java/com/iflytek/skillhub/ratelimit/RateLimitInterceptor.java +++ b/server/skillhub-app/src/main/java/com/iflytek/skillhub/ratelimit/RateLimitInterceptor.java @@ -3,27 +3,34 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.iflytek.skillhub.dto.ApiResponse; import com.iflytek.skillhub.dto.ApiResponseFactory; +import com.iflytek.skillhub.metrics.SkillHubMetrics; import jakarta.servlet.http.HttpServletRequest; import jakarta.servlet.http.HttpServletResponse; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.stereotype.Component; +import org.springframework.web.servlet.HandlerMapping; import org.springframework.web.method.HandlerMethod; import org.springframework.web.servlet.HandlerInterceptor; +import java.util.Map; + @Component public class RateLimitInterceptor implements HandlerInterceptor { private final RateLimiter rateLimiter; private final ApiResponseFactory apiResponseFactory; private final ObjectMapper objectMapper; + private final SkillHubMetrics metrics; public RateLimitInterceptor(RateLimiter rateLimiter, ApiResponseFactory apiResponseFactory, - ObjectMapper objectMapper) { + ObjectMapper objectMapper, + SkillHubMetrics metrics) { this.rateLimiter = rateLimiter; this.apiResponseFactory = apiResponseFactory; this.objectMapper = objectMapper; + this.metrics = metrics; } @Override @@ -48,12 +55,13 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons // Build rate limit key String identifier = isAuthenticated ? "user:" + userId : "ip:" + getClientIp(request); - String key = "ratelimit:" + rateLimit.category() + ":" + identifier; + String key = "ratelimit:" + rateLimit.category() + ":" + identifier + resolveResourceSuffix(rateLimit.category(), request); // Check rate limit boolean allowed = rateLimiter.tryAcquire(key, limit, rateLimit.windowSeconds()); if (!allowed) { + metrics.incrementRateLimitExceeded(rateLimit.category()); response.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); response.setContentType(MediaType.APPLICATION_JSON_VALUE); ApiResponse body = apiResponseFactory.error(429, "error.rateLimit.exceeded"); @@ -78,4 +86,32 @@ private String getClientIp(HttpServletRequest request) { } return ip; } + + @SuppressWarnings("unchecked") + private String resolveResourceSuffix(String category, HttpServletRequest request) { + if (!"download".equals(category)) { + return ""; + } + Object attribute = request.getAttribute(HandlerMapping.URI_TEMPLATE_VARIABLES_ATTRIBUTE); + if (!(attribute instanceof Map templateVariables)) { + return ""; + } + String namespace = stringValue(templateVariables.get("namespace")); + String slug = stringValue(templateVariables.get("slug")); + String version = stringValue(templateVariables.get("version")); + String tagName = stringValue(templateVariables.get("tagName")); + if (namespace == null || slug == null) { + return ""; + } + String target = version != null ? "version:" + version : tagName != null ? "tag:" + tagName : "latest"; + return ":ns:" + namespace + ":slug:" + slug + ":" + target; + } + + private String stringValue(Object value) { + if (value == null) { + return null; + } + String text = value.toString(); + return text.isBlank() ? null : text; + } } diff --git a/server/skillhub-app/src/main/resources/application.yml b/server/skillhub-app/src/main/resources/application.yml index c5e36b50..edf36524 100644 --- a/server/skillhub-app/src/main/resources/application.yml +++ b/server/skillhub-app/src/main/resources/application.yml @@ -84,6 +84,10 @@ skillhub: force-path-style: ${SKILLHUB_STORAGE_S3_FORCE_PATH_STYLE:true} auto-create-bucket: ${SKILLHUB_STORAGE_S3_AUTO_CREATE_BUCKET:false} presign-expiry: ${SKILLHUB_STORAGE_S3_PRESIGN_EXPIRY:PT10M} + max-connections: ${SKILLHUB_STORAGE_S3_MAX_CONNECTIONS:100} + connection-acquisition-timeout: ${SKILLHUB_STORAGE_S3_CONNECTION_ACQUISITION_TIMEOUT:PT2S} + api-call-attempt-timeout: ${SKILLHUB_STORAGE_S3_API_CALL_ATTEMPT_TIMEOUT:PT10S} + api-call-timeout: ${SKILLHUB_STORAGE_S3_API_CALL_TIMEOUT:PT30S} search: engine: postgres rebuild-on-startup: false diff --git a/server/skillhub-app/src/main/resources/db/migration/V14__skill_version_download_state.sql b/server/skillhub-app/src/main/resources/db/migration/V14__skill_version_download_state.sql new file mode 100644 index 00000000..0384d407 --- /dev/null +++ b/server/skillhub-app/src/main/resources/db/migration/V14__skill_version_download_state.sql @@ -0,0 +1,10 @@ +ALTER TABLE skill_version + ADD COLUMN bundle_ready BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN download_ready BOOLEAN NOT NULL DEFAULT FALSE; + +UPDATE skill_version +SET download_ready = CASE + WHEN status = 'PUBLISHED' AND file_count > 0 THEN TRUE + ELSE FALSE + END, + bundle_ready = FALSE; diff --git a/server/skillhub-app/src/main/resources/messages.properties b/server/skillhub-app/src/main/resources/messages.properties index f2852874..a4604545 100644 --- a/server/skillhub-app/src/main/resources/messages.properties +++ b/server/skillhub-app/src/main/resources/messages.properties @@ -49,6 +49,7 @@ error.badRequest=Invalid request error.forbidden=Forbidden error.rateLimit.exceeded=Rate limit exceeded error.internal=An unexpected error occurred +error.storage.unavailable=Object storage is temporarily unavailable error.slug.blank=Slug cannot be blank error.slug.length=Slug length must be between {0} and {1} characters error.slug.pattern=Slug must contain only lowercase letters, numbers, and hyphens, and must start and end with a letter or number diff --git a/server/skillhub-app/src/main/resources/messages_zh.properties b/server/skillhub-app/src/main/resources/messages_zh.properties index 5c652c66..cd5a3b6d 100644 --- a/server/skillhub-app/src/main/resources/messages_zh.properties +++ b/server/skillhub-app/src/main/resources/messages_zh.properties @@ -49,6 +49,7 @@ error.badRequest=请求参数不合法 error.forbidden=没有权限执行该操作 error.rateLimit.exceeded=请求过于频繁，请稍后再试 error.internal=服务器内部错误 +error.storage.unavailable=对象存储暂时不可用，请稍后再试 error.slug.blank=slug 不能为空 error.slug.length=slug 长度必须在 {0} 到 {1} 个字符之间 error.slug.pattern=slug 只能包含小写字母、数字和连字符，且必须以字母或数字开头和结尾 diff --git a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/portal/SkillControllerDownloadTest.java b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/portal/SkillControllerDownloadTest.java index 369155a9..498e97ad 100644 --- a/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/portal/SkillControllerDownloadTest.java +++ b/server/skillhub-app/src/test/java/com/iflytek/skillhub/controller/portal/SkillControllerDownloadTest.java @@ -13,6 +13,9 @@ import com.iflytek.skillhub.domain.shared.exception.DomainForbiddenException; import com.iflytek.skillhub.domain.skill.service.SkillDownloadService; import com.iflytek.skillhub.domain.skill.service.SkillQueryService; +import com.iflytek.skillhub.metrics.SkillHubMetrics; +import com.iflytek.skillhub.ratelimit.RateLimiter; +import com.iflytek.skillhub.storage.StorageAccessException; import java.io.ByteArrayInputStream; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -23,6 +26,10 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.web.servlet.MockMvc; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.verify; + @SpringBootTest @AutoConfigureMockMvc @ActiveProfiles("test") @@ -44,15 +51,23 @@ class SkillControllerDownloadTest { @MockBean private DeviceAuthService deviceAuthService; + @MockBean + private SkillHubMetrics skillHubMetrics; + + @MockBean + private RateLimiter rateLimiter; + @Test void downloadVersion_redirectsToPresignedUrlWhenAvailable() throws Exception { + given(rateLimiter.tryAcquire(anyString(), anyInt(), anyInt())).willReturn(true); given(skillDownloadService.downloadVersion("global", "demo-skill", "1.0.0", null, java.util.Map.of())) .willReturn(new SkillDownloadService.DownloadResult( - new ByteArrayInputStream("zip".getBytes()), + () -> new ByteArrayInputStream("zip".getBytes()), "demo-skill-1.0.0.zip", 128L, "application/zip", - "https://download.example/presigned" + "https://download.example/presigned", + false )); mockMvc.perform(get("/api/v1/skills/global/demo-skill/versions/1.0.0/download") @@ -64,13 +79,15 @@ void downloadVersion_redirectsToPresignedUrlWhenAvailable() throws Exception { @Test void downloadVersion_streamsWhenPresignedUrlIsInsecureForHttpsRequest() throws Exception { + given(rateLimiter.tryAcquire(anyString(), anyInt(), anyInt())).willReturn(true); given(skillDownloadService.downloadVersion("global", "demo-skill", "1.0.0", null, java.util.Map.of())) .willReturn(new SkillDownloadService.DownloadResult( - new ByteArrayInputStream("zip".getBytes()), + () -> new ByteArrayInputStream("zip".getBytes()), "demo-skill-1.0.0.zip", 3L, "application/zip", - "http://download.example/presigned" + "http://download.example/presigned", + false )); mockMvc.perform(get("/api/v1/skills/global/demo-skill/versions/1.0.0/download") @@ -83,13 +100,15 @@ void downloadVersion_streamsWhenPresignedUrlIsInsecureForHttpsRequest() throws E @Test void downloadVersion_streamsWhenPresignedUrlUnavailable() throws Exception { + given(rateLimiter.tryAcquire(anyString(), anyInt(), anyInt())).willReturn(true); given(skillDownloadService.downloadVersion("global", "demo-skill", "1.0.0", null, java.util.Map.of())) .willReturn(new SkillDownloadService.DownloadResult( - new ByteArrayInputStream("zip".getBytes()), + () -> new ByteArrayInputStream("zip".getBytes()), "demo-skill-1.0.0.zip", 3L, "application/zip", - null + null, + false )); mockMvc.perform(get("/api/v1/skills/global/demo-skill/versions/1.0.0/download") @@ -101,13 +120,15 @@ void downloadVersion_streamsWhenPresignedUrlUnavailable() throws Exception { @Test void downloadVersion_allowsAnonymousForGlobalSkill() throws Exception { + given(rateLimiter.tryAcquire(anyString(), anyInt(), anyInt())).willReturn(true); given(skillDownloadService.downloadVersion("global", "demo-skill", "1.0.0", null, java.util.Map.of())) .willReturn(new SkillDownloadService.DownloadResult( - new ByteArrayInputStream("zip".getBytes()), + () -> new ByteArrayInputStream("zip".getBytes()), "demo-skill-1.0.0.zip", 3L, "application/zip", - null + null, + false )); mockMvc.perform(get("/api/v1/skills/global/demo-skill/versions/1.0.0/download") @@ -118,6 +139,7 @@ void downloadVersion_allowsAnonymousForGlobalSkill() throws Exception { @Test void downloadVersion_forbidsAnonymousWhenServiceRejectsSkill() throws Exception { + given(rateLimiter.tryAcquire(anyString(), anyInt(), anyInt())).willReturn(true); given(skillDownloadService.downloadVersion("team-ai", "demo-skill", "1.0.0", null, java.util.Map.of())) .willThrow(new DomainForbiddenException("error.skill.access.denied", "demo-skill")); @@ -125,4 +147,64 @@ void downloadVersion_forbidsAnonymousWhenServiceRejectsSkill() throws Exception .with(csrf())) .andExpect(status().isForbidden()); } + + @Test + void downloadVersion_redirectDoesNotOpenContentStream() throws Exception { + given(rateLimiter.tryAcquire(anyString(), anyInt(), anyInt())).willReturn(true); + given(skillDownloadService.downloadVersion("global", "demo-skill", "1.0.0", null, java.util.Map.of())) + .willReturn(new SkillDownloadService.DownloadResult( + () -> { + throw new AssertionError("content stream should not be opened for redirects"); + }, + "demo-skill-1.0.0.zip", + 128L, + "application/zip", + "https://download.example/presigned", + false + )); + + mockMvc.perform(get("/api/v1/skills/global/demo-skill/versions/1.0.0/download") + .with(user("test-user")) + .with(csrf())) + .andExpect(status().isFound()) + .andExpect(header().string("Location", "https://download.example/presigned")); + } + + @Test + void downloadVersion_usesPerVersionRateLimitKey() throws Exception { + given(rateLimiter.tryAcquire(anyString(), anyInt(), anyInt())).willReturn(true); + given(skillDownloadService.downloadVersion("global", "demo-skill", "1.0.0", "test-user", java.util.Map.of())) + .willReturn(new SkillDownloadService.DownloadResult( + () -> new ByteArrayInputStream("zip".getBytes()), + "demo-skill-1.0.0.zip", + 3L, + "application/zip", + null, + false + )); + + mockMvc.perform(get("/api/v1/skills/global/demo-skill/versions/1.0.0/download") + .requestAttr("userId", "test-user") + .with(user("test-user")) + .with(csrf())) + .andExpect(status().isOk()); + + verify(rateLimiter).tryAcquire( + "ratelimit:download:user:test-user:ns:global:slug:demo-skill:version:1.0.0", + 30, + 60 + ); + } + + @Test + void downloadVersion_returnsServiceUnavailableWhenStorageFails() throws Exception { + given(rateLimiter.tryAcquire(anyString(), anyInt(), anyInt())).willReturn(true); + given(skillDownloadService.downloadVersion("global", "demo-skill", "1.0.0", null, java.util.Map.of())) + .willThrow(new StorageAccessException("getObject", "packages/1/10/bundle.zip", new RuntimeException("boom"))); + + mockMvc.perform(get("/api/v1/skills/global/demo-skill/versions/1.0.0/download") + .with(user("test-user")) + .with(csrf())) + .andExpect(status().isServiceUnavailable()); + } } diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/SkillVersion.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/SkillVersion.java index dddbc29e..80efce80 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/SkillVersion.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/SkillVersion.java @@ -43,6 +43,12 @@ public class SkillVersion { @Column(name = "published_at") private LocalDateTime publishedAt; + @Column(name = "bundle_ready", nullable = false) + private boolean bundleReady; + + @Column(name = "download_ready", nullable = false) + private boolean downloadReady; + @Column(name = "yanked_at") private LocalDateTime yankedAt; @@ -114,6 +120,14 @@ public LocalDateTime getPublishedAt() { return publishedAt; } + public boolean isBundleReady() { + return bundleReady; + } + + public boolean isDownloadReady() { + return downloadReady; + } + public LocalDateTime getYankedAt() { return yankedAt; } @@ -163,6 +177,14 @@ public void setPublishedAt(LocalDateTime publishedAt) { this.publishedAt = publishedAt; } + public void setBundleReady(boolean bundleReady) { + this.bundleReady = bundleReady; + } + + public void setDownloadReady(boolean downloadReady) { + this.downloadReady = downloadReady; + } + public void setYankedAt(LocalDateTime yankedAt) { this.yankedAt = yankedAt; } diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillDownloadService.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillDownloadService.java index 7f26b8a7..5c5f1a75 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillDownloadService.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillDownloadService.java @@ -10,6 +10,8 @@ import com.iflytek.skillhub.domain.skill.*; import com.iflytek.skillhub.storage.ObjectStorageService; import com.iflytek.skillhub.storage.ObjectMetadata; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; @@ -20,11 +22,13 @@ import java.util.Comparator; import java.util.Map; import java.util.List; +import java.util.function.Supplier; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @Service public class SkillDownloadService { + private static final Logger log = LoggerFactory.getLogger(SkillDownloadService.class); private final NamespaceRepository namespaceRepository; private final SkillRepository skillRepository; @@ -58,12 +62,17 @@ public SkillDownloadService( } public record DownloadResult( - InputStream content, + Supplier contentSupplier, String filename, long contentLength, String contentType, - String presignedUrl - ) {} + String presignedUrl, + boolean fallbackBundle + ) { + public InputStream openContent() { + return contentSupplier.get(); + } + } public DownloadResult downloadLatest( String namespaceSlug, @@ -137,9 +146,21 @@ private DownloadResult downloadVersion(Skill skill, SkillVersion version) { ObjectMetadata metadata = objectStorageService.getMetadata(storageKey); String filename = buildFilename(skill, version); String presignedUrl = objectStorageService.generatePresignedUrl(storageKey, Duration.ofMinutes(10), filename); - InputStream content = objectStorageService.getObject(storageKey); - result = new DownloadResult(content, filename, metadata.size(), metadata.contentType(), presignedUrl); + result = new DownloadResult( + () -> objectStorageService.getObject(storageKey), + filename, + metadata.size(), + metadata.contentType(), + presignedUrl, + false + ); } else { + log.warn( + "Bundle missing for published version, falling back to per-file zip [skillId={}, versionId={}, version={}]", + skill.getId(), + version.getId(), + version.getVersion() + ); result = buildBundleFromFiles(skill, version); } @@ -159,11 +180,12 @@ private DownloadResult buildBundleFromFiles(Skill skill, SkillVersion version) { byte[] bundle = createBundle(files); return new DownloadResult( - new ByteArrayInputStream(bundle), + () -> new ByteArrayInputStream(bundle), buildFilename(skill, version), bundle.length, "application/zip", - null + null, + true ); } diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillGovernanceService.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillGovernanceService.java index 8e7d8c9d..c538771f 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillGovernanceService.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillGovernanceService.java @@ -201,6 +201,7 @@ public SkillVersion yankVersion(Long versionId, String actorUserId, String clien version.setYankedAt(LocalDateTime.now()); version.setYankedBy(actorUserId); version.setYankReason(reason); + version.setDownloadReady(false); SkillVersion saved = skillVersionRepository.save(version); auditLogService.record(actorUserId, "YANK_SKILL_VERSION", "SKILL_VERSION", versionId, null, clientIp, userAgent, jsonReason(reason)); return saved; diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillPublishService.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillPublishService.java index aa032a19..7cfed085 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillPublishService.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillPublishService.java @@ -314,6 +314,8 @@ private PublishResult publishFromEntriesInternal( // 11. Update version stats version.setFileCount(skillFiles.size()); version.setTotalSize(totalSize); + version.setBundleReady(true); + version.setDownloadReady(!skillFiles.isEmpty()); skillVersionRepository.save(version); if (!autoPublish) { diff --git a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillQueryService.java b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillQueryService.java index 0ed3bde7..76728b9b 100644 --- a/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillQueryService.java +++ b/server/skillhub-domain/src/main/java/com/iflytek/skillhub/domain/skill/service/SkillQueryService.java @@ -311,14 +311,7 @@ public boolean isDownloadAvailable(SkillVersion version) { if (version.getStatus() != SkillVersionStatus.PUBLISHED) { return false; } - if (objectStorageService.exists(getBundleStorageKey(version.getSkillId(), version.getId()))) { - return true; - } - return skillFileRepository.findByVersionId(version.getId()).stream() - .findAny() - .filter(file -> skillFileRepository.findByVersionId(version.getId()).stream() - .allMatch(candidate -> objectStorageService.exists(candidate.getStorageKey()))) - .isPresent(); + return version.isDownloadReady(); } public ResolvedVersionDTO resolveVersion( diff --git a/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/service/SkillDownloadServiceTest.java b/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/service/SkillDownloadServiceTest.java index 0b407ded..1f099ae0 100644 --- a/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/service/SkillDownloadServiceTest.java +++ b/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/service/SkillDownloadServiceTest.java @@ -90,7 +90,6 @@ void testDownloadLatest_Success() throws Exception { setId(version, 10L); version.setStatus(SkillVersionStatus.PUBLISHED); String storageKey = "packages/1/10/bundle.zip"; - InputStream content = new ByteArrayInputStream("test".getBytes()); ObjectMetadata metadata = new ObjectMetadata(1000L, "application/zip", Instant.now()); when(namespaceRepository.findBySlug(namespaceSlug)).thenReturn(Optional.of(namespace)); @@ -99,7 +98,7 @@ void testDownloadLatest_Success() throws Exception { when(skillVersionRepository.findById(10L)).thenReturn(Optional.of(version)); when(objectStorageService.exists(storageKey)).thenReturn(true); when(objectStorageService.getMetadata(storageKey)).thenReturn(metadata); - when(objectStorageService.getObject(storageKey)).thenReturn(content); + when(objectStorageService.getObject(storageKey)).thenReturn(new ByteArrayInputStream("test".getBytes())); when(objectStorageService.generatePresignedUrl(eq(storageKey), any(), eq("Test Skill-1.0.0.zip"))).thenReturn(null); // Act @@ -109,7 +108,9 @@ void testDownloadLatest_Success() throws Exception { assertNotNull(result); assertEquals("Test Skill-1.0.0.zip", result.filename()); assertEquals(1000L, result.contentLength()); - assertNotNull(result.content()); + try (InputStream content = result.openContent()) { + assertNotNull(content); + } verify(skillRepository).incrementDownloadCount(1L); verify(eventPublisher).publishEvent(any(SkillDownloadedEvent.class)); } @@ -134,7 +135,6 @@ void testDownloadByTag_Success() throws Exception { setId(version, 10L); version.setStatus(SkillVersionStatus.PUBLISHED); String storageKey = "packages/1/10/bundle.zip"; - InputStream content = new ByteArrayInputStream("test".getBytes()); ObjectMetadata metadata = new ObjectMetadata(1000L, "application/zip", Instant.now()); when(namespaceRepository.findBySlug(namespaceSlug)).thenReturn(Optional.of(namespace)); @@ -144,7 +144,7 @@ void testDownloadByTag_Success() throws Exception { when(skillVersionRepository.findById(10L)).thenReturn(Optional.of(version)); when(objectStorageService.exists(storageKey)).thenReturn(true); when(objectStorageService.getMetadata(storageKey)).thenReturn(metadata); - when(objectStorageService.getObject(storageKey)).thenReturn(content); + when(objectStorageService.getObject(storageKey)).thenReturn(new ByteArrayInputStream("test".getBytes())); when(objectStorageService.generatePresignedUrl(eq(storageKey), any(), eq("Test Skill-1.0.0.zip"))).thenReturn(null); // Act @@ -153,7 +153,9 @@ void testDownloadByTag_Success() throws Exception { // Assert assertNotNull(result); assertEquals("Test Skill-1.0.0.zip", result.filename()); - assertNotNull(result.content()); + try (InputStream content = result.openContent()) { + assertNotNull(content); + } verify(skillRepository).incrementDownloadCount(1L); verify(eventPublisher).publishEvent(any(SkillDownloadedEvent.class)); } @@ -176,7 +178,6 @@ void testDownloadVersion_WithPresignedUrlStillProvidesStreamFallback() throws Ex setId(version, 10L); version.setStatus(SkillVersionStatus.PUBLISHED); String storageKey = "packages/1/10/bundle.zip"; - InputStream content = new ByteArrayInputStream("test".getBytes()); ObjectMetadata metadata = new ObjectMetadata(1000L, "application/zip", Instant.now()); when(namespaceRepository.findBySlug(namespaceSlug)).thenReturn(Optional.of(namespace)); @@ -185,14 +186,13 @@ void testDownloadVersion_WithPresignedUrlStillProvidesStreamFallback() throws Ex when(skillVersionRepository.findBySkillIdAndVersion(1L, versionStr)).thenReturn(Optional.of(version)); when(objectStorageService.exists(storageKey)).thenReturn(true); when(objectStorageService.getMetadata(storageKey)).thenReturn(metadata); - when(objectStorageService.getObject(storageKey)).thenReturn(content); when(objectStorageService.generatePresignedUrl(eq(storageKey), any(), eq("Generate Commit Message-1.0.0.zip"))) .thenReturn("http://minio.local/presigned"); SkillDownloadService.DownloadResult result = service.downloadVersion(namespaceSlug, skillSlug, versionStr, userId, userNsRoles); assertEquals("http://minio.local/presigned", result.presignedUrl()); - assertNotNull(result.content()); + verify(objectStorageService, never()).getObject(storageKey); } @Test @@ -252,11 +252,12 @@ void testDownloadVersion_ShouldFallbackToBundledFilesWhenBundleIsMissing() throw SkillDownloadService.DownloadResult result = service.downloadVersion(namespaceSlug, skillSlug, versionStr, userId, userNsRoles); assertNull(result.presignedUrl()); + assertTrue(result.fallbackBundle()); assertEquals("Generate Commit Message-1.0.0.zip", result.filename()); assertEquals("application/zip", result.contentType()); assertTrue(result.contentLength() > 0); - try (ZipInputStream zipInputStream = new ZipInputStream(result.content())) { + try (ZipInputStream zipInputStream = new ZipInputStream(result.openContent())) { var entry = zipInputStream.getNextEntry(); assertNotNull(entry); assertEquals("SKILL.md", entry.getName()); diff --git a/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/service/SkillQueryServiceTest.java b/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/service/SkillQueryServiceTest.java index 4b56ff9b..a454d54e 100644 --- a/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/service/SkillQueryServiceTest.java +++ b/server/skillhub-domain/src/test/java/com/iflytek/skillhub/domain/skill/service/SkillQueryServiceTest.java @@ -415,23 +415,30 @@ void testIsDownloadAvailable_ShouldReturnFalseWhenBundleIsMissing() throws Excep SkillVersion version = new SkillVersion(1L, "1.0.0", "user-100"); setId(version, 10L); version.setStatus(SkillVersionStatus.PUBLISHED); + version.setDownloadReady(false); assertFalse(service.isDownloadAvailable(version)); - verify(objectStorageService).exists("packages/1/10/bundle.zip"); } @Test - void testIsDownloadAvailable_ShouldReturnTrueWhenBundleMissingButFilesExist() throws Exception { + void testIsDownloadAvailable_ShouldReturnTrueWhenPublishedVersionHasFiles() throws Exception { SkillVersion version = new SkillVersion(1L, "1.0.0", "user-100"); setId(version, 10L); version.setStatus(SkillVersionStatus.PUBLISHED); - SkillFile file = new SkillFile(10L, "SKILL.md", 10L, "text/markdown", "hash", "skills/1/10/SKILL.md"); + version.setDownloadReady(true); - when(objectStorageService.exists("packages/1/10/bundle.zip")).thenReturn(false); - when(skillFileRepository.findByVersionId(10L)).thenReturn(List.of(file)); - when(objectStorageService.exists("skills/1/10/SKILL.md")).thenReturn(true); + assertTrue(service.isDownloadAvailable(version)); + } + + @Test + void testIsDownloadAvailable_ShouldNotHitObjectStorageForListSignals() throws Exception { + SkillVersion version = new SkillVersion(1L, "1.0.0", "user-100"); + setId(version, 10L); + version.setStatus(SkillVersionStatus.PUBLISHED); + version.setDownloadReady(true); assertTrue(service.isDownloadAvailable(version)); + verifyNoInteractions(objectStorageService, skillFileRepository); } @Test diff --git a/server/skillhub-storage/pom.xml b/server/skillhub-storage/pom.xml index 9ec5afe1..ef27bb63 100644 --- a/server/skillhub-storage/pom.xml +++ b/server/skillhub-storage/pom.xml @@ -25,6 +25,11 @@ s3 2.20.26 + + software.amazon.awssdk + apache-client + 2.20.26 + org.springframework.boot spring-boot-starter-test diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/LocalFileStorageService.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/LocalFileStorageService.java index e0e35e40..ba87c7be 100644 --- a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/LocalFileStorageService.java +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/LocalFileStorageService.java @@ -28,19 +28,19 @@ public void putObject(String key, InputStream data, long size, String contentTyp data.transferTo(out); } Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException e) { throw new UncheckedIOException("Failed to put object: " + key, e); } + } catch (IOException e) { throw new StorageAccessException("putObject", key, e); } } @Override public InputStream getObject(String key) { try { return Files.newInputStream(resolve(key)); } - catch (IOException e) { throw new UncheckedIOException("Failed to get object: " + key, e); } + catch (IOException e) { throw new StorageAccessException("getObject", key, e); } } @Override public void deleteObject(String key) { try { Files.deleteIfExists(resolve(key)); } - catch (IOException e) { throw new UncheckedIOException("Failed to delete object: " + key, e); } + catch (IOException e) { throw new StorageAccessException("deleteObject", key, e); } } @Override @@ -55,7 +55,7 @@ public ObjectMetadata getMetadata(String key) { Path path = resolve(key); BasicFileAttributes attrs = Files.readAttributes(path, BasicFileAttributes.class); return new ObjectMetadata(attrs.size(), Files.probeContentType(path), attrs.lastModifiedTime().toInstant()); - } catch (IOException e) { throw new UncheckedIOException("Failed to get metadata: " + key, e); } + } catch (IOException e) { throw new StorageAccessException("getMetadata", key, e); } } @Override diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageProperties.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageProperties.java index da961ee5..0e50ef9e 100644 --- a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageProperties.java +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageProperties.java @@ -17,6 +17,10 @@ public class S3StorageProperties { private boolean forcePathStyle = true; private boolean autoCreateBucket = false; private Duration presignExpiry = Duration.ofMinutes(10); + private Integer maxConnections = 100; + private Duration connectionAcquisitionTimeout = Duration.ofSeconds(2); + private Duration apiCallAttemptTimeout = Duration.ofSeconds(10); + private Duration apiCallTimeout = Duration.ofSeconds(30); public String getEndpoint() { return endpoint; } public void setEndpoint(String endpoint) { this.endpoint = endpoint; } @@ -36,4 +40,12 @@ public class S3StorageProperties { public void setAutoCreateBucket(boolean autoCreateBucket) { this.autoCreateBucket = autoCreateBucket; } public Duration getPresignExpiry() { return presignExpiry; } public void setPresignExpiry(Duration presignExpiry) { this.presignExpiry = presignExpiry; } + public Integer getMaxConnections() { return maxConnections; } + public void setMaxConnections(Integer maxConnections) { this.maxConnections = maxConnections; } + public Duration getConnectionAcquisitionTimeout() { return connectionAcquisitionTimeout; } + public void setConnectionAcquisitionTimeout(Duration connectionAcquisitionTimeout) { this.connectionAcquisitionTimeout = connectionAcquisitionTimeout; } + public Duration getApiCallAttemptTimeout() { return apiCallAttemptTimeout; } + public void setApiCallAttemptTimeout(Duration apiCallAttemptTimeout) { this.apiCallAttemptTimeout = apiCallAttemptTimeout; } + public Duration getApiCallTimeout() { return apiCallTimeout; } + public void setApiCallTimeout(Duration apiCallTimeout) { this.apiCallTimeout = apiCallTimeout; } } diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java index 8bb54971..ae50486c 100644 --- a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java @@ -8,6 +8,7 @@ import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.http.apache.ApacheHttpClient; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.*; @@ -33,11 +34,18 @@ public class S3StorageService implements ObjectStorageService { @PostConstruct void init() { + ApacheHttpClient.Builder httpClientBuilder = ApacheHttpClient.builder() + .maxConnections(properties.getMaxConnections()) + .connectionAcquisitionTimeout(properties.getConnectionAcquisitionTimeout()); var builder = S3Client.builder() .region(Region.of(properties.getRegion())) .credentialsProvider(StaticCredentialsProvider.create( AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey()))) - .forcePathStyle(properties.isForcePathStyle()); + .forcePathStyle(properties.isForcePathStyle()) + .httpClientBuilder(httpClientBuilder) + .overrideConfiguration(config -> config + .apiCallAttemptTimeout(properties.getApiCallAttemptTimeout()) + .apiCallTimeout(properties.getApiCallTimeout())); if (properties.getEndpoint() != null && !properties.getEndpoint().isBlank()) { builder.endpointOverride(URI.create(properties.getEndpoint())); } @@ -68,31 +76,52 @@ private void ensureBucketExists() { } @Override public void putObject(String key, InputStream data, long size, String contentType) { - s3Client.putObject(PutObjectRequest.builder().bucket(properties.getBucket()).key(key).contentType(contentType).contentLength(size).build(), RequestBody.fromInputStream(data, size)); + try { + s3Client.putObject(PutObjectRequest.builder().bucket(properties.getBucket()).key(key).contentType(contentType).contentLength(size).build(), RequestBody.fromInputStream(data, size)); + } catch (RuntimeException e) { + throw new StorageAccessException("putObject", key, e); + } } @Override public InputStream getObject(String key) { - return s3Client.getObject(GetObjectRequest.builder().bucket(properties.getBucket()).key(key).build()); + try { + return s3Client.getObject(GetObjectRequest.builder().bucket(properties.getBucket()).key(key).build()); + } catch (RuntimeException e) { + throw new StorageAccessException("getObject", key, e); + } } @Override public void deleteObject(String key) { - s3Client.deleteObject(DeleteObjectRequest.builder().bucket(properties.getBucket()).key(key).build()); + try { + s3Client.deleteObject(DeleteObjectRequest.builder().bucket(properties.getBucket()).key(key).build()); + } catch (RuntimeException e) { + throw new StorageAccessException("deleteObject", key, e); + } } @Override public void deleteObjects(List keys) { if (keys.isEmpty()) return; - List ids = keys.stream().map(k -> ObjectIdentifier.builder().key(k).build()).toList(); - s3Client.deleteObjects(DeleteObjectsRequest.builder().bucket(properties.getBucket()).delete(Delete.builder().objects(ids).build()).build()); + try { + List ids = keys.stream().map(k -> ObjectIdentifier.builder().key(k).build()).toList(); + s3Client.deleteObjects(DeleteObjectsRequest.builder().bucket(properties.getBucket()).delete(Delete.builder().objects(ids).build()).build()); + } catch (RuntimeException e) { + throw new StorageAccessException("deleteObjects", String.join(",", keys), e); + } } @Override public boolean exists(String key) { try { s3Client.headObject(HeadObjectRequest.builder().bucket(properties.getBucket()).key(key).build()); return true; } catch (NoSuchKeyException e) { return false; } + catch (RuntimeException e) { throw new StorageAccessException("exists", key, e); } } @Override public ObjectMetadata getMetadata(String key) { - HeadObjectResponse resp = s3Client.headObject(HeadObjectRequest.builder().bucket(properties.getBucket()).key(key).build()); - return new ObjectMetadata(resp.contentLength(), resp.contentType(), resp.lastModified()); + try { + HeadObjectResponse resp = s3Client.headObject(HeadObjectRequest.builder().bucket(properties.getBucket()).key(key).build()); + return new ObjectMetadata(resp.contentLength(), resp.contentType(), resp.lastModified()); + } catch (RuntimeException e) { + throw new StorageAccessException("getMetadata", key, e); + } } @Override @@ -102,16 +131,20 @@ public String generatePresignedUrl(String key, Duration expiry, String downloadF ? "attachment" : "attachment; filename*=UTF-8''" + java.net.URLEncoder.encode(downloadFilename, StandardCharsets.UTF_8) .replace("+", "%20"); - PresignedGetObjectRequest request = s3Presigner.presignGetObject( - GetObjectPresignRequest.builder() - .signatureDuration(signatureDuration) - .getObjectRequest(GetObjectRequest.builder() - .bucket(properties.getBucket()) - .key(key) - .responseContentDisposition(contentDisposition) - .build()) - .build() - ); - return request.url().toString(); + try { + PresignedGetObjectRequest request = s3Presigner.presignGetObject( + GetObjectPresignRequest.builder() + .signatureDuration(signatureDuration) + .getObjectRequest(GetObjectRequest.builder() + .bucket(properties.getBucket()) + .key(key) + .responseContentDisposition(contentDisposition) + .build()) + .build() + ); + return request.url().toString(); + } catch (RuntimeException e) { + throw new StorageAccessException("generatePresignedUrl", key, e); + } } } diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/StorageAccessException.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/StorageAccessException.java new file mode 100644 index 00000000..81f7db15 --- /dev/null +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/StorageAccessException.java @@ -0,0 +1,21 @@ +package com.iflytek.skillhub.storage; + +public class StorageAccessException extends RuntimeException { + + private final String operation; + private final String key; + + public StorageAccessException(String operation, String key, Throwable cause) { + super("Storage operation failed: " + operation + " [" + key + "]", cause); + this.operation = operation; + this.key = key; + } + + public String getOperation() { + return operation; + } + + public String getKey() { + return key; + } +}