diff --git a/build.gradle b/build.gradle index 54911a6..034be31 100644 --- a/build.gradle +++ b/build.gradle @@ -25,7 +25,7 @@ repositories { } dependencies { - implementation platform('run.halo.tools.platform:plugin:2.21.0-alpha.1') + implementation platform('run.halo.tools.platform:plugin:2.22.0-alpha.1') compileOnly 'run.halo.app:api' implementation platform('software.amazon.awssdk:bom:2.31.58') diff --git a/src/main/java/run/halo/s3os/S3OsAttachmentHandler.java b/src/main/java/run/halo/s3os/S3OsAttachmentHandler.java index d983352..cf4af2c 100644 --- a/src/main/java/run/halo/s3os/S3OsAttachmentHandler.java +++ b/src/main/java/run/halo/s3os/S3OsAttachmentHandler.java @@ -1,15 +1,5 @@ package run.halo.s3os; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.ByteBuffer; -import java.nio.file.FileAlreadyExistsException; -import java.time.Duration; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.pf4j.Extension; @@ -19,17 +9,21 @@ import org.springframework.core.io.buffer.DefaultDataBufferFactory; import org.springframework.http.MediaType; import org.springframework.http.MediaTypeFactory; +import org.springframework.lang.NonNull; import org.springframework.lang.Nullable; import org.springframework.web.server.ServerErrorException; import org.springframework.web.server.ServerWebInputException; +import org.springframework.web.util.UriComponentsBuilder; import reactor.core.Exceptions; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import reactor.core.scheduler.Schedulers; import reactor.util.context.Context; import reactor.util.retry.Retry; +import run.halo.app.core.attachment.ThumbnailSize; import run.halo.app.core.extension.attachment.Attachment; import run.halo.app.core.extension.attachment.Attachment.AttachmentSpec; +import run.halo.app.core.extension.attachment.Attachment.AttachmentStatus; import run.halo.app.core.extension.attachment.Constant; import run.halo.app.core.extension.attachment.Policy; import run.halo.app.core.extension.attachment.endpoint.AttachmentHandler; @@ -44,18 +38,47 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3Configuration; -import software.amazon.awssdk.services.s3.model.*; +import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload; +import software.amazon.awssdk.services.s3.model.CompletedPart; +import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.DeleteObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.HeadObjectResponse; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.services.s3.presigner.S3Presigner; import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; import software.amazon.awssdk.utils.SdkAutoCloseable; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.ByteBuffer; +import java.nio.file.FileAlreadyExistsException; +import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.stream.Collectors; + @Slf4j @Extension public class S3OsAttachmentHandler implements AttachmentHandler { public static final String OBJECT_KEY = "s3os.plugin.halo.run/object-key"; + public static final String URL_SUFFIX_ANNO_KEY = "s3os.plugin.halo.run/url-suffix"; + public static final String SKIP_REMOTE_DELETION_ANNO = "s3os.plugin.halo.run/skip-remote-deletion"; + + private static final MediaType IMAGE_MEDIA_TYPE = MediaType.parseMediaType("image/*"); + public static final int MULTIPART_MIN_PART_SIZE = 5 * 1024 * 1024; /** @@ -156,18 +179,73 @@ public Mono getPermalink(Attachment attachment, Policy policy, ConfigMap co if (!this.shouldHandle(policy)) { return Mono.empty(); } + return Mono.justOrEmpty(doGetPermalink(attachment, S3OsProperties.convertFrom(configMap))); + } + + @Override + public Mono> getThumbnailLinks(Attachment attachment, Policy policy, ConfigMap configMap) { + if (!this.shouldHandle(policy)) { + return Mono.empty(); + } + var properties = S3OsProperties.convertFrom(configMap); + return Mono.just(doGetThumbnailLinks(attachment, properties)); + } + + private Optional doGetPermalink(Attachment attachment, S3OsProperties properties) { var objectKey = getObjectKey(attachment); if (objectKey == null) { // fallback to default handler for backward compatibility - return Mono.empty(); + return Optional.empty(); } - var properties = S3OsProperties.convertFrom(configMap); var objectURL = properties.toObjectURL(objectKey); var urlSuffix = getUrlSuffixAnnotation(attachment); if (StringUtils.isNotBlank(urlSuffix)) { objectURL += urlSuffix; } - return Mono.just(URI.create(objectURL)); + return Optional.of(URI.create(objectURL)); + } + + @NonNull + private Map doGetThumbnailLinks(Attachment attachment, S3OsProperties properties) { + // TODO Support configuring media types that support thumbnails + var support = Optional.ofNullable(attachment.getSpec().getMediaType()) + .map(MediaType::parseMediaType) + .map(IMAGE_MEDIA_TYPE::isCompatibleWith) + .orElse(false); + if (!support) { + if (log.isDebugEnabled()) { + log.debug("Attachment {} media type {} is not compatible with image/*, skip generating thumbnail links", + attachment.getMetadata().getName(), attachment.getSpec().getMediaType()); + } + return Map.of(); + } + + var thumbnailParamPattern = properties.getThumbnailParamPattern(); + if (StringUtils.isBlank(thumbnailParamPattern) || !thumbnailParamPattern.contains("{width}")) { + return Map.of(); + } + return Optional.ofNullable(attachment.getStatus()) + .map(AttachmentStatus::getPermalink) + .filter(StringUtils::isNotBlank) + .map(URI::create) + .or(() -> doGetPermalink(attachment, properties)) + .map(permalink -> Arrays.stream(ThumbnailSize.values()) + .collect(Collectors.toMap(Function.identity(), size -> { + var thumbnailParam = thumbnailParamPattern.replace("{width}", String.valueOf(size.getWidth())); + var isQueryPattern = thumbnailParam.startsWith("?"); + if (isQueryPattern) { + return UriComponentsBuilder.fromUri(permalink) + .query(thumbnailParam.substring(1)) + .build(true) + .toUri(); + } + return UriComponentsBuilder.fromUri(permalink) + .path(thumbnailParam) + .build(true) + .toUri(); + })) + ) + .orElse(Map.of()); } @Nullable @@ -213,6 +291,17 @@ Attachment buildAttachment(S3OsProperties properties, ObjectDetail objectDetail) var attachment = new Attachment(); attachment.setMetadata(metadata); attachment.setSpec(spec); + attachment.setStatus(new AttachmentStatus()); + doGetPermalink(attachment, properties).ifPresent(permalink -> + attachment.getStatus().setPermalink(permalink.toString()) + ); + var thumbnails = doGetThumbnailLinks(attachment, properties); + var mappedThumbnails = thumbnails.keySet() + .stream() + .collect(Collectors.toMap(ThumbnailSize::name, size -> thumbnails.get(size).toString())); + if (!mappedThumbnails.isEmpty()) { + attachment.getStatus().setThumbnails(mappedThumbnails); + } log.info("Built attachment {} successfully", objectDetail.uploadState.objectKey); return attachment; } @@ -369,8 +458,8 @@ Mono checkFileExistsAndRename(UploadState uploadState, if (uploadingFile.put(uploadState.getUploadingMapKey(), uploadState.getUploadingMapKey()) != null) { return Mono.error(new FileAlreadyExistsException("文件 " + uploadState.objectKey - + - " 已存在，建议更名后重试。[local]")); + + + " 已存在，建议更名后重试。[local]")); } uploadState.needRemoveMapKey = true; // check whether file exists @@ -388,7 +477,7 @@ Mono checkFileExistsAndRename(UploadState uploadState, && response.sdkHttpResponse().isSuccessful()) { return Mono.error( new FileAlreadyExistsException("文件 " + uploadState.objectKey - + " 已存在，建议更名后重试。[remote]")); + + " 已存在，建议更名后重试。[remote]")); } else { return Mono.just(uploadState); } @@ -463,18 +552,29 @@ boolean shouldHandle(Policy policy) { } record ObjectDetail(UploadState uploadState, HeadObjectResponse objectMetadata) { + } static class UploadState { + final S3OsProperties properties; + final String originalFileName; + String uploadId; + int partCounter; + Map completedParts = new HashMap<>(); + int buffered = 0; + String contentType; + String fileName; + String objectKey; + boolean needRemoveMapKey = false; public UploadState(S3OsProperties properties, String fileName, boolean needRandomJudge) { diff --git a/src/main/java/run/halo/s3os/S3ThumbnailProvider.java b/src/main/java/run/halo/s3os/S3ThumbnailProvider.java deleted file mode 100644 index f7cc36e..0000000 --- a/src/main/java/run/halo/s3os/S3ThumbnailProvider.java +++ /dev/null @@ -1,106 +0,0 @@ -package run.halo.s3os; - -import java.net.URI; -import java.net.URL; -import java.util.Map; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import lombok.Builder; -import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Component; -import org.springframework.web.util.UriComponentsBuilder; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import run.halo.app.core.attachment.ThumbnailProvider; -import run.halo.app.core.attachment.ThumbnailSize; -import run.halo.app.extension.ConfigMap; -import run.halo.app.extension.ReactiveExtensionClient; - -@Component -@RequiredArgsConstructor -public class S3ThumbnailProvider implements ThumbnailProvider { - static final String WIDTH_PLACEHOLDER = "{width}"; - private final Cache s3PropsCache = CacheBuilder.newBuilder() - .maximumSize(50) - .build(); - - private final ReactiveExtensionClient client; - private final S3LinkService s3LinkService; - - @Override - public Mono generate(ThumbnailContext thumbnailContext) { - var url = thumbnailContext.getImageUrl().toString(); - var size = thumbnailContext.getSize(); - return getCacheValue(url) - .mapNotNull(cacheValue -> placedPattern(cacheValue.pattern(), size)) - .map(param -> { - if (param.startsWith("?")) { - return UriComponentsBuilder.fromHttpUrl(url) - .queryParam(param.substring(1)) - .build() - .toString(); - } - return url + param; - }) - .map(URI::create); - } - - private static String placedPattern(String pattern, ThumbnailSize size) { - return StringUtils.replace(pattern, WIDTH_PLACEHOLDER, String.valueOf(size.getWidth())); - } - - @Override - public Mono delete(URL url) { - // do nothing for s3 - return Mono.empty(); - } - - @Override - public Mono supports(ThumbnailContext thumbnailContext) { - var url = thumbnailContext.getImageUrl().toString(); - return getCacheValue(url).hasElement(); - } - - private Mono getCacheValue(String imageUrl) { - return Flux.fromIterable(s3PropsCache.asMap().entrySet()) - .filter(entry -> imageUrl.startsWith(entry.getKey())) - .next() - .map(Map.Entry::getValue) - .switchIfEmpty(Mono.defer(() -> listAllS3ObjectDomain() - .filter(entry -> imageUrl.startsWith(entry.getKey())) - .map(Map.Entry::getValue) - .next() - )); - } - - @Builder - record S3PropsCacheValue(String pattern, String configMapName) { - } - - private Flux> listAllS3ObjectDomain() { - return s3LinkService.listS3Policies() - .flatMap(s3Policy -> { - var s3ConfigMapName = s3Policy.getSpec().getConfigMapName(); - return fetchS3PropsByConfigMapName(s3ConfigMapName) - .mapNotNull(properties -> { - var thumbnailParam = properties.getThumbnailParamPattern(); - if (StringUtils.isBlank(thumbnailParam)) { - return null; - } - var objectDomain = properties.toObjectURL(""); - var cacheValue = S3PropsCacheValue.builder() - .pattern(thumbnailParam) - .configMapName(s3ConfigMapName) - .build(); - return Map.entry(objectDomain, cacheValue); - }); - }) - .doOnNext(cache -> s3PropsCache.put(cache.getKey(), cache.getValue())); - } - - private Mono fetchS3PropsByConfigMapName(String name) { - return client.fetch(ConfigMap.class, name) - .map(S3OsProperties::convertFrom); - } -} diff --git a/src/main/resources/plugin.yaml b/src/main/resources/plugin.yaml index 3d569f5..05de9ff 100644 --- a/src/main/resources/plugin.yaml +++ b/src/main/resources/plugin.yaml @@ -4,7 +4,7 @@ metadata: name: PluginS3ObjectStorage spec: enabled: true - requires: ">=2.21.0" + requires: ">=2.22.0" author: name: Halo website: https://github.com/halo-dev diff --git a/src/test/java/run/halo/s3os/S3OsAttachmentHandlerTest.java b/src/test/java/run/halo/s3os/S3OsAttachmentHandlerTest.java index 48018b6..cc22319 100644 --- a/src/test/java/run/halo/s3os/S3OsAttachmentHandlerTest.java +++ b/src/test/java/run/halo/s3os/S3OsAttachmentHandlerTest.java @@ -1,20 +1,28 @@ package run.halo.s3os; -import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.core.io.buffer.DataBuffer; import org.springframework.core.io.buffer.DefaultDataBufferFactory; import reactor.core.publisher.Flux; import reactor.test.StepVerifier; +import run.halo.app.core.attachment.ThumbnailSize; +import run.halo.app.core.extension.attachment.Attachment; import run.halo.app.core.extension.attachment.Policy; +import run.halo.app.extension.ConfigMap; +import run.halo.app.extension.Metadata; + +import java.net.URI; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; class S3OsAttachmentHandlerTest { @@ -93,4 +101,112 @@ void reshapeDataBuffersWithBiggerBufferSize() { }) .verifyComplete(); } + + @Test + void shouldGetThumbnailsIfPatternIsQueryParam() { + var attachment = createAttachment("https://s3.halo.run/halo.png?existing-query=existing-value"); + + var policy = new Policy(); + policy.setSpec(new Policy.PolicySpec()); + policy.getSpec().setTemplateName("s3os"); + + var configMap = new ConfigMap(); + configMap.setData(new HashMap<>()); + configMap.getData().put("default", """ + { + "thumbnailParamPattern": "?width={width}&quality=80" + } + """); + handler.getThumbnailLinks(attachment, policy, configMap) + .as(StepVerifier::create) + .expectNext(Map.of( + ThumbnailSize.S, URI.create("https://s3.halo.run/halo.png?existing-query=existing-value&width=400&quality=80"), + ThumbnailSize.M, URI.create("https://s3.halo.run/halo.png?existing-query=existing-value&width=800&quality=80"), + ThumbnailSize.L, URI.create("https://s3.halo.run/halo.png?existing-query=existing-value&width=1200&quality=80"), + ThumbnailSize.XL, URI.create("https://s3.halo.run/halo.png?existing-query=existing-value&width=1600&quality=80") + )) + .verifyComplete(); + } + + @Test + void shouldGetThumbnailsIfPatternIsPath() { + var attachment = createAttachment("https://s3.halo.run/existing-path/halo.png"); + + var policy = new Policy(); + policy.setSpec(new Policy.PolicySpec()); + policy.getSpec().setTemplateName("s3os"); + + var configMap = new ConfigMap(); + configMap.setData(new HashMap<>()); + configMap.getData().put("default", """ + { + "thumbnailParamPattern": "!path/width/{width}" + } + """); + handler.getThumbnailLinks(attachment, policy, configMap) + .as(StepVerifier::create) + .expectNext(Map.of( + ThumbnailSize.S, URI.create("https://s3.halo.run/existing-path/halo.png!path/width/400"), + ThumbnailSize.M, URI.create("https://s3.halo.run/existing-path/halo.png!path/width/800"), + ThumbnailSize.L, URI.create("https://s3.halo.run/existing-path/halo.png!path/width/1200"), + ThumbnailSize.XL, URI.create("https://s3.halo.run/existing-path/halo.png!path/width/1600") + )) + .verifyComplete(); + } + + @Test + void shouldGetEmptyThumbnailsIfNoPattern() { + var attachment = createAttachment("https://s3.halo.run/halo.png"); + + var policy = new Policy(); + policy.setSpec(new Policy.PolicySpec()); + policy.getSpec().setTemplateName("s3os"); + + var configMap = new ConfigMap(); + configMap.setData(new HashMap<>()); + configMap.getData().put("default", """ + {} + """); + handler.getThumbnailLinks(attachment, policy, configMap) + .as(StepVerifier::create) + .expectNext(Map.of()) + .verifyComplete(); + } + + @Test + void shouldGetEmptyThumbnailsIfNotImage() { + var attachment = createAttachment("application/pdf", "https://s3.halo.run/halo.pdf"); + + var policy = new Policy(); + policy.setSpec(new Policy.PolicySpec()); + policy.getSpec().setTemplateName("s3os"); + + var configMap = new ConfigMap(); + configMap.setData(new HashMap<>()); + configMap.getData().put("default", """ + { + "thumbnailParamPattern": "!path/width/{width}" + } + """); + handler.getThumbnailLinks(attachment, policy, configMap) + .as(StepVerifier::create) + .expectNext(Map.of()) + .verifyComplete(); + } + + static Attachment createAttachment(String permalink) { + return createAttachment("image/png", permalink); + } + + static Attachment createAttachment(String mediaType, String permalink) { + var attachment = new Attachment(); + attachment.setMetadata(new Metadata()); + attachment.getMetadata().setName("fake-attachment"); + attachment.setSpec(new Attachment.AttachmentSpec()); + attachment.getSpec().setMediaType(mediaType); + attachment.setStatus(new Attachment.AttachmentStatus()); + attachment.getStatus().setPermalink(permalink); + return attachment; + } + }