migrate web s3 to azure blob storage

yiliuTo · yiliuTo · commit f0fb5f90e190 · 2025-04-07T14:04:38.000+08:00
diff --git a/asset-manager/web/pom.xml b/asset-manager/web/pom.xml
@@ -31,8 +31,14 @@
             <artifactId>spring-boot-starter-amqp</artifactId>
         </dependency>
         <dependency>
-            <groupId>software.amazon.awssdk</groupId>
-            <artifactId>s3</artifactId>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-storage-blob</artifactId>
+            <version>12.29.0</version>
+        </dependency>
+        <dependency>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-identity</artifactId>
+            <version>1.14.2</version>
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
@@ -69,9 +75,9 @@
     <dependencyManagement>
         <dependencies>
             <dependency>
-                <groupId>software.amazon.awssdk</groupId>
-                <artifactId>s3</artifactId>
-                <version>${aws-sdk.version}</version>
+                <groupId>com.azure</groupId>
+                <artifactId>azure-storage-blob-batch</artifactId>
+                <version>12.25.0</version>
             </dependency>
         </dependencies>
     </dependencyManagement>
diff --git a/asset-manager/web/src/main/java/com/microsoft/migration/assets/config/AwsS3Config.java b/asset-manager/web/src/main/java/com/microsoft/migration/assets/config/AwsS3Config.java
@@ -1,33 +1,23 @@
 package com.microsoft.migration.assets.config;
 
+import com.azure.identity.DefaultAzureCredentialBuilder;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.s3.S3Client;
-
 @Configuration
 public class AwsS3Config {
 
-    @Value("${aws.accessKey}")
-    private String accessKey;
-
-    @Value("${aws.secretKey}")
-    private String secretKey;
-
-    @Value("${aws.region}")
-    private String region;
+    @Value("${azure.storage.account-name}")
+    private String accountName;
 
     @Bean
-    public S3Client s3Client() {
-        AwsBasicCredentials awsCredentials = AwsBasicCredentials.create(accessKey, secretKey);
-        
-        return S3Client.builder()
-                .region(Region.of(region))
-                .credentialsProvider(StaticCredentialsProvider.create(awsCredentials))
-                .build();
+    public BlobServiceClient blobServiceClient() {
+        return new BlobServiceClientBuilder()
+                .endpoint("https://" + accountName + ".blob.core.windows.net")
+                .credential(new DefaultAzureCredentialBuilder().build())
+                .buildClient();
     }
 }
diff --git a/asset-manager/web/src/main/java/com/microsoft/migration/assets/service/AwsS3Service.java b/asset-manager/web/src/main/java/com/microsoft/migration/assets/service/AwsS3Service.java
@@ -1,5 +1,11 @@
 package com.microsoft.migration.assets.service;
 
+import com.azure.identity.DefaultAzureCredentialBuilder;
+import com.azure.storage.blob.BlobServiceClient;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.blob.models.BlobHttpHeaders;
+import com.azure.storage.blob.models.BlobItem;
+import com.azure.storage.blob.options.BlobParallelUploadOptions;
 import com.microsoft.migration.assets.model.ImageMetadata;
 import com.microsoft.migration.assets.model.ImageProcessingMessage;
 import com.microsoft.migration.assets.model.S3StorageItem;
@@ -10,9 +16,6 @@
 import org.springframework.context.annotation.Profile;
 import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
-import software.amazon.awssdk.core.sync.RequestBody;
-import software.amazon.awssdk.services.s3.S3Client;
-import software.amazon.awssdk.services.s3.model.*;
 
 import java.io.IOException;
 import java.io.InputStream;
@@ -28,37 +31,31 @@
 @Profile("!dev") // Active when not in dev profile
 public class AwsS3Service implements StorageService {
 
-    private final S3Client s3Client;
+    private final BlobServiceClient blobServiceClient;
     private final RabbitTemplate rabbitTemplate;
     private final ImageMetadataRepository imageMetadataRepository;
 
-    @Value("${aws.s3.bucket}")
-    private String bucketName;
+    @Value("${azure.storage.blob.container-name}")
+    private String containerName;
 
     @Override
     public List<S3StorageItem> listObjects() {
-        ListObjectsV2Request request = ListObjectsV2Request.builder()
-                .bucket(bucketName)
-                .build();
-
-        ListObjectsV2Response response = s3Client.listObjectsV2(request);
-
-        return response.contents().stream()
-                .map(s3Object -> {
+        return blobServiceClient.getBlobContainerClient(containerName).listBlobs().stream()
+                .map(blobItem -> {
                     // Try to get metadata for upload time
                     Instant uploadedAt = imageMetadataRepository.findAll().stream()
-                            .filter(metadata -> metadata.getS3Key().equals(s3Object.key()))
+                            .filter(metadata -> metadata.getS3Key().equals(blobItem.getName()))
                             .map(metadata -> metadata.getUploadedAt().atZone(java.time.ZoneId.systemDefault()).toInstant())
                             .findFirst()
-                            .orElse(s3Object.lastModified()); // fallback to lastModified if metadata not found
+                            .orElse(blobItem.getProperties().getLastModified().toInstant()); // fallback to lastModified if metadata not found
 
                     return new S3StorageItem(
-                            s3Object.key(),
-                            extractFilename(s3Object.key()),
-                            s3Object.size(),
-                            s3Object.lastModified(),
+                            blobItem.getName(),
+                            extractFilename(blobItem.getName()),
+                            blobItem.getProperties().getContentLength(),
+                            blobItem.getProperties().getLastModified().toInstant(),
                             uploadedAt,
-                            generateUrl(s3Object.key())
+                            generateUrl(blobItem.getName())
                     );
                 })
                 .collect(Collectors.toList());
@@ -67,14 +64,11 @@ public List<S3StorageItem> listObjects() {
     @Override
     public void uploadObject(MultipartFile file) throws IOException {
         String key = generateKey(file.getOriginalFilename());
-        
-        PutObjectRequest request = PutObjectRequest.builder()
-                .bucket(bucketName)
-                .key(key)
-                .contentType(file.getContentType())
-                .build();
-        
-        s3Client.putObject(request, RequestBody.fromInputStream(file.getInputStream(), file.getSize()));
+
+        var blobClient = blobServiceClient.getBlobContainerClient(containerName).getBlobClient(key);
+        BlobHttpHeaders headers = new BlobHttpHeaders().setContentType(file.getContentType());
+        BlobParallelUploadOptions options = new BlobParallelUploadOptions(file.getInputStream()).setHeaders(headers);
+        blobClient.uploadWithResponse(options, null, null);
 
         // Send message to queue for thumbnail generation
         ImageProcessingMessage message = new ImageProcessingMessage(
@@ -99,31 +93,23 @@ public void uploadObject(MultipartFile file) throws IOException {
 
     @Override
     public InputStream getObject(String key) throws IOException {
-        GetObjectRequest request = GetObjectRequest.builder()
-                .bucket(bucketName)
-                .key(key)
-                .build();
-        
-        return s3Client.getObject(request);
+        return blobServiceClient.getBlobContainerClient(containerName)
+                .getBlobClient(key)
+                .openInputStream();
     }
 
     @Override
     public void deleteObject(String key) throws IOException {
         // Delete both original and thumbnail if it exists
-        DeleteObjectRequest request = DeleteObjectRequest.builder()
-                .bucket(bucketName)
-                .key(key)
-                .build();
-        
-        s3Client.deleteObject(request);
+        blobServiceClient.getBlobContainerClient(containerName)
+                .getBlobClient(key)
+                .delete();
 
         try {
             // Try to delete thumbnail if it exists
-            DeleteObjectRequest thumbnailRequest = DeleteObjectRequest.builder()
-                    .bucket(bucketName)
-                    .key(getThumbnailKey(key))
-                    .build();
-            s3Client.deleteObject(thumbnailRequest);
+            blobServiceClient.getBlobContainerClient(containerName)
+                    .getBlobClient(getThumbnailKey(key))
+                    .delete();
         } catch (Exception e) {
             // Ignore if thumbnail doesn't exist
         }
@@ -137,21 +123,19 @@ public void deleteObject(String key) throws IOException {
 
     @Override
     public String getStorageType() {
-        return "s3";
+        return "azure";
     }
 
     private String extractFilename(String key) {
         // Extract filename from the object key
         int lastSlashIndex = key.lastIndexOf('/');
         return lastSlashIndex >= 0 ? key.substring(lastSlashIndex + 1) : key;
     }
-    
+
     private String generateUrl(String key) {
-        GetUrlRequest request = GetUrlRequest.builder()
-                .bucket(bucketName)
-                .key(key)
-                .build();
-        return s3Client.utilities().getUrl(request).toString();
+        return blobServiceClient.getBlobContainerClient(containerName)
+                .getBlobClient(key)
+                .getBlobUrl();
     }
 
     private String generateKey(String filename) {
diff --git a/asset-manager/web/src/main/resources/application.properties b/asset-manager/web/src/main/resources/application.properties
@@ -1,10 +1,8 @@
 spring.application.name=assets-manager
 
-# AWS S3 Configuration
-aws.accessKey=your-access-key
-aws.secretKey=your-secret-key
-aws.region=us-east-1
-aws.s3.bucket=your-bucket-name
+# Azure Blob Storage Configuration
+azure.storage.account-name=${AZURE_STORAGE_ACCOUNT_NAME}
+azure.storage.blob.container-name=${AZURE_STORAGE_BLOB_CONTAINER_NAME}
 
 # Max file size for uploads
 spring.servlet.multipart.max-file-size=10MB
@@ -22,4 +20,4 @@ spring.datasource.username=postgres
 spring.datasource.password=postgres
 spring.jpa.hibernate.ddl-auto=update
 spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQLDialect
-spring.jpa.show-sql=true
+spring.jpa.show-sql=true
