GCS batch request test

README.md

@@ -1,5 +1,7 @@
 # java21-spring3-maven-reference
 
+[![Java CI with Maven (multi-job)](https://github.com/squidmin/java21-spring3-maven-reference/actions/workflows/ci.yml/badge.svg?event=pull_request)](https://github.com/squidmin/java21-spring3-maven-reference/actions/workflows/ci.yml)
+
 ## Prerequisites
 
 - Java 21

google-cloud-storage/docs/batch-uploads-using-gcs-client-libraries.md

@@ -0,0 +1,19 @@
+# Batch file uploads using GCS client libraries
+
+The `google-cloud-storage` Java client’s `StorageBatch` only batches metadata ops (update/patch/delete, ACL changes, etc.).
+Uploading multiple files in one request using the client library for Java is not supported.
+
+Media uploads (creating objects with content) use the upload protocols (multipart or resumable) and can’t be combined into a single HTTP batch request.
+In a Java application, you’ll need to upload each file separately — ideally in parallel for throughput.
+
+### Batch updating metadata
+
+Below is an example of using the GCS client library for Java to update the metadata of five JSON files stored under the `batch_uploads/` prefix.
+
+![Batch update GCS object metadata](./img/12_batch-update-gcs-object-metadata.gif)
+
+### Batch uploading files
+
+To upload multiple files, you can use Java's concurrency features to upload files in parallel.
+
+![Batch upload multiple files in parallel to GCS](./img/13_batch-upload-to-gcs-parallel.gif)

google-cloud-storage/pom.xml

@@ -3,7 +3,7 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
-    <!-- Using Spring Boot Starter Parent since the local parent is not compatible with Spring Cloud -->
+    <!-- Using Spring Boot Starter Parent to isolate dependencies from project parent -->
     <parent>
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-starter-parent</artifactId>
@@ -76,21 +76,18 @@
         <dependency>
             <groupId>com.google.api-client</groupId>
             <artifactId>google-api-client</artifactId>
-            <version>1.31.1</version> <!-- ✅ Compatible with Spring Boot 3 -->
+            <version>1.31.1</version>
         </dependency>
 
-        <!-- Google Cloud BigQuery -->
+        <!-- Google Cloud Storage -->
         <dependency>
             <groupId>com.google.cloud</groupId>
-            <artifactId>google-cloud-bigquery</artifactId>
-            <version>2.53.0</version>
+            <artifactId>google-cloud-storage</artifactId>
+            <version>2.1.5</version>
         </dependency>
-
-        <!-- Google Cloud Storage -->
         <dependency>
             <groupId>com.google.cloud</groupId>
-            <artifactId>google-cloud-storage</artifactId>
-            <version>2.1.5</version> <!-- ✅ Boot 3 compatible -->
+            <artifactId>google-cloud-storage-control</artifactId>
         </dependency>
 
         <!-- (Optional) JSON simple for manual parsing -->
@@ -112,6 +109,7 @@
             <optional>true</optional>
         </dependency>
 
+        <!-- Apache Avro (excluding commons-lang3 to avoid conflicts) -->
         <dependency>
             <groupId>org.apache.avro</groupId>
             <artifactId>avro</artifactId>
@@ -124,6 +122,7 @@
             </exclusions>
         </dependency>
 
+        <!-- Spring Boot DevTools -->
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-devtools</artifactId>
@@ -150,6 +149,13 @@
 
     <dependencyManagement>
         <dependencies>
+            <dependency>
+                <groupId>com.google.cloud</groupId>
+                <artifactId>libraries-bom</artifactId>
+                <version>26.70.0</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
             <dependency>
                 <groupId>com.google.cloud</groupId>
                 <artifactId>spring-cloud-gcp-dependencies</artifactId>

google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/config/GcsConfig.java

@@ -5,6 +5,7 @@
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageOptions;
 import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
@@ -14,22 +15,29 @@
 
 @Configuration
 @Getter
+@Slf4j
 public class GcsConfig {
 
     private final String projectId;
+    private final String gcsPrefix;
     private final String bucketName;
+    private final String batchUploadPrefix;
     private final String impersonationTarget;
     private final String accessToken;
     private final String gkmsKeyName;
 
     public GcsConfig(@Value("${spring.cloud.gcp.project-id:#{systemEnvironment['PROJECT_ID']}}") String projectId,
+                     @Value("${gcp.storage.gcs-prefix:#{systemEnvironment['GCS_PREFIX']}}") String gcsPrefix,
                      @Value("${gcp.storage.bucket.name:#{systemEnvironment['GCS_BUCKET_NAME']}}") String bucketName,
+                     @Value("${gcp.storage.bucket.batch-upload-prefix:#{systemEnvironment['BATCH_UPLOAD_PREFIX']}}") String batchUploadPrefix,
                      @Value("${gcp.auth.impersonation-target:#{systemEnvironment['IMPERSONATION_TARGET']}}") String impersonationTarget,
                      @Value("${gcp.auth.access-token:#{systemEnvironment['OAUTH_ACCESS_TOKEN']}}") String accessToken,
                      @Value("${gcp.kms.key-name:#{systemEnvironment['GKMS_KEY_NAME']}}") String gkmsKeyName) {
 
         this.projectId = projectId;
+        this.gcsPrefix = gcsPrefix;
         this.bucketName = bucketName;
+        this.batchUploadPrefix = batchUploadPrefix;
         this.impersonationTarget = impersonationTarget;
         this.accessToken = accessToken;
         this.gkmsKeyName = gkmsKeyName;

google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/controller/GcsController.java

@@ -4,8 +4,10 @@
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
 import org.springframework.web.bind.annotation.*;
-import org.squidmin.java.spring.maven.gcs.dto.ExampleRequest;
-import org.squidmin.java.spring.maven.gcs.dto.ExampleResponse;
+import org.squidmin.java.spring.maven.gcs.dto.BatchFileUploadRequest;
+import org.squidmin.java.spring.maven.gcs.dto.BatchFileUploadResponse;
+import org.squidmin.java.spring.maven.gcs.dto.FileUploadRequest;
+import org.squidmin.java.spring.maven.gcs.dto.FileUploadResponse;
 import org.squidmin.java.spring.maven.gcs.service.impl.GcsServiceImpl;
 
 import java.io.IOException;
@@ -26,17 +28,37 @@ public GcsController(GcsServiceImpl gcsServiceImpl) {
         consumes = MediaType.APPLICATION_JSON_VALUE,
         produces = MediaType.APPLICATION_JSON_VALUE
     )
-    public ResponseEntity<ExampleResponse> uploadFile(@RequestBody ExampleRequest request) throws IOException {
+    public ResponseEntity<FileUploadResponse> uploadFile(@RequestBody FileUploadRequest request) throws IOException {
         URL url = gcsServiceImpl.uploadAvro(request.getFilename(), request);
-        return ResponseEntity.ok(ExampleResponse.builder().url(url).build());
+        return ResponseEntity.ok(FileUploadResponse.builder().url(url).build());
+    }
+
+    @PostMapping(
+        value = "/batch-upload",
+        consumes = MediaType.APPLICATION_JSON_VALUE,
+        produces = MediaType.APPLICATION_JSON_VALUE
+    )
+    public ResponseEntity<BatchFileUploadResponse> batchUpload(@RequestBody BatchFileUploadRequest request) {
+        BatchFileUploadResponse res = gcsServiceImpl.batchUpload(request);
+        return ResponseEntity.ok(res);
+    }
+
+    @PostMapping(
+        value = "/batch-update-gcs-metadata",
+        consumes = MediaType.APPLICATION_JSON_VALUE,
+        produces = MediaType.APPLICATION_JSON_VALUE
+    )
+    public ResponseEntity<BatchFileUploadResponse> batchUpdateGcsMetadata() {
+        BatchFileUploadResponse res = gcsServiceImpl.batchUpdateGcsMetadata();
+        return ResponseEntity.ok(res);
     }
 
     @GetMapping("/download")
-    public ResponseEntity<ExampleResponse> downloadFile(@RequestParam String filename) {
+    public ResponseEntity<FileUploadResponse> downloadFile(@RequestParam String filename) {
         URL url = gcsServiceImpl.downloadAvro(filename);
         return ResponseEntity.ok()
             .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\"")
-            .body(ExampleResponse.builder().url(url).build());
+            .body(FileUploadResponse.builder().url(url).build());
     }
 
 }

google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/BatchFileUploadRequest.java

@@ -0,0 +1,16 @@
+package org.squidmin.java.spring.maven.gcs.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@AllArgsConstructor
+@NoArgsConstructor
+@Getter
+public class BatchFileUploadRequest {
+
+    private List<FileUploadRequest> fileUploadRequests;
+
+}

google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/BatchFileUploadResponse.java

@@ -0,0 +1,21 @@
+package org.squidmin.java.spring.maven.gcs.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.List;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class BatchFileUploadResponse {
+
+    private int totalRequested;
+    private int successCount;
+    private int failureCount;
+    private List<UploadResult> results;
+
+}

google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/ExampleRequest.java → google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/FileUploadRequest.java

@@ -2,15 +2,16 @@
 
 import lombok.AllArgsConstructor;
 import lombok.Getter;
+import org.squidmin.java.spring.maven.gcs.entity.ExampleEntity;
 
 import java.util.List;
 
 @AllArgsConstructor
 @Getter
-public class ExampleRequest {
+public class FileUploadRequest {
 
     private final String filename;
 
-    private final List<ExampleUploadItem> uploadItems;
+    private final List<ExampleEntity> uploadItems;
 
 }

google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/ExampleResponse.java → google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/FileUploadResponse.java

@@ -11,7 +11,7 @@
 @Data
 @Builder
 @Getter
-public class ExampleResponse {
+public class FileUploadResponse {
 
     private URL url;
 

google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/UploadResult.java

@@ -0,0 +1,28 @@
+package org.squidmin.java.spring.maven.gcs.dto;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+import java.util.concurrent.CompletionException;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class UploadResult {
+
+    private String filename;      // logical filename from DTO
+    private String objectName;    // actual GCS object (if known)
+    private String signedUrl;     // present on success
+    private String error;         // present on failure
+
+    public static UploadResult success(String filename, String objectName, String signedUrl) {
+        return new UploadResult(filename, objectName, signedUrl, null);
+    }
+    public static UploadResult failure(String filename, Throwable t) {
+        String msg = (t instanceof CompletionException && t.getCause() != null) ? t.getCause().toString() : t.toString();
+        return new UploadResult(filename, null, null, msg);
+    }
+    public boolean isSuccess() { return error == null; }
+
+}

google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/ExampleUploadItem.java → google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/entity/ExampleEntity.java

@@ -1,4 +1,4 @@
-package org.squidmin.java.spring.maven.gcs.dto;
+package org.squidmin.java.spring.maven.gcs.entity;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
 import lombok.AllArgsConstructor;
@@ -10,7 +10,7 @@
 @NoArgsConstructor
 @AllArgsConstructor
 @Slf4j
-public class ExampleUploadItem {
+public class ExampleEntity {
 
     private String id;
 

google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/service/GcsService.java

@@ -1,13 +1,13 @@
 package org.squidmin.java.spring.maven.gcs.service;
 
-import org.squidmin.java.spring.maven.gcs.dto.ExampleRequest;
+import org.squidmin.java.spring.maven.gcs.dto.FileUploadRequest;
 
 import java.io.IOException;
 import java.net.URL;
 
 public interface GcsService {
 
-    URL uploadAvro(String filename, ExampleRequest request) throws IOException;
+    URL uploadAvro(String filename, FileUploadRequest request) throws IOException;
 
     URL downloadAvro(String filename) throws IOException;