diff --git a/README.md b/README.md index 1e18424..b0c95df 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # java21-spring3-maven-reference +[![Java CI with Maven (multi-job)](https://github.com/squidmin/java21-spring3-maven-reference/actions/workflows/ci.yml/badge.svg?event=pull_request)](https://github.com/squidmin/java21-spring3-maven-reference/actions/workflows/ci.yml) + ## Prerequisites - Java 21 diff --git a/google-cloud-storage/docs/batch-uploads-using-gcs-client-libraries.md b/google-cloud-storage/docs/batch-uploads-using-gcs-client-libraries.md new file mode 100644 index 0000000..b99f8fc --- /dev/null +++ b/google-cloud-storage/docs/batch-uploads-using-gcs-client-libraries.md @@ -0,0 +1,19 @@ +# Batch file uploads using GCS client libraries + +The `google-cloud-storage` Java client’s `StorageBatch` only batches metadata ops (update/patch/delete, ACL changes, etc.). +Uploading multiple files in one request using the client library for Java is not supported. + +Media uploads (creating objects with content) use the upload protocols (multipart or resumable) and can’t be combined into a single HTTP batch request. +In a Java application, you’ll need to upload each file separately — ideally in parallel for throughput. + +### Batch updating metadata + +Below is an example of using the GCS client library for Java to update the metadata of five JSON files stored under the `batch_uploads/` prefix. + +![Batch update GCS object metadata](./img/12_batch-update-gcs-object-metadata.gif) + +### Batch uploading files + +To upload multiple files, you can use Java's concurrency features to upload files in parallel. + +![Batch upload multiple files in parallel to GCS](./img/13_batch-upload-to-gcs-parallel.gif) diff --git a/google-cloud-storage/docs/img/12_batch-update-gcs-object-metadata.gif b/google-cloud-storage/docs/img/12_batch-update-gcs-object-metadata.gif new file mode 100644 index 0000000..be83827 Binary files /dev/null and b/google-cloud-storage/docs/img/12_batch-update-gcs-object-metadata.gif differ diff --git a/google-cloud-storage/docs/img/13_batch-upload-to-gcs-parallel.gif b/google-cloud-storage/docs/img/13_batch-upload-to-gcs-parallel.gif new file mode 100644 index 0000000..1aaa1cd Binary files /dev/null and b/google-cloud-storage/docs/img/13_batch-upload-to-gcs-parallel.gif differ diff --git a/google-cloud-storage/pom.xml b/google-cloud-storage/pom.xml index 9bbb5ed..f28998d 100644 --- a/google-cloud-storage/pom.xml +++ b/google-cloud-storage/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - + org.springframework.boot spring-boot-starter-parent @@ -76,21 +76,18 @@ com.google.api-client google-api-client - 1.31.1 + 1.31.1 - + com.google.cloud - google-cloud-bigquery - 2.53.0 + google-cloud-storage + 2.1.5 - - com.google.cloud - google-cloud-storage - 2.1.5 + google-cloud-storage-control @@ -112,6 +109,7 @@ true + org.apache.avro avro @@ -124,6 +122,7 @@ + org.springframework.boot spring-boot-devtools @@ -150,6 +149,13 @@ + + com.google.cloud + libraries-bom + 26.70.0 + pom + import + com.google.cloud spring-cloud-gcp-dependencies diff --git a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/config/GcsConfig.java b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/config/GcsConfig.java index c70adf6..7f68a77 100644 --- a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/config/GcsConfig.java +++ b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/config/GcsConfig.java @@ -5,6 +5,7 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -14,22 +15,29 @@ @Configuration @Getter +@Slf4j public class GcsConfig { private final String projectId; + private final String gcsPrefix; private final String bucketName; + private final String batchUploadPrefix; private final String impersonationTarget; private final String accessToken; private final String gkmsKeyName; public GcsConfig(@Value("${spring.cloud.gcp.project-id:#{systemEnvironment['PROJECT_ID']}}") String projectId, + @Value("${gcp.storage.gcs-prefix:#{systemEnvironment['GCS_PREFIX']}}") String gcsPrefix, @Value("${gcp.storage.bucket.name:#{systemEnvironment['GCS_BUCKET_NAME']}}") String bucketName, + @Value("${gcp.storage.bucket.batch-upload-prefix:#{systemEnvironment['BATCH_UPLOAD_PREFIX']}}") String batchUploadPrefix, @Value("${gcp.auth.impersonation-target:#{systemEnvironment['IMPERSONATION_TARGET']}}") String impersonationTarget, @Value("${gcp.auth.access-token:#{systemEnvironment['OAUTH_ACCESS_TOKEN']}}") String accessToken, @Value("${gcp.kms.key-name:#{systemEnvironment['GKMS_KEY_NAME']}}") String gkmsKeyName) { this.projectId = projectId; + this.gcsPrefix = gcsPrefix; this.bucketName = bucketName; + this.batchUploadPrefix = batchUploadPrefix; this.impersonationTarget = impersonationTarget; this.accessToken = accessToken; this.gkmsKeyName = gkmsKeyName; diff --git a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/controller/GcsController.java b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/controller/GcsController.java index 3425843..acee75c 100644 --- a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/controller/GcsController.java +++ b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/controller/GcsController.java @@ -4,8 +4,10 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; -import org.squidmin.java.spring.maven.gcs.dto.ExampleRequest; -import org.squidmin.java.spring.maven.gcs.dto.ExampleResponse; +import org.squidmin.java.spring.maven.gcs.dto.BatchFileUploadRequest; +import org.squidmin.java.spring.maven.gcs.dto.BatchFileUploadResponse; +import org.squidmin.java.spring.maven.gcs.dto.FileUploadRequest; +import org.squidmin.java.spring.maven.gcs.dto.FileUploadResponse; import org.squidmin.java.spring.maven.gcs.service.impl.GcsServiceImpl; import java.io.IOException; @@ -26,17 +28,37 @@ public GcsController(GcsServiceImpl gcsServiceImpl) { consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE ) - public ResponseEntity uploadFile(@RequestBody ExampleRequest request) throws IOException { + public ResponseEntity uploadFile(@RequestBody FileUploadRequest request) throws IOException { URL url = gcsServiceImpl.uploadAvro(request.getFilename(), request); - return ResponseEntity.ok(ExampleResponse.builder().url(url).build()); + return ResponseEntity.ok(FileUploadResponse.builder().url(url).build()); + } + + @PostMapping( + value = "/batch-upload", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE + ) + public ResponseEntity batchUpload(@RequestBody BatchFileUploadRequest request) { + BatchFileUploadResponse res = gcsServiceImpl.batchUpload(request); + return ResponseEntity.ok(res); + } + + @PostMapping( + value = "/batch-update-gcs-metadata", + consumes = MediaType.APPLICATION_JSON_VALUE, + produces = MediaType.APPLICATION_JSON_VALUE + ) + public ResponseEntity batchUpdateGcsMetadata() { + BatchFileUploadResponse res = gcsServiceImpl.batchUpdateGcsMetadata(); + return ResponseEntity.ok(res); } @GetMapping("/download") - public ResponseEntity downloadFile(@RequestParam String filename) { + public ResponseEntity downloadFile(@RequestParam String filename) { URL url = gcsServiceImpl.downloadAvro(filename); return ResponseEntity.ok() .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + filename + "\"") - .body(ExampleResponse.builder().url(url).build()); + .body(FileUploadResponse.builder().url(url).build()); } } diff --git a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/BatchFileUploadRequest.java b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/BatchFileUploadRequest.java new file mode 100644 index 0000000..8f32e9c --- /dev/null +++ b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/BatchFileUploadRequest.java @@ -0,0 +1,16 @@ +package org.squidmin.java.spring.maven.gcs.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.util.List; + +@AllArgsConstructor +@NoArgsConstructor +@Getter +public class BatchFileUploadRequest { + + private List fileUploadRequests; + +} diff --git a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/BatchFileUploadResponse.java b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/BatchFileUploadResponse.java new file mode 100644 index 0000000..bb73d7a --- /dev/null +++ b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/BatchFileUploadResponse.java @@ -0,0 +1,21 @@ +package org.squidmin.java.spring.maven.gcs.dto; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class BatchFileUploadResponse { + + private int totalRequested; + private int successCount; + private int failureCount; + private List results; + +} diff --git a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/ExampleRequest.java b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/FileUploadRequest.java similarity index 57% rename from google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/ExampleRequest.java rename to google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/FileUploadRequest.java index a5eb176..28aec43 100644 --- a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/ExampleRequest.java +++ b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/FileUploadRequest.java @@ -2,15 +2,16 @@ import lombok.AllArgsConstructor; import lombok.Getter; +import org.squidmin.java.spring.maven.gcs.entity.ExampleEntity; import java.util.List; @AllArgsConstructor @Getter -public class ExampleRequest { +public class FileUploadRequest { private final String filename; - private final List uploadItems; + private final List uploadItems; } diff --git a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/ExampleResponse.java b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/FileUploadResponse.java similarity index 87% rename from google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/ExampleResponse.java rename to google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/FileUploadResponse.java index e7c9bfd..041e413 100644 --- a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/ExampleResponse.java +++ b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/FileUploadResponse.java @@ -11,7 +11,7 @@ @Data @Builder @Getter -public class ExampleResponse { +public class FileUploadResponse { private URL url; diff --git a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/UploadResult.java b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/UploadResult.java new file mode 100644 index 0000000..e3d08c2 --- /dev/null +++ b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/UploadResult.java @@ -0,0 +1,28 @@ +package org.squidmin.java.spring.maven.gcs.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.concurrent.CompletionException; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class UploadResult { + + private String filename; // logical filename from DTO + private String objectName; // actual GCS object (if known) + private String signedUrl; // present on success + private String error; // present on failure + + public static UploadResult success(String filename, String objectName, String signedUrl) { + return new UploadResult(filename, objectName, signedUrl, null); + } + public static UploadResult failure(String filename, Throwable t) { + String msg = (t instanceof CompletionException && t.getCause() != null) ? t.getCause().toString() : t.toString(); + return new UploadResult(filename, null, null, msg); + } + public boolean isSuccess() { return error == null; } + +} diff --git a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/ExampleUploadItem.java b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/entity/ExampleEntity.java similarity index 86% rename from google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/ExampleUploadItem.java rename to google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/entity/ExampleEntity.java index f721192..8fc520f 100644 --- a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/dto/ExampleUploadItem.java +++ b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/entity/ExampleEntity.java @@ -1,4 +1,4 @@ -package org.squidmin.java.spring.maven.gcs.dto; +package org.squidmin.java.spring.maven.gcs.entity; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.AllArgsConstructor; @@ -10,7 +10,7 @@ @NoArgsConstructor @AllArgsConstructor @Slf4j -public class ExampleUploadItem { +public class ExampleEntity { private String id; diff --git a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/service/GcsService.java b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/service/GcsService.java index 5e70c2e..f512bba 100644 --- a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/service/GcsService.java +++ b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/service/GcsService.java @@ -1,13 +1,13 @@ package org.squidmin.java.spring.maven.gcs.service; -import org.squidmin.java.spring.maven.gcs.dto.ExampleRequest; +import org.squidmin.java.spring.maven.gcs.dto.FileUploadRequest; import java.io.IOException; import java.net.URL; public interface GcsService { - URL uploadAvro(String filename, ExampleRequest request) throws IOException; + URL uploadAvro(String filename, FileUploadRequest request) throws IOException; URL downloadAvro(String filename) throws IOException; diff --git a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/service/impl/GcsServiceImpl.java b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/service/impl/GcsServiceImpl.java index 6dfd5d9..8f77315 100644 --- a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/service/impl/GcsServiceImpl.java +++ b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/service/impl/GcsServiceImpl.java @@ -1,19 +1,24 @@ package org.squidmin.java.spring.maven.gcs.service.impl; -import com.google.cloud.storage.BlobId; -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.Storage; +import com.google.api.gax.paging.Page; +import com.google.cloud.storage.*; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.squidmin.java.spring.maven.gcs.config.GcsConfig; -import org.squidmin.java.spring.maven.gcs.dto.ExampleRequest; +import org.squidmin.java.spring.maven.gcs.dto.BatchFileUploadRequest; +import org.squidmin.java.spring.maven.gcs.dto.BatchFileUploadResponse; +import org.squidmin.java.spring.maven.gcs.dto.FileUploadRequest; +import org.squidmin.java.spring.maven.gcs.dto.UploadResult; import org.squidmin.java.spring.maven.gcs.service.GcsService; import org.squidmin.java.spring.maven.gcs.util.AvroUtil; import java.io.IOException; import java.net.URL; -import java.util.UUID; +import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @Service @@ -31,7 +36,7 @@ public GcsServiceImpl(GcsConfig gcsConfig, Storage storage, AvroUtil avroUtil) { this.avroUtil = avroUtil; } - public URL uploadAvro(String filename, ExampleRequest request) throws IOException { + public URL uploadAvro(String filename, FileUploadRequest request) throws IOException { byte[] avroBytes = avroUtil.serializeToAvro(request.getUploadItems()); BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(gcsConfig.getBucketName(), filename + "_" + UUID.randomUUID().toString().substring(0, 6) + ".avro")) @@ -47,6 +52,134 @@ public URL uploadAvro(String filename, ExampleRequest request) throws IOExceptio return storage.signUrl(blobInfo, 5, TimeUnit.MINUTES, Storage.SignUrlOption.withV4Signature()); } + public BatchFileUploadResponse batchUpload(BatchFileUploadRequest request) { + List items = request.getFileUploadRequests(); + int poolSize = Math.min(Math.max(2, Runtime.getRuntime().availableProcessors() * 2), Math.max(2, items.size())); + + try (ExecutorService pool = Executors.newFixedThreadPool(poolSize)) { + try { + List> futures = items.stream() + .map(r -> CompletableFuture.supplyAsync(() -> { + try { + return doUpload(r); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, pool) + .handle((res, ex) -> ex == null ? res + : UploadResult.failure(safeName(r), ex))) + .toList(); + + List results = futures.stream().map(CompletableFuture::join).toList(); + + long success = results.stream().filter(UploadResult::isSuccess).count(); + long failure = results.size() - success; + + return BatchFileUploadResponse.builder() + .totalRequested(results.size()) + .successCount((int) success) + .failureCount((int) failure) + .results(results) + .build(); + + } finally { + pool.shutdown(); + } + } + } + + private UploadResult doUpload(FileUploadRequest r) throws IOException { + var filename = safeName(r); + var signedUrl = uploadAvro(filename, r); + var objectName = filenameForObject(filename); + return UploadResult.success(filename, objectName, signedUrl.toString()); + } + + private String safeName(FileUploadRequest r) { + return (r.getFilename() == null || r.getFilename().isBlank()) + ? "file-" + UUID.randomUUID() + : r.getFilename(); + } + + private String filenameForObject(String base) { + return base; // Modify for name of object in GCS as needed. + } + + public BatchFileUploadResponse batchUpdateGcsMetadata() { + var batchUploadPrefix = gcsConfig.getBatchUploadPrefix(); + var gcsBucketName = gcsConfig.getBucketName(); + + Map newMetadata = new HashMap<>(); + newMetadata.put("keyToAddOrUpdate", "value"); + + Page blobs = + storage.list( + gcsBucketName, + Storage.BlobListOption.prefix(batchUploadPrefix), + Storage.BlobListOption.delimiter("/")); + + // Print blob metadata (keys and values available to update) + for (Blob blob : blobs.iterateAll()) { + log.info("Blob name: " + blob.getName()); + Map metadata = blob.getMetadata(); + if (metadata != null) { + for (Map.Entry entry : metadata.entrySet()) { + log.info(" Metadata - Key: " + entry.getKey() + ", Value: " + entry.getValue()); + } + } else { + log.info(" No metadata found for this blob."); + } + } + + StorageBatch batchRequest = storage.batch(); + + // Add all blobs with the given prefix to the batch request + List> batchResults = + blobs + .streamAll() + .map(blob -> batchRequest.update(blob.toBuilder().setMetadata(newMetadata).build())) + .toList(); + + // Execute the batch request + batchRequest.submit(); + List failures = + batchResults.stream() + .map( + r -> { + try { + BlobInfo blob = r.get(); + log.info("Updated metadata for blob: " + blob.getName()); + return null; + } catch (StorageException e) { + return e; + } + }) + .filter(Objects::nonNull) + .toList(); + + log.info( + (batchResults.size() - failures.size()) + + " blobs in bucket " + + gcsBucketName + + " with prefix '" + + batchUploadPrefix + + "' had their metadata updated successfully."); + + if (!failures.isEmpty()) { + log.info("While processing, there were " + failures.size() + " failures"); + + for (StorageException failure : failures) { + failure.printStackTrace(System.out); + } + } + + return BatchFileUploadResponse.builder() + .totalRequested(batchResults.size()) + .successCount(batchResults.size() - failures.size()) + .failureCount(failures.size()) + .build(); + } + public URL downloadAvro(String filename) { BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(gcsConfig.getBucketName(), filename)).build(); diff --git a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/util/AvroUtil.java b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/util/AvroUtil.java index b57482b..024fdeb 100644 --- a/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/util/AvroUtil.java +++ b/google-cloud-storage/src/main/java/org/squidmin/java/spring/maven/gcs/util/AvroUtil.java @@ -7,7 +7,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.logging.log4j.util.Strings; import org.springframework.stereotype.Component; -import org.squidmin.java.spring.maven.gcs.dto.ExampleUploadItem; +import org.squidmin.java.spring.maven.gcs.entity.ExampleEntity; import org.squidmin.java.spring.maven.gcs.service.impl.GcsServiceImpl; import java.io.ByteArrayOutputStream; @@ -29,14 +29,14 @@ public String getAvroSchema() throws IOException { return schema; } - public byte[] serializeToAvro(List items) throws IOException { + public byte[] serializeToAvro(List items) throws IOException { Schema schema = new Schema.Parser().parse(getAvroSchema()); ByteArrayOutputStream out = new ByteArrayOutputStream(); GenericDatumWriter datumWriter = new GenericDatumWriter<>(schema); DataFileWriter dataFileWriter = new DataFileWriter<>(datumWriter); dataFileWriter.create(schema, out); - for (ExampleUploadItem item : items) { + for (ExampleEntity item : items) { GenericRecord record = new GenericData.Record(schema); record.put("id", item.getId()); record.put("creationTimestamp", item.getCreationTimestamp()); diff --git a/google-cloud-storage/src/main/resources/application.yml b/google-cloud-storage/src/main/resources/application.yml index 00ea796..3fea127 100644 --- a/google-cloud-storage/src/main/resources/application.yml +++ b/google-cloud-storage/src/main/resources/application.yml @@ -13,5 +13,7 @@ management: gcp: storage: + gcs-prefix: gs:// bucket: + batch-upload-prefix: batch_uploads/ name: jm_lofty-root-cmek-test diff --git a/google-cloud-storage/src/test/java/org/squidmin/java/spring/maven/gcs/service/impl/GcsServiceImplTest.java b/google-cloud-storage/src/test/java/org/squidmin/java/spring/maven/gcs/service/impl/GcsServiceImplTest.java index 6834c4f..11a4ebe 100644 --- a/google-cloud-storage/src/test/java/org/squidmin/java/spring/maven/gcs/service/impl/GcsServiceImplTest.java +++ b/google-cloud-storage/src/test/java/org/squidmin/java/spring/maven/gcs/service/impl/GcsServiceImplTest.java @@ -12,8 +12,8 @@ import org.springframework.test.context.ActiveProfiles; import org.squidmin.java.spring.maven.gcs.GcsModuleTestUtil; import org.squidmin.java.spring.maven.gcs.config.GcsConfig; -import org.squidmin.java.spring.maven.gcs.dto.ExampleRequest; -import org.squidmin.java.spring.maven.gcs.dto.ExampleUploadItem; +import org.squidmin.java.spring.maven.gcs.dto.FileUploadRequest; +import org.squidmin.java.spring.maven.gcs.entity.ExampleEntity; import org.squidmin.java.spring.maven.gcs.util.AvroUtil; import java.lang.reflect.Field; @@ -68,8 +68,8 @@ void setUp() throws Exception { void uploadAvro_shouldUploadAvroAndReturnSignedUrl() throws Exception { String signedUrl = "https://signed-url"; - ExampleUploadItem item = new ExampleUploadItem("1", "2023-01-01T00:00:00Z", "2023-01-02T00:00:00Z", "A", "B"); - ExampleRequest request = new ExampleRequest("test.avro", List.of(item)); + ExampleEntity item = new ExampleEntity("1", "2023-01-01T00:00:00Z", "2023-01-02T00:00:00Z", "A", "B"); + FileUploadRequest request = new FileUploadRequest("test.avro", List.of(item)); Blob mockBlob = Mockito.mock(Blob.class); byte[] mockAvroBytes = new byte[]{1, 2, 3}; // Mocked byte array for Avro serialization diff --git a/google-cloud-storage/src/test/java/org/squidmin/java/spring/maven/gcs/util/AvroUtilTest.java b/google-cloud-storage/src/test/java/org/squidmin/java/spring/maven/gcs/util/AvroUtilTest.java index b71742a..c11367b 100644 --- a/google-cloud-storage/src/test/java/org/squidmin/java/spring/maven/gcs/util/AvroUtilTest.java +++ b/google-cloud-storage/src/test/java/org/squidmin/java/spring/maven/gcs/util/AvroUtilTest.java @@ -5,7 +5,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.squidmin.java.spring.maven.gcs.GcsModuleTestUtil; -import org.squidmin.java.spring.maven.gcs.dto.ExampleUploadItem; +import org.squidmin.java.spring.maven.gcs.entity.ExampleEntity; import java.util.List; @@ -28,8 +28,8 @@ void getAvroSchema() throws Exception { @Test void serializeToAvro_shouldSerializeCorrectly() throws Exception { - ExampleUploadItem item = new ExampleUploadItem("123", "2023-01-01", "2023-01-02", "valA", "valB"); - List items = List.of(item); + ExampleEntity item = new ExampleEntity("123", "2023-01-01", "2023-01-02", "valA", "valB"); + List items = List.of(item); byte[] avroBytes = avroUtil.serializeToAvro(items); diff --git a/google-cloud-storage/src/test/resources/batch_upload_test/test_file_1.json b/google-cloud-storage/src/test/resources/batch_upload_test/test_file_1.json new file mode 100644 index 0000000..33fc7aa --- /dev/null +++ b/google-cloud-storage/src/test/resources/batch_upload_test/test_file_1.json @@ -0,0 +1,7 @@ +{ + "id": "d4f6e1b2-8a43-4efb-bf31-c41b5127b01f", + "creation_timestamp": "2025-10-26T09:15:42Z", + "last_update_timestamp": "2025-10-26T10:03:12Z", + "column_a": "Alpha record", + "column_b": "Initial import" +} diff --git a/google-cloud-storage/src/test/resources/batch_upload_test/test_file_2.json b/google-cloud-storage/src/test/resources/batch_upload_test/test_file_2.json new file mode 100644 index 0000000..4cdcba1 --- /dev/null +++ b/google-cloud-storage/src/test/resources/batch_upload_test/test_file_2.json @@ -0,0 +1,7 @@ +{ + "id": "a91d0d88-5730-4a87-9d71-3e9a5d2de0f8", + "creation_timestamp": "2025-10-25T14:22:17Z", + "last_update_timestamp": "2025-10-25T15:48:30Z", + "column_a": "Beta entry", + "column_b": "Updated after QA review" +} diff --git a/google-cloud-storage/src/test/resources/batch_upload_test/test_file_3.json b/google-cloud-storage/src/test/resources/batch_upload_test/test_file_3.json new file mode 100644 index 0000000..02393fe --- /dev/null +++ b/google-cloud-storage/src/test/resources/batch_upload_test/test_file_3.json @@ -0,0 +1,7 @@ +{ + "id": "c2362e91-1fd9-4523-86ae-f1aa02b6e7a4", + "creation_timestamp": "2025-10-20T08:01:05Z", + "last_update_timestamp": "2025-10-22T11:44:09Z", + "column_a": "Gamma dataset", + "column_b": "Scheduled batch load" +} diff --git a/google-cloud-storage/src/test/resources/batch_upload_test/test_file_4.json b/google-cloud-storage/src/test/resources/batch_upload_test/test_file_4.json new file mode 100644 index 0000000..c4c93e0 --- /dev/null +++ b/google-cloud-storage/src/test/resources/batch_upload_test/test_file_4.json @@ -0,0 +1,7 @@ +{ + "id": "64b4eb6f-7727-4c2f-b1cb-1c7718cba17e", + "creation_timestamp": "2025-09-15T16:35:55Z", + "last_update_timestamp": "2025-10-01T09:23:48Z", + "column_a": "Delta archive", + "column_b": "Restored from backup" +} diff --git a/google-cloud-storage/src/test/resources/batch_upload_test/test_file_5.json b/google-cloud-storage/src/test/resources/batch_upload_test/test_file_5.json new file mode 100644 index 0000000..6edf329 --- /dev/null +++ b/google-cloud-storage/src/test/resources/batch_upload_test/test_file_5.json @@ -0,0 +1,7 @@ +{ + "id": "f51482bb-6d6a-4e72-b0e3-c1fc0e3c4aa3", + "creation_timestamp": "2025-10-01T12:00:00Z", + "last_update_timestamp": "2025-10-26T08:49:27Z", + "column_a": "Epsilon config", + "column_b": "Automated sync job" +} diff --git a/java21-spring3-maven-reference/batch-update-gcs-metadata.bru b/java21-spring3-maven-reference/batch-update-gcs-metadata.bru new file mode 100644 index 0000000..22e75f5 --- /dev/null +++ b/java21-spring3-maven-reference/batch-update-gcs-metadata.bru @@ -0,0 +1,24 @@ +meta { + name: /batch-update-gcs-metadata + type: http + seq: 12 +} + +post { + url: http://localhost:8080/gcs/batch-update-gcs-metadata + body: json + auth: inherit +} + +headers { + Content-Type: application/json + ~Authorization: Bearer {{ACCESS_TOKEN}} +} + +body:json { + {} +} + +settings { + encodeUrl: true +} diff --git a/java21-spring3-maven-reference/batch-upload.bru b/java21-spring3-maven-reference/batch-upload.bru new file mode 100644 index 0000000..dc0f778 --- /dev/null +++ b/java21-spring3-maven-reference/batch-upload.bru @@ -0,0 +1,84 @@ +meta { + name: /batch-upload + type: http + seq: 13 +} + +post { + url: http://localhost:8080/gcs/batch-upload + body: json + auth: inherit +} + +headers { + Content-Type: application/json + ~Authorization: Bearer {{ACCESS_TOKEN}} +} + +body:json { + { + "fileUploadRequests": [ + { + "filename": "test-file15", + "uploadItems": [ + { + "id": "af-24", + "creation_timestamp": "2013-06-24T00:00:00", + "last_update_timestamp": "2015-04-20T00:00:00", + "column_a": "column_a_val_1", + "column_b": "column_b_val_1" + }, + { + "id": "af-35", + "creation_timestamp": "2013-06-24T00:00:00", + "last_update_timestamp": "2015-04-20T00:00:00", + "column_a": "column_a_val_2", + "column_b": "column_b_val_2" + }, + { + "id": "26", + "creation_timestamp": "2013-06-24T00:00:00", + "last_update_timestamp": "2015-04-20T00:00:00", + "column_a": "column_a_val_3", + "column_b": "column_b_val_3" + } + ] + }, + { + "filename": "test-file16", + "uploadItems": [ + { + "id": "af-134", + "creation_timestamp": "2013-06-24T00:00:00", + "last_update_timestamp": "2015-04-20T00:00:00", + "column_a": "column_a_val_1", + "column_b": "column_b_val_1" + }, + { + "id": "asd-12", + "creation_timestamp": "2013-06-24T00:00:00", + "last_update_timestamp": "2015-04-20T00:00:00", + "column_a": "column_a_val_2", + "column_b": "column_b_val_2" + } + ] + }, + { + "filename": "test-file17", + "uploadItems": [ + { + "id": "as-15", + "creation_timestamp": "2013-06-24T00:00:00", + "last_update_timestamp": "2015-04-20T00:00:00", + "column_a": "column_a_val_2", + "column_b": "column_b_val_2" + } + ] + } + ] + } +} + +settings { + encodeUrl: true +}