diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java index 5d35fec2eb..c0bcd7c81f 100644 --- a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsGlobalStorageStatistics.java @@ -35,6 +35,7 @@ import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_SERVICE_UNAVAILABLE_COUNT; import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_SIDE_ERROR_COUNT; import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_TIMEOUT_COUNT; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.WRITE_CHECKSUM_FAILURE_COUNT; import static com.google.cloud.hadoop.gcsio.StatisticTypeEnum.TYPE_DURATION; import static com.google.cloud.hadoop.gcsio.StatisticTypeEnum.TYPE_DURATION_TOTAL; import static com.google.common.base.Preconditions.checkArgument; @@ -303,6 +304,10 @@ void incrementGcsExceptionCount() { increment(EXCEPTION_COUNT); } + void incrementWriteChecksumFailureCount() { + increment(WRITE_CHECKSUM_FAILURE_COUNT); + } + void incrementGcsTotalRequestCount() { increment(GCS_API_REQUEST_COUNT); } diff --git a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageEventSubscriber.java b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageEventSubscriber.java index 8a578d95a1..81bac1fc3e 100644 --- a/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageEventSubscriber.java +++ b/gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageEventSubscriber.java @@ -21,6 +21,7 @@ import com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics; import com.google.cloud.hadoop.gcsio.StatisticTypeEnum; +import com.google.cloud.hadoop.util.GCSChecksumFailureEvent; import com.google.cloud.hadoop.util.GcsJsonApiEvent; import com.google.cloud.hadoop.util.GcsJsonApiEvent.EventType; import com.google.cloud.hadoop.util.GcsJsonApiEvent.RequestType; @@ -141,6 +142,11 @@ private void subscriberOnException(IOException exception) { storageStatistics.incrementGcsExceptionCount(); } + @Subscribe + private void onGcsChecksumFailure(GCSChecksumFailureEvent exception) { + storageStatistics.incrementWriteChecksumFailureCount(); + } + /** * Updating the required gcs specific statistics based on httpresponse. * diff --git a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageStatisticsTest.java b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageStatisticsTest.java index 75afccbaa8..0a40f9d2d8 100644 --- a/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageStatisticsTest.java +++ b/gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleCloudStorageStatisticsTest.java @@ -31,6 +31,7 @@ import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_SIDE_ERROR_COUNT; import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_API_SERVER_TIMEOUT_COUNT; import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.GCS_METADATA_REQUEST; +import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageStatistics.WRITE_CHECKSUM_FAILURE_COUNT; import static com.google.common.truth.Truth.assertThat; import com.google.cloud.hadoop.util.GcsJsonApiEvent; @@ -184,6 +185,14 @@ public void gcs_clientBadRequestCount() throws IOException { verifyStatistics(verifyCounterStats); } + @Test + public void gcs_writeChecksumFailureCount() throws IOException { + GoogleCloudStorageEventBus.postWriteChecksumFailure(); + GhfsGlobalStorageStatistics verifyCounterStats = new GhfsGlobalStorageStatistics(); + verifyCounterStats.incrementCounter(WRITE_CHECKSUM_FAILURE_COUNT, 1); + verifyStatistics(verifyCounterStats); + } + private class TestGcsApiEvent implements IGcsJsonApiEvent { private final int statusCode; diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatistics.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatistics.java index a8368e1c14..286fa12ec1 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatistics.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageStatistics.java @@ -27,6 +27,10 @@ /** Statistics which are collected in GCS Connector. */ public enum GoogleCloudStorageStatistics { EXCEPTION_COUNT("exception_count", "Counts the number of exceptions encountered", TYPE_COUNTER), + WRITE_CHECKSUM_FAILURE_COUNT( + "write_checksum_failure_count", + "Counts the number of checksum failures during write", + TYPE_COUNTER), /** Status Code Counters for JSON Path */ GCS_API_REQUEST_COUNT( diff --git a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageWriteChannel.java b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageWriteChannel.java index 5d24cd0300..bdcc2b1f9a 100644 --- a/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageWriteChannel.java +++ b/gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageWriteChannel.java @@ -130,6 +130,7 @@ private void verifyChecksums(String serverProvidedCrc32c) throws IOException { String srcCrc = BaseEncoding.base64().encode(Ints.toByteArray(cumulativeCrc32cHasher.hash().asInt())); if (!srcCrc.equals(serverProvidedCrc32c)) { + GoogleCloudStorageEventBus.postWriteChecksumFailure(); throw new IOException( String.format( "Data integrity check failed for resource '%s'. Client-calculated CRC32C (%s) did not match server-provided CRC32C (%s).", diff --git a/util/src/main/java/com/google/cloud/hadoop/util/GCSChecksumFailureEvent.java b/util/src/main/java/com/google/cloud/hadoop/util/GCSChecksumFailureEvent.java new file mode 100644 index 0000000000..0ead4b78da --- /dev/null +++ b/util/src/main/java/com/google/cloud/hadoop/util/GCSChecksumFailureEvent.java @@ -0,0 +1,20 @@ +/* + * Copyright 2025 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.hadoop.util; + +/** A thin class to emit checksum failure event for EventBus notification. */ +public class GCSChecksumFailureEvent extends Exception {} diff --git a/util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java b/util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java index 04a3d0dc20..3b396c1615 100644 --- a/util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java +++ b/util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java @@ -27,6 +27,7 @@ public class GoogleCloudStorageEventBus { private static EventBus eventBus = new EventBus(); private static IOException exception = new IOException(); + private static GCSChecksumFailureEvent checksumFailureEvent = new GCSChecksumFailureEvent(); /** * Method to register an obj to event bus @@ -76,4 +77,8 @@ public static void onGrpcStatus(Status status) { public static void postGcsJsonApiEvent(IGcsJsonApiEvent gcsJsonApiEvent) { eventBus.post(gcsJsonApiEvent); } + + public static void postWriteChecksumFailure() { + eventBus.post(checksumFailureEvent); + } }