feat: Add SchemaViolationReason to IngestionFailureEvent (#2045)

* feat: Add SchemaViolationReason to IngestionFailureEvent
docs: Standardize spelling of "acknowledgment" in Pub/Sub protos

PiperOrigin-RevId: 761506540

Source-Link: googleapis/googleapis@c1af7f2

Source-Link: googleapis/googleapis-gen@260979c
Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMjYwOTc5Y2M2NDg3MjJlY2UyODgxOTY2NWIyZWNkMWFiMGZiNjFhNCJ9

* build: owl-bot-staging can't be linted due to its duplicate .eslintrc.json

* build: also update the owl-bot-staging copy of .eslintignore

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
Co-authored-by: feywind <57276408+feywind@users.noreply.github.com>

Author: gcf-owl-bot[bot]

.eslintignore

@@ -6,3 +6,4 @@ protos/
 system-test/
 samples/generated/
 samples/typescript/
+owl-bot-staging/

.github/scripts/package.json

@@ -14,7 +14,7 @@
         "js-yaml": "^4.1.0"
     },
     "devDependencies": {
-        "@octokit/rest": "^21.0.0",
+        "@octokit/rest": "^19.0.0",
         "mocha": "^10.0.0",
         "sinon": "^18.0.0"
     }

protos/google/pubsub/v1/pubsub.proto

@@ -1,4 +1,4 @@
-// Copyright 2024 Google LLC
+// Copyright 2025 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -26,9 +26,8 @@ import "google/protobuf/field_mask.proto";
 import "google/protobuf/timestamp.proto";
 import "google/pubsub/v1/schema.proto";
 
-option cc_enable_arenas = true;
 option csharp_namespace = "Google.Cloud.PubSub.V1";
-option go_package = "cloud.google.com/go/pubsub/apiv1/pubsubpb;pubsubpb";
+option go_package = "cloud.google.com/go/pubsub/v2/apiv1/pubsubpb;pubsubpb";
 option java_multiple_files = true;
 option java_outer_classname = "PubsubProto";
 option java_package = "com.google.pubsub.v1";
@@ -550,6 +549,10 @@ message IngestionFailureEvent {
   // occurs, one or more Avro objects won't be ingested.
   message AvroFailureReason {}
 
+  // Set when a Pub/Sub message fails to get published due to a schema
+  // validation violation.
+  message SchemaViolationReason {}
+
   // Failure when ingesting from a Cloud Storage source.
   message CloudStorageFailure {
     // Optional. Name of the Cloud Storage bucket used for ingestion.
@@ -573,6 +576,10 @@ message IngestionFailureEvent {
       // being published.
       ApiViolationReason api_violation_reason = 6
           [(google.api.field_behavior) = OPTIONAL];
+
+      // Optional. The Pub/Sub message failed schema validation.
+      SchemaViolationReason schema_violation_reason = 7
+          [(google.api.field_behavior) = OPTIONAL];
     }
   }
 
@@ -597,6 +604,10 @@ message IngestionFailureEvent {
       // being published.
       ApiViolationReason api_violation_reason = 5
           [(google.api.field_behavior) = OPTIONAL];
+
+      // Optional. The Pub/Sub message failed schema validation.
+      SchemaViolationReason schema_violation_reason = 6
+          [(google.api.field_behavior) = OPTIONAL];
     }
   }
 
@@ -621,6 +632,10 @@ message IngestionFailureEvent {
       // being published.
       ApiViolationReason api_violation_reason = 5
           [(google.api.field_behavior) = OPTIONAL];
+
+      // Optional. The Pub/Sub message failed schema validation.
+      SchemaViolationReason schema_violation_reason = 6
+          [(google.api.field_behavior) = OPTIONAL];
     }
   }
 
@@ -645,6 +660,29 @@ message IngestionFailureEvent {
       // being published.
       ApiViolationReason api_violation_reason = 5
           [(google.api.field_behavior) = OPTIONAL];
+
+      // Optional. The Pub/Sub message failed schema validation.
+      SchemaViolationReason schema_violation_reason = 6
+          [(google.api.field_behavior) = OPTIONAL];
+    }
+  }
+
+  // Failure when ingesting from an AWS Kinesis source.
+  message AwsKinesisFailureReason {
+    // Optional. The stream ARN of the Kinesis stream being ingested from.
+    string stream_arn = 1 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The partition key of the message that failed to be ingested.
+    string partition_key = 2 [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. The sequence number of the message that failed to be ingested.
+    string sequence_number = 3 [(google.api.field_behavior) = OPTIONAL];
+
+    // Reason why ingestion failed for the specified message.
+    oneof reason {
+      // Optional. The Pub/Sub message failed schema validation.
+      SchemaViolationReason schema_violation_reason = 4
+          [(google.api.field_behavior) = OPTIONAL];
     }
   }
 
@@ -671,7 +709,69 @@ message IngestionFailureEvent {
     // Optional. Failure when ingesting from Confluent Cloud.
     ConfluentCloudFailureReason confluent_cloud_failure = 6
         [(google.api.field_behavior) = OPTIONAL];
+
+    // Optional. Failure when ingesting from AWS Kinesis.
+    AwsKinesisFailureReason aws_kinesis_failure = 7
+        [(google.api.field_behavior) = OPTIONAL];
+  }
+}
+
+// User-defined JavaScript function that can transform or filter a Pub/Sub
+// message.
+message JavaScriptUDF {
+  // Required. Name of the JavasScript function that should applied to Pub/Sub
+  // messages.
+  string function_name = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // Required. JavaScript code that contains a function `function_name` with the
+  // below signature:
+  //
+  // ```
+  //   /**
+  //   * Transforms a Pub/Sub message.
+  //
+  //   * @return {(Object<string, (string | Object<string, string>)>|null)} - To
+  //   * filter a message, return `null`. To transform a message return a map
+  //   * with the following keys:
+  //   *   - (required) 'data' : {string}
+  //   *   - (optional) 'attributes' : {Object<string, string>}
+  //   * Returning empty `attributes` will remove all attributes from the
+  //   * message.
+  //   *
+  //   * @param  {(Object<string, (string | Object<string, string>)>} Pub/Sub
+  //   * message. Keys:
+  //   *   - (required) 'data' : {string}
+  //   *   - (required) 'attributes' : {Object<string, string>}
+  //   *
+  //   * @param  {Object<string, any>} metadata - Pub/Sub message metadata.
+  //   * Keys:
+  //   *   - (optional) 'message_id'  : {string}
+  //   *   - (optional) 'publish_time': {string} YYYY-MM-DDTHH:MM:SSZ format
+  //   *   - (optional) 'ordering_key': {string}
+  //   */
+  //
+  //   function <function_name>(message, metadata) {
+  //   }
+  // ```
+  string code = 2 [(google.api.field_behavior) = REQUIRED];
+}
+
+// All supported message transforms types.
+message MessageTransform {
+  // The type of transform to apply to messages.
+  oneof transform {
+    // Optional. JavaScript User Defined Function. If multiple JavaScriptUDF's
+    // are specified on a resource, each must have a unique `function_name`.
+    JavaScriptUDF javascript_udf = 2 [(google.api.field_behavior) = OPTIONAL];
   }
+
+  // Optional. This field is deprecated, use the `disabled` field to disable
+  // transforms.
+  bool enabled = 3 [deprecated = true, (google.api.field_behavior) = OPTIONAL];
+
+  // Optional. If true, the transform is disabled and will not be applied to
+  // messages. Defaults to `false`.
+  bool disabled = 4 [(google.api.field_behavior) = OPTIONAL];
 }
 
 // A topic resource.
@@ -680,6 +780,8 @@ message Topic {
     type: "pubsub.googleapis.com/Topic"
     pattern: "projects/{project}/topics/{topic}"
     pattern: "_deleted-topic_"
+    plural: "topics"
+    singular: "topic"
   };
 
   // The state of the topic.
@@ -745,6 +847,11 @@ message Topic {
   // Optional. Settings for ingestion from a data source into this topic.
   IngestionDataSourceSettings ingestion_data_source_settings = 10
       [(google.api.field_behavior) = OPTIONAL];
+
+  // Optional. Transforms to be applied to messages published to the topic.
+  // Transforms are applied in the order specified.
+  repeated MessageTransform message_transforms = 13
+      [(google.api.field_behavior) = OPTIONAL];
 }
 
 // A message that is published by publishers and consumed by subscribers. The
@@ -1064,7 +1171,7 @@ service Subscriber {
   }
 
   // Establishes a stream with the server, which sends messages down to the
-  // client. The client streams acknowledgements and ack deadline modifications
+  // client. The client streams acknowledgments and ack deadline modifications
   // back to the server. The server will close the stream and return the status
   // on any error. The server may close the stream with status `UNAVAILABLE` to
   // reassign server-side resources, in which case, the client should
@@ -1188,6 +1295,8 @@ message Subscription {
   option (google.api.resource) = {
     type: "pubsub.googleapis.com/Subscription"
     pattern: "projects/{project}/subscriptions/{subscription}"
+    plural: "subscriptions"
+    singular: "subscription"
   };
 
   // Possible states for a subscription.
@@ -1204,8 +1313,8 @@ message Subscription {
     RESOURCE_ERROR = 2;
   }
 
-  // Information about an associated Analytics Hub subscription
-  // (https://cloud.google.com/bigquery/docs/analytics-hub-manage-subscriptions).
+  // Information about an associated [Analytics Hub
+  // subscription](https://cloud.google.com/bigquery/docs/analytics-hub-manage-subscriptions).
   message AnalyticsHubSubscriptionInfo {
     // Optional. The name of the associated Analytics Hub listing resource.
     // Pattern:
@@ -1328,8 +1437,8 @@ message Subscription {
   //
   // If not set, the default retry policy is applied. This generally implies
   // that messages will be retried as soon as possible for healthy subscribers.
-  // RetryPolicy will be triggered on NACKs or acknowledgement deadline
-  // exceeded events for a given message.
+  // RetryPolicy will be triggered on NACKs or acknowledgment deadline exceeded
+  // events for a given message.
   RetryPolicy retry_policy = 14 [(google.api.field_behavior) = OPTIONAL];
 
   // Optional. Indicates whether the subscription is detached from its topic.
@@ -1344,7 +1453,7 @@ message Subscription {
   // subscription:
   //
   // * The message sent to a subscriber is guaranteed not to be resent
-  // before the message's acknowledgement deadline expires.
+  // before the message's acknowledgment deadline expires.
   // * An acknowledged message will not be resent to a subscriber.
   //
   // Note that subscribers may still receive multiple copies of a message
@@ -1371,14 +1480,19 @@ message Subscription {
   // Only set if the subscritpion is created by Analytics Hub.
   AnalyticsHubSubscriptionInfo analytics_hub_subscription_info = 23
       [(google.api.field_behavior) = OUTPUT_ONLY];
+
+  // Optional. Transforms to be applied to messages before they are delivered to
+  // subscribers. Transforms are applied in the order specified.
+  repeated MessageTransform message_transforms = 25
+      [(google.api.field_behavior) = OPTIONAL];
 }
 
 // A policy that specifies how Pub/Sub retries message delivery.
 //
 // Retry delay will be exponential based on provided minimum and maximum
 // backoffs. https://en.wikipedia.org/wiki/Exponential_backoff.
 //
-// RetryPolicy will be triggered on NACKs or acknowledgement deadline exceeded
+// RetryPolicy will be triggered on NACKs or acknowledgment deadline exceeded
 // events for a given message.
 //
 // Retry Policy is implemented on a best effort basis. At times, the delay
@@ -1418,7 +1532,7 @@ message DeadLetterPolicy {
   // value must be between 5 and 100.
   //
   // The number of delivery attempts is defined as 1 + (the sum of number of
-  // NACKs and number of times the acknowledgement deadline has been exceeded
+  // NACKs and number of times the acknowledgment deadline has been exceeded
   // for the message).
   //
   // A NACK is any call to ModifyAckDeadline with a 0 deadline. Note that
@@ -1680,7 +1794,7 @@ message CloudStorageConfig {
 
   // Optional. The maximum duration that can elapse before a new Cloud Storage
   // file is created. Min 1 minute, max 10 minutes, default 5 minutes. May not
-  // exceed the subscription's acknowledgement deadline.
+  // exceed the subscription's acknowledgment deadline.
   google.protobuf.Duration max_duration = 6
       [(google.api.field_behavior) = OPTIONAL];
 
@@ -1902,7 +2016,7 @@ message AcknowledgeRequest {
 }
 
 // Request for the `StreamingPull` streaming RPC method. This request is used to
-// establish the initial stream as well as to stream acknowledgements and ack
+// establish the initial stream as well as to stream acknowledgments and ack
 // deadline modifications from the client to the server.
 message StreamingPullRequest {
   // Required. The subscription for which to initialize the new stream. This
@@ -1916,12 +2030,11 @@ message StreamingPullRequest {
     }
   ];
 
-  // Optional. List of acknowledgement IDs for acknowledging previously received
+  // Optional. List of acknowledgment IDs for acknowledging previously received
   // messages (received on this stream or a different stream). If an ack ID has
   // expired, the corresponding message may be redelivered later. Acknowledging
-  // a message more than once will not result in an error. If the
-  // acknowledgement ID is malformed, the stream will be aborted with status
-  // `INVALID_ARGUMENT`.
+  // a message more than once will not result in an error. If the acknowledgment
+  // ID is malformed, the stream will be aborted with status `INVALID_ARGUMENT`.
   repeated string ack_ids = 2 [(google.api.field_behavior) = OPTIONAL];
 
   // Optional. The list of new ack deadlines for the IDs listed in
@@ -1938,7 +2051,7 @@ message StreamingPullRequest {
   repeated int32 modify_deadline_seconds = 3
       [(google.api.field_behavior) = OPTIONAL];
 
-  // Optional. List of acknowledgement IDs whose deadline will be modified based
+  // Optional. List of acknowledgment IDs whose deadline will be modified based
   // on the corresponding element in `modify_deadline_seconds`. This field can
   // be used to indicate that more time is needed to process a message by the
   // subscriber, or to make the message available for redelivery if the
@@ -1987,39 +2100,39 @@ message StreamingPullRequest {
 // Response for the `StreamingPull` method. This response is used to stream
 // messages from the server to the client.
 message StreamingPullResponse {
-  // Acknowledgement IDs sent in one or more previous requests to acknowledge a
+  // Acknowledgment IDs sent in one or more previous requests to acknowledge a
   // previously received message.
   message AcknowledgeConfirmation {
-    // Optional. Successfully processed acknowledgement IDs.
+    // Optional. Successfully processed acknowledgment IDs.
     repeated string ack_ids = 1 [(google.api.field_behavior) = OPTIONAL];
 
-    // Optional. List of acknowledgement IDs that were malformed or whose
-    // acknowledgement deadline has expired.
+    // Optional. List of acknowledgment IDs that were malformed or whose
+    // acknowledgment deadline has expired.
     repeated string invalid_ack_ids = 2
         [(google.api.field_behavior) = OPTIONAL];
 
-    // Optional. List of acknowledgement IDs that were out of order.
+    // Optional. List of acknowledgment IDs that were out of order.
     repeated string unordered_ack_ids = 3
         [(google.api.field_behavior) = OPTIONAL];
 
-    // Optional. List of acknowledgement IDs that failed processing with
+    // Optional. List of acknowledgment IDs that failed processing with
     // temporary issues.
     repeated string temporary_failed_ack_ids = 4
         [(google.api.field_behavior) = OPTIONAL];
   }
 
-  // Acknowledgement IDs sent in one or more previous requests to modify the
+  // Acknowledgment IDs sent in one or more previous requests to modify the
   // deadline for a specific message.
   message ModifyAckDeadlineConfirmation {
-    // Optional. Successfully processed acknowledgement IDs.
+    // Optional. Successfully processed acknowledgment IDs.
     repeated string ack_ids = 1 [(google.api.field_behavior) = OPTIONAL];
 
-    // Optional. List of acknowledgement IDs that were malformed or whose
-    // acknowledgement deadline has expired.
+    // Optional. List of acknowledgment IDs that were malformed or whose
+    // acknowledgment deadline has expired.
     repeated string invalid_ack_ids = 2
         [(google.api.field_behavior) = OPTIONAL];
 
-    // Optional. List of acknowledgement IDs that failed processing with
+    // Optional. List of acknowledgment IDs that failed processing with
     // temporary issues.
     repeated string temporary_failed_ack_ids = 3
         [(google.api.field_behavior) = OPTIONAL];
@@ -2109,6 +2222,8 @@ message Snapshot {
   option (google.api.resource) = {
     type: "pubsub.googleapis.com/Snapshot"
     pattern: "projects/{project}/snapshots/{snapshot}"
+    plural: "snapshots"
+    singular: "snapshot"
   };
 
   // Optional. The name of the snapshot.

protos/google/pubsub/v1/schema.proto

@@ -1,4 +1,4 @@
-// Copyright 2024 Google LLC
+// Copyright 2025 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -23,9 +23,8 @@ import "google/api/resource.proto";
 import "google/protobuf/empty.proto";
 import "google/protobuf/timestamp.proto";
 
-option cc_enable_arenas = true;
 option csharp_namespace = "Google.Cloud.PubSub.V1";
-option go_package = "cloud.google.com/go/pubsub/apiv1/pubsubpb;pubsubpb";
+option go_package = "cloud.google.com/go/pubsub/v2/apiv1/pubsubpb;pubsubpb";
 option java_multiple_files = true;
 option java_outer_classname = "SchemaProto";
 option java_package = "com.google.pubsub.v1";