File to GCS event dropped

[braden00]

Hi

I am using vector 0.34.0 to send logs file to GCP Cloud Storage without any issues for a month. But recently got an error below and some log was disappeared.
Not sure if this is an issue or if there is any existing configuration option to handle this?
Expected behavior: Retry sending to GCS in case of failure and no log was dropped

error1

{
  "span": {
    "name": "request",
    "request_id": 607
  },
  "error": "Failed to make HTTP(S) request: error writing a body to connection: Connection reset by peer (os error 104)",
  "message": "Unexpected error type; dropping the request.",
  "target": "vector::sinks::util::retries",
  "internal_log_rate_limit": true,
  "spans": [
    {
      "component_kind": "sink",
      "name": "sink",
      "component_id": "start_event",
      "component_type": "gcp_cloud_storage"
    },
    {
      "name": "request",
      "request_id": 607
    }
  ],
  "level": "ERROR",
  "timestamp": "2024-02-27T13:22:46.637798Z"
}

error2

{
  "span": {
    "request_id": 607,
    "name": "request"
  },
  "stage": "sending",
  "message": "Service call failed. No retries or retries exhausted.",
  "request_id": 607,
  "timestamp": "2024-02-27T13:22:46.643160Z",
  "spans": [
    {
      "name": "sink",
      "component_id": "start_event",
      "component_type": "gcp_cloud_storage",
      "component_kind": "sink"
    },
    {
      "name": "request",
      "request_id": 607
    }
  ],
  "internal_log_rate_limit": true,
  "error": "Some(CallRequest { source: hyper::Error(BodyWrite, Os { code: 104, kind: ConnectionReset, message: \"Connection reset by peer\" }) })",
  "target": "vector_common::internal_event::service",
  "level": "ERROR",
  "error_type": "request_failed"
}

error3

{
  "message": "Events dropped",
  "timestamp": "2024-02-27T13:22:46.643221Z",
  "count": 491,
  "target": "vector_common::internal_event::component_events_dropped",
  "internal_log_rate_limit": true,
  "reason": "Service call failed. No retries or retries exhausted.",
  "intentional": false,
  "span": {
    "name": "request",
    "request_id": 607
  },
  "spans": [
    {
      "component_id": "start_event",
      "name": "sink",
      "component_kind": "sink",
      "component_type": "gcp_cloud_storage"
    },
    {
      "request_id": 607,
      "name": "request"
    }
  ],
  "level": "ERROR"
}

config

    data_dir: "/usr/share/vector/data"
    sources:
      logs:
        type: "file"
        include:
          - "/log/*"
        ignore_older_secs: 7200
        offset_key: offset
      vector_metrics:
        type: "internal_metrics"
    transforms:
      jsonParse:
        type: remap
        inputs:
          - logs
        source: |-
          .message = parse_json!(.message)
      eventFilter:
        type: filter
        inputs: 
          - jsonParse
        condition: '.message.s_event=="START"'
    sinks:
      prometheus:
        type: prometheus_exporter
        inputs:
          - vector_metrics
      start_event:
        type: gcp_cloud_storage
        inputs:
          - eventFilter
        bucket: <GCS BUCKET>
        encoding:
          codec: json
        batch:
          max_bytes: 268435488
          max_events: 40000
          timeout_secs: 30
        buffer:
          type: disk
          max_size: 2684354880
          when_full: block
        key_prefix: "start/date=%F/"
        filename_extension: ndjson
        framing:
          method: newline_delimited

[jszwedko]

Hmm, yeah, that's odd. That error should have been retried. Do you mind opening a full bug report?

[braden00]

Thanks for your reply.
Issue #19988