errors with label {{ massage }}

[AkakievKD]

A note for the community

No response

Problem

i dont understand why i get these error when i add the field message: "{{ message }}"
without this field all work correct but at the same time I see this label in loki

Configuration

apiVersion: v1
kind: ConfigMap
metadata:
  name: vector
  namespace: loki
data:
  vector.yaml: |-
    api:
      address: 0.0.0.0:8686
      enabled: true
      playground: false
    data_dir: /var/local/lib/vector/
    log_schema:
      host_key: host
      message_key: message
      source_type_key: source_type
      timestamp_key: timestamp
    sinks:
      loki:
        compression: snappy
        encoding:
          codec: json
        endpoint: http://loki-loki-distributed-gateway
        inputs:
          - kube_logs
        labels:
          message: "{{ message }}"
          file: "{{ file }}"
          stream: "{{ stream }}"
          k8s_pod_namespace: "{{ kubernetes.pod_namespace }}"
          k8s_pod_name: "{{ kubernetes.pod_name }}"
          k8s_pod_uid: "{{ kubernetes.pod_uid }}"
          k8s_namespace_labels_*: "{{ kubernetes.namespace_labels }}"
          k8s_pod_node_name: "{{ kubernetes.pod_node_name }}"
          k8s_pod_annotations_*: "{{ kubernetes.pod_annotations }}"
          k8s_node_labels_*: "{{ kubernetes.node_labels }}"
          k8s_container_image: "{{ kubernetes.container_image }}"
          k8s_pod_owner: "{{ kubernetes.pod_owner }}" 
          k8s_container_image_id: "{{ kubernetes.container_image_id }}"
          k8s_container_id: "{{ kubernetes.container_id }}"
          k8s_pod_labels_*: "{{ kubernetes.pod_labels }}"
          k8s_container_name: "{{ kubernetes.container_name }}"
          k8s_pod_ip: "{{ kubernetes.pod_ip }}"
        request:
          in_flight_limit: 1
        out_of_order_action: accept
        path: /loki/api/v1/push
        remove_timestamp: true
        type: loki
    sources:
      kube_logs:
        auto_partial_merge: true
        data_dir: /var/local/lib/vector/
        delay_deletion_ms: 60000
        fingerprint_lines: 1
        glob_minimum_cooldown_ms: 30000
        ignore_older_secs: 600
        ingestion_timestamp_field: .ingest_timestamp
        max_line_bytes: 1000000
        max_read_bytes: 1000000
        read_from: beginning
        self_node_name: ${VECTOR_SELF_NODE_NAME}
        timezone: Europe/Moscow
        type: kubernetes_logs

### Version

latest

### Debug Output

2023-07-25T23:08:31.619567Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:08:41.876097Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 140 times.
2023-07-25T23:08:41.876141Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:08:41.876261Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:09:00.353597Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 4 times.
2023-07-25T23:09:00.353677Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:09:00.353800Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:09:31.136773Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 24 times.
2023-07-25T23:09:31.136815Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:09:33.209788Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:09:47.574007Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 4 times.
2023-07-25T23:09:47.574059Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:09:49.634743Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:10:08.121251Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 25 times.
2023-07-25T23:10:08.121291Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:10:08.121432Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:10:30.697554Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 3 times.
2023-07-25T23:10:30.697616Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:10:30.697759Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:10:47.133987Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 2 times.
2023-07-25T23:10:47.134024Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:10:59.445844Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:10:59.446036Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:11:28.200861Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 27 times.
2023-07-25T23:11:28.200913Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:11:28.201020Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:11:54.871740Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 3 times.
2023-07-25T23:11:54.871783Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:12:03.086185Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:12:07.209317Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 25 times.
2023-07-25T23:12:07.209348Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:12:07.209452Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:12:25.669049Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 6 times.
2023-07-25T23:12:25.669093Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:12:25.669207Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:12:44.138845Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 1 times.
2023-07-25T23:12:44.138885Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:12:44.138999Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.
2023-07-25T23:13:02.616531Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] has been suppressed 2 times.
2023-07-25T23:13:02.616570Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Failed to render template for "label_value "{{ massage }}" with label_key "massage"". error=Missing fields on event: ["massage"] error_type="template_failed" stage="processing" internal_log_rate_limit=true
2023-07-25T23:13:02.616688Z WARN sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}: vector::internal_events::template: Internal log [Failed to render template for "label_value "{{ massage }}" with label_key "massage"".] is being suppressed to avoid flooding.

### Example Data

_No response_

### Additional Context

_No response_

### References

_No response_

[jszwedko]

Hi @Nello-Angelo ,

Your log output there doesn't show any errors, but I'm fairly confident you don't want to be setting a message label as that will create a unique Loki stream for every message. Can you describe what you are trying to do? What motivated you to add the message label?

[AkakievKD]

sorry i updated my errors massage

[AkakievKD]

i want to see additional field in log without additional conditions

[jszwedko]

I think you have a typo: it should be message, not massage. I still don't think it is going to do what you want in Loki though. It'll probably cause bad behavior in Loki to have a separate stream for every single message.

[AkakievKD]

yes sorry i write "message" i did mistake but why get this errors?

[AkakievKD]

can your explane why so happend?

[jszwedko]

You got the error because your configuration was referring to a field that did not exist: massage. I'm going to convert this to a discussion because it isn't a bug.

[AkakievKD]

I have the same errors with correct writing

[AkakievKD]

2023-07-26T11:44:49.245439Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=62}: vector::sinks::util::retries: Internal log [Non-retriable error; dropping the request.] is being suppressed to avoid flooding.
2023-07-26T11:44:49.245507Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=62}: vector_common::internal_event::service: Internal log [Service call failed. No retries or retries exhausted.] is being suppressed to avoid flooding.
2023-07-26T11:44:49.245525Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=62}: vector_common::internal_event::component_events_dropped: Internal log [Events dropped] is being suppressed to avoid flooding.
2023-07-26T11:44:58.938582Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=71}: vector::sinks::util::retries: Internal log [Non-retriable error; dropping the request.] has been suppressed 9 times.
2023-07-26T11:44:58.938619Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=71}: vector::sinks::util::retries: Non-retriable error; dropping the request. error=Server responded with an error: 400 Bad Request internal_log_rate_limit=true
2023-07-26T11:44:58.938645Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=71}: vector_common::internal_event::service: Internal log [Service call failed. No retries or retries exhausted.] has been suppressed 9 times.
2023-07-26T11:44:58.938649Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=71}: vector_common::internal_event::service: Service call failed. No retries or retries exhausted. error=Some(ServerError { code: 400 }) request_id=71 error_type="request_failed" stage="sending" internal_log_rate_limit=true
2023-07-26T11:44:58.938667Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=71}: vector_common::internal_event::component_events_dropped: Internal log [Events dropped] has been suppressed 9 times.
2023-07-26T11:44:58.938671Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=71}: vector_common::internal_event::component_events_dropped: Events dropped intentional=false count=1 reason="Service call failed. No retries or retries exhausted." internal_log_rate_limit=true
2023-07-26T11:44:59.977031Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=72}: vector::sinks::util::retries: Internal log [Non-retriable error; dropping the request.] is being suppressed to avoid flooding.
2023-07-26T11:44:59.977118Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=72}: vector_common::internal_event::service: Internal log [Service call failed. No retries or retries exhausted.] is being suppressed to avoid flooding.
2023-07-26T11:44:59.977137Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=72}: vector_common::internal_event::component_events_dropped: Internal log [Events dropped] is being suppressed to avoid flooding.
2023-07-26T11:45:09.467584Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=81}: vector::sinks::util::retries: Internal log [Non-retriable error; dropping the request.] has been suppressed 9 times.
2023-07-26T11:45:09.467623Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=81}: vector::sinks::util::retries: Non-retriable error; dropping the request. error=Server responded with an error: 400 Bad Request internal_log_rate_limit=true
2023-07-26T11:45:09.467649Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=81}: vector_common::internal_event::service: Internal log [Service call failed. No retries or retries exhausted.] has been suppressed 9 times.
2023-07-26T11:45:09.467654Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=81}: vector_common::internal_event::service: Service call failed. No retries or retries exhausted. error=Some(ServerError { code: 400 }) request_id=81 error_type="request_failed" stage="sending" internal_log_rate_limit=true
2023-07-26T11:45:09.467681Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=81}: vector_common::internal_event::component_events_dropped: Internal log [Events dropped] has been suppressed 9 times.
2023-07-26T11:45:09.467689Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=81}: vector_common::internal_event::component_events_dropped: Events dropped intentional=false count=1 reason="Service call failed. No retries or retries exhausted." internal_log_rate_limit=true
2023-07-26T11:45:10.502244Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=82}: vector::sinks::util::retries: Internal log [Non-retriable error; dropping the request.] is being suppressed to avoid flooding.
2023-07-26T11:45:10.502298Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=82}: vector_common::internal_event::service: Internal log [Service call failed. No retries or retries exhausted.] is being suppressed to avoid flooding.
2023-07-26T11:45:10.502315Z ERROR sink{component_kind="sink" component_id=loki component_type=loki component_name=loki}:request{request_id=82}: vector_common::internal_event::component_events_dropped: Internal log [Events dropped] is being suppressed to avoid flooding.

[AkakievKD]

these messages i get with this config

api:
  address: 0.0.0.0:8686
  enabled: true
  playground: false
data_dir: /var/local/lib/vector/
log_schema:
  host_key: host
  message_key: message
  source_type_key: source_type
  timestamp_key: timestamp
sinks:
  loki:
    compression: snappy
    encoding:
      codec: json
    endpoint: http://loki-loki-distributed-gateway
    inputs:
      - kube_logs
    labels:
      message: "{{ message }}"
      file: "{{ file }}"
      stream: "{{ stream }}"
      k8s_pod_namespace: "{{ kubernetes.pod_namespace }}"
      k8s_pod_name: "{{ kubernetes.pod_name }}"
      k8s_pod_uid: "{{ kubernetes.pod_uid }}"
      k8s_namespace_labels_*: "{{ kubernetes.namespace_labels }}"
      k8s_pod_node_name: "{{ kubernetes.pod_node_name }}"
      k8s_pod_annotations_*: "{{ kubernetes.pod_annotations }}"
      k8s_node_labels_*: "{{ kubernetes.node_labels }}"
      k8s_container_image: "{{ kubernetes.container_image }}"
      k8s_pod_owner: "{{ kubernetes.pod_owner }}" 
      k8s_container_image_id: "{{ kubernetes.container_image_id }}"
      k8s_container_id: "{{ kubernetes.container_id }}"
      k8s_pod_labels_*: "{{ kubernetes.pod_labels }}"
      k8s_container_name: "{{ kubernetes.container_name }}"
      k8s_pod_ip: "{{ kubernetes.pod_ip }}"
    request:
      in_flight_limit: 1
    out_of_order_action: accept
    path: /loki/api/v1/push
    remove_timestamp: true
    type: loki
sources:
  kube_logs:
    auto_partial_merge: true
    data_dir: /var/local/lib/vector/
    delay_deletion_ms: 60000
    fingerprint_lines: 1
    glob_minimum_cooldown_ms: 30000
    ignore_older_secs: 600
    ingestion_timestamp_field: .ingest_timestamp
    max_line_bytes: 1000000
    max_read_bytes: 1000000
    read_from: beginning
    self_node_name: ${VECTOR_SELF_NODE_NAME}
    timezone: Europe/Moscow
    type: kubernetes_logs

[jszwedko]

I'm going to guess it is because you are adding message as a label, but I'd check the Loki logs to see if has any additional data. The error is indicating that Loki doesn't like the incoming data.