diff --git a/changelog/fragments/1760515775-batchingconfig.yaml b/changelog/fragments/1760515775-batchingconfig.yaml new file mode 100644 index 00000000000..98b3a83d49a --- /dev/null +++ b/changelog/fragments/1760515775-batchingconfig.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: other + +# Change summary; a 80ish characters long description of the change. +summary: Updating EDOT gateway collector configuration samples for mOTLP input + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc. +component: elastic-agent + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +#pr: https://github.com/owner/repo/1234 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +#issue: https://github.com/owner/repo/1234 diff --git a/deploy/helm/edot-collector/kube-stack/managed_otlp/logs-values.yaml b/deploy/helm/edot-collector/kube-stack/managed_otlp/logs-values.yaml index 9486d8ead56..e950a4b0818 100644 --- a/deploy/helm/edot-collector/kube-stack/managed_otlp/logs-values.yaml +++ b/deploy/helm/edot-collector/kube-stack/managed_otlp/logs-values.yaml @@ -234,22 +234,27 @@ collectors: http: endpoint: ${env:MY_POD_IP}:4318 processors: - batch: - send_batch_size: 1000 - timeout: 1s - send_batch_max_size: 1500 exporters: debug: otlp/ingest: endpoint: ${env:ELASTIC_OTLP_ENDPOINT} headers: Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_timeout: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed timeout: 15s service: pipelines: logs: receivers: [otlp] - processors: [batch] + processors: [] exporters: [debug, otlp/ingest] # For more details on OpenTelemetry's zero-code instrumentation, see: # https://opentelemetry.io/docs/concepts/instrumentation/zero-code/ diff --git a/deploy/helm/edot-collector/kube-stack/managed_otlp/values.yaml b/deploy/helm/edot-collector/kube-stack/managed_otlp/values.yaml index 8f5099ea6f3..7cca427b7ed 100644 --- a/deploy/helm/edot-collector/kube-stack/managed_otlp/values.yaml +++ b/deploy/helm/edot-collector/kube-stack/managed_otlp/values.yaml @@ -545,35 +545,50 @@ collectors: http: endpoint: ${env:MY_POD_IP}:4318 processors: - batch: - send_batch_size: 1000 - timeout: 1s - send_batch_max_size: 1500 - batch/metrics: - # explicitly set send_batch_max_size to 0, as splitting metrics requests may cause version_conflict_engine_exception in TSDB - send_batch_max_size: 0 - timeout: 1s exporters: debug: - otlp/ingest: + otlp/ingest_logs: endpoint: ${env:ELASTIC_OTLP_ENDPOINT} headers: Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_timeout: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed + timeout: 15s + otlp/ingest_metrics_traces: + endpoint: ${env:ELASTIC_OTLP_ENDPOINT} + headers: + Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_timeout: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed timeout: 15s service: pipelines: metrics: receivers: [otlp] - processors: [batch/metrics] - exporters: [debug, otlp/ingest] + processors: [] + exporters: [debug, otlp/ingest_metrics_traces] logs: receivers: [otlp] - processors: [batch] - exporters: [debug, otlp/ingest] + processors: [] + exporters: [debug, otlp/ingest_logs] traces: receivers: [otlp] - processors: [batch] - exporters: [debug, otlp/ingest] + processors: [] + exporters: [debug, otlp/ingest_metrics_traces] # For more details on OpenTelemetry's zero-code instrumentation, see: # https://opentelemetry.io/docs/concepts/instrumentation/zero-code/ instrumentation: diff --git a/internal/pkg/otel/samples/darwin/managed_otlp/logs_metrics_traces.yml b/internal/pkg/otel/samples/darwin/managed_otlp/logs_metrics_traces.yml index b36283399c3..182924f2525 100644 --- a/internal/pkg/otel/samples/darwin/managed_otlp/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/darwin/managed_otlp/logs_metrics_traces.yml @@ -79,10 +79,32 @@ processors: enabled: true exporters: - otlp/ingest: + otlp/ingest_logs: endpoint: ${env:ELASTIC_OTLP_ENDPOINT} headers: Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_interval: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed + otlp/ingest_metrics_traces: + endpoint: ${env:ELASTIC_OTLP_ENDPOINT} + headers: + Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_interval: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed service: extensions: [file_storage] @@ -90,24 +112,24 @@ service: traces/fromsdk: receivers: [otlp/fromsdk] processors: [] - exporters: [otlp/ingest] + exporters: [otlp/ingest_metrics_traces] metrics/fromsdk: receivers: [otlp/fromsdk] processors: [] - exporters: [otlp/ingest] + exporters: [otlp/ingest_metrics_traces] logs/fromsdk: receivers: [otlp/fromsdk] processors: [] - exporters: [otlp/ingest] + exporters: [otlp/ingest_logs] metrics/hostmetrics: receivers: [hostmetrics/system] processors: [resourcedetection] - exporters: [otlp/ingest] + exporters: [otlp/ingest_metrics_traces] logs/platformlogs: receivers: [filelog/platformlogs] processors: [resourcedetection] - exporters: [otlp/ingest] + exporters: [otlp/ingest_logs] diff --git a/internal/pkg/otel/samples/darwin/managed_otlp/platformlogs.yml b/internal/pkg/otel/samples/darwin/managed_otlp/platformlogs.yml index f5782e2063a..f8543d99ebc 100644 --- a/internal/pkg/otel/samples/darwin/managed_otlp/platformlogs.yml +++ b/internal/pkg/otel/samples/darwin/managed_otlp/platformlogs.yml @@ -57,6 +57,15 @@ exporters: endpoint: ${env:ELASTIC_OTLP_ENDPOINT} headers: Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_interval: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed service: extensions: [file_storage] diff --git a/internal/pkg/otel/samples/darwin/managed_otlp/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/darwin/managed_otlp/platformlogs_hostmetrics.yml index c6405129024..80b55c44a7c 100644 --- a/internal/pkg/otel/samples/darwin/managed_otlp/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/darwin/managed_otlp/platformlogs_hostmetrics.yml @@ -78,6 +78,15 @@ exporters: endpoint: ${env:ELASTIC_OTLP_ENDPOINT} headers: Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_interval: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed service: extensions: [file_storage] diff --git a/internal/pkg/otel/samples/linux/managed_otlp/logs_metrics_traces.yml b/internal/pkg/otel/samples/linux/managed_otlp/logs_metrics_traces.yml index 5d20a31c9fd..250a6263add 100644 --- a/internal/pkg/otel/samples/linux/managed_otlp/logs_metrics_traces.yml +++ b/internal/pkg/otel/samples/linux/managed_otlp/logs_metrics_traces.yml @@ -86,10 +86,32 @@ processors: enabled: true exporters: - otlp/ingest: + otlp/ingest_logs: endpoint: ${env:ELASTIC_OTLP_ENDPOINT} headers: Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_interval: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed + otlp/ingest_metrics_traces: + endpoint: ${env:ELASTIC_OTLP_ENDPOINT} + headers: + Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_interval: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed service: extensions: [file_storage] @@ -97,24 +119,24 @@ service: traces/fromsdk: receivers: [otlp/fromsdk] processors: [] - exporters: [otlp/ingest] + exporters: [otlp/ingest_metrics_traces] metrics/fromsdk: receivers: [otlp/fromsdk] processors: [] - exporters: [otlp/ingest] + exporters: [otlp/ingest_metrics_traces] logs/fromsdk: receivers: [otlp/fromsdk] processors: [] - exporters: [otlp/ingest] + exporters: [otlp/ingest_logs] metrics/hostmetrics: receivers: [hostmetrics/system] processors: [resourcedetection] - exporters: [otlp/ingest] + exporters: [otlp/ingest_metrics_traces] logs/platformlogs: receivers: [filelog/platformlogs] processors: [resourcedetection] - exporters: [otlp/ingest] + exporters: [otlp/ingest_logs] diff --git a/internal/pkg/otel/samples/linux/managed_otlp/platformlogs.yml b/internal/pkg/otel/samples/linux/managed_otlp/platformlogs.yml index e9c6811b999..568f89bbadd 100644 --- a/internal/pkg/otel/samples/linux/managed_otlp/platformlogs.yml +++ b/internal/pkg/otel/samples/linux/managed_otlp/platformlogs.yml @@ -57,6 +57,15 @@ exporters: endpoint: ${env:ELASTIC_OTLP_ENDPOINT} headers: Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_interval: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed service: extensions: [file_storage] diff --git a/internal/pkg/otel/samples/linux/managed_otlp/platformlogs_hostmetrics.yml b/internal/pkg/otel/samples/linux/managed_otlp/platformlogs_hostmetrics.yml index decee7ba5c0..8ecdd18503a 100644 --- a/internal/pkg/otel/samples/linux/managed_otlp/platformlogs_hostmetrics.yml +++ b/internal/pkg/otel/samples/linux/managed_otlp/platformlogs_hostmetrics.yml @@ -85,6 +85,15 @@ exporters: endpoint: ${env:ELASTIC_OTLP_ENDPOINT} headers: Authorization: ApiKey ${env:ELASTIC_API_KEY} + sending_queue: + enabled: true + sizer: bytes + queue_size: 50000000 # 50MB uncompressed + block_on_overflow: true + batch: + flush_interval: 1s + min_size: 1_000_000 # 1MB uncompressed + max_size: 4_000_000 # 4MB uncompressed service: extensions: [file_storage]