Set Default Telemetry Collection Interval in EDOT to 60s (#9987) (#10066)

(cherry picked from commit 402db56)

Co-authored-by: Davide Girardi <[email protected]>
Co-authored-by: Craig MacKenzie <[email protected]>

Author: 3 people

changelog/fragments/1758039757-reduce-default-telemetry-frequency.yaml

@@ -0,0 +1,45 @@
+# REQUIRED
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: bug-fix
+
+# REQUIRED for all kinds
+# Change summary; a 80ish characters long description of the change.
+summary: reduce-default-telemetry-frequency
+
+# REQUIRED for breaking-change, deprecation, known-issue
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+description: 
+    Reduce the default telemetry frequency to 60 seconds.
+    This change aims to lower infrastructure costs and reduce label churn in
+    time-series storage. High-cardinality labels sampled too frequently inflate
+    storage and index size, and increase query latency with limited added
+    value.  
+    Environments that require higher resolution can change the `collection_interval` for `hostmetrics`, `kubeletstats` and `k8s_cluster` receivers to a lower value.
+
+# REQUIRED for all kinds
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent
+
+# AUTOMATED
+# OPTIONAL to manually add other PR URLs
+# PR URL: A link the PR that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/9987
+
+# AUTOMATED
+# OPTIONAL to manually add other issue URLs
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+# issue: https://github.com/owner/repo/1234

deploy/helm/edot-collector/kube-stack/managed_otlp/values.yaml

@@ -139,6 +139,7 @@ collectors:
                 - "DELETED"
         # [K8s Cluster Receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/k8sclusterreceiver)
         k8s_cluster:
+          collection_interval: 60s
           auth_type: serviceAccount # Determines how to authenticate to the K8s API server. This can be one of none (for no auth), serviceAccount (to use the standard service account token provided to the agent pod), or kubeConfig to use credentials from ~/.kube/config.
           node_conditions_to_report:
             - Ready
@@ -375,7 +376,7 @@ collectors:
               type: container
         # [Hostmetrics Receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver)
         hostmetrics:
-          collection_interval: 10s
+          collection_interval: 60s
           root_path: /hostfs # Mounted node's root file system
           scrapers:
             cpu:
@@ -445,7 +446,7 @@ collectors:
         # [Kubelet Stats Receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/kubeletstatsreceiver)
         kubeletstats:
           auth_type: serviceAccount # Authentication mechanism with the Kubelet endpoint, refer to: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/kubeletstatsreceiver#configuration
-          collection_interval: 20s
+          collection_interval: 60s
           endpoint: ${env:OTEL_K8S_NODE_NAME}:10250
           node: "${env:OTEL_K8S_NODE_NAME}"
           # Required to work for all CSPs without an issue

deploy/helm/edot-collector/kube-stack/values.yaml

@@ -119,6 +119,7 @@ collectors:
                 - "DELETED"
         # [K8s Cluster Receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/k8sclusterreceiver)
         k8s_cluster:
+          collection_interval: 60s
           auth_type: serviceAccount # Determines how to authenticate to the K8s API server. This can be one of none (for no auth), serviceAccount (to use the standard service account token provided to the agent pod), or kubeConfig to use credentials from ~/.kube/config.
           node_conditions_to_report:
             - Ready
@@ -335,7 +336,7 @@ collectors:
               type: container
         # [Hostmetrics Receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver)
         hostmetrics:
-          collection_interval: 10s
+          collection_interval: 60s
           root_path: /hostfs # Mounted node's root file system
           scrapers:
             cpu:
@@ -405,7 +406,7 @@ collectors:
         # [Kubelet Stats Receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/kubeletstatsreceiver)
         kubeletstats:
           auth_type: serviceAccount # Authentication mechanism with the Kubelet endpoint, refer to: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/kubeletstatsreceiver#configuration
-          collection_interval: 20s
+          collection_interval: 60s
           endpoint: ${env:OTEL_K8S_NODE_NAME}:10250
           node: '${env:OTEL_K8S_NODE_NAME}'
           # Required to work for all CSPs without an issue

internal/pkg/otel/samples/darwin/logs_metrics_traces.yml

@@ -10,7 +10,7 @@ receivers:
 
   # Receiver for CPU, Disk, Memory, and Filesystem metrics
   hostmetrics/system:
-    collection_interval: 30s
+    collection_interval: 60s
     scrapers:
       filesystem:
       memory:

internal/pkg/otel/samples/darwin/managed_otlp/logs_metrics_traces.yml

@@ -10,7 +10,7 @@ receivers:
 
   # Receiver for CPU, Disk, Memory, and Filesystem metrics
   hostmetrics/system:
-    collection_interval: 30s
+    collection_interval: 60s
     scrapers:
       filesystem:
       memory:

internal/pkg/otel/samples/darwin/managed_otlp/platformlogs_hostmetrics.yml

@@ -10,7 +10,7 @@ receivers:
 
   # Receiver for CPU, Disk, Memory, and Filesystem metrics
   hostmetrics/system:
-    collection_interval: 30s
+    collection_interval: 60s
     scrapers:
       filesystem:
       memory:

internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml

@@ -10,7 +10,7 @@ receivers:
 
   # Receiver for CPU, Disk, Memory, and Filesystem metrics
   hostmetrics/system:
-    collection_interval: 30s
+    collection_interval: 60s
     scrapers:
       filesystem:
       memory:

internal/pkg/otel/samples/linux/logs_metrics_traces.yml

@@ -10,7 +10,7 @@ receivers:
 
   # Receiver for CPU, Disk, Memory, and Filesystem metrics
   hostmetrics/system:
-    collection_interval: 30s
+    collection_interval: 60s
     scrapers:
       disk:
       filesystem:

internal/pkg/otel/samples/linux/managed_otlp/logs_metrics_traces.yml

@@ -10,7 +10,7 @@ receivers:
 
   # Receiver for CPU, Disk, Memory, and Filesystem metrics
   hostmetrics/system:
-    collection_interval: 30s
+    collection_interval: 60s
     scrapers:
       disk:
       filesystem:

internal/pkg/otel/samples/linux/managed_otlp/platformlogs_hostmetrics.yml

@@ -10,7 +10,7 @@ receivers:
 
   # Receiver for CPU, Disk, Memory, and Filesystem metrics
   hostmetrics/system:
-    collection_interval: 30s
+    collection_interval: 60s
     scrapers:
       disk:
       filesystem: