diff --git a/solutions/observability/apps/apm-agent-compatibility.md b/solutions/observability/apps/apm-agent-compatibility.md index 23dad90a7c..5ec8f4c297 100644 --- a/solutions/observability/apps/apm-agent-compatibility.md +++ b/solutions/observability/apps/apm-agent-compatibility.md @@ -44,7 +44,7 @@ The chart below outlines the compatibility between different versions of Elastic | `1.x` | ≥ `6.5` | ::::{note} -Java agent < 1.43.0 not fully compatible with APM Server >= 8.11.0. Read more in [Known issues](../../../troubleshoot/observability/apm/known-issues.md#apm-empty-metricset-values). +Java agent < 1.43.0 not fully compatible with APM Server >= 8.11.0. Read more in [Known issues](/release-notes/known-issues/elastic-apm.md). :::: diff --git a/troubleshoot/observability.md b/troubleshoot/observability.md index 2224ada060..9f81d9c2c5 100644 --- a/troubleshoot/observability.md +++ b/troubleshoot/observability.md @@ -2,6 +2,10 @@ navigation_title: "Observability" mapped_pages: - https://www.elastic.co/guide/en/observability/current/troubleshooting.html +applies_to: + stack: all + serverless: + observability: all --- # Troubleshoot Observability [troubleshooting] diff --git a/troubleshoot/observability/amazon-data-firehose.md b/troubleshoot/observability/amazon-data-firehose.md index 33331def57..239b8a9e74 100644 --- a/troubleshoot/observability/amazon-data-firehose.md +++ b/troubleshoot/observability/amazon-data-firehose.md @@ -2,6 +2,10 @@ navigation_title: "Amazon Data Firehose" mapped_pages: - https://www.elastic.co/guide/en/observability/current/monitor-aws-firehose-troubleshooting.html +applies_to: + stack: all + serverless: + observability: all --- diff --git a/troubleshoot/observability/apm-agent-dotnet/apm-net-agent.md b/troubleshoot/observability/apm-agent-dotnet/apm-net-agent.md index 5051ae50fd..99d7c3de2b 100644 --- a/troubleshoot/observability/apm-agent-dotnet/apm-net-agent.md +++ b/troubleshoot/observability/apm-agent-dotnet/apm-net-agent.md @@ -2,6 +2,10 @@ navigation_title: "APM .NET agent" mapped_pages: - https://www.elastic.co/guide/en/apm/agent/dotnet/current/troubleshooting.html +applies_to: + stack: all + serverless: + observability: all --- # Troubleshoot APM .NET agent diff --git a/troubleshoot/observability/apm-agent-go/apm-go-agent.md b/troubleshoot/observability/apm-agent-go/apm-go-agent.md index 77d317c444..82bd452f85 100644 --- a/troubleshoot/observability/apm-agent-go/apm-go-agent.md +++ b/troubleshoot/observability/apm-agent-go/apm-go-agent.md @@ -2,6 +2,10 @@ navigation_title: "APM Go Agent" mapped_pages: - https://www.elastic.co/guide/en/apm/agent/go/current/troubleshooting.html +applies_to: + stack: all + serverless: + observability: all --- # Troubleshoot APM Go Agent diff --git a/troubleshoot/observability/apm-agent-java/apm-java-agent.md b/troubleshoot/observability/apm-agent-java/apm-java-agent.md index 380b5a05d0..b919204415 100644 --- a/troubleshoot/observability/apm-agent-java/apm-java-agent.md +++ b/troubleshoot/observability/apm-agent-java/apm-java-agent.md @@ -2,6 +2,10 @@ navigation_title: "APM Java Agent" mapped_pages: - https://www.elastic.co/guide/en/apm/agent/java/current/trouble-shooting.html +applies_to: + stack: all + serverless: + observability: all --- # Troubleshoot APM Java Agent [trouble-shooting] diff --git a/troubleshoot/observability/apm-agent-nodejs/apm-nodejs-agent.md b/troubleshoot/observability/apm-agent-nodejs/apm-nodejs-agent.md index eac2c7bded..98e6ccf82d 100644 --- a/troubleshoot/observability/apm-agent-nodejs/apm-nodejs-agent.md +++ b/troubleshoot/observability/apm-agent-nodejs/apm-nodejs-agent.md @@ -2,6 +2,10 @@ navigation_title: "APM Node.js Agent" mapped_pages: - https://www.elastic.co/guide/en/apm/agent/nodejs/current/troubleshooting.html +applies_to: + stack: all + serverless: + observability: all --- # Troubleshoot APM Node.js Agent [troubleshooting] diff --git a/troubleshoot/observability/apm-agent-php/apm-php-agent.md b/troubleshoot/observability/apm-agent-php/apm-php-agent.md index dfaeb308a2..5f83339dc3 100644 --- a/troubleshoot/observability/apm-agent-php/apm-php-agent.md +++ b/troubleshoot/observability/apm-agent-php/apm-php-agent.md @@ -2,6 +2,10 @@ navigation_title: "APM PHP Agent" mapped_pages: - https://www.elastic.co/guide/en/apm/agent/php/current/troubleshooting.html +applies_to: + stack: all + serverless: + observability: all --- # Troubleshoot APM PHP Agent [troubleshooting] diff --git a/troubleshoot/observability/apm-agent-python/apm-python-agent.md b/troubleshoot/observability/apm-agent-python/apm-python-agent.md index b0f1744078..522f8b7c51 100644 --- a/troubleshoot/observability/apm-agent-python/apm-python-agent.md +++ b/troubleshoot/observability/apm-agent-python/apm-python-agent.md @@ -2,6 +2,10 @@ navigation_title: "APM Python Agent" mapped_pages: - https://www.elastic.co/guide/en/apm/agent/python/current/troubleshooting.html +applies_to: + stack: all + serverless: + observability: all --- # Troubleshoot APM Python Agent [troubleshooting] diff --git a/troubleshoot/observability/apm-agent-ruby/apm-ruby-agent.md b/troubleshoot/observability/apm-agent-ruby/apm-ruby-agent.md index 96822ff7b0..188c0e2d0c 100644 --- a/troubleshoot/observability/apm-agent-ruby/apm-ruby-agent.md +++ b/troubleshoot/observability/apm-agent-ruby/apm-ruby-agent.md @@ -2,6 +2,10 @@ navigation_title: "APM Ruby Agent" mapped_pages: - https://www.elastic.co/guide/en/apm/agent/ruby/current/debugging.html +applies_to: + stack: all + serverless: + observability: all --- # Troubleshoot APM Ruby Agent [debugging] diff --git a/troubleshoot/observability/apm-agent-rum-js/apm-real-user-monitoring-javascript-agent.md b/troubleshoot/observability/apm-agent-rum-js/apm-real-user-monitoring-javascript-agent.md index 7be6c7e082..9577110a3c 100644 --- a/troubleshoot/observability/apm-agent-rum-js/apm-real-user-monitoring-javascript-agent.md +++ b/troubleshoot/observability/apm-agent-rum-js/apm-real-user-monitoring-javascript-agent.md @@ -2,6 +2,8 @@ navigation_title: "APM Real User Monitoring JavaScript Agent" mapped_pages: - https://www.elastic.co/guide/en/apm/agent/rum-js/current/troubleshooting.html +applies_to: + stack: all --- # Troubleshoot APM Real User Monitoring JavaScript Agent [troubleshooting] diff --git a/troubleshoot/observability/apm-agent-swift/apm-ios-agent.md b/troubleshoot/observability/apm-agent-swift/apm-ios-agent.md index 7c5e147bdc..dfe482cfd3 100644 --- a/troubleshoot/observability/apm-agent-swift/apm-ios-agent.md +++ b/troubleshoot/observability/apm-agent-swift/apm-ios-agent.md @@ -2,6 +2,8 @@ navigation_title: "APM iOS Agent" mapped_pages: - https://www.elastic.co/guide/en/apm/agent/swift/current/troubleshooting.html +applies_to: + stack: all --- # Troubleshoot APM iOS Agent [troubleshooting] diff --git a/troubleshoot/observability/apm.md b/troubleshoot/observability/apm.md index c94fdaca65..6c5c68e596 100644 --- a/troubleshoot/observability/apm.md +++ b/troubleshoot/observability/apm.md @@ -2,6 +2,10 @@ navigation_title: "APM" mapped_pages: - https://www.elastic.co/guide/en/observability/current/apm-troubleshoot-apm.html +applies_to: + stack: all + serverless: + observability: all --- diff --git a/troubleshoot/observability/apm/_agent_is_not_instrumenting_code.md b/troubleshoot/observability/apm/_agent_is_not_instrumenting_code.md index 4ffcc6a9ef..33ab1fa9e6 100644 --- a/troubleshoot/observability/apm/_agent_is_not_instrumenting_code.md +++ b/troubleshoot/observability/apm/_agent_is_not_instrumenting_code.md @@ -1,6 +1,10 @@ --- mapped_pages: - https://www.elastic.co/guide/en/apm/agent/php/current/_agent_is_not_instrumenting_code.html +applies_to: + stack: all + serverless: + observability: all --- # Agent is not instrumenting code [_agent_is_not_instrumenting_code] diff --git a/troubleshoot/observability/apm/_collection_of_diagnostic_information.md b/troubleshoot/observability/apm/_collection_of_diagnostic_information.md index 623efc3956..016c2b4fbb 100644 --- a/troubleshoot/observability/apm/_collection_of_diagnostic_information.md +++ b/troubleshoot/observability/apm/_collection_of_diagnostic_information.md @@ -1,6 +1,10 @@ --- mapped_pages: - https://www.elastic.co/guide/en/apm/agent/php/current/_collection_of_diagnostic_information.html +applies_to: + stack: all + serverless: + observability: all --- # Collection of diagnostic information [_collection_of_diagnostic_information] diff --git a/troubleshoot/observability/apm/apm-server-performance-diagnostic.md b/troubleshoot/observability/apm/apm-server-performance-diagnostic.md index 547e3ae4b2..9a4d55e97d 100644 --- a/troubleshoot/observability/apm/apm-server-performance-diagnostic.md +++ b/troubleshoot/observability/apm/apm-server-performance-diagnostic.md @@ -1,6 +1,8 @@ --- mapped_pages: - https://www.elastic.co/guide/en/observability/current/apm-performance-diagnostic.html +applies_to: + stack: all --- # APM Server performance diagnostic [apm-performance-diagnostic] diff --git a/troubleshoot/observability/apm/apm-server-response-codes.md b/troubleshoot/observability/apm/apm-server-response-codes.md index 2b8d8336c8..374fbb19f0 100644 --- a/troubleshoot/observability/apm/apm-server-response-codes.md +++ b/troubleshoot/observability/apm/apm-server-response-codes.md @@ -1,6 +1,10 @@ --- mapped_pages: - https://www.elastic.co/guide/en/observability/current/apm-common-response-codes.html +applies_to: + stack: all + serverless: + observability: all --- # APM Server response codes [apm-common-response-codes] @@ -8,17 +12,47 @@ mapped_pages: ## HTTP 400: Data decoding error / Data validation error [apm-bad-request] -The most likely cause for this error is using incompatible versions of {{apm-agent}} and APM Server. See the [agent/server compatibility matrix](../../../solutions/observability/apps/apm-agent-compatibility.md) to verify compatibility. +::::{tab-set} + +:::{tab-item} {{stack}} +The most likely cause for this error is using incompatible versions of {{apm-agent}} and APM Server. See the [agent/server compatibility matrix](/solutions/observability/apps/apm-agent-compatibility.md) to verify compatibility. +::: + +:::{tab-item} {{serverless-short}} +The most likely cause for this error is using an incompatible version of an {{apm-agent}}. See [minimum supported APM agent versions](/solutions/observability/apps/elastic-apm-agents.md#observability-apm-agents-elastic-apm-agents-minimum-supported-versions) to verify compatibility. +::: + +:::: ## HTTP 400: Event too large [apm-event-too-large] -APM agents communicate with the APM server by sending events in an HTTP request. Each event is sent as its own line in the HTTP request body. If events are too large, you should consider increasing the [Max event size](../../../solutions/observability/apps/general-configuration-options.md#apm-max_event_size) setting in the APM integration, and adjusting relevant settings in the agent. +::::{tab-set} + +:::{tab-item} {{stack}} +APM agents communicate with the APM server by sending events in an HTTP request. Each event is sent as its own line in the HTTP request body. If events are too large, you should consider increasing the [Max event size](/solutions/observability/apps/general-configuration-options.md#apm-max_event_size) setting in the APM integration, and adjusting relevant settings in the agent. +::: + +:::{tab-item} {{serverless-short}} +APM agents communicate with the Managed intake service by sending events in an HTTP request. Each event is sent as its own line in the HTTP request body. If events are too large, you can reduce the size of the events that your APM agents send by: [enabling span compression](/solutions/observability/apps/spans.md) or [reducing collected stack trace information](/solutions/observability/apps/reduce-storage.md#observability-apm-reduce-stacktrace). +::: + +:::: ## HTTP 401: Invalid token [apm-unauthorized] -Either the [Secret token](../../../solutions/observability/apps/secret-token.md) in the request header doesn’t match the secret token configured in the APM integration, or the [API keys](../../../solutions/observability/apps/api-keys.md) is invalid. +::::{tab-set} + +:::{tab-item} {{stack}} +Either the [Secret token](/solutions/observability/apps/secret-token.md) in the request header doesn’t match the secret token configured in the APM integration, or the [API keys](/solutions/observability/apps/api-keys.md) is invalid. +::: + +:::{tab-item} {{serverless-short}} +The API key is invalid. +::: + +:::: ## HTTP 403: Forbidden request [apm-forbidden] diff --git a/troubleshoot/observability/apm/common-problems.md b/troubleshoot/observability/apm/common-problems.md index 6c31c63315..f5d7fa5dbd 100644 --- a/troubleshoot/observability/apm/common-problems.md +++ b/troubleshoot/observability/apm/common-problems.md @@ -2,6 +2,10 @@ mapped_pages: - https://www.elastic.co/guide/en/observability/current/apm-common-problems.html - https://www.elastic.co/guide/en/serverless/current/observability-apm-troubleshooting.html +applies_to: + stack: all + serverless: + observability: all --- # Common problems [apm-common-problems] @@ -290,24 +294,3 @@ To fix this, make sure these two fields match exactly. For example, if the APM agent is not configured to use the correct host name, the host name might be set to the container name or the Kubernetes pod name. To get the correct host name, you need to set some additional configuration options, specifically `system.kubernetes.node.name` as described in [Kubernetes data](../../../solutions/observability/apps/elastic-apm-events-intake-api.md#apm-api-kubernetes-data). - -## Common response codes [observability-apm-troubleshooting-common-response-codes] -```yaml {applies_to} -serverless: all -``` - - -### HTTP 400: Data decoding error / Data validation error [bad-request] - -The most likely cause for this error is using an incompatible version of an {{apm-agent}}. See [minimum supported APM agent versions](../../../solutions/observability/apps/elastic-apm-agents.md#observability-apm-agents-elastic-apm-agents-minimum-supported-versions) to verify compatibility. - - -### HTTP 400: Event too large [event-too-large] - -APM agents communicate with the Managed intake service by sending events in an HTTP request. Each event is sent as its own line in the HTTP request body. If events are too large, you can reduce the size of the events that your APM agents send by: [enabling span compression](../../../solutions/observability/apps/spans.md) or [reducing collected stack trace information](../../../solutions/observability/apps/reduce-storage.md#observability-apm-reduce-stacktrace). - - -### HTTP 401: Invalid token [unauthorized] - -The API key is invalid. - diff --git a/troubleshoot/observability/apm/enable-apm-server-binary-debugging.md b/troubleshoot/observability/apm/enable-apm-server-binary-debugging.md index 5818ea27b0..97b0394f70 100644 --- a/troubleshoot/observability/apm/enable-apm-server-binary-debugging.md +++ b/troubleshoot/observability/apm/enable-apm-server-binary-debugging.md @@ -2,6 +2,8 @@ navigation_title: "APM Server binary debugging" mapped_pages: - https://www.elastic.co/guide/en/observability/current/apm-enable-apm-server-debugging.html +applies_to: + stack: all --- diff --git a/troubleshoot/observability/apm/known-issues.md b/troubleshoot/observability/apm/known-issues.md deleted file mode 100644 index 16420ef09e..0000000000 --- a/troubleshoot/observability/apm/known-issues.md +++ /dev/null @@ -1,305 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/observability/current/apm-known-issues.html ---- - -# Known issues [apm-known-issues] - -APM has the following known issues: - - -## `prefer_ilm` required in component templates to create custom lifecycle policies [_prefer_ilm_required_in_component_templates_to_create_custom_lifecycle_policies] - -*Elastic Stack versions: 8.15.1+* - -The issue occurs when creating a *new* cluster using version 8.15.1+. The issue occurs for any APM data streams created in 8.15.1+. The issue does *not* occur if custom component template has been created in or before version 8.15.0. - -In 8.15.0, APM Server began using the [apm-data plugin](https://github.com/elastic/elasticsearch/tree/main/x-pack/plugin/apm-data) to manage data streams, ingest pipelines, lifecycle policies, and more. In 8.15.1, a fix was introduced to address unmanaged indices in older clusters using default ILM policies. This fix added a fallback to the default ILM policy (if it exists) and set the `prefer_ilm` configuration to `false`. This setting impacts clusters where both ILM and data stream lifecycles (DSL) are in effect—such as when configuring custom ILM policies using `@custom` component templates, under the conditions mentioned above. - -To override ILM policies for these new clusters using component template, set the `prefer_ilm` configuration to `true` by following the [updated guide to customize ILM](../../../solutions/observability/apps/index-lifecycle-management.md). - - -## Upgrading to v8.15.x may cause ingestion to fail [_upgrading_to_v8_15_x_may_cause_ingestion_to_fail] - -*Elastic Stack versions: 8.15.0, 8.15.1, 8.15.2, 8.15.3*
*Fixed in Elastic Stack version 8.15.4* - -The issue only occurs when *upgrading* the {{stack}} from 8.12.2 or lower directly to any 8.15.x version prior to 8.15.4. The issue does *not* occur when creating a *new* cluster using any 8.15.x version, or when upgrading from 8.12.2 to 8.13.x or 8.14.x and then to 8.15.x. - -In APM Servers versions prior to 8.13.0, an ingestion pipeline exists to perform a check on the version. The version check would fail any APM document produced with a different version of APM server compared to the version of the installed APM’s ingest pipeline. In 8.13.0 the version check in the ingest pipeline was removed. Due to the combination of an internal change in how apm data management assets are set up from 8.15 onwards and a bug in Elasticsearch, related to [lazy rollover of data streams](https://github.com/elastic/elasticsearch/issues/112781), the ingestion pipeline conducting the version check is not removed on upgrade and prevents the ingestion of data. - -If the deployment is running 8.15.0, upgrade the deployment to 8.15.1 or above. A manual rollover of all APM data streams is required to pick up the new index templates and remove the faulty ingest pipeline version check. Perform the following requests to Elasticsearch (they are assuming the `default` namespace is used, adjust if necessary): - -```txt -POST /traces-apm-default/_rollover -POST /traces-apm.rum-default/_rollover -POST /logs-apm.error-default/_rollover -POST /logs-apm.app-default/_rollover -POST /metrics-apm.app-default/_rollover -POST /metrics-apm.internal-default/_rollover -POST /metrics-apm.service_destination.1m-default/_rollover -POST /metrics-apm.service_destination.10m-default/_rollover -POST /metrics-apm.service_destination.60m-default/_rollover -POST /metrics-apm.service_summary.1m-default/_rollover -POST /metrics-apm.service_summary.10m-default/_rollover -POST /metrics-apm.service_summary.60m-default/_rollover -POST /metrics-apm.service_transaction.1m-default/_rollover -POST /metrics-apm.service_transaction.10m-default/_rollover -POST /metrics-apm.service_transaction.60m-default/_rollover -POST /metrics-apm.transaction.1m-default/_rollover -POST /metrics-apm.transaction.10m-default/_rollover -POST /metrics-apm.transaction.60m-default/_rollover -``` - - -## Upgrading to v8.15.0 may cause APM indices to lose their lifecycle policy [_upgrading_to_v8_15_0_may_cause_apm_indices_to_lose_their_lifecycle_policy] - -*Elastic Stack versions: 8.15.0*
*Fixed in Elastic Stack version 8.15.1* - -The issue only occurs when *upgrading* the {{stack}} to 8.15.0. The issue does *not* occur when creating a *new* cluster using 8.15.0. The issue also does not occur if a custom ILM policy is configured using a custom component template. - -In 8.15.0, APM Server switched to use data stream lifecycle to manage data retention for APM indices for new deployments as well as for upgraded deployments with default lifecycle configurations. Unfortunately, since any data stream created before 8.15.0 does not have a data stream lifecycle configuration, such existing data streams become unmanaged for default lifecycle configurations. - -Upgrading to 8.15.1 resolves the lifecycle issue for any new indices created for APM data streams. However, indices created in version 8.15.0 will remain unmanaged if the default ILM policy is in place. To fix these unmanaged indices, consider one of the following approaches: - -1. Manually delete the unmanaged indices when they are no longer needed. -2. Explicitly configure APM data streams to use the default data stream lifecycle configuration. This approach migrates all affected data streams to use data stream lifecycles, maintaining behavior equivalent to the default ILM policies. Apply this fix only to data streams that have unmanaged indices due to missing default ILM policies. - - ```txt - PUT _data_stream/{{data_stream_name}}-{{data_stream_namespace}}/_lifecycle - { - "data_retention": - } - ``` - - -Default `` for each data stream is available in [this guide](../../../solutions/observability/apps/index-lifecycle-management.md). - -This issue is fixed in 8.15.1 ([elastic/elasticsearch#112432](https://github.com/elastic/elasticsearch/pull/112432)). - - -## Upgrading to v8.13.0 to v8.13.2 breaks APM anomaly rules [broken-apm-anomaly-rule] - -*Elastic Stack versions: 8.13.0, 8.13.1, 8.13.2*
*Fixed in Elastic Stack version 8.13.3* - -This issue occurs when upgrading the Elastic Stack to version 8.13.0, 8.13.1, or 8.13.2. This issue may go unnoticed unless you actively monitor your {{kib}} logs. The following log indicates the presence of this issue: - -```shell -"params invalid: [anomalyDetectorTypes]: expected value of type [array] but got [undefined]" -``` - -This issue occurs because a non-optional parameter, `anomalyDetectorTypes` was added in 8.13.0 without the presence of an automation migration script. This breaks pre-existing rules as they do not have this parameter and will fail validation. This issue is fixed in v8.13.3. - -There are three ways to fix this error: - -* Upgrade to version 8.13.3 -* Fix broken anomaly rules in the APM UI (no upgrade required) -* Fix broken anomaly rules with Kibana APIs (no upgrade required) - -**Fix broken anomaly rules in the APM UI** - -1. From any APM page in Kibana, select **Alerts and rules** → **Manage rules**. -2. Filter your rules by setting **Type** to **APM Anomaly**. -3. For each anomaly rule in the list, select the pencil icon to edit the rule. -4. Add one or more **DETECTOR TYPES** to the rule. - - The detector type determines when the anomaly rule triggers. For example, a latency anomaly rule will trigger when the latency of the service being monitored is abnormal. Supported detector types are `latency`, `throughput`, and `failed transaction rate`. - -5. Click **Save**. - -**Fix broken anomaly rules with Kibana APIs** - -1. Find broken rules - - :::::{{tip} - To identify rules in this exact state, you can use the [find rules endpoint](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-alerting) and search for the APM anomaly rule type as well as this exact error message indicating that the rule is in the broken state. We will also use the `fields` parameter to specify only the fields required when making the update request later. - - * `search_fields=alertTypeId` - * `search=apm.anomaly` - * `filter=alert.attributes.executionStatus.error.message:"params invalid: [anomalyDetectorTypes]: expected value of type [array] but got [undefined]"` - * `fields=[id, name, actions, tags, schedule, notify_when, throttle, params]` - - The encoded request might look something like this: - - ```shell - curl -u "$KIBANA_USER":"$KIBANA_PASSWORD" "$KIBANA_URL/api/alerting/rules/_find?search_fields=alertTypeId&search=apm.anomaly&filter=alert.attributes.executionStatus.error.message%3A%22params%20invalid%3A%20%5BanomalyDetectorTypes%5D%3A%20expected%20value%20of%20type%20%5Barray%5D%20but%20got%20%5Bundefined%5D%22&fields=id&fields=name&fields=actions&fields=tags&fields=schedule&fields=notify_when&fields=throttle&fields=params" - ``` - - ::::{dropdown} Example result: - ```json - { - "page": 1, - "total": 1, - "per_page": 10, - "data": [ - { - "id": "d85e54de-f96a-49b5-99d4-63956f90a6eb", - "name": "APM Anomaly Jason Test FAILING [2]", - "tags": [ - "test", - "jasonrhodes" - ], - "throttle": null, - "schedule": { - "interval": "1m" - }, - "params": { - "windowSize": 30, - "windowUnit": "m", - "anomalySeverityType": "warning", - "environment": "ENVIRONMENT_ALL" - }, - "notify_when": null, - "actions": [] - } - ] - } - ``` - - :::: - - - ::::: - -2. Prepare the update JSON doc(s) - - ::::{tip} - For each broken rule found, create a JSON rule document with what was returned from the API in the previous step. You will need to make two changes to each document: - - 1. Remove the `id` key but keep the value connected to this document (e.g. rename the file to `{{id}}.json`). **The `id` cannot be sent as part of the request body for the PUT request, but you will need it for the URL path.** - 2. Add the `"anomalyDetectorTypes"` to the `"params"` block, using the default value as seen below to mimic the pre-8.13 behavior: - - ```json - { - "params": { - // ... other existing params should stay here, - // with the required one added to this object - "anomalyDetectorTypes": [ - "txLatency", - "txThroughput", - "txFailureRate" - ] - } - } - ``` - - - :::: - -3. Update each rule using the `PUT /api/alerting/rule/{{id}}` API - - ::::{tip} - For each rule, submit a PUT request to the [update rule endpoint](https://www.elastic.co/docs/api/doc/kibana/group/endpoint-alerting) using that rule’s ID and its stored update document from the previous step. For example, assuming the first broken rule’s ID is `046c0d4f`: - - ```shell - curl -u "$KIBANA_USER":"$KIBANA_PASSWORD" -XPUT "$KIBANA_URL/api/alerting/rule/046c0d4f" -H 'Content-Type: application/json' -H 'kbn-xsrf: rule-update' -d @046c0d4f.json - ``` - - Once the PUT request executes successfully, the rule will no longer be broken. - - :::: - - - -## Upgrading APM Server to 8.11+ might break event intake from older APM Java agents [apm-empty-metricset-values] - -*APM Server versions: >=8.11.0*
*Elastic APM Java agent versions: < 1.43.0* - -If you are using APM Server (> v8.11.0) and the Elastic APM Java agent (< v1.43.0), the agent may be sending empty histogram metricsets. - -In previous APM Server versions some data validation was not properly applied, leading the APM Server to accept empty histogram metricsets where it shouldn’t. This bug was fixed in the APM Server in 8.11.0. - -The APM Java agent (< v1.43.0) was sending this kind of invalid data under certain circumstances. If you upgrade the APM Server to v8.11.0+ *without* upgrading the APM Java agent version, metricsets can be rejected by the APM Server and can result in additional error logs in the Java agent. - -The fix is to upgrade the Elastic APM Java agent to a version >= 1.43.0. Find details in [elastic/apm-data#157](https://github.com/elastic/apm-data/pull/157). - - -## traces-apm@custom ingest pipeline applied to certain data streams unintentionally [_traces_apmcustom_ingest_pipeline_applied_to_certain_data_streams_unintentionally] - -*APM Server versions: 8.12.0*
- -If you’re using the Elastic APM Server v8.12.0, the `traces-apm@custom` ingest pipeline is now additionally applied to data streams `traces-apm.sampled-*` and `traces-apm.rum-*`, and applied twice for `traces-apm-*`. This bug impacts users with a non-empty `traces-apm@custom` ingest pipeline. - -If you rely on this unintended behavior in 8.12.0, please rename your pipeline to `traces-apm.integration@custom` to preserve this behavior in later versions. - -A fix was released in 8.12.1: [elastic/kibana#175448](https://github.com/elastic/kibana/pull/175448). - - -## Ingesting new JVM metrics in 8.9 and 8.10 breaks upgrade to 8.11 and stops ingestion [_ingesting_new_jvm_metrics_in_8_9_and_8_10_breaks_upgrade_to_8_11_and_stops_ingestion] - -*APM Server versions: 8.11.0, 8.11.1*
*Elastic APM Java agent versions: 1.39.0+* - -If you’re using the Elastic APM Java agent v1.39.0+ to send new JVM metrics to APM Server v8.9.x and v8.10.x, upgrading to 8.11.0 or 8.11.1 will silently fail and stop ingesting APM metrics. - -After upgrading, you will see the following errors: - -* APM Server error logs: - - ```txt - failed to index document in 'metrics-apm.internal-default' (fail_processor_exception): Document produced by APM Server v8.11.1, which is newer than the installed APM integration (v8.10.3-preview-1695284222). The APM integration must be upgraded. - ``` - -* Fleet error on integration package upgrade: - - ```txt - Failed installing package [apm] due to error: [ResponseError: mapper_parsing_exception - Root causes: - mapper_parsing_exception: Field [jvm.memory.non_heap.pool.committed] attempted to shadow a time_series_metric] - ``` - - -A fix was released in 8.11.2: [elastic/kibana#171712](https://github.com/elastic/kibana/pull/171712). - - -## APM integration package upgrade through Fleet causes excessive data stream rollovers [_apm_integration_package_upgrade_through_fleet_causes_excessive_data_stream_rollovers] - -*APM Server versions: <= 8.12.1 +* - -If you’re upgrading APM integration package to any versions <= 8.12.1, in some rare cases, the upgrade fails with a mapping conflict error. The upgrade process keeps rolling over the data stream in an unsuccessful attempt to work around the error. As a result, many empty backing indices for APM data streams are created. - -During upgrade, you will see errors similar to the one below: - -* Fleet error on integration package upgrade: - - ```txt - Mappings update for metrics-apm.service_destination.10m-default failed due to ResponseError: illegal_argument_exception - Root causes: - illegal_argument_exception: Mapper for [metricset.interval] conflicts with existing mapper: - Cannot update parameter [value] from [10m] to [null] - ``` - - -A fix was released in 8.12.2: [elastic/apm-server#12219](https://github.com/elastic/apm-server/pull/12219). - - -## Performance regression: APM issues too many small bulk requests for Elasticsearch output [_performance_regression_apm_issues_too_many_small_bulk_requests_for_elasticsearch_output] - -*APM Server versions: >=8.13.0, <= 8.14.2*
- -If you’re on APM server version >=8.13.0, <= 8.14.2_, using Elasticsearch output, do not specify any `output.elasticsearch.flush_bytes`, and do not disable compression explicitly by setting `output.elasticsearch.compression_level` to `0`, APM server will issue smaller bulk requests of 24KB size, and more bulk requests will need to be made to maintain the original throughput. This causes Elasticsearch to experience higher load, and APM server may exhibit Elasticsearch backpressure symptoms. - -This happens because a performance regression was introduced, such that the default value of bulk indexer flush bytes was reduced from 1MB to 24KB. - -Affected APM servers will emit the following log: - -```txt -flush_bytes config value is too small (0) and might be ignored by the indexer, increasing value to 24576 -``` - -To workaround the issue, modify the Elasticsearch output configuration in APM. - -* For APM Server binary - - * In `apm-server.yml`, set `output.elasticsearch.flush_bytes: 1mib` - -* For Fleet-managed APM (non-Elastic Cloud) - - * In Fleet, open the Settings tab. - * Under Outputs, identify the Elasticsearch output that receives from APM, select the edit icon. - * In the Edit output flyout, in "Advanced YAML configuration" field, add line `flush_bytes: 1mib`. - -* For Elastic Cloud - - * It is not possible to edit the Fleet "Elastic Cloud internal output". - - -A fix will be released in 8.14.3: [elastic/apm-server#13576](https://github.com/elastic/apm-server/pull/13576). diff --git a/troubleshoot/observability/apm/processing-performance.md b/troubleshoot/observability/apm/processing-performance.md index ea394174b6..cbd7c75c94 100644 --- a/troubleshoot/observability/apm/processing-performance.md +++ b/troubleshoot/observability/apm/processing-performance.md @@ -1,6 +1,8 @@ --- mapped_pages: - https://www.elastic.co/guide/en/observability/current/apm-processing-and-performance.html +applies_to: + stack: all --- # Processing and performance [apm-processing-and-performance] diff --git a/troubleshoot/observability/apm/what-happens-when-apm-server-elasticsearch-is-down.md b/troubleshoot/observability/apm/what-happens-when-apm-server-elasticsearch-is-down.md index e97b4ecc8b..43f97d3987 100644 --- a/troubleshoot/observability/apm/what-happens-when-apm-server-elasticsearch-is-down.md +++ b/troubleshoot/observability/apm/what-happens-when-apm-server-elasticsearch-is-down.md @@ -1,6 +1,8 @@ --- mapped_pages: - https://www.elastic.co/guide/en/observability/current/apm-server-es-down.html +applies_to: + stack: all --- # APM Server or Elasticsearch is down [apm-server-es-down] diff --git a/troubleshoot/observability/explore-data.md b/troubleshoot/observability/explore-data.md index 10b5a8ea80..67ce0cf775 100644 --- a/troubleshoot/observability/explore-data.md +++ b/troubleshoot/observability/explore-data.md @@ -1,6 +1,10 @@ --- mapped_pages: - https://www.elastic.co/guide/en/observability/current/exploratory-data-visualizations.html +applies_to: + stack: all + serverless: + observability: all --- # Explore data [exploratory-data-visualizations] diff --git a/troubleshoot/observability/inspect.md b/troubleshoot/observability/inspect.md index e606c0caeb..62498c045f 100644 --- a/troubleshoot/observability/inspect.md +++ b/troubleshoot/observability/inspect.md @@ -1,6 +1,10 @@ --- mapped_pages: - https://www.elastic.co/guide/en/observability/current/inspect.html +applies_to: + stack: all + serverless: + observability: all --- # Inspect [inspect] diff --git a/troubleshoot/observability/troubleshoot-logs.md b/troubleshoot/observability/troubleshoot-logs.md index 80569cd0ea..05d5ea27aa 100644 --- a/troubleshoot/observability/troubleshoot-logs.md +++ b/troubleshoot/observability/troubleshoot-logs.md @@ -3,6 +3,10 @@ navigation_title: Logs mapped_pages: - https://www.elastic.co/guide/en/observability/current/logs-troubleshooting.html - https://www.elastic.co/guide/en/serverless/current/observability-troubleshoot-logs.html +applies_to: + stack: all + serverless: + observability: all --- # Troubleshoot logs [logs-troubleshooting] diff --git a/troubleshoot/observability/troubleshoot-mapping-issues.md b/troubleshoot/observability/troubleshoot-mapping-issues.md index e51421c1dd..2a9053718a 100644 --- a/troubleshoot/observability/troubleshoot-mapping-issues.md +++ b/troubleshoot/observability/troubleshoot-mapping-issues.md @@ -1,10 +1,12 @@ --- -navigation_title: "Mapping issues" +navigation_title: "Uptime" mapped_pages: - https://www.elastic.co/guide/en/observability/current/troubleshoot-uptime-mapping-issues.html +applies_to: + stack: deprecated 8.15.0 --- -# Troubleshoot mapping issues [troubleshoot-uptime-mapping-issues] +# Troubleshoot Uptime mapping issues [troubleshoot-uptime-mapping-issues] ## Mapping issues [_mapping_issues] diff --git a/troubleshoot/observability/troubleshoot-service-level-objectives-slos.md b/troubleshoot/observability/troubleshoot-service-level-objectives-slos.md index bb97cd81f8..1b6538f201 100644 --- a/troubleshoot/observability/troubleshoot-service-level-objectives-slos.md +++ b/troubleshoot/observability/troubleshoot-service-level-objectives-slos.md @@ -3,6 +3,10 @@ navigation_title: "SLOs" mapped_pages: - https://www.elastic.co/guide/en/observability/current/slo-troubleshoot-slos.html - https://www.elastic.co/guide/en/serverless/current/slo-troubleshoot-slos.html +applies_to: + stack: all + serverless: + observability: all --- # Troubleshoot service-level objectives (SLOs) [slo-troubleshoot-slos] diff --git a/troubleshoot/observability/troubleshoot-your-universal-profiling-agent-deployment.md b/troubleshoot/observability/troubleshoot-your-universal-profiling-agent-deployment.md index 49c901a611..579b8ae3ca 100644 --- a/troubleshoot/observability/troubleshoot-your-universal-profiling-agent-deployment.md +++ b/troubleshoot/observability/troubleshoot-your-universal-profiling-agent-deployment.md @@ -2,6 +2,8 @@ navigation_title: "Universal Profiling" mapped_pages: - https://www.elastic.co/guide/en/observability/current/profiling-troubleshooting.html +applies_to: + stack: all --- diff --git a/troubleshoot/observability/troubleshoot-your-universal-profiling-agent-deployment/troubleshoot-universal-profiling-backend.md b/troubleshoot/observability/troubleshoot-your-universal-profiling-agent-deployment/troubleshoot-universal-profiling-backend.md index 8b8d9f5350..6f2bcb217e 100644 --- a/troubleshoot/observability/troubleshoot-your-universal-profiling-agent-deployment/troubleshoot-universal-profiling-backend.md +++ b/troubleshoot/observability/troubleshoot-your-universal-profiling-agent-deployment/troubleshoot-universal-profiling-backend.md @@ -2,6 +2,8 @@ navigation_title: "Troubleshoot the backend" mapped_pages: - https://www.elastic.co/guide/en/observability/current/profiling-self-managed-troubleshooting.html +applies_to: + stack: all --- diff --git a/troubleshoot/observability/troubleshooting-infrastructure-monitoring.md b/troubleshoot/observability/troubleshooting-infrastructure-monitoring.md index 46eb1a5a4e..1baeacb912 100644 --- a/troubleshoot/observability/troubleshooting-infrastructure-monitoring.md +++ b/troubleshoot/observability/troubleshooting-infrastructure-monitoring.md @@ -3,6 +3,10 @@ navigation_title: "Infrastructure monitoring" mapped_pages: - https://www.elastic.co/guide/en/observability/current/troubleshooting-infrastructure-monitoring.html - https://www.elastic.co/guide/en/serverless/current/observability-troubleshooting-infrastructure-monitoring.html +applies_to: + stack: all + serverless: + observability: all --- # Troubleshoot infrastructure monitoring [troubleshooting-infrastructure-monitoring] diff --git a/troubleshoot/observability/troubleshooting-infrastructure-monitoring/understanding-no-results-found-message.md b/troubleshoot/observability/troubleshooting-infrastructure-monitoring/understanding-no-results-found-message.md index deae1f8bfa..967bfa6463 100644 --- a/troubleshoot/observability/troubleshooting-infrastructure-monitoring/understanding-no-results-found-message.md +++ b/troubleshoot/observability/troubleshooting-infrastructure-monitoring/understanding-no-results-found-message.md @@ -2,6 +2,10 @@ mapped_pages: - https://www.elastic.co/guide/en/observability/current/handle-no-results-found-message.html - https://www.elastic.co/guide/en/serverless/current/observability-handle-no-results-found-message.html +applies_to: + stack: all + serverless: + observability: all --- # Understand "no results found" message [handle-no-results-found-message] diff --git a/troubleshoot/observability/troubleshooting-synthetics.md b/troubleshoot/observability/troubleshooting-synthetics.md index 0eb3186e3b..c6571209dc 100644 --- a/troubleshoot/observability/troubleshooting-synthetics.md +++ b/troubleshoot/observability/troubleshooting-synthetics.md @@ -3,6 +3,10 @@ navigation_title: "Synthetics" mapped_pages: - https://www.elastic.co/guide/en/observability/current/synthetics-troubleshooting.html - https://www.elastic.co/guide/en/serverless/current/observability-synthetics-troubleshooting.html +applies_to: + stack: all + serverless: + observability: all --- # Troubleshoot Synthetics [synthetics-troubleshooting] diff --git a/troubleshoot/toc.yml b/troubleshoot/toc.yml index 4233ecc435..98cc33e6ac 100644 --- a/troubleshoot/toc.yml +++ b/troubleshoot/toc.yml @@ -134,7 +134,6 @@ toc: - file: observability/apm-agent-python/apm-python-agent.md - file: observability/apm-agent-ruby/apm-ruby-agent.md - file: observability/apm-agent-rum-js/apm-real-user-monitoring-javascript-agent.md - - file: observability/apm/known-issues.md - file: observability/amazon-data-firehose.md - file: observability/troubleshoot-your-universal-profiling-agent-deployment.md children: