diff --git a/redirects.yml b/redirects.yml index 4ca85d4563..5ff6ecbaee 100644 --- a/redirects.yml +++ b/redirects.yml @@ -30,10 +30,16 @@ redirects: 'solutions/observability/apps/real-user-monitoring-rum.md': 'solutions/observability/apm/real-user-monitoring-rum.md' 'solutions/observability/apps/create-upload-source-maps-rum.md': 'solutions/observability/apm/create-upload-source-maps-rum.md' 'solutions/observability/apps/use-opentelemetry-with-apm.md': 'solutions/observability/apm/use-opentelemetry-with-apm.md' - 'solutions/observability/apps/upstream-opentelemetry-collectors-language-sdks.md': 'solutions/observability/apm/upstream-opentelemetry-collectors-language-sdks.md' - 'solutions/observability/apps/collect-metrics.md': 'solutions/observability/apm/collect-metrics.md' - 'solutions/observability/apps/limitations.md': 'solutions/observability/apm/limitations.md' - 'solutions/observability/apps/resource-atrributes.md': 'solutions/observability/apm/resource-attributes.md' + 'solutions/observability/apps/upstream-opentelemetry-collectors-language-sdks.md': 'solutions/observability/apm/use-opentelemetry-with-apm.md' + 'solutions/observability/apps/collect-metrics.md': 'solutions/observability/apm/use-opentelemetry-with-apm.md' + 'solutions/observability/apps/limitations.md': 'solutions/observability/apm/use-opentelemetry-with-apm.md' + 'solutions/observability/apps/resource-atrributes.md': 'solutions/observability/apm/use-opentelemetry-with-apm.md' + 'solutions/observability/apm/collect-metrics.md': 'solutions/observability/apm/use-opentelemetry-with-apm.md' + 'solutions/observability/apm/limitations.md': 'solutions/observability/apm/use-opentelemetry-with-apm.md' + 'solutions/observability/apm/resource-attributes.md': 'solutions/observability/apm/use-opentelemetry-with-apm.md' + 'solutions/observability/apm/upstream-opentelemetry-collectors-language-sdks.md': 'solutions/observability/apm/use-opentelemetry-with-apm.md' + 'solutions/observability/get-started/quickstart-monitor-hosts-with-opentelemetry.md': 'solutions/observability/get-started.md' + 'solutions/observability/get-started/quickstart-unified-kubernetes-observability-with-elastic-distributions-of-opentelemetry-edot.md': 'solutions/observability/get-started.md' 'solutions/observability/apps/apm-k8s-attacher.md': 'solutions/observability/apm/apm-k8s-attacher.md' 'solutions/observability/apps/monitoring-aws-lambda-functions.md': 'solutions/observability/apm/monitor-aws-lambda-functions.md' 'solutions/observability/apps/integrate-with-jaeger-deprecated.md': 'solutions/observability/apm/jaeger.md' diff --git a/solutions/observability/apm/collect-metrics.md b/solutions/observability/apm/collect-metrics.md deleted file mode 100644 index f4d0b70a58..0000000000 --- a/solutions/observability/apm/collect-metrics.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/observability/current/apm-open-telemetry-collect-metrics.html - - https://www.elastic.co/guide/en/serverless/current/observability-apm-agents-opentelemetry-collect-metrics.html -applies_to: - stack: - serverless: ---- - -# Collect metrics [apm-open-telemetry-collect-metrics] - -::::{important} -When collecting metrics, note that the [`DoubleValueRecorder`](https://www.javadoc.io/doc/io.opentelemetry/opentelemetry-api/latest/io/opentelemetry/api/metrics/DoubleValueRecorder.md) and [`LongValueRecorder`](https://www.javadoc.io/doc/io.opentelemetry/opentelemetry-api/latest/io/opentelemetry/api/metrics/LongValueObserver.md) metrics are not yet supported. -:::: - -Here’s an example of how to capture business metrics from a Java application. - -```java -// initialize metric -Meter meter = GlobalMetricsProvider.getMeter("my-frontend"); -DoubleCounter orderValueCounter = meter.doubleCounterBuilder("order_value").build(); - -public void createOrder(HttpServletRequest request) { - - // create order in the database - ... - // increment business metrics for monitoring - orderValueCounter.add(orderPrice); -} -``` - -See the [Open Telemetry Metrics API](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/api.md) for more information. - -## Verify OpenTelemetry metrics data [apm-open-telemetry-verify-metrics] - -Use **Discover** to validate that metrics are successfully reported to {{kib}}. - -1. Open your Observability instance. -2. Find **Discover** in the main menu or use the [global search field](/get-started/the-stack.md#kibana-navigation-search). -3. Click **All logs** → **Data Views** then select **APM**. -4. Filter the data to only show documents with metrics: `processor.name :"metric"` -5. Narrow your search with a known OpenTelemetry field. For example, if you have an `order_value` field, add `order_value: *` to your search to return only OpenTelemetry metrics documents. - -## Visualize your metrics[apm-open-telemetry-visualize] - -Use **Lens** to create visualizations for OpenTelemetry metrics. Lens enables you to build visualizations by dragging and dropping data fields. It makes smart visualization suggestions for your data, allowing you to switch between visualization types. - -To get started with a new Lens visualization: - -1. Go to **Visualizations**. -2. Click **Create new visualization**. -3. Select **Lens**. - -For more information on using Lens, refer to the [Lens documentation](/explore-analyze/visualize/lens.md). \ No newline at end of file diff --git a/solutions/observability/apm/get-started-apm-server-binary.md b/solutions/observability/apm/get-started-apm-server-binary.md index 9e57af02cb..1e70a5a967 100644 --- a/solutions/observability/apm/get-started-apm-server-binary.md +++ b/solutions/observability/apm/get-started-apm-server-binary.md @@ -732,9 +732,9 @@ const apm = initApm({ :::::: ::::::{tab-item} OpenTelemetry -Elastic integrates with OpenTelemetry, allowing you to reuse your existing instrumentation to easily send observability data to the {{stack}}. +Elastic integrates with OpenTelemetry using the Elastic Distribution of OpenTelemetry (EDOT), allowing you to reuse your existing instrumentation to easily send observability data to the {{stack}}. -For more information on how to combine Elastic and OpenTelemetry, see [OpenTelemetry integration](/solutions/observability/apm/use-opentelemetry-with-apm.md). +For more information on how to combine Elastic and OpenTelemetry, see the [EDOT documentation](https://elastic.github.io/opentelemetry/). :::::: ::::::: diff --git a/solutions/observability/apm/get-started-fleet-managed-apm-server.md b/solutions/observability/apm/get-started-fleet-managed-apm-server.md index 9bfca274c5..91b48cb7f0 100644 --- a/solutions/observability/apm/get-started-fleet-managed-apm-server.md +++ b/solutions/observability/apm/get-started-fleet-managed-apm-server.md @@ -773,9 +773,9 @@ const apm = initApm({ :::::: ::::::{tab-item} OpenTelemetry -Elastic integrates with OpenTelemetry, allowing you to reuse your existing instrumentation to easily send observability data to the {{stack}}. +Elastic integrates with OpenTelemetry using the Elastic Distribution of OpenTelemetry (EDOT), allowing you to reuse your existing instrumentation to easily send observability data to the {{stack}}. -For more information on how to combine Elastic and OpenTelemetry, see [OpenTelemetry integration](/solutions/observability/apm/use-opentelemetry-with-apm.md). +For more information on how to combine Elastic and OpenTelemetry, see the [EDOT documentation](https://elastic.github.io/opentelemetry/). :::::: ::::::: diff --git a/solutions/observability/apm/limitations.md b/solutions/observability/apm/limitations.md deleted file mode 100644 index 2de20e6265..0000000000 --- a/solutions/observability/apm/limitations.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/observability/current/apm-open-telemetry-known-limitations.html - - https://www.elastic.co/guide/en/serverless/current/observability-apm-agents-opentelemetry-limitations.html -applies_to: - stack: - serverless: ---- - -# Limitations [apm-open-telemetry-known-limitations] - -## OpenTelemetry traces [apm-open-telemetry-traces-limitations] - -* Traces of applications using `messaging` semantics might be wrongly displayed as `transactions` in the Applications UI, while they should be considered `spans` (see issue [#7001](https://github.com/elastic/apm-server/issues/7001)). -* Inability to see Stack traces in spans. -* Inability in APM views to view the "Time Spent by Span Type" (see issue [#5747](https://github.com/elastic/apm-server/issues/5747)). - -## OpenTelemetry logs [apm-open-telemetry-logs-intake] - -* [preview] The OpenTelemetry logs intake via Elastic is in technical preview. -* The application logs data stream (`app_logs`) has dynamic mapping disabled. This means the automatic detection and mapping of new fields is disabled (see issue [#9093](https://github.com/elastic/apm-server/issues/9093)). - -## OpenTelemetry Line Protocol (OTLP) [apm-open-telemetry-otlp-limitations] - -Elastic supports both the [OTLP/gRPC](https://opentelemetry.io/docs/specs/otlp/#otlpgrpc) and [OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/#otlphttp) protocol with ProtoBuf payload. Elastic does not yet support JSON Encoding for OTLP/HTTP. - -## OpenTelemetry Collector exporter for Elastic [apm-open-telemetry-collector-exporter] - -The [OpenTelemetry Collector exporter for Elastic](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/v0.57.2/exporter/elasticexporter) has been deprecated and replaced by the native support of the OpenTelemetry Line Protocol in Elastic Observability (OTLP). - -The [OpenTelemetry Collector exporter for Elastic](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/elasticsearchexporter) (which is different from the legacy exporter mentioned above) is not intended to be used with Elastic APM and Elastic Observability. Use [Elastic’s native OTLP support](/solutions/observability/apm/upstream-opentelemetry-collectors-language-sdks.md) instead. - -% Statefull only for tail-based sampling? - -## OpenTelemetry’s tail-based sampling [apm-open-telemetry-tbs] -```{applies_to} -stack: all -``` - -Tail-based sampling allows to make sampling decisions after all spans of a trace have been completed. This allows for more powerful and informed sampling rules. - -When using OpenTelemetry with Elastic APM, there are two different implementations available for tail-based sampling: - -* Tail-based sampling using the [tailsamplingprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor) in the OpenTelemetry Collector -* Native [tail-based sampling in the Elastic APM backend](/solutions/observability/apm/transaction-sampling.md#apm-tail-based-sampling) - -Using the [tailsamplingprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor) in the OpenTelemetry Collector comes with an important limitation. Elastic’s APM backend calculates span and transaction metrics based on the incoming span events. These metrics are accurate for 100% sampling scenarios. In scenarios with probabilistic sampling, Elastic’s APM backend is being informed about the sampling rate of spans and can extrapolate throughput metrics based on the incoming, partial data. However, with tail-based sampling there’s no clear probability for sampling decisions as the rules can be more complex and the OpenTelemetry Collector does not provide sampling probability information to the Elastic backend that could be used for extrapolation of data. Therefore, there’s no way for Elastic APM to properly extrapolate throughput and count metrics that are derived from span events that have been tail-based sampled in the OpenTelemetry Collector. In these scenarios, derived throughput and count metrics are likely to be inaccurate. - -Therefore, we recommend using Elastic’s native tail-based sampling when integrating with OpenTelemetry. \ No newline at end of file diff --git a/solutions/observability/apm/managed-intake-service-event-api.md b/solutions/observability/apm/managed-intake-service-event-api.md index 0ed6e85d53..5285df1523 100644 --- a/solutions/observability/apm/managed-intake-service-event-api.md +++ b/solutions/observability/apm/managed-intake-service-event-api.md @@ -4476,7 +4476,5 @@ The managed intake service supports two OTLP communication protocols on the same | OTLP logs intake | `/v1/logs` | ::::{tip} -See our [OpenTelemetry docs](/solutions/observability/apm/upstream-opentelemetry-collectors-language-sdks.md) to learn how to send data to the managed intake service from an OpenTelemetry agent OpenTelemetry collector. - +See our [OpenTelemetry documentation](https://elastic.github.io/opentelemetry/) to learn how to send data to the managed intake service. :::: - diff --git a/solutions/observability/apm/opentelemetry-intake-api.md b/solutions/observability/apm/opentelemetry-intake-api.md index 0066a923c3..73439088c9 100644 --- a/solutions/observability/apm/opentelemetry-intake-api.md +++ b/solutions/observability/apm/opentelemetry-intake-api.md @@ -1,8 +1,6 @@ --- mapped_pages: - https://www.elastic.co/guide/en/observability/current/apm-api-otlp.html -applies_to: - stack: --- # OpenTelemetry intake API [apm-api-otlp] @@ -31,6 +29,6 @@ APM Server supports two OTLP communication protocols on the same port: | OTLP logs intake | `/v1/logs` | ::::{tip} -See our OpenTelemetry documentation to learn how to send data to the APM Server from an [OpenTelemetry agent](/solutions/observability/apm/upstream-opentelemetry-collectors-language-sdks.md#apm-instrument-apps-otel) or [OpenTelemetry collector](/solutions/observability/apm/upstream-opentelemetry-collectors-language-sdks.md#apm-connect-open-telemetry-collector). +See our [OpenTelemetry documentation](https://elastic.github.io/opentelemetry/) to learn how to send data to the APM Server. :::: diff --git a/solutions/observability/apm/resource-attributes.md b/solutions/observability/apm/resource-attributes.md deleted file mode 100644 index 3aa493419b..0000000000 --- a/solutions/observability/apm/resource-attributes.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/observability/current/apm-open-telemetry-resource-attributes.html - - https://www.elastic.co/guide/en/serverless/current/observability-apm-agents-opentelemetry-resource-attributes.html -applies_to: - stack: - serverless: ---- - -# Resource attributes [apm-open-telemetry-resource-attributes] - -A resource attribute is a key/value pair containing information about the entity producing telemetry. Resource attributes are mapped to Elastic Common Schema (ECS) fields like `service.*`, `cloud.*`, `process.*`, etc. These fields describe the service and the environment that the service runs in. - -The examples shown here set the Elastic (ECS) `service.environment` field for the resource, i.e. service, that is producing trace events. Note that Elastic maps the OpenTelemetry `deployment.environment` field to the ECS `service.environment` field on ingestion. - -**OpenTelemetry agent** - -Use the `OTEL_RESOURCE_ATTRIBUTES` environment variable to pass resource attributes at process invocation. - -```bash -export OTEL_RESOURCE_ATTRIBUTES=deployment.environment=production -``` - -**OpenTelemetry collector** - -Use the [resource processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourceprocessor) to set or apply changes to resource attributes. - -```yaml -... -processors: - resource: - attributes: - - key: deployment.environment - action: insert - value: production -... -``` - -::::{tip} -Need to add event attributes instead? Use attributes—​not to be confused with resource attributes—​to add data to span, log, or metric events. Attributes can be added as a part of the OpenTelemetry instrumentation process or with the [attributes processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/attributesprocessor). - -:::: - -% Stateful only after this? - -Elastic integrates with OpenTelemetry, allowing you to reuse your existing instrumentation to easily send observability data to the {{stack}}. - -For more information on how to combine Elastic and OpenTelemetry, see [OpenTelemetry integration](/solutions/observability/apm/use-opentelemetry-with-apm.md). \ No newline at end of file diff --git a/solutions/observability/apm/traces.md b/solutions/observability/apm/traces.md index 516c7ac0f5..65656a832f 100644 --- a/solutions/observability/apm/traces.md +++ b/solutions/observability/apm/traces.md @@ -37,12 +37,6 @@ In this example, Elastic’s Ruby agent communicates with Elastic’s Java agent :alt: How traceparent propagation works ::: -In this example, Elastic’s Ruby agent communicates with OpenTelemetry’s Java agent. Both support the `traceparent` header, and trace data is successfully propagated. - -:::{image} /solutions/images/observability-dt-trace-ex2.png -:alt: How traceparent propagation works -::: - In this example, the trace meets a piece of middleware that doesn’t propagate the `traceparent` header. The distributed trace ends and any further communication will result in a new trace. :::{image} /solutions/images/observability-dt-trace-ex3.png diff --git a/solutions/observability/apm/transaction-sampling.md b/solutions/observability/apm/transaction-sampling.md index 4fa1d929ea..0d93855703 100644 --- a/solutions/observability/apm/transaction-sampling.md +++ b/solutions/observability/apm/transaction-sampling.md @@ -84,23 +84,6 @@ In the example in *Figure 4*, `Service A` and `Service B` are Elastic-monitored :title: Using the `restart` trace continuation strategy ::: -### OpenTelemetry [_opentelemetry] - -Head-based sampling is implemented directly in the APM agents and SDKs. The sample rate must be propagated between services and the managed intake service in order to produce accurate metrics. - -OpenTelemetry offers multiple samplers. However, most samplers do not propagate the sample rate. This results in inaccurate span-based metrics, like APM throughput, latency, and error metrics. - -For accurate span-based metrics when using head-based sampling with OpenTelemetry, you must use a [consistent probability sampler](https://opentelemetry.io/docs/specs/otel/trace/tracestate-probability-sampling/). These samplers propagate the sample rate between services and the managed intake service, resulting in accurate metrics. - -::::{note} -OpenTelemetry does not offer consistent probability samplers in all languages. OpenTelemetry users should consider using tail-based sampling instead. - -Refer to the documentation of your favorite OpenTelemetry agent or SDK for more information on the availability of consistent probability samplers. - -:::: - -% Stateful only for tail-based sampling - ## Tail-based sampling [apm-tail-based-sampling] ```{applies_to} @@ -133,12 +116,6 @@ In this example, `Service A` initiates four transactions. If our sample rate is :alt: Distributed tracing and tail based sampling example one ::: -### OpenTelemetry with tail-based sampling [_opentelemetry_with_tail_based_sampling] - -Tail-based sampling is implemented entirely in APM Server, and will work with traces sent by either Elastic APM agents or OpenTelemetry SDKs. - -Due to [OpenTelemetry tail-based sampling limitations](/solutions/observability/apm/limitations.md#apm-open-telemetry-tbs) when using [tailsamplingprocessor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor), we recommend using APM Server tail-based sampling instead. - ### Tail-based sampling performance and requirements [_tail_based_sampling_performance_and_requirements] Tail-based sampling (TBS), by definition, requires storing events locally temporarily, such that they can be retrieved and forwarded when a sampling decision is made. diff --git a/solutions/observability/apm/upstream-opentelemetry-collectors-language-sdks.md b/solutions/observability/apm/upstream-opentelemetry-collectors-language-sdks.md deleted file mode 100644 index 065fec81d2..0000000000 --- a/solutions/observability/apm/upstream-opentelemetry-collectors-language-sdks.md +++ /dev/null @@ -1,269 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/observability/current/apm-open-telemetry-direct.html - - https://www.elastic.co/guide/en/serverless/current/observability-apm-agents-opentelemetry-opentelemetry-native-support.html -applies_to: - stack: - serverless: ---- - -# Upstream OpenTelemetry Collectors and language SDKs [apm-open-telemetry-direct] - -:::{include} _snippets/apm-server-vs-mis.md -::: - -::::{note} -This is one of several approaches you can use to integrate Elastic with OpenTelemetry. **To compare approaches and choose the best approach for your use case, refer to [OpenTelemetry](/solutions/observability/apm/use-opentelemetry-with-apm.md).** -:::: - -The {{stack}} natively supports the OpenTelemetry protocol (OTLP). This means trace data and metrics collected from your applications and infrastructure can be sent directly to the {{stack}}. - -* Send data to Elastic from an upstream [OpenTelemetry Collector](/solutions/observability/apm/upstream-opentelemetry-collectors-language-sdks.md#apm-connect-open-telemetry-collector) -* Send data to Elastic from an upstream [OpenTelemetry language SDK](/solutions/observability/apm/upstream-opentelemetry-collectors-language-sdks.md#apm-instrument-apps-otel) - -## Send data from an upstream OpenTelemetry Collector [apm-connect-open-telemetry-collector] - -Connect your OpenTelemetry Collector instances to Elastic {{observability}} or {{obs-serverless}} using the OTLP exporter: - -::::{tab-set} -:group: stack-serverless - -:::{tab-item} Elastic Stack -:sync: stack - -```yaml -receivers: <1> - # ... - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 -processors: <2> - # ... - memory_limiter: - check_interval: 1s - limit_mib: 2000 - batch: - -exporters: - debug: - verbosity: detailed <3> - otlp: <4> - # Elastic APM server https endpoint without the "https://" prefix - endpoint: "${env:ELASTIC_APM_SERVER_ENDPOINT}" <5> <7> - headers: - # Elastic APM Server secret token - Authorization: "Bearer ${env:ELASTIC_APM_SECRET_TOKEN}" <6> <7> - -service: - pipelines: - traces: - receivers: [otlp] - processors: [..., memory_limiter, batch] - exporters: [debug, otlp] - metrics: - receivers: [otlp] - processors: [..., memory_limiter, batch] - exporters: [debug, otlp] - logs: <8> - receivers: [otlp] - processors: [..., memory_limiter, batch] - exporters: [debug, otlp] -``` - -1. The receivers, like the [OTLP receiver](https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver/otlpreceiver), that forward data emitted by APM agents, or the [host metrics receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver). -2. We recommend using the [Batch processor](https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/batchprocessor/README.md) and the [memory limiter processor](https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/memorylimiterprocessor/README.md). For more information, see [recommended processors](https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/README.md#recommended-processors). -3. The [debug exporter](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/debugexporter) is helpful for troubleshooting, and supports configurable verbosity levels: `basic` (default), `normal`, and `detailed`. -4. Elastic {{observability}} endpoint configuration. APM Server supports a ProtoBuf payload via both the OTLP protocol over gRPC transport [(OTLP/gRPC)](https://opentelemetry.io/docs/specs/otlp/#otlpgrpc) and the OTLP protocol over HTTP transport [(OTLP/HTTP)](https://opentelemetry.io/docs/specs/otlp/#otlphttp). To learn more about these exporters, see the OpenTelemetry Collector documentation: [OTLP/HTTP Exporter](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter) or [OTLP/gRPC exporter](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlpexporter). When adding an endpoint to an existing configuration an optional name component can be added, like `otlp/elastic`, to distinguish endpoints as described in the [OpenTelemetry Collector Configuration Basics](https://opentelemetry.io/docs/collector/configuration/#basics). -5. Hostname and port of the APM Server endpoint. For example, `elastic-apm-server:8200`. -6. Credential for Elastic APM [secret token authorization](/solutions/observability/apm/secret-token.md) (`Authorization: "Bearer a_secret_token"`) or [API key authorization](/solutions/observability/apm/api-keys.md) (`Authorization: "ApiKey an_api_key"`). -7. Environment-specific configuration parameters can be conveniently passed in as environment variables documented [here](https://opentelemetry.io/docs/collector/configuration/#environment-variables) (e.g. `ELASTIC_APM_SERVER_ENDPOINT` and `ELASTIC_APM_SECRET_TOKEN`). -8. [preview] To send OpenTelemetry logs to {{stack}} version 8.0+, declare a `logs` pipeline. - -::: - -:::{tab-item} Serverless -:sync: serverless - -```yaml -receivers: <1> - # ... - otlp: - -processors: <2> - # ... - memory_limiter: - check_interval: 1s - limit_mib: 2000 - batch: - -exporters: - logging: - loglevel: warn <3> - otlp/elastic: <4> - # Elastic https endpoint without the "https://" prefix - endpoint: "${ELASTIC_APM_SERVER_ENDPOINT}" <5> <7> - headers: - # Elastic API key - Authorization: "ApiKey ${ELASTIC_APM_API_KEY}" <6> <7> - -service: - pipelines: - traces: - receivers: [otlp] - processors: [..., memory_limiter, batch] - exporters: [logging, otlp/elastic] - metrics: - receivers: [otlp] - processors: [..., memory_limiter, batch] - exporters: [logging, otlp/elastic] - logs: <8> - receivers: [otlp] - processors: [..., memory_limiter, batch] - exporters: [logging, otlp/elastic] -``` - -1. The receivers, like the [OTLP receiver](https://github.com/open-telemetry/opentelemetry-collector/tree/main/receiver/otlpreceiver), that forward data emitted by APM agents, or the [host metrics receiver](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/hostmetricsreceiver). -2. We recommend using the [Batch processor](https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/batchprocessor/README.md) and the [memory limiter processor](https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/memorylimiterprocessor/README.md). For more information, see [recommended processors](https://github.com/open-telemetry/opentelemetry-collector/blob/main/processor/README.md#recommended-processors). -3. The [logging exporter](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/loggingexporter) is helpful for troubleshooting and supports various logging levels, like `debug`, `info`, `warn`, and `error`. -4. {{obs-serverless}} endpoint configuration. Elastic supports a ProtoBuf payload via both the OTLP protocol over gRPC transport [(OTLP/gRPC)](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlpgrpc) and the OTLP protocol over HTTP transport [(OTLP/HTTP)](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlphttp). To learn more about these exporters, see the OpenTelemetry Collector documentation: [OTLP/HTTP Exporter](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter) or [OTLP/gRPC exporter](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlpexporter). -5. Hostname and port of the Elastic endpoint. For example, `elastic-apm-server:8200`. -6. Credential for Elastic APM API key authorization (`Authorization: "ApiKey an_api_key"`). -7. Environment-specific configuration parameters can be conveniently passed in as environment variables documented [here](https://opentelemetry.io/docs/collector/configuration/#configuration-environment-variables) (e.g. `ELASTIC_APM_SERVER_ENDPOINT` and `ELASTIC_APM_API_KEY`). -8. [preview] To send OpenTelemetry logs to your project, declare a `logs` pipeline. - -::: - -:::: - -You’re now ready to export traces and metrics from your services and applications. - -::::{tip} -When using the OpenTelemetry Collector, you should always prefer sending data via the [`OTLP` exporter](https://github.com/open-telemetry/opentelemetry-collector/tree/main/exporter/otlphttpexporter). Using other methods, like the [`elasticsearch` exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/elasticsearchexporter), will bypass all of the validation and data processing that Elastic performs. In addition, your data will not be viewable in your Observability project if you use the `elasticsearch` exporter. -:::: - -## Send data from an upstream OpenTelemetry SDK [apm-instrument-apps-otel] - -::::{note} -This document outlines how to send data directly from an upstream OpenTelemetry SDK to Elastic, which is appropriate when getting started. However, in many cases you should use the OpenTelemetry SDK to send data to an OpenTelemetry Collector that processes and exports data to Elastic. Read more about when and how to use a collector in the [OpenTelemetry documentation](https://opentelemetry.io/docs/collector/#when-to-use-a-collector). - -:::: - -To export traces and metrics to Elastic, instrument your services and applications with the OpenTelemetry API, SDK, or both. For example, if you are a Java developer, you need to instrument your Java app with the [OpenTelemetry agent for Java](https://github.com/open-telemetry/opentelemetry-java-instrumentation). See the [OpenTelemetry Instrumentation guides](https://opentelemetry.io/docs/instrumentation/) to download the OpenTelemetry agent or SDK for your language. - -Define environment variables to configure the OpenTelemetry agent or SDK and enable communication with Elastic APM. For example, if you are instrumenting a Java app, define the following environment variables: - -::::{tab-set} -:group: stack-serverless - -:::{tab-item} Elastic Stack -:sync: stack - -```bash -export OTEL_RESOURCE_ATTRIBUTES=service.name=checkoutService,service.version=1.1,deployment.environment=production -export OTEL_EXPORTER_OTLP_ENDPOINT=https://apm_server_url:8200 -export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer an_apm_secret_token" -export OTEL_METRICS_EXPORTER="otlp" \ -export OTEL_LOGS_EXPORTER="otlp" \ <1> -java -javaagent:/path/to/opentelemetry-javaagent-all.jar \ - -classpath lib/*:classes/ \ - com.mycompany.checkout.CheckoutServiceServer -``` - -1. [preview] The OpenTelemetry logs intake via APM Server is currently in technical preview. - -`OTEL_RESOURCE_ATTRIBUTES` -: Fields that describe the service and the environment that the service runs in. See [resource attributes](/solutions/observability/apm/resource-attributes.md) for more information. - -`OTEL_EXPORTER_OTLP_ENDPOINT` -: APM Server URL. The host and port that APM Server listens for events on. - -`OTEL_EXPORTER_OTLP_HEADERS` -: Authorization header that includes the Elastic APM Secret token or API key: `"Authorization=Bearer an_apm_secret_token"` or `"Authorization=ApiKey an_api_key"`. - - For information on how to format an API key, see [API keys](/solutions/observability/apm/api-keys.md). - - Note the required space between `Bearer` and `an_apm_secret_token`, and `ApiKey` and `an_api_key`. - - ::::{note} - If you are using a version of the Python OpenTelemetry agent *before* 1.27.0, the content of the header *must* be URL-encoded. You can use the Python standard library’s `urllib.parse.quote` function to encode the content of the header. - :::: - -`OTEL_METRICS_EXPORTER` -: Metrics exporter to use. See [exporter selection](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#exporter-selection) for more information. - -`OTEL_LOGS_EXPORTER` -: Logs exporter to use. See [exporter selection](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#exporter-selection) for more information. - -::: - -:::{tab-item} Serverless -:sync: serverless - -```bash -export OTEL_RESOURCE_ATTRIBUTES=service.name=checkoutService,service.version=1.1,deployment.environment=production -export OTEL_EXPORTER_OTLP_ENDPOINT=https://apm_server_url:8200 -export OTEL_EXPORTER_OTLP_HEADERS="Authorization=ApiKey an_apm_api_key" -export OTEL_METRICS_EXPORTER="otlp" \ -export OTEL_LOGS_EXPORTER="otlp" \ <1> -java -javaagent:/path/to/opentelemetry-javaagent-all.jar \ - -classpath lib/*:classes/ \ - com.mycompany.checkout.CheckoutServiceServer -``` - -1. [preview] The OpenTelemetry logs intake via Elastic is currently in technical preview. - -`OTEL_RESOURCE_ATTRIBUTES` -: Fields that describe the service and the environment that the service runs in. See [resource attributes](/solutions/observability/apm/resource-attributes.md) for more information. - -`OTEL_EXPORTER_OTLP_ENDPOINT` -: Elastic URL. The host and port that Elastic listens for APM events on. - -`OTEL_EXPORTER_OTLP_HEADERS` -: Authorization header that includes the Elastic APM API key: `"Authorization=ApiKey an_api_key"`. Note the required space between `ApiKey` and `an_api_key`. - - For information on how to format an API key, refer to [Secure communication with APM agents](/solutions/observability/apm/use-apm-securely.md). - - ::::{note} - If you are using a version of the Python OpenTelemetry agent *before* 1.27.0, the content of the header *must* be URL-encoded. You can use the Python standard library’s `urllib.parse.quote` function to encode the content of the header. - - :::: - -`OTEL_METRICS_EXPORTER` -: Metrics exporter to use. See [exporter selection](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#exporter-selection) for more information. - -`OTEL_LOGS_EXPORTER` -: Logs exporter to use. See [exporter selection](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/#exporter-selection) for more information. - -::: - -:::: - -You are now ready to collect traces and [metrics](/solutions/observability/apm/collect-metrics.md) before [verifying metrics](/solutions/observability/apm/collect-metrics.md#apm-open-telemetry-verify-metrics) and [visualizing metrics](/solutions/observability/apm/collect-metrics.md#apm-open-telemetry-visualize). - -## Proxy requests to APM Server [apm-open-telemetry-proxy-apm] - -APM Server supports both the [OTLP/gRPC](https://opentelemetry.io/docs/specs/otlp/#otlpgrpc) and [OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/#otlphttp) protocol on the same port as Elastic APM agent requests. For ease of setup, we recommend using OTLP/HTTP when proxying or load balancing requests to Elastic. - -If you use the OTLP/gRPC protocol, requests to Elastic must use either HTTP/2 over TLS or HTTP/2 Cleartext (H2C). No matter which protocol is used, OTLP/gRPC requests will have the header: `"Content-Type: application/grpc"`. - -When using a layer 7 (L7) proxy like AWS ALB, requests must be proxied in a way that ensures requests to Elastic follow the rules outlined above. For example, with ALB you can create rules to select an alternative backend protocol based on the headers of requests coming into ALB. In this example, you’d select the gRPC protocol when the `"Content-Type: application/grpc"` header exists on a request. - -Many L7 load balancers handle HTTP and gRPC traffic separately and rely on explicitly defined routes and service configurations to correctly proxy requests. Since APM Server serves both protocols on the same port, it may not be compatible with some L7 load balancers. For example, to work around this issue in [Ingress NGINX Controller for Kubernetes](https://github.com/kubernetes/ingress-nginx), either: - -* Use the `otlp` exporter in the OTel collector. Set annotation `nginx.ingress.kubernetes.io/backend-protocol: "GRPC"` on the K8s Ingress object proxying to APM Server. -* Use the `otlphttp` exporter in the OTel collector. Set annotation `nginx.ingress.kubernetes.io/backend-protocol: "HTTP"` (or `"HTTPS"` if APM Server expects TLS) on the K8s Ingress object proxying to APM Server. - -The preferred approach is to deploy a L4 (TCP) load balancer (e.g. [NLB](https://docs.aws.amazon.com/elasticloadbalancing/latest/network/introduction.html) on AWS) in front of APM Server, which forwards raw TCP traffic transparently without protocol inspection. - -For more information on how to configure an AWS ALB to support gRPC, see this AWS blog post: [Application Load Balancer Support for End-to-End HTTP/2 and gRPC](https://aws.amazon.com/blogs/aws/new-application-load-balancer-support-for-end-to-end-http-2-and-grpc/). - -For more information on how APM Server services gRPC requests, see [Muxing gRPC and HTTP/1.1](https://github.com/elastic/apm-server/blob/main/dev_docs/otel.md#muxing-grpc-and-http11). - -## Next steps [apm-open-telemetry-direct-next] - -* [Collect metrics](/solutions/observability/apm/collect-metrics.md) -* Add [Resource attributes](/solutions/observability/apm/resource-attributes.md) -* Learn about the [limitations of this integration](/solutions/observability/apm/limitations.md) diff --git a/solutions/observability/apm/use-opentelemetry-with-apm.md b/solutions/observability/apm/use-opentelemetry-with-apm.md index e5b882c0d4..44f554cbd6 100644 --- a/solutions/observability/apm/use-opentelemetry-with-apm.md +++ b/solutions/observability/apm/use-opentelemetry-with-apm.md @@ -19,12 +19,10 @@ For a complete overview of using OpenTelemetry with Elastic, explore [**Elastic [OpenTelemetry](https://opentelemetry.io/docs/concepts/what-is-opentelemetry/) is a set of APIs, SDKs, tooling, and integrations that enable the capture and management of telemetry data from your services and applications. -Elastic integrates with OpenTelemetry, allowing you to reuse your existing instrumentation to easily send observability data to the {{stack}}. There are several ways to integrate OpenTelemetry with the {{stack}}: +Elastic integrates with OpenTelemetry, allowing you to reuse your existing instrumentation to easily send observability data to the {{stack}}. You can integrate OpenTelemetry with the {{stack}} the following ways: -* [Elastic Distributions of OpenTelemetry language SDKs](/solutions/observability/apm/use-opentelemetry-with-apm.md#apm-otel-elastic-distros) -* [Upstream OpenTelemetry API/SDK + Elastic APM agent](/solutions/observability/apm/use-opentelemetry-with-apm.md#apm-otel-api-sdk-elastic-agent) -* [Upstream OpenTelemetry Collector and language SDKs](/solutions/observability/apm/use-opentelemetry-with-apm.md#apm-otel-upstream) -* [AWS Lambda collector exporter](/solutions/observability/apm/use-opentelemetry-with-apm.md#apm-otel-lambda) +* [Elastic Distributions of OpenTelemetry language SDKs](#apm-otel-elastic-distros) +* [AWS Lambda collector exporter](#apm-otel-lambda) ## Elastic Distributions of OpenTelemetry language SDKs [apm-otel-elastic-distros] @@ -46,65 +44,17 @@ With an Elastic Distribution of OpenTelemetry language SDK you have access to al Get started with an Elastic Distribution of OpenTelemetry language SDK: -* [**Elastic Distribution of OpenTelemetry Java →**](https://github.com/elastic/elastic-otel-java) -* [preview] [**Elastic Distribution of OpenTelemetry .NET →**](https://github.com/elastic/elastic-otel-dotnet) -* [preview] [**Elastic Distribution of OpenTelemetry Node.js →**](https://github.com/elastic/elastic-otel-node) -* [preview] [**Elastic Distribution of OpenTelemetry Python →**](https://github.com/elastic/elastic-otel-python) -* [preview] [**Elastic Distribution of OpenTelemetry PHP →**](https://github.com/elastic/elastic-otel-php) +* [**Elastic Distribution of OpenTelemetry Java →**](https://elastic.github.io/opentelemetry/edot-sdks/java/index.html) +* [preview] [**Elastic Distribution of OpenTelemetry .NET →**](https://elastic.github.io/opentelemetry/edot-sdks/dotnet/index.html) +* [preview] [**Elastic Distribution of OpenTelemetry Node.js →**](https://elastic.github.io/opentelemetry/edot-sdks/nodejs/index.html) +* [preview] [**Elastic Distribution of OpenTelemetry Python →**](https://elastic.github.io/opentelemetry/edot-sdks/python/index.html) +* [preview] [**Elastic Distribution of OpenTelemetry PHP →**](https://elastic.github.io/opentelemetry/edot-sdks/php/index.html) ::::{note} For more details about OpenTelemetry distributions in general, visit the [OpenTelemetry documentation](https://opentelemetry.io/docs/concepts/distributions). :::: -## Upstream OpenTelemetry API/SDK + Elastic APM agent [apm-otel-api-sdk-elastic-agent] - -Use the OpenTelemetry API/SDKs with [Elastic APM agents](/solutions/observability/apm/get-started-fleet-managed-apm-server.md#_step_3_install_apm_agents) to translate OpenTelemetry API calls to Elastic APM API calls. - -:::{image} /solutions/images/observability-apm-otel-api-sdk-elastic-agent.png -:alt: apm otel api sdk elastic agent -:screenshot: -::: - -This allows you to reuse your existing OpenTelemetry instrumentation to create Elastic APM transactions and spans — ​avoiding vendor lock-in and having to redo manual instrumentation. - -However, not all features of the OpenTelemetry API are supported when using this approach, and not all Elastic APM agents support this approach. - -Find more details about how to use an OpenTelemetry API or SDK with an Elastic APM agent and which OpenTelemetry API features are supported in the APM agent documentation: - -* [**APM Java agent →**](apm-agent-java://reference/opentelemetry-bridge.md) -* [**APM .NET agent →**](apm-agent-dotnet://reference/opentelemetry-bridge.md) -* [**APM Node.js agent →**](apm-agent-nodejs://reference/opentelemetry-bridge.md) -* [**APM Python agent →**](apm-agent-python://reference/opentelemetry-api-bridge.md) - -## Upstream OpenTelemetry Collector and language SDKs [apm-otel-upstream] - -The {{stack}} natively supports the OpenTelemetry protocol (OTLP). This means trace data and metrics collected from your applications and infrastructure by an OpenTelemetry Collector or OpenTelemetry language SDK can be sent to the {{stack}}. - -You can set up an [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/), instrument your application with an [OpenTelemetry language SDK](https://opentelemetry.io/docs/languages/) that sends data to the collector, and use the collector to process and export the data to either {{apm-server-or-mis}}. - -:::{image} /solutions/images/observability-apm-otel-api-sdk-collector.png -:alt: apm otel api sdk collector -:screenshot: -::: - -::::{note} -It’s also possible to send data directly to either {{apm-server-or-mis}} from an upstream OpenTelemetry SDK. You might do this during development or if you’re monitoring a small-scale application. Read more about when to use a collector in the [OpenTelemetry documentation](https://opentelemetry.io/docs/collector/#when-to-use-a-collector). - -:::: - -This approach works well when you need to instrument a technology that Elastic doesn’t provide a solution for. For example, if you want to instrument C or C++ you could use the [OpenTelemetry C++ client](https://github.com/open-telemetry/opentelemetry-cpp). - -However, there are some limitations when using collectors and language SDKs built and maintained by OpenTelemetry, including: - -* Elastic can’t provide implementation support on how to use upstream OpenTelemetry tools. -* You won’t have access to Elastic enterprise APM features. -* You may experience problems with performance efficiency. - -For more on the limitations associated with using upstream OpenTelemetry tools, refer to [Limitations](/solutions/observability/apm/limitations.md). - -[**Get started with upstream OpenTelemetry Collectors and language SDKs →**](/solutions/observability/apm/upstream-opentelemetry-collectors-language-sdks.md) - ## AWS Lambda collector exporter [apm-otel-lambda] AWS Lambda functions can be instrumented with OpenTelemetry and monitored with Elastic {{observability}} or {{obs-serverless}}. diff --git a/solutions/observability/get-started.md b/solutions/observability/get-started.md index f3082669a9..a55a261c2f 100644 --- a/solutions/observability/get-started.md +++ b/solutions/observability/get-started.md @@ -52,8 +52,8 @@ Follow the steps in these guides to get started quickly: * [Quickstart: Monitor hosts with {{agent}}](/solutions/observability/get-started/quickstart-monitor-hosts-with-elastic-agent.md) * [Quickstart: Monitor your Kubernetes cluster with {{agent}}](/solutions/observability/get-started/quickstart-monitor-kubernetes-cluster-with-elastic-agent.md) -* [Quickstart: Monitor hosts with OpenTelemetry](/solutions/observability/get-started/quickstart-monitor-hosts-with-opentelemetry.md) -* [Quickstart: Unified Kubernetes Observability with Elastic Distributions of OpenTelemetry (EDOT)](/solutions/observability/get-started/quickstart-unified-kubernetes-observability-with-elastic-distributions-of-opentelemetry-edot.md) +* [Quickstart: Monitor hosts with OpenTelemetry](https://elastic.github.io/opentelemetry/quickstart//) +* [Quickstart: Unified Kubernetes Observability with Elastic Distributions of OpenTelemetry (EDOT)](https://elastic.github.io/opentelemetry/use-cases/kubernetes/) * [Quickstart: Collect data with AWS Firehose](/solutions/observability/get-started/quickstart-collect-data-with-aws-firehose.md) diff --git a/solutions/observability/get-started/quickstart-monitor-hosts-with-opentelemetry.md b/solutions/observability/get-started/quickstart-monitor-hosts-with-opentelemetry.md deleted file mode 100644 index b7543cc55b..0000000000 --- a/solutions/observability/get-started/quickstart-monitor-hosts-with-opentelemetry.md +++ /dev/null @@ -1,146 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/observability/current/quickstart-monitor-hosts-with-otel.html - - https://www.elastic.co/guide/en/serverless/current/quickstart-monitor-hosts-with-otel.html -applies_to: - stack: - serverless: ---- - -# Quickstart: Monitor hosts with OpenTelemetry [quickstart-monitor-hosts-with-otel] - -::::{warning} -This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. -:::: - - -In this quickstart guide, you’ll learn how to monitor your hosts using the Elastic Distribution of OpenTelemetry (EDOT) Collector. You’ll also learn how to use {{observability}} features to gain deeper insight into your observability data after collecting it. - - -## Prerequisites [_prerequisites] - -::::{tab-set} -:group: stack-serverless - -:::{tab-item} Elastic Stack -:sync: stack - -* An {{es}} cluster for storing and searching your data, and {{kib}} for visualizing and managing your data. This quickstart is available for all Elastic deployment models. The quickest way to get started with this quickstart is using a trial project on [Elastic serverless](https://docs.elastic.co/serverless/quickstart-monitor-hosts-with-otel.html). -* This quickstart is only available for Linux and MacOS systems. -* A user with the **Admin** role or higher—required to onboard system logs and metrics. To learn more, refer to [User roles and privileges](/deploy-manage/users-roles/cloud-organization/user-roles.md). -* Root privileges on the host—required to run the OpenTelemetry collector because of these components: - - * `hostmetrics` receiver to read all system metrics (all processes, memory, etc.). - * `filelog` to allow the collector to read any user or application log files. - -::: - -:::{tab-item} Serverless -:sync: serverless - -* An {{observability}} project. To learn more, refer to [Create an Observability project](/solutions/observability/get-started/create-an-observability-project.md). -* This quickstart is only available for Linux and MacOS systems. -* A user with the **Admin** role or higher—required to onboard system logs and metrics. To learn more, refer to [Assign user roles and privileges](/deploy-manage/users-roles/cloud-organization/user-roles.md#general-assign-user-roles). -* Root privileges on the host—required to run the OpenTelemetry collector because of these components: - - * `hostmetrics` receiver to read all system metrics (all processes, memory, etc.). - * `filelog` to allow the collector to read any user or application log files. - -::: - -:::: - - -## Limitations [_limitations] - -Refer to [Elastic OpenTelemetry Collector limitations](https://github.com/elastic/opentelemetry/blob/main/docs/EDOT-collector/edot-collector-limitations.md) for known limitations when using the EDOT Collector. - - -## Collect your data [_collect_your_data] - -Follow these steps to collect logs and metrics using the EDOT Collector: - -:::::{tab-set} -:group: stack-serverless - -::::{tab-item} Elastic Stack -:sync: stack - -1. In {{kib}}, go to the **Observability** UI and click **Add Data**. -2. Under **What do you want to monitor?** select **Host**, and then select **OpenTelemetry: Logs & Metrics**. - - :::{image} /solutions/images/observability-quickstart-monitor-hosts-otel-entry-point.png - :alt: Host monitoring entry point - :screenshot: - ::: - -3. Select the appropriate platform. -4. Copy the command under step 1, open a terminal on your host, and run the command. - - This command downloads the {{agent}} package, extracts it in a EDOT directory. For example, `elastic-distro-8.16.0-linux-x86_64`. It also adds a sample `otel.yml` configuration file to the directory and updates the storage directory, Elastic endpoint, and API key in the file. - - The default log path is `/var/log/*.log`. To update the path, modify the `otel.yml` in the EDOT directory. - - Find additional sample `otel.yml` configuration files in the EDOT directory in the `otel_samples` folder. - -5. Copy the command under Step 2 and run it in your terminal to start the EDOT Collector. - -::::{note} -Logs are collected from setup onward, so you won’t see logs that occurred before starting the EDOT Collector. -:::: - -:::: - -::::{tab-item} Serverless -:sync: serverless - -1. [Create a new {{obs-serverless}} project](/solutions/observability/get-started/create-an-observability-project.md), or open an existing one. -2. To open the quickstart, go to **Add Data**. -3. Select **Collect and analyze logs**, and then select **OpenTelemetry**. -4. Under **What do you want to monitor?** select **Host**, and then select **Elastic Agent: Logs & Metrics**. - - :::{image} /solutions/images/serverless-quickstart-monitor-hosts-otel-entry-point.png - :alt: Host monitoring entry point - :screenshot: - ::: - -5. Select the appropriate platform, and complete the following: -6. For **MacOS and Linux**, copy the command, open a terminal on your host, and run the command to download and configure the OpenTelemetry collector. -7. For **Kubernetes**, download the manifest. -8. Copy the command under Step 2: -9. For **MacOS and Linux**, run the command in your terminal to start the EDOT Collector. -10. For **Kubernetes**, run the command from the directory where you downloaded the manifest to install the EDOT Collector on every node of your cluster. - -Logs are collected from setup onward, so you won’t see logs that occurred before starting the EDOT Collector. The default log path is `/var/log/*`. To update the path, modify `otel.yml`. - -:::: - -::::: - - -Under **Visualize your data**, you’ll see links to **Discover** to view your logs and **Hosts** to view your host metrics. - - -## Gain deeper insight into your host data [_get_value_out_of_your_data] - -After using the Hosts page and Discover to confirm you’ve ingested all the host logs and metrics you want to monitor, use Elastic {{observability}} to gain deeper insight into your host data with the following capabilities and features: - -* In the [Infrastructure UI](/solutions/observability/infra-and-hosts/analyze-infrastructure-host-metrics.md), analyze and compare data collected from your hosts. You can also: - - * [Detect anomalies](/solutions/observability/infra-and-hosts/detect-metric-anomalies.md) for memory usage and network traffic on hosts. - * [Create alerts](/solutions/observability/incident-management/create-manage-rules.md) that notify you when an anomaly is detected or a metric exceeds a given value. - -* In [Discover](/solutions/observability/logs/discover-logs.md), search and filter your log data, get information about the structure of log fields, and display your findings in a visualization. You can also: - - * [Monitor log data set quality](/solutions/observability/data-set-quality-monitoring.md) to find degraded documents. - * [Run a pattern analysis](/explore-analyze/machine-learning/machine-learning-in-kibana/xpack-ml-aiops.md#log-pattern-analysis) to find patterns in unstructured log messages. - * [Create alerts](/solutions/observability/incident-management/create-manage-rules.md) that notify you when an Observability data type reaches or exceeds a given value. - -* Use [machine learning](/explore-analyze/machine-learning/machine-learning-in-kibana.md) to apply predictive analytics to your data: - - * [Detect anomalies](/explore-analyze/machine-learning/anomaly-detection.md) by comparing real-time and historical data from different sources to look for unusual, problematic patterns. - * [Analyze log spikes and drops](/explore-analyze/machine-learning/machine-learning-in-kibana/xpack-ml-aiops.md#log-rate-analysis). - * [Detect change points](/explore-analyze/machine-learning/machine-learning-in-kibana/xpack-ml-aiops.md#change-point-detection) in your time series data. - - -Refer to the [Elastic Observability](/solutions/observability.md) for a description of other useful features. \ No newline at end of file diff --git a/solutions/observability/get-started/quickstart-unified-kubernetes-observability-with-elastic-distributions-of-opentelemetry-edot.md b/solutions/observability/get-started/quickstart-unified-kubernetes-observability-with-elastic-distributions-of-opentelemetry-edot.md deleted file mode 100644 index f05435d369..0000000000 --- a/solutions/observability/get-started/quickstart-unified-kubernetes-observability-with-elastic-distributions-of-opentelemetry-edot.md +++ /dev/null @@ -1,142 +0,0 @@ ---- -mapped_pages: - - https://www.elastic.co/guide/en/observability/current/monitor-k8s-otel-edot.html - - https://www.elastic.co/guide/en/serverless/current/monitor-k8s-otel-edot.html -applies_to: - stack: - serverless: ---- - -# Quickstart: Unified Kubernetes Observability with Elastic Distributions of OpenTelemetry (EDOT) [monitor-k8s-otel-edot] - -::::{warning} -This functionality is in technical preview and may be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. -:::: - - -In this quickstart guide, you’ll learn how to send Kubernetes logs, metrics, and application traces to Elasticsearch, using the [OpenTelemetry Operator](https://github.com/open-telemetry/opentelemetry-operator/) to orchestrate [Elastic Distributions of OpenTelemetry](https://github.com/elastic/opentelemetry/tree/main) (EDOT) Collectors and SDK instances. - -All the components will be deployed through the [opentelemetry-kube-stack](https://github.com/open-telemetry/opentelemetry-helm-charts/tree/main/charts/opentelemetry-kube-stack) helm chart. They include: - -* [OpenTelemetry Operator](https://github.com/open-telemetry/opentelemetry-operator/). -* `DaemonSet` EDOT Collector configured for node level metrics. -* `Deployment` EDOT Collector configured for cluster level metrics. -* `Instrumentation` object for applications [auto-instrumentation](https://opentelemetry.io/docs/kubernetes/operator/automatic/). - -For a more detailed description of the components and advanced configuration, refer to the [elastic/opentelemetry](https://github.com/elastic/opentelemetry/blob/main/docs/kubernetes/operator/README.md) GitHub repository. - - -## Prerequisites [_prerequisites_2] - -::::{tab-set} -:group: stack-serverless - -:::{tab-item} Elastic Stack -:sync: stack - -* An {{es}} cluster for storing and searching your data, and {{kib}} for visualizing and managing your data. This quickstart is available for all Elastic deployment models. To get started quickly, try out [{{ecloud}}](https://cloud.elastic.co/registration?page=docs&placement=docs-body). -* A running Kubernetes cluster (v1.23 or newer). -* [Kubectl](https://kubernetes.io/docs/reference/kubectl/). -* [Helm](https://helm.sh/docs/intro/install/). -* (optional) [Cert-manager](https://cert-manager.io/docs/installation/), if you opt for automatic generation and renewal of TLS certificates. - -::: - -:::{tab-item} Serverless -:sync: serverless - -* An {{obs-serverless}} project. To learn more, refer to [Create an Observability project](/solutions/observability/get-started/create-an-observability-project.md). -* A running Kubernetes cluster (v1.23 or newer). -* [Kubectl](https://kubernetes.io/docs/reference/kubectl/). -* [Helm](https://helm.sh/docs/intro/install/). -* (optional) [Cert-manager](https://cert-manager.io/docs/installation/), if you opt for automatic generation and renewal of TLS certificates. - -::: - -:::: - -## Collect your data [_collect_your_data_2] - -::::{tab-set} -:group: stack-serverless - -:::{tab-item} Elastic Stack -:sync: stack - -1. In {{kib}}, go to the **Observability** UI and click **Add Data**. -2. Under **What do you want to monitor?** select **Kubernetes**, and then select **OpenTelemetry: Full Observability**. - - :::{image} /solutions/images/observability-quickstart-k8s-otel-entry-point.png - :alt: Kubernetes-OTel entry point - :screenshot: - ::: - -3. Follow the on-screen instructions to install all needed components. - - ::::{note} - The default installation deploys the OpenTelemetry Operator with a self-signed TLS certificate valid for 365 days. This certificate **won’t be renewed** unless the Helm Chart release is manually updated. Refer to the [cert-manager integrated installation](https://github.com/elastic/opentelemetry/blob/main/docs/kubernetes/operator/README.md#cert-manager) guide to enable automatic certificate generation and renewal using [cert-manager](https://cert-manager.io/docs/installation/). - - :::: - - - Deploy the OpenTelemetry Operator and EDOT Collectors using the kube-stack Helm chart with the provided `values.yaml` file. You will run a few commands to: - - * Add the helm chart repository needed for the installation. - * Create a namespace. - * Create a secret with an API Key and the {{es}} endpoint to be used by the collectors. - * Install the `opentelemetry-kube-stack` helm chart with the provided `values.yaml`. - * Optionally, for instrumenting applications, apply the corresponding `annotations` as shown in {{kib}}. - -::: - -:::{tab-item} Serverless -:sync: serverless - -1. [Create a new {{obs-serverless}} project](/solutions/observability/get-started/create-an-observability-project.md), or open an existing one. -2. In your {{obs-serverless}} project, go to **Add Data**. -3. Under **What do you want to monitor?** select **Kubernetes**, and then select **OpenTelemetry: Full Observability**. - - :::{image} /solutions/images/serverless-quickstart-k8s-otel-entry-point.png - :alt: Kubernetes-OTel entry point - :screenshot: - ::: - -4. Follow the on-screen instructions to install all needed components. - - ::::{note} - The default installation deploys the OpenTelemetry Operator with a self-signed TLS certificate valid for 365 days. This certificate **won’t be renewed** unless the Helm Chart release is manually updated. Refer to the [cert-manager integrated installation](https://github.com/elastic/opentelemetry/blob/main/docs/kubernetes/operator/README.md#cert-manager) guide to enable automatic certificate generation and renewal using [cert-manager](https://cert-manager.io/docs/installation/). - - :::: - - - Deploy the OpenTelemetry Operator and EDOT Collectors using the kube-stack Helm chart with the provided `values.yaml` file. You will run a few commands to: - - * Add the helm chart repository needed for the installation. - * Create a namespace. - * Create a secret with an API Key and the {{es}} endpoint to be used by the collectors. - * Install the `opentelemetry-kube-stack` helm chart with the provided `values.yaml`. - * Optionally, for instrumenting applications, apply the corresponding `annotations` as shown in {{kib}}. - - -::: - -:::: - - - -## Visualize your data [_visualize_your_data] - -After installation is complete and all relevant data is flowing into Elastic, the **Visualize your data** section provides a link to the **[OTEL][Metrics Kubernetes]Cluster Overview** dashboard used to monitor the health of the cluster. - -:::{image} /solutions/images/observability-quickstart-k8s-otel-dashboard.png -:alt: Kubernetes overview dashboard -:screenshot: -::: - - -## Troubleshooting and more [_troubleshooting_and_more] - -* To troubleshoot deployment and installation, refer to [installation verification](https://github.com/elastic/opentelemetry/tree/main/docs/kubernetes/operator#installation-verification). -* For application instrumentation details, refer to [Instrumenting applications with EDOT SDKs on Kubernetes](https://github.com/elastic/opentelemetry/blob/main/docs/kubernetes/operator/instrumenting-applications.md). -* To customize the configuration, refer to [custom configuration](https://github.com/elastic/opentelemetry/tree/main/docs/kubernetes/operator#custom-configuration). -* Refer to [Observability overview](/solutions/observability/get-started/what-is-elastic-observability.md) for a description of other useful features. \ No newline at end of file diff --git a/solutions/observability/infra-and-hosts/understanding-no-results-found-message.md b/solutions/observability/infra-and-hosts/understanding-no-results-found-message.md index 0aff695063..3e2ea3e691 100644 --- a/solutions/observability/infra-and-hosts/understanding-no-results-found-message.md +++ b/solutions/observability/infra-and-hosts/understanding-no-results-found-message.md @@ -27,13 +27,7 @@ This could be for any of these reasons: :::: -* You are not using the Elastic Distribution of the OpenTelemetry Collector, which automatically maps data to the Elastic Common Schema (ECS) fields expected by the visualization. - - ::::{tip} - Follow our OpenTelemetry quickstart under **Observability** → **Add data** → **Monitor infrastructure** to make sure OpenTelemetry data is correctly mapped to ECS-compliant fields. - - :::: - +* You are not using the Elastic Distribution of the OpenTelemetry (EDOT) collector, which automatically maps data to the Elastic Common Schema (ECS) fields expected by the visualization. Refer to the [EDOT documenation](https://elastic.github.io/opentelemetry/) for more information. * You have explicitly chosen not to send these metrics. You may choose to limit the metrics sent to Elastic to save on space and improve cluster performance. For example, the System integration has options to choose which metrics you want to send. You can [edit the integration policy](https://www.elastic.co/guide/en/fleet/current/edit-or-delete-integration-policy.html) to begin collecting the missing metrics. For example: diff --git a/solutions/toc.yml b/solutions/toc.yml index 3809a8ba18..1e79d1858e 100644 --- a/solutions/toc.yml +++ b/solutions/toc.yml @@ -92,8 +92,6 @@ toc: - file: observability/get-started/create-an-observability-project.md - file: observability/get-started/quickstart-monitor-hosts-with-elastic-agent.md - file: observability/get-started/quickstart-monitor-kubernetes-cluster-with-elastic-agent.md - - file: observability/get-started/quickstart-monitor-hosts-with-opentelemetry.md - - file: observability/get-started/quickstart-unified-kubernetes-observability-with-elastic-distributions-of-opentelemetry-edot.md - file: observability/get-started/quickstart-collect-data-with-aws-firehose.md - file: observability/get-started/quickstart-elastic-cloud-otel-endpoint.md - file: observability/get-started/add-data-from-splunk.md @@ -126,11 +124,6 @@ toc: - file: observability/apm/real-user-monitoring-rum.md - file: observability/apm/create-upload-source-maps-rum.md - file: observability/apm/use-opentelemetry-with-apm.md - children: - - file: observability/apm/upstream-opentelemetry-collectors-language-sdks.md - - file: observability/apm/collect-metrics.md - - file: observability/apm/limitations.md - - file: observability/apm/resource-attributes.md - file: observability/apm/apm-k8s-attacher.md - file: observability/apm/monitor-aws-lambda-functions.md - file: observability/apm/jaeger.md