From 33db2a1aa51c7f329b35b5e167250eb08ec494f5 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Thu, 20 Nov 2025 21:39:02 +0100 Subject: [PATCH 1/5] Add OTLP proxy endpoint --- Directory.Packages.props | 4 +- .../Elastic.Documentation.Api.Core.csproj | 1 + .../Telemetry/IOtlpGateway.cs | 25 ++ .../Telemetry/OtlpProxyOptions.cs | 58 ++++ .../Telemetry/OtlpProxyRequest.cs | 18 ++ .../Telemetry/OtlpProxyUsecase.cs | 46 +++ .../Telemetry/README.md | 276 ++++++++++++++++++ .../TelemetryConstants.cs | 6 + .../Adapters/Telemetry/AdotOtlpGateway.cs | 74 +++++ .../MappingsExtensions.cs | 41 +++ .../OpenTelemetry/OpenTelemetryExtensions.cs | 1 + .../ServicesExtension.cs | 19 ++ ....Documentation.Api.IntegrationTests.csproj | 1 + .../Examples/ServiceMockingExampleTests.cs | 95 ++++++ .../Fixtures/ApiWebApplicationFactory.cs | 187 +++++++++--- .../OtlpProxyIntegrationTests.cs | 198 +++++++++++++ 16 files changed, 1002 insertions(+), 48 deletions(-) create mode 100644 src/api/Elastic.Documentation.Api.Core/Telemetry/IOtlpGateway.cs create mode 100644 src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyOptions.cs create mode 100644 src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyRequest.cs create mode 100644 src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyUsecase.cs create mode 100644 src/api/Elastic.Documentation.Api.Core/Telemetry/README.md create mode 100644 src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Telemetry/AdotOtlpGateway.cs create mode 100644 tests-integration/Elastic.Documentation.Api.IntegrationTests/Examples/ServiceMockingExampleTests.cs create mode 100644 tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 8e69507a4..05320f0cd 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -27,6 +27,7 @@ + @@ -41,6 +42,7 @@ + @@ -106,4 +108,4 @@ - \ No newline at end of file + diff --git a/src/api/Elastic.Documentation.Api.Core/Elastic.Documentation.Api.Core.csproj b/src/api/Elastic.Documentation.Api.Core/Elastic.Documentation.Api.Core.csproj index e6e497c1b..a1770fca2 100644 --- a/src/api/Elastic.Documentation.Api.Core/Elastic.Documentation.Api.Core.csproj +++ b/src/api/Elastic.Documentation.Api.Core/Elastic.Documentation.Api.Core.csproj @@ -9,6 +9,7 @@ + diff --git a/src/api/Elastic.Documentation.Api.Core/Telemetry/IOtlpGateway.cs b/src/api/Elastic.Documentation.Api.Core/Telemetry/IOtlpGateway.cs new file mode 100644 index 000000000..9591bb53c --- /dev/null +++ b/src/api/Elastic.Documentation.Api.Core/Telemetry/IOtlpGateway.cs @@ -0,0 +1,25 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Documentation.Api.Core.Telemetry; + +/// +/// Gateway for forwarding OTLP telemetry to a collector. +/// +public interface IOtlpGateway +{ + /// + /// Forwards OTLP telemetry data to the collector. + /// + /// The OTLP signal type: traces, logs, or metrics + /// The raw OTLP payload stream + /// Content-Type of the payload + /// Cancellation token + /// HTTP status code and response content + Task<(int StatusCode, string? Content)> ForwardOtlp( + string signalType, + Stream requestBody, + string contentType, + Cancel ctx = default); +} diff --git a/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyOptions.cs b/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyOptions.cs new file mode 100644 index 000000000..d976b72eb --- /dev/null +++ b/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyOptions.cs @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Microsoft.Extensions.Configuration; + +namespace Elastic.Documentation.Api.Core.Telemetry; + +/// +/// Configuration options for the OTLP proxy. +/// When using ADOT Lambda Layer, the proxy forwards to the local collector at localhost:4318. +/// The ADOT layer handles authentication and forwarding to the backend (Elastic APM, etc). +/// +/// +/// ADOT Lambda Layer runs a local OpenTelemetry Collector that accepts OTLP/HTTP on: +/// - localhost:4318 (HTTP/JSON and HTTP/protobuf) +/// - localhost:4317 (gRPC) +/// +/// The ADOT layer is configured via environment variables: +/// - OTEL_EXPORTER_OTLP_ENDPOINT: Where ADOT forwards telemetry +/// - OTEL_EXPORTER_OTLP_HEADERS: Authentication headers ADOT uses +/// - AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument (enables ADOT) +/// +public class OtlpProxyOptions +{ + /// + /// OTLP endpoint URL for the local ADOT collector. + /// Defaults to localhost:4318 when running in Lambda with ADOT layer. + /// + public string Endpoint { get; } + + public OtlpProxyOptions(IConfiguration configuration) + { + // Check for test override first (for integration tests with WireMock) + var configEndpoint = configuration["OtlpProxy:Endpoint"]; + if (!string.IsNullOrEmpty(configEndpoint)) + { + Endpoint = configEndpoint; + return; + } + + // Check if we're in Lambda with ADOT layer + var execWrapper = Environment.GetEnvironmentVariable("AWS_LAMBDA_EXEC_WRAPPER"); + var isAdotEnabled = execWrapper?.Contains("otel-instrument") == true; + + if (isAdotEnabled) + { + // ADOT Lambda Layer runs collector on localhost:4318 + Endpoint = "http://localhost:4318"; + } + else + { + // Fallback to configured endpoint for local development + Endpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT") + ?? "http://localhost:4318"; + } + } +} diff --git a/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyRequest.cs b/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyRequest.cs new file mode 100644 index 000000000..90d45c093 --- /dev/null +++ b/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyRequest.cs @@ -0,0 +1,18 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +namespace Elastic.Documentation.Api.Core.Telemetry; + +/// +/// Request model for OTLP proxy endpoint. +/// Accepts raw OTLP payload from frontend and forwards to configured OTLP endpoint. +/// +public class OtlpProxyRequest +{ + /// + /// The OTLP signal type: traces, logs, or metrics + /// + public required string SignalType { get; init; } +} + diff --git a/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyUsecase.cs b/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyUsecase.cs new file mode 100644 index 000000000..4c8c8f2f7 --- /dev/null +++ b/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyUsecase.cs @@ -0,0 +1,46 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Diagnostics; +using Microsoft.Extensions.Logging; + +namespace Elastic.Documentation.Api.Core.Telemetry; + +/// +/// Proxies OTLP telemetry from the frontend to the local ADOT Lambda Layer collector. +/// The ADOT layer handles authentication and forwarding to the backend. +/// +public class OtlpProxyUsecase( + IOtlpGateway gateway, + ILogger logger) +{ + private static readonly ActivitySource ActivitySource = new(TelemetryConstants.OtlpProxySourceName); + + /// + /// Proxies OTLP data from the frontend to the local ADOT collector. + /// + /// The OTLP signal type: traces, logs, or metrics + /// The raw OTLP payload (JSON or protobuf) + /// Content-Type header from the original request + /// Cancellation token + /// HTTP status code and response content + public async Task<(int StatusCode, string? Content)> ProxyOtlp( + string signalType, + Stream requestBody, + string contentType, + Cancel ctx = default) + { + using var activity = ActivitySource.StartActivity("ProxyOtlp", ActivityKind.Client); + + // Validate signal type + if (signalType is not ("traces" or "logs" or "metrics")) + { + logger.LogWarning("Invalid OTLP signal type: {SignalType}", signalType); + return (400, $"Invalid signal type: {signalType}. Must be traces, logs, or metrics"); + } + + // Forward to gateway + return await gateway.ForwardOtlp(signalType, requestBody, contentType, ctx); + } +} diff --git a/src/api/Elastic.Documentation.Api.Core/Telemetry/README.md b/src/api/Elastic.Documentation.Api.Core/Telemetry/README.md new file mode 100644 index 000000000..980c3c6f4 --- /dev/null +++ b/src/api/Elastic.Documentation.Api.Core/Telemetry/README.md @@ -0,0 +1,276 @@ +# OTLP Proxy for Frontend Telemetry + +This OTLP (OpenTelemetry Protocol) proxy allows frontend JavaScript code to send telemetry (logs, traces, metrics) to the same OTLP collector used by the backend **without exposing authentication credentials to the browser**. + +## Security Model + +### ✅ Secure: Backend handles authentication + +``` +Frontend (Browser) → API Proxy → OTLP Collector (Elastic APM/OTel) + ↑ + Adds auth headers + (credentials stay secure) +``` + +The proxy: +- Reads credentials from environment variables on the backend +- Automatically adds authentication headers to forwarded requests +- Prevents credential exposure to browser DevTools or network inspection + +### ❌ Insecure: Direct frontend connection + +``` +Frontend (Browser) → OTLP Collector + ↑ + Requires auth credentials + (exposed in browser code) +``` + +## Configuration + +The proxy uses standard OpenTelemetry environment variables: + +```bash +# Required: OTLP collector endpoint +OTEL_EXPORTER_OTLP_ENDPOINT=https://your-apm-server.elastic.co:443 + +# Optional: Authentication headers (multiple headers separated by comma) +OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer secret-token" + +# Or for Elastic APM with API Key: +OTEL_EXPORTER_OTLP_HEADERS="Authorization=ApiKey base64-encoded-api-key" + +# Or multiple headers: +OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer token,X-Custom-Header=value" +``` + +## API Endpoints + +The proxy provides three endpoints matching the OTLP specification: + +``` +POST /docs/_api/v1/otlp/v1/traces - Forward trace spans +POST /docs/_api/v1/otlp/v1/logs - Forward log records +POST /docs/_api/v1/otlp/v1/metrics - Forward metrics +``` + +### Content Types Supported + +- `application/json` - OTLP JSON encoding (recommended for browser) +- `application/x-protobuf` - OTLP protobuf encoding (smaller but requires encoding) + +## Frontend Usage + +### Option 1: Using OpenTelemetry JS SDK (Recommended) + +```typescript +import { WebTracerProvider } from '@opentelemetry/sdk-trace-web'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { Resource } from '@opentelemetry/resources'; +import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; + +// Configure the tracer to use the proxy endpoint +const provider = new WebTracerProvider({ + resource: new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: 'docs-frontend', + [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0', + }), +}); + +// Point the exporter to the proxy endpoint (no credentials needed!) +const exporter = new OTLPTraceExporter({ + url: 'https://docs.elastic.co/_api/v1/otlp/v1/traces', + headers: {}, // No auth headers needed - proxy handles it +}); + +provider.addSpanProcessor(new BatchSpanProcessor(exporter)); +provider.register(); + +// Now you can create spans +const tracer = provider.getTracer('docs-frontend'); +const span = tracer.startSpan('page-load'); +span.end(); +``` + +### Option 2: Using OpenTelemetry Logs API + +```typescript +import { LoggerProvider } from '@opentelemetry/sdk-logs'; +import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; + +const loggerProvider = new LoggerProvider({ + resource: new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: 'docs-frontend', + }), +}); + +const exporter = new OTLPLogExporter({ + url: 'https://docs.elastic.co/_api/v1/otlp/v1/logs', +}); + +loggerProvider.addLogRecordProcessor(new BatchLogRecordProcessor(exporter)); + +const logger = loggerProvider.getLogger('docs-frontend'); +logger.emit({ + severityNumber: 9, + severityText: 'INFO', + body: 'User clicked button', + attributes: { + 'user.action': 'click', + 'button.id': 'submit', + }, +}); +``` + +### Option 3: Manual Fetch (for debugging) + +```typescript +// Send logs manually +await fetch('https://docs.elastic.co/_api/v1/otlp/v1/logs', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + resourceLogs: [ + { + resource: { + attributes: [ + { key: 'service.name', value: { stringValue: 'docs-frontend' } }, + ], + }, + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: String(Date.now() * 1000000), + severityNumber: 9, + severityText: 'INFO', + body: { + stringValue: 'Test log from browser', + }, + attributes: [ + { key: 'page.url', value: { stringValue: window.location.href } }, + ], + }, + ], + }, + ], + }, + ], + }), +}); +``` + +## CORS Configuration + +If your frontend is served from a different domain, you'll need to configure CORS: + +```csharp +// In Program.cs or startup configuration +app.UseCors(policy => policy + .WithOrigins("https://docs.elastic.co") + .AllowAnyMethod() + .AllowAnyHeader()); +``` + +## Monitoring the Proxy + +The proxy creates its own spans under the `Elastic.Documentation.Api.OtlpProxy` activity source. + +Each proxied request includes: +- `otel.signal_type` - The signal type (traces/logs/metrics) +- `otel.content_type` - The content type of the request +- `otel.target_url` - The target OTLP collector URL +- `http.response.status_code` - Response status from collector + +## Example: Full Frontend Integration + +```typescript +// frontend/telemetry.ts +import { WebTracerProvider } from '@opentelemetry/sdk-trace-web'; +import { ZoneContextManager } from '@opentelemetry/context-zone'; +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { DocumentLoadInstrumentation } from '@opentelemetry/instrumentation-document-load'; +import { UserInteractionInstrumentation } from '@opentelemetry/instrumentation-user-interaction'; +import { registerInstrumentations } from '@opentelemetry/instrumentation'; + +export function initTelemetry() { + const provider = new WebTracerProvider({ + resource: new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: 'docs-frontend', + [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: + window.location.hostname.includes('localhost') ? 'dev' : 'prod', + }), + }); + + // Use the proxy endpoint - no credentials needed! + const exporter = new OTLPTraceExporter({ + url: `${window.location.origin}/_api/v1/otlp/v1/traces`, + }); + + provider.addSpanProcessor(new BatchSpanProcessor(exporter, { + maxQueueSize: 100, + scheduledDelayMillis: 5000, + })); + + provider.register({ + contextManager: new ZoneContextManager(), + }); + + // Auto-instrument page loads and user interactions + registerInstrumentations({ + instrumentations: [ + new DocumentLoadInstrumentation(), + new UserInteractionInstrumentation(), + ], + }); + + console.log('OpenTelemetry initialized with proxy'); +} +``` + +## Troubleshooting + +### Proxy returns 503 "OTLP proxy is not configured" + +The `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable is not set. Configure it in your Lambda environment variables. + +### Proxy returns 401/403 from collector + +The authentication headers in `OTEL_EXPORTER_OTLP_HEADERS` are invalid or expired. + +### Frontend gets CORS errors + +Add CORS configuration to allow requests from your frontend domain. + +### Data not appearing in Elastic APM + +1. Check the proxy logs for errors +2. Verify the OTLP endpoint is correct +3. Ensure the collector is configured to accept OTLP/HTTP on `/v1/traces`, `/v1/logs`, `/v1/metrics` +4. Check that your OTLP payload format is correct (use the OpenTelemetry SDK to avoid formatting errors) + +## Performance Considerations + +- The proxy uses streaming to avoid buffering large payloads in memory +- Batch telemetry in the frontend before sending (use `BatchSpanProcessor` and `BatchLogRecordProcessor`) +- Consider sampling high-volume traces in production +- Monitor proxy latency via the `Elastic.Documentation.Api.OtlpProxy` spans + +## Security Best Practices + +✅ **DO:** +- Use HTTPS for the proxy endpoint in production +- Set appropriate rate limits on the proxy endpoint +- Monitor for unusual traffic patterns +- Use resource attributes to identify the frontend service + +❌ **DON'T:** +- Expose OTLP collector credentials in frontend code +- Allow unauthenticated access to the collector directly +- Send PII (personally identifiable information) in telemetry without user consent +- Forget to configure CORS for cross-origin requests + diff --git a/src/api/Elastic.Documentation.Api.Core/TelemetryConstants.cs b/src/api/Elastic.Documentation.Api.Core/TelemetryConstants.cs index b6a36c7c6..6a8d2683c 100644 --- a/src/api/Elastic.Documentation.Api.Core/TelemetryConstants.cs +++ b/src/api/Elastic.Documentation.Api.Core/TelemetryConstants.cs @@ -25,4 +25,10 @@ public static class TelemetryConstants /// Tag/baggage name used to annotate spans with the user's EUID value. /// public const string UserEuidAttributeName = "user.euid"; + + /// + /// ActivitySource name for OTLP proxy operations. + /// Used to trace frontend telemetry proxying. + /// + public const string OtlpProxySourceName = "Elastic.Documentation.Api.OtlpProxy"; } diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Telemetry/AdotOtlpGateway.cs b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Telemetry/AdotOtlpGateway.cs new file mode 100644 index 000000000..f045528ca --- /dev/null +++ b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Telemetry/AdotOtlpGateway.cs @@ -0,0 +1,74 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Documentation.Api.Core.Telemetry; +using Microsoft.Extensions.Logging; + +namespace Elastic.Documentation.Api.Infrastructure.Adapters.Telemetry; + +/// +/// Gateway that forwards OTLP telemetry to the ADOT Lambda Layer collector. +/// +public class AdotOtlpGateway( + OtlpProxyOptions options, + ILogger logger) : IOtlpGateway +{ + private static readonly HttpClient HttpClient = new() + { + Timeout = TimeSpan.FromSeconds(30) + }; + + /// + public async Task<(int StatusCode, string? Content)> ForwardOtlp( + string signalType, + Stream requestBody, + string contentType, + Cancel ctx = default) + { + try + { + // Build the target URL: http://localhost:4318/v1/{signalType} + var targetUrl = $"{options.Endpoint.TrimEnd('/')}/v1/{signalType}"; + + logger.LogDebug("Forwarding OTLP {SignalType} to ADOT collector at {TargetUrl}", signalType, targetUrl); + + using var request = new HttpRequestMessage(HttpMethod.Post, targetUrl); + + // Forward the content + request.Content = new StreamContent(requestBody); + request.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType); + + // No need to add authentication headers - ADOT layer handles auth to backend + // Just forward the telemetry to the local collector + + // Forward to ADOT collector + using var response = await HttpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, ctx); + var responseContent = response.Content.Headers.ContentLength > 0 + ? await response.Content.ReadAsStringAsync(ctx) + : string.Empty; + + if (!response.IsSuccessStatusCode) + { + logger.LogError("OTLP forward to ADOT failed with status {StatusCode}: {Content}", + response.StatusCode, responseContent); + } + else + { + logger.LogDebug("Successfully forwarded OTLP {SignalType} to ADOT collector", signalType); + } + + return ((int)response.StatusCode, responseContent); + } + catch (HttpRequestException ex) when (ex.Message.Contains("Connection refused") || ex.InnerException?.Message?.Contains("Connection refused") == true) + { + logger.LogError(ex, "Failed to connect to ADOT collector at {Endpoint}. Is ADOT Lambda Layer enabled?", options.Endpoint); + return (503, "ADOT collector not available. Ensure AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument is set"); + } + catch (Exception ex) + { + logger.LogError(ex, "Error forwarding OTLP {SignalType}", signalType); + return (500, $"Error forwarding OTLP: {ex.Message}"); + } + } +} diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/MappingsExtensions.cs b/src/api/Elastic.Documentation.Api.Infrastructure/MappingsExtensions.cs index c95372a69..ce7724699 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/MappingsExtensions.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/MappingsExtensions.cs @@ -4,6 +4,7 @@ using Elastic.Documentation.Api.Core.AskAi; using Elastic.Documentation.Api.Core.Search; +using Elastic.Documentation.Api.Core.Telemetry; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; @@ -19,6 +20,7 @@ public static void MapElasticDocsApiEndpoints(this IEndpointRouteBuilder group) _ = group.MapPost("/", () => Results.Empty); MapAskAiEndpoint(group); MapSearchEndpoint(group); + MapOtlpProxyEndpoint(group); } private static void MapAskAiEndpoint(IEndpointRouteBuilder group) @@ -55,4 +57,43 @@ Cancel ctx return Results.Ok(searchResponse); }); } + + private static void MapOtlpProxyEndpoint(IEndpointRouteBuilder group) + { + // Use /o/* to avoid adblocker detection (common blocklists target /otlp, /telemetry, etc.) + var otlpGroup = group.MapGroup("/o"); + + // Proxy endpoint for traces + // Frontend: POST /_api/v1/o/t → ADOT: POST localhost:4318/v1/traces + _ = otlpGroup.MapPost("/t", + async (HttpContext context, OtlpProxyUsecase proxyUsecase, Cancel ctx) => + { + var contentType = context.Request.ContentType ?? "application/json"; + var (statusCode, content) = await proxyUsecase.ProxyOtlp("traces", context.Request.Body, contentType, ctx); + return Results.Content(content ?? string.Empty, contentType, statusCode: statusCode); + }) + .DisableAntiforgery(); // Frontend requests won't have antiforgery tokens + + // Proxy endpoint for logs + // Frontend: POST /_api/v1/o/l → ADOT: POST localhost:4318/v1/logs + _ = otlpGroup.MapPost("/l", + async (HttpContext context, OtlpProxyUsecase proxyUsecase, Cancel ctx) => + { + var contentType = context.Request.ContentType ?? "application/json"; + var (statusCode, content) = await proxyUsecase.ProxyOtlp("logs", context.Request.Body, contentType, ctx); + return Results.Content(content ?? string.Empty, contentType, statusCode: statusCode); + }) + .DisableAntiforgery(); + + // Proxy endpoint for metrics + // Frontend: POST /_api/v1/o/m → ADOT: POST localhost:4318/v1/metrics + _ = otlpGroup.MapPost("/m", + async (HttpContext context, OtlpProxyUsecase proxyUsecase, Cancel ctx) => + { + var contentType = context.Request.ContentType ?? "application/json"; + var (statusCode, content) = await proxyUsecase.ProxyOtlp("metrics", context.Request.Body, contentType, ctx); + return Results.Content(content ?? string.Empty, contentType, statusCode: statusCode); + }) + .DisableAntiforgery(); + } } diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/OpenTelemetry/OpenTelemetryExtensions.cs b/src/api/Elastic.Documentation.Api.Infrastructure/OpenTelemetry/OpenTelemetryExtensions.cs index 7927ab1a7..55ad64151 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/OpenTelemetry/OpenTelemetryExtensions.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/OpenTelemetry/OpenTelemetryExtensions.cs @@ -36,6 +36,7 @@ public static TracerProviderBuilder AddDocsApiTracing(this TracerProviderBuilder _ = builder .AddSource(TelemetryConstants.AskAiSourceName) .AddSource(TelemetryConstants.StreamTransformerSourceName) + .AddSource(TelemetryConstants.OtlpProxySourceName) .AddAspNetCoreInstrumentation(aspNetCoreOptions => { // Enrich spans with custom attributes from HTTP context diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs b/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs index 328bff119..416035992 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs @@ -6,10 +6,13 @@ using Elastic.Documentation.Api.Core; using Elastic.Documentation.Api.Core.AskAi; using Elastic.Documentation.Api.Core.Search; +using Elastic.Documentation.Api.Core.Telemetry; using Elastic.Documentation.Api.Infrastructure.Adapters.AskAi; using Elastic.Documentation.Api.Infrastructure.Adapters.Search; +using Elastic.Documentation.Api.Infrastructure.Adapters.Telemetry; using Elastic.Documentation.Api.Infrastructure.Aws; using Elastic.Documentation.Api.Infrastructure.Gcp; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using NetEscapades.EnumGenerators; @@ -71,6 +74,7 @@ private static void AddElasticDocsApiUsecases(this IServiceCollection services, AddParameterProvider(services, appEnv); AddAskAiUsecase(services, appEnv); AddSearchUsecase(services, appEnv); + AddOtlpProxyUsecase(services, appEnv); } // https://docs.aws.amazon.com/systems -manager/latest/userguide/ps-integration-lambda-extensions.html @@ -171,4 +175,19 @@ private static void AddSearchUsecase(IServiceCollection services, AppEnv appEnv) _ = services.AddScoped(); _ = services.AddScoped(); } + + private static void AddOtlpProxyUsecase(IServiceCollection services, AppEnv appEnv) + { + var logger = GetLogger(services); + logger?.LogInformation("Configuring OTLP proxy use case for environment {AppEnvironment}", appEnv); + + _ = services.AddSingleton(sp => + { + var config = sp.GetRequiredService(); + return new OtlpProxyOptions(config); + }); + _ = services.AddScoped(); + _ = services.AddScoped(); + logger?.LogInformation("OTLP proxy configured to forward to ADOT Lambda Layer collector"); + } } diff --git a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Elastic.Documentation.Api.IntegrationTests.csproj b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Elastic.Documentation.Api.IntegrationTests.csproj index 32e364eba..d319a74c5 100644 --- a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Elastic.Documentation.Api.IntegrationTests.csproj +++ b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Elastic.Documentation.Api.IntegrationTests.csproj @@ -14,6 +14,7 @@ + diff --git a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Examples/ServiceMockingExampleTests.cs b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Examples/ServiceMockingExampleTests.cs new file mode 100644 index 000000000..e27fd61d3 --- /dev/null +++ b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Examples/ServiceMockingExampleTests.cs @@ -0,0 +1,95 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Net; +using Elastic.Documentation.Api.Core.Search; +using Elastic.Documentation.Api.Core.Telemetry; +using Elastic.Documentation.Api.IntegrationTests.Fixtures; +using FakeItEasy; +using FluentAssertions; +using Xunit; + +namespace Elastic.Documentation.Api.IntegrationTests.Examples; + +/// +/// Example test demonstrating how to mock multiple services in integration tests. +/// This serves as documentation for the service mocking pattern. +/// +public class ServiceMockingExampleTests +{ + [Fact] + public async Task ExampleWithMultipleServiceMocks() + { + // Arrange - Create multiple mocks + var mockOtlpGateway = A.Fake(); + var mockSearchGateway = A.Fake(); + + // Configure mock behaviors + A.CallTo(() => mockOtlpGateway.ForwardOtlp( + A._, + A._, + A._, + A._)) + .Returns(Task.FromResult<(int StatusCode, string? Content)>((200, "{}}"))); + + A.CallTo(() => mockSearchGateway.SearchAsync( + A._, + A._, + A._, + A._)) + .Returns(Task.FromResult((TotalHits: 1, Results: new List + { + new() + { + Type = "page", + Url = "/docs/test", + Title = "Test Result", + Description = "A test result", + Parents = [] + } + }))); + + // Create factory with multiple mocked services using fluent API + using var factory = ApiWebApplicationFactory.WithMockedServices(services => + services + .Replace(mockOtlpGateway) // Replace IOtlpGateway + .Replace(mockSearchGateway)); // Replace ISearchGateway + + using var client = factory.CreateClient(); + + // Act - Make a search request that uses the mocked search gateway + var searchResponse = await client.GetAsync( + "/docs/_api/v1/search?q=test&page=1&pageSize=5", + TestContext.Current.CancellationToken); + + // Assert + searchResponse.StatusCode.Should().Be(HttpStatusCode.OK); + + // Verify the search gateway was called with correct parameters + A.CallTo(() => mockSearchGateway.SearchAsync( + "test", // query + 1, // page + 5, // pageSize (default in API) + A._)) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task ExampleWithSingletonMock() + { + // Arrange + var mockGateway = A.Fake(); + A.CallTo(() => mockGateway.ForwardOtlp(A._, A._, A._, A._)) + .Returns(Task.FromResult<(int, string?)>((503, "Unavailable"))); + + // Use ReplaceSingleton for services registered as singletons + using var factory = ApiWebApplicationFactory.WithMockedServices(services => + services.ReplaceSingleton(mockGateway)); + + using var client = factory.CreateClient(); + + // Act & Assert + // Your test logic here... + } +} diff --git a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Fixtures/ApiWebApplicationFactory.cs b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Fixtures/ApiWebApplicationFactory.cs index 96c60e7aa..3e3139515 100644 --- a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Fixtures/ApiWebApplicationFactory.cs +++ b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Fixtures/ApiWebApplicationFactory.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Text; using Elastic.Documentation.Api.Core.AskAi; +using Elastic.Documentation.Api.Core.Telemetry; using Elastic.Documentation.Api.Infrastructure; using Elastic.Documentation.Api.Infrastructure.Aws; using Elastic.Documentation.Api.Infrastructure.OpenTelemetry; @@ -12,6 +13,7 @@ using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; using OpenTelemetry; using OpenTelemetry.Logs; @@ -28,53 +30,76 @@ public class ApiWebApplicationFactory : WebApplicationFactory public List ExportedActivities { get; } = []; public List ExportedLogRecords { get; } = []; private readonly List _mockMemoryStreams = []; - protected override void ConfigureWebHost(IWebHostBuilder builder) => - builder.ConfigureServices(services => - { - var otelBuilder = services.AddOpenTelemetry(); - _ = otelBuilder.WithTracing(tracing => - { - _ = tracing - .AddDocsApiTracing() // Reuses production configuration - .AddInMemoryExporter(ExportedActivities); - }); - _ = otelBuilder.WithLogging(logging => - { - _ = logging - .AddDocsApiLogging() // Reuses production configuration - .AddInMemoryExporter(ExportedLogRecords); - }); - - // Mock IParameterProvider to avoid AWS dependencies - var mockParameterProvider = A.Fake(); - A.CallTo(() => mockParameterProvider.GetParam(A._, A._, A._)) - .Returns(Task.FromResult("mock-value")); - _ = services.AddSingleton(mockParameterProvider); - - // Mock IAskAiGateway to avoid external AI service calls - var mockAskAiGateway = A.Fake>(); - A.CallTo(() => mockAskAiGateway.AskAi(A._, A._)) - .ReturnsLazily(() => - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes("data: test\n\n")); - _mockMemoryStreams.Add(stream); - return Task.FromResult(stream); - }); - _ = services.AddSingleton(mockAskAiGateway); - - // Mock IStreamTransformer - var mockTransformer = A.Fake(); - A.CallTo(() => mockTransformer.AgentProvider).Returns("test-provider"); - A.CallTo(() => mockTransformer.AgentId).Returns("test-agent"); - A.CallTo(() => mockTransformer.TransformAsync(A._, A._, A._, A._)) - .ReturnsLazily((Stream s, string? _, Activity? activity, Cancel _) => - { - // Dispose the activity if provided (simulating what the real transformer does) - activity?.Dispose(); - return Task.FromResult(s); - }); - _ = services.AddSingleton(mockTransformer); - }); + private readonly Action? _configureServices; + + public ApiWebApplicationFactory() : this(null) + { + } + + internal ApiWebApplicationFactory(Action? configureServices) => _configureServices = configureServices; + + /// + /// Creates a factory with specific services replaced by mocks. + /// This allows tests to inject fake implementations for testing specific scenarios. + /// + /// Action to configure service replacements + /// New factory instance with replaced services + public static ApiWebApplicationFactory WithMockedServices(Action serviceReplacements) + { + var builder = new ServiceReplacementBuilder(); + serviceReplacements(builder); + return new ApiWebApplicationFactory(builder.Build()); + } + + protected override void ConfigureWebHost(IWebHostBuilder builder) => builder.ConfigureServices(services => + { + var otelBuilder = services.AddOpenTelemetry(); + _ = otelBuilder.WithTracing(tracing => + { + _ = tracing + .AddDocsApiTracing() // Reuses production configuration + .AddInMemoryExporter(ExportedActivities); + }); + _ = otelBuilder.WithLogging(logging => + { + _ = logging + .AddDocsApiLogging() // Reuses production configuration + .AddInMemoryExporter(ExportedLogRecords); + }); + + // Mock IParameterProvider to avoid AWS dependencies + var mockParameterProvider = A.Fake(); + A.CallTo(() => mockParameterProvider.GetParam(A._, A._, A._)) + .Returns(Task.FromResult("mock-value")); + _ = services.AddSingleton(mockParameterProvider); + + // Mock IAskAiGateway to avoid external AI service calls + var mockAskAiGateway = A.Fake>(); + A.CallTo(() => mockAskAiGateway.AskAi(A._, A._)) + .ReturnsLazily(() => + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes("data: test\n\n")); + _mockMemoryStreams.Add(stream); + return Task.FromResult(stream); + }); + _ = services.AddSingleton(mockAskAiGateway); + + // Mock IStreamTransformer + var mockTransformer = A.Fake(); + A.CallTo(() => mockTransformer.AgentProvider).Returns("test-provider"); + A.CallTo(() => mockTransformer.AgentId).Returns("test-agent"); + A.CallTo(() => mockTransformer.TransformAsync(A._, A._, A._, A._)) + .ReturnsLazily((Stream s, string? _, Activity? activity, Cancel _) => + { + // Dispose the activity if provided (simulating what the real transformer does) + activity?.Dispose(); + return Task.FromResult(s); + }); + _ = services.AddSingleton(mockTransformer); + + // Allow tests to override services - RemoveAll + Add to properly replace + _configureServices?.Invoke(services); + }); protected override void Dispose(bool disposing) { @@ -89,3 +114,71 @@ protected override void Dispose(bool disposing) base.Dispose(disposing); } } + +/// +/// Builder for replacing services in integration tests. +/// Provides a fluent API for replacing multiple services with mocks. +/// +public class ServiceReplacementBuilder +{ + private readonly List> _replacements = []; + + /// + /// Replace a service of type TService with a specific instance. + /// + /// The service interface type to replace + /// The mock/fake instance to use + /// This builder for chaining + public ServiceReplacementBuilder Replace(TService instance) where TService : class + { + _replacements.Add(services => + { + services.RemoveAll(); + _ = services.AddScoped(_ => instance); + }); + return this; + } + + /// + /// Replace a service of type TService with a factory function. + /// + /// The service interface type to replace + /// Factory function to create the service + /// This builder for chaining + public ServiceReplacementBuilder Replace(Func factory) where TService : class + { + _replacements.Add(services => + { + services.RemoveAll(); + _ = services.AddScoped(factory); + }); + return this; + } + + /// + /// Replace a service with a singleton instance. + /// + /// The service interface type to replace + /// The singleton instance to use + /// This builder for chaining + public ServiceReplacementBuilder ReplaceSingleton(TService instance) where TService : class + { + _replacements.Add(services => + { + services.RemoveAll(); + _ = services.AddSingleton(_ => instance); + }); + return this; + } + + /// + /// Builds the final service configuration action. + /// + internal Action Build() => services => + { + foreach (var replacement in _replacements) + { + replacement(services); + } + }; +} diff --git a/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs b/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs new file mode 100644 index 000000000..ed31c714d --- /dev/null +++ b/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs @@ -0,0 +1,198 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Net; +using System.Text; +using Elastic.Documentation.Api.Core.Telemetry; +using Elastic.Documentation.Api.IntegrationTests.Fixtures; +using FakeItEasy; +using FluentAssertions; +using Xunit; + +namespace Elastic.Documentation.Api.IntegrationTests; + +public class OtlpProxyIntegrationTests +{ + [Fact] + public async Task OtlpProxyTracesEndpointReturnsSuccess() + { + // Arrange + var mockGateway = A.Fake(); + A.CallTo(() => mockGateway.ForwardOtlp( + A._, + A._, + A._, + A._)) + .Returns(Task.FromResult<(int StatusCode, string? Content)>((200, "{}"))); + + using var factory = ApiWebApplicationFactory.WithMockedServices(services => + services.Replace(mockGateway)); + using var client = factory.CreateClient(); + + var otlpPayload = /*lang=json,strict*/ """ + { + "resourceSpans": [{ + "scopeSpans": [{ + "spans": [{ + "traceId": "0123456789abcdef0123456789abcdef", + "spanId": "0123456789abcdef", + "name": "test-span" + }] + }] + }] + } + """; + var content = new StringContent(otlpPayload, Encoding.UTF8, "application/json"); + + // Act + var response = await client.PostAsync("/docs/_api/v1/o/t", content, TestContext.Current.CancellationToken); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + + // Verify gateway was called + A.CallTo(() => mockGateway.ForwardOtlp( + "traces", + A._, + A._, + A._)) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task OtlpProxyLogsEndpointReturnsSuccess() + { + // Arrange + var mockGateway = A.Fake(); + A.CallTo(() => mockGateway.ForwardOtlp( + A._, + A._, + A._, + A._)) + .Returns(Task.FromResult<(int StatusCode, string? Content)>((200, "{}"))); + + using var factory = ApiWebApplicationFactory.WithMockedServices(services => + services.Replace(mockGateway)); + using var client = factory.CreateClient(); + + var otlpPayload = /*lang=json,strict*/ """ + { + "resourceLogs": [{ + "scopeLogs": [{ + "logRecords": [{ + "timeUnixNano": "1672531200000000000", + "severityNumber": 9, + "severityText": "INFO", + "body": { + "stringValue": "Test log" + } + }] + }] + }] + } + """; + var content = new StringContent(otlpPayload, Encoding.UTF8, "application/json"); + + // Act + var response = await client.PostAsync("/docs/_api/v1/o/l", content, TestContext.Current.CancellationToken); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + + // Verify gateway was called + A.CallTo(() => mockGateway.ForwardOtlp( + "logs", + A._, + A._, + A._)) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task OtlpProxyMetricsEndpointReturnsSuccess() + { + // Arrange + var mockGateway = A.Fake(); + A.CallTo(() => mockGateway.ForwardOtlp( + A._, + A._, + A._, + A._)) + .Returns(Task.FromResult<(int StatusCode, string? Content)>((200, "{}"))); + + using var factory = ApiWebApplicationFactory.WithMockedServices(services => + services.Replace(mockGateway)); + using var client = factory.CreateClient(); + + var otlpPayload = /*lang=json,strict*/ """ + { + "resourceMetrics": [{ + "scopeMetrics": [{ + "metrics": [{ + "name": "test_metric", + "unit": "1" + }] + }] + }] + } + """; + var content = new StringContent(otlpPayload, Encoding.UTF8, "application/json"); + + // Act + var response = await client.PostAsync("/docs/_api/v1/o/m", content, TestContext.Current.CancellationToken); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.OK); + + // Verify gateway was called + A.CallTo(() => mockGateway.ForwardOtlp( + "metrics", + A._, + A._, + A._)) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task OtlpProxyReturnsGatewayErrorStatusCode() + { + // Arrange + var mockGateway = A.Fake(); + A.CallTo(() => mockGateway.ForwardOtlp( + A._, + A._, + A._, + A._)) + .Returns(Task.FromResult<(int StatusCode, string? Content)>((503, "Service unavailable"))); + + using var factory = ApiWebApplicationFactory.WithMockedServices(services => + services.Replace(mockGateway)); + using var client = factory.CreateClient(); + + var content = new StringContent("{}", Encoding.UTF8, "application/json"); + + // Act + var response = await client.PostAsync("/docs/_api/v1/o/t", content, TestContext.Current.CancellationToken); + + // Assert + response.StatusCode.Should().Be(HttpStatusCode.ServiceUnavailable); + var responseBody = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); + responseBody.Should().Contain("Service unavailable"); + } + + [Fact] + public async Task OtlpProxyInvalidSignalTypeReturns404() + { + // Arrange + using var factory = new ApiWebApplicationFactory(); + using var client = factory.CreateClient(); + var content = new StringContent("{}", Encoding.UTF8, "application/json"); + + // Act - use invalid signal type + var response = await client.PostAsync("/docs/_api/v1/o/invalid", content, TestContext.Current.CancellationToken); + + // Assert - route doesn't exist + response.StatusCode.Should().Be(HttpStatusCode.NotFound); + } +} From a8c7d98f23830542e2e506441ee4e94964516189 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Thu, 20 Nov 2025 21:55:26 +0100 Subject: [PATCH 2/5] Fix OtlpProxyOptions --- .../Telemetry/OtlpProxyOptions.cs | 34 +-- .../Telemetry/README.md | 276 ------------------ 2 files changed, 12 insertions(+), 298 deletions(-) delete mode 100644 src/api/Elastic.Documentation.Api.Core/Telemetry/README.md diff --git a/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyOptions.cs b/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyOptions.cs index d976b72eb..4872cb9b8 100644 --- a/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyOptions.cs +++ b/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyOptions.cs @@ -8,18 +8,19 @@ namespace Elastic.Documentation.Api.Core.Telemetry; /// /// Configuration options for the OTLP proxy. -/// When using ADOT Lambda Layer, the proxy forwards to the local collector at localhost:4318. -/// The ADOT layer handles authentication and forwarding to the backend (Elastic APM, etc). +/// The proxy forwards telemetry to a local OTLP collector (typically ADOT Lambda Layer). /// /// /// ADOT Lambda Layer runs a local OpenTelemetry Collector that accepts OTLP/HTTP on: /// - localhost:4318 (HTTP/JSON and HTTP/protobuf) /// - localhost:4317 (gRPC) /// -/// The ADOT layer is configured via environment variables: -/// - OTEL_EXPORTER_OTLP_ENDPOINT: Where ADOT forwards telemetry -/// - OTEL_EXPORTER_OTLP_HEADERS: Authentication headers ADOT uses -/// - AWS_LAMBDA_EXEC_WRAPPER: /opt/otel-instrument (enables ADOT) +/// Configuration priority: +/// 1. OtlpProxy:Endpoint in IConfiguration (for tests/overrides) +/// 2. OTEL_EXPORTER_OTLP_ENDPOINT environment variable +/// 3. Default: http://localhost:4318 +/// +/// The proxy will return 503 if the collector is not available. /// public class OtlpProxyOptions { @@ -31,7 +32,7 @@ public class OtlpProxyOptions public OtlpProxyOptions(IConfiguration configuration) { - // Check for test override first (for integration tests with WireMock) + // Check for explicit configuration override first (for tests or custom deployments) var configEndpoint = configuration["OtlpProxy:Endpoint"]; if (!string.IsNullOrEmpty(configEndpoint)) { @@ -39,20 +40,9 @@ public OtlpProxyOptions(IConfiguration configuration) return; } - // Check if we're in Lambda with ADOT layer - var execWrapper = Environment.GetEnvironmentVariable("AWS_LAMBDA_EXEC_WRAPPER"); - var isAdotEnabled = execWrapper?.Contains("otel-instrument") == true; - - if (isAdotEnabled) - { - // ADOT Lambda Layer runs collector on localhost:4318 - Endpoint = "http://localhost:4318"; - } - else - { - // Fallback to configured endpoint for local development - Endpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT") - ?? "http://localhost:4318"; - } + // Default to localhost:4318 - this is where ADOT Lambda Layer collector runs + // If ADOT layer is not present, the proxy will fail gracefully and return 503 + Endpoint = Environment.GetEnvironmentVariable("OTEL_EXPORTER_OTLP_ENDPOINT") + ?? "http://localhost:4318"; } } diff --git a/src/api/Elastic.Documentation.Api.Core/Telemetry/README.md b/src/api/Elastic.Documentation.Api.Core/Telemetry/README.md deleted file mode 100644 index 980c3c6f4..000000000 --- a/src/api/Elastic.Documentation.Api.Core/Telemetry/README.md +++ /dev/null @@ -1,276 +0,0 @@ -# OTLP Proxy for Frontend Telemetry - -This OTLP (OpenTelemetry Protocol) proxy allows frontend JavaScript code to send telemetry (logs, traces, metrics) to the same OTLP collector used by the backend **without exposing authentication credentials to the browser**. - -## Security Model - -### ✅ Secure: Backend handles authentication - -``` -Frontend (Browser) → API Proxy → OTLP Collector (Elastic APM/OTel) - ↑ - Adds auth headers - (credentials stay secure) -``` - -The proxy: -- Reads credentials from environment variables on the backend -- Automatically adds authentication headers to forwarded requests -- Prevents credential exposure to browser DevTools or network inspection - -### ❌ Insecure: Direct frontend connection - -``` -Frontend (Browser) → OTLP Collector - ↑ - Requires auth credentials - (exposed in browser code) -``` - -## Configuration - -The proxy uses standard OpenTelemetry environment variables: - -```bash -# Required: OTLP collector endpoint -OTEL_EXPORTER_OTLP_ENDPOINT=https://your-apm-server.elastic.co:443 - -# Optional: Authentication headers (multiple headers separated by comma) -OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer secret-token" - -# Or for Elastic APM with API Key: -OTEL_EXPORTER_OTLP_HEADERS="Authorization=ApiKey base64-encoded-api-key" - -# Or multiple headers: -OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer token,X-Custom-Header=value" -``` - -## API Endpoints - -The proxy provides three endpoints matching the OTLP specification: - -``` -POST /docs/_api/v1/otlp/v1/traces - Forward trace spans -POST /docs/_api/v1/otlp/v1/logs - Forward log records -POST /docs/_api/v1/otlp/v1/metrics - Forward metrics -``` - -### Content Types Supported - -- `application/json` - OTLP JSON encoding (recommended for browser) -- `application/x-protobuf` - OTLP protobuf encoding (smaller but requires encoding) - -## Frontend Usage - -### Option 1: Using OpenTelemetry JS SDK (Recommended) - -```typescript -import { WebTracerProvider } from '@opentelemetry/sdk-trace-web'; -import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; -import { Resource } from '@opentelemetry/resources'; -import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; - -// Configure the tracer to use the proxy endpoint -const provider = new WebTracerProvider({ - resource: new Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: 'docs-frontend', - [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0', - }), -}); - -// Point the exporter to the proxy endpoint (no credentials needed!) -const exporter = new OTLPTraceExporter({ - url: 'https://docs.elastic.co/_api/v1/otlp/v1/traces', - headers: {}, // No auth headers needed - proxy handles it -}); - -provider.addSpanProcessor(new BatchSpanProcessor(exporter)); -provider.register(); - -// Now you can create spans -const tracer = provider.getTracer('docs-frontend'); -const span = tracer.startSpan('page-load'); -span.end(); -``` - -### Option 2: Using OpenTelemetry Logs API - -```typescript -import { LoggerProvider } from '@opentelemetry/sdk-logs'; -import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http'; - -const loggerProvider = new LoggerProvider({ - resource: new Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: 'docs-frontend', - }), -}); - -const exporter = new OTLPLogExporter({ - url: 'https://docs.elastic.co/_api/v1/otlp/v1/logs', -}); - -loggerProvider.addLogRecordProcessor(new BatchLogRecordProcessor(exporter)); - -const logger = loggerProvider.getLogger('docs-frontend'); -logger.emit({ - severityNumber: 9, - severityText: 'INFO', - body: 'User clicked button', - attributes: { - 'user.action': 'click', - 'button.id': 'submit', - }, -}); -``` - -### Option 3: Manual Fetch (for debugging) - -```typescript -// Send logs manually -await fetch('https://docs.elastic.co/_api/v1/otlp/v1/logs', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - resourceLogs: [ - { - resource: { - attributes: [ - { key: 'service.name', value: { stringValue: 'docs-frontend' } }, - ], - }, - scopeLogs: [ - { - logRecords: [ - { - timeUnixNano: String(Date.now() * 1000000), - severityNumber: 9, - severityText: 'INFO', - body: { - stringValue: 'Test log from browser', - }, - attributes: [ - { key: 'page.url', value: { stringValue: window.location.href } }, - ], - }, - ], - }, - ], - }, - ], - }), -}); -``` - -## CORS Configuration - -If your frontend is served from a different domain, you'll need to configure CORS: - -```csharp -// In Program.cs or startup configuration -app.UseCors(policy => policy - .WithOrigins("https://docs.elastic.co") - .AllowAnyMethod() - .AllowAnyHeader()); -``` - -## Monitoring the Proxy - -The proxy creates its own spans under the `Elastic.Documentation.Api.OtlpProxy` activity source. - -Each proxied request includes: -- `otel.signal_type` - The signal type (traces/logs/metrics) -- `otel.content_type` - The content type of the request -- `otel.target_url` - The target OTLP collector URL -- `http.response.status_code` - Response status from collector - -## Example: Full Frontend Integration - -```typescript -// frontend/telemetry.ts -import { WebTracerProvider } from '@opentelemetry/sdk-trace-web'; -import { ZoneContextManager } from '@opentelemetry/context-zone'; -import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; -import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; -import { DocumentLoadInstrumentation } from '@opentelemetry/instrumentation-document-load'; -import { UserInteractionInstrumentation } from '@opentelemetry/instrumentation-user-interaction'; -import { registerInstrumentations } from '@opentelemetry/instrumentation'; - -export function initTelemetry() { - const provider = new WebTracerProvider({ - resource: new Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: 'docs-frontend', - [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: - window.location.hostname.includes('localhost') ? 'dev' : 'prod', - }), - }); - - // Use the proxy endpoint - no credentials needed! - const exporter = new OTLPTraceExporter({ - url: `${window.location.origin}/_api/v1/otlp/v1/traces`, - }); - - provider.addSpanProcessor(new BatchSpanProcessor(exporter, { - maxQueueSize: 100, - scheduledDelayMillis: 5000, - })); - - provider.register({ - contextManager: new ZoneContextManager(), - }); - - // Auto-instrument page loads and user interactions - registerInstrumentations({ - instrumentations: [ - new DocumentLoadInstrumentation(), - new UserInteractionInstrumentation(), - ], - }); - - console.log('OpenTelemetry initialized with proxy'); -} -``` - -## Troubleshooting - -### Proxy returns 503 "OTLP proxy is not configured" - -The `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable is not set. Configure it in your Lambda environment variables. - -### Proxy returns 401/403 from collector - -The authentication headers in `OTEL_EXPORTER_OTLP_HEADERS` are invalid or expired. - -### Frontend gets CORS errors - -Add CORS configuration to allow requests from your frontend domain. - -### Data not appearing in Elastic APM - -1. Check the proxy logs for errors -2. Verify the OTLP endpoint is correct -3. Ensure the collector is configured to accept OTLP/HTTP on `/v1/traces`, `/v1/logs`, `/v1/metrics` -4. Check that your OTLP payload format is correct (use the OpenTelemetry SDK to avoid formatting errors) - -## Performance Considerations - -- The proxy uses streaming to avoid buffering large payloads in memory -- Batch telemetry in the frontend before sending (use `BatchSpanProcessor` and `BatchLogRecordProcessor`) -- Consider sampling high-volume traces in production -- Monitor proxy latency via the `Elastic.Documentation.Api.OtlpProxy` spans - -## Security Best Practices - -✅ **DO:** -- Use HTTPS for the proxy endpoint in production -- Set appropriate rate limits on the proxy endpoint -- Monitor for unusual traffic patterns -- Use resource attributes to identify the frontend service - -❌ **DON'T:** -- Expose OTLP collector credentials in frontend code -- Allow unauthenticated access to the collector directly -- Send PII (personally identifiable information) in telemetry without user consent -- Forget to configure CORS for cross-origin requests - From 1abb9bd13d9de2fdefa875309f2457c3d06a22b9 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Thu, 20 Nov 2025 22:31:30 +0100 Subject: [PATCH 3/5] Refactor --- .../Elastic.Documentation.Api.Core.csproj | 1 + .../Telemetry/IOtlpGateway.cs | 4 +- .../Telemetry/OtlpProxyRequest.cs | 30 +++- .../Telemetry/OtlpProxyUsecase.cs | 16 +- .../Adapters/Telemetry/AdotOtlpGateway.cs | 18 +-- .../MappingsExtensions.cs | 6 +- .../ServicesExtension.cs | 8 + .../Examples/ServiceMockingExampleTests.cs | 4 +- .../Fixtures/ApiWebApplicationFactory.cs | 8 + .../OtlpProxyIntegrationTests.cs | 149 ++++++++++-------- 10 files changed, 145 insertions(+), 99 deletions(-) diff --git a/src/api/Elastic.Documentation.Api.Core/Elastic.Documentation.Api.Core.csproj b/src/api/Elastic.Documentation.Api.Core/Elastic.Documentation.Api.Core.csproj index a1770fca2..366861512 100644 --- a/src/api/Elastic.Documentation.Api.Core/Elastic.Documentation.Api.Core.csproj +++ b/src/api/Elastic.Documentation.Api.Core/Elastic.Documentation.Api.Core.csproj @@ -12,6 +12,7 @@ + diff --git a/src/api/Elastic.Documentation.Api.Core/Telemetry/IOtlpGateway.cs b/src/api/Elastic.Documentation.Api.Core/Telemetry/IOtlpGateway.cs index 9591bb53c..ae2518af2 100644 --- a/src/api/Elastic.Documentation.Api.Core/Telemetry/IOtlpGateway.cs +++ b/src/api/Elastic.Documentation.Api.Core/Telemetry/IOtlpGateway.cs @@ -12,13 +12,13 @@ public interface IOtlpGateway /// /// Forwards OTLP telemetry data to the collector. /// - /// The OTLP signal type: traces, logs, or metrics + /// The OTLP signal type (traces, logs, or metrics) /// The raw OTLP payload stream /// Content-Type of the payload /// Cancellation token /// HTTP status code and response content Task<(int StatusCode, string? Content)> ForwardOtlp( - string signalType, + OtlpSignalType signalType, Stream requestBody, string contentType, Cancel ctx = default); diff --git a/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyRequest.cs b/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyRequest.cs index 90d45c093..6b79762cc 100644 --- a/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyRequest.cs +++ b/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyRequest.cs @@ -2,8 +2,37 @@ // Elasticsearch B.V licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information +using System.ComponentModel.DataAnnotations; +using NetEscapades.EnumGenerators; + namespace Elastic.Documentation.Api.Core.Telemetry; +/// +/// OTLP signal types supported by the proxy. +/// The Display names match the OTLP path segments (lowercase). +/// +[EnumExtensions] +public enum OtlpSignalType +{ + /// + /// Distributed traces - maps to /v1/traces + /// + [Display(Name = "traces")] + Traces, + + /// + /// Log records - maps to /v1/logs + /// + [Display(Name = "logs")] + Logs, + + /// + /// Metrics data - maps to /v1/metrics + /// + [Display(Name = "metrics")] + Metrics +} + /// /// Request model for OTLP proxy endpoint. /// Accepts raw OTLP payload from frontend and forwards to configured OTLP endpoint. @@ -15,4 +44,3 @@ public class OtlpProxyRequest /// public required string SignalType { get; init; } } - diff --git a/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyUsecase.cs b/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyUsecase.cs index 4c8c8f2f7..a0eb1fb9f 100644 --- a/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyUsecase.cs +++ b/src/api/Elastic.Documentation.Api.Core/Telemetry/OtlpProxyUsecase.cs @@ -3,7 +3,6 @@ // See the LICENSE file in the project root for more information using System.Diagnostics; -using Microsoft.Extensions.Logging; namespace Elastic.Documentation.Api.Core.Telemetry; @@ -11,35 +10,26 @@ namespace Elastic.Documentation.Api.Core.Telemetry; /// Proxies OTLP telemetry from the frontend to the local ADOT Lambda Layer collector. /// The ADOT layer handles authentication and forwarding to the backend. /// -public class OtlpProxyUsecase( - IOtlpGateway gateway, - ILogger logger) +public class OtlpProxyUsecase(IOtlpGateway gateway) { private static readonly ActivitySource ActivitySource = new(TelemetryConstants.OtlpProxySourceName); /// /// Proxies OTLP data from the frontend to the local ADOT collector. /// - /// The OTLP signal type: traces, logs, or metrics + /// The OTLP signal type (traces, logs, or metrics) /// The raw OTLP payload (JSON or protobuf) /// Content-Type header from the original request /// Cancellation token /// HTTP status code and response content public async Task<(int StatusCode, string? Content)> ProxyOtlp( - string signalType, + OtlpSignalType signalType, Stream requestBody, string contentType, Cancel ctx = default) { using var activity = ActivitySource.StartActivity("ProxyOtlp", ActivityKind.Client); - // Validate signal type - if (signalType is not ("traces" or "logs" or "metrics")) - { - logger.LogWarning("Invalid OTLP signal type: {SignalType}", signalType); - return (400, $"Invalid signal type: {signalType}. Must be traces, logs, or metrics"); - } - // Forward to gateway return await gateway.ForwardOtlp(signalType, requestBody, contentType, ctx); } diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Telemetry/AdotOtlpGateway.cs b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Telemetry/AdotOtlpGateway.cs index f045528ca..600af78e5 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Telemetry/AdotOtlpGateway.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/Telemetry/AdotOtlpGateway.cs @@ -11,17 +11,16 @@ namespace Elastic.Documentation.Api.Infrastructure.Adapters.Telemetry; /// Gateway that forwards OTLP telemetry to the ADOT Lambda Layer collector. /// public class AdotOtlpGateway( + IHttpClientFactory httpClientFactory, OtlpProxyOptions options, ILogger logger) : IOtlpGateway { - private static readonly HttpClient HttpClient = new() - { - Timeout = TimeSpan.FromSeconds(30) - }; + public const string HttpClientName = "OtlpProxy"; + private readonly HttpClient _httpClient = httpClientFactory.CreateClient(HttpClientName); /// public async Task<(int StatusCode, string? Content)> ForwardOtlp( - string signalType, + OtlpSignalType signalType, Stream requestBody, string contentType, Cancel ctx = default) @@ -29,21 +28,22 @@ public class AdotOtlpGateway( try { // Build the target URL: http://localhost:4318/v1/{signalType} - var targetUrl = $"{options.Endpoint.TrimEnd('/')}/v1/{signalType}"; + // Use ToStringFast(true) from generated enum extensions (returns Display name: "traces", "logs", "metrics") + var targetUrl = $"{options.Endpoint.TrimEnd('/')}/v1/{signalType.ToStringFast(true)}"; logger.LogDebug("Forwarding OTLP {SignalType} to ADOT collector at {TargetUrl}", signalType, targetUrl); using var request = new HttpRequestMessage(HttpMethod.Post, targetUrl); - // Forward the content + // Forward the content with the original content type request.Content = new StreamContent(requestBody); - request.Content.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType); + _ = request.Content.Headers.TryAddWithoutValidation("Content-Type", contentType); // No need to add authentication headers - ADOT layer handles auth to backend // Just forward the telemetry to the local collector // Forward to ADOT collector - using var response = await HttpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, ctx); + using var response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, ctx); var responseContent = response.Content.Headers.ContentLength > 0 ? await response.Content.ReadAsStringAsync(ctx) : string.Empty; diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/MappingsExtensions.cs b/src/api/Elastic.Documentation.Api.Infrastructure/MappingsExtensions.cs index ce7724699..1c605e4a4 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/MappingsExtensions.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/MappingsExtensions.cs @@ -69,7 +69,7 @@ private static void MapOtlpProxyEndpoint(IEndpointRouteBuilder group) async (HttpContext context, OtlpProxyUsecase proxyUsecase, Cancel ctx) => { var contentType = context.Request.ContentType ?? "application/json"; - var (statusCode, content) = await proxyUsecase.ProxyOtlp("traces", context.Request.Body, contentType, ctx); + var (statusCode, content) = await proxyUsecase.ProxyOtlp(OtlpSignalType.Traces, context.Request.Body, contentType, ctx); return Results.Content(content ?? string.Empty, contentType, statusCode: statusCode); }) .DisableAntiforgery(); // Frontend requests won't have antiforgery tokens @@ -80,7 +80,7 @@ private static void MapOtlpProxyEndpoint(IEndpointRouteBuilder group) async (HttpContext context, OtlpProxyUsecase proxyUsecase, Cancel ctx) => { var contentType = context.Request.ContentType ?? "application/json"; - var (statusCode, content) = await proxyUsecase.ProxyOtlp("logs", context.Request.Body, contentType, ctx); + var (statusCode, content) = await proxyUsecase.ProxyOtlp(OtlpSignalType.Logs, context.Request.Body, contentType, ctx); return Results.Content(content ?? string.Empty, contentType, statusCode: statusCode); }) .DisableAntiforgery(); @@ -91,7 +91,7 @@ private static void MapOtlpProxyEndpoint(IEndpointRouteBuilder group) async (HttpContext context, OtlpProxyUsecase proxyUsecase, Cancel ctx) => { var contentType = context.Request.ContentType ?? "application/json"; - var (statusCode, content) = await proxyUsecase.ProxyOtlp("metrics", context.Request.Body, contentType, ctx); + var (statusCode, content) = await proxyUsecase.ProxyOtlp(OtlpSignalType.Metrics, context.Request.Body, contentType, ctx); return Results.Content(content ?? string.Empty, contentType, statusCode: statusCode); }) .DisableAntiforgery(); diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs b/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs index 416035992..84c761922 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/ServicesExtension.cs @@ -186,6 +186,14 @@ private static void AddOtlpProxyUsecase(IServiceCollection services, AppEnv appE var config = sp.GetRequiredService(); return new OtlpProxyOptions(config); }); + + // Register named HttpClient for OTLP proxy + _ = services.AddHttpClient(AdotOtlpGateway.HttpClientName) + .ConfigureHttpClient(client => + { + client.Timeout = TimeSpan.FromSeconds(30); + }); + _ = services.AddScoped(); _ = services.AddScoped(); logger?.LogInformation("OTLP proxy configured to forward to ADOT Lambda Layer collector"); diff --git a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Examples/ServiceMockingExampleTests.cs b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Examples/ServiceMockingExampleTests.cs index e27fd61d3..fb22503a4 100644 --- a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Examples/ServiceMockingExampleTests.cs +++ b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Examples/ServiceMockingExampleTests.cs @@ -27,7 +27,7 @@ public async Task ExampleWithMultipleServiceMocks() // Configure mock behaviors A.CallTo(() => mockOtlpGateway.ForwardOtlp( - A._, + A._, A._, A._, A._)) @@ -80,7 +80,7 @@ public async Task ExampleWithSingletonMock() { // Arrange var mockGateway = A.Fake(); - A.CallTo(() => mockGateway.ForwardOtlp(A._, A._, A._, A._)) + A.CallTo(() => mockGateway.ForwardOtlp(A._, A._, A._, A._)) .Returns(Task.FromResult<(int, string?)>((503, "Unavailable"))); // Use ReplaceSingleton for services registered as singletons diff --git a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Fixtures/ApiWebApplicationFactory.cs b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Fixtures/ApiWebApplicationFactory.cs index 3e3139515..c39275ee1 100644 --- a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Fixtures/ApiWebApplicationFactory.cs +++ b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Fixtures/ApiWebApplicationFactory.cs @@ -51,6 +51,14 @@ public static ApiWebApplicationFactory WithMockedServices(Action + /// Creates a factory with custom service configuration. + /// + /// Action to configure services directly + /// New factory instance with custom service configuration + public static ApiWebApplicationFactory WithMockedServices(Action configureServices) + => new(configureServices); + protected override void ConfigureWebHost(IWebHostBuilder builder) => builder.ConfigureServices(services => { var otelBuilder = services.AddOpenTelemetry(); diff --git a/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs b/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs index ed31c714d..e4a22ad2e 100644 --- a/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs +++ b/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs @@ -4,10 +4,11 @@ using System.Net; using System.Text; -using Elastic.Documentation.Api.Core.Telemetry; +using Elastic.Documentation.Api.Infrastructure.Adapters.Telemetry; using Elastic.Documentation.Api.IntegrationTests.Fixtures; using FakeItEasy; using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; using Xunit; namespace Elastic.Documentation.Api.IntegrationTests; @@ -15,21 +16,26 @@ namespace Elastic.Documentation.Api.IntegrationTests; public class OtlpProxyIntegrationTests { [Fact] - public async Task OtlpProxyTracesEndpointReturnsSuccess() + public async Task OtlpProxyTracesEndpointForwardsToCorrectUrl() { // Arrange - var mockGateway = A.Fake(); - A.CallTo(() => mockGateway.ForwardOtlp( - A._, - A._, - A._, - A._)) - .Returns(Task.FromResult<(int StatusCode, string? Content)>((200, "{}"))); + var mockHandler = A.Fake(); + var capturedRequest = (HttpRequestMessage?)null; + + A.CallTo(mockHandler) + .Where(call => call.Method.Name == "SendAsync") + .WithReturnType>() + .Invokes((HttpRequestMessage req, CancellationToken ct) => capturedRequest = req) + .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{}") })); using var factory = ApiWebApplicationFactory.WithMockedServices(services => - services.Replace(mockGateway)); - using var client = factory.CreateClient(); + { + // Replace the named HttpClient with our mock + _ = services.AddHttpClient(AdotOtlpGateway.HttpClientName) + .ConfigurePrimaryHttpMessageHandler(() => mockHandler); + }); + var client = factory.CreateClient(); var otlpPayload = /*lang=json,strict*/ """ { "resourceSpans": [{ @@ -48,34 +54,42 @@ public async Task OtlpProxyTracesEndpointReturnsSuccess() // Act var response = await client.PostAsync("/docs/_api/v1/o/t", content, TestContext.Current.CancellationToken); - // Assert - response.StatusCode.Should().Be(HttpStatusCode.OK); + // Assert - verify the request was forwarded to the correct URL + if (!response.IsSuccessStatusCode) + { + var errorBody = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); + throw new Exception($"Test failed with {response.StatusCode}: {errorBody}"); + } - // Verify gateway was called - A.CallTo(() => mockGateway.ForwardOtlp( - "traces", - A._, - A._, - A._)) - .MustHaveHappenedOnceExactly(); + response.StatusCode.Should().Be(HttpStatusCode.OK); + capturedRequest.Should().NotBeNull(); + capturedRequest!.RequestUri.Should().NotBeNull(); + capturedRequest.RequestUri!.ToString().Should().Be("http://localhost:4318/v1/traces"); + capturedRequest.Method.Should().Be(HttpMethod.Post); + capturedRequest.Content.Should().NotBeNull(); + capturedRequest.Content!.Headers.ContentType!.MediaType.Should().Be("application/json"); } [Fact] - public async Task OtlpProxyLogsEndpointReturnsSuccess() + public async Task OtlpProxyLogsEndpointForwardsToCorrectUrl() { // Arrange - var mockGateway = A.Fake(); - A.CallTo(() => mockGateway.ForwardOtlp( - A._, - A._, - A._, - A._)) - .Returns(Task.FromResult<(int StatusCode, string? Content)>((200, "{}"))); + var mockHandler = A.Fake(); + var capturedRequest = (HttpRequestMessage?)null; + + A.CallTo(mockHandler) + .Where(call => call.Method.Name == "SendAsync") + .WithReturnType>() + .Invokes((HttpRequestMessage req, CancellationToken ct) => capturedRequest = req) + .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{}") })); using var factory = ApiWebApplicationFactory.WithMockedServices(services => - services.Replace(mockGateway)); - using var client = factory.CreateClient(); + { + _ = services.AddHttpClient(AdotOtlpGateway.HttpClientName) + .ConfigurePrimaryHttpMessageHandler(() => mockHandler); + }); + var client = factory.CreateClient(); var otlpPayload = /*lang=json,strict*/ """ { "resourceLogs": [{ @@ -97,34 +111,32 @@ public async Task OtlpProxyLogsEndpointReturnsSuccess() // Act var response = await client.PostAsync("/docs/_api/v1/o/l", content, TestContext.Current.CancellationToken); - // Assert + // Assert - verify the enum ToStringFast() generates "logs" (lowercase) response.StatusCode.Should().Be(HttpStatusCode.OK); - - // Verify gateway was called - A.CallTo(() => mockGateway.ForwardOtlp( - "logs", - A._, - A._, - A._)) - .MustHaveHappenedOnceExactly(); + capturedRequest.Should().NotBeNull(); + capturedRequest!.RequestUri!.ToString().Should().Be("http://localhost:4318/v1/logs"); } [Fact] - public async Task OtlpProxyMetricsEndpointReturnsSuccess() + public async Task OtlpProxyMetricsEndpointForwardsToCorrectUrl() { // Arrange - var mockGateway = A.Fake(); - A.CallTo(() => mockGateway.ForwardOtlp( - A._, - A._, - A._, - A._)) - .Returns(Task.FromResult<(int StatusCode, string? Content)>((200, "{}"))); + var mockHandler = A.Fake(); + var capturedRequest = (HttpRequestMessage?)null; + + A.CallTo(mockHandler) + .Where(call => call.Method.Name == "SendAsync") + .WithReturnType>() + .Invokes((HttpRequestMessage req, CancellationToken ct) => capturedRequest = req) + .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{}") })); using var factory = ApiWebApplicationFactory.WithMockedServices(services => - services.Replace(mockGateway)); - using var client = factory.CreateClient(); + { + _ = services.AddHttpClient(AdotOtlpGateway.HttpClientName) + .ConfigurePrimaryHttpMessageHandler(() => mockHandler); + }); + var client = factory.CreateClient(); var otlpPayload = /*lang=json,strict*/ """ { "resourceMetrics": [{ @@ -142,40 +154,39 @@ public async Task OtlpProxyMetricsEndpointReturnsSuccess() // Act var response = await client.PostAsync("/docs/_api/v1/o/m", content, TestContext.Current.CancellationToken); - // Assert + // Assert - verify the enum ToStringFast() generates "metrics" (lowercase) response.StatusCode.Should().Be(HttpStatusCode.OK); - - // Verify gateway was called - A.CallTo(() => mockGateway.ForwardOtlp( - "metrics", - A._, - A._, - A._)) - .MustHaveHappenedOnceExactly(); + capturedRequest.Should().NotBeNull(); + capturedRequest!.RequestUri!.ToString().Should().Be("http://localhost:4318/v1/metrics"); } [Fact] - public async Task OtlpProxyReturnsGatewayErrorStatusCode() + public async Task OtlpProxyReturnsCollectorErrorStatusCode() { // Arrange - var mockGateway = A.Fake(); - A.CallTo(() => mockGateway.ForwardOtlp( - A._, - A._, - A._, - A._)) - .Returns(Task.FromResult<(int StatusCode, string? Content)>((503, "Service unavailable"))); + var mockHandler = A.Fake(); + + A.CallTo(mockHandler) + .Where(call => call.Method.Name == "SendAsync") + .WithReturnType>() + .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable) + { + Content = new StringContent("Service unavailable") + })); using var factory = ApiWebApplicationFactory.WithMockedServices(services => - services.Replace(mockGateway)); - using var client = factory.CreateClient(); + { + _ = services.AddHttpClient(AdotOtlpGateway.HttpClientName) + .ConfigurePrimaryHttpMessageHandler(() => mockHandler); + }); + var client = factory.CreateClient(); var content = new StringContent("{}", Encoding.UTF8, "application/json"); // Act var response = await client.PostAsync("/docs/_api/v1/o/t", content, TestContext.Current.CancellationToken); - // Assert + // Assert - verify error responses are properly forwarded response.StatusCode.Should().Be(HttpStatusCode.ServiceUnavailable); var responseBody = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); responseBody.Should().Contain("Service unavailable"); From 006b09629ee645f3df62aa5aeb06e231f7a4bdf0 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Thu, 20 Nov 2025 22:33:32 +0100 Subject: [PATCH 4/5] Fix CodeQL --- .../OtlpProxyIntegrationTests.cs | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs b/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs index e4a22ad2e..0d9b2abdb 100644 --- a/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs +++ b/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs @@ -26,7 +26,10 @@ public async Task OtlpProxyTracesEndpointForwardsToCorrectUrl() .Where(call => call.Method.Name == "SendAsync") .WithReturnType>() .Invokes((HttpRequestMessage req, CancellationToken ct) => capturedRequest = req) - .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{}") })); + .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{}") + })); using var factory = ApiWebApplicationFactory.WithMockedServices(services => { @@ -49,10 +52,11 @@ public async Task OtlpProxyTracesEndpointForwardsToCorrectUrl() }] } """; - var content = new StringContent(otlpPayload, Encoding.UTF8, "application/json"); + + using var content = new StringContent(otlpPayload, Encoding.UTF8, "application/json"); // Act - var response = await client.PostAsync("/docs/_api/v1/o/t", content, TestContext.Current.CancellationToken); + using var response = await client.PostAsync("/docs/_api/v1/o/t", content, TestContext.Current.CancellationToken); // Assert - verify the request was forwarded to the correct URL if (!response.IsSuccessStatusCode) @@ -81,7 +85,10 @@ public async Task OtlpProxyLogsEndpointForwardsToCorrectUrl() .Where(call => call.Method.Name == "SendAsync") .WithReturnType>() .Invokes((HttpRequestMessage req, CancellationToken ct) => capturedRequest = req) - .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{}") })); + .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{}") + })); using var factory = ApiWebApplicationFactory.WithMockedServices(services => { @@ -106,10 +113,11 @@ public async Task OtlpProxyLogsEndpointForwardsToCorrectUrl() }] } """; - var content = new StringContent(otlpPayload, Encoding.UTF8, "application/json"); + + using var content = new StringContent(otlpPayload, Encoding.UTF8, "application/json"); // Act - var response = await client.PostAsync("/docs/_api/v1/o/l", content, TestContext.Current.CancellationToken); + using var response = await client.PostAsync("/docs/_api/v1/o/l", content, TestContext.Current.CancellationToken); // Assert - verify the enum ToStringFast() generates "logs" (lowercase) response.StatusCode.Should().Be(HttpStatusCode.OK); @@ -128,7 +136,10 @@ public async Task OtlpProxyMetricsEndpointForwardsToCorrectUrl() .Where(call => call.Method.Name == "SendAsync") .WithReturnType>() .Invokes((HttpRequestMessage req, CancellationToken ct) => capturedRequest = req) - .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{}") })); + .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{}") + })); using var factory = ApiWebApplicationFactory.WithMockedServices(services => { @@ -149,10 +160,11 @@ public async Task OtlpProxyMetricsEndpointForwardsToCorrectUrl() }] } """; - var content = new StringContent(otlpPayload, Encoding.UTF8, "application/json"); + + using var content = new StringContent(otlpPayload, Encoding.UTF8, "application/json"); // Act - var response = await client.PostAsync("/docs/_api/v1/o/m", content, TestContext.Current.CancellationToken); + using var response = await client.PostAsync("/docs/_api/v1/o/m", content, TestContext.Current.CancellationToken); // Assert - verify the enum ToStringFast() generates "metrics" (lowercase) response.StatusCode.Should().Be(HttpStatusCode.OK); @@ -181,10 +193,10 @@ public async Task OtlpProxyReturnsCollectorErrorStatusCode() }); var client = factory.CreateClient(); - var content = new StringContent("{}", Encoding.UTF8, "application/json"); + using var content = new StringContent("{}", Encoding.UTF8, "application/json"); // Act - var response = await client.PostAsync("/docs/_api/v1/o/t", content, TestContext.Current.CancellationToken); + using var response = await client.PostAsync("/docs/_api/v1/o/t", content, TestContext.Current.CancellationToken); // Assert - verify error responses are properly forwarded response.StatusCode.Should().Be(HttpStatusCode.ServiceUnavailable); @@ -198,10 +210,10 @@ public async Task OtlpProxyInvalidSignalTypeReturns404() // Arrange using var factory = new ApiWebApplicationFactory(); using var client = factory.CreateClient(); - var content = new StringContent("{}", Encoding.UTF8, "application/json"); + using var content = new StringContent("{}", Encoding.UTF8, "application/json"); // Act - use invalid signal type - var response = await client.PostAsync("/docs/_api/v1/o/invalid", content, TestContext.Current.CancellationToken); + using var response = await client.PostAsync("/docs/_api/v1/o/invalid", content, TestContext.Current.CancellationToken); // Assert - route doesn't exist response.StatusCode.Should().Be(HttpStatusCode.NotFound); From 05fbd7563e7465f37878eb230b8547c763b55945 Mon Sep 17 00:00:00 2001 From: Jan Calanog Date: Thu, 20 Nov 2025 22:48:17 +0100 Subject: [PATCH 5/5] Refactor and dispose disposable --- .../AskAi/AgentBuilderAskAiGateway.cs | 4 +- .../Adapters/AskAi/LlmGatewayAskAiGateway.cs | 4 +- .../EuidEnrichmentIntegrationTests.cs | 49 +++++++-- .../Examples/ServiceMockingExampleTests.cs | 95 ---------------- .../Fixtures/ApiWebApplicationFactory.cs | 101 ++++++------------ .../OtlpProxyIntegrationTests.cs | 56 +++++++--- 6 files changed, 117 insertions(+), 192 deletions(-) delete mode 100644 tests-integration/Elastic.Documentation.Api.IntegrationTests/Examples/ServiceMockingExampleTests.cs diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/AgentBuilderAskAiGateway.cs b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/AgentBuilderAskAiGateway.cs index cdf53b3c9..6748a40b1 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/AgentBuilderAskAiGateway.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/AgentBuilderAskAiGateway.cs @@ -37,7 +37,7 @@ public async Task AskAi(AskAiRequest askAiRequest, Cancel ctx = default) var kibanaUrl = await parameterProvider.GetParam("docs-kibana-url", false, ctx); var kibanaApiKey = await parameterProvider.GetParam("docs-kibana-apikey", true, ctx); - var request = new HttpRequestMessage(HttpMethod.Post, + using var request = new HttpRequestMessage(HttpMethod.Post, $"{kibanaUrl}/api/agent_builder/converse/async") { Content = new StringContent(requestBody, Encoding.UTF8, "application/json") @@ -45,7 +45,7 @@ public async Task AskAi(AskAiRequest askAiRequest, Cancel ctx = default) request.Headers.Add("kbn-xsrf", "true"); request.Headers.Authorization = new AuthenticationHeaderValue("ApiKey", kibanaApiKey); - var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx); + using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx); // Ensure the response is successful before streaming if (!response.IsSuccessStatusCode) diff --git a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/LlmGatewayAskAiGateway.cs b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/LlmGatewayAskAiGateway.cs index f7d1cdf70..64e3c72ca 100644 --- a/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/LlmGatewayAskAiGateway.cs +++ b/src/api/Elastic.Documentation.Api.Infrastructure/Adapters/AskAi/LlmGatewayAskAiGateway.cs @@ -25,7 +25,7 @@ public async Task AskAi(AskAiRequest askAiRequest, Cancel ctx = default) { var llmGatewayRequest = LlmGatewayRequest.CreateFromRequest(askAiRequest); var requestBody = JsonSerializer.Serialize(llmGatewayRequest, LlmGatewayContext.Default.LlmGatewayRequest); - var request = new HttpRequestMessage(HttpMethod.Post, options.FunctionUrl) + using var request = new HttpRequestMessage(HttpMethod.Post, options.FunctionUrl) { Content = new StringContent(requestBody, Encoding.UTF8, "application/json") }; @@ -37,7 +37,7 @@ public async Task AskAi(AskAiRequest askAiRequest, Cancel ctx = default) // Use HttpCompletionOption.ResponseHeadersRead to get headers immediately // This allows us to start streaming as soon as headers are received - var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx); + using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, ctx); // Ensure the response is successful before streaming if (!response.IsSuccessStatusCode) diff --git a/tests-integration/Elastic.Documentation.Api.IntegrationTests/EuidEnrichmentIntegrationTests.cs b/tests-integration/Elastic.Documentation.Api.IntegrationTests/EuidEnrichmentIntegrationTests.cs index 3920b3296..f2f4e1425 100644 --- a/tests-integration/Elastic.Documentation.Api.IntegrationTests/EuidEnrichmentIntegrationTests.cs +++ b/tests-integration/Elastic.Documentation.Api.IntegrationTests/EuidEnrichmentIntegrationTests.cs @@ -7,18 +7,18 @@ using Elastic.Documentation.Api.Core; using Elastic.Documentation.Api.Core.AskAi; using Elastic.Documentation.Api.IntegrationTests.Fixtures; +using FakeItEasy; using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; namespace Elastic.Documentation.Api.IntegrationTests; /// /// Integration tests for euid cookie enrichment in OpenTelemetry traces and logging. -/// Uses WebApplicationFactory to test the real API configuration with mocked services. +/// Uses WebApplicationFactory to test the real API configuration with mocked AskAi services. /// -public class EuidEnrichmentIntegrationTests(ApiWebApplicationFactory factory) : IClassFixture +public class EuidEnrichmentIntegrationTests { - private readonly ApiWebApplicationFactory _factory = factory; - /// /// Test that verifies euid cookie is added to both HTTP span and custom AskAi span, /// and appears in log entries - using the real API configuration. @@ -29,8 +29,39 @@ public async Task AskAiEndpointPropagatatesEuidToAllSpansAndLogs() // Arrange const string expectedEuid = "integration-test-euid-12345"; + // Track streams created by mocks so we can dispose them after the test + var mockStreams = new List(); + + // Create factory with mocked AskAi services + using var factory = ApiWebApplicationFactory.WithMockedServices(services => + { + // Mock IAskAiGateway to avoid external AI service calls + var mockAskAiGateway = A.Fake>(); + A.CallTo(() => mockAskAiGateway.AskAi(A._, A._)) + .ReturnsLazily(() => + { + var stream = new MemoryStream(Encoding.UTF8.GetBytes("data: test\n\n")); + mockStreams.Add(stream); + return Task.FromResult(stream); + }); + services.AddSingleton(mockAskAiGateway); + + // Mock IStreamTransformer + var mockTransformer = A.Fake(); + A.CallTo(() => mockTransformer.AgentProvider).Returns("test-provider"); + A.CallTo(() => mockTransformer.AgentId).Returns("test-agent"); + A.CallTo(() => mockTransformer.TransformAsync(A._, A._, A._, A._)) + .ReturnsLazily((Stream s, string? _, Activity? activity, Cancel _) => + { + // Dispose the activity if provided (simulating what the real transformer does) + activity?.Dispose(); + return Task.FromResult(s); + }); + services.AddSingleton(mockTransformer); + }); + // Create client - using var client = _factory.CreateClient(); + using var client = factory.CreateClient(); // Act - Make request to /ask-ai/stream with euid cookie using var request = new HttpRequestMessage(HttpMethod.Post, "/docs/_api/v1/ask-ai/stream"); @@ -48,7 +79,7 @@ public async Task AskAiEndpointPropagatatesEuidToAllSpansAndLogs() response.IsSuccessStatusCode.Should().BeTrue(); // Assert - Verify spans were captured - var activities = _factory.ExportedActivities; + var activities = factory.ExportedActivities; activities.Should().NotBeEmpty("OpenTelemetry should have captured activities"); // Verify HTTP span has euid @@ -67,7 +98,7 @@ public async Task AskAiEndpointPropagatatesEuidToAllSpansAndLogs() askAiEuidTag.Value.Should().Be(expectedEuid, "AskAi span euid should match cookie value"); // Assert - Verify logs have euid in attributes - var logRecords = _factory.ExportedLogRecords; + var logRecords = factory.ExportedLogRecords; logRecords.Should().NotBeEmpty("Should have captured log records"); // Find a log entry from AskAiUsecase @@ -80,5 +111,9 @@ public async Task AskAiEndpointPropagatatesEuidToAllSpansAndLogs() var euidAttribute = askAiLogRecord!.Attributes?.FirstOrDefault(a => a.Key == TelemetryConstants.UserEuidAttributeName) ?? default; euidAttribute.Should().NotBe(default(KeyValuePair), "Log record should include user.euid attribute"); (euidAttribute.Value?.ToString() ?? string.Empty).Should().Be(expectedEuid, "Log record euid should match cookie value"); + + // Cleanup - dispose all mock streams + foreach (var stream in mockStreams) + stream.Dispose(); } } diff --git a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Examples/ServiceMockingExampleTests.cs b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Examples/ServiceMockingExampleTests.cs deleted file mode 100644 index fb22503a4..000000000 --- a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Examples/ServiceMockingExampleTests.cs +++ /dev/null @@ -1,95 +0,0 @@ -// Licensed to Elasticsearch B.V under one or more agreements. -// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information - -using System.Net; -using Elastic.Documentation.Api.Core.Search; -using Elastic.Documentation.Api.Core.Telemetry; -using Elastic.Documentation.Api.IntegrationTests.Fixtures; -using FakeItEasy; -using FluentAssertions; -using Xunit; - -namespace Elastic.Documentation.Api.IntegrationTests.Examples; - -/// -/// Example test demonstrating how to mock multiple services in integration tests. -/// This serves as documentation for the service mocking pattern. -/// -public class ServiceMockingExampleTests -{ - [Fact] - public async Task ExampleWithMultipleServiceMocks() - { - // Arrange - Create multiple mocks - var mockOtlpGateway = A.Fake(); - var mockSearchGateway = A.Fake(); - - // Configure mock behaviors - A.CallTo(() => mockOtlpGateway.ForwardOtlp( - A._, - A._, - A._, - A._)) - .Returns(Task.FromResult<(int StatusCode, string? Content)>((200, "{}}"))); - - A.CallTo(() => mockSearchGateway.SearchAsync( - A._, - A._, - A._, - A._)) - .Returns(Task.FromResult((TotalHits: 1, Results: new List - { - new() - { - Type = "page", - Url = "/docs/test", - Title = "Test Result", - Description = "A test result", - Parents = [] - } - }))); - - // Create factory with multiple mocked services using fluent API - using var factory = ApiWebApplicationFactory.WithMockedServices(services => - services - .Replace(mockOtlpGateway) // Replace IOtlpGateway - .Replace(mockSearchGateway)); // Replace ISearchGateway - - using var client = factory.CreateClient(); - - // Act - Make a search request that uses the mocked search gateway - var searchResponse = await client.GetAsync( - "/docs/_api/v1/search?q=test&page=1&pageSize=5", - TestContext.Current.CancellationToken); - - // Assert - searchResponse.StatusCode.Should().Be(HttpStatusCode.OK); - - // Verify the search gateway was called with correct parameters - A.CallTo(() => mockSearchGateway.SearchAsync( - "test", // query - 1, // page - 5, // pageSize (default in API) - A._)) - .MustHaveHappenedOnceExactly(); - } - - [Fact] - public async Task ExampleWithSingletonMock() - { - // Arrange - var mockGateway = A.Fake(); - A.CallTo(() => mockGateway.ForwardOtlp(A._, A._, A._, A._)) - .Returns(Task.FromResult<(int, string?)>((503, "Unavailable"))); - - // Use ReplaceSingleton for services registered as singletons - using var factory = ApiWebApplicationFactory.WithMockedServices(services => - services.ReplaceSingleton(mockGateway)); - - using var client = factory.CreateClient(); - - // Act & Assert - // Your test logic here... - } -} diff --git a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Fixtures/ApiWebApplicationFactory.cs b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Fixtures/ApiWebApplicationFactory.cs index c39275ee1..f6483b21e 100644 --- a/tests-integration/Elastic.Documentation.Api.IntegrationTests/Fixtures/ApiWebApplicationFactory.cs +++ b/tests-integration/Elastic.Documentation.Api.IntegrationTests/Fixtures/ApiWebApplicationFactory.cs @@ -3,9 +3,6 @@ // See the LICENSE file in the project root for more information using System.Diagnostics; -using System.Text; -using Elastic.Documentation.Api.Core.AskAi; -using Elastic.Documentation.Api.Core.Telemetry; using Elastic.Documentation.Api.Infrastructure; using Elastic.Documentation.Api.Infrastructure.Aws; using Elastic.Documentation.Api.Infrastructure.OpenTelemetry; @@ -14,7 +11,6 @@ using Microsoft.AspNetCore.Mvc.Testing; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; -using Microsoft.Extensions.Logging; using OpenTelemetry; using OpenTelemetry.Logs; using OpenTelemetry.Trace; @@ -24,12 +20,13 @@ namespace Elastic.Documentation.Api.IntegrationTests.Fixtures; /// /// Custom WebApplicationFactory for testing the API with mocked services. /// This fixture can be reused across multiple test classes. +/// Only mocks services that ALL tests need (OpenTelemetry, AWS Parameters). +/// Test-specific mocks should be configured using WithMockedServices. /// public class ApiWebApplicationFactory : WebApplicationFactory { public List ExportedActivities { get; } = []; public List ExportedLogRecords { get; } = []; - private readonly List _mockMemoryStreams = []; private readonly Action? _configureServices; public ApiWebApplicationFactory() : this(null) @@ -60,67 +57,31 @@ public static ApiWebApplicationFactory WithMockedServices(Action new(configureServices); protected override void ConfigureWebHost(IWebHostBuilder builder) => builder.ConfigureServices(services => - { - var otelBuilder = services.AddOpenTelemetry(); - _ = otelBuilder.WithTracing(tracing => - { - _ = tracing - .AddDocsApiTracing() // Reuses production configuration - .AddInMemoryExporter(ExportedActivities); - }); - _ = otelBuilder.WithLogging(logging => - { - _ = logging - .AddDocsApiLogging() // Reuses production configuration - .AddInMemoryExporter(ExportedLogRecords); - }); - - // Mock IParameterProvider to avoid AWS dependencies - var mockParameterProvider = A.Fake(); - A.CallTo(() => mockParameterProvider.GetParam(A._, A._, A._)) - .Returns(Task.FromResult("mock-value")); - _ = services.AddSingleton(mockParameterProvider); - - // Mock IAskAiGateway to avoid external AI service calls - var mockAskAiGateway = A.Fake>(); - A.CallTo(() => mockAskAiGateway.AskAi(A._, A._)) - .ReturnsLazily(() => - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes("data: test\n\n")); - _mockMemoryStreams.Add(stream); - return Task.FromResult(stream); - }); - _ = services.AddSingleton(mockAskAiGateway); - - // Mock IStreamTransformer - var mockTransformer = A.Fake(); - A.CallTo(() => mockTransformer.AgentProvider).Returns("test-provider"); - A.CallTo(() => mockTransformer.AgentId).Returns("test-agent"); - A.CallTo(() => mockTransformer.TransformAsync(A._, A._, A._, A._)) - .ReturnsLazily((Stream s, string? _, Activity? activity, Cancel _) => - { - // Dispose the activity if provided (simulating what the real transformer does) - activity?.Dispose(); - return Task.FromResult(s); - }); - _ = services.AddSingleton(mockTransformer); - - // Allow tests to override services - RemoveAll + Add to properly replace - _configureServices?.Invoke(services); - }); - - protected override void Dispose(bool disposing) { - if (disposing) + // Configure OpenTelemetry with in-memory exporters for all tests + var otelBuilder = services.AddOpenTelemetry(); + _ = otelBuilder.WithTracing(tracing => { - foreach (var stream in _mockMemoryStreams) - { - stream.Dispose(); - } - _mockMemoryStreams.Clear(); - } - base.Dispose(disposing); - } + _ = tracing + .AddDocsApiTracing() // Reuses production configuration + .AddInMemoryExporter(ExportedActivities); + }); + _ = otelBuilder.WithLogging(logging => + { + _ = logging + .AddDocsApiLogging() // Reuses production configuration + .AddInMemoryExporter(ExportedLogRecords); + }); + + // Mock IParameterProvider to avoid AWS dependencies in all tests + var mockParameterProvider = A.Fake(); + A.CallTo(() => mockParameterProvider.GetParam(A._, A._, A._)) + .Returns(Task.FromResult("mock-value")); + _ = services.AddSingleton(mockParameterProvider); + + // Apply test-specific service replacements (if any) + _configureServices?.Invoke(services); + }); } /// @@ -183,10 +144,10 @@ public ServiceReplacementBuilder ReplaceSingleton(TService instance) w /// Builds the final service configuration action. /// internal Action Build() => services => - { - foreach (var replacement in _replacements) - { - replacement(services); - } - }; + { + foreach (var replacement in _replacements) + { + replacement(services); + } + }; } diff --git a/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs b/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs index 0d9b2abdb..bcecdab1d 100644 --- a/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs +++ b/tests-integration/Elastic.Documentation.Api.IntegrationTests/OtlpProxyIntegrationTests.cs @@ -22,14 +22,17 @@ public async Task OtlpProxyTracesEndpointForwardsToCorrectUrl() var mockHandler = A.Fake(); var capturedRequest = (HttpRequestMessage?)null; + // Create mock response (will be disposed by HttpClient) + var mockResponse = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{}") + }; + A.CallTo(mockHandler) .Where(call => call.Method.Name == "SendAsync") .WithReturnType>() .Invokes((HttpRequestMessage req, CancellationToken ct) => capturedRequest = req) - .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent("{}") - })); + .Returns(Task.FromResult(mockResponse)); using var factory = ApiWebApplicationFactory.WithMockedServices(services => { @@ -72,6 +75,9 @@ public async Task OtlpProxyTracesEndpointForwardsToCorrectUrl() capturedRequest.Method.Should().Be(HttpMethod.Post); capturedRequest.Content.Should().NotBeNull(); capturedRequest.Content!.Headers.ContentType!.MediaType.Should().Be("application/json"); + + // Cleanup mock response + mockResponse.Dispose(); } [Fact] @@ -81,14 +87,17 @@ public async Task OtlpProxyLogsEndpointForwardsToCorrectUrl() var mockHandler = A.Fake(); var capturedRequest = (HttpRequestMessage?)null; + // Create mock response (will be disposed by HttpClient) + var mockResponse = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{}") + }; + A.CallTo(mockHandler) .Where(call => call.Method.Name == "SendAsync") .WithReturnType>() .Invokes((HttpRequestMessage req, CancellationToken ct) => capturedRequest = req) - .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent("{}") - })); + .Returns(Task.FromResult(mockResponse)); using var factory = ApiWebApplicationFactory.WithMockedServices(services => { @@ -123,6 +132,9 @@ public async Task OtlpProxyLogsEndpointForwardsToCorrectUrl() response.StatusCode.Should().Be(HttpStatusCode.OK); capturedRequest.Should().NotBeNull(); capturedRequest!.RequestUri!.ToString().Should().Be("http://localhost:4318/v1/logs"); + + // Cleanup mock response + mockResponse.Dispose(); } [Fact] @@ -132,14 +144,17 @@ public async Task OtlpProxyMetricsEndpointForwardsToCorrectUrl() var mockHandler = A.Fake(); var capturedRequest = (HttpRequestMessage?)null; + // Create mock response (will be disposed by HttpClient) + var mockResponse = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{}") + }; + A.CallTo(mockHandler) .Where(call => call.Method.Name == "SendAsync") .WithReturnType>() .Invokes((HttpRequestMessage req, CancellationToken ct) => capturedRequest = req) - .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent("{}") - })); + .Returns(Task.FromResult(mockResponse)); using var factory = ApiWebApplicationFactory.WithMockedServices(services => { @@ -170,6 +185,9 @@ public async Task OtlpProxyMetricsEndpointForwardsToCorrectUrl() response.StatusCode.Should().Be(HttpStatusCode.OK); capturedRequest.Should().NotBeNull(); capturedRequest!.RequestUri!.ToString().Should().Be("http://localhost:4318/v1/metrics"); + + // Cleanup mock response + mockResponse.Dispose(); } [Fact] @@ -178,13 +196,16 @@ public async Task OtlpProxyReturnsCollectorErrorStatusCode() // Arrange var mockHandler = A.Fake(); + // Create mock response (will be disposed by HttpClient) + var mockResponse = new HttpResponseMessage(HttpStatusCode.ServiceUnavailable) + { + Content = new StringContent("Service unavailable") + }; + A.CallTo(mockHandler) .Where(call => call.Method.Name == "SendAsync") .WithReturnType>() - .Returns(Task.FromResult(new HttpResponseMessage(HttpStatusCode.ServiceUnavailable) - { - Content = new StringContent("Service unavailable") - })); + .Returns(Task.FromResult(mockResponse)); using var factory = ApiWebApplicationFactory.WithMockedServices(services => { @@ -202,6 +223,9 @@ public async Task OtlpProxyReturnsCollectorErrorStatusCode() response.StatusCode.Should().Be(HttpStatusCode.ServiceUnavailable); var responseBody = await response.Content.ReadAsStringAsync(TestContext.Current.CancellationToken); responseBody.Should().Contain("Service unavailable"); + + // Cleanup mock response + mockResponse.Dispose(); } [Fact]