Add OTLP proxy endpoint

Author: reakaleek

Directory.Packages.props

@@ -27,6 +27,7 @@
     <PackageVersion Include="AWSSDK.S3" Version="4.0.7.14" />
     <PackageVersion Include="Elastic.OpenTelemetry" Version="1.1.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration.UserSecrets" Version="10.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.Abstractions" Version="10.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Telemetry.Abstractions" Version="10.0.0" />
     <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.13.0" />
     <PackageVersion Include="Generator.Equals" Version="3.2.1" PrivateAssets="all" IncludeAssets="runtime; build; native; contentfiles; analyzers; buildtransitive" />
@@ -41,6 +42,7 @@
     <PackageVersion Include="Microsoft.OpenApi" Version="3.0.1" />
     <PackageVersion Include="TUnit" Version="0.25.21" />
     <PackageVersion Include="xunit.v3.extensibility.core" Version="2.0.2" />
+    <PackageVersion Include="WireMock.Net" Version="1.6.11" />
   </ItemGroup>
   <!-- Build -->
   <ItemGroup>
@@ -106,4 +108,4 @@
     </PackageVersion>
     <PackageVersion Include="xunit.v3" Version="2.0.2" />
   </ItemGroup>
-</Project>
+</Project>

src/api/Elastic.Documentation.Api.Core/Elastic.Documentation.Api.Core.csproj

@@ -9,6 +9,7 @@
     </PropertyGroup>
 
     <ItemGroup>
+      <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" />
       <PackageReference Include="Microsoft.Extensions.Logging" />
       <PackageReference Include="Microsoft.Extensions.Telemetry.Abstractions" />
     </ItemGroup>

src/api/Elastic.Documentation.Api.Core/Telemetry/ADOT_PROXY_GUIDE.md

@@ -0,0 +1,314 @@
+# OTLP Proxy for Frontend Telemetry with ADOT Lambda Layer
+
+This OTLP (OpenTelemetry Protocol) proxy allows frontend JavaScript code to send telemetry (logs, traces, metrics) through the ADOT (AWS Distro for OpenTelemetry) Lambda Layer **without exposing authentication credentials to the browser**.
+
+## Architecture with ADOT Lambda Layer
+
+```
+Frontend (Browser) → API Proxy → ADOT Lambda Layer (localhost:4318) → Elastic APM
+                                         ↑
+                                   Handles authentication
+                                   (credentials in env vars)
+```
+
+### Benefits
+
+1. **Secure**: Frontend never sees authentication credentials
+2. **Simple**: ADOT layer handles all backend authentication
+3. **Fast**: Local proxy (no external network hop in Lambda)
+4. **Standard**: Uses OpenTelemetry Protocol (OTLP) - works with any OTEL SDK
+
+## Lambda Configuration
+
+### Required Environment Variables
+
+```bash
+# Enable ADOT Lambda Layer
+AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument
+
+# Configure where ADOT forwards telemetry (your Elastic APM endpoint)
+OTEL_EXPORTER_OTLP_ENDPOINT=https://your-apm-server.elastic.co:443
+
+# Authentication for the backend (ADOT uses this, not exposed to frontend)
+OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer your-secret-token"
+# Or for Elastic Cloud:
+# OTEL_EXPORTER_OTLP_HEADERS="Authorization=ApiKey base64-api-key"
+
+# Service name for backend telemetry
+OTEL_SERVICE_NAME=docs-api
+
+# Resource attributes
+OTEL_RESOURCE_ATTRIBUTES="deployment.environment=prod,service.version=1.0.0"
+```
+
+### Lambda Layer ARN
+
+Add the ADOT Lambda Layer to your Lambda function. Find the latest ARN at:
+https://aws-otel.github.io/docs/getting-started/lambda/lambda-dotnet
+
+Example for us-east-1:
+```
+arn:aws:lambda:us-east-1:901920570463:layer:aws-otel-collector-amd64-ver-0-90-1:1
+```
+
+## API Endpoints
+
+The proxy provides three endpoints:
+
+```
+POST /_api/v1/otlp/v1/traces   - Forward trace spans
+POST /_api/v1/otlp/v1/logs     - Forward log records  
+POST /_api/v1/otlp/v1/metrics  - Forward metrics
+```
+
+##Frontend Usage
+
+### Quick Start with OpenTelemetry JS
+
+```typescript
+import { WebTracerProvider, BatchSpanProcessor } from '@opentelemetry/sdk-trace-web';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { Resource } from '@opentelemetry/resources';
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+
+// Configure tracer to use the proxy endpoint
+const provider = new WebTracerProvider({
+  resource: new Resource({
+    [SemanticResourceAttributes.SERVICE_NAME]: 'docs-frontend',
+    [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]: 
+      window.location.hostname.includes('localhost') ? 'dev' : 'prod',
+  }),
+});
+
+// Point to the proxy - no credentials needed!
+const exporter = new OTLPTraceExporter({
+  url: 'https://docs.elastic.co/_api/v1/otlp/v1/traces',
+  // No Authorization header needed - proxy handles it!
+});
+
+provider.addSpanProcessor(new BatchSpanProcessor(exporter));
+provider.register();
+
+// Now create spans
+const tracer = provider.getTracer('docs-frontend');
+const span = tracer.startSpan('page-load');
+span.setAttribute('page.url', window.location.href);
+span.end();
+```
+
+### Logs from Frontend
+
+```typescript
+import { LoggerProvider, BatchLogRecordProcessor } from '@opentelemetry/sdk-logs';
+import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
+
+const loggerProvider = new LoggerProvider({
+  resource: new Resource({
+    [SemanticResourceAttributes.SERVICE_NAME]: 'docs-frontend',
+  }),
+});
+
+const exporter = new OTLPLogExporter({
+  url: 'https://docs.elastic.co/_api/v1/otlp/v1/logs',
+});
+
+loggerProvider.addLogRecordProcessor(new BatchLogRecordProcessor(exporter));
+
+const logger = loggerProvider.getLogger('docs-frontend');
+
+// Log user actions
+logger.emit({
+  severityNumber: 9, // INFO
+  severityText: 'INFO',
+  body: 'User clicked search button',
+  attributes: {
+    'user.action': 'search',
+    'search.query': 'elasticsearch',
+  },
+});
+```
+
+### Complete Frontend Integration
+
+```typescript
+// telemetry.ts - Initialize once at app startup
+import { trace } from '@opentelemetry/api';
+import { WebTracerProvider, BatchSpanProcessor } from '@opentelemetry/sdk-trace-web';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { ZoneContextManager } from '@opentelemetry/context-zone';
+import { DocumentLoadInstrumentation } from '@opentelemetry/instrumentation-document-load';
+import { UserInteractionInstrumentation } from '@opentelemetry/instrumentation-user-interaction';
+import { registerInstrumentations } from '@opentelemetry/instrumentation';
+import { Resource } from '@opentelemetry/resources';
+import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+
+export function initTelemetry() {
+  const provider = new WebTracerProvider({
+    resource: new Resource({
+      [SemanticResourceAttributes.SERVICE_NAME]: 'docs-frontend',
+      [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0',
+      [SemanticResourceAttributes.DEPLOYMENT_ENVIRONMENT]:
+        window.location.hostname === 'docs.elastic.co' ? 'prod' : 'dev',
+    }),
+  });
+
+  // Use the proxy endpoint (same origin, no CORS issues!)
+  const exporter = new OTLPTraceExporter({
+    url: `${window.location.origin}/_api/v1/otlp/v1/traces`,
+  });
+
+  provider.addSpanProcessor(new BatchSpanProcessor(exporter, {
+    maxQueueSize: 100,
+    maxExportBatchSize: 10,
+    scheduledDelayMillis: 5000, // Send every 5 seconds
+  }));
+
+  provider.register({
+    contextManager: new ZoneContextManager(),
+  });
+
+  // Auto-instrument page loads and clicks
+  registerInstrumentations({
+    instrumentations: [
+      new DocumentLoadInstrumentation(),
+      new UserInteractionInstrumentation({
+        eventNames: ['click', 'submit'],
+      }),
+    ],
+  });
+
+  console.log('OpenTelemetry initialized via ADOT proxy');
+}
+
+// Call this in your app entry point
+initTelemetry();
+
+// Now you can manually create spans anywhere
+const tracer = trace.getTracer('docs-frontend');
+const span = tracer.startSpan('search-request');
+span.setAttribute('search.query', 'elasticsearch');
+span.end();
+```
+
+## How It Works
+
+### Request Flow
+
+1. **Frontend** sends OTLP JSON payload to `/_api/v1/otlp/v1/traces`
+2. **API Proxy** forwards to `http://localhost:4318/v1/traces` (ADOT layer)
+3. **ADOT Layer** adds authentication headers and forwards to Elastic APM
+4. **Elastic APM** receives and indexes the telemetry
+
+### Security
+
+- Frontend code **never** contains authentication credentials
+- Credentials live in Lambda environment variables (encrypted at rest)
+- ADOT layer handles all authentication to Elastic APM
+- Proxy acts as a secure gateway
+
+## Monitoring
+
+### Proxy Telemetry
+
+The proxy itself creates spans you can monitor:
+
+- **Activity Source**: `Elastic.Documentation.Api.OtlpProxy`
+- **Tags**:
+  - `otel.signal_type` - traces/logs/metrics
+  - `otel.content_type` - application/json or application/x-protobuf
+  - `otel.adot_endpoint` - http://localhost:4318
+  - `otel.target_url` - Full URL forwarded to
+  - `http.response.status_code` - Response from ADOT
+
+### CloudWatch Logs
+
+```bash
+# Search for proxy logs
+aws logs tail /aws/lambda/docs-api --filter "ProxyOtlp" --follow
+```
+
+## Troubleshooting
+
+### Connection refused to localhost:4318
+
+**Cause**: ADOT Lambda Layer is not enabled  
+**Solution**: Ensure `AWS_LAMBDA_EXEC_WRAPPER=/opt/otel-instrument` is set
+
+### Telemetry not appearing in Elastic APM
+
+1. Check ADOT layer logs in CloudWatch
+2. Verify `OTEL_EXPORTER_OTLP_ENDPOINT` points to your APM server
+3. Verify `OTEL_EXPORTER_OTLP_HEADERS` contains valid credentials
+4. Check APM server is reachable from Lambda (VPC/security groups)
+
+### CORS errors from frontend
+
+If your docs are served from a different domain than the API:
+
+```csharp
+// In Program.cs
+app.UseCors(policy => policy
+    .WithOrigins("https://docs.elastic.co")
+    .AllowAnyMethod()
+    .AllowAnyHeader());
+```
+
+### High Lambda costs
+
+- Use **BatchSpanProcessor** in frontend (batches multiple spans)
+- Set reasonable `scheduledDelayMillis` (5-10 seconds)
+- Consider sampling in production:
+
+```typescript
+import { TraceIdRatioBasedSampler } from '@opentelemetry/sdk-trace-base';
+
+const provider = new WebTracerProvider({
+  sampler: new TraceIdRatioBasedSampler(0.1), // Sample 10% of traces
+  // ...
+});
+```
+
+## Performance Considerations
+
+- **Batching**: Frontend batches telemetry before sending
+- **No buffering**: Proxy streams data (no memory overhead)
+- **Local forwarding**: ADOT on localhost:4318 (no network latency)
+- **Async**: ADOT forwards asynchronously to backend
+
+## Example: Track Search Queries
+
+```typescript
+import { trace } from '@opentelemetry/api';
+
+function handleSearch(query: string) {
+  const tracer = trace.getTracer('docs-frontend');
+  const span = tracer.startSpan('search', {
+    attributes: {
+      'search.query': query,
+      'search.source': 'search-box',
+    },
+  });
+
+  try {
+    // Perform search
+    const results = await fetch(`/api/search?q=${query}`);
+    span.setAttribute('search.results_count', results.length);
+    span.setStatus({ code: SpanStatusCode.OK });
+  } catch (error) {
+    span.setStatus({
+      code: SpanStatusCode.ERROR,
+      message: error.message,
+    });
+  } finally {
+    span.end();
+  }
+}
+```
+
+## Next Steps
+
+1. **Deploy**: Ensure ADOT layer is attached to your Lambda
+2. **Configure**: Set `AWS_LAMBDA_EXEC_WRAPPER` and OTEL environment variables  
+3. **Instrument**: Add OpenTelemetry SDK to your frontend
+4. **Test**: Send test telemetry from browser DevTools
+5. **Monitor**: Check Elastic APM for incoming frontend telemetry