diff --git a/opentelemetry-exporter-geneva/geneva-uploader-ffi/src/lib.rs b/opentelemetry-exporter-geneva/geneva-uploader-ffi/src/lib.rs index 5aa4a377..98772f4e 100644 --- a/opentelemetry-exporter-geneva/geneva-uploader-ffi/src/lib.rs +++ b/opentelemetry-exporter-geneva/geneva-uploader-ffi/src/lib.rs @@ -288,8 +288,21 @@ pub unsafe extern "C" fn geneva_client_new( // Auth method conversion let auth_method = match config.auth_method { - // 0 => Managed Identity (default to system-assigned when coming from FFI for now) - 0 => AuthMethod::SystemManagedIdentity, + 0 => { + // Unified: Workload Identity (AKS) or System Managed Identity (VM) + // Auto-detect based on environment + if std::env::var("AZURE_FEDERATED_TOKEN_FILE").is_ok() { + // Workload Identity: azure_identity crate reads AZURE_CLIENT_ID, AZURE_TENANT_ID, + // and AZURE_FEDERATED_TOKEN_FILE automatically from environment + let resource = std::env::var("GENEVA_WORKLOAD_IDENTITY_RESOURCE") + .unwrap_or_else(|_| "https://monitor.azure.com".to_string()); + + AuthMethod::WorkloadIdentity { resource } + } else { + AuthMethod::SystemManagedIdentity + } + } + 1 => { // Certificate authentication: read fields from tagged union let cert = unsafe { config.auth.cert }; diff --git a/opentelemetry-exporter-geneva/geneva-uploader/Cargo.toml b/opentelemetry-exporter-geneva/geneva-uploader/Cargo.toml index 9fb54e0b..729745d2 100644 --- a/opentelemetry-exporter-geneva/geneva-uploader/Cargo.toml +++ b/opentelemetry-exporter-geneva/geneva-uploader/Cargo.toml @@ -16,8 +16,9 @@ serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0", features = ["raw_value"] } uuid = { version = "1.0", features = ["v4"] } # TODO - support both native-tls and rustls -reqwest = { version = "0.12", features = ["native-tls", "native-tls-alpn"], default-features = false} -native-tls = "0.2" +# http2 feature is required by hyper-util even when using http1_only() +reqwest = { version = "0.12", features = ["native-tls", "native-tls-alpn", "http2"], default-features = false} +native-tls = "0.2" thiserror = "2.0" chrono = "0.4" url = "2.2" diff --git a/opentelemetry-exporter-geneva/geneva-uploader/src/client.rs b/opentelemetry-exporter-geneva/geneva-uploader/src/client.rs index f1b06b20..8994e06b 100644 --- a/opentelemetry-exporter-geneva/geneva-uploader/src/client.rs +++ b/opentelemetry-exporter-geneva/geneva-uploader/src/client.rs @@ -57,6 +57,7 @@ impl GenevaClient { } } AuthMethod::Certificate { .. } => {} + AuthMethod::WorkloadIdentity { .. } => {} #[cfg(feature = "mock_auth")] AuthMethod::MockAuth => {} } diff --git a/opentelemetry-exporter-geneva/geneva-uploader/src/config_service/client.rs b/opentelemetry-exporter-geneva/geneva-uploader/src/config_service/client.rs index 3f4053ad..55dd2124 100644 --- a/opentelemetry-exporter-geneva/geneva-uploader/src/config_service/client.rs +++ b/opentelemetry-exporter-geneva/geneva-uploader/src/config_service/client.rs @@ -1,4 +1,4 @@ -// Geneva Config Client with TLS (PKCS#12) and TODO: Managed Identity support +// Geneva Config Client with TLS (PKCS#12) and Azure Workload Identity support TODO: Azure Arc support use base64::{engine::general_purpose, Engine as _}; use reqwest::{ @@ -18,15 +18,20 @@ use std::fs; use std::path::PathBuf; use std::sync::RwLock; -// Azure Identity imports for MSI authentication +// Azure Identity imports for MSI and Workload Identity authentication use azure_core::credentials::TokenCredential; -use azure_identity::{ManagedIdentityCredential, ManagedIdentityCredentialOptions, UserAssignedId}; +use azure_identity::{ + ManagedIdentityCredential, ManagedIdentityCredentialOptions, UserAssignedId, + WorkloadIdentityCredential, +}; /// Authentication methods for the Geneva Config Client. /// -/// The client supports two authentication methods: -/// - Certificate-based authentication using PKCS#12 (.p12) files -/// - Managed Identity (Azure) - planned for future implementation +/// The client supports the following authentication methods: +/// - Certificate-based authentication (mTLS) using PKCS#12 (.p12) files +/// - Azure Managed Identity (System-assigned or User-assigned) +/// - Azure Workload Identity (Federated Identity for Kubernetes) +/// - Mock authentication for testing (feature-gated) /// /// # Certificate Format /// Certificates should be in PKCS#12 (.p12) format for client TLS authentication. @@ -65,6 +70,19 @@ pub enum AuthMethod { UserManagedIdentityByObjectId { object_id: String }, /// User-assigned managed identity by resource ID UserManagedIdentityByResourceId { resource_id: String }, + /// Azure Workload Identity authentication (Federated Identity for Kubernetes) + /// + /// The following environment variables must be set in the pod spec: + /// * `AZURE_CLIENT_ID` - Azure AD Application (client) ID (set explicitly in pod env) + /// * `AZURE_TENANT_ID` - Azure AD Tenant ID (set explicitly in pod env) + /// * `AZURE_FEDERATED_TOKEN_FILE` - Path to service account token file (auto-injected by workload identity webhook) + /// + /// These variables are automatically read by the Azure Identity SDK at runtime. + /// + /// # Arguments + /// * `resource` - Azure AD resource URI for token acquisition + /// (e.g., for Azure Public Cloud) + WorkloadIdentity { resource: String }, #[cfg(feature = "mock_auth")] MockAuth, // No authentication, used for testing purposes } @@ -78,6 +96,8 @@ pub(crate) enum GenevaConfigClientError { JwtTokenError(String), #[error("Certificate error: {0}")] Certificate(String), + #[error("Workload Identity authentication error: {0}")] + WorkloadIdentityAuth(String), #[error("MSI authentication error: {0}")] MsiAuth(String), @@ -257,6 +277,10 @@ impl GenevaConfigClient { .map_err(|e| GenevaConfigClientError::Certificate(e.to_string()))?; client_builder = client_builder.use_preconfigured_tls(tls_connector); } + AuthMethod::WorkloadIdentity { .. } => { + // No special HTTP client configuration needed for Workload Identity + // Authentication is done via Bearer token in request headers + } AuthMethod::SystemManagedIdentity | AuthMethod::UserManagedIdentity { .. } | AuthMethod::UserManagedIdentityByObjectId { .. } @@ -276,13 +300,14 @@ impl GenevaConfigClient { let version_str = format!("Ver{0}v0", config.config_major_version); // Use different API endpoints based on authentication method - // Certificate auth uses "api", MSI auth uses "userapi" + // Certificate auth uses "api", MSI auth and Workload Identity use "userapi" let api_path = match &config.auth_method { AuthMethod::Certificate { .. } => "api", AuthMethod::SystemManagedIdentity | AuthMethod::UserManagedIdentity { .. } | AuthMethod::UserManagedIdentityByObjectId { .. } - | AuthMethod::UserManagedIdentityByResourceId { .. } => "userapi", + | AuthMethod::UserManagedIdentityByResourceId { .. } + | AuthMethod::WorkloadIdentity { .. } => "userapi", #[cfg(feature = "mock_auth")] AuthMethod::MockAuth => "api", // treat mock like certificate path for URL shape }; @@ -329,6 +354,55 @@ impl GenevaConfigClient { headers } + /// Get Azure AD token using Workload Identity (Federated Identity) + /// + /// Reads AZURE_CLIENT_ID, AZURE_TENANT_ID, and AZURE_FEDERATED_TOKEN_FILE from environment variables. + /// In Kubernetes: + /// - AZURE_CLIENT_ID and AZURE_TENANT_ID must be set explicitly in the pod spec + /// - AZURE_FEDERATED_TOKEN_FILE is auto-injected by the workload identity webhook + async fn get_workload_identity_token(&self) -> Result { + let resource = + match &self.config.auth_method { + AuthMethod::WorkloadIdentity { resource } => resource, + _ => return Err(GenevaConfigClientError::WorkloadIdentityAuth( + "get_workload_identity_token called but auth method is not WorkloadIdentity" + .to_string(), + )), + }; + + // TODO: Extract scope generation logic into helper function shared with get_msi_token() + let base = resource.trim_end_matches("/.default").trim_end_matches('/'); + let mut scope_candidates: Vec = vec![format!("{base}/.default"), base.to_string()]; + // TODO - below check is not required, as we alread trim "/" + if !base.ends_with('/') { + scope_candidates.push(format!("{base}/")); + } + + // TODO: Consider caching WorkloadIdentityCredential if profiling shows credential creation overhead + // Pass None to let azure_identity crate read AZURE_CLIENT_ID, AZURE_TENANT_ID, + // and AZURE_FEDERATED_TOKEN_FILE from environment variables automatically + let credential = WorkloadIdentityCredential::new(None).map_err(|e| { + GenevaConfigClientError::WorkloadIdentityAuth(format!( + "Failed to create WorkloadIdentityCredential. Ensure AZURE_CLIENT_ID, AZURE_TENANT_ID, and AZURE_FEDERATED_TOKEN_FILE environment variables are set: {e}" + )) + })?; + + let mut last_err: Option = None; + for scope in &scope_candidates { + //TODO - It looks like the get_token API accepts a slice of &str + match credential.get_token(&[scope.as_str()], None).await { + Ok(token) => return Ok(token.token.secret().to_string()), + Err(e) => last_err = Some(e.to_string()), + } + } + + let detail = last_err.unwrap_or_else(|| "no error detail".into()); + Err(GenevaConfigClientError::WorkloadIdentityAuth(format!( + "Workload Identity token acquisition failed. Scopes tried: {scopes}. Last error: {detail}", + scopes = scope_candidates.join(", ") + ))) + } + /// Get MSI token for GCS authentication async fn get_msi_token(&self) -> Result { let resource = self.config.msi_resource.as_ref().ok_or_else(|| { @@ -337,17 +411,14 @@ impl GenevaConfigClient { ) })?; - // Normalize resource (strip trailing "/.default" if provided by user) + // TODO: Extract scope generation logic into helper function shared with get_workload_identity_token() let base = resource.trim_end_matches("/.default").trim_end_matches('/'); - - // Candidate scopes tried with Azure Identity let mut scope_candidates: Vec = vec![format!("{base}/.default"), base.to_string()]; - // Add variant with trailing slash if not already present + // TODO - below check is not required, as we alread trim "/" if !base.ends_with('/') { scope_candidates.push(format!("{base}/")); } - // Build credential based on selector let user_assigned_id = match &self.config.auth_method { AuthMethod::SystemManagedIdentity => None, AuthMethod::UserManagedIdentity { client_id } => { @@ -367,6 +438,7 @@ impl GenevaConfigClient { } }; + // TODO: Consider caching ManagedIdentityCredential if profiling shows credential creation overhead let options = ManagedIdentityCredentialOptions { user_assigned_id, ..Default::default() @@ -382,6 +454,7 @@ impl GenevaConfigClient { Err(e) => last_err = Some(e.to_string()), } } + let detail = last_err.unwrap_or_else(|| "no error detail".into()); Err(GenevaConfigClientError::MsiAuth(format!( "Managed Identity token acquisition failed. Scopes tried: {scopes}. Last error: {detail}. IMDS fallback intentionally disabled.", @@ -506,8 +579,8 @@ impl GenevaConfigClient { /// Internal method that actually fetches data from Geneva Config Service async fn fetch_ingestion_info(&self) -> Result<(IngestionGatewayInfo, MonikerInfo)> { - let tag_id = Uuid::new_v4().to_string(); //TODO - uuid is costly, check if counter is enough? - let mut url = String::with_capacity(self.precomputed_url_prefix.len() + 50); // Pre-allocate with reasonable capacity + let tag_id = Uuid::new_v4().to_string(); // TODO: consider cheaper counter if perf-critical + let mut url = String::with_capacity(self.precomputed_url_prefix.len() + 50); write!(&mut url, "{}&TagId={tag_id}", self.precomputed_url_prefix).map_err(|e| { GenevaConfigClientError::InternalError(format!("Failed to write URL: {e}")) })?; @@ -518,40 +591,37 @@ impl GenevaConfigClient { request = request.header("x-ms-client-request-id", req_id); - // Add MSI authentication for managed identity auth method + // Add appropriate authentication header match &self.config.auth_method { + AuthMethod::WorkloadIdentity { .. } => { + let token = self.get_workload_identity_token().await?; + request = request.header(AUTHORIZATION, format!("Bearer {}", token)); + } AuthMethod::SystemManagedIdentity | AuthMethod::UserManagedIdentity { .. } | AuthMethod::UserManagedIdentityByObjectId { .. } | AuthMethod::UserManagedIdentityByResourceId { .. } => { - let msi_token = self.get_msi_token().await?; - request = request.header(AUTHORIZATION, format!("Bearer {}", msi_token)); + let token = self.get_msi_token().await?; + request = request.header(AUTHORIZATION, format!("Bearer {}", token)); } AuthMethod::Certificate { .. } => { /* mTLS only */ } #[cfg(feature = "mock_auth")] AuthMethod::MockAuth => { /* no auth header */ } } - // Log the request details for debugging + // Send HTTP request let response = match request.send().await { - Ok(response) => response, - Err(e) => { - return Err(GenevaConfigClientError::Http(e)); - } + Ok(resp) => resp, + Err(e) => return Err(GenevaConfigClientError::Http(e)), }; - // Check if the response is successful let status = response.status(); let body = response.text().await?; + if status.is_success() { - let parsed = match serde_json::from_str::(&body) { - Ok(response) => response, - Err(e) => { - return Err(GenevaConfigClientError::AuthInfoNotFound(format!( - "Failed to parse response: {e}" - ))); - } - }; + let parsed = serde_json::from_str::(&body).map_err(|e| { + GenevaConfigClientError::AuthInfoNotFound(format!("Failed to parse response: {e}")) + })?; for account in parsed.storage_account_keys { if account.is_primary_moniker && account.account_moniker_name.contains("diag") { @@ -559,7 +629,6 @@ impl GenevaConfigClient { name: account.account_moniker_name, account_group: account.account_group_name, }; - return Ok((parsed.ingestion_gateway_info, moniker_info)); } } @@ -610,16 +679,21 @@ fn extract_endpoint_from_token(token: &str) -> Result { _ => payload.to_string(), }; - // Decode the Base64-encoded payload into raw bytes with a more tolerant approach. + // Decode the Base64-encoded payload into raw bytes. + // Try URL-safe (with and without padding), then fall back to standard Base64. let decoded = match general_purpose::URL_SAFE_NO_PAD.decode(&payload) { Ok(b) => b, - Err(e_url) => match general_purpose::STANDARD.decode(&payload) { + Err(e_url_no_pad) => match general_purpose::URL_SAFE.decode(&payload) { Ok(b) => b, - Err(e_std) => { - return Err(GenevaConfigClientError::JwtTokenError(format!( - "Failed to decode JWT (url_safe and standard): url_err={e_url}; std_err={e_std}" - ))) - } + Err(e_url_pad) => match general_purpose::STANDARD.decode(&payload) { + Ok(b) => b, + Err(e_std) => { + return Err(GenevaConfigClientError::JwtTokenError(format!( + "Failed to decode JWT (URL_SAFE_NO_PAD, URL_SAFE, and STANDARD): \ + no_pad_err={e_url_no_pad}; pad_err={e_url_pad}; std_err={e_std}" + ))) + } + }, }, }; diff --git a/opentelemetry-exporter-geneva/geneva-uploader/src/config_service/mod.rs b/opentelemetry-exporter-geneva/geneva-uploader/src/config_service/mod.rs index 132b8d8d..dbf454ce 100644 --- a/opentelemetry-exporter-geneva/geneva-uploader/src/config_service/mod.rs +++ b/opentelemetry-exporter-geneva/geneva-uploader/src/config_service/mod.rs @@ -20,16 +20,19 @@ mod tests { namespace: "ns".to_string(), region: "region".to_string(), config_major_version: 1, - auth_method: AuthMethod::SystemManagedIdentity, + auth_method: AuthMethod::WorkloadIdentity { + resource: "https://monitor.azure.com".to_string(), + }, msi_resource: None, }; assert_eq!(config.environment, "env"); assert_eq!(config.account, "acct"); - assert!(matches!( - config.auth_method, - AuthMethod::SystemManagedIdentity - )); + + match config.auth_method { + AuthMethod::WorkloadIdentity { .. } => {} + _ => panic!("expected WorkloadIdentity variant"), + } } fn generate_self_signed_p12() -> (NamedTempFile, String) { diff --git a/opentelemetry-exporter-geneva/opentelemetry-exporter-geneva/examples/Dockerfile b/opentelemetry-exporter-geneva/opentelemetry-exporter-geneva/examples/Dockerfile new file mode 100644 index 00000000..8e45c454 --- /dev/null +++ b/opentelemetry-exporter-geneva/opentelemetry-exporter-geneva/examples/Dockerfile @@ -0,0 +1,45 @@ +# Dockerfile for Geneva Uploader Workload Identity Test + # + # This Dockerfile must be built from the repository root to access the workspace: + # cd /path/to/opentelemetry-rust-contrib + # docker build -f opentelemetry-exporter-geneva/opentelemetry-exporter-geneva/examples/Dockerfile -t geneva-uploader-test:latest . + # + # Or using ACR: + # az acr build --registry --image geneva-uploader-test:latest \ + # --file opentelemetry-exporter-geneva/opentelemetry-exporter-geneva/examples/Dockerfile . + + FROM rust:1.85-slim AS builder + + # Install build dependencies + RUN apt-get update && apt-get install -y \ + pkg-config \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* + + WORKDIR /app + + # Copy the entire workspace from repository root + COPY . . + + # Build the example + WORKDIR /app/opentelemetry-exporter-geneva/opentelemetry-exporter-geneva + RUN cargo build --release --example basic_workload_identity_test + + # Runtime stage + FROM debian:bookworm-slim + + # Install runtime dependencies + RUN apt-get update && apt-get install -y \ + ca-certificates \ + libssl3 \ + && rm -rf /var/lib/apt/lists/* + + # Copy the binary + COPY --from=builder /app/target/release/examples/basic_workload_identity_test /usr/local/bin/geneva-uploader-test + + # Run as non-root user + RUN useradd -m -u 1000 appuser + USER appuser + + ENTRYPOINT ["/usr/local/bin/geneva-uploader-test"] + diff --git a/opentelemetry-exporter-geneva/opentelemetry-exporter-geneva/examples/README.md b/opentelemetry-exporter-geneva/opentelemetry-exporter-geneva/examples/README.md new file mode 100644 index 00000000..37ab414c --- /dev/null +++ b/opentelemetry-exporter-geneva/opentelemetry-exporter-geneva/examples/README.md @@ -0,0 +1,360 @@ +# Geneva Exporter - Workload Identity Example + +This example demonstrates how to use Azure Workload Identity to authenticate to Geneva Config Service (GCS) from an Azure Kubernetes Service (AKS) cluster. + +## Prerequisites + +- Azure CLI (`az`) installed and authenticated +- `kubectl` configured to access your AKS cluster +- AKS cluster with OIDC Issuer and Workload Identity enabled +- Azure Container Registry (ACR) attached to your AKS cluster +- Access to Geneva/Jarvis portal for registering managed identities + +## Architecture + +Azure Workload Identity enables Kubernetes pods to authenticate to Azure services using **User-Assigned Managed Identities** with federated identity credentials. This approach uses Managed Identities, NOT App Registrations, simplifying credential management. + +**Authentication Flow**: +1. Pod runs with a Kubernetes service account +2. Kubernetes injects a service account JWT token into the pod +3. Application exchanges the Kubernetes token for an Azure AD access token using the Managed Identity +4. Azure AD access token is used to authenticate to Geneva Config Service + +**Key Difference**: Traditional Workload Identity setups often use App Registrations with client secrets. This implementation uses **User-Assigned Managed Identities** instead, which eliminates the need to manage secrets or certificates. + +## Step 1: Enable Workload Identity on AKS (if not already enabled) + +```bash +# Check if OIDC issuer is enabled +az aks show --resource-group --name --query "oidcIssuerProfile.issuerUrl" -o tsv + +# If not enabled, enable it +az aks update \ + --resource-group \ + --name \ + --enable-oidc-issuer \ + --enable-workload-identity +``` + +## Step 2: Create User-Assigned Managed Identity + +**Important**: We create a **User-Assigned Managed Identity**, NOT an Azure AD App Registration. Workload Identity with Managed Identities is simpler and doesn't require managing client secrets or certificates. + +```bash +# Set variables +RESOURCE_GROUP="" +LOCATION="" # e.g., eastus2 +IDENTITY_NAME="geneva-uploader-identity-$(openssl rand -hex 3)" + +# Create the managed identity (NOT an App Registration) +az identity create \ + --resource-group $RESOURCE_GROUP \ + --name $IDENTITY_NAME \ + --location $LOCATION + +# Get the client ID and principal ID +export AZURE_CLIENT_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $IDENTITY_NAME --query clientId -o tsv) +export PRINCIPAL_ID=$(az identity show --resource-group $RESOURCE_GROUP --name $IDENTITY_NAME --query principalId -o tsv) + +echo "Client ID: $AZURE_CLIENT_ID" +echo "Principal ID: $PRINCIPAL_ID" + +# Note: The AZURE_CLIENT_ID here is the managed identity's client ID, not an App Registration +``` + +## Step 3: Create Kubernetes Service Account + +```bash +# Set Kubernetes variables +NAMESPACE="default" # or your preferred namespace +SERVICE_ACCOUNT_NAME="geneva-uploader-sa" + +# Create service account with workload identity annotation +cat < --query "oidcIssuerProfile.issuerUrl" -o tsv) + +# Create federated credential +FEDERATED_CREDENTIAL_NAME="geneva-fedcred-$(openssl rand -hex 3)" + +az identity federated-credential create \ + --name $FEDERATED_CREDENTIAL_NAME \ + --identity-name $IDENTITY_NAME \ + --resource-group $RESOURCE_GROUP \ + --issuer $AKS_OIDC_ISSUER \ + --subject system:serviceaccount:$NAMESPACE:$SERVICE_ACCOUNT_NAME \ + --audience api://AzureADTokenExchange + +echo "Federated credential created: $FEDERATED_CREDENTIAL_NAME" +``` + +## Step 5: Register Managed Identity in Geneva Portal + +Register the managed identity using the **Principal ID (Object ID)** from Step 2. Wait 5-10 minutes for propagation. + +## Step 6: Get Your Azure Tenant ID + +```bash +export AZURE_TENANT_ID=$(az account show --query tenantId -o tsv) +echo "Tenant ID: $AZURE_TENANT_ID" +``` + +## Step 7: Build and Push Docker Image + +```bash +# Navigate to the workspace root +cd /path/to/opentelemetry-rust-contrib + +# Set ACR variables +ACR_NAME="" +IMAGE_NAME="geneva-uploader-workload-identity-test" +IMAGE_TAG="latest" + +# Build the image +docker build \ + -f opentelemetry-exporter-geneva/opentelemetry-exporter-geneva/examples/Dockerfile \ + -t $ACR_NAME.azurecr.io/$IMAGE_NAME:$IMAGE_TAG \ + . + +# Push to ACR +az acr login --name $ACR_NAME +docker push $ACR_NAME.azurecr.io/$IMAGE_NAME:$IMAGE_TAG +``` + +## Step 8: Create ConfigMap with Geneva Configuration + +```bash +# Create ConfigMap with your Geneva environment configuration +cat <` +- Workload identity webhook is running in the cluster (it injects `AZURE_FEDERATED_TOKEN_FILE`) + +### Token exchange fails with "invalid_client" + +**Cause**: Federated credential not configured correctly. + +**Fix**: Verify: +- Federated credential issuer matches AKS OIDC issuer exactly +- Subject is `system:serviceaccount::` +- Audience is `api://AzureADTokenExchange` + +### "Invalid scope" error + +**Cause**: Wrong resource URI for your Azure cloud. + +**Fix**: Update `GENEVA_WORKLOAD_IDENTITY_RESOURCE` in ConfigMap: +- Azure Public: `https://monitor.azure.com` +- Azure Government: `https://monitor.azure.us` +- Azure China: `https://monitor.azure.cn` + +### Logs show success but no data in Geneva + +**Possible causes**: +1. Managed identity not registered in Geneva (wait 5-10 minutes after registration) +2. Identity doesn't have correct permissions in Geneva account +3. Wrong Geneva endpoint or account configuration + +**Fix**: +- Verify identity in Geneva portal +- Check Geneva account permissions +- Review ConfigMap values against Geneva documentation + +### Check workload identity webhook status + +```bash +kubectl get pods -n kube-system | grep workload-identity +kubectl logs -n kube-system -l app.kubernetes.io/name=workload-identity-webhook +``` + +## Example kubectl Commands + +```bash +# Watch pod status +kubectl get pod geneva-uploader-test -n $NAMESPACE -w + +# Get detailed pod info +kubectl describe pod geneva-uploader-test -n $NAMESPACE + +# Stream logs +kubectl logs -f geneva-uploader-test -n $NAMESPACE + +# Check service account +kubectl get serviceaccount $SERVICE_ACCOUNT_NAME -n $NAMESPACE -o yaml + +# Check ConfigMap +kubectl get configmap geneva-config -n $NAMESPACE -o yaml + +# Delete and redeploy +kubectl delete pod geneva-uploader-test -n $NAMESPACE +# Then re-run Step 9 +``` + +## Cleanup + +```bash +# Delete Kubernetes resources +kubectl delete pod geneva-uploader-test -n $NAMESPACE +kubectl delete configmap geneva-config -n $NAMESPACE +kubectl delete serviceaccount $SERVICE_ACCOUNT_NAME -n $NAMESPACE + +# Delete Azure resources +az identity federated-credential delete \ + --name $FEDERATED_CREDENTIAL_NAME \ + --identity-name $IDENTITY_NAME \ + --resource-group $RESOURCE_GROUP + +az identity delete \ + --resource-group $RESOURCE_GROUP \ + --name $IDENTITY_NAME + +# Remove from Jarvis (Geneva portal) manually +``` + +## References + +- [Azure Workload Identity Documentation](https://azure.github.io/azure-workload-identity/) +- [AKS Workload Identity Overview](https://learn.microsoft.com/azure/aks/workload-identity-overview) + diff --git a/opentelemetry-exporter-geneva/opentelemetry-exporter-geneva/examples/basic_workload_identity_test.rs b/opentelemetry-exporter-geneva/opentelemetry-exporter-geneva/examples/basic_workload_identity_test.rs new file mode 100644 index 00000000..6777cfc4 --- /dev/null +++ b/opentelemetry-exporter-geneva/opentelemetry-exporter-geneva/examples/basic_workload_identity_test.rs @@ -0,0 +1,158 @@ +//! run with `$ cargo run --example basic_workload_identity_test` + +use geneva_uploader::client::{GenevaClient, GenevaClientConfig}; +use geneva_uploader::AuthMethod; +use opentelemetry_appender_tracing::layer; +use opentelemetry_exporter_geneva::GenevaExporter; +use opentelemetry_sdk::logs::log_processor_with_async_runtime::BatchLogProcessor; +use opentelemetry_sdk::runtime::Tokio; +use opentelemetry_sdk::{ + logs::{BatchConfig, SdkLoggerProvider}, + Resource, +}; +use std::env; +use std::thread; +use std::time::Duration; +use tracing::{error, info, warn}; +use tracing_subscriber::{prelude::*, EnvFilter}; + +/* +Environment variables required: + +export GENEVA_ENDPOINT="https://abc.azurewebsites.net" +export GENEVA_ENVIRONMENT="Test" +export GENEVA_ACCOUNT="PipelineAgent2Demo" +export GENEVA_NAMESPACE="PAdemo2" +export GENEVA_REGION="eastus" +export GENEVA_CONFIG_MAJOR_VERSION=2 +export MONITORING_GCS_AUTH_ID_TYPE="AuthWorkloadIdentity" +export GENEVA_WORKLOAD_IDENTITY_RESOURCE="https://abc.azurewebsites.net" # Resource (audience) base for token exchange + +# Azure Workload Identity configuration: +export AZURE_CLIENT_ID="" # Azure AD Application (client) ID +export AZURE_TENANT_ID="" # Azure AD Tenant ID +export AZURE_FEDERATED_TOKEN_FILE="/var/run/secrets/azure/tokens/azure-identity-token" # Path to service account token (Kubernetes default) + +# Optional: Override the token file path +# export WORKLOAD_IDENTITY_TOKEN_FILE="/custom/path/to/token" +*/ + +#[tokio::main] +async fn main() { + let endpoint = env::var("GENEVA_ENDPOINT").expect("GENEVA_ENDPOINT is required"); + let environment = env::var("GENEVA_ENVIRONMENT").expect("GENEVA_ENVIRONMENT is required"); + let account = env::var("GENEVA_ACCOUNT").expect("GENEVA_ACCOUNT is required"); + let namespace = env::var("GENEVA_NAMESPACE").expect("GENEVA_NAMESPACE is required"); + let region = env::var("GENEVA_REGION").expect("GENEVA_REGION is required"); + let config_major_version: u32 = env::var("GENEVA_CONFIG_MAJOR_VERSION") + .expect("GENEVA_CONFIG_MAJOR_VERSION is required") + .parse() + .expect("GENEVA_CONFIG_MAJOR_VERSION must be a u32"); + + let tenant = env::var("GENEVA_TENANT").unwrap_or_else(|_| "default-tenant".to_string()); + let role_name = env::var("GENEVA_ROLE_NAME").unwrap_or_else(|_| "default-role".to_string()); + let role_instance = + env::var("GENEVA_ROLE_INSTANCE").unwrap_or_else(|_| "default-instance".to_string()); + + // Determine authentication method based on environment variables + let auth_method = match env::var("MONITORING_GCS_AUTH_ID_TYPE").as_deref() { + Ok("AuthWorkloadIdentity") => { + let resource = env::var("GENEVA_WORKLOAD_IDENTITY_RESOURCE") + .expect("GENEVA_WORKLOAD_IDENTITY_RESOURCE required for Workload Identity auth"); + + // Note: AZURE_CLIENT_ID, AZURE_TENANT_ID, and AZURE_FEDERATED_TOKEN_FILE + // are read automatically by the azure_identity crate from environment variables. + // These are typically set by the Azure Workload Identity webhook in Kubernetes. + AuthMethod::WorkloadIdentity { + resource, + } + } + _ => panic!( + "This example requires Workload Identity authentication. Set MONITORING_GCS_AUTH_ID_TYPE=AuthWorkloadIdentity" + ), + }; + + let config = GenevaClientConfig { + endpoint, + environment, + account, + namespace, + region, + config_major_version, + tenant, + role_name, + role_instance, + auth_method, + msi_resource: None, // Not used for Workload Identity + }; + + // GenevaClient::new is synchronous (returns Result), so no await is needed here. + let geneva_client = GenevaClient::new(config).expect("Failed to create GenevaClient"); + + let exporter = GenevaExporter::new(geneva_client); + let batch_processor = BatchLogProcessor::builder(exporter, Tokio) + .with_batch_config(BatchConfig::default()) + .build(); + + let provider: SdkLoggerProvider = SdkLoggerProvider::builder() + .with_resource( + Resource::builder() + .with_service_name("geneva-exporter-workload-identity-test") + .build(), + ) + .with_log_processor(batch_processor) + .build(); + + // To prevent a telemetry-induced-telemetry loop, OpenTelemetry's own internal + // logging is properly suppressed. However, logs emitted by external components + // (such as reqwest, tonic, etc.) are not suppressed as they do not propagate + // OpenTelemetry context. Until this issue is addressed + // (https://github.com/open-telemetry/opentelemetry-rust/issues/2877), + // filtering like this is the best way to suppress such logs. + // + // The filter levels are set as follows: + // - Allow `info` level and above by default. + // - Completely restrict logs from `hyper`, `tonic`, `h2`, and `reqwest`. + // + // Note: This filtering will also drop logs from these components even when + // they are used outside of the OTLP Exporter. + let filter_otel = EnvFilter::new("info") + .add_directive("hyper=off".parse().unwrap()) + .add_directive("opentelemetry=off".parse().unwrap()) + .add_directive("tonic=off".parse().unwrap()) + .add_directive("h2=off".parse().unwrap()) + .add_directive("reqwest=off".parse().unwrap()); + let otel_layer = layer::OpenTelemetryTracingBridge::new(&provider).with_filter(filter_otel); + + // Create a new tracing::Fmt layer to print the logs to stdout. It has a + // default filter of `info` level and above, and `debug` and above for logs + // from OpenTelemetry crates. The filter levels can be customized as needed. + let filter_fmt = EnvFilter::new("info") + .add_directive("hyper=debug".parse().unwrap()) + .add_directive("reqwest=debug".parse().unwrap()) + .add_directive("opentelemetry=debug".parse().unwrap()); + let fmt_layer = tracing_subscriber::fmt::layer() + .with_thread_names(true) + .with_filter(filter_fmt); + + tracing_subscriber::registry() + .with(otel_layer) + .with(fmt_layer) + .init(); + + // Generate logs to trigger batch processing and GCS calls + info!(name: "Log", target: "my-system", event_id = 20, user_name = "user1", user_email = "user1@opentelemetry.io", message = "Registration successful"); + info!(name: "Log", target: "my-system", event_id = 51, user_name = "user2", user_email = "user2@opentelemetry.io", message = "Checkout successful"); + info!(name: "Log", target: "my-system", event_id = 30, user_name = "user3", user_email = "user3@opentelemetry.io", message = "User login successful"); + info!(name: "Log", target: "my-system", event_id = 52, user_name = "user2", user_email = "user2@opentelemetry.io", message = "Payment processed successfully"); + error!(name: "Log", target: "my-system", event_id = 31, user_name = "user4", user_email = "user4@opentelemetry.io", message = "Login failed - invalid credentials"); + warn!(name: "Log", target: "my-system", event_id = 53, user_name = "user5", user_email = "user5@opentelemetry.io", message = "Shopping cart abandoned"); + info!(name: "Log", target: "my-system", event_id = 32, user_name = "user1", user_email = "user1@opentelemetry.io", message = "Password reset requested"); + info!(name: "Log", target: "my-system", event_id = 54, user_name = "user2", user_email = "user2@opentelemetry.io", message = "Order shipped successfully"); + + println!("Sleeping for 30 seconds..."); + thread::sleep(Duration::from_secs(30)); + + let _ = provider.shutdown(); + println!("Shutting down provider"); +}