Refine AppSignals AutoMonitor and SDK Injection checks (#342)

* Add validation to skip ADOT SDK injection for incompatible configurations

Skip ADOT SDK injection in the following scenarios:
- runAsNonRoot=true without runAsUser (prevents init container failures)
- Custom OTLP endpoints without Application Signals enabled (avoids conflicts)
- Application Signals explicitly disabled (allows custom OTel configuration)

Key changes:
- Add envFrom ConfigMap resolution for validation (N+1 query optimization)
- Add comprehensive unit tests (33 scenarios in helper_test.go)
- Skip injection when OTel Collector container already exists
- Expand excluded namespaces for auto-monitor (40+ common system namespaces)
- Check for OpenTelemetry operator installation before enabling auto-monitor
- Protocol-aware CloudWatch agent endpoint detection

Files modified:
- pkg/instrumentation/helper.go: Core validation logic
- pkg/instrumentation/helper_test.go: Comprehensive unit tests
- pkg/instrumentation/sdk.go: envFrom caching, OTC detection
- pkg/instrumentation/sdk_test.go: Integration tests
- pkg/instrumentation/{java,python,nodejs,dotnet}.go: Pass container params
- pkg/instrumentation/auto/monitor.go: Expanded excluded namespaces
- pkg/instrumentation/auto/util.go: OpenTelemetry operator check

* Update logging for configure AutoMonitor functions

* Replace -a to -r for cp cmd on Python and NodeJs sdk injection

* fix: skip auto-monitor when api-resource fetching returns uncaptured errors

* nit refactoring - use consts for otel env vars

Author: mxiamxia

pkg/instrumentation/auto/monitor.go

@@ -26,7 +26,50 @@ import (
 	"github.com/aws/amazon-cloudwatch-agent-operator/pkg/instrumentation"
 )
 
-var excludedNamespaces = []string{"kube-system", "amazon-cloudwatch"}
+// List of namespaces excluded from ApplicationSignals auto-monitoring by default (best-effort basis).
+// Customers must explicitly opt in to enable ApplicationSignals for workloads in these namespaces.
+var excludedNamespaces = []string{
+	// --- Monitoring & Observability ---
+	"monitoring",                    // Prometheus, kube-prometheus-stack, Grafana
+	"loki",                          // Loki, Promtail
+	"observability",                 // Tempo, Jaeger
+	"opentelemetry-operator-system", // OpenTelemetry Collector / Operator
+	"amazon-cloudwatch",             // Amazon CloudWatch Agent
+	"tracing",                       // Zipkin or similar tracing tools
+
+	// --- Ingress & Networking ---
+	"ingress-nginx", // NGINX Ingress Controller
+	"traefik",       // Traefik Ingress Controller
+	"istio-system",  // Istio control plane
+	"linkerd",       // Linkerd service mesh
+	"cert-manager",  // Cert-Manager for TLS certificates
+	"external-dns",  // ExternalDNS controller
+
+	// --- Cloud Integrations ---
+	"aws-load-balancer-controller", // AWS Load Balancer Controller
+	"kube-system",                  // Cluster Autoscaler, CSI Drivers, Metrics Server
+	"external-dns",                 // ExternalDNS (sometimes deployed here too)
+
+	// --- Security & Policy ---
+	"kyverno",           // Kyverno policy engine
+	"gatekeeper-system", // OPA Gatekeeper
+	"falco",             // Falco runtime security
+	"vault",             // HashiCorp Vault for secrets
+
+	// --- CI/CD & Management ---
+	"argocd",           // Argo CD
+	"flux-system",      // Flux CD
+	"jenkins",          // Jenkins CI/CD
+	"tekton-pipelines", // Tekton pipelines
+	"spinnaker",        // Spinnaker
+	"harbor",           // Harbor container registry
+	"reloader",         // Reloader for configmap/secret reloads
+
+	// --- Other Utilities ---
+	"keda",   // KEDA event-driven autoscaler
+	"velero", // Velero backup/restore
+	"minio",  // MinIO object storage
+}
 
 const (
 	ByLabel              = "IndexByLabel"

pkg/instrumentation/auto/util.go

@@ -10,6 +10,7 @@ import (
 	"os"
 
 	"github.com/go-logr/logr"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -62,6 +63,21 @@ func configureAutoMonitor(ctx context.Context, autoMonitorConfigStr string, clie
 		return nil, fmt.Errorf("unable to unmarshal auto-monitor config: %w", err)
 	}
 
+	resources, err := clientSet.Discovery().ServerResourcesForGroupVersion("opentelemetry.io/v1alpha1")
+	if err == nil {
+		for _, r := range resources.APIResources {
+			if r.Name == "instrumentations" {
+				setupLog.Info("W! auto-monitor is disabled due to the presence of opentelemetry.io/v1alpha1 group version")
+				return nil, nil
+			}
+		}
+	} else {
+		if !errors.IsNotFound(err) {
+			setupLog.Info(fmt.Sprintf("W! auto-monitor is disabled due to failures in retrieving server groups: %v", err))
+			return nil, nil
+		}
+	}
+
 	logger := ctrl.Log.WithName("auto_monitor")
 	return NewMonitor(ctx, *autoMonitorConfig, clientSet, client, reader, logger), nil
 }

pkg/instrumentation/auto/util_test.go

@@ -11,8 +11,9 @@ import (
 
 	"github.com/go-logr/logr/testr"
 	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	discoveryfake "k8s.io/client-go/discovery/fake"
 	"k8s.io/client-go/kubernetes/fake"
-
 	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
 )
 
@@ -28,6 +29,7 @@ func TestCreateInstrumentationAnnotator(t *testing.T) {
 		envDisableMonitor    bool
 		autoAnnotationConfig string
 		autoMonitorConfig    string
+		otelSetupExists      bool
 		expectNilAnnotator   bool
 		expectedType         string
 	}{
@@ -94,6 +96,16 @@ func TestCreateInstrumentationAnnotator(t *testing.T) {
 			expectNilAnnotator:   false,
 			expectedType:         "*auto.AnnotationMutators",
 		},
+		{
+			name:                 "Disable auto monitor with otel setup found",
+			envDisableAnnotation: false,
+			envDisableMonitor:    false,
+			otelSetupExists:      true,
+			autoAnnotationConfig: ``,
+			autoMonitorConfig:    `{"monitorAllServices":true}`,
+			expectNilAnnotator:   true,
+			expectedType:         "",
+		},
 	}
 
 	for _, tt := range tests {
@@ -111,8 +123,29 @@ func TestCreateInstrumentationAnnotator(t *testing.T) {
 				os.Unsetenv("DISABLE_AUTO_MONITOR")
 			}
 
+			fakeClientset := fake.NewSimpleClientset()
+			if tt.otelSetupExists {
+				fakeDiscovery, ok := fakeClientset.Discovery().(*discoveryfake.FakeDiscovery)
+				if !ok {
+					t.Fatal("couldn't convert Discovery() to *FakeDiscovery")
+				}
+
+				fakeDiscovery.Resources = []*metav1.APIResourceList{
+					{
+						GroupVersion: "opentelemetry.io/v1alpha1",
+						APIResources: []metav1.APIResource{
+							{
+								Name:       "instrumentations",
+								Namespaced: true,
+								Kind:       "Instrumentation",
+								Verbs:      []string{"get", "list", "watch", "create", "update", "patch", "delete"},
+							},
+						},
+					},
+				}
+			}
 			// Call the function
-			annotator := createInstrumentationAnnotatorWithClientset(tt.autoMonitorConfig, tt.autoAnnotationConfig, ctx, fake.NewSimpleClientset(), fakeClient, fakeClient, logger)
+			annotator := createInstrumentationAnnotatorWithClientset(tt.autoMonitorConfig, tt.autoAnnotationConfig, ctx, fakeClientset, fakeClient, fakeClient, logger)
 
 			// Check results
 			if tt.expectNilAnnotator {

pkg/instrumentation/dotnet.go

@@ -54,19 +54,22 @@ var (
 	dotNetCommandWindows = []string{"CMD", "/c", "xcopy", "/e", "autoinstrumentation\\*", dotnetInstrMountPathWindows}
 )
 
-func injectDotNetSDK(dotNetSpec v1alpha1.DotNet, pod corev1.Pod, index int, runtime string) (corev1.Pod, error) {
-
-	// caller checks if there is at least one container.
+func injectDotNetSDK(dotNetSpec v1alpha1.DotNet, pod corev1.Pod, index int, runtime string, allEnvs []corev1.EnvVar) (corev1.Pod, error) {
 	container := &pod.Spec.Containers[index]
 
 	err := validateContainerEnv(container.Env, envDotNetStartupHook, envDotNetAdditionalDeps, envDotNetSharedStore)
 	if err != nil {
 		return pod, err
 	}
 
-	// check if OTEL_DOTNET_AUTO_HOME env var is already set in the container
+	// Check if ADOT SDK should be injected based on all environment variables and security context
+	if !shouldInjectADOTSDK(allEnvs, pod, container) {
+		return pod, fmt.Errorf("ADOT DOTNET SDK injection skipped due to incompatible OTel configuration")
+	}
+
+	// check if OTEL_DOTNET_AUTO_HOME env var is already set
 	// if it is already set, then we assume that .NET Auto-instrumentation is already configured for this container
-	if getIndexOfEnv(container.Env, envDotNetOTelAutoHome) > -1 {
+	if getIndexOfEnv(allEnvs, envDotNetOTelAutoHome) > -1 {
 		return pod, errors.New("OTEL_DOTNET_AUTO_HOME environment variable is already set in the container")
 	}
 
@@ -86,10 +89,9 @@ func injectDotNetSDK(dotNetSpec v1alpha1.DotNet, pod corev1.Pod, index int, runt
 		return pod, fmt.Errorf("provided instrumentation.opentelemetry.io/dotnet-runtime annotation value '%s' is not supported", runtime)
 	}
 
-	// inject .NET instrumentation spec env vars.
+	// inject .NET instrumentation spec env vars with validation
 	for _, env := range dotNetSpec.Env {
-		idx := getIndexOfEnv(container.Env, env.Name)
-		if idx == -1 {
+		if shouldInjectEnvVar(allEnvs, env.Name) {
 			container.Env = append(container.Env, env)
 		}
 	}

pkg/instrumentation/dotnet_test.go

@@ -539,7 +539,12 @@ func TestInjectDotNetSDK(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			pod, err := injectDotNetSDK(test.DotNet, test.pod, 0, test.runtime)
+			// Pass container's env vars for validation
+			allEnvs := []corev1.EnvVar{}
+			if len(test.pod.Spec.Containers) > 0 {
+				allEnvs = test.pod.Spec.Containers[0].Env
+			}
+			pod, err := injectDotNetSDK(test.DotNet, test.pod, 0, test.runtime, allEnvs)
 			assert.Equal(t, test.expected, pod)
 			assert.Equal(t, test.err, err)
 		})