feat: add support for multi container instrumentation pod (#1901)

* feat(annotations): add new instrumentation specific annotations

* feat(dotnet): add instrumentation specific volume and initcont

* feat(go): update containers names specific annotation

* feat(injection): improve check for instrumentation status

* feat(java): add instrumentation specific volume and initcont

* feat(nodejs): add instrumentation specific volume and initcont

* feat(python): add instrumentation specific volume and initcont

* feat(mutator): add possibility to specify instrumentation specific containers

* feat(sdkinjection): add possibility to instrument language specific containers

* chore(e2e): update tests

* chore: update go.sum

* chore(e2e): bring back container-names annotation

* chore(e2e): add multi-instrumentation tests

* feat(instr): add multi-instrumentation support feature gate

* chore(e2e): add multi-instrumentation tests to actions

* chore(go): update container-names annotation check

* chore(e2e): fix tests

* chore: add changelog

* chore(e2e): add and fix tests

* chore(go): fix go tests

* chore(mutator): add helpers and tests

* feat(instr): improve multi instrumentation and specific cases handling

* chore(helper): sort duplicates - fix tests

* chore(feature): add from version

* chore(instr): move funcs to languageInstrumentations struct

* chore(instr): avoid reflection

* feat(docs): add information about multi-instrumentation support

* Update pkg/instrumentation/podmutator.go

Co-authored-by: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com>

* chore(instr): add new cases

* chore(instr): log error for invalid instr conf

* chore(instr): ret err for duplicates, improve conditions

* chore(go): improve containers check condition

* feat(sdk): fix initcontainer caps

* feat(sdk): fix securitycontext injection

* feat: update featuregate version

* feat: update fg registration version

* chore: update tests with vol sizelimit

* feat: update readme

* fix: dotnet tests

* fix dotnet e2e test

* fix: sort only if duplicates

* chore: featuregate update fromversion

---------

Co-authored-by: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com>

Author: mat-rumian

.chloggen/1901-support-multi-cont-instr.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. operator, target allocator, github action)
+component: operator
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Add support for multi instrumentation"
+
+# One or more tracking issues related to the change
+issues: [1717]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

.github/workflows/e2e.yaml

@@ -29,6 +29,7 @@ jobs:
          - e2e-upgrade
          - e2e-prometheuscr
          - e2e-autoscale
+         - e2e-multi-instrumentation
 
     steps:
     - name: Set up Go

Makefile

@@ -207,9 +207,14 @@ e2e-log-operator:
 	kubectl get pod -n opentelemetry-operator-system | grep "opentelemetry-operator" | awk '{print $$1}' | xargs -I {} kubectl logs -n opentelemetry-operator-system {} manager
 	kubectl get deploy -A
 
+# end-to-tests for multi-instrumentation
+.PHONY: e2e-multi-instrumentation
+e2e-multi-instrumentation:
+	$(KUTTL) test --config kuttl-test-multi-instr.yaml
+
 .PHONY: prepare-e2e
 prepare-e2e: kuttl set-image-controller container container-target-allocator container-operator-opamp-bridge start-kind cert-manager install-metrics-server load-image-all deploy
-	TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) SED_BIN="$(SED)" ./hack/modify-test-images.sh
+	TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) OPERATOR_IMG=$(IMG) SED_BIN="$(SED)" ./hack/modify-test-images.sh
 
 .PHONY: enable-prometheus-feature-flag
 enable-prometheus-feature-flag:

README.md

@@ -302,7 +302,7 @@ The possible values for the annotation can be
 * `"my-other-namespace/my-instrumentation"` - name and namespace of `Instrumentation` CR instance in another namespace.
 * `"false"` - do not inject
 
-#### Multi-container pods
+#### Multi-container pods with single instrumentation
 
 If nothing else is specified, instrumentation is performed on the first container available in the pod spec.
 In some cases (for example in the case of the injection of an Istio sidecar) it becomes necessary to specify on which container(s) this injection must be performed.
@@ -342,6 +342,90 @@ In the above case, `myapp` and `myapp2` containers will be instrumented, `myapp3
 
 > 🚨 **NOTE**: Go auto-instrumentation **does not** support multicontainer pods. When injecting Go auto-instrumentation the first pod should be the only pod you want instrumented.
 
+#### Multi-container pods with multiple instrumentations
+
+Works only when `operator.autoinstrumentation.multi-instrumentation` feature is `enabled`.
+
+Annotations defining which language instrumentation will be injected are required. When feature is enabled, specific for Instrumentation language containers annotations are used:
+
+Java:
+```bash
+instrumentation.opentelemetry.io/java-container-names: "java1,java2"
+```
+
+NodeJS:
+```bash
+instrumentation.opentelemetry.io/nodejs-container-names: "nodejs1,nodejs2"
+```
+
+Python:
+```bash
+instrumentation.opentelemetry.io/python-container-names: "python1,python3"
+```
+
+DotNet:
+```bash
+instrumentation.opentelemetry.io/dotnet-container-names: "dotnet1,dotnet2"
+```
+
+Go:
+```bash
+instrumentation.opentelemetry.io/go-container-names: "go1"
+```
+
+ApacheHttpD:
+```bash
+instrumentation.opentelemetry.io/apache-httpd-container-names: "apache1,apache2"
+```
+
+SDK:
+```bash
+instrumentation.opentelemetry.io/sdk-container-names: "app1,app2"
+```
+
+If language instrumentation specific container names are not specified, instrumentation is performed on the first container available in the pod spec (only if single instrumentation injection is configured).
+
+In some cases containers in the pod are using different technologies. It becomes necessary to specify language instrumentation for container(s) on which this injection must be performed.
+
+For this, we will use language instrumentation specific container names annotation for which we will indicate one or more container names (`.spec.containers.name`) on which the injection must be made:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: my-deployment-with-multi-containers-multi-instrumentations
+spec:
+  selector:
+    matchLabels:
+      app: my-pod-with-multi-containers-multi-instrumentations
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: my-pod-with-multi-containers-multi-instrumentations
+      annotations:
+        instrumentation.opentelemetry.io/inject-java: "true"
+        instrumentation.opentelemetry.io/java-container-names: "myapp,myapp2"
+        instrumentation.opentelemetry.io/inject-python: "true"
+        instrumentation.opentelemetry.io/python-container-names: "myapp3"
+    spec:
+      containers:
+      - name: myapp
+        image: myImage1
+      - name: myapp2
+        image: myImage2
+      - name: myapp3
+        image: myImage3
+```
+
+In the above case, `myapp` and `myapp2` containers will be instrumented using Java and `myapp3` using Python instrumentation.
+
+**NOTE**: Go auto-instrumentation **does not** support multicontainer pods. When injecting Go auto-instrumentation the first container should be the only you want to instrument.
+
+**NOTE**: This type of instrumentation **does not** allow to instrument a container with multiple language instrumentations.
+
+**NOTE**: `instrumentation.opentelemetry.io/container-names` annotation is not used for this feature.
+
 #### Use customized or vendor instrumentation
 
 By default, the operator uses upstream auto-instrumentation libraries. Custom auto-instrumentation can be configured by
@@ -403,8 +487,8 @@ instrumentation.opentelemetry.io/inject-sdk: "true"
 The operator allows specifying, via the feature gates,  which languages the Instrumentation resource may instrument.
 These feature gates must be passed to the operator via the `--feature-gates` flag.
 The flag allows for a comma-delimited list of feature gate identifiers.
-Prefix a gate with '-' to disable support for the corresponding language.
-Prefixing a gate with '+' or no prefix will enable support for the corresponding language.
+Prefix a gate with '-' to disable support for the corresponding language or multi instrumentation feature.
+Prefixing a gate with '+' or no prefix will enable support for the corresponding language or multi instrumentation feature.
 If a language is enabled by default its gate only needs to be supplied when disabling the gate.
 
 | Language      | Gate                                        | Default Value |
@@ -418,6 +502,10 @@ If a language is enabled by default its gate only needs to be supplied when disa
 
 Language not specified in the table are always supported and cannot be disabled.
 
+OpenTelemetry Operator allows to instrument multiple containers using multiple language specific instrumentations.
+These feature can be enabled using `operator.autoinstrumentation.multi-instrumentation` flag. By default flag is `disabled`.
+For more information about multi-instrumentation feature capabilities please see [Multi-container pods with multiple instrumentations](#Multi-container-pods-with-multiple-instrumentations).
+
 ### Target Allocator
 
 The OpenTelemetry Operator comes with an optional component, the [Target Allocator](/cmd/otel-allocator/README.md) (TA). When creating an OpenTelemetryCollector Custom Resource (CR) and setting the TA as enabled, the Operator will create a new deployment and service to serve specific `http_sd_config` directives for each Collector pod as part of that CR. It will also change the Prometheus receiver configuration in the CR, so that it uses the [http_sd_config](https://prometheus.io/docs/prometheus/latest/http_sd/) from the TA. The following example shows how to get started with the Target Allocator:

hack/modify-test-images.sh

@@ -5,3 +5,5 @@ SED_BIN=${SED_BIN:-sed}
 ${SED_BIN} -i "s#local/opentelemetry-operator-targetallocator:e2e#${TARGETALLOCATOR_IMG}#g" tests/e2e/smoke-targetallocator/*.yaml
 ${SED_BIN} -i "s#local/opentelemetry-operator-targetallocator:e2e#${TARGETALLOCATOR_IMG}#g" tests/e2e/targetallocator-features/00-install.yaml
 ${SED_BIN} -i "s#local/opentelemetry-operator-targetallocator:e2e#${TARGETALLOCATOR_IMG}#g" tests/e2e/prometheus-config-validation/*.yaml
+
+${SED_BIN} -i "s#local/opentelemetry-operator:e2e#${OPERATOR_IMG}#g" tests/e2e-multi-instrumentation/*.yaml

kuttl-test-multi-instr.yaml

@@ -0,0 +1,9 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+artifactsDir: ./tests/_build/artifacts/
+commands:
+  - command: kubectl apply -f ./tests/e2e-multi-instrumentation/manager_deployment_feature_gate.yaml
+  - command: go run hack/check-operator-ready.go
+testDirs:
+  - ./tests/e2e-multi-instrumentation/
+timeout: 150

pkg/featuregate/featuregate.go

@@ -62,6 +62,12 @@ var (
 		featuregate.WithRegisterFromVersion("v0.80.0"),
 	)
 
+	EnableMultiInstrumentationSupport = featuregate.GlobalRegistry().MustRegister(
+		"operator.autoinstrumentation.multi-instrumentation",
+		featuregate.StageAlpha,
+		featuregate.WithRegisterFromVersion("0.86.0"),
+		featuregate.WithRegisterDescription("controls whether the operator supports multi instrumentation"))
+
 	// EnableTargetAllocatorRewrite is the feature gate that controls whether the collector's configuration should
 	// automatically be rewritten when the target allocator is enabled.
 	EnableTargetAllocatorRewrite = featuregate.GlobalRegistry().MustRegister(

pkg/instrumentation/annotation.go

@@ -23,16 +23,23 @@ import (
 const (
 	// annotationInjectJava indicates whether java auto-instrumentation should be injected or not.
 	// Possible values are "true", "false" or "<Instrumentation>" name.
-	annotationInjectJava          = "instrumentation.opentelemetry.io/inject-java"
-	annotationInjectNodeJS        = "instrumentation.opentelemetry.io/inject-nodejs"
-	annotationInjectPython        = "instrumentation.opentelemetry.io/inject-python"
-	annotationInjectDotNet        = "instrumentation.opentelemetry.io/inject-dotnet"
-	annotationDotNetRuntime       = "instrumentation.opentelemetry.io/otel-dotnet-auto-runtime"
-	annotationInjectGo            = "instrumentation.opentelemetry.io/inject-go"
-	annotationGoExecPath          = "instrumentation.opentelemetry.io/otel-go-auto-target-exe"
-	annotationInjectSdk           = "instrumentation.opentelemetry.io/inject-sdk"
-	annotationInjectContainerName = "instrumentation.opentelemetry.io/container-names"
-	annotationInjectApacheHttpd   = "instrumentation.opentelemetry.io/inject-apache-httpd"
+	annotationInjectContainerName             = "instrumentation.opentelemetry.io/container-names"
+	annotationInjectJava                      = "instrumentation.opentelemetry.io/inject-java"
+	annotationInjectJavaContainersName        = "instrumentation.opentelemetry.io/java-container-names"
+	annotationInjectNodeJS                    = "instrumentation.opentelemetry.io/inject-nodejs"
+	annotationInjectNodeJSContainersName      = "instrumentation.opentelemetry.io/nodejs-container-names"
+	annotationInjectPython                    = "instrumentation.opentelemetry.io/inject-python"
+	annotationInjectPythonContainersName      = "instrumentation.opentelemetry.io/python-container-names"
+	annotationInjectDotNet                    = "instrumentation.opentelemetry.io/inject-dotnet"
+	annotationDotNetRuntime                   = "instrumentation.opentelemetry.io/otel-dotnet-auto-runtime"
+	annotationInjectDotnetContainersName      = "instrumentation.opentelemetry.io/dotnet-container-names"
+	annotationInjectGo                        = "instrumentation.opentelemetry.io/inject-go"
+	annotationInjectGoContainersName          = "instrumentation.opentelemetry.io/go-container-names"
+	annotationGoExecPath                      = "instrumentation.opentelemetry.io/otel-go-auto-target-exe"
+	annotationInjectSdk                       = "instrumentation.opentelemetry.io/inject-sdk"
+	annotationInjectSdkContainersName         = "instrumentation.opentelemetry.io/sdk-container-names"
+	annotationInjectApacheHttpd               = "instrumentation.opentelemetry.io/inject-apache-httpd"
+	annotationInjectApacheHttpdContainersName = "instrumentation.opentelemetry.io/apache-httpd-container-names"
 )
 
 // annotationValue returns the effective annotationInjectJava value, based on the annotations from the pod and namespace.

pkg/instrumentation/dotnet.go

@@ -33,12 +33,15 @@ const (
 	envDotNetOTelAutoHome               = "OTEL_DOTNET_AUTO_HOME"
 	dotNetCoreClrEnableProfilingEnabled = "1"
 	dotNetCoreClrProfilerID             = "{918728DD-259F-4A6A-AC2B-B85E1B658318}"
-	dotNetCoreClrProfilerGlibcPath      = "/otel-auto-instrumentation/linux-x64/OpenTelemetry.AutoInstrumentation.Native.so"
-	dotNetCoreClrProfilerMuslPath       = "/otel-auto-instrumentation/linux-musl-x64/OpenTelemetry.AutoInstrumentation.Native.so"
-	dotNetAdditionalDepsPath            = "/otel-auto-instrumentation/AdditionalDeps"
-	dotNetOTelAutoHomePath              = "/otel-auto-instrumentation"
-	dotNetSharedStorePath               = "/otel-auto-instrumentation/store"
-	dotNetStartupHookPath               = "/otel-auto-instrumentation/net/OpenTelemetry.AutoInstrumentation.StartupHook.dll"
+	dotNetCoreClrProfilerGlibcPath      = "/otel-auto-instrumentation-dotnet/linux-x64/OpenTelemetry.AutoInstrumentation.Native.so"
+	dotNetCoreClrProfilerMuslPath       = "/otel-auto-instrumentation-dotnet/linux-musl-x64/OpenTelemetry.AutoInstrumentation.Native.so"
+	dotNetAdditionalDepsPath            = "/otel-auto-instrumentation-dotnet/AdditionalDeps"
+	dotNetOTelAutoHomePath              = "/otel-auto-instrumentation-dotnet"
+	dotNetSharedStorePath               = "/otel-auto-instrumentation-dotnet/store"
+	dotNetStartupHookPath               = "/otel-auto-instrumentation-dotnet/net/OpenTelemetry.AutoInstrumentation.StartupHook.dll"
+	dotnetVolumeName                    = volumeName + "-dotnet"
+	dotnetInitContainerName             = initContainerName + "-dotnet"
+	dotnetInstrMountPath                = "/otel-auto-instrumentation-dotnet"
 )
 
 // Supported .NET runtime identifiers (https://learn.microsoft.com/en-us/dotnet/core/rid-catalog), can be set by instrumentation.opentelemetry.io/inject-dotnet.
@@ -108,28 +111,28 @@ func injectDotNetSDK(dotNetSpec v1alpha1.DotNet, pod corev1.Pod, index int) (cor
 	setDotNetEnvVar(container, envDotNetSharedStore, dotNetSharedStorePath, concatEnvValues)
 
 	container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
-		Name:      volumeName,
-		MountPath: "/otel-auto-instrumentation",
+		Name:      dotnetVolumeName,
+		MountPath: dotnetInstrMountPath,
 	})
 
 	// We just inject Volumes and init containers for the first processed container.
-	if isInitContainerMissing(pod) {
+	if isInitContainerMissing(pod, dotnetInitContainerName) {
 		pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
-			Name: volumeName,
+			Name: dotnetVolumeName,
 			VolumeSource: corev1.VolumeSource{
 				EmptyDir: &corev1.EmptyDirVolumeSource{
 					SizeLimit: volumeSize(dotNetSpec.VolumeSizeLimit),
 				},
 			}})
 
 		pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{
-			Name:      initContainerName,
+			Name:      dotnetInitContainerName,
 			Image:     dotNetSpec.Image,
-			Command:   []string{"cp", "-a", "/autoinstrumentation/.", "/otel-auto-instrumentation/"},
+			Command:   []string{"cp", "-a", "/autoinstrumentation/.", dotnetInstrMountPath},
 			Resources: dotNetSpec.Resources,
 			VolumeMounts: []corev1.VolumeMount{{
-				Name:      volumeName,
-				MountPath: "/otel-auto-instrumentation",
+				Name:      dotnetVolumeName,
+				MountPath: dotnetInstrMountPath,
 			}},
 		})
 	}