Version gate the CONFIG_PATH==STATE_PATH change to 8.13 and later (#8869) (#8873)

We ran into issues with Fleet server no longer enroling with the changes from #8856
This proposes to version gate the functionality to Elastic agent versions that actually support advanced configuration.

(cherry picked from commit 70d91eb)

Author: pebrc

pkg/controller/agent/pod.go

@@ -148,8 +148,10 @@ func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret
 		if builder, err = amendBuilderForFleetMode(params, fleetCerts, fleetToken, builder, configHash); err != nil {
 			return corev1.PodTemplateSpec{}, err
 		}
-		// Point agent to static config file mounted from a secret to /etc/agent/elastic-agent.yml
-		builder = builder.WithArgs("-e", "-c", path.Join(ConfigMountPath, ConfigFileName))
+		if params.AgentVersion.GTE(agentv1alpha1.FleetAdvancedConfigMinVersion) {
+			// Point agent to static config file mounted from a secret to /etc/agent/elastic-agent.yml for versions that support advanced configuration to be used in addition to the config coming from Fleet.
+			builder = builder.WithArgs("-e", "-c", path.Join(ConfigMountPath, ConfigFileName))
+		}
 	} else if spec.StandaloneModeEnabled() {
 		// cleanup secret used in Fleet mode
 		if err := cleanupEnvVarsSecret(params); err != nil {
@@ -197,6 +199,15 @@ func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret
 	return builder.PodTemplate, nil
 }
 
+func fleetConfigPath(v version.Version) string {
+	if v.LT(agentv1alpha1.FleetAdvancedConfigMinVersion) {
+		// default to the in-container config directory for older versions of Elastic Agent
+		// that still try to rewrite the config file in Fleet mode during enrollment.
+		return "/usr/share/elastic-agent"
+	}
+	return DataMountPath
+}
+
 func amendBuilderForFleetMode(params Params, fleetCerts *certificates.CertificatesSecret, fleetToken EnrollmentAPIKey, builder *defaults.PodTemplateBuilder, configHash hash.Hash) (*defaults.PodTemplateBuilder, error) {
 	esAssociation, err := getRelatedEsAssoc(params)
 	if err != nil {
@@ -238,7 +249,7 @@ func amendBuilderForFleetMode(params Params, fleetCerts *certificates.Certificat
 		WithResources(defaultFleetResources).
 		WithEnv(
 			corev1.EnvVar{Name: "STATE_PATH", Value: DataMountPath},
-			corev1.EnvVar{Name: "CONFIG_PATH", Value: DataMountPath},
+			corev1.EnvVar{Name: "CONFIG_PATH", Value: fleetConfigPath(params.AgentVersion)},
 		)
 
 	return builder, nil

pkg/controller/agent/pod_test.go

@@ -87,7 +87,7 @@ func Test_amendBuilderForFleetMode(t *testing.T) {
 					},
 					{
 						Name:  "CONFIG_PATH",
-						Value: "/usr/share/elastic-agent/state",
+						Value: "/usr/share/elastic-agent",
 					},
 				}
 

test/e2e/agent/config.go

@@ -45,7 +45,40 @@ const (
         name: system-1
       - package:
           name: kubernetes
-        name: kubernetes-1`
+        name: kubernetes-1
+        inputs:
+        - type: kubelet-kubernetes/metrics
+          enabled: false
+        - type: kube-state-metrics-kubernetes/metrics
+          enabled: false
+        - type:  kube-apiserver-kubernetes/metrics
+          enabled: false
+        - type:  kube-proxy-kubernetes/metrics
+          enabled: false
+        - type:  kube-scheduler-kubernetes/metrics
+          enabled: false
+        - type: kube-controller-manager-kubernetes/metrics
+          enabled: false
+        - type: events-kubernetes/metrics
+          enabled: false
+        - type: container-logs-filestream
+          enabled: true
+          streams:
+          - data_stream:
+              dataset: kubernetes.container_logs
+            enabled: true
+            vars:
+            - name: paths
+              value:
+              - /var/log/containers/*${kubernetes.container.id}.log
+            - name: symlinks
+              value: true
+            - name: containerParserStream
+              value: all
+            - name: containerParserFormat
+              value: auto
+        - type: audit-logs-filestream
+          enabled: false`
 
 	E2EAgentSystemIntegrationConfig = `id: 2d70a6f0-33a5-11eb-bb2f-418d0388a8cf
 revision: 2
@@ -396,4 +429,35 @@ inputs:
   securityContext:
     runAsUser: 0
 `
+	E2EAgentFleetModeHostPathPodTemplate = `spec:
+   hostNetwork: true
+   dnsPolicy: ClusterFirstWithHostNet
+   automountServiceAccountToken: true
+   securityContext:
+     runAsUser: 0
+   containers:
+   - name: agent
+     volumeMounts:
+     - mountPath: /var/lib/docker/containers
+       name: varlibdockercontainers
+     - mountPath: /var/log/containers
+       name: varlogcontainers
+     - mountPath: /var/log/pods
+       name: varlogpods
+   volumes:
+   - name: varlibdockercontainers
+     hostPath:
+       path: /var/lib/docker/containers
+   - name: varlogcontainers
+     hostPath:
+       path: /var/log/containers
+   - name: varlogpods
+     hostPath:
+       path: /var/log/pods`
+
+	E2EAgentFleetModeAdvancedConfig = `fleet:
+  enabled: true
+providers.kubernetes:
+  add_resource_metadata:
+    deployment: true`
 )

test/e2e/agent/config_test.go

@@ -15,6 +15,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	agentv1alpha1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/agent/v1alpha1"
 	v1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1"
 	"github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version"
 	"github.com/elastic/cloud-on-k8s/v3/test/e2e/test"
@@ -196,6 +197,46 @@ func TestFleetMode(t *testing.T) {
 	})
 }
 
+func TestFleetModeAdvancedConfig(t *testing.T) {
+	if version.MustParse(test.Ctx().ElasticStackVersion).LT(agentv1alpha1.FleetAdvancedConfigMinVersion) {
+		t.Skipf("Skipping test %s because Fleet advanced config is not supported for stack version %s", t.Name(), test.Ctx().ElasticStackVersion)
+	}
+
+	name := "fleet-adv-cfg"
+	esBuilder := elasticsearch.NewBuilder(name).
+		WithESMasterDataNodes(3, elasticsearch.DefaultResources)
+
+	kbBuilder := kibana.NewBuilder(name).
+		WithElasticsearchRef(esBuilder.Ref()).
+		WithNodeCount(1)
+
+	fleetServerBuilder := agent.NewBuilder(name+"-fs").
+		WithRoles(agent.AgentFleetModeRoleName).
+		WithOpenShiftRoles(test.UseSCCRole).
+		WithDeployment().
+		WithFleetMode().
+		WithFleetServer().
+		WithElasticsearchRefs(agent.ToOutput(esBuilder.Ref(), "default")).
+		WithKibanaRef(kbBuilder.Ref()).
+		// Check that we have Kubernetes deployment metadata in any log index (this can only be done on the Fleet server builder which has an ES output):
+		WithESValidation(agent.HasEvent("/logs-*/_search?q=_exists_:kubernetes.deployment.name"), "default")
+
+	agentBuilder := agent.NewBuilder(name + "-ea").
+		WithRoles(agent.AgentFleetModeRoleName).
+		WithOpenShiftRoles(test.UseSCCRole).
+		WithFleetMode().
+		WithKibanaRef(kbBuilder.Ref()).
+		WithFleetServerRef(fleetServerBuilder.Ref())
+
+	kbBuilder = kbBuilder.WithConfig(fleetConfigForKibana(t, fleetServerBuilder.Agent.Spec.Version, esBuilder.Ref(), fleetServerBuilder.Ref(), true))
+
+	fleetServerBuilder = agent.ApplyYamls(t, fleetServerBuilder, "", E2EAgentFleetModePodTemplate)
+	agentBuilder = agent.ApplyYamls(t, agentBuilder, E2EAgentFleetModeAdvancedConfig, E2EAgentFleetModeHostPathPodTemplate)
+
+	test.Sequence(nil, test.EmptySteps, esBuilder, kbBuilder, fleetServerBuilder, agentBuilder).RunSequential(t)
+
+}
+
 func fleetConfigForKibana(t *testing.T, agentVersion string, esRef v1.ObjectSelector, fsRef v1.ObjectSelector, tlsEnabled bool) map[string]interface{} {
 	t.Helper()
 	kibanaConfig := map[string]interface{}{}