Tune and enhance neutron-operator probes

This patch tunes the liveness and readiness probes to match other
operators [0][1][2][3].

We noticed that at times neutron tests would fail with 5XX errors,
indicating that the service was not avaiable during the execution of
the test, after investigation it was reported that neutron containers
were restarted due to probe failures.

Prior to this patch, neutron had the liveness and readiness probes set
to 5s where the majority of other operators had it set to something as
high as 30s (see links below).

This patch also removes the readiness probe from the neutron-api container
as it's redundant since it checks for readiness of the httpd container
(which is already checked by the httpd container readiness probe itself).

Resolves: https://issues.redhat.com/browse/OSPRH-9305

[0] https://github.com/openstack-k8s-operators/nova-operator/blob/27895863ccb7a5ef552e3d924d778fc7f664d056/pkg/novaapi/deployment.go#L49
[1] https://github.com/openstack-k8s-operators/keystone-operator/blob/1c577c9ef57612991d4139ab232eca61f0f43e83/pkg/keystone/deployment.go#L46
[2] https://github.com/openstack-k8s-operators/placement-operator/blob/d3a775bb32a963806949acd6556bd43ad901864d/pkg/placement/deployment.go#L41
[3] https://github.com/openstack-k8s-operators/glance-operator/blob/aa9b49b5810f76b4d06061191c5cb297eaa18ba8/pkg/glanceapi/statefulset.go#L62

Signed-off-by: Lucas Alvares Gomes <[email protected]>

Author: umago

pkg/neutronapi/deployment.go

@@ -41,17 +41,21 @@ func Deployment(
 	labels map[string]string,
 	annotations map[string]string,
 ) (*appsv1.Deployment, error) {
+	// TODO(lucasagomes): Look into how to implement separated probes
+	// for the httpd and neutron-api containers. Right now the code uses
+	// the same liveness and readiness probes for both containers which
+	// only checks the port 9696 (NeutronPublicPort) which is the port
+	// that httpd is listening to. Ideally, we should also include a
+	// probe on port 9697 which is the port that neutron-api binds to
 	livenessProbe := &corev1.Probe{
-		// TODO might need tuning
-		TimeoutSeconds:      5,
-		PeriodSeconds:       5,
-		InitialDelaySeconds: 20,
+		TimeoutSeconds:      30,
+		PeriodSeconds:       30,
+		InitialDelaySeconds: 5,
 	}
 	readinessProbe := &corev1.Probe{
-		// TODO might need tuning
-		TimeoutSeconds:      5,
-		PeriodSeconds:       5,
-		InitialDelaySeconds: 20,
+		TimeoutSeconds:      30,
+		PeriodSeconds:       30,
+		InitialDelaySeconds: 5,
 	}
 	args := []string{"-c", ServiceCommand}
 	httpdArgs := []string{"-DFOREGROUND"}
@@ -148,7 +152,6 @@ func Deployment(
 							Env:                      env.MergeEnvs([]corev1.EnvVar{}, envVars),
 							VolumeMounts:             apiVolumeMounts,
 							Resources:                instance.Spec.Resources,
-							ReadinessProbe:           readinessProbe,
 							LivenessProbe:            livenessProbe,
 							TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
 						},

test/functional/neutronapi_controller_test.go

@@ -861,7 +861,6 @@ var _ = Describe("NeutronAPI controller", func() {
 
 			nSvcContainer := deployment.Spec.Template.Spec.Containers[0]
 			Expect(nSvcContainer.LivenessProbe.HTTPGet.Port.IntVal).To(Equal(int32(9696)))
-			Expect(nSvcContainer.ReadinessProbe.HTTPGet.Port.IntVal).To(Equal(int32(9696)))
 			Expect(nSvcContainer.VolumeMounts).To(HaveLen(2))
 			Expect(nSvcContainer.Image).To(Equal(util.GetEnvVar("RELATED_IMAGE_NEUTRON_API_IMAGE_URL_DEFAULT", neutronv1.NeutronAPIContainerImage)))
 

test/kuttl/common/assert_sample_deployment.yaml

@@ -76,21 +76,11 @@ spec:
             path: /
             port: 9696
             scheme: HTTP
-          initialDelaySeconds: 20
-          periodSeconds: 5
+          initialDelaySeconds: 5
+          periodSeconds: 30
           successThreshold: 1
-          timeoutSeconds: 5
+          timeoutSeconds: 30
         name: neutron-api
-        readinessProbe:
-          failureThreshold: 3
-          httpGet:
-            path: /
-            port: 9696
-            scheme: HTTP
-          initialDelaySeconds: 20
-          periodSeconds: 5
-          successThreshold: 1
-          timeoutSeconds: 5
         resources: {}
         securityContext:
           runAsUser: 42435
@@ -105,21 +95,21 @@ spec:
             path: /
             port: 9696
             scheme: HTTP
-          initialDelaySeconds: 20
-          periodSeconds: 5
+          initialDelaySeconds: 5
+          periodSeconds: 30
           successThreshold: 1
-          timeoutSeconds: 5
+          timeoutSeconds: 30
         name: neutron-httpd
         readinessProbe:
           failureThreshold: 3
           httpGet:
             path: /
             port: 9696
             scheme: HTTP
-          initialDelaySeconds: 20
-          periodSeconds: 5
+          initialDelaySeconds: 5
+          periodSeconds: 30
           successThreshold: 1
-          timeoutSeconds: 5
+          timeoutSeconds: 30
         resources: {}
         securityContext:
           runAsUser: 0