CLOUDP-58851: Include readiness probe in Agent image (#25)

Author: chatton

.evergreen.yml

@@ -63,28 +63,16 @@ functions:
         env:
           KUBECONFIG: ${workdir}/kube_config
 
-  deploy_operator:
-    - command: subprocess.exec
-      type: setup
-      params:
-        add_to_path:
-          - ${workdir}/bin
-        working_dir: mongodb-kubernetes-operator
-        binary: scripts/ci/deploy_operator.sh
-        include_expansions_in_env:
-          - version_id
-        env:
-          KUBECONFIG: ${workdir}/kube_config
-
   run_e2e_test:
     - command: subprocess.exec
-      type: setup
+      type: test
       params:
         add_to_path:
           - ${workdir}/bin
         working_dir: mongodb-kubernetes-operator
         include_expansions_in_env:
           - version_id
+          - test
         binary: scripts/ci/run_test.sh
         env:
           KUBECONFIG: ${workdir}/kube_config
@@ -103,25 +91,47 @@ functions:
         binary: scripts/ci/build_and_push_image.sh
 
 tasks:
-  - name: unit_tests
+  - name: build_operator_image
+    priority: 60
+    exec_timeout_secs: 600
     commands:
       - func: clone
-      - func: go_test
+      - func: build_and_push_image
+        vars:
+          dockerfile: docker/Dockerfile.operator
+          image: quay.io/mongodb/community-operator-dev:${version_id}
 
-  - name: e2e_test
+  - name: build_e2e_image
+    priority: 60
+    exec_timeout_secs: 600
     commands:
       - func: clone
       - func: build_and_push_image
         vars:
-         dockerfile: docker/Dockerfile.operator
-         image: quay.io/mongodb/community-operator-dev:${version_id}
+          dockerfile: docker/Dockerfile.e2e
+          image: quay.io/mongodb/community-operator-e2e:${version_id}
+
+  - name: unit_tests
+    commands:
+      - func: clone
+      - func: go_test
+
+  - name: e2e_test_replica_set
+    commands:
+      - func: clone
       - func: setup_kubernetes_environment
-      - func: deploy_operator
-      - func: build_and_push_image
+      - func: run_e2e_test
         vars:
-         dockerfile: docker/Dockerfile.e2e
-         image: quay.io/mongodb/community-operator-e2e:${version_id}
+          test: replica_set
+
+  - name: e2e_test_replica_set_readiness_probe
+    commands:
+      - func: clone
+      - func: setup_kubernetes_environment
       - func: run_e2e_test
+        vars:
+          test: replica_set_readiness_probe
+
 
 buildvariants:
   - name: go_unit_tests
@@ -133,7 +143,21 @@ buildvariants:
 
   - name: e2e_tests
     display_name: e2e_tests
+    run_on:
+      - ubuntu1604-build
+    depends_on:
+      - name: build_operator_image
+        variant: init_test_run
+      - name: build_e2e_image
+        variant: init_test_run
+    tasks:
+      - name: e2e_test_replica_set
+      - name: e2e_test_replica_set_readiness_probe
+
+  - name: init_test_run
+    display_name: init_test_run
     run_on:
       - ubuntu1604-build
     tasks:
-      - name: e2e_test
+      - name: build_operator_image
+      - name: build_e2e_image

agent/Dockerfile

@@ -26,4 +26,11 @@ RUN curl -LO http://downloads.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1604-
     mv mongodb-linux-x86_64-ubuntu1604-4.0.6/bin/mongo /usr/bin && \
     rm -rf mongodb-linux-x86_64-ubuntu1604-4.0.6.tgz mongodb-linux-x86_64-ubuntu1604-4.0.6
 
+
+RUN mkdir -p /var/lib/mongodb-mms-automation/probes/ \
+    && curl --retry 3 https://readinessprobe.s3-us-west-1.amazonaws.com/readinessprobe -o /var/lib/mongodb-mms-automation/probes/readinessprobe \
+    && chmod +x /var/lib/mongodb-mms-automation/probes/readinessprobe \
+    && mkdir -p /var/log/mongodb-mms-automation/ \
+    && chmod -R +wr /var/log/mongodb-mms-automation/
+
 CMD ["agent/mongodb-agent", "-cluster=/var/lib/automation/config/automation-config.json"]

cmd/testrunner/crds/crds.go

@@ -64,7 +64,7 @@ func allCrds(deployDir string) ([]string, error) {
 	crdDir := path.Join(deployDir, "crds")
 	var crdFilePaths []string
 	err := filepath.Walk(crdDir, func(path string, info os.FileInfo, err error) error {
-		if strings.HasSuffix(info.Name(), "_crd.yaml") {
+		if info != nil && strings.HasSuffix(info.Name(), "_crd.yaml") {
 			fmt.Printf("Found CRD: %s\n", info.Name())
 			crdFilePaths = append(crdFilePaths, path)
 		}

cmd/testrunner/main.go

@@ -34,21 +34,24 @@ type flags struct {
 	namespace     string
 	operatorImage string
 	testImage     string
+	test          string
 }
 
 func parseFlags() flags {
-	var namespace, deployDir, operatorImage, testImage *string
+	var namespace, deployDir, operatorImage, testImage, test *string
 	namespace = flag.String("namespace", "default", "the namespace the operator and tests should be deployed in")
 	deployDir = flag.String("deployDir", "deploy/", "the path to the directory which contains the yaml deployment files")
 	operatorImage = flag.String("operatorImage", "quay.io/mongodb/community-operator-dev:latest", "the image which should be used for the operator deployment")
 	testImage = flag.String("testImage", "quay.io/mongodb/community-operator-e2e:latest", "the image which should be used for the operator e2e tests")
+	test = flag.String("test", "", "test e2e test that should be run. (name of folder containing the test)")
 	flag.Parse()
 
 	return flags{
 		deployDir:     *deployDir,
 		namespace:     *namespace,
 		operatorImage: *operatorImage,
 		testImage:     *testImage,
+		test:          *test,
 	}
 }
 
@@ -87,13 +90,12 @@ func runCmd(f flags) error {
 	}
 	fmt.Println("Successfully deployed the operator")
 
-	testToRun := "test/replica_set_test.yaml" // TODO: this should be configurable
-	if err := buildKubernetesResourceFromYamlFile(c, testToRun, &corev1.Pod{}, withNamespace(f.namespace), withTestImage(f.testImage)); err != nil {
+	testToRun := "test/operator-sdk-test.yaml"
+	if err := buildKubernetesResourceFromYamlFile(c, testToRun, &corev1.Pod{}, withNamespace(f.namespace), withTestImage(f.testImage), withTest(f.test)); err != nil {
 		return fmt.Errorf("error deploying test: %v", err)
 	}
 
-	// TODO: configurable
-	nsName := types.NamespacedName{Name: "my-replica-set-test", Namespace: f.namespace}
+	nsName := types.NamespacedName{Name: "operator-sdk-test", Namespace: f.namespace}
 
 	fmt.Println("Waiting for pod to be ready...")
 	testPod, err := pod.WaitForPhase(c, nsName, time.Second*5, time.Minute*5, corev1.PodRunning)
@@ -183,6 +185,8 @@ func withNamespace(ns string) func(runtime.Object) {
 			v.Namespace = ns
 		case *corev1.Pod:
 			v.Namespace = ns
+		case *appsv1.Deployment:
+			v.Namespace = ns
 		}
 	}
 }
@@ -208,6 +212,26 @@ func withOperatorImage(image string) func(runtime.Object) {
 	}
 }
 
+// withTest configures the test Pod to launch with the correct
+// command which will target the given test
+func withTest(test string) func(obj runtime.Object) {
+	return func(obj runtime.Object) {
+		if testPod, ok := obj.(*corev1.Pod); ok {
+			testPod.Spec.Containers[0].Command = []string{
+				"/bin/operator-sdk",
+				"test",
+				"local",
+				fmt.Sprintf("./test/e2e/%s", test),
+				"--namespace",
+				testPod.Namespace,
+				"--verbose",
+				"--kubeconfig",
+				"/etc/config/kubeconfig",
+			}
+		}
+	}
+}
+
 // buildKubernetesResourceFromYamlFile will create the kubernetes resource defined in yamlFilePath. All of the functional options
 // provided will be applied before creation.
 func buildKubernetesResourceFromYamlFile(c client.Client, yamlFilePath string, obj runtime.Object, options ...func(obj runtime.Object)) error {

docker/Dockerfile.testrunner

@@ -23,4 +23,8 @@ FROM registry.access.redhat.com/ubi8/ubi-minimal:latest
 # copy the testrunner binary
 COPY --from=builder /go/main runner
 
+# add yaml files required by the test runner
+ADD deploy/ deploy/
+ADD test/ test/
+
 CMD ["./runner"]

pkg/controller/mongodb/mongodb_controller.go

@@ -27,10 +27,13 @@ import (
 )
 
 const (
-	AutomationConfigKey   = "automation-config"
-	agentName             = "mongodb-agent"
-	mongodbName           = "mongod"
-	agentImageEnvVariable = "AGENT_IMAGE"
+	AutomationConfigKey       = "automation-config"
+	agentName                 = "mongodb-agent"
+	mongodbName               = "mongod"
+	agentImageEnvVariable     = "AGENT_IMAGE"
+	readinessProbePath        = "/var/lib/mongodb-mms-automation/probes/readinessprobe"
+	agentHealthStatusFilePath = "/var/log/mongodb-mms-automation/agent-health-status.json"
+	clusterFilePath           = "/var/lib/automation/config/automation-config"
 )
 
 // Add creates a new MongoDB Controller and adds it to the Manager. The Manager will set fields on the Controller
@@ -210,16 +213,21 @@ func buildAutomationConfigConfigMap(mdb mdbv1.MongoDB) (corev1.ConfigMap, error)
 func buildContainers(mdb mdbv1.MongoDB) ([]corev1.Container, error) {
 	agentCommand := []string{
 		"agent/mongodb-agent",
-		"-cluster=/var/lib/automation/config/automation-config",
+		"-cluster=" + clusterFilePath,
 		"-skipMongoStart",
 		"-noDaemonize",
+		"-healthCheckFilePath=" + agentHealthStatusFilePath,
+		"-serveStatusPort=5000",
 	}
+
+	readinessProbe := defaultReadinessProbe()
 	agentContainer := corev1.Container{
 		Name:            agentName,
 		Image:           os.Getenv(agentImageEnvVariable),
 		ImagePullPolicy: corev1.PullAlways,
 		Resources:       resourcerequirements.Defaults(),
 		Command:         agentCommand,
+		ReadinessProbe:  &readinessProbe,
 	}
 
 	mongoDbCommand := []string{
@@ -236,6 +244,19 @@ func buildContainers(mdb mdbv1.MongoDB) ([]corev1.Container, error) {
 	return []corev1.Container{agentContainer, mongodbContainer}, nil
 }
 
+func defaultReadinessProbe() corev1.Probe {
+	return corev1.Probe{
+		Handler: corev1.Handler{
+			Exec: &corev1.ExecAction{Command: []string{readinessProbePath}},
+		},
+		// Setting the failure threshold to quite big value as the agent may spend some time to reach the goal
+		FailureThreshold: 240,
+		// The agent may be not on time to write the status file right after the container is created - we need to wait
+		// for some time
+		InitialDelaySeconds: 5,
+	}
+}
+
 // buildStatefulSet takes a MongoDB resource and converts it into
 // the corresponding stateful set
 func buildStatefulSet(mdb mdbv1.MongoDB) (appsv1.StatefulSet, error) {

pkg/controller/mongodb/replicaset_controller_test.go

@@ -3,6 +3,7 @@ package mongodb
 import (
 	"context"
 	"os"
+	"reflect"
 	"testing"
 
 	mdbv1 "github.com/mongodb/mongodb-kubernetes-operator/pkg/apis/mongodb/v1"
@@ -68,6 +69,8 @@ func TestStatefulSet_IsCorrectlyConfigured(t *testing.T) {
 	agentContainer := sts.Spec.Template.Spec.Containers[0]
 	assert.Equal(t, agentName, agentContainer.Name)
 	assert.Equal(t, os.Getenv(agentImageEnvVariable), agentContainer.Image)
+	expectedProbe := defaultReadinessProbe()
+	assert.True(t, reflect.DeepEqual(&expectedProbe, agentContainer.ReadinessProbe))
 
 	mongodbContainer := sts.Spec.Template.Spec.Containers[1]
 	assert.Equal(t, mongodbName, mongodbContainer.Name)

scripts/ci/deploy_operator.sh

@@ -12,16 +12,16 @@ contents="$(cat ${temp})"
 kubectl create cm kube-config --from-literal=kubeconfig="${contents}"
 rm ${temp}
 
-
 # create roles and service account required for the test runner
 kubectl apply -f deploy/testrunner
 
 # start the test runner pod
 kubectl run test-runner --generator=run-pod/v1 \
   --restart=Never \
+  --image-pull-policy=Always \
   --image=quay.io/chatton/test-runner \
   --serviceaccount=test-runner \
-  --command -- ./runner  --operatorImage quay.io/mongodb/community-operator-dev:${version_id} --testImage quay.io/mongodb/community-operator-e2e:${version_id}
+  --command -- ./runner  --operatorImage quay.io/mongodb/community-operator-dev:${version_id} --testImage quay.io/mongodb/community-operator-e2e:${version_id} --test=${test}
 
 
 echo "Test pod is ready to begin"