✨ add metrics check and further helpers for the e2e tests

Provide further improvements for e2e tests test to help
users be aware of how to tests using the metrics endpoint
and validate if the metrics are properly expose.

Author: camilamacedo86

.github/workflows/test-e2e-samples.yml

@@ -38,6 +38,7 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '27s/^#//' $KUSTOMIZATION_FILE_PATH
           sed -i '51s/^#//' $KUSTOMIZATION_FILE_PATH
           sed -i '55,151s/^#//' $KUSTOMIZATION_FILE_PATH
           cd testdata/project-v4/
@@ -58,6 +59,7 @@ jobs:
         run: |
           KUSTOMIZATION_FILE_PATH="testdata/project-v4-with-deploy-image/config/default/kustomization.yaml"
           sed -i '25s/^#//' $KUSTOMIZATION_FILE_PATH
+          sed -i '27s/^#//' $KUSTOMIZATION_FILE_PATH
           sed -i '51s/^#//' $KUSTOMIZATION_FILE_PATH
           # Uncomment only ValidatingWebhookConfiguration
           # from cert-manager replaces

docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_suite_test.go

@@ -57,6 +57,9 @@ func TestE2E(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
+	By("Ensure that Prometheus is enable")
+	_ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#")
+
 	By("generating files")
 	cmd := exec.Command("make", "generate")
 	_, err := utils.Run(cmd)

docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go

@@ -17,8 +17,12 @@ limitations under the License.
 package e2e
 
 import (
+	"encoding/json"
 	"fmt"
+	"os"
 	"os/exec"
+	"path/filepath"
+	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -27,10 +31,19 @@ import (
 	"tutorial.kubebuilder.io/project/test/utils"
 )
 
+// namespace where the project is deployed in
 const namespace = "project-system"
 
-// Define a set of end-to-end (e2e) tests to validate the behavior of the controller.
-var _ = Describe("controller", Ordered, func() {
+// serviceAccountName created for the project
+const serviceAccountName = "project-controller-manager"
+
+// metricsServiceName is the name of the metrics service of the project
+const metricsServiceName = "project-controller-manager-metrics-service"
+
+// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data
+const metricsRoleBindingName = "project-metrics-binding"
+
+var _ = Describe("Manager", Ordered, func() {
 	// Before running the tests, set up the environment by creating the namespace,
 	// installing CRDs, and deploying the controller.
 	BeforeAll(func() {
@@ -53,8 +66,12 @@ var _ = Describe("controller", Ordered, func() {
 	// After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs,
 	// and deleting the namespace.
 	AfterAll(func() {
+		By("cleaning up the curl pod for metrics")
+		cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace)
+		_, _ = utils.Run(cmd)
+
 		By("undeploying the controller-manager")
-		cmd := exec.Command("make", "undeploy")
+		cmd = exec.Command("make", "undeploy")
 		_, _ = utils.Run(cmd)
 
 		By("uninstalling CRDs")
@@ -66,11 +83,10 @@ var _ = Describe("controller", Ordered, func() {
 		_, _ = utils.Run(cmd)
 	})
 
-	// The Context block contains the actual tests that validate the operator's behavior.
-	Context("Operator", func() {
+	// The Context block contains the actual tests that validate the manager's behavior.
+	Context("Manager", func() {
+		var controllerPodName string
 		It("should run successfully", func() {
-			var controllerPodName string
-
 			By("validating that the controller-manager pod is running as expected")
 			verifyControllerUp := func() error {
 				// Get the name of the controller-manager pod
@@ -110,5 +126,163 @@ var _ = Describe("controller", Ordered, func() {
 
 		// TODO(user): Customize the e2e test suite to include
 		// additional scenarios specific to your project.
+
+		It("should ensure the metrics endpoint is serving metrics", func() {
+			By("creating a ClusterRoleBinding for the service account to allow access to metrics")
+			cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName,
+				"--clusterrole=project-metrics-reader",
+				fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName),
+			)
+			_, err := utils.Run(cmd)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding")
+
+			By("validating that the metrics service is available")
+			cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace)
+			_, err = utils.Run(cmd)
+			ExpectWithOffset(2, err).NotTo(HaveOccurred(), "Metrics service should exist")
+
+			By("validating that the ServiceMonitor for Prometheus is applied in the namespace")
+			cmd = exec.Command("kubectl", "get", "ServiceMonitor", "-n", namespace)
+			_, err = utils.Run(cmd)
+			ExpectWithOffset(2, err).NotTo(HaveOccurred(), "ServiceMonitor should exist")
+
+			By("getting the service account token")
+			token, err := serviceAccountToken()
+			ExpectWithOffset(2, err).NotTo(HaveOccurred())
+			ExpectWithOffset(2, token).NotTo(BeEmpty())
+
+			By("waiting for the metrics endpoint to be ready")
+			verifyMetricsEndpointReady := func() error {
+				cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
+				output, err := utils.Run(cmd)
+				if err != nil {
+					return err
+				}
+				if !strings.Contains(string(output), "8443") {
+					return fmt.Errorf("metrics endpoint is not ready")
+				}
+				return nil
+			}
+			EventuallyWithOffset(2, verifyMetricsEndpointReady, 2*time.Minute, 10*time.Second).Should(Succeed())
+
+			By("verifying that the controller manager is serving the metrics server")
+			Eventually(func() error {
+				cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
+				logs, err := utils.Run(cmd)
+				if err != nil {
+					return err
+				}
+				if !strings.Contains(string(logs), "controller-runtime.metrics\tServing metrics server") {
+					return fmt.Errorf("metrics server not yet started")
+				}
+				return nil
+			}, 2*time.Minute, 10*time.Second).Should(Succeed(), "Controller manager did not start serving metrics server")
+
+			By("creating the curl-metrics pod to access the metrics endpoint")
+			cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
+				"--namespace", namespace,
+				"--image=curlimages/curl:7.78.0",
+				"--", "/bin/sh", "-c", fmt.Sprintf(
+					"curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics",
+					token, metricsServiceName, namespace))
+			_, err = utils.Run(cmd)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod")
+
+			By("waiting for the curl-metrics pod to complete.")
+			verifyCurlUp := func() error {
+				cmd := exec.Command("kubectl", "get", "pods", "curl-metrics",
+					"-o", "jsonpath={.status.phase}",
+					"-n", namespace)
+				status, err := utils.Run(cmd)
+				ExpectWithOffset(3, err).NotTo(HaveOccurred())
+				if string(status) != "Succeeded" {
+					return fmt.Errorf("curl pod in %s status", status)
+				}
+				return nil
+			}
+			EventuallyWithOffset(2, verifyCurlUp, 5*time.Minute, 10*time.Second).Should(Succeed())
+
+			By("getting the metrics by checking curl-metrics logs")
+			metricsOutput := getMetricsOutput()
+			ExpectWithOffset(1, metricsOutput).To(ContainSubstring(
+				"controller_runtime_reconcile_total",
+			))
+		})
+
+		// TODO: Customize the e2e test suite with scenarios specific to your project.
+		// Consider applying sample/CR(s) and check their status and/or verifying
+		// the reconciliation by using the metrics, i.e.:
+		// metricsOutput := getMetricsOutput()
+		// ExpectWithOffset(1, metricsOutput).To(ContainSubstring(
+		//    fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`,
+		//    strings.ToLower(<Kind>),
+		// ))
 	})
 })
+
+// serviceAccountToken returns a token for the specified service account in the given namespace.
+// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request
+// and parsing the resulting token from the API response.
+func serviceAccountToken() (string, error) {
+	const tokenRequestRawString = `{
+		"apiVersion": "authentication.k8s.io/v1",
+		"kind": "TokenRequest"
+	}`
+
+	// Temporary file to store the token request
+	secretName := fmt.Sprintf("%s-token-request", serviceAccountName)
+	tokenRequestFile := filepath.Join("/tmp", secretName)
+	err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o755))
+	if err != nil {
+		return "", err
+	}
+
+	var out string
+	var rawJson string
+	Eventually(func() error {
+		// Execute kubectl command to create the token
+		cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf(
+			"/api/v1/namespaces/%s/serviceaccounts/%s/token",
+			namespace,
+			serviceAccountName,
+		), "-f", tokenRequestFile)
+
+		output, err := cmd.CombinedOutput()
+		if err != nil {
+			return err
+		}
+
+		rawJson = string(output)
+
+		// Parse the JSON output to extract the token
+		var token tokenRequest
+		err = json.Unmarshal([]byte(rawJson), &token)
+		if err != nil {
+			return err
+		}
+
+		out = token.Status.Token
+		return nil
+	}, time.Minute, time.Second).Should(Succeed())
+
+	return out, err
+}
+
+// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint.
+func getMetricsOutput() string {
+	By("getting the curl-metrics logs")
+	cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace)
+	metricsOutput, err := utils.Run(cmd)
+	ExpectWithOffset(3, err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod")
+	metricsOutputStr := string(metricsOutput)
+	ExpectWithOffset(3, metricsOutputStr).To(ContainSubstring("< HTTP/1.1 200 OK"))
+	return metricsOutputStr
+}
+
+// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response,
+// containing only the token field that we need to extract.
+type tokenRequest struct {
+	Status struct {
+		Token string `json:"token"`
+	} `json:"status"`
+}

docs/book/src/cronjob-tutorial/testdata/project/test/utils/utils.go

@@ -17,6 +17,8 @@ limitations under the License.
 package utils
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
 	"os"
 	"os/exec"
@@ -198,3 +200,52 @@ func GetProjectDir() (string, error) {
 	wd = strings.Replace(wd, "/test/e2e", "", -1)
 	return wd, nil
 }
+
+// UncommentCode searches for target in the file and remove the comment prefix
+// of the target content. The target content may span multiple lines.
+func UncommentCode(filename, target, prefix string) error {
+	// false positive
+	// nolint:gosec
+	content, err := os.ReadFile(filename)
+	if err != nil {
+		return err
+	}
+	strContent := string(content)
+
+	idx := strings.Index(strContent, target)
+	if idx < 0 {
+		return fmt.Errorf("unable to find the code %s to be uncomment", target)
+	}
+
+	out := new(bytes.Buffer)
+	_, err = out.Write(content[:idx])
+	if err != nil {
+		return err
+	}
+
+	scanner := bufio.NewScanner(bytes.NewBufferString(target))
+	if !scanner.Scan() {
+		return nil
+	}
+	for {
+		_, err := out.WriteString(strings.TrimPrefix(scanner.Text(), prefix))
+		if err != nil {
+			return err
+		}
+		// Avoid writing a newline in case the previous line was the last in target.
+		if !scanner.Scan() {
+			break
+		}
+		if _, err := out.WriteString("\n"); err != nil {
+			return err
+		}
+	}
+
+	_, err = out.Write(content[idx+len(target):])
+	if err != nil {
+		return err
+	}
+	// false positive
+	// nolint:gosec
+	return os.WriteFile(filename, out.Bytes(), 0644)
+}

docs/book/src/getting-started/testdata/project/test/e2e/e2e_suite_test.go

@@ -57,6 +57,9 @@ func TestE2E(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
+	By("Ensure that Prometheus is enable")
+	_ = utils.UncommentCode("config/default/kustomization.yaml", "#- ../prometheus", "#")
+
 	By("generating files")
 	cmd := exec.Command("make", "generate")
 	_, err := utils.Run(cmd)