Added testing of custom metrics to e2e test deploy_app.

Author: jweite-amazon

test/e2e/common.go

@@ -192,6 +192,37 @@ func DownloadFromAppInWorkloadCluster(ctx context.Context, workloadKubeconfigPat
 	return KubectlExec(ctx, "run", workloadKubeconfigPath, runArgs...)
 }
 
+func DownloadMetricsFromCAPCManager(ctx context.Context, bootstrapKubeconfigPath string) (string, error) {
+	// Expose the CAPC manager metrics port via a K8S service
+	runArgs := []string{
+		"--port=8080", "--target-port=metrics", "--name=capc-controller-manager-metrics", "--namespace=capc-system", "deployment", "capc-controller-manager",
+	}
+	_, err := KubectlExec(ctx, "expose", bootstrapKubeconfigPath, runArgs...)
+	if err != nil {
+		fmt.Println(err)
+		return "", err
+	}
+
+	// Scrape the metrics from the service
+	runArgs = []string{
+		"-i", "--restart=Never", "dummy", "--image=dockerqa/curl:ubuntu-trusty", "--command", "--", "curl", "--silent", "capc-controller-manager-metrics.capc-system:8080/metrics",
+	}
+	result, err := KubectlExec(ctx, "run", bootstrapKubeconfigPath, runArgs...)
+	if err != nil {
+		return result, err
+	}
+
+	// Remove the metrics service
+	runArgs = []string{
+		"--namespace=capc-system", "service", "capc-controller-manager-metrics",
+	}
+	_, err = KubectlExec(ctx, "delete", bootstrapKubeconfigPath, runArgs...)
+	if err != nil {
+		return "", err
+	}
+	return result, err
+}
+
 type cloudConfig struct {
 	APIURL    string `ini:"api-url"`
 	APIKey    string `ini:"api-key"`

test/e2e/config/cloudstack.yaml

@@ -101,6 +101,8 @@ providers:
         replacements:
           - old: --metrics-bind-addr=localhost:8080
             new: --metrics-bind-addr=:8080
+          - old: "--leader-elect"
+            new: "--leader-elect\n        - --metrics-bind-addr=:8080"
 
 variables:
   KUBERNETES_VERSION_MANAGEMENT: "v1.20.10"

test/e2e/deploy_app.go

@@ -110,6 +110,14 @@ func DeployAppSpec(ctx context.Context, inputGetter func() CommonSpecInput) {
 
 		Expect(actualHtml).To(Equal(expectedHtml))
 
+		By("Confirming that the custom reconciliation error metric is scrape-able")
+		// BIG NOTE: The first reconciliation attempt of isolated_network!AssociatePublicIPAddress() returns
+		//  a CloudStack error 9999. This test expects that to happen.
+		//  No acs_reconciliation_errors appear in the scrape until logged.
+		//  If that error ever gets fixed, this test will break.
+		metricsScrape, err := DownloadMetricsFromCAPCManager(ctx, input.BootstrapClusterProxy.GetKubeconfigPath())
+		Expect(err).To(BeNil())
+		Expect(metricsScrape).To(MatchRegexp("acs_reconciliation_errors\\{acs_error_code=\"9999\"\\} [0-9]+"))
 		By("PASSED!")
 	})