Expose metrics

By default, the metrics are exposed at localhost:8383. Note that this
means in a pod, workloads outside the pod *can not* access them.

Refs:
 - https://issues.redhat.com/browse/ACM-2179

Signed-off-by: Justin Kulikauskas <[email protected]>

Author: JustinKuli

main.go

@@ -162,7 +162,7 @@ func main() {
 	mgrOptionsBase := manager.Options{
 		LeaderElection: tool.Options.EnableLeaderElection,
 		// Disable the metrics endpoint
-		MetricsBindAddress: "0",
+		MetricsBindAddress: tool.Options.MetricsAddr,
 		Scheme:             scheme,
 		// Override the EventBroadcaster so that the spam filter will not ignore events for the policy but with
 		// different messages if a large amount of events for that policy are sent in a short time.
@@ -446,6 +446,10 @@ func getHubManager(
 	options.Namespace = tool.Options.ClusterNamespaceOnHub
 	options.NewCache = newCacheFunc
 
+	// Disable the metrics endpoint for this manager. Note that since they both use the global
+	// metrics registry, metrics for this manager are still exposed by the other manager.
+	options.MetricsBindAddress = "0"
+
 	// Create a new manager to provide shared dependencies and start components
 	mgr, err := ctrl.NewManager(hubCfg, options)
 	if err != nil {

test/e2e/case13_metrics_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2020 Red Hat, Inc.
+// Copyright Contributors to the Open Cluster Management project
+
+package e2e
+
+import (
+	"os/exec"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// getMetrics curls the metrics endpoint, filters the response with the given patterns,
+// and returns the value(s) for the matching metric(s).
+func getMetrics(metricPatterns ...string) []string {
+	metricFilter := " | grep " + strings.Join(metricPatterns, " | grep ")
+	metricsCmd := `curl localhost:8383/metrics` + metricFilter
+	cmd := exec.Command("bash", "-c", metricsCmd)
+
+	matchingMetricsRaw, err := cmd.Output()
+	if err != nil {
+		if err.Error() == "exit status 1" {
+			return []string{} // exit 1 indicates that grep couldn't find a match.
+		}
+
+		return []string{err.Error()}
+	}
+
+	matchingMetrics := strings.Split(strings.TrimSpace(string(matchingMetricsRaw)), "\n")
+	values := make([]string, len(matchingMetrics))
+
+	for i, metric := range matchingMetrics {
+		fields := strings.Fields(metric)
+		if len(fields) > 0 {
+			values[i] = fields[len(fields)-1]
+		}
+	}
+
+	return values
+}
+
+var _ = Describe("Test metrics are exposed", Ordered, func() {
+	It("Should expose the controller runtime reconcile total for each controller", func() {
+		controllers := []string{
+			"policy-spec-sync",
+			"policy-status-sync",
+			"policy-template-sync",
+			"secret-sync",
+		}
+
+		for _, ctrl := range controllers {
+			By("Checking for the " + ctrl + " controller metric")
+			matches := getMetrics("controller_runtime_reconcile_total", "success", ctrl)
+			Expect(matches).To(HaveLen(1))
+		}
+	})
+})

tool/options.go

@@ -20,6 +20,7 @@ type SyncerOptions struct {
 	EnableLeaderElection      bool
 	LegacyLeaderElection      bool
 	ProbeAddr                 string
+	MetricsAddr               string
 	// The namespace that the replicated policies should be synced to. This defaults to the same namespace as on the
 	// Hub.
 	ClusterNamespace string
@@ -95,4 +96,11 @@ func ProcessFlags() {
 		":8080",
 		"The address the first probe endpoint binds to.",
 	)
+
+	flag.StringVar(
+		&Options.MetricsAddr,
+		"metrics-bind-address",
+		"localhost:8383",
+		"The address the metrics endpoint binds to.",
+	)
 }