loadtest: add support for push metrics

Equip the test orchestrator with the ability to push
metrics on execution time to a configurable remote prometheus
push gateway.

Author: calvinrzachman

itest/loadtest/config.go

@@ -58,6 +58,17 @@ type BitcoinConfig struct {
 	TLSPath  string `long:"tlspath" description:"Path to btcd's TLS certificate, if TLS is enabled"`
 }
 
+// PrometheusGatewayConfig defines exported config options for connecting to the
+// Prometheus PushGateway.
+type PrometheusGatewayConfig struct {
+	// nolint: lll
+	Enabled bool `long:"enabled" description:"Enable pushing metrics to Prometheus PushGateway"`
+	// nolint: lll
+	Host    string `long:"host" description:"Prometheus PushGateway host address"`
+	Port    int    `long:"port" description:"Prometheus PushGateway port"`
+	PushURL string
+}
+
 // Config holds the main configuration for the performance testing binary.
 type Config struct {
 	// TestCases is a comma separated list of test cases that will be
@@ -97,6 +108,12 @@ type Config struct {
 
 	// TestTimeout is the timeout for each test.
 	TestTimeout time.Duration `long:"test-timeout" description:"the timeout for each test"`
+
+	// PrometheusGateway is the configuration for the Prometheus
+	// PushGateway.
+	//
+	// nolint: lll
+	PrometheusGateway *PrometheusGatewayConfig `group:"prometheus-gateway" namespace:"prometheus-gateway" description:"Prometheus PushGateway configuration"`
 }
 
 // DefaultConfig returns the default configuration for the performance testing
@@ -120,6 +137,11 @@ func DefaultConfig() Config {
 		SendType:         taprpc.AssetType_COLLECTIBLE,
 		TestSuiteTimeout: defaultSuiteTimeout,
 		TestTimeout:      defaultTestTimeout,
+		PrometheusGateway: &PrometheusGatewayConfig{
+			Enabled: false,
+			Host:    "localhost",
+			Port:    9091,
+		},
 	}
 }
 
@@ -156,6 +178,28 @@ func LoadConfig() (*Config, error) {
 // of it with sane defaults.
 func ValidateConfig(cfg Config) (*Config, error) {
 	// TODO (positiveblue): add validation logic.
+
+	// Validate Prometheus PushGateway configuration.
+	if cfg.PrometheusGateway.Enabled {
+		gatewayHost := cfg.PrometheusGateway.Host
+		gatewayPort := cfg.PrometheusGateway.Port
+
+		if gatewayHost == "" {
+			return nil, fmt.Errorf(
+				"gateway hostname may not be empty",
+			)
+		}
+
+		if gatewayPort == 0 {
+			return nil, fmt.Errorf("gateway port is not set")
+		}
+
+		// Construct the endpoint for Prometheus PushGateway.
+		cfg.PrometheusGateway.PushURL = fmt.Sprintf(
+			"%s:%d", gatewayHost, gatewayPort,
+		)
+	}
+
 	return &cfg, nil
 }
 

itest/loadtest/load_test.go

@@ -4,11 +4,30 @@ package loadtest
 
 import (
 	"context"
+	"fmt"
 	"testing"
+	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/push"
 	"github.com/stretchr/testify/require"
 )
 
+var (
+	testDuration = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "test_duration_seconds",
+			Help: "Duration of the test execution, in seconds",
+		},
+		[]string{"test_name"},
+	)
+)
+
+func init() {
+	// Register the metric with Prometheus's default registry.
+	prometheus.MustRegister(testDuration)
+}
+
 type testCase struct {
 	name string
 	fn   func(t *testing.T, ctx context.Context, cfg *Config)
@@ -48,6 +67,9 @@ func TestPerformance(t *testing.T) {
 			continue
 		}
 
+		// Record the start time of the test case.
+		startTime := time.Now()
+
 		success := t.Run(tc.name, func(tt *testing.T) {
 			ctxt, cancel := context.WithTimeout(
 				ctxt, cfg.TestTimeout,
@@ -59,6 +81,33 @@ func TestPerformance(t *testing.T) {
 		if !success {
 			t.Fatalf("test case %v failed", tc.name)
 		}
+
+		// Calculate the test duration and push metrics if the test case succeeded.
+		if cfg.PrometheusGateway.Enabled {
+			duration := time.Since(startTime).Seconds()
+
+			timeTag := fmt.Sprintf("%d", time.Now().Unix())
+
+			label := tc.name + timeTag
+
+			// Update the metric with the test duration.
+			testDuration.WithLabelValues(label).Set(duration)
+
+			t.Logf("Pushing testDuration %v with label %v to gateway", duration, label)
+
+			// Create a new pusher to push the metrics.
+			pusher := push.New(cfg.PrometheusGateway.PushURL, "load_test").
+				Collector(testDuration)
+
+			// Push the metrics to Prometheus PushGateway.
+			if err := pusher.Add(); err != nil {
+				t.Logf("Could not push metrics to Prometheus PushGateway: %v",
+					err)
+			} else {
+				t.Logf("Metrics pushed for test case '%s': duration = %v seconds",
+					tc.name, duration)
+			}
+		}
 	}
 }