Add the ability to configure a ratio of proxy metrics to be recorded

This ratio defaults to 1.0 (i.e. all metrics will be recorded), but we will set
it much lower in sandbox and production, probably something closer to 0.01. This
will reduce recorded metrics volume and thus StackDriver cost, while still
retaining enough data for overall performance monitoring.

This is handled stochastically, so as to not require any coordination between
Java threads or GKE pods/clusters, as alternative approaches would (i.e. using a
counter and recording every Nth, or throttling to a max metrics qps).

Author: CydeWeys

proxy/src/main/java/google/registry/proxy/ProxyConfig.java

@@ -113,6 +113,8 @@ public static class Metrics {
     public int stackdriverMaxQps;
     public int stackdriverMaxPointsPerRequest;
     public int writeIntervalSeconds;
+    public double frontendMetricsRatio;
+    public double backendMetricsRatio;
   }
 
   /** Configuration options that apply to quota management. */

proxy/src/main/java/google/registry/proxy/ProxyModule.java

@@ -395,6 +395,20 @@ static Duration provideCertCachingDuration(ProxyConfig config) {
     return Duration.ofSeconds(config.serverCertificateCacheSeconds);
   }
 
+  @Singleton
+  @Provides
+  @Named("frontendMetricsRatio")
+  static double provideFrontendMetricsRatio(ProxyConfig config) {
+    return config.metrics.frontendMetricsRatio;
+  }
+
+  @Singleton
+  @Provides
+  @Named("backendMetricsRatio")
+  static double provideBackendMetricsRatio(ProxyConfig config) {
+    return config.metrics.backendMetricsRatio;
+  }
+
   /** Root level component that exposes the port-to-protocol map. */
   @Singleton
   @Component(

proxy/src/main/java/google/registry/proxy/config/default-config.yaml

@@ -200,3 +200,15 @@ metrics:
 
   # How often metrics are written.
   writeIntervalSeconds: 60
+
+  # What ratio of frontend request metrics should be stochastically recorded
+  # (0.0 means none, 1.0 means all). This is useful for reducing metrics volume,
+  # and thus cost, while still recording some information for performance
+  # monitoring purposes.
+  frontendMetricsRatio: 1.0
+
+  # What ratio of backend request metrics should be stochastically recorded
+  # (0.0 means none, 1.0 means all). This is useful for reducing metrics volume,
+  # and thus cost, while still recording some information for performance
+  # monitoring purposes.
+  backendMetricsRatio: 1.0

proxy/src/main/java/google/registry/proxy/metric/BackendMetrics.java

@@ -14,6 +14,7 @@
 
 package google.registry.proxy.metric;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableSet;
 import com.google.monitoring.metrics.EventMetric;
 import com.google.monitoring.metrics.IncrementableMetric;
@@ -22,7 +23,9 @@
 import google.registry.util.NonFinalForTesting;
 import io.netty.handler.codec.http.FullHttpResponse;
 import jakarta.inject.Inject;
+import jakarta.inject.Named;
 import jakarta.inject.Singleton;
+import java.util.Random;
 import org.joda.time.Duration;
 
 /** Backend metrics instrumentation. */
@@ -75,8 +78,18 @@ public class BackendMetrics extends BaseMetrics {
               LABELS,
               DEFAULT_LATENCY_FITTER);
 
+  @NonFinalForTesting
+  @VisibleForTesting
+  Random random = new Random();
+
+  @NonFinalForTesting
+  @VisibleForTesting
+  double backendMetricsRatio;
+
   @Inject
-  BackendMetrics() {}
+  BackendMetrics(@Named("backendMetricsRatio") double backendMetricsRatio) {
+    this.backendMetricsRatio = backendMetricsRatio;
+  }
 
   @Override
   void resetMetrics() {
@@ -89,13 +102,21 @@ void resetMetrics() {
 
   @NonFinalForTesting
   public void requestSent(String protocol, String certHash, int bytes) {
+    // Short-circuit metrics recording randomly according to the configured ratio.
+    if (random.nextDouble() > backendMetricsRatio) {
+      return;
+    }
     requestsCounter.increment(protocol, certHash);
     requestBytes.record(bytes, protocol, certHash);
   }
 
   @NonFinalForTesting
   public void responseReceived(
       String protocol, String certHash, FullHttpResponse response, Duration latency) {
+    // Short-circuit metrics recording randomly according to the configured ratio.
+    if (random.nextDouble() > backendMetricsRatio) {
+      return;
+    }
     latencyMs.record(latency.getMillis(), protocol, certHash);
     responseBytes.record(response.content().readableBytes(), protocol, certHash);
     responsesCounter.increment(protocol, certHash, response.status().toString());

proxy/src/main/java/google/registry/proxy/metric/FrontendMetrics.java

@@ -14,6 +14,7 @@
 
 package google.registry.proxy.metric;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.monitoring.metrics.EventMetric;
@@ -26,8 +27,10 @@
 import io.netty.channel.group.DefaultChannelGroup;
 import io.netty.util.concurrent.GlobalEventExecutor;
 import jakarta.inject.Inject;
+import jakarta.inject.Named;
 import jakarta.inject.Singleton;
 import java.util.Map;
+import java.util.Random;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import org.joda.time.Duration;
@@ -78,8 +81,16 @@ public class FrontendMetrics extends BaseMetrics {
               LABELS,
               DEFAULT_LATENCY_FITTER);
 
+  @NonFinalForTesting
+  @VisibleForTesting
+  Random random = new Random();
+
+  double frontendMetricsRatio;
+
   @Inject
-  public FrontendMetrics() {}
+  FrontendMetrics(@Named("frontendMetricsRatio") double frontendMetricsRatio) {
+    this.frontendMetricsRatio = frontendMetricsRatio;
+  }
 
   @Override
   void resetMetrics() {
@@ -109,6 +120,10 @@ public void registerQuotaRejection(String protocol, String certHash) {
 
   @NonFinalForTesting
   public void responseSent(String protocol, String certHash, Duration latency) {
+    // Short-circuit metrics recording randomly according to the configured ratio.
+    if (random.nextDouble() > frontendMetricsRatio) {
+      return;
+    }
     latencyMs.record(latency.getMillis(), protocol, certHash);
   }
 }

proxy/src/test/java/google/registry/proxy/ProtocolModuleTest.java

@@ -312,6 +312,20 @@ static Duration provideCertCachingDuration() {
       return Duration.ofHours(1);
     }
 
+    @Singleton
+    @Provides
+    @Named("frontendMetricsRatio")
+    static double provideFrontendMetricsRatio() {
+      return 1.0;
+    }
+
+    @Singleton
+    @Provides
+    @Named("backendMetricsRatio")
+    static double providebackendMetricsRatio() {
+      return 1.0;
+    }
+
     // This method is only here to satisfy Dagger binding, but is never used. In test environment,
     // it is the self-signed certificate and its key that ends up being used.
     @Singleton

proxy/src/test/java/google/registry/proxy/metric/BackendMetricsTest.java

@@ -18,11 +18,14 @@
 import static com.google.monitoring.metrics.contrib.LongMetricSubject.assertThat;
 import static google.registry.proxy.TestUtils.makeHttpPostRequest;
 import static google.registry.proxy.TestUtils.makeHttpResponse;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 import com.google.common.collect.ImmutableSet;
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.FullHttpResponse;
 import io.netty.handler.codec.http.HttpResponseStatus;
+import java.util.Random;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -34,7 +37,7 @@ class BackendMetricsTest {
   private final String certHash = "blah12345";
   private final String protocol = "frontend protocol";
 
-  private final BackendMetrics metrics = new BackendMetrics();
+  private final BackendMetrics metrics = new BackendMetrics(1.0);
 
   @BeforeEach
   void beforeEach() {
@@ -107,15 +110,21 @@ void testSuccess_oneResponse() {
 
   @Test
   void testSuccess_multipleResponses() {
+    metrics.backendMetricsRatio = 0.2;
+    metrics.random = mock(Random.class);
+    // The third response won't be logged.
+    when(metrics.random.nextDouble()).thenReturn(.1, .04, .5, .15);
     String content1 = "some response";
     String content2 = "other response";
     String content3 = "a very bad response";
     FullHttpResponse response1 = makeHttpResponse(content1, HttpResponseStatus.OK);
     FullHttpResponse response2 = makeHttpResponse(content2, HttpResponseStatus.OK);
-    FullHttpResponse response3 = makeHttpResponse(content3, HttpResponseStatus.BAD_REQUEST);
+    FullHttpResponse response3 = makeHttpResponse(content2, HttpResponseStatus.OK);
+    FullHttpResponse response4 = makeHttpResponse(content3, HttpResponseStatus.BAD_REQUEST);
     metrics.responseReceived(protocol, certHash, response1, Duration.millis(5));
     metrics.responseReceived(protocol, certHash, response2, Duration.millis(8));
-    metrics.responseReceived(protocol, certHash, response3, Duration.millis(2));
+    metrics.responseReceived(protocol, certHash, response3, Duration.millis(15));
+    metrics.responseReceived(protocol, certHash, response4, Duration.millis(2));
 
     assertThat(BackendMetrics.requestsCounter).hasNoOtherValues();
     assertThat(BackendMetrics.requestBytes).hasNoOtherValues();

proxy/src/test/java/google/registry/proxy/metric/FrontendMetricsTest.java

@@ -15,11 +15,17 @@
 package google.registry.proxy.metric;
 
 import static com.google.common.truth.Truth.assertThat;
+import static com.google.monitoring.metrics.contrib.DistributionMetricSubject.assertThat;
 import static com.google.monitoring.metrics.contrib.LongMetricSubject.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
+import com.google.common.collect.ImmutableSet;
 import io.netty.channel.ChannelFuture;
 import io.netty.channel.DefaultChannelId;
 import io.netty.channel.embedded.EmbeddedChannel;
+import java.util.Random;
+import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -28,7 +34,7 @@ class FrontendMetricsTest {
 
   private static final String PROTOCOL = "some protocol";
   private static final String CERT_HASH = "abc_blah_1134zdf";
-  private final FrontendMetrics metrics = new FrontendMetrics();
+  private final FrontendMetrics metrics = new FrontendMetrics(1.0);
 
   @BeforeEach
   void beforeEach() {
@@ -60,8 +66,13 @@ void testSuccess_oneConnection() {
 
   @Test
   void testSuccess_twoConnections_sameClient() {
+    metrics.frontendMetricsRatio = 0.2;
+    metrics.random = mock(Random.class);
+    // The third response won't be logged.
+    when(metrics.random.nextDouble()).thenReturn(.1, .04, .5);
     EmbeddedChannel channel1 = new EmbeddedChannel();
     EmbeddedChannel channel2 = new EmbeddedChannel(DefaultChannelId.newInstance());
+    EmbeddedChannel channel3 = new EmbeddedChannel();
 
     metrics.registerActiveConnection(PROTOCOL, CERT_HASH, channel1);
     assertThat(channel1.isActive()).isTrue();
@@ -85,6 +96,27 @@ void testSuccess_twoConnections_sameClient() {
         .and()
         .hasNoOtherValues();
 
+    metrics.responseSent(PROTOCOL, CERT_HASH, Duration.millis(10));
+    metrics.responseSent(PROTOCOL, CERT_HASH, Duration.millis(8));
+    metrics.responseSent(PROTOCOL, CERT_HASH, Duration.millis(13));
+
+    metrics.registerActiveConnection(PROTOCOL, CERT_HASH, channel3);
+    assertThat(channel3.isActive()).isTrue();
+    assertThat(FrontendMetrics.activeConnectionsGauge)
+        .hasValueForLabels(2, PROTOCOL, CERT_HASH)
+        .and()
+        .hasNoOtherValues();
+    // All connection counts are recorded as metrics, but ...
+    assertThat(FrontendMetrics.totalConnectionsCounter)
+        .hasValueForLabels(3, PROTOCOL, CERT_HASH)
+        .and()
+        .hasNoOtherValues();
+    // Latency stats are subject to the metrics ratio.
+    assertThat(FrontendMetrics.latencyMs).hasDataSetForLabels(ImmutableSet.of(10, 8), PROTOCOL,
+            CERT_HASH)
+        .and()
+        .hasNoOtherValues();
+
     @SuppressWarnings("unused")
     ChannelFuture unusedFuture1 = channel1.close();
     assertThat(channel1.isActive()).isFalse();
@@ -93,7 +125,7 @@ void testSuccess_twoConnections_sameClient() {
         .and()
         .hasNoOtherValues();
     assertThat(FrontendMetrics.totalConnectionsCounter)
-        .hasValueForLabels(2, PROTOCOL, CERT_HASH)
+        .hasValueForLabels(3, PROTOCOL, CERT_HASH)
         .and()
         .hasNoOtherValues();
 
@@ -102,7 +134,16 @@ void testSuccess_twoConnections_sameClient() {
     assertThat(channel2.isActive()).isFalse();
     assertThat(FrontendMetrics.activeConnectionsGauge).hasNoOtherValues();
     assertThat(FrontendMetrics.totalConnectionsCounter)
-        .hasValueForLabels(2, PROTOCOL, CERT_HASH)
+        .hasValueForLabels(3, PROTOCOL, CERT_HASH)
+        .and()
+        .hasNoOtherValues();
+
+    @SuppressWarnings("unused")
+    ChannelFuture unusedFuture3 = channel3.close();
+    assertThat(channel3.isActive()).isFalse();
+    assertThat(FrontendMetrics.activeConnectionsGauge).hasNoOtherValues();
+    assertThat(FrontendMetrics.totalConnectionsCounter)
+        .hasValueForLabels(3, PROTOCOL, CERT_HASH)
         .and()
         .hasNoOtherValues();
   }