Add the ability to configure a ratio of proxy metrics to be recorded (#2772)

This ratio defaults to 1.0 (i.e. all metrics will be recorded), but we will set
it much lower in sandbox and production, probably something closer to 0.01. This
will reduce recorded metrics volume and thus StackDriver cost, while still
retaining enough data for overall performance monitoring.

This is handled stochastically, so as to not require any coordination between
Java threads or GKE pods/clusters, as alternative approaches would (i.e. using a
counter and recording every Nth, or throttling to a max metrics qps).

proxy/src/test/java/google/registry/proxy/ProtocolModuleTest.java

@@ -56,6 +56,7 @@ import jakarta.inject.Named;
 import jakarta.inject.Provider;
 import jakarta.inject.Singleton;
 import java.time.Duration;
+import java.util.Random;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
@@ -312,6 +313,26 @@ public abstract class ProtocolModuleTest {
       return Duration.ofHours(1);
     }
 
+    @Singleton
+    @Provides
+    @Named("frontendMetricsRatio")
+    static double provideFrontendMetricsRatio() {
+      return 1.0;
+    }
+
+    @Singleton
+    @Provides
+    @Named("backendMetricsRatio")
+    static double providebackendMetricsRatio() {
+      return 1.0;
+    }
+
+    @Singleton
+    @Provides
+    static Random provideRandom() {
+      return new Random();
+    }
+
     // This method is only here to satisfy Dagger binding, but is never used. In test environment,
     // it is the self-signed certificate and its key that ends up being used.
     @Singleton

proxy/src/test/java/google/registry/proxy/metric/BackendMetricsTest.java

@@ -18,11 +18,14 @@ import static com.google.monitoring.metrics.contrib.DistributionMetricSubject.as
 import static com.google.monitoring.metrics.contrib.LongMetricSubject.assertThat;
 import static google.registry.proxy.TestUtils.makeHttpPostRequest;
 import static google.registry.proxy.TestUtils.makeHttpResponse;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 import com.google.common.collect.ImmutableSet;
 import io.netty.handler.codec.http.FullHttpRequest;
 import io.netty.handler.codec.http.FullHttpResponse;
 import io.netty.handler.codec.http.HttpResponseStatus;
+import java.util.Random;
 import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -34,10 +37,11 @@ class BackendMetricsTest {
   private final String certHash = "blah12345";
   private final String protocol = "frontend protocol";
 
-  private final BackendMetrics metrics = new BackendMetrics();
+  private BackendMetrics metrics;
 
   @BeforeEach
   void beforeEach() {
+    metrics = new BackendMetrics(1.0, new Random());
     metrics.resetMetrics();
   }
 
@@ -107,15 +111,21 @@ class BackendMetricsTest {
 
   @Test
   void testSuccess_multipleResponses() {
+    Random mockRandom = mock(Random.class);
+    metrics = new BackendMetrics(0.2, mockRandom);
+    // The third response won't be logged.
+    when(mockRandom.nextDouble()).thenReturn(.1, .04, .5, .15);
     String content1 = "some response";
     String content2 = "other response";
     String content3 = "a very bad response";
     FullHttpResponse response1 = makeHttpResponse(content1, HttpResponseStatus.OK);
     FullHttpResponse response2 = makeHttpResponse(content2, HttpResponseStatus.OK);
-    FullHttpResponse response3 = makeHttpResponse(content3, HttpResponseStatus.BAD_REQUEST);
+    FullHttpResponse response3 = makeHttpResponse(content2, HttpResponseStatus.OK);
+    FullHttpResponse response4 = makeHttpResponse(content3, HttpResponseStatus.BAD_REQUEST);
     metrics.responseReceived(protocol, certHash, response1, Duration.millis(5));
     metrics.responseReceived(protocol, certHash, response2, Duration.millis(8));
-    metrics.responseReceived(protocol, certHash, response3, Duration.millis(2));
+    metrics.responseReceived(protocol, certHash, response3, Duration.millis(15));
+    metrics.responseReceived(protocol, certHash, response4, Duration.millis(2));
 
     assertThat(BackendMetrics.requestsCounter).hasNoOtherValues();
     assertThat(BackendMetrics.requestBytes).hasNoOtherValues();

proxy/src/test/java/google/registry/proxy/metric/FrontendMetricsTest.java

@@ -15,11 +15,17 @@
 package google.registry.proxy.metric;
 
 import static com.google.common.truth.Truth.assertThat;
+import static com.google.monitoring.metrics.contrib.DistributionMetricSubject.assertThat;
 import static com.google.monitoring.metrics.contrib.LongMetricSubject.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
+import com.google.common.collect.ImmutableSet;
 import io.netty.channel.ChannelFuture;
 import io.netty.channel.DefaultChannelId;
 import io.netty.channel.embedded.EmbeddedChannel;
+import java.util.Random;
+import org.joda.time.Duration;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
@@ -28,10 +34,11 @@ class FrontendMetricsTest {
 
   private static final String PROTOCOL = "some protocol";
   private static final String CERT_HASH = "abc_blah_1134zdf";
-  private final FrontendMetrics metrics = new FrontendMetrics();
+  private FrontendMetrics metrics;
 
   @BeforeEach
   void beforeEach() {
+    metrics = new FrontendMetrics(1.0, new Random());
     metrics.resetMetrics();
   }
 
@@ -60,8 +67,13 @@ class FrontendMetricsTest {
 
   @Test
   void testSuccess_twoConnections_sameClient() {
+    Random mockRandom = mock(Random.class);
+    metrics = new FrontendMetrics(0.2, mockRandom);
+    // The third response won't be logged.
+    when(mockRandom.nextDouble()).thenReturn(.1, .04, .5);
     EmbeddedChannel channel1 = new EmbeddedChannel();
     EmbeddedChannel channel2 = new EmbeddedChannel(DefaultChannelId.newInstance());
+    EmbeddedChannel channel3 = new EmbeddedChannel();
 
     metrics.registerActiveConnection(PROTOCOL, CERT_HASH, channel1);
     assertThat(channel1.isActive()).isTrue();
@@ -85,6 +97,27 @@ class FrontendMetricsTest {
         .and()
         .hasNoOtherValues();
 
+    metrics.responseSent(PROTOCOL, CERT_HASH, Duration.millis(10));
+    metrics.responseSent(PROTOCOL, CERT_HASH, Duration.millis(8));
+    metrics.responseSent(PROTOCOL, CERT_HASH, Duration.millis(13));
+
+    metrics.registerActiveConnection(PROTOCOL, CERT_HASH, channel3);
+    assertThat(channel3.isActive()).isTrue();
+    assertThat(FrontendMetrics.activeConnectionsGauge)
+        .hasValueForLabels(2, PROTOCOL, CERT_HASH)
+        .and()
+        .hasNoOtherValues();
+    // All connection counts are recorded as metrics, but ...
+    assertThat(FrontendMetrics.totalConnectionsCounter)
+        .hasValueForLabels(3, PROTOCOL, CERT_HASH)
+        .and()
+        .hasNoOtherValues();
+    // Latency stats are subject to the metrics ratio.
+    assertThat(FrontendMetrics.latencyMs)
+        .hasDataSetForLabels(ImmutableSet.of(10, 8), PROTOCOL, CERT_HASH)
+        .and()
+        .hasNoOtherValues();
+
     @SuppressWarnings("unused")
     ChannelFuture unusedFuture1 = channel1.close();
     assertThat(channel1.isActive()).isFalse();
@@ -93,7 +126,7 @@ class FrontendMetricsTest {
         .and()
         .hasNoOtherValues();
     assertThat(FrontendMetrics.totalConnectionsCounter)
-        .hasValueForLabels(2, PROTOCOL, CERT_HASH)
+        .hasValueForLabels(3, PROTOCOL, CERT_HASH)
         .and()
         .hasNoOtherValues();
 
@@ -102,7 +135,16 @@ class FrontendMetricsTest {
     assertThat(channel2.isActive()).isFalse();
     assertThat(FrontendMetrics.activeConnectionsGauge).hasNoOtherValues();
     assertThat(FrontendMetrics.totalConnectionsCounter)
-        .hasValueForLabels(2, PROTOCOL, CERT_HASH)
+        .hasValueForLabels(3, PROTOCOL, CERT_HASH)
+        .and()
+        .hasNoOtherValues();
+
+    @SuppressWarnings("unused")
+    ChannelFuture unusedFuture3 = channel3.close();
+    assertThat(channel3.isActive()).isFalse();
+    assertThat(FrontendMetrics.activeConnectionsGauge).hasNoOtherValues();
+    assertThat(FrontendMetrics.totalConnectionsCounter)
+        .hasValueForLabels(3, PROTOCOL, CERT_HASH)
         .and()
         .hasNoOtherValues();
   }