Use LongUpDownCounter for Linked Project Error Metrics (elastic#139657)

Replaces LongCounter with LongUpDownCounter for the metric
used for linked project connection errors.  Since the metric
may not be incremented for long periods of time it was not
appearing in the mappings of the current index of the APM
datastream.  This results in 'not found' errors when developing
dashboards and alerts.  The change required adding an initial
zero value so the metric will be collected every polling cycle.

Relates: ES-12696

Author: JeremyDahlgren

server/src/main/java/org/elasticsearch/transport/RemoteClusterService.java

@@ -102,9 +102,13 @@ public boolean isRemoteClusterServerEnabled() {
          *  the functionality to do it the right way is not yet ready -- replace this code when it's ready.
          */
         this.crossProjectEnabled = settings.getAsBoolean("serverless.cross_project.enabled", false);
+        // Since this counter may never be incremented we need to force an initial observed value of zero via add(0) so the metric will be
+        // in the mappings of the indices of the APM data stream, otherwise we will encounter 'not found' errors in dashboards and alerts.
+        // See observability-dev issue #3042.
         transportService.getTelemetryProvider()
             .getMeterRegistry()
-            .registerLongCounter(CONNECTION_ATTEMPT_FAILURES_COUNTER_NAME, "linked project connection attempt failure count", "count");
+            .registerLongUpDownCounter(CONNECTION_ATTEMPT_FAILURES_COUNTER_NAME, "linked project connection attempt failure count", "count")
+            .add(0);
     }
 
     public RemoteClusterCredentialsManager getRemoteClusterCredentialsManager() {

server/src/main/java/org/elasticsearch/transport/RemoteConnectionStrategy.java

@@ -21,7 +21,7 @@
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.telemetry.TelemetryProvider;
-import org.elasticsearch.telemetry.metric.LongCounter;
+import org.elasticsearch.telemetry.metric.LongUpDownCounter;
 import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.Closeable;
@@ -78,10 +78,9 @@ enum ConnectionAttempt {
         reconnect
     }
 
-    private static final String metricLabelPrefix = "es_linked_project";
-    static final String linkedProjectIdLabel = metricLabelPrefix + "_id";
-    static final String linkedProjectAliasLabel = metricLabelPrefix + "_alias";
-    static final String connectionAtemptLabel = metricLabelPrefix + "_attempt";
+    static final String linkedProjectIdLabel = "es_linked_project_id";
+    static final String linkedProjectAliasLabel = "es_linked_project_alias";
+    static final String connectionAtemptLabel = "es_linked_project_attempt";
 
     private final int maxPendingConnectionListeners;
 
@@ -91,7 +90,7 @@ enum ConnectionAttempt {
     private final Object mutex = new Object();
     private List<ActionListener<Void>> listeners = new ArrayList<>();
     private final AtomicBoolean initialConnectionAttempted = new AtomicBoolean(false);
-    private final LongCounter connectionAttemptFailures;
+    private final LongUpDownCounter connectionAttemptFailures;
 
     protected final TransportService transportService;
     protected final RemoteConnectionManager connectionManager;
@@ -110,9 +109,11 @@ enum ConnectionAttempt {
         connectionManager.addListener(this);
     }
 
-    private LongCounter lookupConnectionFailureMetric(TelemetryProvider telemetryProvider) {
+    private LongUpDownCounter lookupConnectionFailureMetric(TelemetryProvider telemetryProvider) {
         final var meterRegistry = telemetryProvider == null ? null : telemetryProvider.getMeterRegistry();
-        return meterRegistry == null ? null : meterRegistry.getLongCounter(RemoteClusterService.CONNECTION_ATTEMPT_FAILURES_COUNTER_NAME);
+        return meterRegistry == null
+            ? null
+            : meterRegistry.getLongUpDownCounter(RemoteClusterService.CONNECTION_ATTEMPT_FAILURES_COUNTER_NAME);
     }
 
     static ConnectionProfile buildConnectionProfile(LinkedProjectConfig config, String transportProfile) {
@@ -242,7 +243,7 @@ private void connectionAttemptCompleted(@Nullable Exception e) {
         } else {
             logger.warn(msgSupplier, e);
             if (connectionAttemptFailures != null) {
-                connectionAttemptFailures.incrementBy(
+                connectionAttemptFailures.add(
                     1,
                     Map.of(
                         linkedProjectIdLabel,

server/src/test/java/org/elasticsearch/transport/RemoteConnectionStrategyTests.java

@@ -266,7 +266,7 @@ public void testConnectionAttemptMetricsAndLogging() {
                     if (shouldConnectFail) {
                         metricRecorder.collect();
                         final var counterName = RemoteClusterService.CONNECTION_ATTEMPT_FAILURES_COUNTER_NAME;
-                        final var measurements = metricRecorder.getMeasurements(InstrumentType.LONG_COUNTER, counterName);
+                        final var measurements = metricRecorder.getMeasurements(InstrumentType.LONG_UP_DOWN_COUNTER, counterName);
                         assertFalse(measurements.isEmpty());
                         final var measurement = measurements.getLast();
                         assertThat(measurement.getLong(), equalTo(1L));