SOLR-17865: Support optional node level metrics recording with OTEL

solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java

@@ -42,13 +42,13 @@
 import org.apache.solr.core.PluginInfo;
 import org.apache.solr.core.SolrCore;
 import org.apache.solr.core.SolrInfoBean;
-import org.apache.solr.metrics.SolrDelegateRegistryMetricsContext;
 import org.apache.solr.metrics.SolrMetricManager;
-import org.apache.solr.metrics.SolrMetricProducer;
 import org.apache.solr.metrics.SolrMetricsContext;
 import org.apache.solr.metrics.otel.OtelUnit;
 import org.apache.solr.metrics.otel.instruments.AttributedLongCounter;
 import org.apache.solr.metrics.otel.instruments.AttributedLongTimer;
+import org.apache.solr.metrics.otel.instruments.DualRegistryAttributedLongCounter;
+import org.apache.solr.metrics.otel.instruments.DualRegistryAttributedLongTimer;
 import org.apache.solr.request.SolrQueryRequest;
 import org.apache.solr.request.SolrRequestHandler;
 import org.apache.solr.response.SolrQueryResponse;
@@ -168,21 +168,13 @@ public SolrMetricsContext getSolrMetricsContext() {
   @Override
   public void initializeMetrics(
       SolrMetricsContext parentContext, Attributes attributes, String scope) {
-    if (aggregateNodeLevelMetricsEnabled) {
-      this.solrMetricsContext =
-          new SolrDelegateRegistryMetricsContext(
-              parentContext.getMetricManager(),
-              parentContext.getRegistryName(),
-              SolrMetricProducer.getUniqueMetricTag(this, parentContext.getTag()),
-              SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));
-    } else {
-      this.solrMetricsContext = parentContext.getChildContext(this);
-    }
+    this.solrMetricsContext = parentContext.getChildContext(this);
 
     metrics =
         new HandlerMetrics(
             solrMetricsContext,
-            attributes.toBuilder().put(CATEGORY_ATTR, getCategory().toString()).build());
+            attributes.toBuilder().put(CATEGORY_ATTR, getCategory().toString()).build(),
+            aggregateNodeLevelMetricsEnabled);
 
     // NOCOMMIT: I don't see value in this metric
     solrMetricsContext.gauge(
@@ -198,62 +190,133 @@ public static class HandlerMetrics {
                     null, new MetricsConfig.MetricsConfigBuilder().setEnabled(false).build()),
                 "NO_OP",
                 "NO_OP"),
-            Attributes.empty());
+            Attributes.empty(),
+            false);
 
     public AttributedLongCounter requests;
     public AttributedLongCounter numServerErrors;
     public AttributedLongCounter numClientErrors;
     public AttributedLongCounter numTimeouts;
     public AttributedLongTimer requestTimes;
 
-    public HandlerMetrics(SolrMetricsContext solrMetricsContext, Attributes attributes) {
-
-      LongCounter requestMetric;
-      LongCounter errorRequestMetric;
-      LongCounter timeoutRequestMetric;
-      LongHistogram requestTimeMetric;
-
-      if (solrMetricsContext.getRegistryName().equals("solr.node")) {
-        requestMetric =
-            solrMetricsContext.longCounter("solr_node_requests", "Http Solr node requests");
-        errorRequestMetric =
-            solrMetricsContext.longCounter(
-                "solr_node_requests_errors", "HTTP Solr node request errors");
-        timeoutRequestMetric =
-            solrMetricsContext.longCounter(
-                "solr_node_requests_timeout", "HTTP Solr node request timeouts");
-        requestTimeMetric =
-            solrMetricsContext.longHistogram(
-                "solr_node_requests_times", "HTTP Solr node request times", OtelUnit.MILLISECONDS);
+    public HandlerMetrics(
+        SolrMetricsContext solrMetricsContext,
+        Attributes coreAttributes,
+        boolean aggregateNodeLevelMetricsEnabled) {
+
+      if (aggregateNodeLevelMetricsEnabled) {
+        createDualRegistryMetrics(solrMetricsContext, coreAttributes);
       } else {
-        requestMetric =
-            solrMetricsContext.longCounter("solr_core_requests", "HTTP Solr core requests");
-        errorRequestMetric =
-            solrMetricsContext.longCounter(
-                "solr_core_requests_errors", "HTTP Solr core request errors");
-        timeoutRequestMetric =
-            solrMetricsContext.longCounter(
-                "solr_core_requests_timeout", "HTTP Solr core request timeouts");
-        requestTimeMetric =
-            solrMetricsContext.longHistogram(
-                "solr_core_requests_times", "HTTP Solr core request times", OtelUnit.MILLISECONDS);
+        createSingleRegistryMetrics(solrMetricsContext, coreAttributes);
       }
+    }
 
-      requests = new AttributedLongCounter(requestMetric, attributes);
+    private void createDualRegistryMetrics(
+        SolrMetricsContext solrCoreMetricsContext, Attributes coreAttributes) {
+      SolrMetricsContext solrNodeMetricsContext =
+          new SolrMetricsContext(
+              solrCoreMetricsContext.getMetricManager(),
+              SolrMetricManager.getRegistryName(SolrInfoBean.Group.node),
+              null);
+
+      Attributes nodeAttributes =
+          Attributes.builder()
+              .put(CATEGORY_ATTR, coreAttributes.get(CATEGORY_ATTR))
+              .put(HANDLER_ATTR, coreAttributes.get(HANDLER_ATTR))
+              .build();
+
+      requests =
+          new DualRegistryAttributedLongCounter(
+              requestCounter(solrCoreMetricsContext, false), coreAttributes,
+              requestCounter(solrNodeMetricsContext, true), nodeAttributes);
+      numServerErrors =
+          new DualRegistryAttributedLongCounter(
+              requestErrorCounter(solrCoreMetricsContext, false),
+              coreAttributes.toBuilder().put(AttributeKey.stringKey("source"), "server").build(),
+              requestErrorCounter(solrNodeMetricsContext, true),
+              nodeAttributes.toBuilder().put(AttributeKey.stringKey("source"), "server").build());
+
+      numClientErrors =
+          new DualRegistryAttributedLongCounter(
+              requestErrorCounter(solrCoreMetricsContext, false),
+              coreAttributes.toBuilder().put(AttributeKey.stringKey("source"), "client").build(),
+              requestErrorCounter(solrNodeMetricsContext, true),
+              nodeAttributes.toBuilder().put(AttributeKey.stringKey("source"), "client").build());
+      numTimeouts =
+          new DualRegistryAttributedLongCounter(
+              requestTimeoutCounter(solrCoreMetricsContext, false), coreAttributes,
+              requestTimeoutCounter(solrNodeMetricsContext, true), nodeAttributes);
+      requestTimes =
+          new DualRegistryAttributedLongTimer(
+              requestTimeHistogram(solrCoreMetricsContext, false), coreAttributes,
+              requestTimeHistogram(solrNodeMetricsContext, true), nodeAttributes);
+    }
 
+    private void createSingleRegistryMetrics(
+        SolrMetricsContext solrMetricsContext, Attributes coreAttributes) {
+      boolean isNodeRegistry =
+          solrMetricsContext
+              .getRegistryName()
+              .equals(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node));
+
+      Attributes attributes =
+          isNodeRegistry
+              ? Attributes.builder()
+                  .put(CATEGORY_ATTR, coreAttributes.get(CATEGORY_ATTR))
+                  .put(HANDLER_ATTR, coreAttributes.get(HANDLER_ATTR))
+                  .build()
+              : coreAttributes;
+
+      requests =
+          new AttributedLongCounter(requestCounter(solrMetricsContext, isNodeRegistry), attributes);
       numServerErrors =
           new AttributedLongCounter(
-              errorRequestMetric,
+              requestErrorCounter(solrMetricsContext, isNodeRegistry),
               attributes.toBuilder().put(AttributeKey.stringKey("source"), "server").build());
-
       numClientErrors =
           new AttributedLongCounter(
-              errorRequestMetric,
+              requestErrorCounter(solrMetricsContext, isNodeRegistry),
               attributes.toBuilder().put(AttributeKey.stringKey("source"), "client").build());
+      numTimeouts =
+          new AttributedLongCounter(
+              requestTimeoutCounter(solrMetricsContext, isNodeRegistry), attributes);
+      requestTimes =
+          new AttributedLongTimer(
+              requestTimeHistogram(solrMetricsContext, isNodeRegistry), attributes);
+    }
+
+    private LongCounter requestCounter(
+        SolrMetricsContext solrMetricsContext, boolean isNodeRegistry) {
+      return (isNodeRegistry)
+          ? solrMetricsContext.longCounter("solr_node_requests", "HTTP Solr node requests")
+          : solrMetricsContext.longCounter("solr_core_requests", "HTTP Solr core requests");
+    }
+
+    private LongCounter requestErrorCounter(
+        SolrMetricsContext solrMetricsContext, boolean isNodeRegistry) {
+      return (isNodeRegistry)
+          ? solrMetricsContext.longCounter(
+              "solr_node_requests_errors", "HTTP Solr node request errors")
+          : solrMetricsContext.longCounter(
+              "solr_core_requests_errors", "HTTP Solr core request errors");
+    }
 
-      numTimeouts = new AttributedLongCounter(timeoutRequestMetric, attributes);
+    private LongCounter requestTimeoutCounter(
+        SolrMetricsContext solrMetricsContext, boolean isNodeRegistry) {
+      return (isNodeRegistry)
+          ? solrMetricsContext.longCounter(
+              "solr_node_requests_timeout", "HTTP Solr node request timeouts")
+          : solrMetricsContext.longCounter(
+              "solr_core_requests_timeout", "HTTP Solr core request timeouts");
+    }
 
-      requestTimes = new AttributedLongTimer(requestTimeMetric, attributes);
+    private LongHistogram requestTimeHistogram(
+        SolrMetricsContext solrMetricsContext, boolean isNodeRegistry) {
+      return (isNodeRegistry)
+          ? solrMetricsContext.longHistogram(
+              "solr_node_requests_times", "HTTP Solr node request times", OtelUnit.MILLISECONDS)
+          : solrMetricsContext.longHistogram(
+              "solr_core_requests_times", "HTTP Solr core request times", OtelUnit.MILLISECONDS);
     }
   }
 

solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java

@@ -98,8 +98,6 @@ public class SearchHandler extends RequestHandlerBase
   static final String INIT_FIRST_COMPONENTS = "first-components";
   static final String INIT_LAST_COMPONENTS = "last-components";
 
-  protected static final String SHARD_HANDLER_SUFFIX = "[shard]";
-
   private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
   /**
@@ -171,7 +169,8 @@ public void initializeMetrics(
                 .putAll(attributes)
                 .put(CATEGORY_ATTR, getCategory().toString())
                 .put(INTERNAL_ATTR, true)
-                .build());
+                .build(),
+            false);
   }
 
   @Override

solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java

@@ -30,7 +30,7 @@ public AttributedDoubleCounter(DoubleCounter counter, Attributes attributes) {
   }
 
   public void inc() {
-    add(1.0);
+    counter.add(1.0, attributes);
   }
 
   public void add(Double value) {

solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java

@@ -30,11 +30,11 @@ public AttributedDoubleUpDownCounter(DoubleUpDownCounter upDownCounter, Attribut
   }
 
   public void inc() {
-    add(1.0);
+    upDownCounter.add(1.0, attributes);
   }
 
   public void dec() {
-    add(-1.0);
+    upDownCounter.add(-1.0, attributes);
   }
 
   public void add(Double value) {

solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java

@@ -30,7 +30,7 @@ public AttributedLongCounter(LongCounter baseCounter, Attributes attributes) {
   }
 
   public void inc() {
-    add(1L);
+    baseCounter.add(1L, attributes);
   }
 
   public void add(Long value) {

solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java

@@ -30,11 +30,11 @@ public AttributedLongUpDownCounter(LongUpDownCounter upDownCounter, Attributes a
   }
 
   public void inc() {
-    add(1L);
+    upDownCounter.add(1L, attributes);
   }
 
   public void dec() {
-    add(-1L);
+    upDownCounter.add(-1L, attributes);
   }
 
   public void add(Long value) {

solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.otel.instruments;
+
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.LongCounter;
+
+/**
+ * An AttributedLongCounter that writes to both core and node registries with corresponding
+ * attributes.
+ */
+public class DualRegistryAttributedLongCounter extends AttributedLongCounter {
+
+  private final AttributedLongCounter nodeCounter;
+
+  public DualRegistryAttributedLongCounter(
+      LongCounter coreCounter,
+      Attributes coreAttributes,
+      LongCounter nodeCounter,
+      Attributes nodeAttributes) {
+    super(coreCounter, coreAttributes);
+    this.nodeCounter = new AttributedLongCounter(nodeCounter, nodeAttributes);
+  }
+
+  @Override
+  public void inc() {
+    super.inc();
+    nodeCounter.inc();
+  }
+
+  @Override
+  public void add(Long value) {
+    super.add(value);
+    nodeCounter.add(value);
+  }
+}

solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.java

@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.metrics.otel.instruments;
+
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.metrics.LongHistogram;
+
+/**
+ * An AttributedLongTimer that records to both core and node registries with corresponding
+ * attributes.
+ */
+public class DualRegistryAttributedLongTimer extends AttributedLongTimer {
+
+  private final AttributedLongTimer nodeTimer;
+
+  public DualRegistryAttributedLongTimer(
+      LongHistogram coreHistogram,
+      Attributes coreAttributes,
+      LongHistogram nodeHistogram,
+      Attributes nodeAttributes) {
+    super(coreHistogram, coreAttributes);
+    this.nodeTimer = new AttributedLongTimer(nodeHistogram, nodeAttributes);
+  }
+
+  @Override
+  public void record(Long value) {
+    super.record(value);
+    nodeTimer.record(value);
+  }
+}

solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java

@@ -196,6 +196,8 @@ private RequestHandlerBase.HandlerMetrics createHandlerMetrics() {
     when(metricsContext.longHistogram(any(), any())).thenReturn(mockLongHistogram);
 
     return new RequestHandlerBase.HandlerMetrics(
-        metricsContext, Attributes.of(AttributeKey.stringKey("/handler"), "/someBaseMetricPath"));
+        metricsContext,
+        Attributes.of(AttributeKey.stringKey("/handler"), "/someBaseMetricPath"),
+        false);
   }
 }