diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java index f726842d773..f2407a4273e 100644 --- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java +++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java @@ -21,8 +21,6 @@ import io.opentelemetry.api.common.AttributeKey; import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.LongCounter; -import io.opentelemetry.api.metrics.LongHistogram; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.util.Collection; @@ -41,11 +39,10 @@ import org.apache.solr.core.PluginInfo; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrInfoBean; -import org.apache.solr.metrics.SolrDelegateRegistryMetricsContext; import org.apache.solr.metrics.SolrMetricManager; -import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.metrics.otel.OtelUnit; +import org.apache.solr.metrics.otel.instruments.AttributedInstrumentFactory; import org.apache.solr.metrics.otel.instruments.AttributedLongCounter; import org.apache.solr.metrics.otel.instruments.AttributedLongTimer; import org.apache.solr.request.SolrQueryRequest; @@ -72,6 +69,7 @@ public abstract class RequestHandlerBase PermissionNameProvider { public static final String REQUEST_CPU_TIMER_CONTEXT = "publishCpuTime"; + public static final AttributeKey SOURCE_ATTR = AttributeKey.stringKey("source"); protected NamedList initArgs = null; protected SolrParams defaults; protected SolrParams appends; @@ -167,21 +165,13 @@ public SolrMetricsContext getSolrMetricsContext() { @Override public void initializeMetrics( SolrMetricsContext parentContext, Attributes attributes, String scope) { - if (aggregateNodeLevelMetricsEnabled) { - this.solrMetricsContext = - new SolrDelegateRegistryMetricsContext( - parentContext.getMetricManager(), - parentContext.getRegistryName(), - SolrMetricProducer.getUniqueMetricTag(this, parentContext.getTag()), - SolrMetricManager.getRegistryName(SolrInfoBean.Group.node)); - } else { - this.solrMetricsContext = parentContext.getChildContext(this); - } + this.solrMetricsContext = parentContext.getChildContext(this); metrics = new HandlerMetrics( solrMetricsContext, - attributes.toBuilder().put(CATEGORY_ATTR, getCategory().toString()).build()); + attributes.toBuilder().put(CATEGORY_ATTR, getCategory().toString()).build(), + aggregateNodeLevelMetricsEnabled); } /** Metrics for this handler. */ @@ -189,7 +179,8 @@ public static class HandlerMetrics { public static final HandlerMetrics NO_OP = new HandlerMetrics( new SolrMetricsContext(new SolrMetricManager(null), "NO_OP", "NO_OP"), - Attributes.empty()); + Attributes.empty(), + false); public AttributedLongCounter requests; public AttributedLongCounter numServerErrors; @@ -197,53 +188,41 @@ public static class HandlerMetrics { public AttributedLongCounter numTimeouts; public AttributedLongTimer requestTimes; - public HandlerMetrics(SolrMetricsContext solrMetricsContext, Attributes attributes) { - LongCounter requestMetric; - LongCounter errorRequestMetric; - LongCounter timeoutRequestMetric; - LongHistogram requestTimeMetric; - - if (solrMetricsContext.getRegistryName().equals("solr.node")) { - requestMetric = - solrMetricsContext.longCounter("solr_node_requests", "Http Solr node requests"); - errorRequestMetric = - solrMetricsContext.longCounter( - "solr_node_requests_errors", "HTTP Solr node request errors"); - timeoutRequestMetric = - solrMetricsContext.longCounter( - "solr_node_requests_timeout", "HTTP Solr node request timeouts"); - requestTimeMetric = - solrMetricsContext.longHistogram( - "solr_node_requests_times", "HTTP Solr node request times", OtelUnit.MILLISECONDS); - } else { - requestMetric = - solrMetricsContext.longCounter("solr_core_requests", "HTTP Solr core requests"); - errorRequestMetric = - solrMetricsContext.longCounter( - "solr_core_requests_errors", "HTTP Solr core request errors"); - timeoutRequestMetric = - solrMetricsContext.longCounter( - "solr_core_requests_timeout", "HTTP Solr core request timeouts"); - requestTimeMetric = - solrMetricsContext.longHistogram( - "solr_core_requests_times", "HTTP Solr core request times", OtelUnit.MILLISECONDS); - } + public HandlerMetrics( + SolrMetricsContext solrMetricsContext, + Attributes coreAttributes, + boolean aggregateNodeLevelMetricsEnabled) { - requests = new AttributedLongCounter(requestMetric, attributes); + AttributedInstrumentFactory factory = + new AttributedInstrumentFactory( + solrMetricsContext, coreAttributes, aggregateNodeLevelMetricsEnabled); + + requests = + factory.attributedLongCounter( + "solr_core_requests", "HTTP Solr requests", Attributes.empty()); numServerErrors = - new AttributedLongCounter( - errorRequestMetric, - attributes.toBuilder().put(AttributeKey.stringKey("source"), "server").build()); + factory.attributedLongCounter( + "solr_core_requests_errors", + "HTTP Solr request errors", + Attributes.of(SOURCE_ATTR, "server")); numClientErrors = - new AttributedLongCounter( - errorRequestMetric, - attributes.toBuilder().put(AttributeKey.stringKey("source"), "client").build()); - - numTimeouts = new AttributedLongCounter(timeoutRequestMetric, attributes); - - requestTimes = new AttributedLongTimer(requestTimeMetric, attributes); + factory.attributedLongCounter( + "solr_core_requests_errors", + "HTTP Solr request errors", + Attributes.of(SOURCE_ATTR, "client")); + + numTimeouts = + factory.attributedLongCounter( + "solr_core_requests_timeout", "HTTP Solr request timeouts", Attributes.empty()); + + requestTimes = + factory.attributedLongTimer( + "solr_core_requests_times", + "HTTP Solr request times", + OtelUnit.MILLISECONDS, + Attributes.empty()); } } diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index b42e62d187b..cc7ad157651 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -98,8 +98,6 @@ public class SearchHandler extends RequestHandlerBase static final String INIT_FIRST_COMPONENTS = "first-components"; static final String INIT_LAST_COMPONENTS = "last-components"; - protected static final String SHARD_HANDLER_SUFFIX = "[shard]"; - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); /** @@ -171,7 +169,8 @@ public void initializeMetrics( .putAll(attributes) .put(CATEGORY_ATTR, getCategory().toString()) .put(INTERNAL_ATTR, true) - .build()); + .build(), + false); } @Override diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java index 43adf8ad087..e2f28293e4a 100644 --- a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java +++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleCounter.java @@ -29,7 +29,7 @@ public AttributedDoubleCounter(DoubleCounter counter, Attributes attributes) { this.attributes = attributes; } - public void inc() { + public final void inc() { add(1.0); } diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java index 23c0d3b9a53..ca2808a596f 100644 --- a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java +++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedDoubleUpDownCounter.java @@ -29,11 +29,11 @@ public AttributedDoubleUpDownCounter(DoubleUpDownCounter upDownCounter, Attribut this.attributes = attributes; } - public void inc() { + public final void inc() { add(1.0); } - public void dec() { + public final void dec() { add(-1.0); } diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.java new file mode 100644 index 00000000000..a443eab92b0 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedInstrumentFactory.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.metrics.otel.instruments; + +import static org.apache.solr.handler.component.SearchHandler.INTERNAL_ATTR; +import static org.apache.solr.metrics.SolrCoreMetricManager.COLLECTION_ATTR; +import static org.apache.solr.metrics.SolrCoreMetricManager.CORE_ATTR; +import static org.apache.solr.metrics.SolrCoreMetricManager.REPLICA_TYPE_ATTR; +import static org.apache.solr.metrics.SolrCoreMetricManager.SHARD_ATTR; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.LongHistogram; +import io.opentelemetry.api.metrics.LongUpDownCounter; +import java.util.Set; +import org.apache.solr.core.SolrInfoBean; +import org.apache.solr.metrics.SolrMetricManager; +import org.apache.solr.metrics.SolrMetricsContext; +import org.apache.solr.metrics.otel.OtelUnit; + +/** + * Factory for creating metrics instruments that can write to either single or dual registries (core + * and node). + */ +public class AttributedInstrumentFactory { + + private static final Set> FILTER_ATTRS_SET = + Set.of(COLLECTION_ATTR, CORE_ATTR, SHARD_ATTR, REPLICA_TYPE_ATTR, INTERNAL_ATTR); + private final SolrMetricsContext primaryMetricsContext; + private final Attributes primaryAttributes; + private final boolean aggregateToNodeRegistry; + private final boolean primaryIsNodeRegistry; + private SolrMetricsContext nodeMetricsContext = null; + private Attributes nodeAttributes = null; + + public AttributedInstrumentFactory( + SolrMetricsContext primaryMetricsContext, + Attributes primaryAttributes, + boolean aggregateToNodeRegistry) { + this.primaryMetricsContext = primaryMetricsContext; + this.primaryAttributes = primaryAttributes; + this.aggregateToNodeRegistry = aggregateToNodeRegistry; + + // Primary could be a node + this.primaryIsNodeRegistry = + primaryMetricsContext + .getRegistryName() + .equals(SolrMetricManager.getRegistryName(SolrInfoBean.Group.node)); + + // Only create node registry if we want dual registry mode AND primary is not already a node + // registry + if (aggregateToNodeRegistry && !primaryIsNodeRegistry) { + this.nodeMetricsContext = + new SolrMetricsContext( + primaryMetricsContext.getMetricManager(), + SolrMetricManager.getRegistryName(SolrInfoBean.Group.node), + null); + this.nodeAttributes = createNodeAttributes(primaryAttributes); + } + } + + public AttributedLongCounter attributedLongCounter( + String metricName, String description, Attributes additionalAttributes) { + Attributes finalPrimaryAttrs = appendAttributes(primaryAttributes, additionalAttributes); + + if (aggregateToNodeRegistry && nodeMetricsContext != null) { + Attributes finalNodeAttrs = appendAttributes(nodeAttributes, additionalAttributes); + + LongCounter primaryCounter = primaryMetricsContext.longCounter(metricName, description); + LongCounter nodeCounter = + nodeMetricsContext.longCounter(toNodeMetricName(metricName), description); + return new DualRegistryAttributedLongCounter( + primaryCounter, finalPrimaryAttrs, nodeCounter, finalNodeAttrs); + } else { + String finalMetricName = primaryIsNodeRegistry ? toNodeMetricName(metricName) : metricName; + LongCounter counter = primaryMetricsContext.longCounter(finalMetricName, description); + return new AttributedLongCounter(counter, finalPrimaryAttrs); + } + } + + public AttributedLongUpDownCounter attributedLongUpDownCounter( + String metricName, String description, Attributes additionalAttributes) { + Attributes finalPrimaryAttrs = appendAttributes(primaryAttributes, additionalAttributes); + + if (aggregateToNodeRegistry && nodeMetricsContext != null) { + Attributes finalNodeAttrs = appendAttributes(nodeAttributes, additionalAttributes); + + LongUpDownCounter primaryCounter = + primaryMetricsContext.longUpDownCounter(metricName, description); + LongUpDownCounter nodeCounter = + nodeMetricsContext.longUpDownCounter(toNodeMetricName(metricName), description); + return new DualRegistryAttributedLongUpDownCounter( + primaryCounter, finalPrimaryAttrs, nodeCounter, finalNodeAttrs); + } else { + String finalMetricName = primaryIsNodeRegistry ? toNodeMetricName(metricName) : metricName; + LongUpDownCounter counter = + primaryMetricsContext.longUpDownCounter(finalMetricName, description); + return new AttributedLongUpDownCounter(counter, finalPrimaryAttrs); + } + } + + public AttributedLongTimer attributedLongTimer( + String metricName, String description, OtelUnit unit, Attributes additionalAttributes) { + Attributes finalPrimaryAttrs = appendAttributes(primaryAttributes, additionalAttributes); + + if (aggregateToNodeRegistry && nodeMetricsContext != null) { + Attributes finalNodeAttrs = appendAttributes(nodeAttributes, additionalAttributes); + LongHistogram primaryHistogram = + primaryMetricsContext.longHistogram(metricName, description, unit); + LongHistogram nodeHistogram = + nodeMetricsContext.longHistogram(toNodeMetricName(metricName), description, unit); + return new DualRegistryAttributedLongTimer( + primaryHistogram, finalPrimaryAttrs, nodeHistogram, finalNodeAttrs); + } else { + String finalMetricName = primaryIsNodeRegistry ? toNodeMetricName(metricName) : metricName; + LongHistogram histogram = + primaryMetricsContext.longHistogram(finalMetricName, description, unit); + return new AttributedLongTimer(histogram, finalPrimaryAttrs); + } + } + + /** Replace core metric name prefix to node prefix */ + private String toNodeMetricName(String coreMetricName) { + return coreMetricName.replace("solr_core", "solr_node"); + } + + /** Filter out core attributes and keep all others for node-level metrics */ + @SuppressWarnings("unchecked") + private Attributes createNodeAttributes(Attributes coreAttributes) { + var builder = Attributes.builder(); + + coreAttributes.forEach( + (key, value) -> { + if (!FILTER_ATTRS_SET.contains(key)) { + builder.put((AttributeKey) key, value); + } + }); + + return builder.build(); + } + + private Attributes appendAttributes(Attributes base, Attributes additional) { + return base.toBuilder().putAll(additional).build(); + } +} diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java index f33b4784c53..e2a477eac35 100644 --- a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java +++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongCounter.java @@ -29,7 +29,7 @@ public AttributedLongCounter(LongCounter baseCounter, Attributes attributes) { this.attributes = attributes; } - public void inc() { + public final void inc() { add(1L); } diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java index 91487c9e677..e3ae3e98edd 100644 --- a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java +++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/AttributedLongUpDownCounter.java @@ -29,11 +29,11 @@ public AttributedLongUpDownCounter(LongUpDownCounter upDownCounter, Attributes a this.attributes = attributes; } - public void inc() { + public final void inc() { add(1L); } - public void dec() { + public final void dec() { add(-1L); } diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.java new file mode 100644 index 00000000000..15fc27b85c5 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongCounter.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.metrics.otel.instruments; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; + +/** + * An AttributedLongCounter that writes to both core and node registries with corresponding + * attributes. + */ +public class DualRegistryAttributedLongCounter extends AttributedLongCounter { + + private final AttributedLongCounter nodeCounter; + + public DualRegistryAttributedLongCounter( + LongCounter coreCounter, + Attributes coreAttributes, + LongCounter nodeCounter, + Attributes nodeAttributes) { + super(coreCounter, coreAttributes); + assert coreCounter != nodeCounter; + this.nodeCounter = new AttributedLongCounter(nodeCounter, nodeAttributes); + } + + @Override + public void add(Long value) { + super.add(value); + nodeCounter.add(value); + } +} diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.java new file mode 100644 index 00000000000..b42d869892c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongTimer.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.metrics.otel.instruments; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongHistogram; + +/** + * An AttributedLongTimer that records to both core and node registries with corresponding + * attributes. + */ +public class DualRegistryAttributedLongTimer extends AttributedLongTimer { + + private final AttributedLongTimer nodeTimer; + + public DualRegistryAttributedLongTimer( + LongHistogram coreHistogram, + Attributes coreAttributes, + LongHistogram nodeHistogram, + Attributes nodeAttributes) { + super(coreHistogram, coreAttributes); + assert coreHistogram != nodeHistogram; + this.nodeTimer = new AttributedLongTimer(nodeHistogram, nodeAttributes); + } + + @Override + public void record(Long value) { + super.record(value); + nodeTimer.record(value); + } +} diff --git a/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongUpDownCounter.java b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongUpDownCounter.java new file mode 100644 index 00000000000..6cf505fb54e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/metrics/otel/instruments/DualRegistryAttributedLongUpDownCounter.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.metrics.otel.instruments; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongUpDownCounter; + +/** + * An AttributedLongUpDownCounter that writes to both core and node registries with corresponding + * attributes. + */ +public class DualRegistryAttributedLongUpDownCounter extends AttributedLongUpDownCounter { + + private final AttributedLongUpDownCounter nodeUpDownCounter; + + public DualRegistryAttributedLongUpDownCounter( + LongUpDownCounter coreUpDownCounter, + Attributes coreAttributes, + LongUpDownCounter nodeUpDownCounter, + Attributes nodeAttributes) { + super(coreUpDownCounter, coreAttributes); + assert coreUpDownCounter != nodeUpDownCounter; + this.nodeUpDownCounter = new AttributedLongUpDownCounter(nodeUpDownCounter, nodeAttributes); + } + + @Override + public void add(Long value) { + super.add(value); + nodeUpDownCounter.add(value); + } +} diff --git a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java index 0a027282c46..dc802afc2b7 100644 --- a/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java +++ b/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java @@ -53,11 +53,9 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrConfig.UpdateHandlerInfo; import org.apache.solr.core.SolrCore; -import org.apache.solr.core.SolrInfoBean; -import org.apache.solr.metrics.SolrDelegateRegistryMetricsContext; -import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; +import org.apache.solr.metrics.otel.instruments.AttributedInstrumentFactory; import org.apache.solr.metrics.otel.instruments.AttributedLongCounter; import org.apache.solr.metrics.otel.instruments.AttributedLongUpDownCounter; import org.apache.solr.request.LocalSolrQueryRequest; @@ -227,84 +225,121 @@ public DirectUpdateHandler2(SolrCore core, UpdateHandler updateHandler) { @Override public void initializeMetrics( SolrMetricsContext parentContext, Attributes attributes, String scope) { - if (core.getSolrConfig().getUpdateHandlerInfo().aggregateNodeLevelMetricsEnabled) { - this.solrMetricsContext = - new SolrDelegateRegistryMetricsContext( - parentContext.getMetricManager(), - parentContext.getRegistryName(), - SolrMetricProducer.getUniqueMetricTag(this, parentContext.getTag()), - SolrMetricManager.getRegistryName(SolrInfoBean.Group.node)); - } else { - this.solrMetricsContext = parentContext.getChildContext(this); - } - final List observables = new ArrayList<>(); + this.solrMetricsContext = parentContext.getChildContext(this); var baseAttributes = attributes.toBuilder() .put(AttributeKey.stringKey("category"), getCategory().toString()) .build(); - var baseCommandsMetric = - solrMetricsContext.longUpDownCounter( - "solr_core_update_cumulative_ops", - "Cumulative number of update commands processed. Cumulative can decrease from rollback command"); + boolean aggregateNodeLevelMetricsEnabled = + core.getSolrConfig().getUpdateHandlerInfo().aggregateNodeLevelMetricsEnabled; + + createMetrics(baseAttributes, aggregateNodeLevelMetricsEnabled); + } + + private void createMetrics(Attributes baseAttributes, boolean aggregateToNodeRegistry) { + final List observables = new ArrayList<>(); + + AttributedInstrumentFactory factory = + new AttributedInstrumentFactory( + solrMetricsContext, baseAttributes, aggregateToNodeRegistry); addCommandsCumulative = - new AttributedLongUpDownCounter( - baseCommandsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "adds").build()); + factory.attributedLongUpDownCounter( + "solr_core_update_cumulative_ops", + "Cumulative number of update commands processed. Cumulative can decrease from rollback command", + Attributes.of(OPERATION_ATTR, "adds")); deleteByIdCommandsCumulative = - new AttributedLongUpDownCounter( - baseCommandsMetric, - baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_id").build()); + factory.attributedLongUpDownCounter( + "solr_core_update_cumulative_ops", + "Cumulative number of update commands processed. Cumulative can decrease from rollback command", + Attributes.of(OPERATION_ATTR, "deletes_by_id")); deleteByQueryCommandsCumulative = - new AttributedLongUpDownCounter( - baseCommandsMetric, - baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_query").build()); - - var baseCommitOpsMetric = - solrMetricsContext.longCounter( - "solr_core_update_commit_ops", "Total number of commit operations"); + factory.attributedLongUpDownCounter( + "solr_core_update_cumulative_ops", + "Cumulative number of update commands processed. Cumulative can decrease from rollback command", + Attributes.of(OPERATION_ATTR, "deletes_by_query")); commitCommands = - new AttributedLongCounter( - baseCommitOpsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "commits").build()); + factory.attributedLongCounter( + "solr_core_update_commit_ops", + "Total number of commit operations", + Attributes.of(OPERATION_ATTR, "commits")); optimizeCommands = - new AttributedLongCounter( - baseCommitOpsMetric, - baseAttributes.toBuilder().put(OPERATION_ATTR, "optimize").build()); + factory.attributedLongCounter( + "solr_core_update_commit_ops", + "Total number of commit operations", + Attributes.of(OPERATION_ATTR, "optimize")); mergeIndexesCommands = - new AttributedLongCounter( - baseCommitOpsMetric, - baseAttributes.toBuilder().put(OPERATION_ATTR, "merge_indexes").build()); + factory.attributedLongCounter( + "solr_core_update_commit_ops", + "Total number of commit operations", + Attributes.of(OPERATION_ATTR, "merge_indexes")); expungeDeleteCommands = - new AttributedLongCounter( - baseCommitOpsMetric, - baseAttributes.toBuilder().put(OPERATION_ATTR, "expunge_deletes").build()); - - var baseMaintenanceMetric = - solrMetricsContext.longCounter( - "solr_core_update_maintenance_ops", "Total number of maintenance operations"); + factory.attributedLongCounter( + "solr_core_update_commit_ops", + "Total number of commit operations", + Attributes.of(OPERATION_ATTR, "expunge_deletes")); rollbackCommands = - new AttributedLongCounter( - baseMaintenanceMetric, - baseAttributes.toBuilder().put(OPERATION_ATTR, "rollback").build()); + factory.attributedLongCounter( + "solr_core_update_maintenance_ops", + "Total number of maintenance operations", + Attributes.of(OPERATION_ATTR, "rollback")); splitCommands = - new AttributedLongCounter( - baseMaintenanceMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "split").build()); - - var baseErrorsMetric = - solrMetricsContext.longCounter("solr_core_update_errors", "Total number of update errors"); + factory.attributedLongCounter( + "solr_core_update_maintenance_ops", + "Total number of maintenance operations", + Attributes.of(OPERATION_ATTR, "split")); numErrorsCumulative = - new AttributedLongCounter(baseErrorsMetric, baseAttributes.toBuilder().build()); + factory.attributedLongCounter( + "solr_core_update_errors", "Total number of update errors", Attributes.empty()); + + submittedAdds = + factory.attributedLongCounter( + "solr_core_update_submitted_ops", + "Total number of submitted update operations", + Attributes.of(OPERATION_ATTR, "adds")); + + submittedDeleteById = + factory.attributedLongCounter( + "solr_core_update_submitted_ops", + "Total number of submitted update operations", + Attributes.of(OPERATION_ATTR, "deletes_by_id")); + submittedDeleteByQuery = + factory.attributedLongCounter( + "solr_core_update_submitted_ops", + "Total number of submitted update operations", + Attributes.of(OPERATION_ATTR, "deletes_by_query")); + + committedAdds = + factory.attributedLongCounter( + "solr_core_update_committed_ops", + "Total number of committed update operations", + Attributes.of(OPERATION_ATTR, "adds")); + + committedDeleteById = + factory.attributedLongCounter( + "solr_core_update_committed_ops", + "Total number of committed update operations", + Attributes.of(OPERATION_ATTR, "deletes_by_id")); + + committedDeleteByQuery = + factory.attributedLongCounter( + "solr_core_update_committed_ops", + "Total number of committed update operations", + Attributes.of(OPERATION_ATTR, "deletes_by_query")); + + // Create observable metrics only for core registry observables.add( solrMetricsContext.observableLongCounter( "solr_core_update_auto_commits", @@ -361,38 +396,6 @@ public void initializeMetrics( })); this.toClose = Collections.unmodifiableList(observables); - - var baseSubmittedOpsMetric = - solrMetricsContext.longCounter( - "solr_core_update_submitted_ops", "Total number of submitted update operations"); - - var baseCommittedOpsMetric = - solrMetricsContext.longCounter( - "solr_core_update_committed_ops", "Total number of committed update operations"); - - submittedAdds = - new AttributedLongCounter( - baseSubmittedOpsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "adds").build()); - submittedDeleteById = - new AttributedLongCounter( - baseSubmittedOpsMetric, - baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_id").build()); - submittedDeleteByQuery = - new AttributedLongCounter( - baseSubmittedOpsMetric, - baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_query").build()); - - committedAdds = - new AttributedLongCounter( - baseCommittedOpsMetric, baseAttributes.toBuilder().put(OPERATION_ATTR, "adds").build()); - committedDeleteById = - new AttributedLongCounter( - baseCommittedOpsMetric, - baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_id").build()); - committedDeleteByQuery = - new AttributedLongCounter( - baseCommittedOpsMetric, - baseAttributes.toBuilder().put(OPERATION_ATTR, "deletes_by_query").build()); } private void deleteAll() throws IOException { diff --git a/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml index f23456d060a..2b1586ae253 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-aggregate-node-metrics/conf/solrconfig.xml @@ -29,7 +29,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.commitwithin.softcommit:true} @@ -52,4 +52,4 @@ - \ No newline at end of file + diff --git a/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java b/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java index 74d3d6b2510..2320b526b08 100644 --- a/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java +++ b/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java @@ -17,6 +17,7 @@ package org.apache.solr.handler; +import static org.apache.solr.metrics.SolrMetricProducer.HANDLER_ATTR; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; @@ -196,6 +197,6 @@ private RequestHandlerBase.HandlerMetrics createHandlerMetrics() { when(metricsContext.longHistogram(any(), any())).thenReturn(mockLongHistogram); return new RequestHandlerBase.HandlerMetrics( - metricsContext, Attributes.of(AttributeKey.stringKey("/handler"), "/someBaseMetricPath")); + metricsContext, Attributes.of(HANDLER_ATTR, "/someBaseMetricPath"), false); } } diff --git a/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java b/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java index e9618297760..e338939b30c 100644 --- a/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java +++ b/solr/core/src/test/org/apache/solr/handler/RequestHandlerMetricsTest.java @@ -16,19 +16,17 @@ */ package org.apache.solr.handler; +import io.prometheus.metrics.model.snapshots.CounterSnapshot; +import io.prometheus.metrics.model.snapshots.Labels; import java.io.IOException; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrCore; +import org.apache.solr.util.SolrMetricTestUtils; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -54,10 +52,7 @@ public static void afterClass() { System.clearProperty("metricsEnabled"); } - // TODO: Migrate aggregateNodeLevelMetricsEnabled for OTEL metrics @Test - @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17865") - @SuppressWarnings({"unchecked"}) public void testAggregateNodeLevelMetrics() throws SolrServerException, IOException { String collection1 = "testRequestHandlerMetrics1"; String collection2 = "testRequestHandlerMetrics2"; @@ -82,97 +77,83 @@ public void testAggregateNodeLevelMetrics() throws SolrServerException, IOExcept cloudClient.query(collection1, solrQuery); cloudClient.query(collection2, solrQuery); - NamedList response = - cloudClient.request( - new GenericSolrRequest( - SolrRequest.METHOD.GET, "/admin/metrics", SolrRequest.SolrRequestType.ADMIN)); - - NamedList metrics = (NamedList) response.get("metrics"); - - final double[] minQueryTime = {Double.MAX_VALUE}; - final double[] maxQueryTime = {-1.0}; - final double[] minUpdateTime = {Double.MAX_VALUE}; - final double[] maxUpdateTime = {-1.0}; - Set> coreMetrics = new HashSet<>(); - metrics.forEach( - (key, coreMetric) -> { - if (key.startsWith("solr.core.testRequestHandlerMetrics")) { - coreMetrics.add((NamedList) coreMetric); - } - }); - assertEquals(2, coreMetrics.size()); - coreMetrics.forEach( - metric -> { - assertEquals( - 1L, - ((Map) metric.get("QUERY./select.requestTimes")) - .get("count") - .longValue()); - minQueryTime[0] = - Math.min( - minQueryTime[0], - ((Map) metric.get("QUERY./select.requestTimes")) - .get("min_ms") - .doubleValue()); - maxQueryTime[0] = - Math.max( - maxQueryTime[0], - ((Map) metric.get("QUERY./select.requestTimes")) - .get("max_ms") - .doubleValue()); - assertEquals( - 1L, - ((Map) metric.get("UPDATE./update.requestTimes")) - .get("count") - .longValue()); - minUpdateTime[0] = - Math.min( - minUpdateTime[0], - ((Map) metric.get("UPDATE./update.requestTimes")) - .get("min_ms") - .doubleValue()); - maxUpdateTime[0] = - Math.max( - maxUpdateTime[0], - ((Map) metric.get("UPDATE./update.requestTimes")) - .get("max_ms") - .doubleValue()); - }); - - NamedList nodeMetrics = (NamedList) metrics.get("solr.node"); - assertEquals( - 2L, - ((Map) nodeMetrics.get("QUERY./select.requestTimes")) - .get("count") - .longValue()); - assertEquals( - minQueryTime[0], - ((Map) nodeMetrics.get("QUERY./select.requestTimes")) - .get("min_ms") - .doubleValue(), - 0.0); - assertEquals( - maxQueryTime[0], - ((Map) nodeMetrics.get("QUERY./select.requestTimes")) - .get("max_ms") - .doubleValue(), - 0.0); - assertEquals( - 2L, - ((Map) nodeMetrics.get("UPDATE./update.requestTimes")) - .get("count") - .longValue()); - assertEquals( - minUpdateTime[0], - ((Map) nodeMetrics.get("UPDATE./update.requestTimes")) - .get("min_ms") - .doubleValue(), - 0.0); - assertEquals( - maxUpdateTime[0], - ((Map) nodeMetrics.get("UPDATE./update.requestTimes")) - .get("max_ms") - .doubleValue(), - 0.0); + var coreContainer = cluster.getJettySolrRunners().get(0).getCoreContainer(); + + try (SolrCore core1 = coreContainer.getCore(coreContainer.getAllCoreNames().get(0)); + SolrCore core2 = coreContainer.getCore(coreContainer.getAllCoreNames().get(1))) { + + CounterSnapshot.CounterDataPointSnapshot actualCore1Selects = + SolrMetricTestUtils.newCloudSelectRequestsDatapoint(core1); + CounterSnapshot.CounterDataPointSnapshot actualCore1Updates = + SolrMetricTestUtils.newCloudUpdateRequestsDatapoint(core1); + CounterSnapshot.CounterDataPointSnapshot actualCore2Selects = + SolrMetricTestUtils.newCloudSelectRequestsDatapoint(core2); + CounterSnapshot.CounterDataPointSnapshot actualCore2Updates = + SolrMetricTestUtils.newCloudUpdateRequestsDatapoint(core2); + CounterSnapshot.CounterDataPointSnapshot actualCore1SubmittedOps = + SolrMetricTestUtils.getCounterDatapoint( + core1, + "solr_core_update_submitted_ops", + SolrMetricTestUtils.newCloudLabelsBuilder(core1) + .label("category", "UPDATE") + .label("ops", "adds") + .build()); + CounterSnapshot.CounterDataPointSnapshot actualCore2SubmittedOps = + SolrMetricTestUtils.getCounterDatapoint( + core2, + "solr_core_update_submitted_ops", + SolrMetricTestUtils.newCloudLabelsBuilder(core2) + .label("category", "UPDATE") + .label("ops", "adds") + .build()); + + assertEquals(1.0, actualCore1Selects.getValue(), 0.0); + assertEquals(1.0, actualCore1Updates.getValue(), 0.0); + assertEquals(1.0, actualCore2Updates.getValue(), 0.0); + assertEquals(1.0, actualCore2Selects.getValue(), 0.0); + assertEquals(1.0, actualCore1SubmittedOps.getValue(), 0.0); + assertEquals(1.0, actualCore2SubmittedOps.getValue(), 0.0); + + // Get node metrics and the select/update requests should be the sum of both cores requests + var nodeReader = SolrMetricTestUtils.getPrometheusMetricReader(coreContainer, "solr.node"); + + CounterSnapshot.CounterDataPointSnapshot nodeSelectRequests = + (CounterSnapshot.CounterDataPointSnapshot) + SolrMetricTestUtils.getDataPointSnapshot( + nodeReader, + "solr_node_requests", + Labels.builder() + .label("category", "QUERY") + .label("handler", "/select") + .label("otel_scope_name", "org.apache.solr") + .build()); + CounterSnapshot.CounterDataPointSnapshot nodeUpdateRequests = + (CounterSnapshot.CounterDataPointSnapshot) + SolrMetricTestUtils.getDataPointSnapshot( + nodeReader, + "solr_node_requests", + Labels.builder() + .label("category", "UPDATE") + .label("handler", "/update") + .label("otel_scope_name", "org.apache.solr") + .build()); + CounterSnapshot.CounterDataPointSnapshot nodeSubmittedOps = + (CounterSnapshot.CounterDataPointSnapshot) + SolrMetricTestUtils.getDataPointSnapshot( + nodeReader, + "solr_node_update_submitted_ops", + Labels.builder() + .label("category", "UPDATE") + .label("ops", "adds") + .label("otel_scope_name", "org.apache.solr") + .build()); + + assertNotNull("Node select requests should be recorded", nodeSelectRequests); + assertNotNull("Node update requests should be recorded", nodeUpdateRequests); + assertNotNull("Node submitted update operations should be recorded", nodeSubmittedOps); + assertEquals(2.0, nodeSelectRequests.getValue(), 0.0); + assertEquals(2.0, nodeUpdateRequests.getValue(), 0.0); + assertEquals(2.0, nodeSubmittedOps.getValue(), 0.0); + } } }