SOLR-17856: Solr ref-guide OpenTelemetry documentation (#3811)

Solr 10 ref-guide additions/changes after migrating from Dropwizard to OpenTelemetry.
Additionally, added configurable property for changing OTLP endpoint via gRPC or HTTP.

Author: mlbiscoc

changelog/unreleased/SOLR-17458-switch-from-dropwizard-to-opentelemetry.yml

@@ -0,0 +1,16 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: Switch from Dropwizard to OpenTelemetry. This change provides native Prometheus support on the /admin/metrics API, OTLP support, exemplar support for tracing correlation with OpenMetrics format and native attributes and labels on all metrics.
+type: changed
+authors:
+  - name: Matthew Biscocho
+  - name: David Smiley
+  - name: Sanjay Dutt
+  - name: Jude Muriithi
+  - name: Luke Kot-Zaniewski
+  - name: Carlos Ugarte
+  - name: Kevin Liang
+  - name: Bryan Jacobowitz
+  - name: Adam Quigley
+links:
+  - name: SOLR-17458
+    url: https://issues.apache.org/jira/browse/SOLR-17458

solr/benchmark/src/resources/solr.xml

@@ -21,8 +21,6 @@
 -->
 <solr>
 
-  <metrics enabled="${metricsEnabled:true}"/>
-
   <str name="shareSchema">${shareSchema:false}</str>
   <str name="configSetBaseDir">${configSetBaseDir:configsets}</str>
   <str name="coreRootDirectory">${coreRootDirectory:.}</str>

solr/core/src/java/org/apache/solr/handler/admin/MetricsHandler.java

@@ -60,15 +60,16 @@ public class MetricsHandler extends RequestHandlerBase implements PermissionName
   public static final String KEY_PARAM = "key";
   public static final String EXPR_PARAM = "expr";
   public static final String TYPE_PARAM = "type";
+
   // Prometheus filtering parameters
   public static final String CATEGORY_PARAM = "category";
   public static final String CORE_PARAM = "core";
   public static final String COLLECTION_PARAM = "collection";
   public static final String SHARD_PARAM = "shard";
-  public static final String REPLICA_PARAM = "replica";
+  public static final String REPLICA_TYPE_PARAM = "replica_type";
   public static final String METRIC_NAME_PARAM = "name";
   private static final Set<String> labelFilterKeys =
-      Set.of(CATEGORY_PARAM, CORE_PARAM, COLLECTION_PARAM, SHARD_PARAM, REPLICA_PARAM);
+      Set.of(CATEGORY_PARAM, CORE_PARAM, COLLECTION_PARAM, SHARD_PARAM, REPLICA_TYPE_PARAM);
 
   public static final String PROMETHEUS_METRICS_WT = "prometheus";
   public static final String OPEN_METRICS_WT = "openmetrics";

solr/core/src/java/org/apache/solr/metrics/otel/MetricExporterFactory.java

@@ -30,5 +30,12 @@ public interface MetricExporterFactory {
   public static final int OTLP_EXPORTER_INTERVAL =
       Integer.parseInt(EnvUtils.getProperty("solr.metrics.otlpExporterInterval", "60000"));
 
+  public static final String OTLP_EXPORTER_GRPC_ENDPOINT =
+      EnvUtils.getProperty("solr.metrics.otlpGrpcExporterEndpoint", "http://localhost:4317");
+
+  public static final String OTLP_EXPORTER_HTTP_ENDPOINT =
+      EnvUtils.getProperty(
+          "solr.metrics.otlpHttpExporterEndpoint", "http://localhost:4318/v1/metrics");
+
   MetricExporter getExporter();
 }

solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtlpExporterFactory.java

@@ -42,8 +42,12 @@ public MetricExporter getExporter() {
     }
 
     return switch (OTLP_EXPORTER_PROTOCOL) {
-      case "grpc" -> OtlpGrpcMetricExporter.getDefault();
-      case "http" -> OtlpHttpMetricExporter.getDefault();
+      case "grpc" -> OtlpGrpcMetricExporter.getDefault().toBuilder()
+          .setEndpoint(OTLP_EXPORTER_GRPC_ENDPOINT)
+          .build();
+      case "http" -> OtlpHttpMetricExporter.getDefault().toBuilder()
+          .setEndpoint(OTLP_EXPORTER_HTTP_ENDPOINT)
+          .build();
       case "none" -> null;
       default -> {
         log.warn(

solr/server/solr/solr.xml

@@ -59,16 +59,4 @@
     <int name="connTimeout">${connTimeout:60000}</int>
   </shardHandlerFactory>
 
-  <metrics enabled="${metricsEnabled:true}">
-    <!--    Solr computes JVM metrics for threads. Computing these metrics, esp. computing deadlocks etc.,
-     requires potentially expensive computations, and can be avoided for every metrics call by
-     setting a high caching expiration interval (in seconds).
-      <caching>
-        <int name="threadsIntervalSeconds">5</int>
-      </caching>
-    -->
-    <!--reporter name="jmx_metrics" group="core" class="org.apache.solr.metrics.reporters.SolrJmxReporter"/-->
-  </metrics>
-
-
 </solr>

solr/solr-ref-guide/modules/configuration-guide/pages/configuring-solr-xml.adoc

@@ -65,10 +65,6 @@ The default `solr.xml` file is found in `$SOLR_TIP/server/solr/solr.xml` and loo
     <int name="connTimeout">${connTimeout:60000}</int>
   </shardHandlerFactory>
 
-  <metrics enabled="${metricsEnabled:true}">
-    <!--reporter name="jmx_metrics" group="core" class="org.apache.solr.metrics.reporters.SolrJmxReporter"/-->
-  </metrics>
-
 </solr>
 ----
 
@@ -696,13 +692,6 @@ The `name` attribute is required and must be unique for each `clusterSingleton`.
 The `class` attribute should be set to the FQN (fully qualified name) of a class that extends `ClusterSingleton`.
 Sub-elements are specific to the implementation, `value1` is provided as an example here.
 
-=== The <metrics> Element
-
-The `<metrics>` element in `solr.xml` allows you to customize the metrics reported by Solr.
-You can define system properties that should not be returned, or define custom suppliers and reporters.
-
-If you would like to customize the metrics for your installation, see the xref:deployment-guide:metrics-reporting.adoc#metrics-configuration[Metrics Configuration] section.
-
 === The <caches> Element
 
 The `<caches>` element in `solr.xml` supports defining and configuring named node-level caches.

solr/solr-ref-guide/modules/configuration-guide/pages/solr-plugins.adoc

@@ -42,7 +42,7 @@ Examples of these are query parsers, request handlers, update request processors
 
 * Cluster level (or Core Container level) plugins.
 These are plugins that are installed at a cluster level and every Solr node has one instance each of these plugins.
-Examples of these are xref:deployment-guide:authentication-and-authorization-plugins.adoc[], xref:deployment-guide:metrics-reporting.adoc#reporters[metrics reporters], https://issues.apache.org/jira/browse/SOLR-14404[cluster level request handlers], etc.
+Examples of these are xref:deployment-guide:authentication-and-authorization-plugins.adoc[], https://issues.apache.org/jira/browse/SOLR-14404[cluster level request handlers], etc.
 
 == Installing Plugins ==
 

solr/solr-ref-guide/modules/deployment-guide/deployment-nav.adoc

@@ -61,8 +61,6 @@
 ** xref:metrics-reporting.adoc[]
 ** xref:performance-statistics-reference.adoc[]
 ** xref:plugins-stats-screen.adoc[]
-** xref:mbean-request-handler.adoc[]
-** xref:monitoring-with-prometheus-and-grafana.adoc[]
 ** xref:jmx-with-solr.adoc[]
 ** xref:thread-dump.adoc[]
 ** xref:distributed-tracing.adoc[]

solr/solr-ref-guide/modules/deployment-guide/pages/jmx-with-solr.adoc

@@ -22,21 +22,14 @@ In essence, it is a standard interface by which complex systems can be viewed an
 Solr, like any other good citizen of the Java universe, can be controlled via a JMX interface.
 Once enabled, you can use a JMX client, like jconsole, to connect with Solr.
 
-If you are unfamiliar with JMX, you may  find the following overview useful: http://docs.oracle.com/javase/8/docs/technotes/guides/management/agent.html.
+If you are unfamiliar with JMX, you may find the following overview useful: http://docs.oracle.com/javase/8/docs/technotes/guides/management/agent.html.
 
 == Configuring JMX
 
-JMX support is configured by defining a metrics reporter, as described in the section the section xref:metrics-reporting.adoc#jmx-reporter[JMX Reporter].
-
-If you have an existing MBean server running in Solr's JVM, or if you start Solr with the system property `-Dcom.sun.management.jmxremote`, Solr will automatically identify its location on startup even if you have not defined a reporter explicitly in `solr.xml`.
-You can also define the location of the MBean server with parameters defined in the reporter definition.
+If you have an existing MBean server running in Solr's JVM, or if you start Solr with the system property `-Dcom.sun.management.jmxremote`, Solr will automatically identify its location on startup.
 
 == Configuring MBean Servers
 
-Versions of Solr prior to 7.0 defined JMX support in `solrconfig.xml`.
-This has been changed to the metrics reporter configuration defined above.
-Parameters for the reporter configuration allow defining the location or address of an existing MBean server.
-
 An MBean server can be started at the time of Solr's startup by passing the system parameter `-Dcom.sun.management.jmxremote`.
 See Oracle's documentation for additional settings available to start and control an MBean server at http://docs.oracle.com/javase/8/docs/technotes/guides/management/agent.html.