add bootstrap metrics to v0.x (#828)

bootstrap metrics

Author: waziqi89

src/main/java/com/yelp/nrtsearch/server/grpc/LuceneServer.java

@@ -35,6 +35,7 @@
 import com.google.protobuf.util.JsonFormat;
 import com.yelp.nrtsearch.LuceneServerModule;
 import com.yelp.nrtsearch.server.MetricsRequestHandler;
+import com.yelp.nrtsearch.server.Version;
 import com.yelp.nrtsearch.server.backup.Archiver;
 import com.yelp.nrtsearch.server.config.LuceneServerConfiguration;
 import com.yelp.nrtsearch.server.config.QueryCacheConfig;
@@ -72,6 +73,7 @@
 import io.grpc.stub.ServerCallStreamObserver;
 import io.grpc.stub.StreamObserver;
 import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.Gauge;
 import io.prometheus.client.hotspot.DefaultExports;
 import java.io.File;
 import java.io.FileOutputStream;
@@ -120,6 +122,8 @@ public LuceneServer(
 
   @VisibleForTesting
   public void start() throws IOException {
+    Gauge.Timer timer =
+        BootstrapMetrics.nrtsearchBootstrapTimer.labels(Version.CURRENT.toString()).startTimer();
     List<Plugin> plugins = pluginsService.loadPlugins();
     String serviceName = luceneServerConfiguration.getServiceName();
     String nodeName = luceneServerConfiguration.getNodeName();
@@ -196,6 +200,7 @@ public void start() throws IOException {
             .start();
     logger.info(
         "Server started, listening on " + luceneServerConfiguration.getPort() + " for messages");
+    timer.close();
   }
 
   @VisibleForTesting
@@ -228,6 +233,8 @@ private void registerMetrics(GlobalState globalState) {
     // register thread pool metrics
     new ThreadPoolCollector().register(collectorRegistry);
     collectorRegistry.register(RejectionCounterWrapper.rejectionCounter);
+    // register bootstrap metrics
+    BootstrapMetrics.register(collectorRegistry);
     // register nrt metrics
     NrtMetrics.register(collectorRegistry);
     // register index metrics

src/main/java/com/yelp/nrtsearch/server/luceneserver/ShardState.java

@@ -15,6 +15,7 @@
  */
 package com.yelp.nrtsearch.server.luceneserver;
 
+import com.yelp.nrtsearch.server.Version;
 import com.yelp.nrtsearch.server.config.LuceneServerConfiguration;
 import com.yelp.nrtsearch.server.grpc.DeadlineUtils;
 import com.yelp.nrtsearch.server.grpc.IndexLiveSettings;
@@ -26,10 +27,12 @@
 import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager;
 import com.yelp.nrtsearch.server.luceneserver.index.NrtIndexWriter;
 import com.yelp.nrtsearch.server.luceneserver.warming.WarmerConfig;
+import com.yelp.nrtsearch.server.monitoring.BootstrapMetrics;
 import com.yelp.nrtsearch.server.monitoring.IndexMetrics;
 import com.yelp.nrtsearch.server.utils.FileUtil;
 import com.yelp.nrtsearch.server.utils.HostPort;
 import io.grpc.StatusRuntimeException;
+import io.prometheus.client.Gauge;
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -966,8 +969,13 @@ public synchronized void startReplica(ReplicationServerClient primaryAddress, lo
       }
 
       if (configuration.getSyncInitialNrtPoint()) {
+        Gauge.Timer timer =
+            BootstrapMetrics.initialNRTTimer
+                .labels(name.split(":")[0], Version.CURRENT.toString())
+                .startTimer();
         nrtReplicaNode.syncFromCurrentPrimary(
             configuration.getInitialSyncPrimaryWaitMs(), configuration.getInitialSyncMaxTimeMs());
+        timer.close();
       }
 
       startSearcherPruningThread(indexState.getGlobalState().getShutdownLatch());

src/main/java/com/yelp/nrtsearch/server/luceneserver/StartIndexHandler.java

@@ -15,6 +15,7 @@
  */
 package com.yelp.nrtsearch.server.luceneserver;
 
+import com.yelp.nrtsearch.server.Version;
 import com.yelp.nrtsearch.server.backup.Archiver;
 import com.yelp.nrtsearch.server.grpc.Mode;
 import com.yelp.nrtsearch.server.grpc.ReplicationServerClient;
@@ -23,7 +24,10 @@
 import com.yelp.nrtsearch.server.grpc.StartIndexRequest;
 import com.yelp.nrtsearch.server.grpc.StartIndexResponse;
 import com.yelp.nrtsearch.server.luceneserver.index.IndexStateManager;
+import com.yelp.nrtsearch.server.luceneserver.state.BackendGlobalState;
+import com.yelp.nrtsearch.server.monitoring.BootstrapMetrics;
 import com.yelp.nrtsearch.server.utils.FileUtil;
+import io.prometheus.client.Gauge;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -90,13 +94,20 @@ public StartIndexResponse handle(IndexState indexState, StartIndexRequest startI
                 Files.createDirectories(indexState.getRootDir());
                 deleteDownloadedBackupDirectories(restoreIndex.getResourceName());
               }
-
-              dataPath =
-                  downloadArtifact(
-                      restoreIndex.getServiceName(),
-                      restoreIndex.getResourceName(),
-                      INDEXED_DATA_TYPE.DATA,
-                      restoreFromIncArchiver);
+              try (Gauge.Timer _timer =
+                  BootstrapMetrics.dataRestoreTimer
+                      .labels(
+                          BackendGlobalState.getBaseIndexName(startIndexRequest.getIndexName()),
+                          startIndexRequest.getIndexName(),
+                          Version.CURRENT.toString())
+                      .startTimer()) {
+                dataPath =
+                    downloadArtifact(
+                        restoreIndex.getServiceName(),
+                        restoreIndex.getResourceName(),
+                        INDEXED_DATA_TYPE.DATA,
+                        restoreFromIncArchiver);
+              }
             } else {
               throw new IllegalStateException(
                   "Index " + indexState.getName() + " already restored");

src/main/java/com/yelp/nrtsearch/server/luceneserver/index/ImmutableIndexState.java

@@ -401,7 +401,6 @@ public void start(
     if (isStarted()) {
       throw new IllegalStateException("index \"" + getName() + "\" was already started");
     }
-
     // restore data if provided
     if (dataPath != null) {
       restoreIndexData(dataPath, getRootDir());

src/main/java/com/yelp/nrtsearch/server/luceneserver/state/BackendGlobalState.java

@@ -97,6 +97,22 @@ public static String getUniqueIndexName(String indexName, String id) {
     return indexName + "-" + id;
   }
 
+  /**
+   * Get the base index name from the unique index identifier
+   *
+   * @param uniqueIndexName unique index identifier
+   * @return index name
+   */
+  public static String getBaseIndexName(String uniqueIndexName) {
+    String[] parts = uniqueIndexName.split("-");
+    // suffix is 32 hexadecimal characters with four hyphens, so if there are additional hyphens,
+    // they belongs to the base index name.
+    if (parts.length > 5) {
+      return String.join("-", java.util.Arrays.copyOf(parts, parts.length - 5));
+    }
+    return uniqueIndexName; // Return original if it doesn't match expected format
+  }
+
   /**
    * Constructor.
    *

src/main/java/com/yelp/nrtsearch/server/luceneserver/warming/Warmer.java

@@ -19,10 +19,14 @@
 import com.google.common.base.Strings;
 import com.google.protobuf.InvalidProtocolBufferException;
 import com.google.protobuf.util.JsonFormat;
+import com.yelp.nrtsearch.server.Version;
 import com.yelp.nrtsearch.server.backup.Archiver;
 import com.yelp.nrtsearch.server.grpc.SearchRequest;
 import com.yelp.nrtsearch.server.luceneserver.IndexState;
 import com.yelp.nrtsearch.server.luceneserver.SearchHandler;
+import com.yelp.nrtsearch.server.luceneserver.state.BackendGlobalState;
+import com.yelp.nrtsearch.server.monitoring.BootstrapMetrics;
+import io.prometheus.client.Gauge;
 import java.io.BufferedReader;
 import java.io.BufferedWriter;
 import java.io.IOException;
@@ -125,8 +129,14 @@ public synchronized void backupWarmingQueriesToS3(String service) throws IOExcep
 
   public void warmFromS3(IndexState indexState, int parallelism)
       throws IOException, SearchHandler.SearchHandlerException, InterruptedException {
-    SearchHandler searchHandler = new SearchHandler(indexState.getSearchThreadPoolExecutor(), true);
-    warmFromS3(indexState, parallelism, searchHandler);
+    try (Gauge.Timer _timer =
+        BootstrapMetrics.warmingQueryTimer
+            .labels(service, BackendGlobalState.getBaseIndexName(index), Version.CURRENT.toString())
+            .startTimer()) {
+      SearchHandler searchHandler =
+          new SearchHandler(indexState.getSearchThreadPoolExecutor(), true);
+      warmFromS3(indexState, parallelism, searchHandler);
+    }
   }
 
   @VisibleForTesting

src/main/java/com/yelp/nrtsearch/server/monitoring/BootstrapMetrics.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright 2025 Yelp Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yelp.nrtsearch.server.monitoring;
+
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.Gauge;
+
+/**
+ * Class for collecting the timing of the overall bootstrap time and the major components. The
+ * timers shall only be updated during the bootstrap.
+ */
+public class BootstrapMetrics {
+
+  public static final Gauge nrtsearchBootstrapTimer =
+      Gauge.build()
+          .name("total_bootstrap_time_seconds")
+          .help("timer to record the total bootstrap time.")
+          .labelNames("nrtsearch_version")
+          .create();
+
+  public static final Gauge pluginInitializationTimer =
+      Gauge.build()
+          .name("plugin_initialization_time_seconds")
+          .help("timer to record the boostrap time spent on plugin initialization.")
+          .labelNames("plugin_name", "plugin_version", "nrtsearch_version")
+          .create();
+
+  public static final Gauge dataRestoreTimer =
+      Gauge.build()
+          .name("data_restore_time_seconds")
+          .help(
+              "timer to record the boostrap time spent on restoring the stored data from local or remote source.")
+          .labelNames("index", "unique_index_name", "nrtsearch_version")
+          .create();
+
+  public static final Gauge initialNRTTimer =
+      Gauge.build()
+          .name("initial_nrt_time_seconds")
+          .help("timer to record the boostrap time spent on initial nrt")
+          .labelNames("index", "nrtsearch_version")
+          .create();
+
+  public static final Gauge warmingQueryTimer =
+      Gauge.build()
+          .name("warming_time_seconds")
+          .help("timer to record the boostrap time spent on plugin initialization.")
+          .labelNames("service", "index", "nrtsearch_version")
+          .create();
+
+  /**
+   * Add all bootstrap metrics to the collector registry.
+   *
+   * @param registry collector registry
+   */
+  public static void register(CollectorRegistry registry) {
+    registry.register(nrtsearchBootstrapTimer);
+    registry.register(pluginInitializationTimer);
+    registry.register(dataRestoreTimer);
+    registry.register(initialNRTTimer);
+    registry.register(warmingQueryTimer);
+  }
+}

src/main/java/com/yelp/nrtsearch/server/plugins/PluginsService.java

@@ -15,8 +15,12 @@
  */
 package com.yelp.nrtsearch.server.plugins;
 
+import static io.prometheus.client.Collector.NANOSECONDS_PER_SECOND;
+
 import com.amazonaws.services.s3.AmazonS3;
+import com.yelp.nrtsearch.server.Version;
 import com.yelp.nrtsearch.server.config.LuceneServerConfiguration;
+import com.yelp.nrtsearch.server.monitoring.BootstrapMetrics;
 import io.prometheus.client.CollectorRegistry;
 import java.io.File;
 import java.net.MalformedURLException;
@@ -69,10 +73,17 @@ public List<Plugin> loadPlugins() {
     List<Plugin> loadedPlugins = new ArrayList<>();
     PluginDownloader pluginDownloader = new PluginDownloader(amazonS3, config);
     for (String plugin : config.getPlugins()) {
+      long startNs = System.nanoTime();
       logger.info("Loading plugin: " + plugin);
       PluginDescriptor descriptor = loadPlugin(plugin, pluginSearchPath, pluginDownloader);
       loadedPluginDescriptors.add(descriptor);
       loadedPlugins.add(descriptor.getPlugin());
+      BootstrapMetrics.pluginInitializationTimer
+          .labels(
+              descriptor.getPluginMetadata().getName(),
+              descriptor.getPluginMetadata().getVersion(),
+              Version.CURRENT.toString())
+          .set((System.nanoTime() - startNs) / NANOSECONDS_PER_SECOND);
     }
     pluginDownloader.close();
     return loadedPlugins;