update to use ES logger instead of infostream

Author: john-wagster

server/src/main/java/org/elasticsearch/index/codec/vectors/DefaultIVFVectorsWriter.java

@@ -9,6 +9,8 @@
 
 package org.elasticsearch.index.codec.vectors;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.codecs.hnsw.FlatVectorsWriter;
 import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.FloatVectorValues;
@@ -17,7 +19,6 @@
 import org.apache.lucene.internal.hppc.IntArrayList;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
 import org.elasticsearch.index.codec.vectors.cluster.HierarchicalKMeans;
@@ -31,14 +32,14 @@
 import static org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat.INDEX_BITS;
 import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.discretize;
 import static org.apache.lucene.util.quantization.OptimizedScalarQuantizer.packAsBinary;
-import static org.elasticsearch.index.codec.vectors.IVFVectorsFormat.IVF_VECTOR_COMPONENT;
 
 /**
  * Default implementation of {@link IVFVectorsWriter}. It uses {@link HierarchicalKMeans} algorithm to
  * partition the vector space, and then stores the centroids and posting list in a sequential
  * fashion.
  */
 public class DefaultIVFVectorsWriter extends IVFVectorsWriter {
+    private static final Logger logger = LogManager.getLogger(DefaultIVFVectorsWriter.class);
 
     private final int vectorPerCluster;
 
@@ -53,7 +54,6 @@ long[] buildAndWritePostingsLists(
         CentroidSupplier centroidSupplier,
         FloatVectorValues floatVectorValues,
         IndexOutput postingsOutput,
-        InfoStream infoStream,
         IntArrayList[] assignmentsByCluster
     ) throws IOException {
         // write the posting lists
@@ -79,14 +79,14 @@ long[] buildAndWritePostingsLists(
             writePostingList(cluster, postingsOutput, binarizedByteVectorValues);
         }
 
-        if (infoStream.isEnabled(IVF_VECTOR_COMPONENT)) {
-            printClusterQualityStatistics(assignmentsByCluster, infoStream);
+        if(logger.isDebugEnabled()) {
+            printClusterQualityStatistics(assignmentsByCluster);
         }
 
         return offsets;
     }
 
-    private static void printClusterQualityStatistics(IntArrayList[] clusters, InfoStream infoStream) {
+    private static void printClusterQualityStatistics(IntArrayList[] clusters) {
         float min = Float.MAX_VALUE;
         float max = Float.MIN_VALUE;
         float mean = 0;
@@ -105,21 +105,8 @@ private static void printClusterQualityStatistics(IntArrayList[] clusters, InfoS
             max = Math.max(max, cluster.size());
         }
         float variance = m2 / (clusters.length - 1);
-        infoStream.message(
-            IVF_VECTOR_COMPONENT,
-            "Centroid count: "
-                + clusters.length
-                + " min: "
-                + min
-                + " max: "
-                + max
-                + " mean: "
-                + mean
-                + " stdDev: "
-                + Math.sqrt(variance)
-                + " variance: "
-                + variance
-        );
+        logger.debug("Centroid count: {} min: {} max: {} mean: {} stdDev: {} variance: {}",
+            clusters.length, min, max, mean, Math.sqrt(variance), variance);
     }
 
     private void writePostingList(IntArrayList cluster, IndexOutput postingsOutput, BinarizedFloatVectorValues binarizedByteVectorValues)
@@ -208,17 +195,16 @@ CentroidAssignments calculateAndWriteCentroids(
         float[] globalCentroid
     ) throws IOException {
         // TODO: take advantage of prior generated clusters from mergeState in the future
-        return calculateAndWriteCentroids(fieldInfo, floatVectorValues, centroidOutput, mergeState.infoStream, globalCentroid, false);
+        return calculateAndWriteCentroids(fieldInfo, floatVectorValues, centroidOutput, globalCentroid, false);
     }
 
     CentroidAssignments calculateAndWriteCentroids(
         FieldInfo fieldInfo,
         FloatVectorValues floatVectorValues,
         IndexOutput centroidOutput,
-        InfoStream infoStream,
         float[] globalCentroid
     ) throws IOException {
-        return calculateAndWriteCentroids(fieldInfo, floatVectorValues, centroidOutput, infoStream, globalCentroid, true);
+        return calculateAndWriteCentroids(fieldInfo, floatVectorValues, centroidOutput, globalCentroid, true);
     }
 
     /**
@@ -228,7 +214,6 @@ CentroidAssignments calculateAndWriteCentroids(
      * @param fieldInfo merging field info
      * @param floatVectorValues the float vector values to merge
      * @param centroidOutput the centroid output
-     * @param infoStream the merge state
      * @param globalCentroid the global centroid, calculated by this method and used to quantize the centroids
      * @param cacheCentroids whether the centroids are kept or discarded once computed
      * @return the vector assignments, soar assignments, and if asked the centroids themselves that were computed
@@ -238,7 +223,6 @@ CentroidAssignments calculateAndWriteCentroids(
         FieldInfo fieldInfo,
         FloatVectorValues floatVectorValues,
         IndexOutput centroidOutput,
-        InfoStream infoStream,
         float[] globalCentroid,
         boolean cacheCentroids
     ) throws IOException {
@@ -266,12 +250,9 @@ CentroidAssignments calculateAndWriteCentroids(
         // write centroids
         writeCentroids(centroids, fieldInfo, globalCentroid, centroidOutput);
 
-        if (infoStream.isEnabled(IVF_VECTOR_COMPONENT)) {
-            infoStream.message(
-                IVF_VECTOR_COMPONENT,
-                "calculate centroids and assign vectors time ms: " + ((System.nanoTime() - nanoTime) / 1000000.0)
-            );
-            infoStream.message(IVF_VECTOR_COMPONENT, "final centroid count: " + centroids.length);
+        if (logger.isDebugEnabled()) {
+            logger.debug("calculate centroids and assign vectors time ms: {}", (System.nanoTime() - nanoTime) / 1000000.0);
+            logger.debug("final centroid count: {}", centroids.length);
         }
 
         IntArrayList[] assignmentsByCluster = new IntArrayList[centroids.length];

server/src/main/java/org/elasticsearch/index/codec/vectors/IVFVectorsFormat.java

@@ -45,7 +45,6 @@
  */
 public class IVFVectorsFormat extends KnnVectorsFormat {
 
-    public static final String IVF_VECTOR_COMPONENT = "IVF";
     public static final String NAME = "IVFVectorsFormat";
     // centroid ordinals -> centroid values, offsets
     public static final String CENTROID_EXTENSION = "cenivf";

server/src/main/java/org/elasticsearch/index/codec/vectors/IVFVectorsWriter.java

@@ -28,7 +28,6 @@
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.InfoStream;
 import org.apache.lucene.util.VectorUtil;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.core.SuppressForbidden;
@@ -134,7 +133,6 @@ abstract CentroidAssignments calculateAndWriteCentroids(
         FieldInfo fieldInfo,
         FloatVectorValues floatVectorValues,
         IndexOutput centroidOutput,
-        InfoStream infoStream,
         float[] globalCentroid
     ) throws IOException;
 
@@ -143,7 +141,6 @@ abstract long[] buildAndWritePostingsLists(
         CentroidSupplier centroidSupplier,
         FloatVectorValues floatVectorValues,
         IndexOutput postingsOutput,
-        InfoStream infoStream,
         IntArrayList[] assignmentsByCluster
     ) throws IOException;
 
@@ -168,7 +165,6 @@ public final void flush(int maxDoc, Sorter.DocMap sortMap) throws IOException {
                 fieldWriter.fieldInfo,
                 floatVectorValues,
                 ivfCentroids,
-                segmentWriteState.infoStream,
                 globalCentroid
             );
 
@@ -180,7 +176,6 @@ public final void flush(int maxDoc, Sorter.DocMap sortMap) throws IOException {
                 centroidSupplier,
                 floatVectorValues,
                 ivfClusters,
-                segmentWriteState.infoStream,
                 centroidAssignments.assignmentsByCluster()
             );
             // write posting lists
@@ -313,7 +308,6 @@ public final void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) thro
                             centroidSupplier,
                             floatVectorValues,
                             ivfClusters,
-                            mergeState.infoStream,
                             centroidAssignments.assignmentsByCluster()
                         );
                         assert offsets.length == centroidSupplier.size();