Merge pull request #7 from SentriusLLC/copilot/fix-6

Fix VectorIndexFooter build issues with defensive programming and validation

Author: phrocker

core/src/main/java/org/apache/accumulo/core/file/rfile/VectorCompression.java

@@ -66,8 +66,8 @@ public static CompressedVector compress8Bit(float[] vector) {
     byte[] quantized = new byte[vector.length];
     float scale = 255.0f / range;
     for (int i = 0; i < vector.length; i++) {
-      int quantizedValue = Math.round((vector[i] - min) * scale) - 128;
-      quantized[i] = (byte) Math.max(-128, Math.min(127, quantizedValue));
+      int quantizedValue = Math.round((vector[i] - min) * scale);
+      quantized[i] = (byte) Math.max(0, Math.min(255, quantizedValue));
     }
 
     return new CompressedVector(quantized, min, max, COMPRESSION_QUANTIZED_8BIT);
@@ -107,8 +107,8 @@ public static CompressedVector compress16Bit(float[] vector) {
     ByteBuffer buffer = ByteBuffer.allocate(vector.length * 2);
     float scale = 65535.0f / range;
     for (float v : vector) {
-      int quantizedValue = Math.round((v - min) * scale) - 32768;
-      short shortValue = (short) Math.max(-32768, Math.min(32767, quantizedValue));
+      int quantizedValue = Math.round((v - min) * scale);
+      short shortValue = (short) Math.max(0, Math.min(65535, quantizedValue));
       buffer.putShort(shortValue);
     }
 
@@ -160,7 +160,7 @@ private static float[] decompress8Bit(CompressedVector compressed) {
 
     float scale = range / 255.0f;
     for (int i = 0; i < data.length; i++) {
-      int unsignedByte = (data[i] & 0xFF) + 128;
+      int unsignedByte = data[i] & 0xFF;
       result[i] = min + (unsignedByte * scale);
     }
 
@@ -184,7 +184,7 @@ private static float[] decompress16Bit(CompressedVector compressed) {
 
     float scale = range / 65535.0f;
     for (int i = 0; i < result.length; i++) {
-      int unsignedShort = (buffer.getShort() & 0xFFFF) + 32768;
+      int unsignedShort = buffer.getShort() & 0xFFFF;
       result[i] = min + (unsignedShort * scale);
     }
 

core/src/main/java/org/apache/accumulo/core/file/rfile/VectorIndexFooter.java

@@ -221,11 +221,22 @@ private float[][] performKMeansClustering(List<float[]> points, int k) {
     k = Math.min(k, points.size()); // Can't have more clusters than points
     int dimension = points.get(0).length;
 
+    // Validate that all points have the same dimension
+    for (float[] point : points) {
+      if (point.length != dimension) {
+        throw new IllegalArgumentException("All points must have the same dimension: expected "
+            + dimension + ", got " + point.length);
+      }
+    }
+
     // Initialize centroids randomly
     float[][] centroids = new float[k][dimension];
     for (int i = 0; i < k; i++) {
       // Use point i as initial centroid (simple initialization)
-      System.arraycopy(points.get(i * points.size() / k), 0, centroids[i], 0, dimension);
+      int pointIndex = (i * points.size()) / k;
+      // Ensure we don't go out of bounds
+      pointIndex = Math.min(pointIndex, points.size() - 1);
+      System.arraycopy(points.get(pointIndex), 0, centroids[i], 0, dimension);
     }
 
     // K-means iterations (simplified - normally would do multiple iterations)
@@ -302,6 +313,10 @@ private int[] findTopKNearestClusters(float[] point, float[][] centroids, int k)
   }
 
   private float euclideanDistance(float[] a, float[] b) {
+    if (a.length != b.length) {
+      throw new IllegalArgumentException(
+          "Vector dimensions must match: " + a.length + " != " + b.length);
+    }
     float sum = 0.0f;
     for (int i = 0; i < a.length; i++) {
       float diff = a[i] - b[i];

core/src/main/java/org/apache/accumulo/core/file/rfile/VectorIterator.java

@@ -504,4 +504,13 @@ public void setVectorIndex(VectorIndex vectorIndex) {
     this.vectorIndex = vectorIndex;
   }
 
+  /**
+   * Sets the vector index footer for this iterator, enabling advanced indexing capabilities.
+   *
+   * @param indexFooter the vector index footer containing hierarchical indexing structures
+   */
+  public void setVectorIndexFooter(VectorIndexFooter indexFooter) {
+    this.indexFooter = indexFooter;
+  }
+
 }

core/src/test/java/org/apache/accumulo/core/file/rfile/ProductionVectorStoreExampleTest.java

@@ -23,17 +23,24 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
 
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
 /**
  * Comprehensive example demonstrating production-ready vector store features including: - Metadata
  * integration for per-vector categories - Compression for storage efficiency - Batching/staging for
  * performance - Advanced indexing for scalability - Vector chunking for large embeddings
  */
+@SuppressFBWarnings(value = "PREDICTABLE_RANDOM",
+    justification = "This class is an example/demo, not security-sensitive production code.")
 public class ProductionVectorStoreExampleTest {
 
+  static Random rand = new Random(1234);
+
   public static void main(String[] args) {
     System.out.println("=== Production Vector Store Capabilities ===\n");
 
@@ -72,9 +79,9 @@ public static void demonstrateCategoryIntegration() {
     iteratorOptions.put(VectorIterator.TOP_K_OPTION, "5");
 
     System.out.println("User with category filter = internal can access:");
-    System.out.println("  ✓ Public vectors (always available)");
-    System.out.println("  ✓ Internal vectors (category matches)");
-    System.out.println("  ✗ Restricted vectors (not included in filter)");
+    System.out.println("  + Public vectors (always available)");
+    System.out.println("  + Internal vectors (category matches)");
+    System.out.println("  - Restricted vectors (not included in filter)");
 
     System.out.println();
   }
@@ -172,7 +179,7 @@ public static void demonstrateVectorChunking() {
 
     float[] largeEmbedding = new float[1024];
     for (int i = 0; i < largeEmbedding.length; i++) {
-      largeEmbedding[i] = (float) (Math.random() * 2.0 - 1.0);
+      largeEmbedding[i] = (float) (rand.nextFloat() * 2.0 - 1.0);
     }
 
     int chunkSize = 256;
@@ -191,7 +198,7 @@ private static List<VectorBuffer.VectorBlock.VectorEntry> createSampleVectorBloc
     List<VectorBuffer.VectorBlock.VectorEntry> entries = new ArrayList<>();
     for (int i = 0; i < count; i++) {
       Key key = new Key(prefix + "_" + i, "embedding", "vector", System.currentTimeMillis());
-      float[] vector = {(float) Math.random(), (float) Math.random(), (float) Math.random()};
+      float[] vector = {rand.nextFloat(), rand.nextFloat(), rand.nextFloat()};
       byte[] category = "public".getBytes();
       entries.add(new VectorBuffer.VectorBlock.VectorEntry(key, vector, category));
     }

core/src/test/java/org/apache/accumulo/core/file/rfile/VectorIndexFooterTest.java

@@ -118,4 +118,21 @@ public void testEmptyIndexBehavior() {
 
     assertTrue(candidates.isEmpty());
   }
+
+  @Test
+  public void testDimensionValidation() {
+    VectorIndexFooter footer =
+        new VectorIndexFooter(2, VectorIndexFooter.IndexingType.HIERARCHICAL);
+
+    // Create centroids with mismatched dimensions
+    List<float[]> centroids = Arrays.asList(new float[] {1.0f, 0.0f}, // 2D
+        new float[] {0.0f, 1.0f, 0.0f}); // 3D - this should cause an exception
+
+    try {
+      footer.buildHierarchicalIndex(centroids, 2);
+      assertTrue(false, "Expected IllegalArgumentException for mismatched dimensions");
+    } catch (IllegalArgumentException e) {
+      assertTrue(e.getMessage().contains("All points must have the same dimension"));
+    }
+  }
 }