Skip to content

Commit d089204

Browse files
lukewhitinglukewhiting
authored
Merge branch 'main' into ES-12330-prevent-auto-shard-on-lookup-index
2 parents d580af0 + f9eee6c commit d089204

File tree

3 files changed

+23
-25
lines changed

3 files changed

+23
-25
lines changed

‎rest-api-spec/src/main/resources/rest-api-spec/api/searchable_snapshots.clear_cache.json‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,6 @@
5050
],
5151
"default": "open",
5252
"description": "Whether to expand wildcard expression to concrete indices that are open, closed or both."
53-
},
54-
"index": {
55-
"type": "list",
56-
"description": "A comma-separated list of index name to limit the operation"
5753
}
5854
}
5955
}

‎server/src/main/java/org/elasticsearch/index/codec/vectors/DefaultIVFVectorsWriter.java‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,11 @@
1717
import org.apache.lucene.store.IOContext;
1818
import org.apache.lucene.store.IndexInput;
1919
import org.apache.lucene.store.IndexOutput;
20+
import org.apache.lucene.util.LongValues;
2021
import org.apache.lucene.util.VectorUtil;
2122
import org.apache.lucene.util.hnsw.IntToIntFunction;
23+
import org.apache.lucene.util.packed.PackedInts;
24+
import org.apache.lucene.util.packed.PackedLongValues;
2225
import org.elasticsearch.index.codec.vectors.cluster.HierarchicalKMeans;
2326
import org.elasticsearch.index.codec.vectors.cluster.KMeansResult;
2427
import org.elasticsearch.logging.LogManager;
@@ -46,7 +49,7 @@ public DefaultIVFVectorsWriter(SegmentWriteState state, FlatVectorsWriter rawVec
4649
}
4750

4851
@Override
49-
long[] buildAndWritePostingsLists(
52+
LongValues buildAndWritePostingsLists(
5053
FieldInfo fieldInfo,
5154
CentroidSupplier centroidSupplier,
5255
FloatVectorValues floatVectorValues,
@@ -81,7 +84,7 @@ long[] buildAndWritePostingsLists(
8184
}
8285
}
8386
// write the posting lists
84-
final long[] offsets = new long[centroidSupplier.size()];
87+
final PackedLongValues.Builder offsets = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
8588
DocIdsWriter docIdsWriter = new DocIdsWriter();
8689
DiskBBQBulkWriter bulkWriter = new DiskBBQBulkWriter.OneBitDiskBBQBulkWriter(ES91OSQVectorsScorer.BULK_SIZE, postingsOutput);
8790
OnHeapQuantizedVectors onHeapQuantizedVectors = new OnHeapQuantizedVectors(
@@ -93,7 +96,7 @@ long[] buildAndWritePostingsLists(
9396
float[] centroid = centroidSupplier.centroid(c);
9497
int[] cluster = assignmentsByCluster[c];
9598
// TODO align???
96-
offsets[c] = postingsOutput.getFilePointer();
99+
offsets.add(postingsOutput.getFilePointer());
97100
int size = cluster.length;
98101
postingsOutput.writeVInt(size);
99102
postingsOutput.writeInt(Float.floatToIntBits(VectorUtil.dotProduct(centroid, centroid)));
@@ -109,11 +112,11 @@ long[] buildAndWritePostingsLists(
109112
printClusterQualityStatistics(assignmentsByCluster);
110113
}
111114

112-
return offsets;
115+
return offsets.build();
113116
}
114117

115118
@Override
116-
long[] buildAndWritePostingsLists(
119+
LongValues buildAndWritePostingsLists(
117120
FieldInfo fieldInfo,
118121
CentroidSupplier centroidSupplier,
119122
FloatVectorValues floatVectorValues,
@@ -199,7 +202,7 @@ long[] buildAndWritePostingsLists(
199202
}
200203
// now we can read the quantized vectors from the temporary file
201204
try (IndexInput quantizedVectorsInput = mergeState.segmentInfo.dir.openInput(quantizedVectorsTempName, IOContext.DEFAULT)) {
202-
final long[] offsets = new long[centroidSupplier.size()];
205+
final PackedLongValues.Builder offsets = PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
203206
OffHeapQuantizedVectors offHeapQuantizedVectors = new OffHeapQuantizedVectors(
204207
quantizedVectorsInput,
205208
fieldInfo.getVectorDimension()
@@ -210,9 +213,9 @@ long[] buildAndWritePostingsLists(
210213
float[] centroid = centroidSupplier.centroid(c);
211214
int[] cluster = assignmentsByCluster[c];
212215
boolean[] isOverspill = isOverspillByCluster[c];
213-
// TODO align???
214-
offsets[c] = postingsOutput.getFilePointer();
216+
offsets.add(postingsOutput.getFilePointer());
215217
int size = cluster.length;
218+
// TODO align???
216219
postingsOutput.writeVInt(size);
217220
postingsOutput.writeInt(Float.floatToIntBits(VectorUtil.dotProduct(centroid, centroid)));
218221
offHeapQuantizedVectors.reset(size, ord -> isOverspill[ord], ord -> cluster[ord]);
@@ -226,7 +229,7 @@ long[] buildAndWritePostingsLists(
226229
if (logger.isDebugEnabled()) {
227230
printClusterQualityStatistics(assignmentsByCluster);
228231
}
229-
return offsets;
232+
return offsets.build();
230233
}
231234
}
232235

@@ -270,7 +273,7 @@ void writeCentroids(
270273
FieldInfo fieldInfo,
271274
CentroidSupplier centroidSupplier,
272275
float[] globalCentroid,
273-
long[] offsets,
276+
LongValues offsets,
274277
IndexOutput centroidOutput
275278
) throws IOException {
276279

@@ -302,7 +305,7 @@ void writeCentroids(
302305
// write the centroids
303306
centroidOutput.writeBytes(buffer.array(), buffer.array().length);
304307
// write the offset of this posting list
305-
centroidOutput.writeLong(offsets[i]);
308+
centroidOutput.writeLong(offsets.get(i));
306309
}
307310
}
308311

‎server/src/main/java/org/elasticsearch/index/codec/vectors/IVFVectorsWriter.java‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.lucene.store.IndexInput;
2929
import org.apache.lucene.store.IndexOutput;
3030
import org.apache.lucene.store.RandomAccessInput;
31+
import org.apache.lucene.util.LongValues;
3132
import org.apache.lucene.util.VectorUtil;
3233
import org.elasticsearch.core.IOUtils;
3334
import org.elasticsearch.core.SuppressForbidden;
@@ -126,11 +127,11 @@ abstract void writeCentroids(
126127
FieldInfo fieldInfo,
127128
CentroidSupplier centroidSupplier,
128129
float[] globalCentroid,
129-
long[] centroidOffset,
130+
LongValues centroidOffset,
130131
IndexOutput centroidOutput
131132
) throws IOException;
132133

133-
abstract long[] buildAndWritePostingsLists(
134+
abstract LongValues buildAndWritePostingsLists(
134135
FieldInfo fieldInfo,
135136
CentroidSupplier centroidSupplier,
136137
FloatVectorValues floatVectorValues,
@@ -139,7 +140,7 @@ abstract long[] buildAndWritePostingsLists(
139140
int[] overspillAssignments
140141
) throws IOException;
141142

142-
abstract long[] buildAndWritePostingsLists(
143+
abstract LongValues buildAndWritePostingsLists(
143144
FieldInfo fieldInfo,
144145
CentroidSupplier centroidSupplier,
145146
FloatVectorValues floatVectorValues,
@@ -160,25 +161,24 @@ abstract CentroidSupplier createCentroidSupplier(
160161
public final void flush(int maxDoc, Sorter.DocMap sortMap) throws IOException {
161162
rawVectorDelegate.flush(maxDoc, sortMap);
162163
for (FieldWriter fieldWriter : fieldWriters) {
163-
float[] globalCentroid = new float[fieldWriter.fieldInfo.getVectorDimension()];
164+
final float[] globalCentroid = new float[fieldWriter.fieldInfo.getVectorDimension()];
164165
// build a float vector values with random access
165166
final FloatVectorValues floatVectorValues = getFloatVectorValues(fieldWriter.fieldInfo, fieldWriter.delegate, maxDoc);
166167
// build centroids
167168
final CentroidAssignments centroidAssignments = calculateCentroids(fieldWriter.fieldInfo, floatVectorValues, globalCentroid);
168169
// wrap centroids with a supplier
169170
final CentroidSupplier centroidSupplier = new OnHeapCentroidSupplier(centroidAssignments.centroids());
170171
// write posting lists
171-
final long[] offsets = buildAndWritePostingsLists(
172+
final LongValues offsets = buildAndWritePostingsLists(
172173
fieldWriter.fieldInfo,
173174
centroidSupplier,
174175
floatVectorValues,
175176
ivfClusters,
176177
centroidAssignments.assignments(),
177178
centroidAssignments.overspillAssignments()
178179
);
179-
assert offsets.length == centroidSupplier.size();
180-
final long centroidOffset = ivfCentroids.alignFilePointer(Float.BYTES);
181180
// write centroids
181+
final long centroidOffset = ivfCentroids.alignFilePointer(Float.BYTES);
182182
writeCentroids(fieldWriter.fieldInfo, centroidSupplier, globalCentroid, offsets, ivfCentroids);
183183
final long centroidLength = ivfCentroids.getFilePointer() - centroidOffset;
184184
// write meta file
@@ -338,7 +338,7 @@ private void mergeOneFieldIVF(FieldInfo fieldInfo, MergeState mergeState) throws
338338
calculatedGlobalCentroid
339339
);
340340
// write posting lists
341-
final long[] offsets = buildAndWritePostingsLists(
341+
final LongValues offsets = buildAndWritePostingsLists(
342342
fieldInfo,
343343
centroidSupplier,
344344
floatVectorValues,
@@ -347,9 +347,8 @@ private void mergeOneFieldIVF(FieldInfo fieldInfo, MergeState mergeState) throws
347347
assignments,
348348
overspillAssignments
349349
);
350-
assert offsets.length == centroidSupplier.size();
351-
centroidOffset = ivfCentroids.alignFilePointer(Float.BYTES);
352350
// write centroids
351+
centroidOffset = ivfCentroids.alignFilePointer(Float.BYTES);
353352
writeCentroids(fieldInfo, centroidSupplier, calculatedGlobalCentroid, offsets, ivfCentroids);
354353
centroidLength = ivfCentroids.getFilePointer() - centroidOffset;
355354
// write meta

0 commit comments

Comments
 (0)