Skip to content

Commit 1f742f4

Browse files
Add more tests
1 parent 7a2cd83 commit 1f742f4

File tree

7 files changed

+187
-32
lines changed

7 files changed

+187
-32
lines changed

server/src/main/java/org/elasticsearch/index/IndexSettings.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1182,7 +1182,6 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
11821182
}
11831183

11841184
scopedSettings.addSettingsUpdateConsumer(VECTORS_INDEXING_USE_GPU_SETTING, this::setUseGpuForVectorsIndexing);
1185-
11861185
scopedSettings.addSettingsUpdateConsumer(
11871186
MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING,
11881187
mergePolicyConfig::setCompoundFormatThreshold

x-pack/plugin/gpu/src/internalClusterTest/java/org/elasticsearch/plugin/gpu/GPUIndexIT.java

Lines changed: 75 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,23 @@
77

88
package org.elasticsearch.plugin.gpu;
99

10+
import org.apache.lucene.tests.util.LuceneTestCase;
11+
import org.elasticsearch.action.bulk.BulkRequestBuilder;
12+
import org.elasticsearch.action.bulk.BulkResponse;
1013
import org.elasticsearch.common.settings.Settings;
1114
import org.elasticsearch.plugins.Plugin;
15+
import org.elasticsearch.search.vectors.KnnSearchBuilder;
1216
import org.elasticsearch.test.ESIntegTestCase;
1317
import org.elasticsearch.xpack.gpu.GPUPlugin;
1418

1519
import java.util.Collection;
1620
import java.util.List;
21+
import java.util.Locale;
1722

1823
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
24+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailuresAndResponse;
1925

26+
@LuceneTestCase.SuppressCodecs("*") // use our custom codec
2027
public class GPUIndexIT extends ESIntegTestCase {
2128

2229
@Override
@@ -25,40 +32,82 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
2532
}
2633

2734
public void testBasic() {
35+
final int dims = randomIntBetween(4, 128);
36+
final int[] numDocs = new int[] { randomIntBetween(1, 100), 1, 2, randomIntBetween(1, 100) };
37+
createIndex(dims);
38+
int totalDocs = 0;
39+
for (int i = 0; i < numDocs.length; i++) {
40+
indexDocs(numDocs[i], dims, i * 100);
41+
totalDocs += numDocs[i];
42+
}
43+
refresh();
44+
assertSearch(randomFloatVector(dims), totalDocs);
45+
}
46+
47+
public void testSearchWithoutGPU() {
48+
final int dims = randomIntBetween(4, 128);
49+
final int numDocs = randomIntBetween(1, 500);
50+
createIndex(dims);
51+
ensureGreen();
52+
53+
indexDocs(numDocs, dims, 0);
54+
refresh();
55+
56+
// update settings to disable GPU usage
57+
Settings.Builder settingsBuilder = Settings.builder().put("index.vectors.indexing.use_gpu", false);
58+
assertAcked(client().admin().indices().prepareUpdateSettings("foo-index").setSettings(settingsBuilder.build()));
59+
ensureGreen();
60+
assertSearch(randomFloatVector(dims), numDocs);
61+
}
62+
63+
private void createIndex(int dims) {
2864
var settings = Settings.builder().put(indexSettings());
2965
settings.put("index.number_of_shards", 1);
30-
settings.put("index.vectors.indexing.use_gpu", "true");
31-
assertAcked(prepareCreate("foo-index").setSettings(settings.build()).setMapping("""
32-
"properties": {
33-
"my_vector": {
34-
"type": "dense_vector",
35-
"dims": 5,
36-
"similarity": "l2_norm",
37-
"index_options": {
38-
"type": "hnsw"
66+
settings.put("index.vectors.indexing.use_gpu", true);
67+
assertAcked(prepareCreate("foo-index").setSettings(settings.build()).setMapping(String.format(Locale.ROOT, """
68+
{
69+
"properties": {
70+
"my_vector": {
71+
"type": "dense_vector",
72+
"dims": %d,
73+
"similarity": "l2_norm",
74+
"index_options": {
75+
"type": "hnsw"
76+
}
3977
}
4078
}
4179
}
42-
"""));
80+
""", dims)));
4381
ensureGreen();
82+
}
4483

45-
prepareIndex("foo-index").setId("1").setSource("my_vector", new float[] { 230.0f, 300.33f, -34.8988f, 15.555f, -200.0f }).get();
46-
47-
// TODO: add more docs...
84+
private void indexDocs(int numDocs, int dims, int startDoc) {
85+
BulkRequestBuilder bulkRequest = client().prepareBulk();
86+
for (int i = 0; i < numDocs; i++) {
87+
String id = String.valueOf(startDoc + i);
88+
bulkRequest.add(prepareIndex("foo-index").setId(id).setSource("my_vector", randomFloatVector(dims)));
89+
}
90+
BulkResponse bulkResponse = bulkRequest.get();
91+
assertFalse("Bulk request failed: " + bulkResponse.buildFailureMessage(), bulkResponse.hasFailures());
92+
}
4893

49-
ensureGreen();
50-
refresh();
94+
private void assertSearch(float[] queryVector, int totalDocs) {
95+
int k = Math.min(randomIntBetween(1, 20), totalDocs);
96+
int numCandidates = k * 10;
97+
assertNoFailuresAndResponse(
98+
prepareSearch("foo-index").setSize(k)
99+
.setKnnSearch(List.of(new KnnSearchBuilder("my_vector", queryVector, k, numCandidates, null, null))),
100+
response -> {
101+
assertEquals("Expected k hits to be returned", k, response.getHits().getHits().length);
102+
}
103+
);
104+
}
51105

52-
// TODO: do some basic search
53-
// var knn = new KnnSearchBuilder("nested.vector", new float[] { -0.5f, 90.0f, -10f, 14.8f, -156.0f }, 2, 3, null, null);
54-
// var request = prepareSearch("test").addFetchField("name").setKnnSearch(List.of(knn));
55-
// assertNoFailuresAndResponse(request, response -> {
56-
// assertHitCount(response, 2);
57-
// assertEquals("2", response.getHits().getHits()[0].getId());
58-
// assertEquals("cat", response.getHits().getHits()[0].field("name").getValue());
59-
// assertEquals("3", response.getHits().getHits()[1].getId());
60-
// assertEquals("rat", response.getHits().getHits()[1].field("name").getValue());
61-
// });
62-
// }
106+
private static float[] randomFloatVector(int dims) {
107+
float[] vector = new float[dims];
108+
for (int i = 0; i < dims; i++) {
109+
vector[i] = randomFloat();
110+
}
111+
return vector;
63112
}
64113
}

x-pack/plugin/gpu/src/main/java/module-info.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,5 @@
1717
exports org.elasticsearch.xpack.gpu.codec;
1818

1919
provides org.apache.lucene.codecs.KnnVectorsFormat with org.elasticsearch.xpack.gpu.codec.GPUVectorsFormat;
20+
provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.gpu.GPUFeatures;
2021
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.gpu;
9+
10+
import org.elasticsearch.features.FeatureSpecification;
11+
import org.elasticsearch.features.NodeFeature;
12+
13+
import java.util.Set;
14+
15+
public class GPUFeatures implements FeatureSpecification {
16+
17+
public static final NodeFeature VECTORS_INDEXING_USE_GPU = new NodeFeature("vectors.indexing.use_gpu");
18+
19+
@Override
20+
public Set<NodeFeature> getFeatures() {
21+
return Set.of();
22+
}
23+
24+
@Override
25+
public Set<NodeFeature> getTestFeatures() {
26+
return Set.of(VECTORS_INDEXING_USE_GPU);
27+
}
28+
}

x-pack/plugin/gpu/src/main/java/org/elasticsearch/xpack/gpu/GPUPlugin.java

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
import org.elasticsearch.common.util.FeatureFlag;
1212
import org.elasticsearch.index.IndexSettings;
13+
import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
1314
import org.elasticsearch.index.mapper.vectors.VectorsFormatProvider;
1415
import org.elasticsearch.plugins.MapperPlugin;
1516
import org.elasticsearch.plugins.Plugin;
@@ -25,20 +26,33 @@ public VectorsFormatProvider getVectorsFormatProvider() {
2526
if (GPU_FORMAT.isEnabled()) {
2627
IndexSettings.GpuMode gpuMode = indexSettings.getValue(IndexSettings.VECTORS_INDEXING_USE_GPU_SETTING);
2728
if (gpuMode == IndexSettings.GpuMode.TRUE) {
29+
if (vectorIndexTypeSupported(indexOptions.getType()) == false) {
30+
throw new IllegalArgumentException(
31+
"[index.vectors.indexing.use_gpu] was set to [true], but GPU vector indexing is only supported "
32+
+ "for [hnsw] index_options.type, got: ["
33+
+ indexOptions.getType()
34+
+ "]"
35+
);
36+
}
2837
CuVSResources resources = GPUVectorsFormat.cuVSResourcesOrNull(true);
2938
if (resources == null) {
3039
throw new IllegalArgumentException(
3140
"[index.vectors.indexing.use_gpu] was set to [true], but GPU resources are not accessible on the node."
3241
);
33-
} else {
34-
return new GPUVectorsFormat();
3542
}
43+
return new GPUVectorsFormat();
3644
}
37-
if ((gpuMode == IndexSettings.GpuMode.AUTO) && GPUVectorsFormat.cuVSResourcesOrNull(false) != null) {
45+
if ((gpuMode == IndexSettings.GpuMode.AUTO)
46+
&& vectorIndexTypeSupported(indexOptions.getType())
47+
&& GPUVectorsFormat.cuVSResourcesOrNull(false) != null) {
3848
return new GPUVectorsFormat();
3949
}
4050
}
4151
return null;
4252
};
4353
}
54+
55+
private boolean vectorIndexTypeSupported(DenseVectorFieldMapper.VectorIndexType type) {
56+
return type == DenseVectorFieldMapper.VectorIndexType.HNSW;
57+
}
4458
}
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#
2+
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
# or more contributor license agreements. Licensed under the Elastic License
4+
# 2.0; you may not use this file except in compliance with the Elastic License
5+
# 2.0.
6+
#
7+
8+
org.elasticsearch.xpack.gpu.GPUFeatures

x-pack/plugin/gpu/src/yamlRestTest/resources/rest-api-spec/test/gpu/10_basic.yml

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
---
22
"Test GPU vector operations":
3+
4+
- requires:
5+
cluster_features: [ "vectors.indexing.use_gpu" ]
6+
reason: "A cluster should have a GPU plugin to run these tests"
7+
8+
# creating an index is successful even if the GPU is not available
39
- do:
410
indices.create:
511
index: my_vectors
@@ -12,6 +18,11 @@
1218
similarity: l2_norm
1319
index_options:
1420
type: hnsw
21+
settings:
22+
index.number_of_shards: 1
23+
index.vectors.indexing.use_gpu: true
24+
- match: { error: null }
25+
1526

1627
- do:
1728
bulk:
@@ -32,14 +43,59 @@
3243
embedding: [0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
3344
- match: { errors: false }
3445

46+
- do:
47+
bulk:
48+
index: my_vectors
49+
refresh: true
50+
body:
51+
- index:
52+
_id: "4"
53+
- text: "Fourth document"
54+
embedding: [0.05, 0.12, 0.18, 0.22, 0.29, 0.33, 0.41, 0.47, 0.53, 0.59, 0.65, 0.71, 0.77, 0.83, 0.89, 0.95, 0.11, 0.17, 0.23, 0.29, 0.35, 0.41, 0.47, 0.53, 0.59, 0.65, 0.71, 0.77, 0.83, 0.89, 0.95, 0.01, 0.07, 0.13, 0.19, 0.25, 0.31, 0.37, 0.43, 0.49, 0.55, 0.61, 0.67, 0.73, 0.79, 0.85, 0.91, 0.97, 0.03, 0.09, 0.15, 0.21, 0.27, 0.33, 0.39, 0.45, 0.51, 0.57, 0.63, 0.69, 0.75, 0.81, 0.87, 0.93, 0.99, 0.05, 0.11, 0.17, 0.23, 0.29, 0.35, 0.41, 0.47, 0.53, 0.59, 0.65, 0.71, 0.77, 0.83, 0.89, 0.95, 0.01, 0.07, 0.13, 0.19, 0.25, 0.31, 0.37, 0.43, 0.49, 0.55, 0.61, 0.67, 0.73, 0.79, 0.85, 0.91, 0.97, 0.03, 0.09, 0.15, 0.21, 0.27, 0.33, 0.39, 0.45, 0.51, 0.57, 0.63, 0.69, 0.75, 0.81, 0.87, 0.93, 0.99, 0.05, 0.11, 0.17, 0.23, 0.29, 0.35, 0.41, 0.47, 0.53, 0.59, 0.65, 0.71, 0.46]
55+
- index:
56+
_id: "5"
57+
- text: "Fifth document"
58+
embedding: [0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 0.13, 0.23, 0.33, 0.43, 0.53, 0.63, 0.73, 0.83, 0.93, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 0.17, 0.27, 0.37, 0.47, 0.57, 0.67, 0.77, 0.87, 0.97, 0.19, 0.29, 0.39, 0.49, 0.59, 0.69, 0.79, 0.89, 0.99, 0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 0.13, 0.23, 0.33, 0.43, 0.53, 0.63, 0.73, 0.83, 0.93, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 0.17, 0.27, 0.37, 0.47, 0.57, 0.67, 0.77, 0.87, 0.97, 0.19, 0.29, 0.39, 0.49, 0.59, 0.69, 0.79, 0.89, 0.99, 0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 0.13, 0.23, 0.33, 0.43, 0.53, 0.63, 0.73, 0.83, 0.93, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 0.17, 0.27, 0.37, 0.47, 0.57, 0.67, 0.77, 0.87, 0.97, 0.19, 0.29, 0.39]
59+
- index:
60+
_id: "6"
61+
- text: "Sixth document"
62+
embedding: [0.12, 0.22, 0.32, 0.42, 0.52, 0.62, 0.72, 0.82, 0.92, 0.14, 0.24, 0.34, 0.44, 0.54, 0.64, 0.74, 0.84, 0.94, 0.16, 0.26, 0.36, 0.46, 0.56, 0.66, 0.76, 0.86, 0.96, 0.18, 0.28, 0.38, 0.48, 0.58, 0.68, 0.78, 0.88, 0.98, 0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 0.13, 0.23, 0.33, 0.43, 0.53, 0.63, 0.73, 0.83, 0.93, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 0.17, 0.27, 0.37, 0.47, 0.57, 0.67, 0.77, 0.87, 0.97, 0.19, 0.29, 0.39, 0.49, 0.59, 0.69, 0.79, 0.89, 0.99, 0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 0.13, 0.23, 0.33, 0.43, 0.53, 0.63, 0.73, 0.83, 0.93, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 0.17, 0.27, 0.37, 0.47, 0.57, 0.67, 0.77, 0.87, 0.97, 0.19, 0.29, 0.39, 0.49, 0.59, 0.69, 0.79, 0.89, 0.29, 0.39, 0.49]
63+
- index:
64+
_id: "7"
65+
- text: "Seventh document"
66+
embedding: [0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 0.05, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 0.07, 0.17, 0.27, 0.37, 0.47, 0.57, 0.67, 0.77, 0.87, 0.97, 0.09, 0.19, 0.29, 0.39, 0.49, 0.59, 0.69, 0.79, 0.89, 0.99, 0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 0.13, 0.23, 0.33, 0.43, 0.53, 0.63, 0.73, 0.83, 0.93, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 0.17, 0.27, 0.37, 0.47, 0.57, 0.67, 0.77, 0.87, 0.97, 0.19, 0.29, 0.39, 0.49, 0.59, 0.69, 0.79, 0.89, 0.99, 0.11, 0.21, 0.31, 0.41, 0.51, 0.61, 0.71, 0.81, 0.91, 0.13, 0.23, 0.33, 0.43, 0.53, 0.63, 0.73, 0.83, 0.93, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 0.17, 0.27, 0.37, 0.47, 0.57, 0.67, 0.77, 0.87, 0.97, 0.19, 0.29, 0.39, 0.49, 0.59, 0.69, 0.79, 0.89, 0.90]
67+
- match: { errors: false }
68+
3569
- do:
3670
search:
3771
index: my_vectors
3872
body:
3973
knn:
4074
field: embedding
4175
query_vector: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]
42-
k: 3
76+
k: 10
4377

44-
- match: { hits.total.value: 3 }
78+
- match: { hits.total.value: 7 }
4579
- match: { hits.hits.0._id: "1" }
80+
81+
- do:
82+
bulk:
83+
index: my_vectors
84+
refresh: true
85+
body:
86+
- delete:
87+
_id: "1"
88+
- delete:
89+
_id: "7"
90+
- match: { errors: false }
91+
92+
- do:
93+
search:
94+
index: my_vectors
95+
body:
96+
knn:
97+
field: embedding
98+
query_vector: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]
99+
k: 10
100+
- match: { hits.total.value: 5 }
101+
- match: { hits.hits.0._id: "2" }

0 commit comments

Comments
 (0)