Skip to content

Commit 2c20c49

Browse files
dimitris-athanasioudimitris-athanasiou
authored
Add usage stats for semantic_text fields (#135262)
This change enhances usage reporting for the inference plugin to account for usage of `semantic_text` fields. Usage is bucketed by service and task_type. For each bucket that corresponds to a task_type that is compatible with semantic_text this adds a `semantic_text` object that contains: - `field_count`: the number of `semantic_text` fields that use an inference endpoint of that service/task_type. - `indices_count`: the number of indices that contain at least one `semantic_field` referencing an inference endpoint of that service/task_type. - `inference_id_count`: the number of distinct inference endpoints of that service/task_type used by `semantic_text` fields. In addition, this change adds two new kinds of buckets that facilitate getting aggregate usage information: - `_all` buckets are added by `task_type`. These allow summation of usage info for all inference endpoints of a particular `task_type`. - default model buckets are added for models compatible with `semantic_text`. Those contain usage info for models that are included by default.
1 parent d5fc11a commit 2c20c49

File tree

11 files changed

+1099
-66
lines changed

11 files changed

+1099
-66
lines changed

‎docs/changelog/135262.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 135262
2+
summary: Add usage stats for `semantic_text` fields
3+
area: "Vector Search"
4+
type: enhancement
5+
issues: []

‎server/src/main/resources/transport/definitions/referable/inference_telemetry_added_semantic_text_stats.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
9182000

‎server/src/main/resources/transport/upper_bounds/9.2.csv‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
index_reshard_shardcount_small,9181000
1+
inference_telemetry_added_semantic_text_stats,9182000

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/usage/ModelStats.java‎

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,12 @@
77

88
package org.elasticsearch.xpack.core.inference.usage;
99

10+
import org.elasticsearch.TransportVersion;
1011
import org.elasticsearch.common.io.stream.StreamInput;
1112
import org.elasticsearch.common.io.stream.StreamOutput;
1213
import org.elasticsearch.common.io.stream.Writeable;
14+
import org.elasticsearch.core.Nullable;
15+
import org.elasticsearch.features.NodeFeature;
1316
import org.elasticsearch.inference.TaskType;
1417
import org.elasticsearch.xcontent.ToXContentObject;
1518
import org.elasticsearch.xcontent.XContentBuilder;
@@ -19,28 +22,34 @@
1922

2023
public class ModelStats implements ToXContentObject, Writeable {
2124

25+
public static final NodeFeature SEMANTIC_TEXT_USAGE = new NodeFeature("inference.semantic_text_usage");
26+
27+
static final TransportVersion INFERENCE_TELEMETRY_ADDED_SEMANTIC_TEXT_STATS = TransportVersion.fromName(
28+
"inference_telemetry_added_semantic_text_stats"
29+
);
30+
2231
private final String service;
2332
private final TaskType taskType;
2433
private long count;
34+
@Nullable
35+
private final SemanticTextStats semanticTextStats;
2536

26-
public ModelStats(String service, TaskType taskType) {
27-
this(service, taskType, 0L);
28-
}
29-
30-
public ModelStats(String service, TaskType taskType, long count) {
37+
public ModelStats(String service, TaskType taskType, long count, @Nullable SemanticTextStats semanticTextStats) {
3138
this.service = service;
3239
this.taskType = taskType;
3340
this.count = count;
34-
}
35-
36-
public ModelStats(ModelStats stats) {
37-
this(stats.service, stats.taskType, stats.count);
41+
this.semanticTextStats = semanticTextStats;
3842
}
3943

4044
public ModelStats(StreamInput in) throws IOException {
4145
this.service = in.readString();
4246
this.taskType = in.readEnum(TaskType.class);
4347
this.count = in.readLong();
48+
if (in.getTransportVersion().supports(INFERENCE_TELEMETRY_ADDED_SEMANTIC_TEXT_STATS)) {
49+
this.semanticTextStats = in.readOptional(SemanticTextStats::new);
50+
} else {
51+
this.semanticTextStats = null;
52+
}
4453
}
4554

4655
public void add() {
@@ -59,6 +68,11 @@ public long count() {
5968
return count;
6069
}
6170

71+
@Nullable
72+
public SemanticTextStats semanticTextStats() {
73+
return semanticTextStats;
74+
}
75+
6276
@Override
6377
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
6478
builder.startObject();
@@ -71,25 +85,34 @@ public void addXContentFragment(XContentBuilder builder, Params params) throws I
7185
builder.field("service", service);
7286
builder.field("task_type", taskType.name());
7387
builder.field("count", count);
88+
if (semanticTextStats != null) {
89+
builder.field("semantic_text", semanticTextStats);
90+
}
7491
}
7592

7693
@Override
7794
public void writeTo(StreamOutput out) throws IOException {
7895
out.writeString(service);
7996
out.writeEnum(taskType);
8097
out.writeLong(count);
98+
if (out.getTransportVersion().supports(INFERENCE_TELEMETRY_ADDED_SEMANTIC_TEXT_STATS)) {
99+
out.writeOptionalWriteable(semanticTextStats);
100+
}
81101
}
82102

83103
@Override
84104
public boolean equals(Object o) {
85105
if (this == o) return true;
86106
if (o == null || getClass() != o.getClass()) return false;
87107
ModelStats that = (ModelStats) o;
88-
return count == that.count && Objects.equals(service, that.service) && taskType == that.taskType;
108+
return count == that.count
109+
&& Objects.equals(service, that.service)
110+
&& taskType == that.taskType
111+
&& Objects.equals(semanticTextStats, that.semanticTextStats);
89112
}
90113

91114
@Override
92115
public int hashCode() {
93-
return Objects.hash(service, taskType, count);
116+
return Objects.hash(service, taskType, count, semanticTextStats);
94117
}
95118
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/usage/SemanticTextStats.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.core.inference.usage;
9+
10+
import org.elasticsearch.common.io.stream.StreamInput;
11+
import org.elasticsearch.common.io.stream.StreamOutput;
12+
import org.elasticsearch.common.io.stream.Writeable;
13+
import org.elasticsearch.xcontent.ToXContentObject;
14+
import org.elasticsearch.xcontent.XContentBuilder;
15+
16+
import java.io.IOException;
17+
import java.util.Objects;
18+
19+
public class SemanticTextStats implements ToXContentObject, Writeable {
20+
21+
private static final String FIELD_COUNT = "field_count";
22+
private static final String INDICES_COUNT = "indices_count";
23+
private static final String INFERENCE_ID_COUNT = "inference_id_count";
24+
25+
private long fieldCount;
26+
private long indicesCount;
27+
private long inferenceIdCount;
28+
29+
public SemanticTextStats() {}
30+
31+
public SemanticTextStats(long fieldCount, long indicesCount, long inferenceIdCount) {
32+
this.fieldCount = fieldCount;
33+
this.indicesCount = indicesCount;
34+
this.inferenceIdCount = inferenceIdCount;
35+
}
36+
37+
public SemanticTextStats(StreamInput in) throws IOException {
38+
fieldCount = in.readVLong();
39+
indicesCount = in.readVLong();
40+
inferenceIdCount = in.readVLong();
41+
}
42+
43+
@Override
44+
public void writeTo(StreamOutput out) throws IOException {
45+
out.writeVLong(fieldCount);
46+
out.writeVLong(indicesCount);
47+
out.writeVLong(inferenceIdCount);
48+
}
49+
50+
@Override
51+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
52+
builder.startObject();
53+
builder.field(FIELD_COUNT, fieldCount);
54+
builder.field(INDICES_COUNT, indicesCount);
55+
builder.field(INFERENCE_ID_COUNT, inferenceIdCount);
56+
builder.endObject();
57+
return builder;
58+
}
59+
60+
@Override
61+
public boolean equals(Object o) {
62+
if (this == o) return true;
63+
if (o == null || getClass() != o.getClass()) return false;
64+
SemanticTextStats that = (SemanticTextStats) o;
65+
return fieldCount == that.fieldCount && indicesCount == that.indicesCount && inferenceIdCount == that.inferenceIdCount;
66+
}
67+
68+
@Override
69+
public int hashCode() {
70+
return Objects.hash(fieldCount, indicesCount, inferenceIdCount);
71+
}
72+
73+
public long getFieldCount() {
74+
return fieldCount;
75+
}
76+
77+
public long getIndicesCount() {
78+
return indicesCount;
79+
}
80+
81+
public long getInferenceIdCount() {
82+
return inferenceIdCount;
83+
}
84+
85+
public void addFieldCount(long fieldCount) {
86+
this.fieldCount += fieldCount;
87+
}
88+
89+
public void incIndicesCount() {
90+
this.indicesCount++;
91+
}
92+
93+
public void setInferenceIdCount(long inferenceIdCount) {
94+
this.inferenceIdCount = inferenceIdCount;
95+
}
96+
97+
public boolean isEmpty() {
98+
return fieldCount == 0 && indicesCount == 0 && inferenceIdCount == 0;
99+
}
100+
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/usage/ModelStatsTests.java‎

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,17 @@
77

88
package org.elasticsearch.xpack.core.inference.usage;
99

10+
import org.elasticsearch.TransportVersion;
1011
import org.elasticsearch.common.io.stream.Writeable;
1112
import org.elasticsearch.inference.TaskType;
12-
import org.elasticsearch.test.AbstractWireSerializingTestCase;
1313
import org.elasticsearch.test.ESTestCase;
14+
import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase;
1415

1516
import java.io.IOException;
1617

1718
import static org.hamcrest.Matchers.equalTo;
1819

19-
public class ModelStatsTests extends AbstractWireSerializingTestCase<ModelStats> {
20+
public class ModelStatsTests extends AbstractBWCWireSerializationTestCase<ModelStats> {
2021

2122
@Override
2223
protected Writeable.Reader<ModelStats> instanceReader() {
@@ -33,16 +34,28 @@ protected ModelStats mutateInstance(ModelStats modelStats) throws IOException {
3334
String service = modelStats.service();
3435
TaskType taskType = modelStats.taskType();
3536
long count = modelStats.count();
36-
return switch (randomInt(2)) {
37-
case 0 -> new ModelStats(randomValueOtherThan(service, ESTestCase::randomIdentifier), taskType, count);
38-
case 1 -> new ModelStats(service, randomValueOtherThan(taskType, () -> randomFrom(TaskType.values())), count);
39-
case 2 -> new ModelStats(service, taskType, randomValueOtherThan(count, ESTestCase::randomLong));
37+
SemanticTextStats semanticTextStats = modelStats.semanticTextStats();
38+
return switch (randomInt(3)) {
39+
case 0 -> new ModelStats(randomValueOtherThan(service, ESTestCase::randomIdentifier), taskType, count, semanticTextStats);
40+
case 1 -> new ModelStats(
41+
service,
42+
randomValueOtherThan(taskType, () -> randomFrom(TaskType.values())),
43+
count,
44+
semanticTextStats
45+
);
46+
case 2 -> new ModelStats(service, taskType, randomValueOtherThan(count, ESTestCase::randomLong), semanticTextStats);
47+
case 3 -> new ModelStats(
48+
service,
49+
taskType,
50+
count,
51+
randomValueOtherThan(semanticTextStats, SemanticTextStatsTests::createRandomInstance)
52+
);
4053
default -> throw new IllegalArgumentException();
4154
};
4255
}
4356

4457
public void testAdd() {
45-
ModelStats stats = new ModelStats("test_service", randomFrom(TaskType.values()));
58+
ModelStats stats = new ModelStats("test_service", randomFrom(TaskType.values()), 0, null);
4659
assertThat(stats.count(), equalTo(0L));
4760

4861
stats.add();
@@ -56,6 +69,20 @@ public void testAdd() {
5669
}
5770

5871
public static ModelStats createRandomInstance() {
59-
return new ModelStats(randomIdentifier(), randomFrom(TaskType.values()), randomLong());
72+
TaskType taskType = randomValueOtherThan(TaskType.ANY, () -> randomFrom(TaskType.values()));
73+
return new ModelStats(
74+
randomIdentifier(),
75+
taskType,
76+
randomLong(),
77+
randomBoolean() ? SemanticTextStatsTests.createRandomInstance() : null
78+
);
79+
}
80+
81+
@Override
82+
protected ModelStats mutateInstanceForVersion(ModelStats instance, TransportVersion version) {
83+
if (version.supports(ModelStats.INFERENCE_TELEMETRY_ADDED_SEMANTIC_TEXT_STATS) == false) {
84+
return new ModelStats(instance.service(), instance.taskType(), instance.count(), null);
85+
}
86+
return instance;
6087
}
6188
}

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/inference/usage/SemanticTextStatsTests.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.core.inference.usage;
9+
10+
import org.elasticsearch.TransportVersion;
11+
import org.elasticsearch.common.io.stream.Writeable;
12+
import org.elasticsearch.test.ESTestCase;
13+
import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase;
14+
15+
import java.io.IOException;
16+
17+
import static org.hamcrest.Matchers.equalTo;
18+
import static org.hamcrest.Matchers.is;
19+
20+
public class SemanticTextStatsTests extends AbstractBWCWireSerializationTestCase<SemanticTextStats> {
21+
22+
@Override
23+
protected Writeable.Reader<SemanticTextStats> instanceReader() {
24+
return SemanticTextStats::new;
25+
}
26+
27+
@Override
28+
protected SemanticTextStats createTestInstance() {
29+
return createRandomInstance();
30+
}
31+
32+
static SemanticTextStats createRandomInstance() {
33+
return new SemanticTextStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
34+
}
35+
36+
@Override
37+
protected SemanticTextStats mutateInstance(SemanticTextStats instance) throws IOException {
38+
return switch (randomInt(2)) {
39+
case 0 -> new SemanticTextStats(
40+
randomValueOtherThan(instance.getFieldCount(), ESTestCase::randomNonNegativeLong),
41+
instance.getIndicesCount(),
42+
instance.getInferenceIdCount()
43+
);
44+
case 1 -> new SemanticTextStats(
45+
instance.getFieldCount(),
46+
randomValueOtherThan(instance.getIndicesCount(), ESTestCase::randomNonNegativeLong),
47+
instance.getInferenceIdCount()
48+
);
49+
case 2 -> new SemanticTextStats(
50+
instance.getFieldCount(),
51+
instance.getIndicesCount(),
52+
randomValueOtherThan(instance.getInferenceIdCount(), ESTestCase::randomNonNegativeLong)
53+
);
54+
default -> throw new IllegalArgumentException();
55+
};
56+
}
57+
58+
public void testDefaultConstructor() {
59+
var stats = new SemanticTextStats();
60+
assertThat(stats.getFieldCount(), equalTo(0L));
61+
assertThat(stats.getIndicesCount(), equalTo(0L));
62+
assertThat(stats.getInferenceIdCount(), equalTo(0L));
63+
}
64+
65+
public void testAddFieldCount() {
66+
var stats = new SemanticTextStats();
67+
stats.addFieldCount(10L);
68+
assertThat(stats.getFieldCount(), equalTo(10L));
69+
stats.addFieldCount(32L);
70+
assertThat(stats.getFieldCount(), equalTo(42L));
71+
}
72+
73+
public void testIsEmpty() {
74+
assertThat(new SemanticTextStats().isEmpty(), is(true));
75+
assertThat(new SemanticTextStats(randomLongBetween(1, Long.MAX_VALUE), 0, 0).isEmpty(), is(false));
76+
assertThat(new SemanticTextStats(0, randomLongBetween(1, Long.MAX_VALUE), 0).isEmpty(), is(false));
77+
assertThat(new SemanticTextStats(0, 0, randomLongBetween(1, Long.MAX_VALUE)).isEmpty(), is(false));
78+
}
79+
80+
@Override
81+
protected SemanticTextStats mutateInstanceForVersion(SemanticTextStats instance, TransportVersion version) {
82+
return instance;
83+
}
84+
}

‎x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
import org.elasticsearch.features.FeatureSpecification;
1111
import org.elasticsearch.features.NodeFeature;
12+
import org.elasticsearch.xpack.core.inference.usage.ModelStats;
1213
import org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsMapper;
1314
import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper;
1415
import org.elasticsearch.xpack.inference.queries.InterceptedInferenceQueryBuilder;
@@ -94,7 +95,8 @@ public Set<NodeFeature> getTestFeatures() {
9495
SemanticQueryBuilder.SEMANTIC_QUERY_MULTIPLE_INFERENCE_IDS,
9596
SemanticQueryBuilder.SEMANTIC_QUERY_FILTER_FIELD_CAPS_FIX,
9697
InterceptedInferenceQueryBuilder.NEW_SEMANTIC_QUERY_INTERCEPTORS,
97-
TEXT_SIMILARITY_RERANKER_SNIPPETS
98+
TEXT_SIMILARITY_RERANKER_SNIPPETS,
99+
ModelStats.SEMANTIC_TEXT_USAGE
98100
)
99101
);
100102
testFeatures.addAll(getFeatures());

0 commit comments

Comments
 (0)