Skip to content

Commit 81b4cce

Browse files
authored
Update EIS sparse and dense embedding max batch size to 16 (#132646)
1 parent 8d12540 commit 81b4cce

File tree

2 files changed

+13
-3
lines changed

2 files changed

+13
-3
lines changed

docs/changelog/132646.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 132646
2+
summary: Update EIS sparse and dense embedding max batch size to 16
3+
area: Machine Learning
4+
type: bug
5+
issues: []

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/ElasticInferenceService.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,10 @@ public class ElasticInferenceService extends SenderService {
9191
public static final String NAME = "elastic";
9292
public static final String ELASTIC_INFERENCE_SERVICE_IDENTIFIER = "Elastic Inference Service";
9393
public static final Integer DENSE_TEXT_EMBEDDINGS_DIMENSIONS = 1024;
94-
public static final Integer SPARSE_TEXT_EMBEDDING_MAX_BATCH_SIZE = 512;
94+
// The maximum batch size for sparse text embeddings is set to 16.
95+
// This value was reduced from 512 due to memory constraints; batch sizes above 32 can cause GPU out-of-memory errors.
96+
// A batch size of 16 provides optimal throughput and stability, especially on lower-tier instance types.
97+
public static final Integer SPARSE_TEXT_EMBEDDING_MAX_BATCH_SIZE = 16;
9598

9699
private static final EnumSet<TaskType> IMPLEMENTED_TASK_TYPES = EnumSet.of(
97100
TaskType.SPARSE_EMBEDDING,
@@ -101,8 +104,10 @@ public class ElasticInferenceService extends SenderService {
101104
);
102105
private static final String SERVICE_NAME = "Elastic";
103106

104-
// TODO: check with team, what makes the most sense
105-
private static final Integer DENSE_TEXT_EMBEDDINGS_MAX_BATCH_SIZE = 32;
107+
// TODO: revisit this value once EIS supports dense models
108+
// The maximum batch size for dense text embeddings is proactively set to 16.
109+
// This mirrors the memory constraints observed with sparse embeddings
110+
private static final Integer DENSE_TEXT_EMBEDDINGS_MAX_BATCH_SIZE = 16;
106111

107112
// rainbow-sprinkles
108113
static final String DEFAULT_CHAT_COMPLETION_MODEL_ID_V1 = "rainbow-sprinkles";

0 commit comments

Comments
 (0)