diff --git a/muted-tests.yml b/muted-tests.yml index b2abae986ac29..9785e69f20544 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -393,15 +393,6 @@ tests: - class: org.elasticsearch.action.admin.cluster.node.tasks.CancellableTasksIT method: testChildrenTasksCancelledOnTimeout issue: https://github.com/elastic/elasticsearch/issues/123568 -- class: org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests - method: testSnapshotRecovery {p0=false p1=false} - issue: https://github.com/elastic/elasticsearch/issues/124385 -- class: org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests - method: testSnapshotRecovery {p0=false p1=true} - issue: https://github.com/elastic/elasticsearch/issues/124383 -- class: org.elasticsearch.xpack.inference.mapper.SemanticInferenceMetadataFieldsRecoveryTests - method: testSnapshotRecovery {p0=true p1=false} - issue: https://github.com/elastic/elasticsearch/issues/124384 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT method: test {yaml=reference/cat/nodes/line_361} issue: https://github.com/elastic/elasticsearch/issues/124103 diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 34925607a0424..6d3b77901aff3 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -1378,7 +1378,8 @@ public static void assertConsistentHistoryBetweenTranslogAndLuceneIndex(Engine e assertThat(luceneOp.toString(), luceneOp.primaryTerm(), equalTo(translogOp.primaryTerm())); assertThat(luceneOp.opType(), equalTo(translogOp.opType())); if (luceneOp.opType() == Translog.Operation.Type.INDEX) { - if (engine.engineConfig.getIndexSettings().isRecoverySourceSyntheticEnabled()) { + if (engine.engineConfig.getIndexSettings().isRecoverySourceSyntheticEnabled() + || engine.engineConfig.getMapperService().mappingLookup().inferenceFields().isEmpty() == false) { assertTrue( "luceneOp=" + luceneOp + " != translogOp=" + translogOp, translogOperationAsserter.assertSameIndexOperation((Translog.Index) luceneOp, (Translog.Index) translogOp) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java index add130da2d368..175c3e90f798d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.engine.LuceneChangesSnapshot; import org.elasticsearch.index.engine.LuceneSyntheticSourceChangesSnapshot; import org.elasticsearch.index.engine.SearchBasedChangesSnapshot; +import org.elasticsearch.index.engine.TranslogOperationAsserter; import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.SourceFieldMapper; @@ -42,7 +43,6 @@ import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.generateRandomChunkingSettings; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomChunkedInferenceEmbeddingByte; import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomChunkedInferenceEmbeddingFloat; @@ -137,7 +137,7 @@ protected String defaultMapping() { } public void testSnapshotRecovery() throws IOException { - List expectedOperations = new ArrayList<>(); + List expectedOperations = new ArrayList<>(); int size = randomIntBetween(10, 50); for (int i = 0; i < size; i++) { var source = randomSource(); @@ -159,8 +159,19 @@ public void testSnapshotRecovery() throws IOException { } } var op = indexForDoc(doc); - expectedOperations.add(op); - engine.index(op); + var result = engine.index(op); + expectedOperations.add( + new Translog.Index( + result.getId(), + result.getSeqNo(), + result.getTerm(), + result.getVersion(), + op.source(), + op.routing(), + op.getAutoGeneratedIdTimestamp() + ) + ); + if (frequently()) { engine.flush(); } @@ -181,13 +192,12 @@ public void testSnapshotRecovery() throws IOException { IndexVersion.current() ) ) { + var asserter = TranslogOperationAsserter.withEngineConfig(engine.config()); for (int i = 0; i < size; i++) { var op = snapshot.next(); assertThat(op.opType(), equalTo(Translog.Operation.Type.INDEX)); Translog.Index indexOp = (Translog.Index) op; - assertThat(indexOp.id(), equalTo(expectedOperations.get(i).id())); - assertThat(indexOp.routing(), equalTo(expectedOperations.get(i).routing())); - assertToXContentEquivalent(indexOp.source(), expectedOperations.get(i).source(), XContentType.JSON); + asserter.assertSameIndexOperation(indexOp, expectedOperations.get(i)); } assertNull(snapshot.next()); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java index b4ac5c475d425..6e90e9d3af12e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java @@ -194,7 +194,8 @@ public static ChunkedInferenceEmbedding randomChunkedInferenceEmbeddingByte(Mode for (String input : inputs) { byte[] values = new byte[embeddingLength]; for (int j = 0; j < values.length; j++) { - values[j] = randomByte(); + // to avoid vectors with zero magnitude + values[j] = (byte) Math.max(1, randomByte()); } chunks.add( new EmbeddingResults.Chunk( @@ -215,7 +216,8 @@ public static ChunkedInferenceEmbedding randomChunkedInferenceEmbeddingFloat(Mod for (String input : inputs) { float[] values = new float[embeddingLength]; for (int j = 0; j < values.length; j++) { - values[j] = randomFloat(); + // to avoid vectors with zero magnitude + values[j] = Math.max(1e-6f, randomFloat()); } chunks.add( new EmbeddingResults.Chunk( @@ -236,7 +238,7 @@ public static ChunkedInferenceEmbedding randomChunkedInferenceEmbeddingSparse(Li for (String input : inputs) { var tokens = new ArrayList(); for (var token : input.split("\\s+")) { - tokens.add(new WeightedToken(token, withFloats ? randomFloat() : randomIntBetween(1, 255))); + tokens.add(new WeightedToken(token, withFloats ? Math.max(Float.MIN_NORMAL, randomFloat()) : randomIntBetween(1, 255))); } chunks.add( new EmbeddingResults.Chunk(