Fix and unmute SemanticInferenceMetadataFieldsRecoveryTests (#126784) (#126815) (#132420)

Backport of #126784 to 8.18

Closes #131570 Closes #131571  Closes #131572

Author: jimczi

test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java

@@ -1378,7 +1378,8 @@ public static void assertConsistentHistoryBetweenTranslogAndLuceneIndex(Engine e
             assertThat(luceneOp.toString(), luceneOp.primaryTerm(), equalTo(translogOp.primaryTerm()));
             assertThat(luceneOp.opType(), equalTo(translogOp.opType()));
             if (luceneOp.opType() == Translog.Operation.Type.INDEX) {
-                if (engine.engineConfig.getIndexSettings().isRecoverySourceSyntheticEnabled()) {
+                if (engine.engineConfig.getIndexSettings().isRecoverySourceSyntheticEnabled()
+                    || engine.engineConfig.getMapperService().mappingLookup().inferenceFields().isEmpty() == false) {
                     assertTrue(
                         "luceneOp=" + luceneOp + " != translogOp=" + translogOp,
                         translogOperationAsserter.assertSameIndexOperation((Translog.Index) luceneOp, (Translog.Index) translogOp)

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsRecoveryTests.java

@@ -21,6 +21,7 @@
 import org.elasticsearch.index.engine.LuceneChangesSnapshot;
 import org.elasticsearch.index.engine.LuceneSyntheticSourceChangesSnapshot;
 import org.elasticsearch.index.engine.SearchBasedChangesSnapshot;
+import org.elasticsearch.index.engine.TranslogOperationAsserter;
 import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
@@ -42,7 +43,6 @@
 import java.util.ArrayList;
 import java.util.List;
 
-import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomChunkedInferenceEmbeddingByte;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.randomChunkedInferenceEmbeddingSparse;
 import static org.elasticsearch.xpack.inference.mapper.SemanticTextFieldTests.semanticTextFieldFromChunkedInferenceResults;
@@ -124,7 +124,7 @@ protected String defaultMapping() {
     }
 
     public void testSnapshotRecovery() throws IOException {
-        List<Engine.Index> expectedOperations = new ArrayList<>();
+        List<Translog.Index> expectedOperations = new ArrayList<>();
         int size = randomIntBetween(10, 50);
         for (int i = 0; i < size; i++) {
             var source = randomSource();
@@ -146,8 +146,19 @@ public void testSnapshotRecovery() throws IOException {
                 }
             }
             var op = indexForDoc(doc);
-            expectedOperations.add(op);
-            engine.index(op);
+            var result = engine.index(op);
+            expectedOperations.add(
+                new Translog.Index(
+                    result.getId(),
+                    result.getSeqNo(),
+                    result.getTerm(),
+                    result.getVersion(),
+                    op.source(),
+                    op.routing(),
+                    op.getAutoGeneratedIdTimestamp()
+                )
+            );
+
             if (frequently()) {
                 engine.flush();
             }
@@ -168,13 +179,12 @@ public void testSnapshotRecovery() throws IOException {
                 IndexVersion.current()
             )
         ) {
+            var asserter = TranslogOperationAsserter.withEngineConfig(engine.config());
             for (int i = 0; i < size; i++) {
                 var op = snapshot.next();
                 assertThat(op.opType(), equalTo(Translog.Operation.Type.INDEX));
                 Translog.Index indexOp = (Translog.Index) op;
-                assertThat(indexOp.id(), equalTo(expectedOperations.get(i).id()));
-                assertThat(indexOp.routing(), equalTo(expectedOperations.get(i).routing()));
-                assertToXContentEquivalent(indexOp.source(), expectedOperations.get(i).source(), XContentType.JSON);
+                asserter.assertSameIndexOperation(indexOp, expectedOperations.get(i));
             }
             assertNull(snapshot.next());
         }

x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java

@@ -174,7 +174,8 @@ public static ChunkedInferenceEmbeddingByte randomChunkedInferenceEmbeddingByte(
         for (String input : inputs) {
             byte[] values = new byte[model.getServiceSettings().dimensions()];
             for (int j = 0; j < values.length; j++) {
-                values[j] = randomByte();
+                // to avoid vectors with zero magnitude
+                values[j] = (byte) Math.max(1, randomByte());
             }
             chunks.add(
                 new ChunkedInferenceEmbeddingByte.ByteEmbeddingChunk(values, input, new ChunkedInference.TextOffset(0, input.length()))
@@ -188,7 +189,8 @@ public static ChunkedInferenceEmbeddingFloat randomChunkedInferenceEmbeddingFloa
         for (String input : inputs) {
             float[] values = new float[model.getServiceSettings().dimensions()];
             for (int j = 0; j < values.length; j++) {
-                values[j] = randomFloat();
+                // to avoid vectors with zero magnitude
+                values[j] = Math.max(1e-6f, randomFloat());
             }
             chunks.add(
                 new ChunkedInferenceEmbeddingFloat.FloatEmbeddingChunk(values, input, new ChunkedInference.TextOffset(0, input.length()))
@@ -206,7 +208,7 @@ public static ChunkedInferenceEmbeddingSparse randomChunkedInferenceEmbeddingSpa
         for (String input : inputs) {
             var tokens = new ArrayList<WeightedToken>();
             for (var token : input.split("\\s+")) {
-                tokens.add(new WeightedToken(token, withFloats ? randomFloat() : randomIntBetween(1, 255)));
+                tokens.add(new WeightedToken(token, withFloats ? Math.max(Float.MIN_NORMAL, randomFloat()) : randomIntBetween(1, 255)));
             }
             chunks.add(
                 new ChunkedInferenceEmbeddingSparse.SparseEmbeddingChunk(tokens, input, new ChunkedInference.TextOffset(0, input.length()))