Fixing indexing regression and bug fixes for grouping criteria (#20145)

Signed-off-by: RS146BIJAY <rishavsagar4b1@gmail.com>

Author: RS146BIJAY

CHANGELOG.md

@@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix SearchPhaseExecutionException to properly initCause ([#20320](https://github.com/opensearch-project/OpenSearch/pull/20320))
 - Fix `cluster.remote.<cluster_alias>.server_name` setting no populating SNI ([#20321](https://github.com/opensearch-project/OpenSearch/pull/20321))
 - Fix X-Opaque-Id header propagation (along with other response headers) for streaming Reactor Netty 4 transport ([#20371](https://github.com/opensearch-project/OpenSearch/pull/20371))
+- Fix indexing regression and bug fixes for grouping criteria. ([20145](https://github.com/opensearch-project/OpenSearch/pull/20145))
 
 ### Dependencies
 - Bump `com.google.auth:google-auth-library-oauth2-http` from 1.38.0 to 1.41.0 ([#20183](https://github.com/opensearch-project/OpenSearch/pull/20183))

server/src/main/java/org/opensearch/OpenSearchServerException.java

@@ -24,7 +24,6 @@
 import static org.opensearch.Version.V_2_7_0;
 import static org.opensearch.Version.V_3_0_0;
 import static org.opensearch.Version.V_3_2_0;
-import static org.opensearch.Version.V_3_3_0;
 
 /**
  * Utility class to register server exceptions
@@ -1242,13 +1241,5 @@ public static void registerExceptions() {
                 V_3_2_0
             )
         );
-        registerExceptionHandle(
-            new OpenSearchExceptionHandle(
-                org.opensearch.index.engine.LookupMapLockAcquisitionException.class,
-                org.opensearch.index.engine.LookupMapLockAcquisitionException::new,
-                CUSTOM_ELASTICSEARCH_EXCEPTIONS_BASE_ID + 2,
-                V_3_3_0
-            )
-        );
     }
 }

server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java

@@ -87,7 +87,6 @@
 import org.opensearch.index.IndexingPressureService;
 import org.opensearch.index.SegmentReplicationPressureService;
 import org.opensearch.index.engine.Engine;
-import org.opensearch.index.engine.LookupMapLockAcquisitionException;
 import org.opensearch.index.engine.VersionConflictEngineException;
 import org.opensearch.index.get.GetResult;
 import org.opensearch.index.mapper.MapperException;
@@ -728,15 +727,7 @@ && isConflictException(executionResult.getFailure().getCause())
             && context.getRetryCounter() < ((UpdateRequest) docWriteRequest).retryOnConflict()) {
             context.resetForExecutionForRetry();
             return;
-        } else if (isFailed
-            && context.getPrimary() != null
-            && context.getPrimary().indexSettings() != null
-            && context.getPrimary().indexSettings().isContextAwareEnabled()
-            && isLookupMapLockAcquisitionException(executionResult.getFailure().getCause())
-            && context.getRetryCounter() < context.getPrimary().indexSettings().getMaxRetryOnLookupMapAcquisitionException()) {
-                context.resetForExecutionForRetry();
-                return;
-            }
+        }
         final BulkItemResponse response;
         if (isUpdate) {
             response = processUpdateResponse((UpdateRequest) docWriteRequest, context.getConcreteIndex(), executionResult, updateResult);
@@ -765,10 +756,6 @@ private static boolean isConflictException(final Exception e) {
         return ExceptionsHelper.unwrapCause(e) instanceof VersionConflictEngineException;
     }
 
-    private static boolean isLookupMapLockAcquisitionException(final Exception e) {
-        return ExceptionsHelper.unwrapCause(e) instanceof LookupMapLockAcquisitionException;
-    }
-
     /**
      * Creates a new bulk item result from the given requests and result of performing the update operation on the shard.
      */

server/src/main/java/org/opensearch/index/engine/CompositeIndexWriter.java

@@ -18,6 +18,7 @@
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -52,13 +53,13 @@ public interface DocumentIndexWriter extends Closeable, ReferenceManager.Refresh
 
     void deleteUnusedFiles() throws IOException;
 
-    long addDocuments(Iterable<ParseContext.Document> docs, Term uid) throws IOException;
+    long addDocuments(List<ParseContext.Document> docs, Term uid) throws IOException;
 
     long addDocument(ParseContext.Document doc, Term uid) throws IOException;
 
     void softUpdateDocuments(
         Term uid,
-        Iterable<ParseContext.Document> docs,
+        List<ParseContext.Document> docs,
         long version,
         long seqNo,
         long primaryTerm,

server/src/main/java/org/opensearch/index/engine/DocumentIndexWriter.java

@@ -16,6 +16,7 @@
 import org.opensearch.index.mapper.ParseContext;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -131,7 +132,7 @@ public void deleteUnusedFiles() throws IOException {
     }
 
     @Override
-    public long addDocuments(Iterable<ParseContext.Document> docs, Term uid) throws IOException {
+    public long addDocuments(final List<ParseContext.Document> docs, Term uid) throws IOException {
         return indexWriter.addDocuments(docs);
     }
 
@@ -143,7 +144,7 @@ public long addDocument(ParseContext.Document doc, Term uid) throws IOException
     @Override
     public void softUpdateDocuments(
         Term uid,
-        Iterable<ParseContext.Document> docs,
+        List<ParseContext.Document> docs,
         long version,
         long seqNo,
         long primaryTerm,

server/src/main/java/org/opensearch/index/engine/LookupMapLockAcquisitionException.java

@@ -195,6 +195,10 @@ public IndexWriterConfig buildIndexWriterConfig() {
             iwc.setMergePolicy(new OpenSearchMergePolicy(mergePolicy));
             iwc.setSimilarity(engineConfig.getSimilarity());
             iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac());
+            // We are setting the codec here rather than in the CodecService because each CriteriaBasedCodec requires
+            // associatedCriteria to be attached upon creation during IndexWriter initialisation. This criteria is
+            // determined on a per-document basis and is only available within the InternalEngine. Therefore, the codec
+            // for the child writer is created here where the necessary criteria information is accessible
             if (engineConfig.getIndexSettings().isContextAwareEnabled()) {
                 iwc.setCodec(new CriteriaBasedCodec(engineConfig.getCodec(), associatedCriteria));
             } else {

server/src/main/java/org/opensearch/index/engine/LuceneIndexWriter.java

@@ -8,6 +8,8 @@
 
 package org.opensearch.index.mapper;
 
+import org.apache.lucene.index.LeafReader;
+import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.script.ContextAwareGroupingScript;
 import org.opensearch.script.Script;
 
@@ -181,4 +183,18 @@ public ContextAwareGroupingFieldType fieldType() {
     protected String contentType() {
         return CONTENT_TYPE;
     }
+
+    /**
+     * Context Aware Segment field is not a part of an ingested document, so omitting it from Context Aware Segment
+     * validation.
+     */
+    @Override
+    public void canDeriveSource() {}
+
+    /**
+     * Context Aware Segment field is not a part of an ingested document, so omitting it from Context Aware Segment
+     * generation.
+     */
+    @Override
+    public void deriveSource(XContentBuilder builder, LeafReader leafReader, int docId) throws IOException {}
 }

server/src/main/java/org/opensearch/index/engine/NativeLuceneIndexWriterFactory.java

@@ -84,6 +84,7 @@
 import java.util.function.BooleanSupplier;
 import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
@@ -690,7 +691,9 @@ public boolean isCompositeIndexPresent() {
     }
 
     public Set<CompositeMappedFieldType> getCompositeFieldTypes() {
-        return compositeMappedFieldTypes;
+        return compositeMappedFieldTypes.stream()
+            .filter(compositeMappedFieldType -> compositeMappedFieldType instanceof CompositeDataCubeFieldType)
+            .collect(Collectors.toSet());
     }
 
     private Set<CompositeMappedFieldType> getCompositeFieldTypesFromMapper() {