Feature/datafusion (#38)

* Abstracting lucene away: part 1

* initial abstractions to reduce indexing engine coupling

* Text backed engine testing

---------

Co-authored-by: Mohit Godwani <[email protected]>

Author: bharath-techie

server/src/main/java/org/opensearch/index/engine/DataFormatPlugin.java

@@ -0,0 +1,21 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.engine;
+
+import org.opensearch.index.engine.exec.DataFormat;
+import org.opensearch.index.engine.exec.IndexingExecutionEngine;
+
+import javax.xml.crypto.Data;
+
+public interface DataFormatPlugin  {
+
+    <T extends DataFormat> IndexingExecutionEngine<T> indexingEngine();
+
+    DataFormat getDataFormat();
+}

server/src/main/java/org/opensearch/index/engine/Engine.java

@@ -79,6 +79,8 @@
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.VersionType;
+import org.opensearch.index.engine.exec.DocumentInput;
+import org.opensearch.index.engine.exec.composite.CompositeDataFormatWriter;
 import org.opensearch.index.mapper.IdFieldMapper;
 import org.opensearch.index.mapper.Mapping;
 import org.opensearch.index.mapper.ParseContext.Document;
@@ -1610,6 +1612,7 @@ public static class Index extends Operation {
         private final boolean isRetry;
         private final long ifSeqNo;
         private final long ifPrimaryTerm;
+        public CompositeDataFormatWriter.CompositeDocumentInput documentInput;
 
         public Index(
             Term uid,

server/src/main/java/org/opensearch/index/engine/exec/FileMetadata.java

@@ -0,0 +1,11 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.engine.exec;
+
+public record FileMetadata(DataFormat df, String fileName) { }

server/src/main/java/org/opensearch/index/engine/exec/FlushIn.java

@@ -0,0 +1,13 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.engine.exec;
+
+public interface FlushIn {
+
+}

server/src/main/java/org/opensearch/index/engine/exec/IndexingExecutionEngine.java

@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.engine.exec;
+
+import org.opensearch.index.mapper.MappedFieldType;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+public interface IndexingExecutionEngine<T extends DataFormat> {
+    List<String> supportedFieldTypes();
+
+    Writer<? extends DocumentInput<?>> createWriter() throws IOException; // A writer responsible for data format vended by this engine.
+
+    RefreshResult refresh(RefreshInput refreshInput) throws IOException;
+
+    DataFormat getDataFormat();
+}

server/src/main/java/org/opensearch/index/engine/exec/RefreshInput.java

@@ -0,0 +1,29 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.engine.exec;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class RefreshInput {
+
+    private List<FileMetadata> files;
+
+    public RefreshInput() {
+        this.files = new ArrayList<>();
+    }
+
+    public void add(FileMetadata fileMetadata) {
+        this.files.add(fileMetadata);
+    }
+
+    public List<FileMetadata> getFiles() {
+        return files;
+    }
+}

server/src/main/java/org/opensearch/index/engine/exec/RefreshResult.java

@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.engine.exec;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class RefreshResult {
+    private Map<DataFormat, List<FileMetadata>> refreshedFiles = new HashMap<>();
+
+    public RefreshResult() {
+
+    }
+
+    public void add(DataFormat df, List<FileMetadata> fileMetadata) {
+        refreshedFiles.computeIfAbsent(df, ddf -> new ArrayList<>()).addAll(fileMetadata);
+    }
+
+    public Map<DataFormat, List<FileMetadata>> getRefreshedFiles() {
+        return refreshedFiles;
+    }
+}

server/src/main/java/org/opensearch/index/engine/exec/WriteResult.java

@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.engine.exec;
+
+public record WriteResult(boolean success, Exception e, long version, long term, long seqNo) {
+}

server/src/main/java/org/opensearch/index/engine/exec/Writer.java

@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.engine.exec;
+
+import java.io.IOException;
+import java.util.Optional;
+
+public interface Writer<P extends DocumentInput<?>> {
+    WriteResult addDoc(P d) throws IOException;
+
+    FileMetadata flush(FlushIn flushIn) throws IOException;
+
+    void sync() throws IOException;
+
+    void close();
+
+    Optional<FileMetadata> getMetadata();
+
+    P newDocumentInput();
+}

server/src/main/java/org/opensearch/index/engine/exec/composite/CompositeIndexingExecutionEngine.java

@@ -8,6 +8,7 @@
 
 package org.opensearch.index.engine.exec.composite;
 
+import org.opensearch.index.engine.DataFormatPlugin;
 import org.opensearch.index.engine.exec.DataFormat;
 import org.opensearch.index.engine.exec.FileMetadata;
 import org.opensearch.index.engine.exec.IndexingExecutionEngine;