absolute rowId and derived source integration for query_then_fetch

Author: Aniket Modak

modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java

@@ -553,7 +553,7 @@ protected void canDeriveSourceInternal() {
      *       both doc values and stored field
      */
     @Override
-    protected DerivedFieldGenerator derivedFieldGenerator() {
+    public DerivedFieldGenerator derivedFieldGenerator() {
         return new DerivedFieldGenerator(
             mappedFieldType,
             new SortedNumericDocValuesFetcher(mappedFieldType, simpleName()),

modules/parquet-data-format/build.gradle

@@ -99,7 +99,7 @@ dependencies {
   implementation 'com.fasterxml.jackson.core:jackson-annotations:2.18.2'
 
   // FlatBuffers dependency required by Arrow
-  implementation 'com.google.flatbuffers:flatbuffers-java:2.0.0'
+  implementation "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}"
 
   // Netty dependencies required by Arrow memory management
   implementation 'io.netty:netty-buffer:4.1.118.Final'

plugins/dataformat-csv/jni/Cargo.toml

@@ -0,0 +1,53 @@
+[package]
+name = "opensearch-datafusion-csv-jni"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "opensearch_datafusion_csv_jni"
+crate-type = ["cdylib"]
+
+[dependencies]
+# DataFusion dependencies
+datafusion = "49.0.0"
+datafusion-substrait = "49.0.0"
+arrow = "54.0.0"
+arrow-array = "54.0.0"
+arrow-schema = "54.0.0"
+arrow-buffer = "54.0.0"
+
+# JNI dependencies
+jni = "0.21"
+
+# Async runtime
+tokio = { version = "1.0", features = ["full"] }
+futures = "0.3"
+futures-util = "0.3"
+
+# Serialization
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+
+# Error handling
+anyhow = "1.0"
+thiserror = "1.0"
+
+# Logging
+log = "0.4"
+
+# Parquet support
+parquet = "54.0.0"
+
+# Object store for file access
+object_store = "0.11"
+url = "2.0"
+
+# Substrait support
+substrait = "0.47"
+prost = "0.13"
+
+# Temporary directory support
+tempfile = "3.0"
+
+[build-dependencies]
+cbindgen = "0.27"

plugins/engine-datafusion/build.gradle

@@ -47,12 +47,12 @@ dependencies {
     // CheckerFramework annotations required by Arrow 17.0.0
     implementation "org.checkerframework:checker-qual:3.42.0"
     // FlatBuffers dependency required by Arrow 17.0.0
-    implementation "com.google.flatbuffers:flatbuffers-java:23.5.26"
+    implementation "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}"
 
     testImplementation "junit:junit:${versions.junit}"
     testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
     testImplementation "org.mockito:mockito-core:${versions.mockito}"
-
+    testImplementation project(":modules:parquet-data-format")
     // Add CSV plugin for testing
     // testImplementation project(':plugins:dataformat-csv')
 }