move front-end test plugin to a separate plugin package.

add sandbox/qa with a rest test that will test full wiring & execution

Signed-off-by: Marc Handalian <marc.handalian@gmail.com>

Author: mch2

.gitignore

@@ -76,15 +76,3 @@ testfixtures_shared/
 
 # build files generated
 doc-tools/missing-doclet/bin/
-/plugins/dataformat-csv/jni/target
-/plugins/dataformat-csv/jni/Cargo.lock
-
-/modules/parquet-data-format/src/main/rust/target
-/modules/parquet-data-format/src/main/rust/debug
-/modules/parquet-data-format/src/main/resources/native/
-/modules/parquet-data-format/jni/target/debug
-/modules/parquet-data-format/jni/target/.rustc_info.json
-
-/modules/parquet-data-format/jni/target/release
-**/Cargo.lock
-/modules/parquet-data-format/jni/

gradle/run.gradle

@@ -88,7 +88,9 @@ testClusters {
       for (String p : installedPlugins) {
         // check if its a local plugin first
         if (project.findProject(':plugins:' + p) != null) {
-          plugin('plugins:' + p)
+          plugin(':plugins:' + p)
+        } else if (project.findProject(':sandbox:plugins:' + p) != null) {
+          plugin(':sandbox:plugins:' + p)
         } else {
           // attempt to fetch it from maven
           project.repositories.mavenLocal()

modules/parquet-data-format/build.gradle

@@ -114,7 +114,7 @@ dependencies {
   implementation 'org.apache.arrow:arrow-c-data:18.3.0'
 
   // Checker Framework annotations (required by Arrow)
-  implementation 'org.checkerframework:checker-qual:3.42.0'
+  implementation 'org.checkerframework:checker-qual:3.43.0'
 
   // Jackson dependencies required by Arrow
   implementation 'com.fasterxml.jackson.core:jackson-core:2.18.2'

modules/parquet-data-format/src/main/java/com/parquet/parquetdataformat/engine/read/ParquetDataSourceCodec.java

@@ -32,6 +32,6 @@ public class ParquetDataSourceCodec implements DataSourceCodec {
     }
 
     public DataFormat getDataFormat() {
-        return DataFormat.CSV;
+        return DataFormat.PARQUET;
     }
 }

plugins/engine-datafusion/build.gradle

@@ -23,13 +23,15 @@ opensearchplugin {
     name = pluginName
     description = pluginDescription
     classname = "${projectPath}.${pathToPlugin}"
+    extendedPlugins = ['analytics-engine']
     licenseFile = rootProject.file('LICENSE.txt')
     noticeFile = rootProject.file('NOTICE.txt')
 }
 
 
 dependencies {
     api project(':libs:opensearch-vectorized-exec-spi')
+    api project(':sandbox:libs:analytics-framework')
     implementation "org.apache.logging.log4j:log4j-api:${versions.log4j}"
     implementation "org.apache.logging.log4j:log4j-core:${versions.log4j}"
 
@@ -47,10 +49,32 @@ dependencies {
     // SLF4J API for Arrow logging compatibility
     implementation "org.slf4j:slf4j-api:${versions.slf4j}"
     // CheckerFramework annotations required by Arrow 17.0.0
-    implementation "org.checkerframework:checker-qual:3.42.0"
+    implementation "org.checkerframework:checker-qual:3.43.0"
     // FlatBuffers dependency required by Arrow 17.0.0
     implementation "com.google.flatbuffers:flatbuffers-java:${versions.flatbuffers}"
 
+    // Substrait: RelNode → Substrait plan conversion via Calcite Isthmus
+    implementation('io.substrait:core:0.67.0') {
+        exclude group: 'ch.qos.logback', module: 'logback-classic'
+        exclude group: 'ch.qos.logback', module: 'logback-core'
+        exclude group: 'org.apache.calcite', module: 'calcite-core'
+        exclude group: 'org.apache.calcite', module: 'calcite-linq4j'
+        exclude group: 'com.google.code.gson', module: 'gson'
+        exclude group: 'org.antlr', module: 'antlr4-runtime'
+        exclude group: 'org.antlr', module: 'antlr4'
+    }
+    implementation('io.substrait:isthmus:0.67.0') {
+        exclude group: 'ch.qos.logback', module: 'logback-classic'
+        exclude group: 'ch.qos.logback', module: 'logback-core'
+        exclude group: 'org.apache.calcite', module: 'calcite-core'
+        exclude group: 'org.apache.calcite', module: 'calcite-linq4j'
+        exclude group: 'com.google.code.gson', module: 'gson'
+        exclude group: 'org.antlr', module: 'antlr4-runtime'
+        exclude group: 'org.antlr', module: 'antlr4'
+    }
+    implementation "com.google.protobuf:protobuf-java:${versions.protobuf}"
+    implementation "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}"
+
     testImplementation "junit:junit:${versions.junit}"
     testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
     testImplementation "org.mockito:mockito-core:${versions.mockito}"
@@ -131,6 +155,47 @@ task copyNativeLibrary(type: Copy, dependsOn: buildRustLibrary) {
     }
 }
 
+// Exclude jars provided by analytics-engine plugin (shared via extendedPlugins classloader)
+// Exclude jars already in the analytics-engine parent plugin ZIP (via analytics-framework)
+bundlePlugin {
+    exclude 'analytics-framework-*.jar'
+    exclude 'calcite-core-*.jar'
+    exclude 'calcite-linq4j-*.jar'
+    exclude 'avatica-core-*.jar'
+    exclude 'avatica-metrics-*.jar'
+    exclude 'guava-*.jar'
+    exclude 'failureaccess-*.jar'
+    exclude 'slf4j-api-*.jar'
+    exclude 'checker-qual-*.jar'
+    exclude 'jackson-annotations-*.jar'
+    exclude 'jackson-databind-*.jar'
+    exclude 'jackson-core-*.jar'
+    exclude 'commons-codec-*.jar'
+    exclude 'janino-*.jar'
+    exclude 'commons-compiler-*.jar'
+    exclude 'joou-java-6-*.jar'
+    exclude 'commons-lang3-*.jar'
+    exclude 'commons-text-*.jar'
+    exclude 'commons-math3-*.jar'
+    exclude 'value-annotations-*.jar'
+    exclude 'json-path-*.jar'
+    exclude 'json-smart-*.jar'
+    exclude 'accessors-smart-*.jar'
+    exclude 'asm-*.jar'
+    exclude 'jts-core-*.jar'
+    exclude 'jts-io-common-*.jar'
+    exclude 'proj4j-*.jar'
+    exclude 'uzaygezen-core-*.jar'
+    exclude 'sketches-core-*.jar'
+    exclude 'memory-*.jar'
+    exclude 'httpcore5-*.jar'
+    exclude 'httpcore5-h2-*.jar'
+    exclude 'httpclient5-*.jar'
+    exclude 'jts-core-*.jar'
+    exclude 'jackson-core-*.jar'
+    exclude 'error_prone_annotations-*.jar'
+}
+
 // Ensure native library is built before Java compilation
 compileJava.dependsOn copyNativeLibrary
 
@@ -194,13 +259,37 @@ tasks.named("dependencyLicenses").configure {
     mapping from: /slf4j-.*/, to: 'slf4j-api'
     mapping from: /checker-qual.*/, to: 'checker-qual'
     mapping from: /flatbuffers-.*/, to: 'flatbuffers-java'
+    mapping from: /substrait-.*/, to: 'substrait'
+    mapping from: /protobuf-.*/, to: 'protobuf-java'
 }
 
 // Configure third party audit to handle Apache Arrow dependencies
 tasks.named('thirdPartyAudit').configure {
     ignoreMissingClasses(
         // Apache Commons Codec (missing dependency)
-        'org.apache.commons.codec.binary.Hex'
+        'org.apache.commons.codec.binary.Hex',
+        // Immutables annotations (used by Substrait generated code, annotation-only)
+        'org.immutables.value.Value',
+        'org.immutables.value.Value$Check',
+        'org.immutables.value.Value$Default',
+        'org.immutables.value.Value$Derived',
+        'org.immutables.value.Value$Enclosing',
+        'org.immutables.value.Value$Immutable',
+        'org.immutables.value.Value$Lazy',
+        'org.immutables.value.Value$Parameter',
+        'org.immutables.value.Value$Style',
+        // Error Prone annotations (used by Substrait/Protobuf, annotation-only)
+        'com.google.errorprone.annotations.CanIgnoreReturnValue',
+        'com.google.errorprone.annotations.CheckReturnValue',
+        'com.google.errorprone.annotations.Immutable',
+        'com.google.errorprone.annotations.InlineMe',
+        'com.google.errorprone.annotations.InlineMeValidationDisabled',
+        'com.google.errorprone.annotations.concurrent.LazyInit',
+        // JSR-305 annotations (used by Substrait, annotation-only)
+        'javax.annotation.Nonnull',
+        'javax.annotation.Nullable',
+        'javax.annotation.concurrent.Immutable',
+        'javax.annotation.concurrent.NotThreadSafe'
     )
     ignoreViolations(
         // Apache Arrow internal classes that use Unsafe operations

plugins/engine-datafusion/src/main/java/org/opensearch/datafusion/DataFusionAnalyticsBackend.java

@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.datafusion;
+
+import org.apache.calcite.sql.SqlOperatorTable;
+import org.opensearch.analytics.backend.EngineBridge;
+import org.opensearch.analytics.spi.AnalyticsBackEndPlugin;
+import org.opensearch.index.engine.exec.coord.CatalogSnapshot;
+
+/**
+ * SPI adapter that delegates to the real {@link DataFusionPlugin} instance.
+ *
+ * <p>{@code PluginsService} extension discovery requires either a no-arg constructor
+ * or a constructor taking the enclosing plugin. Since {@code DataFusionPlugin}
+ * itself needs a {@code Settings} constructor for plugin loading, this thin
+ * adapter bridges the two requirements.
+ */
+public class DataFusionAnalyticsBackend implements AnalyticsBackEndPlugin {
+
+    private final DataFusionPlugin plugin;
+
+    public DataFusionAnalyticsBackend(DataFusionPlugin plugin) {
+        this.plugin = plugin;
+    }
+
+    @Override
+    public String name() {
+        return plugin.name();
+    }
+
+    @Override
+    public EngineBridge<?, ?, ?> bridge(CatalogSnapshot snapshot) {
+        return plugin.bridge(snapshot);
+    }
+
+    @Override
+    public SqlOperatorTable operatorTable() {
+        return plugin.operatorTable();
+    }
+}