kevintang2022 · kevintang2022 · Sep 3, 2025 · Sep 2, 2025 · Sep 5, 2025 · Sep 4, 2025
diff --git a/.github/workflows/prestocpp-linux-build-and-unit-test.yml b/.github/workflows/prestocpp-linux-build-and-unit-test.yml
@@ -370,7 +370,7 @@ jobs:
           # Use different Maven options to install.
           MAVEN_OPTS: "-Xmx2G -XX:+ExitOnOutOfMemoryError"
         run: |
-          for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-execution' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
+          for i in $(seq 1 3); do ./mvnw clean install $MAVEN_FAST_INSTALL -pl 'presto-native-sidecar-plugin' -am && s=0 && break || s=$? && sleep 10; done; (exit $s)
 
       - name: Run presto-native sidecar tests
         if: |

diff --git a/CODEOWNERS b/CODEOWNERS
@@ -123,6 +123,7 @@ CODEOWNERS @prestodb/team-tsc
 #####################################################################
 # Presto on Spark module
 /presto-spark* @shrinidhijoshi @prestodb/committers
+/presto-native-execution/*/com/facebook/presto/spark/* @shrinidhijoshi @prestodb/committers
 
 #####################################################################
 # Presto connectors and plugins
@@ -166,4 +167,3 @@ CODEOWNERS @prestodb/team-tsc
 # Presto CI and builds
 /.github @czentgr @unidevel @prestodb/committers
 /docker @czentgr @unidevel @prestodb/committers
-
diff --git a/pom.xml b/pom.xml
@@ -90,8 +90,7 @@
         <dep.guice.version>6.0.0</dep.guice.version>
         <dep.arrow.version>17.0.0</dep.arrow.version>
         <dep.mariadb.version>3.5.4</dep.mariadb.version>
-
-        <dep.pos.classloader.module-name.suffix>2</dep.pos.classloader.module-name.suffix>
+        <dep.spark.version>2.0.2-6</dep.spark.version>
 
         <!--
           America/Bahia_Banderas has:
@@ -225,7 +224,7 @@
         <module>presto-router-example-plugin-scheduler</module>
         <module>presto-plan-checker-router-plugin</module>
         <module>presto-sql-invoked-functions-plugin</module>
-        <module>presto-spark-classloader-spark${dep.pos.classloader.module-name.suffix}</module>
+        <module>presto-native-sql-invoked-functions-plugin</module>
     </modules>
 
     <dependencyManagement>
@@ -944,12 +943,6 @@
                 <version>${project.version}</version>
             </dependency>
 
-            <dependency>
-                <groupId>com.facebook.presto</groupId>
-                <artifactId>presto-spark-classloader-spark${dep.pos.classloader.module-name.suffix}</artifactId>
-                <version>${project.version}</version>
-            </dependency>
-
             <dependency>
                 <groupId>com.facebook.presto</groupId>
                 <artifactId>presto-spark</artifactId>
@@ -987,6 +980,13 @@
                 <version>${project.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>com.facebook.presto.spark</groupId>
+                <artifactId>spark-core</artifactId>
+                <version>${dep.spark.version}</version>
+                <scope>provided</scope>
+            </dependency>
+
             <dependency>
                 <groupId>com.facebook.presto</groupId>
                 <artifactId>presto-base-arrow-flight</artifactId>
@@ -2538,13 +2538,6 @@
                 <version>2.0.1</version>
             </dependency>
 
-            <dependency>
-                <groupId>com.facebook.presto.spark</groupId>
-                <artifactId>spark-core</artifactId>
-                <version>2.0.2-6</version>
-                <scope>provided</scope>
-            </dependency>
-
             <dependency>
                 <groupId>com.clearspring.analytics</groupId>
                 <artifactId>stream</artifactId>
@@ -3162,6 +3155,78 @@
                 </plugins>
             </build>
         </profile>
+        <profile>
+            <id>spark2</id>
+
+            <activation>
+                <activeByDefault>true</activeByDefault>
+                <property>
+                    <name>!spark-version</name>
+                </property>
+            </activation>
+
+            <properties>
+                <dep.spark.version>2.0.2-6</dep.spark.version>
+            </properties>
+
+            <modules>
+                <module>presto-spark-classloader-spark2</module>
+            </modules>
+
+            <build>
+                <pluginManagement>
+                    <plugins>
+                        <plugin>
+                            <groupId>org.apache.maven.plugins</groupId>
+                            <artifactId>maven-enforcer-plugin</artifactId>
+                            <version>3.3.0</version>
+                            <dependencies>
+                                <dependency>
+                                    <groupId>org.codehaus.mojo</groupId>
+                                    <artifactId>extra-enforcer-rules</artifactId>
+                                    <version>1.6.2</version>
+                                </dependency>
+                            </dependencies>
+                            <configuration>
+                                <skip>true</skip>
+                                <rules>
+                                    <requireUpperBoundDeps>
+                                        <excludes combine.children="append">
+                                            <!-- TODO: fix this in Airlift resolver -->
+                                            <exclude>org.codehaus.plexus:plexus-utils</exclude>
+                                            <exclude>com.google.guava:guava</exclude>
+                                            <exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
+                                            <exclude>com.fasterxml.jackson.core:jackson-core</exclude>
+                                            <exclude>com.fasterxml.jackson.core:jackson-databind</exclude>
+                                        </excludes>
+                                    </requireUpperBoundDeps>
+                                </rules>
+                            </configuration>
+                        </plugin>
+
+                        <plugin>
+                            <groupId>org.basepom.maven</groupId>
+                            <artifactId>duplicate-finder-maven-plugin</artifactId>
+                            <configuration>
+                                <skip>true</skip>
+                                <ignoredClassPatterns combine.children="append">
+                                    <ignoredClassPattern>com.github.benmanes.caffeine.*</ignoredClassPattern>
+                                    <!-- Duplicate class is being brought in by commons-io & log4j-api -->
+                                    <ignoredClassPattern>META-INF.versions.9.module-info</ignoredClassPattern>
+                                    <!-- Duplicate class is being brought in by several netty dependencies-->
+                                    <ignoredClassPattern>META-INF.versions.11.module-info</ignoredClassPattern>
+                                    <!-- Ignore duplicate classes related to lucene-core and ranger-apache -->
+                                    <ignoredClassPattern>META-INF.versions.9.org.apache.lucene.*</ignoredClassPattern>
+                                </ignoredClassPatterns>
+
+                            </configuration>
+                        </plugin>
+
+                    </plugins>
+                </pluginManagement>
+            </build>
+
+        </profile>
         <profile>
             <id>spark3</id>
 
@@ -3173,19 +3238,12 @@
             </activation>
 
             <properties>
-                <dep.pos.classloader.module-name.suffix>3</dep.pos.classloader.module-name.suffix>
+                <dep.spark.version>3.4.1-1</dep.spark.version>
             </properties>
 
-            <dependencyManagement>
-                <dependencies>
-                    <dependency>
-                        <groupId>com.facebook.presto.spark</groupId>
-                        <artifactId>spark-core</artifactId>
-                        <version>3.4.1-1</version>
-                        <scope>provided</scope>
-                    </dependency>
-                </dependencies>
-            </dependencyManagement>
+            <modules>
+                <module>presto-spark-classloader-spark3</module>
+            </modules>
 
             <build>
                 <pluginManagement>

diff --git a/...se/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java b/...se/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java
@@ -67,6 +67,7 @@
 import com.facebook.presto.operator.aggregation.DoubleCovarianceAggregation;
 import com.facebook.presto.operator.aggregation.DoubleHistogramAggregation;
 import com.facebook.presto.operator.aggregation.DoubleRegressionAggregation;
+import com.facebook.presto.operator.aggregation.DoubleRegressionExtendedAggregation;
 import com.facebook.presto.operator.aggregation.DoubleSumAggregation;
 import com.facebook.presto.operator.aggregation.EntropyAggregation;
 import com.facebook.presto.operator.aggregation.GeometricMeanAggregations;
@@ -84,6 +85,7 @@
 import com.facebook.presto.operator.aggregation.RealGeometricMeanAggregations;
 import com.facebook.presto.operator.aggregation.RealHistogramAggregation;
 import com.facebook.presto.operator.aggregation.RealRegressionAggregation;
+import com.facebook.presto.operator.aggregation.RealRegressionExtendedAggregation;
 import com.facebook.presto.operator.aggregation.RealSumAggregation;
 import com.facebook.presto.operator.aggregation.ReduceAggregationFunction;
 import com.facebook.presto.operator.aggregation.SumDataSizeForStats;
@@ -744,7 +746,9 @@ private List<? extends SqlFunction> getBuiltInFunctions(FunctionsConfig function
                 .aggregates(DoubleCovarianceAggregation.class)
                 .aggregates(RealCovarianceAggregation.class)
                 .aggregates(DoubleRegressionAggregation.class)
+                .aggregates(DoubleRegressionExtendedAggregation.class)
                 .aggregates(RealRegressionAggregation.class)
+                .aggregates(RealRegressionExtendedAggregation.class)
                 .aggregates(DoubleCorrelationAggregation.class)
                 .aggregates(RealCorrelationAggregation.class)
                 .aggregates(BitwiseOrAggregation.class)

diff --git a/...to-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationUtils.java b/...to-main-base/src/main/java/com/facebook/presto/operator/aggregation/AggregationUtils.java
@@ -22,6 +22,7 @@
 import com.facebook.presto.operator.aggregation.state.CentralMomentsState;
 import com.facebook.presto.operator.aggregation.state.CorrelationState;
 import com.facebook.presto.operator.aggregation.state.CovarianceState;
+import com.facebook.presto.operator.aggregation.state.ExtendedRegressionState;
 import com.facebook.presto.operator.aggregation.state.RegressionState;
 import com.facebook.presto.operator.aggregation.state.VarianceState;
 import com.facebook.presto.spi.function.AggregationFunctionImplementation;
@@ -145,9 +146,14 @@ public static double getCorrelation(CorrelationState state)
     public static void updateRegressionState(RegressionState state, double x, double y)
     {
         double oldMeanX = state.getMeanX();
-        double oldMeanY = state.getMeanY();
         updateCovarianceState(state, x, y);
         state.setM2X(state.getM2X() + (x - oldMeanX) * (x - state.getMeanX()));
+    }
+
+    public static void updateExtendedRegressionState(ExtendedRegressionState state, double x, double y)
+    {
+        double oldMeanY = state.getMeanY();
+        updateRegressionState(state, x, y);
         state.setM2Y(state.getM2Y() + (y - oldMeanY) * (y - state.getMeanY()));
     }
 
@@ -189,12 +195,12 @@ public static double getRegressionSxy(RegressionState state)
         return state.getC2();
     }
 
-    public static double getRegressionSyy(RegressionState state)
+    public static double getRegressionSyy(ExtendedRegressionState state)
     {
         return state.getM2Y();
     }
 
-    public static double getRegressionR2(RegressionState state)
+    public static double getRegressionR2(ExtendedRegressionState state)
     {
         if (state.getM2X() != 0 && state.getM2Y() == 0) {
             return 1.0;
@@ -311,10 +317,21 @@ public static void mergeRegressionState(RegressionState state, RegressionState o
         long na = state.getCount();
         long nb = otherState.getCount();
         state.setM2X(state.getM2X() + otherState.getM2X() + na * nb * Math.pow(state.getMeanX() - otherState.getMeanX(), 2) / (double) (na + nb));
-        state.setM2Y(state.getM2Y() + otherState.getM2Y() + na * nb * Math.pow(state.getMeanY() - otherState.getMeanY(), 2) / (double) (na + nb));
         updateCovarianceState(state, otherState);
     }
 
+    public static void mergeExtendedRegressionState(ExtendedRegressionState state, ExtendedRegressionState otherState)
+    {
+        if (otherState.getCount() == 0) {
+            return;
+        }
+
+        long na = state.getCount();
+        long nb = otherState.getCount();
+        state.setM2Y(state.getM2Y() + otherState.getM2Y() + na * nb * Math.pow(state.getMeanY() - otherState.getMeanY(), 2) / (double) (na + nb));
+        mergeRegressionState(state, otherState);
+    }
+
     public static String generateAggregationName(String baseName, TypeSignature outputType, List<TypeSignature> inputTypes)
     {
         StringBuilder sb = new StringBuilder();

diff --git a/...e/src/main/java/com/facebook/presto/operator/aggregation/DoubleRegressionAggregation.java b/...e/src/main/java/com/facebook/presto/operator/aggregation/DoubleRegressionAggregation.java
@@ -24,15 +24,8 @@
 import com.facebook.presto.spi.function.SqlType;
 
 import static com.facebook.presto.common.type.DoubleType.DOUBLE;
-import static com.facebook.presto.operator.aggregation.AggregationUtils.getRegressionAvgx;
-import static com.facebook.presto.operator.aggregation.AggregationUtils.getRegressionAvgy;
-import static com.facebook.presto.operator.aggregation.AggregationUtils.getRegressionCount;
 import static com.facebook.presto.operator.aggregation.AggregationUtils.getRegressionIntercept;
-import static com.facebook.presto.operator.aggregation.AggregationUtils.getRegressionR2;
 import static com.facebook.presto.operator.aggregation.AggregationUtils.getRegressionSlope;
-import static com.facebook.presto.operator.aggregation.AggregationUtils.getRegressionSxx;
-import static com.facebook.presto.operator.aggregation.AggregationUtils.getRegressionSxy;
-import static com.facebook.presto.operator.aggregation.AggregationUtils.getRegressionSyy;
 import static com.facebook.presto.operator.aggregation.AggregationUtils.mergeRegressionState;
 import static com.facebook.presto.operator.aggregation.AggregationUtils.updateRegressionState;
 
@@ -78,100 +71,4 @@ public static void regrIntercept(@AggregationState RegressionState state, BlockB
             out.appendNull();
         }
     }
-
-    @AggregationFunction("regr_sxy")
-    @OutputFunction(StandardTypes.DOUBLE)
-    public static void regrSxy(@AggregationState RegressionState state, BlockBuilder out)
-    {
-        double result = getRegressionSxy(state);
-        double count = getRegressionCount(state);
-        if (Double.isFinite(result) && Double.isFinite(count) && count > 0) {
-            DOUBLE.writeDouble(out, result);
-        }
-        else {
-            out.appendNull();
-        }
-    }
-
-    @AggregationFunction("regr_sxx")
-    @OutputFunction(StandardTypes.DOUBLE)
-    public static void regrSxx(@AggregationState RegressionState state, BlockBuilder out)
-    {
-        double result = getRegressionSxx(state);
-        double count = getRegressionCount(state);
-        if (Double.isFinite(result) && Double.isFinite(count) && count > 0) {
-            DOUBLE.writeDouble(out, result);
-        }
-        else {
-            out.appendNull();
-        }
-    }
-
-    @AggregationFunction("regr_syy")
-    @OutputFunction(StandardTypes.DOUBLE)
-    public static void regrSyy(@AggregationState RegressionState state, BlockBuilder out)
-    {
-        double result = getRegressionSyy(state);
-        double count = getRegressionCount(state);
-        if (Double.isFinite(result) && Double.isFinite(count) && count > 0) {
-            DOUBLE.writeDouble(out, result);
-        }
-        else {
-            out.appendNull();
-        }
-    }
-
-    @AggregationFunction("regr_r2")
-    @OutputFunction(StandardTypes.DOUBLE)
-    public static void regrR2(@AggregationState RegressionState state, BlockBuilder out)
-    {
-        double result = getRegressionR2(state);
-        if (Double.isFinite(result)) {
-            DOUBLE.writeDouble(out, result);
-        }
-        else {
-            out.appendNull();
-        }
-    }
-
-    @AggregationFunction("regr_count")
-    @OutputFunction(StandardTypes.DOUBLE)
-    public static void regrCount(@AggregationState RegressionState state, BlockBuilder out)
-    {
-        double result = getRegressionCount(state);
-        if (Double.isFinite(result) && result > 0) {
-            DOUBLE.writeDouble(out, result);
-        }
-        else {
-            out.appendNull();
-        }
-    }
-
-    @AggregationFunction("regr_avgy")
-    @OutputFunction(StandardTypes.DOUBLE)
-    public static void regrAvgy(@AggregationState RegressionState state, BlockBuilder out)
-    {
-        double result = getRegressionAvgy(state);
-        double count = getRegressionCount(state);
-        if (Double.isFinite(result) && Double.isFinite(count) && count > 0) {
-            DOUBLE.writeDouble(out, result);
-        }
-        else {
-            out.appendNull();
-        }
-    }
-
-    @AggregationFunction("regr_avgx")
-    @OutputFunction(StandardTypes.DOUBLE)
-    public static void regrAvgx(@AggregationState RegressionState state, BlockBuilder out)
-    {
-        double result = getRegressionAvgx(state);
-        double count = getRegressionCount(state);
-        if (Double.isFinite(result) && Double.isFinite(count) && count > 0) {
-            DOUBLE.writeDouble(out, result);
-        }
-        else {
-            out.appendNull();
-        }
-    }
 }