Merge remote-tracking branch 'origin/main'

Author: Isso-W

.github/workflows/javaTests.yml

@@ -73,7 +73,7 @@ jobs:
           "**.functions.builtin.part1.**",
           "**.functions.builtin.part2.**",
           "**.functions.frame.**,**.functions.indexing.**,**.functions.io.**,**.functions.iogen.**",
-          "**.functions.dnn.**",
+          "**.functions.dnn.**,**.functions.ooc.**",
           "**.functions.paramserv.**",
           "**.functions.recompile.**,**.functions.misc.**",
           "**.functions.mlcontext.**",
@@ -149,7 +149,7 @@ jobs:
       run: mvn jacoco:report
 
     - name: Upload coverage to Codecov
-      uses: codecov/[email protected].2
+      uses: codecov/[email protected].3
       if: github.repository_owner == 'apache'
       with:
         fail_ci_if_error: false

.github/workflows/python.yml

@@ -118,7 +118,7 @@ jobs:
           black \
           opt-einsum \
           nltk
-        
+
     - name: Build Python Package
       run: |
         cd src/main/python

bin/systemds

@@ -413,6 +413,7 @@ if [ $WORKER == 1 ]; then
   print_out "#  starting Federated worker on port $PORT"
   CMD=" \
   java $SYSTEMDS_STANDALONE_OPTS \
+  --add-modules=jdk.incubator.vector \
   $LOG4JPROPFULL \
   -jar $SYSTEMDS_JAR_FILE \
   -w $PORT \
@@ -422,6 +423,7 @@ elif [ "$FEDMONITORING" == 1 ]; then
   print_out "#  starting Federated backend monitoring on port $PORT"
   CMD=" \
   java $SYSTEMDS_STANDALONE_OPTS \
+  --add-modules=jdk.incubator.vector \
   $LOG4JPROPFULL \
   -jar $SYSTEMDS_JAR_FILE \
   -fedMonitoring $PORT \
@@ -433,6 +435,7 @@ elif [ $SYSDS_DISTRIBUTED == 0 ]; then
   CMD=" \
   java $SYSTEMDS_STANDALONE_OPTS \
   $LOG4JPROPFULL \
+  --add-modules=jdk.incubator.vector \
   -jar $SYSTEMDS_JAR_FILE \
   -f $SCRIPT_FILE \
   -exec $SYSDS_EXEC_MODE \
@@ -442,6 +445,7 @@ else
   print_out "#  Running script $SCRIPT_FILE distributed with opts: $*"
   CMD=" \
   spark-submit $SYSTEMDS_DISTRIBUTED_OPTS \
+  --add-modules=jdk.incubator.vector \
   $SYSTEMDS_JAR_FILE \
   -f $SCRIPT_FILE \
   -exec $SYSDS_EXEC_MODE \

docker/build.sh

@@ -23,7 +23,7 @@
 # Build the docker containers
 
 # The first build is for running systemds through docker.
-docker image build -f docker/sysds.Dockerfile -t apache/systemds:latest .
+# docker image build -f docker/sysds.Dockerfile -t apache/systemds:latest .
 
 # The second build is for testing systemds. This image installs the R dependencies needed to run the tests.
 docker image build -f docker/testsysds.Dockerfile -t apache/systemds:testing-latest .

docker/entrypoint.sh

@@ -30,7 +30,9 @@ export MAVEN_OPTS="-Xmx512m"
 
 log="/tmp/sysdstest.log"
 mvn -ntp -B test-compile 2>&1 | grep -E "BUILD|Total time:|---|Building SystemDS"
-mvn -ntp -B test -D maven.test.skip=false -D automatedtestbase.outputbuffering=true -D test=$1 2>&1 | grep -v "already exists in destination." | tee $log
+mvn -ntp -B test -D maven.test.skip=false -D automatedtestbase.outputbuffering=true -D test=$1 2>&1 \
+	| grep -v "already exists in destination." \
+	| grep -v 'WARNING: Using incubator modules' | tee $log
 
 # Merge Federated test runs.
 [ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec

pom.xml

@@ -67,7 +67,7 @@
 		<!-- aws-java-sdk-bundle version should align with hadoop-aws version -->
 		<!-- aws-java-sdk-bundle.version>1.12.367</aws-java-sdk-bundle.version -->
 		<!-- Set java compile level via argument, ex: 1.8 1.9 10 11-->
-		<java.level>11</java.level>
+		<java.level>17</java.level>
 		<java.version>{java.level}</java.version>
 		<!-->Testing settings<!-->
 		<maven.test.skip>false</maven.test.skip>
@@ -92,6 +92,7 @@
 			--add-opens=java.base/java.lang.ref=ALL-UNNAMED
 			--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
 			--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+			--add-modules=jdk.incubator.vector
 		</jvm.addopens>
 	</properties>
 
@@ -357,6 +358,9 @@
 					<source>${java.level}</source>
 					<target>${java.level}</target>
 					<release>${java.level}</release>
+					<compilerArgs>
+						<arg>--add-modules=jdk.incubator.vector</arg>
+					</compilerArgs>
 				</configuration>
 			</plugin>
 
@@ -904,6 +908,7 @@
 							<notimestamp>true</notimestamp>
 							<failOnWarnings>false</failOnWarnings>
 							<quiet>true</quiet>
+							<additionalJOption>--add-modules=jdk.incubator.vector</additionalJOption>
 							<skip>${doc.skip}</skip>
 							<show>public</show>
 							<source>${java.level}</source>

scripts/builtin/raGroupby.dml

@@ -132,7 +132,22 @@ m_raGroupby = function (Matrix[Double] X, Integer col, String method)
     # Set value of final output
     Y = matrix(0, rows=numGroups, cols=totalCells)
     Y[,1] = key_unique
-    Y[,2:ncol(Y)] = matrix(Y_temp_reduce, rows=numGroups, cols=totalCells-1)
+
+    # The permutation matrix creates a structure where each group's data
+    # may not fill exactly maxRowsInGroup rows.
+    # If needed, we need to pad to the expected size first.
+    expectedRows = numGroups * maxRowsInGroup
+    actualRows = nrow(Y_temp_reduce)
+
+    if(actualRows < expectedRows) {
+      # Pad Y_temp_reduce with zeros to match expected structure
+      Y_tmp_padded = matrix(0, rows=expectedRows, cols=ncol(Y_temp_reduce))
+      Y_tmp_padded[1:actualRows,] = Y_temp_reduce
+    } else {
+      Y_tmp_padded = Y_temp_reduce
+    }
+
+    Y[,2:ncol(Y)] = matrix(Y_tmp_padded, rows=numGroups, cols=totalCells-1)
   }
 }