Update CI workflow

Author: irfanghat

.github/workflows/build_and_test.yml

@@ -25,15 +25,28 @@ jobs:
 
       - name: Install Spark
         run: |
-          curl -fL https://dlcdn.apache.org/spark/spark-3.5.8/spark-3.5.8-bin-hadoop3.tgz -o spark.tgz
+          curl -fL https://dlcdn.apache.org/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz -o spark.tgz
           tar -xzf spark.tgz
-          mv spark-3.5.8-bin-hadoop3 $HOME/spark
+          mv spark-3.5.3-bin-hadoop3 $HOME/spark
 
       - name: Start Spark Connect Server
         run: |
           $HOME/spark/sbin/start-connect-server.sh \
-            --packages org.apache.spark:spark-connect_2.12:3.5.1
-          sleep 10
+            --packages "org.apache.spark:spark-connect_2.12:3.5.3,io.delta:delta-spark_2.12:3.2.0,io.graphframes:graphframes-spark3_2.12:0.10.0,io.graphframes:graphframes-connect-spark3_2.12:0.10.0" \
+            --conf "spark.connect.extensions.relation.classes=org.apache.spark.sql.graphframes.GraphFramesConnect" \
+            --conf "spark.driver.extraJavaOptions=-Divy.cache.dir=/tmp -Divy.home=/tmp -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError" \
+            --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \
+            --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" \
+            --conf "spark.driver.memory=4g" \
+            --conf "spark.executor.memory=4g" \
+            --conf "spark.memory.fraction=0.8" \
+            --conf "spark.memory.storageFraction=0.3" \
+            --conf "spark.sql.shuffle.partitions=8" \
+            --conf "spark.default.parallelism=8" \
+            --conf "spark.driver.maxResultSize=2g"
+
+          # wait for server
+          sleep 15
 
       - name: Install gcovr
         run: |
@@ -51,7 +64,7 @@ jobs:
 
       - name: Run Tests
         run: |
-          export SPARK_REMOTE=sc://localhost
+          export SPARK_REMOTE=sc://localhost:15002
           cd build
           ctest -LE dbrx --output-on-failure