[SPARK-CONNECT][CPP] Implement GraphFrames Support and Core API Enhancements #68
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build, Test & Coverage | |
| permissions: | |
| contents: read | |
| on: | |
| push: | |
| branches: ["main"] | |
| pull_request: | |
| branches: ["main"] | |
| workflow_dispatch: | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 17 | |
| - name: Install Spark | |
| run: | | |
| curl -fL https://dlcdn.apache.org/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz -o spark.tgz | |
| tar -xzf spark.tgz | |
| mv spark-3.5.3-bin-hadoop3 $HOME/spark | |
| - name: Start Spark Connect Server | |
| run: | | |
| $HOME/spark/sbin/start-connect-server.sh \ | |
| --packages "org.apache.spark:spark-connect_2.12:3.5.3,io.delta:delta-spark_2.12:3.2.0,io.graphframes:graphframes-spark3_2.12:0.10.0,io.graphframes:graphframes-connect-spark3_2.12:0.10.0" \ | |
| --conf "spark.connect.extensions.relation.classes=org.apache.spark.sql.graphframes.GraphFramesConnect" \ | |
| --conf "spark.driver.extraJavaOptions=-Divy.cache.dir=/tmp -Divy.home=/tmp -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError" \ | |
| --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ | |
| --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" \ | |
| --conf "spark.driver.memory=4g" \ | |
| --conf "spark.executor.memory=4g" \ | |
| --conf "spark.memory.fraction=0.8" \ | |
| --conf "spark.memory.storageFraction=0.3" \ | |
| --conf "spark.sql.shuffle.partitions=8" \ | |
| --conf "spark.default.parallelism=8" \ | |
| --conf "spark.driver.maxResultSize=2g" | |
| # wait for server | |
| sleep 15 | |
| - name: Install gcovr | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y gcovr | |
| - name: Install dependencies | |
| run: chmod +x ./install_deps.sh && ./install_deps.sh | |
| - name: Configure with Coverage | |
| run: cmake -S . -B build -DENABLE_COVERAGE=ON | |
| - name: Build | |
| run: cmake --build build --parallel | |
| - name: Run Tests | |
| run: | | |
| export SPARK_REMOTE=sc://localhost:15002 | |
| cd build | |
| ctest -LE dbrx --output-on-failure | |
| - name: Generate Coverage (XML & HTML) | |
| run: | | |
| gcovr -r src \ | |
| --object-directory build \ | |
| --exclude '.*\.pb\.cc' \ | |
| --exclude '.*\.grpc\.pb\.cc' \ | |
| --exclude '.*\.h' \ | |
| --xml-pretty -o coverage.xml \ | |
| --html-details coverage.html \ | |
| --fail-under-line 70 \ | |
| --print-summary | |
| - name: Upload Coverage XML | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-xml | |
| path: coverage.xml | |
| - name: Upload Coverage HTML | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-html | |
| path: coverage.html |