[SPARK-CONNECT][CPP] VCPKG toolchain #80
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build, Test & Coverage | |
| permissions: | |
| contents: read | |
| on: | |
| push: | |
| branches: ["main"] | |
| pull_request: | |
| branches: ["main"] | |
| workflow_dispatch: | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| env: | |
| # Set the vcpkg root directory | |
| VCPKG_ROOT: ${{ github.workspace }}/vcpkg | |
| # Set the path to the vcpkg binary cache | |
| VCPKG_BINARY_CACHE: ${{ github.workspace }}/vcpkg-cache | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # ---------------- vcpkg Setup and Caching ---------------- | |
| - name: Bootstrap vcpkg | |
| run: | | |
| git clone https://github.com/microsoft/vcpkg.git ${{ env.VCPKG_ROOT }} | |
| ${{ env.VCPKG_ROOT }}/bootstrap-vcpkg.sh | |
| shell: bash | |
| - name: Cache vcpkg binaries | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.VCPKG_BINARY_CACHE }} | |
| key: ${{ runner.os }}-vcpkg-${{ hashFiles('**/vcpkg.json') }} | |
| restore-keys: | | |
| ${{ runner.os }}-vcpkg- | |
| # ---------------- Install Dependencies ---------------- | |
| - name: Install dependencies via vcpkg | |
| run: | | |
| export VCPKG_BINARY_SOURCES="clear;files,${{ env.VCPKG_BINARY_CACHE }},readwrite" | |
| ${{ env.VCPKG_ROOT }}/vcpkg install --triplet x64-linux | |
| shell: bash | |
| # ---------------- Spark and System Setup ---------------- | |
| - name: Install Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: 17 | |
| - name: Install Spark | |
| run: | | |
| curl -fL https://dlcdn.apache.org/spark/spark-3.5.8/spark-3.5.8-bin-hadoop3.tgz -o spark.tgz | |
| tar -xzf spark.tgz | |
| mv spark-3.5.8-bin-hadoop3 $HOME/spark | |
| - name: Start Spark Connect Server | |
| run: | | |
| $HOME/spark/sbin/start-connect-server.sh \ | |
| --packages "org.apache.spark:spark-connect_2.12:3.5.3,io.delta:delta-spark_2.12:3.2.0,io.graphframes:graphframes-spark3_2.12:0.10.0,io.graphframes:graphframes-connect-spark3_2.12:0.10.0" \ | |
| --conf "spark.connect.extensions.relation.classes=org.apache.spark.sql.graphframes.GraphFramesConnect" \ | |
| --conf "spark.driver.extraJavaOptions=-Divy.cache.dir=/tmp -Divy.home=/tmp -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError" \ | |
| --conf "spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension" \ | |
| --conf "spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog" \ | |
| --conf "spark.driver.memory=4g" \ | |
| --conf "spark.executor.memory=4g" \ | |
| --conf "spark.memory.fraction=0.8" \ | |
| --conf "spark.memory.storageFraction=0.3" \ | |
| --conf "spark.sql.shuffle.partitions=8" \ | |
| --conf "spark.default.parallelism=8" \ | |
| --conf "spark.driver.maxResultSize=2g" | |
| # wait for server | |
| sleep 15 | |
| - name: Install gcovr | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y gcovr | |
| - name: Install dependencies | |
| run: chmod +x ./install_deps.sh && ./install_deps.sh | |
| - name: Configure with Coverage | |
| run: cmake --preset prod | |
| - name: Build | |
| run: cmake --build --preset release | |
| - name: Run Tests | |
| run: | | |
| export SPARK_REMOTE=sc://localhost | |
| ctest --preset test_coverage | |
| - name: Generate Coverage (XML & HTML) | |
| run: | | |
| gcovr -r src \ | |
| --object-directory build \ | |
| --exclude '.*\.pb\.cc' \ | |
| --exclude '.*\.grpc\.pb\.cc' \ | |
| --exclude '.*\.h' \ | |
| --xml-pretty -o coverage.xml \ | |
| --html-details coverage.html \ | |
| --fail-under-line 70 \ | |
| --print-summary | |
| - name: Upload Coverage XML | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-xml | |
| path: coverage.xml | |
| - name: Upload Coverage HTML | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-html | |
| path: coverage.html |