Support delta and iceberg via builder pattern #279
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Validate code | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| branches: [ main ] | |
| jobs: | |
| validate_code: | |
| name: Validate code in spark container | |
| runs-on: ubuntu-latest | |
| if: ${{ github.ref != 'refs/heads/main' }} | |
| permissions: | |
| contents: read | |
| packages: read | |
| pull-requests: write | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Build the Docker Compose stack | |
| run: docker compose up --quiet-pull -d | |
| - name: Get Compose Logs | |
| if: always() | |
| run: docker compose logs | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11.x' # Version range or exact version of a Python version to use, using SemVer's version range syntax | |
| architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified | |
| - name: Install Poetry and dependencies | |
| uses: SneaksAndData/github-actions/install_poetry@v0.1.11 | |
| with: | |
| install_extras: all | |
| - name: Black | |
| shell: bash | |
| run: | | |
| set -e | |
| poetry run black . --check --diff | |
| - name: Lint | |
| run: | | |
| set -e | |
| find ./spark_utils -type f -name "*.py" | xargs poetry run pylint | |
| - name: Unit test | |
| shell: bash | |
| env: | |
| PYSPARK_SUBMIT_ARGS: "--packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.10.0,org.apache.iceberg:iceberg-aws-bundle:1.10.0,io.delta:delta-spark_2.12:3.2.1 pyspark-shell" | |
| run: | | |
| set -euxo pipefail | |
| poetry run pytest --cov-config=.coveragerc --doctest-modules --junitxml=junit/test-results.xml --cov=. --cov-report=term-missing:skip-covered | tee pytest-coverage.txt | |
| - name: Publish Code Coverage | |
| uses: MishaKav/pytest-coverage-comment@main | |
| with: | |
| pytest-coverage-path: ./pytest-coverage.txt | |
| junitxml-path: ./junit/test-results.xml |