Add delta table support #90
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| types: [opened, synchronize, reopened, labeled] | |
| workflow_dispatch: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| permissions: | |
| contents: read | |
| jobs: | |
| check: | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| check: [fmt, clippy] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Rust | |
| uses: dtolnay/[email protected] | |
| with: | |
| components: ${{ matrix.check == 'fmt' && 'rustfmt' || 'clippy' }} | |
| - name: Cache Cargo | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: ${{ matrix.check }} | |
| - name: Run Cargo Fmt | |
| if: matrix.check == 'fmt' | |
| run: cargo fmt --check | |
| - name: Run Cargo Clippy | |
| if: matrix.check == 'clippy' | |
| run: cargo clippy --all-targets --all-features --no-deps -- -D warnings | |
| test-partial: | |
| name: Tests (Partial) | |
| runs-on: ubuntu-latest | |
| # Run on forks. | |
| if: github.event.pull_request.head.repo.fork == true | |
| permissions: | |
| contents: read | |
| strategy: | |
| matrix: | |
| postgres_version: [17, 16, 15, 14] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Rust | |
| uses: dtolnay/[email protected] | |
| - name: Cache Cargo | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: test-partial | |
| - name: Start Docker Compose (Postgres ${{ matrix.postgres_version }}) | |
| run: | | |
| POSTGRES_VERSION=${{ matrix.postgres_version }} docker compose -f ./scripts/docker-compose.yaml up -d | |
| - name: Install sqlx-cli | |
| run: | | |
| cargo install sqlx-cli \ | |
| --features native-tls,postgres \ | |
| --no-default-features \ | |
| --locked | |
| - name: Migrate Database | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_PORT: 5430 | |
| POSTGRES_HOST: localhost | |
| run: | | |
| sudo apt-get install libpq-dev -y | |
| ./etl-api/scripts/run_migrations.sh | |
| - name: Run Tests | |
| run: | | |
| cargo test \ | |
| --workspace \ | |
| --all-features \ | |
| --no-fail-fast \ | |
| --exclude etl-destinations \ | |
| && \ | |
| cargo test \ | |
| -p etl-destinations \ | |
| --no-default-features \ | |
| --no-fail-fast \ | |
| --features iceberg | |
| test-full: | |
| name: Tests (Full) | |
| runs-on: ubuntu-latest | |
| # Run on non-forks. | |
| if: github.event.pull_request.head.repo.fork == false | |
| permissions: | |
| contents: read | |
| id-token: write | |
| strategy: | |
| matrix: | |
| postgres_version: [17, 16, 15, 14] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Rust | |
| uses: dtolnay/[email protected] | |
| - name: Cache Cargo | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: test-full | |
| - name: Start Docker Compose (Postgres ${{ matrix.postgres_version }}) | |
| run: | | |
| POSTGRES_VERSION=${{ matrix.postgres_version }} docker compose -f ./scripts/docker-compose.yaml up -d | |
| - name: Install sqlx-cli | |
| run: | | |
| cargo install sqlx-cli \ | |
| --features native-tls,postgres \ | |
| --no-default-features \ | |
| --locked | |
| - name: Migrate Database | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_PORT: 5430 | |
| POSTGRES_HOST: localhost | |
| run: | | |
| sudo apt-get install libpq-dev -y | |
| ./etl-api/scripts/run_migrations.sh | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Set up BigQuery Credentials | |
| run: | | |
| printf '%s' '${{ secrets.TESTS_BIGQUERY_SA_KEY_JSON }}' > /tmp/bigquery-sa-key.json | |
| echo "TESTS_BIGQUERY_PROJECT_ID=${{ secrets.TESTS_BIGQUERY_PROJECT_ID }}" >> $GITHUB_ENV | |
| echo "TESTS_BIGQUERY_SA_KEY_PATH=/tmp/bigquery-sa-key.json" >> $GITHUB_ENV | |
| - name: Generate Code Coverage | |
| id: coverage | |
| run: | | |
| cargo llvm-cov test \ | |
| --workspace --no-fail-fast \ | |
| --all-features \ | |
| --lcov --output-path lcov.info | |
| - name: Upload Coverage to Coveralls | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| fail-on-error: false | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| path-to-lcov: lcov.info |