Add delta table support #66
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| types: [opened, synchronize, reopened, labeled] | |
| workflow_dispatch: | |
| env: | |
| CARGO_TERM_COLOR: always | |
| permissions: | |
| contents: read | |
| jobs: | |
| check: | |
| name: Check | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| check: [fmt, clippy] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Rust | |
| uses: dtolnay/[email protected] | |
| with: | |
| components: ${{ matrix.check == 'fmt' && 'rustfmt' || 'clippy' }} | |
| - name: Setup Rust cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: ${{ matrix.check }} | |
| - name: Run cargo fmt | |
| if: matrix.check == 'fmt' | |
| run: cargo fmt --check | |
| - name: Run cargo clippy | |
| if: matrix.check == 'clippy' | |
| run: cargo clippy --all-targets --all-features --no-deps -- -D warnings | |
| test-partial: | |
| name: Test Partial | |
| runs-on: ubuntu-latest | |
| # Run on forks. | |
| if: github.event.pull_request.head.repo.fork == true | |
| permissions: | |
| contents: read | |
| strategy: | |
| matrix: | |
| postgres_version: [17, 16, 15, 14] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Rust | |
| uses: dtolnay/[email protected] | |
| - name: Setup Rust cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: test-partial | |
| - name: Start Docker Compose Environment (Postgres ${{ matrix.postgres_version }}) | |
| run: | | |
| POSTGRES_VERSION=${{ matrix.postgres_version }} docker compose -f ./scripts/docker-compose.yaml up -d | |
| - name: Install sqlx-cli | |
| run: | | |
| cargo install sqlx-cli \ | |
| --features native-tls,postgres \ | |
| --no-default-features \ | |
| --locked | |
| - name: Migrate database | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_PORT: 5430 | |
| POSTGRES_HOST: localhost | |
| run: | | |
| sudo apt-get install libpq-dev -y | |
| ./etl-api/scripts/run_migrations.sh | |
| - name: Run tests | |
| run: | | |
| cargo test \ | |
| --workspace \ | |
| --all-features \ | |
| --no-fail-fast \ | |
| --exclude etl-destinations \ | |
| && \ | |
| cargo test \ | |
| -p etl-destinations \ | |
| --no-default-features \ | |
| --no-fail-fast \ | |
| --features iceberg | |
| test-full: | |
| name: Test Full | |
| runs-on: ubuntu-latest | |
| # Run on non-forks. | |
| if: github.event.pull_request.head.repo.fork == false | |
| permissions: | |
| contents: read | |
| id-token: write | |
| strategy: | |
| matrix: | |
| postgres_version: [17, 16, 15, 14] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Rust | |
| uses: dtolnay/[email protected] | |
| - name: Setup Rust cache | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| key: test-full | |
| - name: Start Docker Compose Environment (Postgres ${{ matrix.postgres_version }}) | |
| run: | | |
| POSTGRES_VERSION=${{ matrix.postgres_version }} docker compose -f ./scripts/docker-compose.yaml up -d | |
| - name: Install sqlx-cli | |
| run: | | |
| cargo install sqlx-cli \ | |
| --features native-tls,postgres \ | |
| --no-default-features \ | |
| --locked | |
| - name: Migrate database | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: postgres | |
| POSTGRES_PORT: 5430 | |
| POSTGRES_HOST: localhost | |
| run: | | |
| sudo apt-get install libpq-dev -y | |
| ./etl-api/scripts/run_migrations.sh | |
| - name: Install cargo-llvm-cov | |
| uses: taiki-e/install-action@cargo-llvm-cov | |
| - name: Set up BigQuery environment variables and credentials | |
| run: | | |
| printf '%s' '${{ secrets.TESTS_BIGQUERY_SA_KEY_JSON }}' > /tmp/bigquery-sa-key.json | |
| echo "TESTS_BIGQUERY_PROJECT_ID=${{ secrets.TESTS_BIGQUERY_PROJECT_ID }}" >> $GITHUB_ENV | |
| echo "TESTS_BIGQUERY_SA_KEY_PATH=/tmp/bigquery-sa-key.json" >> $GITHUB_ENV | |
| - name: Generate code coverage | |
| id: coverage | |
| run: | | |
| cargo llvm-cov test \ | |
| --workspace --no-fail-fast \ | |
| --all-features \ | |
| --lcov --output-path lcov.info | |
| - name: Upload coverage to Coveralls | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| fail-on-error: false | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| path-to-lcov: lcov.info |