test-examples.yml

name: Examples Test [auth]

on:
  push:
    branches:
      - main
  pull_request:

env:
  CARGO_TERM_COLOR: always

permissions:
  contents: read # Needed to checkout code

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

jobs:
  docker:
    uses: ./.github/workflows/docker_build.yml
    with:
      platform: "amd64"
      dockerfile: "docker/full.Dockerfile"
      image_name: "lakekeeper-local"

  # Depends on all actions that are required for a "successful" CI run.
  tests-pass:
    name: all examples pass
    runs-on: ubuntu-latest
    needs:
      - test-example-access-control-simple
      - test-example-minimal
      - test-example-access-control-advanced
      - test-docker-compose-with-keycloak
      - test-example-kafka-connect-iceberg-sink
    steps:
      - run: exit 0

  test-example-minimal:
    needs: docker
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6

      - name: Restore binary
        uses: actions/download-artifact@v6
        with:
          name: lakekeeper-image
          path: artifacts

      - name: Display structure of downloaded files
        run: ls -Rlh artifacts

      - name: Restore Docker image
        run: |
          docker load -i artifacts/lakekeeper-local-amd64.tar

      - name: Test Minimal
        run: |
          cd examples/minimal
          sed -i '/pull_policy: always/d' docker-compose.yaml
          docker compose run jupyter bash -c "cd /home/jovyan/examples/ && jupyter execute Pyiceberg.ipynb && jupyter execute Spark.ipynb && jupyter execute Trino.ipynb && jupyter execute Starrocks.ipynb && jupyter execute \"Multiple Warehouses.ipynb\" && jupyter execute DuckDB.ipynb"
        env:
          LAKEKEEPER_TEST__SERVER_IMAGE: localhost/lakekeeper-local:amd64
      - name: Dump docker logs on failure
        if: failure()
        uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e

  test-example-access-control-simple:
    needs: docker
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6

      - name: Restore binary
        uses: actions/download-artifact@v6
        with:
          name: lakekeeper-image
          path: artifacts

      - name: Display structure of downloaded files
        run: ls -Rlh artifacts

      - name: Restore Docker image
        run: |
          docker load -i artifacts/lakekeeper-local-amd64.tar

      - name: Test Aeccess Control Simple
        run: |
          cd examples/access-control-simple
          sed -i '/pull_policy: always/d' docker-compose.yaml
          docker compose run jupyter bash -c "cd /home/jovyan/examples/ && jupyter execute 01-Bootstrap.ipynb && jupyter execute 02-Create-Warehouse.ipynb && jupyter execute 03-01-Spark.ipynb && jupyter execute 03-02-Trino.ipynb && jupyter execute 03-03-Starrocks.ipynb && jupyter execute 03-04-PyIceberg.ipynb && jupyter execute 03-05-DuckDB.ipynb"
        env:
          LAKEKEEPER_TEST__SERVER_IMAGE: localhost/lakekeeper-local:amd64
      - name: Dump docker logs on failure
        if: failure()
        uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e

  test-example-access-control-advanced:
    needs: docker
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6

      - name: Test Make all
        run: |
          cd examples/kafka-connect-iceberg-sink
          make all

      - name: Test Make all
        run: |
          cd examples/kafka-connect-iceberg-sink
          make run-producer

  test-example-kafka-connect-iceberg-sink:
    needs: docker
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6

      - name: Restore binary
        uses: actions/download-artifact@v6
        with:
          name: lakekeeper-image
          path: artifacts

      - name: Display structure of downloaded files
        run: ls -Rlh artifacts

      - name: Restore Docker image
        run: |
          docker load -i artifacts/lakekeeper-local-amd64.tar

      - name: Test Access Control Advanced
        run: |
          cd examples/access-control-advanced
          sed -i '/pull_policy: always/d' docker-compose.yaml
          docker compose run jupyter bash -c "cd /home/jovyan/examples/ && jupyter execute 01-Bootstrap.ipynb && jupyter execute 02-Create-Warehouse.ipynb"
        env:
          LAKEKEEPER_TEST__SERVER_IMAGE: localhost/lakekeeper-local:amd64
      - name: Dump docker logs on failure
        if: failure()
        uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e

  test-docker-compose-with-keycloak:
    needs: docker
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6

      - name: Restore binary
        uses: actions/download-artifact@v6
        with:
          name: lakekeeper-image
          path: artifacts

      - name: Display structure of downloaded files
        run: ls -Rlh artifacts

      - name: Restore Docker image
        run: |
          docker load -i artifacts/lakekeeper-local-amd64.tar

      - name: Launch Docker Compose with Keycloak
        run: |
          cd docker-compose
          sed -i '/pull_policy: /d' docker-compose.yaml
          docker compose -f docker-compose.yaml -f keycloak-overlay.yaml up -d --wait
        env:
          LAKEKEEPER__SERVER_IMAGE: localhost/lakekeeper-local:amd64

      - name: Test Docker Compose with Keycloak
        run: |
          set -euo pipefail

          CATALOG_URL="http://localhost:8181/catalog"
          MANAGEMENT_URL="http://localhost:8181/management"
          KEYCLOAK_TOKEN_URL="http://localhost:30080/realms/iceberg/protocol/openid-connect/token"
          WAREHOUSE="demo"

          CLIENT_ID="spark"
          CLIENT_SECRET="2OR3eRvYfSZzzZ16MlPd95jhLnOaLM52"

          # Get an access token from keycloak (the idP server)
          echo "Getting access token from Keycloak..."
          ACCESS_TOKEN=$(curl -s -X POST \
            "$KEYCLOAK_TOKEN_URL" \
            -H "Content-Type: application/x-www-form-urlencoded" \
            -d "grant_type=client_credentials" \
            -d "client_id=$CLIENT_ID" \
            -d "client_secret=$CLIENT_SECRET" \
            -d "scope=lakekeeper" \
            | jq -r '.access_token')

          echo "Access token received"

          # Check the 'info' endpoint of the Lakekeeper management API
          echo "Checking Lakekeeper management API info endpoint..."
          curl -s -X GET \
            "$MANAGEMENT_URL/v1/info" \
            -H "Authorization: Bearer $ACCESS_TOKEN" \
            | jq

          # Bootstrap
          echo "Bootstrapping Lakekeeper..."
          curl -s -X POST \
            "$MANAGEMENT_URL/v1/bootstrap" \
            -H "Authorization: Bearer $ACCESS_TOKEN" \
            -H "Content-Type: application/json" \
            -d '{
              "accept-terms-of-use": true
            }' \
            || echo "Bootstrap may have already been completed"

      - name: Dump docker logs on failure
        if: failure()
        uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e