lakekeeper/.github/workflows/integration_test_workflow.yml at main · askvinni/lakekeeper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
name: integration test workflow

on:
  workflow_call:
    inputs:
      test_name:
        required: true
        type: string

env:
  CARGO_TERM_COLOR: always

permissions:
  contents: read # Needed to checkout code

jobs:
  test:
    name: ${{ inputs.test_name }}
    env:
      current_test: ${{ inputs.test_name }}
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6

      - name: Restore binary
        uses: actions/download-artifact@v6
        with:
          name: lakekeeper-image
          path: artifacts

      - name: Display structure of downloaded files
        run: ls -Rlh artifacts

      - name: Restore Docker image
        run: |
          docker load -i artifacts/lakekeeper-local-amd64.tar

      - name: Test ${{ inputs.test_name }}
        uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # v3
        with:
          max_attempts: 2
          timeout_minutes: 25
          command: |
            cd tests &&
            if [[ "${{ inputs.test_name }}" == *"openfga"* ]]; then
              docker compose -f docker-compose.yaml -f docker-compose-openfga-overlay.yaml run --quiet-pull spark /opt/entrypoint.sh bash -c "cd /opt/tests && bash run.sh ${current_test}"
            elif [[ "${{ inputs.test_name }}" == *"kv2"* ]]; then
              docker compose -f docker-compose.yaml -f docker-compose-vault-overlay.yaml run --quiet-pull spark /opt/entrypoint.sh bash -c "cd /opt/tests && bash run.sh ${current_test}"
            elif [[ "${{ inputs.test_name }}" == *"opa"* ]]; then
              docker compose -f docker-compose.yaml -f docker-compose-openfga-overlay.yaml -f docker-compose-trino-opa-overlay.yaml run --quiet-pull spark /opt/entrypoint.sh bash -c "cd /opt/tests && bash run.sh ${current_test}"
            elif [[ "${{ inputs.test_name }}" == *"starrocks"* ]]; then
              docker compose -f docker-compose.yaml -f docker-compose-starrocks-overlay.yaml run --quiet-pull spark /opt/entrypoint.sh bash -c "cd /opt/tests && bash run.sh ${current_test}"
            elif [[ "${{ inputs.test_name }}" == *"aws_system_identity"* ]]; then
              docker compose -f docker-compose.yaml -f docker-compose-s3-system-identity-overlay.yaml run --quiet-pull spark /opt/entrypoint.sh bash -c "cd /opt/tests && bash run.sh ${current_test}"
            elif [[ "${{ inputs.test_name }}" == *"legacy_md5"* ]]; then
              docker compose -f docker-compose.yaml -f docker-compose-legacy-md5-overlay.yaml run --quiet-pull spark /opt/entrypoint.sh bash -c "cd /opt/tests && bash run.sh ${current_test}"
            else
              docker compose run --quiet-pull spark /opt/entrypoint.sh bash -c "cd /opt/tests && bash run.sh ${current_test}"
            fi
        env:
          LAKEKEEPER_TEST__SPARK_IMAGE: apache/spark:3.5.6-java17-python3
          LAKEKEEPER_TEST__SERVER_IMAGE: localhost/lakekeeper-local:amd64
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
          AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
          AZURE_STORAGE_ACCOUNT_NAME: ${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
          AZURE_STORAGE_FILESYSTEM: ${{ secrets.AZURE_STORAGE_FILESYSTEM }}
          GCS_CREDENTIAL: ${{ secrets.GCS_CREDENTIAL }}
          GCS_BUCKET: ${{ secrets.GCS_BUCKET }}
          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
          AWS_S3_REGION: ${{ secrets.AWS_S3_REGION }}
          AWS_S3_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY_ID }}
          AWS_S3_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }}
          AWS_S3_STS_ROLE_ARN: ${{ secrets.AWS_S3_STS_ROLE_ARN }}
      - name: Dump DB
        if: always()
        uses: tj-actions/pg-dump@f68d6b3b0a96a30fda60109a19a5d9fb3c536c9c
        with:
          database_url: "postgresql://postgres:postgres@localhost:31102/postgres"
          postgresql_version: "17"
          path: dump.sql
      - name: compress
        if: always()
        run: cat dump.sql | gzip -9 > dump.sql.gz
      - name: Upload dump
        if: always()
        uses: actions/upload-artifact@v5
        with:
          name: db-dump-${{ inputs.test_name }}
          path: dump.sql.gz
      - name: Dump docker logs on failure
        if: always()
        uses: jwalton/gh-docker-logs@2741064ab9d7af54b0b1ffb6076cf64c16f0220e