lance-duckdb/.github/workflows/Build.yml at 9ac53013e5a310f5d3568b90c9b7d46836e6dcb8 · lance-format/lance-duckdb · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
name: Extension Build
on:
  pull_request:
    paths-ignore:
      - "**.md"
  push:
    branches:
      - "main"
    paths-ignore:
      - "**.md"
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

jobs:
  build:
    name: Build
    runs-on: ubuntu-24.04
    env:
      CUSTOM_LINKER: mold
      CCACHE_BASEDIR: ${{ github.workspace }}
      CCACHE_COMPRESS: "true"
      CCACHE_DIR: ${{ github.workspace }}/.ccache
      CCACHE_MAXSIZE: 500M
      CCACHE_NOHASHDIR: "true"
      EXT_FLAGS: -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
    steps:
      - uses: actions/checkout@v5
        with:
          submodules: recursive

      - name: Compute DuckDB SHA
        id: duckdb-sha
        run: |
          echo "sha=$(git -C duckdb rev-parse HEAD)" >> "$GITHUB_OUTPUT"

      - name: Compute Build SHA
        id: build-sha
        run: |
          echo "sha=${{ hashFiles('CMakeLists.txt', 'extension_config.cmake', 'Makefile') }}" >> "$GITHUB_OUTPUT"

      - name: Prepare cache directories
        run: |
          mkdir -p .ccache

      - name: Restore ccache
        id: cache-ccache
        uses: actions/cache/restore@v4
        with:
          path: |
            .ccache
          key: ccache-${{ runner.os }}-${{ steps.duckdb-sha.outputs.sha }}-${{ steps.build-sha.outputs.sha }}-${{ github.sha }}
          restore-keys: |
            ccache-${{ runner.os }}-${{ steps.duckdb-sha.outputs.sha }}-${{ steps.build-sha.outputs.sha }}-

      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev ninja-build ccache mold

      - name: Build
        run: |
          GEN=ninja make release -j 4

      - name: Upload build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: build-release-${{ github.sha }}
          path: |
            build/release/test/unittest
            build/release/src/libduckdb.so*
          if-no-files-found: error

      - name: Save ccache
        if: github.ref == 'refs/heads/main' && success()
        uses: actions/cache/save@v4
        with:
          path: |
            .ccache
          key: ccache-${{ runner.os }}-${{ steps.duckdb-sha.outputs.sha }}-${{ steps.build-sha.outputs.sha }}-${{ github.sha }}

  test:
    name: Test
    runs-on: ubuntu-24.04
    needs: build
    steps:
      - uses: actions/checkout@v5

      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-release-${{ github.sha }}
          path: build/release

      - name: Ensure test runner is executable
        run: |
          chmod +x ./build/release/test/unittest

      - name: Run tests
        run: |
          ./build/release/test/unittest "test/*"

  test_s3:
    name: Test S3 (MinIO)
    runs-on: ubuntu-24.04
    needs: build
    steps:
      - uses: actions/checkout@v5

      - name: Download build artifacts
        uses: actions/download-artifact@v4
        with:
          name: build-release-${{ github.sha }}
          path: build/release

      - name: Ensure test runner is executable
        run: |
          chmod +x ./build/release/test/unittest

      - name: Start MinIO
        run: |
          docker run -d --rm \
            --name lance-minio \
            -p 9000:9000 \
            -e MINIO_ROOT_USER=minioadmin \
            -e MINIO_ROOT_PASSWORD=minioadmin \
            minio/minio:RELEASE.2025-01-20T14-49-07Z \
            server /data

      - name: Wait for MinIO
        run: |
          for i in $(seq 1 60); do
            if curl -fsS "http://127.0.0.1:9000/minio/health/ready" >/dev/null; then
              exit 0
            fi
            sleep 1
          done
          echo "MinIO did not become ready" >&2
          exit 1

      - name: Upload test dataset to MinIO
        run: |
          curl -fsSL "https://dl.min.io/client/mc/release/linux-amd64/mc" -o mc
          chmod +x mc
          ./mc alias set local http://127.0.0.1:9000 minioadmin minioadmin
          ./mc mb -p local/lance-test || true
          ./mc mirror --overwrite test/data/test_data.lance local/lance-test/test_data.lance
          ./mc mirror --overwrite test/data/search_test_data.lance local/lance-test/search_test_data.lance

      - name: Run S3 tests
        env:
          LANCE_TEST_S3: "1"
        run: |
          ./build/release/test/unittest "test/*"

      - name: Cleanup MinIO
        if: always()
        run: |
          docker logs lance-minio || true
          docker rm -f lance-minio || true