From 22220a45744e911e93f18327aa99e91f470d6031 Mon Sep 17 00:00:00 2001 From: Cory LaNou Date: Wed, 26 Nov 2025 14:50:58 -0600 Subject: [PATCH 1/7] feat(release): add automated VFS builds for releases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add VFS (SQLite loadable extension) builds to the release workflow for: - linux/amd64 (native) - linux/arm64 (cross-compiled) - darwin/amd64 (native Intel) - darwin/arm64 (native M1) The VFS jobs run after GoReleaser completes and upload artifacts to the same release. This makes VFS extensions available to users without requiring manual compilation. Closes #863 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/release.yml | 100 ++++++++++++++++++++++++++++++++++ .goreleaser.yml | 13 +++++ Makefile | 43 +++++++++++++++ 3 files changed, 156 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 334964e0..aebb5a0a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,6 +62,106 @@ jobs: GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }} HOMEBREW_TAP_GITHUB_TOKEN: ${{ secrets.HOMEBREW_TAP_GITHUB_TOKEN }} + vfs-build-linux: + name: Build VFS (Linux ${{ matrix.arch }}) + runs-on: ubuntu-latest + needs: goreleaser + strategy: + matrix: + arch: [amd64, arm64] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Install cross-compiler (arm64) + if: matrix.arch == 'arm64' + run: | + sudo apt-get update + sudo apt-get install -y gcc-aarch64-linux-gnu + + - name: Get version + id: version + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "version=${{ inputs.tag }}" >> $GITHUB_OUTPUT + else + echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + fi + + - name: Build VFS extension + run: make vfs-linux-${{ matrix.arch }} + + - name: Create archive + run: | + cd dist + tar -czvf litestream-vfs-${{ steps.version.outputs.version }}-linux-${{ matrix.arch }}.tar.gz \ + litestream-vfs-linux-${{ matrix.arch }}.so + + - name: Upload to release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload ${{ steps.version.outputs.version }} \ + dist/litestream-vfs-${{ steps.version.outputs.version }}-linux-${{ matrix.arch }}.tar.gz \ + --clobber + + vfs-build-darwin: + name: Build VFS (macOS ${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + needs: goreleaser + strategy: + matrix: + include: + - arch: amd64 + runner: macos-13 + - arch: arm64 + runner: macos-14 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Get version + id: version + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then + echo "version=${{ inputs.tag }}" >> $GITHUB_OUTPUT + else + echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + fi + + - name: Build VFS extension + run: make vfs-darwin-${{ matrix.arch }} + + - name: Create archive + run: | + cd dist + tar -czvf litestream-vfs-${{ steps.version.outputs.version }}-darwin-${{ matrix.arch }}.tar.gz \ + litestream-vfs-darwin-${{ matrix.arch }}.dylib + + - name: Upload to release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload ${{ steps.version.outputs.version }} \ + dist/litestream-vfs-${{ steps.version.outputs.version }}-darwin-${{ matrix.arch }}.tar.gz \ + --clobber + # macos-sign: # runs-on: macos-latest # needs: goreleaser diff --git a/.goreleaser.yml b/.goreleaser.yml index a8f7209b..8713b987 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -151,6 +151,19 @@ release: ### Binary installation Download the appropriate archive for your platform, extract, and move to your PATH. + ## VFS Extension (Experimental) + + SQLite loadable extensions for read-only access to Litestream replicas are available for supported platforms: + + | Platform | File | + |----------|------| + | Linux x86_64 | `litestream-vfs-VERSION-linux-amd64.tar.gz` | + | Linux ARM64 | `litestream-vfs-VERSION-linux-arm64.tar.gz` | + | macOS Intel | `litestream-vfs-VERSION-darwin-amd64.tar.gz` | + | macOS Apple Silicon | `litestream-vfs-VERSION-darwin-arm64.tar.gz` | + + See the [VFS documentation](https://litestream.io/guides/vfs/) for usage instructions. + # Signing configuration # signs: # - id: macos diff --git a/Makefile b/Makefile index 5c55a014..0566aa12 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,13 @@ default: docker: docker build -t litestream . +# VFS build configuration +VFS_BUILD_TAGS := vfs,SQLITE3VFS_LOADABLE_EXT +VFS_SRC := ./cmd/litestream-vfs +VFS_C_SRC := src/litestream-vfs.c +DARWIN_LDFLAGS := -framework CoreFoundation -framework Security -lresolv +LINUX_LDFLAGS := -lpthread -ldl -lm + .PHONY: vfs vfs: mkdir -p dist @@ -10,6 +17,42 @@ vfs: mv dist/litestream-vfs.h src/litestream-vfs.h gcc -framework CoreFoundation -framework Security -lresolv -g -fPIC -shared -o dist/litestream-vfs.so src/litestream-vfs.c dist/litestream-vfs.a +.PHONY: vfs-linux-amd64 +vfs-linux-amd64: + mkdir -p dist + CGO_ENABLED=1 GOOS=linux GOARCH=amd64 \ + go build -tags $(VFS_BUILD_TAGS) -o dist/litestream-vfs-linux-amd64.a -buildmode=c-archive $(VFS_SRC) + cp dist/litestream-vfs-linux-amd64.h src/litestream-vfs.h + gcc -g -fPIC -shared -o dist/litestream-vfs-linux-amd64.so \ + $(VFS_C_SRC) dist/litestream-vfs-linux-amd64.a $(LINUX_LDFLAGS) + +.PHONY: vfs-linux-arm64 +vfs-linux-arm64: + mkdir -p dist + CGO_ENABLED=1 GOOS=linux GOARCH=arm64 CC=aarch64-linux-gnu-gcc \ + go build -tags $(VFS_BUILD_TAGS) -o dist/litestream-vfs-linux-arm64.a -buildmode=c-archive $(VFS_SRC) + cp dist/litestream-vfs-linux-arm64.h src/litestream-vfs.h + aarch64-linux-gnu-gcc -g -fPIC -shared -o dist/litestream-vfs-linux-arm64.so \ + $(VFS_C_SRC) dist/litestream-vfs-linux-arm64.a $(LINUX_LDFLAGS) + +.PHONY: vfs-darwin-amd64 +vfs-darwin-amd64: + mkdir -p dist + CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 \ + go build -tags $(VFS_BUILD_TAGS) -o dist/litestream-vfs-darwin-amd64.a -buildmode=c-archive $(VFS_SRC) + cp dist/litestream-vfs-darwin-amd64.h src/litestream-vfs.h + clang -arch x86_64 -g -fPIC -shared -o dist/litestream-vfs-darwin-amd64.dylib \ + $(VFS_C_SRC) dist/litestream-vfs-darwin-amd64.a $(DARWIN_LDFLAGS) + +.PHONY: vfs-darwin-arm64 +vfs-darwin-arm64: + mkdir -p dist + CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 \ + go build -tags $(VFS_BUILD_TAGS) -o dist/litestream-vfs-darwin-arm64.a -buildmode=c-archive $(VFS_SRC) + cp dist/litestream-vfs-darwin-arm64.h src/litestream-vfs.h + clang -arch arm64 -g -fPIC -shared -o dist/litestream-vfs-darwin-arm64.dylib \ + $(VFS_C_SRC) dist/litestream-vfs-darwin-arm64.a $(DARWIN_LDFLAGS) + vfs-test: go test -v -tags=vfs ./cmd/litestream-vfs From 26681b905146203182e9889da2e519e3deddfa68 Mon Sep 17 00:00:00 2001 From: Cory LaNou Date: Wed, 26 Nov 2025 15:23:21 -0600 Subject: [PATCH 2/7] fix(release): improve VFS build compatibility and validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Use ubuntu-20.04 for Linux builds (glibc 2.31) for broader compatibility - Add smoke tests to verify .so/.dylib can be loaded by SQLite - Fix release notes to use GoReleaser templating for version 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/release.yml | 16 +++++++++++++++- .goreleaser.yml | 8 ++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aebb5a0a..40f3dd14 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,7 +64,7 @@ jobs: vfs-build-linux: name: Build VFS (Linux ${{ matrix.arch }}) - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 # Use older Ubuntu for broader glibc compatibility (2.31) needs: goreleaser strategy: matrix: @@ -99,6 +99,14 @@ jobs: - name: Build VFS extension run: make vfs-linux-${{ matrix.arch }} + - name: Verify artifact + run: | + file dist/litestream-vfs-linux-${{ matrix.arch }}.so + # For native arch, verify it can be loaded by SQLite + if [ "${{ matrix.arch }}" == "amd64" ]; then + sqlite3 ':memory:' ".load dist/litestream-vfs-linux-amd64.so" ".exit" && echo "Extension loads successfully" + fi + - name: Create archive run: | cd dist @@ -148,6 +156,12 @@ jobs: - name: Build VFS extension run: make vfs-darwin-${{ matrix.arch }} + - name: Verify artifact + run: | + file dist/litestream-vfs-darwin-${{ matrix.arch }}.dylib + # Verify the extension can be loaded by SQLite + sqlite3 ':memory:' ".load dist/litestream-vfs-darwin-${{ matrix.arch }}.dylib" ".exit" && echo "Extension loads successfully" + - name: Create archive run: | cd dist diff --git a/.goreleaser.yml b/.goreleaser.yml index 8713b987..152e6584 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -157,10 +157,10 @@ release: | Platform | File | |----------|------| - | Linux x86_64 | `litestream-vfs-VERSION-linux-amd64.tar.gz` | - | Linux ARM64 | `litestream-vfs-VERSION-linux-arm64.tar.gz` | - | macOS Intel | `litestream-vfs-VERSION-darwin-amd64.tar.gz` | - | macOS Apple Silicon | `litestream-vfs-VERSION-darwin-arm64.tar.gz` | + | Linux x86_64 | `litestream-vfs-v{{.Version}}-linux-amd64.tar.gz` | + | Linux ARM64 | `litestream-vfs-v{{.Version}}-linux-arm64.tar.gz` | + | macOS Intel | `litestream-vfs-v{{.Version}}-darwin-amd64.tar.gz` | + | macOS Apple Silicon | `litestream-vfs-v{{.Version}}-darwin-arm64.tar.gz` | See the [VFS documentation](https://litestream.io/guides/vfs/) for usage instructions. From 474d038baa73261fb7038dbc88a8b696dc78cbb4 Mon Sep 17 00:00:00 2001 From: Cory LaNou Date: Wed, 26 Nov 2025 15:31:57 -0600 Subject: [PATCH 3/7] fix(release): add checksums and arm64 validation for VFS builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Generate SHA256 checksums for all VFS artifacts (.sha256 files) - Add readelf validation for cross-compiled arm64 Linux builds - Upload checksums alongside tarballs to release 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/release.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 40f3dd14..574e1a33 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -106,6 +106,11 @@ jobs: if [ "${{ matrix.arch }}" == "amd64" ]; then sqlite3 ':memory:' ".load dist/litestream-vfs-linux-amd64.so" ".exit" && echo "Extension loads successfully" fi + # For cross-compiled arch, verify ELF header and architecture + if [ "${{ matrix.arch }}" == "arm64" ]; then + readelf -h dist/litestream-vfs-linux-arm64.so | grep -E "(Class|Machine)" + echo "ARM64 ELF header verified" + fi - name: Create archive run: | @@ -113,12 +118,19 @@ jobs: tar -czvf litestream-vfs-${{ steps.version.outputs.version }}-linux-${{ matrix.arch }}.tar.gz \ litestream-vfs-linux-${{ matrix.arch }}.so + - name: Generate checksum + run: | + cd dist + sha256sum litestream-vfs-${{ steps.version.outputs.version }}-linux-${{ matrix.arch }}.tar.gz \ + > litestream-vfs-${{ steps.version.outputs.version }}-linux-${{ matrix.arch }}.tar.gz.sha256 + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh release upload ${{ steps.version.outputs.version }} \ dist/litestream-vfs-${{ steps.version.outputs.version }}-linux-${{ matrix.arch }}.tar.gz \ + dist/litestream-vfs-${{ steps.version.outputs.version }}-linux-${{ matrix.arch }}.tar.gz.sha256 \ --clobber vfs-build-darwin: @@ -168,12 +180,19 @@ jobs: tar -czvf litestream-vfs-${{ steps.version.outputs.version }}-darwin-${{ matrix.arch }}.tar.gz \ litestream-vfs-darwin-${{ matrix.arch }}.dylib + - name: Generate checksum + run: | + cd dist + shasum -a 256 litestream-vfs-${{ steps.version.outputs.version }}-darwin-${{ matrix.arch }}.tar.gz \ + > litestream-vfs-${{ steps.version.outputs.version }}-darwin-${{ matrix.arch }}.tar.gz.sha256 + - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | gh release upload ${{ steps.version.outputs.version }} \ dist/litestream-vfs-${{ steps.version.outputs.version }}-darwin-${{ matrix.arch }}.tar.gz \ + dist/litestream-vfs-${{ steps.version.outputs.version }}-darwin-${{ matrix.arch }}.tar.gz.sha256 \ --clobber # macos-sign: From de5ef5b0da20837b5a52de56a103e9f16e2d713d Mon Sep 17 00:00:00 2001 From: Cory LaNou Date: Wed, 26 Nov 2025 15:49:15 -0600 Subject: [PATCH 4/7] fix(release): set macOS minimum deployment target for VFS builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set MACOSX_MIN_VERSION=11.0 to ensure VFS dylibs are compatible with macOS Big Sur and later, rather than being locked to the CI runner's macOS version. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0566aa12..ebd06fb3 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,8 @@ docker: VFS_BUILD_TAGS := vfs,SQLITE3VFS_LOADABLE_EXT VFS_SRC := ./cmd/litestream-vfs VFS_C_SRC := src/litestream-vfs.c -DARWIN_LDFLAGS := -framework CoreFoundation -framework Security -lresolv +MACOSX_MIN_VERSION := 11.0 +DARWIN_LDFLAGS := -framework CoreFoundation -framework Security -lresolv -mmacosx-version-min=$(MACOSX_MIN_VERSION) LINUX_LDFLAGS := -lpthread -ldl -lm .PHONY: vfs From 23acbf9e3f5979e830230a68d691adc3a90867d9 Mon Sep 17 00:00:00 2001 From: Cory LaNou Date: Wed, 26 Nov 2025 16:47:57 -0600 Subject: [PATCH 5/7] fix(release): remove reference to non-existent VFS documentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .goreleaser.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 152e6584..029674f7 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -162,8 +162,6 @@ release: | macOS Intel | `litestream-vfs-v{{.Version}}-darwin-amd64.tar.gz` | | macOS Apple Silicon | `litestream-vfs-v{{.Version}}-darwin-arm64.tar.gz` | - See the [VFS documentation](https://litestream.io/guides/vfs/) for usage instructions. - # Signing configuration # signs: # - id: macos From 62a3e7255d08a1de76095f38f5539ba97fbc478f Mon Sep 17 00:00:00 2001 From: Cory LaNou Date: Thu, 27 Nov 2025 08:12:16 -0600 Subject: [PATCH 6/7] fix(release): use platform-agnostic names in VFS tarballs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename files inside tarballs to litestream-vfs.so (Linux) and litestream-vfs.dylib (macOS) regardless of build architecture. This provides a consistent filename for users regardless of platform. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/release.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 574e1a33..32a524f4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -115,8 +115,9 @@ jobs: - name: Create archive run: | cd dist + cp litestream-vfs-linux-${{ matrix.arch }}.so litestream-vfs.so tar -czvf litestream-vfs-${{ steps.version.outputs.version }}-linux-${{ matrix.arch }}.tar.gz \ - litestream-vfs-linux-${{ matrix.arch }}.so + litestream-vfs.so - name: Generate checksum run: | @@ -177,8 +178,9 @@ jobs: - name: Create archive run: | cd dist + cp litestream-vfs-darwin-${{ matrix.arch }}.dylib litestream-vfs.dylib tar -czvf litestream-vfs-${{ steps.version.outputs.version }}-darwin-${{ matrix.arch }}.tar.gz \ - litestream-vfs-darwin-${{ matrix.arch }}.dylib + litestream-vfs.dylib - name: Generate checksum run: | From a19b9ae391c3a9c4b69457adab28fa471336a62c Mon Sep 17 00:00:00 2001 From: Cory LaNou Date: Thu, 27 Nov 2025 08:13:08 -0600 Subject: [PATCH 7/7] fix(release): rename VFS extension to litestream.so MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rename the VFS extension file inside tarballs from litestream-vfs.so to litestream.so (and .dylib on macOS) to match the VFS registration name "litestream". 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 32a524f4..9dca294d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -115,9 +115,9 @@ jobs: - name: Create archive run: | cd dist - cp litestream-vfs-linux-${{ matrix.arch }}.so litestream-vfs.so + cp litestream-vfs-linux-${{ matrix.arch }}.so litestream.so tar -czvf litestream-vfs-${{ steps.version.outputs.version }}-linux-${{ matrix.arch }}.tar.gz \ - litestream-vfs.so + litestream.so - name: Generate checksum run: | @@ -178,9 +178,9 @@ jobs: - name: Create archive run: | cd dist - cp litestream-vfs-darwin-${{ matrix.arch }}.dylib litestream-vfs.dylib + cp litestream-vfs-darwin-${{ matrix.arch }}.dylib litestream.dylib tar -czvf litestream-vfs-${{ steps.version.outputs.version }}-darwin-${{ matrix.arch }}.tar.gz \ - litestream-vfs.dylib + litestream.dylib - name: Generate checksum run: |