Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 76 additions & 0 deletions .github/workflows/check.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
name: CI Check

on:
push:
branches: [ main ]
pull_request:
branches: [ main ]

jobs:
check-rust-ffi:
name: Check Rust Build (${{ matrix.os }}-${{ matrix.arch }})
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
arch: amd64
target: x86_64-unknown-linux-gnu
- os: ubuntu-latest
arch: arm64
target: aarch64-unknown-linux-gnu
- os: macos-latest
arch: amd64
target: x86_64-apple-darwin
- os: macos-latest
arch: arm64
target: aarch64-apple-darwin
- os: windows-latest
arch: amd64
target: x86_64-pc-windows-gnu

steps:
- uses: actions/checkout@v4

- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}

- name: Install Cross-Compilation Tools (Linux ARM64)
if: matrix.target == 'aarch64-unknown-linux-gnu'
run: |
sudo apt-get update
sudo apt-get install -y gcc-aarch64-linux-gnu

- name: Check Rust Compilation
working-directory: zerobus-ffi
run: |
cargo check --release --target ${{ matrix.target }}
env:
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc

check-go:
name: Check Go SDK
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4

- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
cache: true

- name: Go Vet
run: go vet ./...

- name: Go Format Check
run: |
if [ "$(gofmt -s -l . | wc -l)" -gt 0 ]; then
echo "The following files are not formatted correctly:"
gofmt -s -l .
exit 1
fi

4 changes: 2 additions & 2 deletions .github/workflows/push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
run: go generate -v

- name: Verify library built
run: test -f libzerobus_ffi.a
run: test -f lib/linux_amd64/libzerobus_ffi.a

- name: Test go build works (user step 2)
run: go build -v
Expand Down Expand Up @@ -53,7 +53,7 @@ jobs:
shell: bash

- name: Verify library built
run: test -f libzerobus_ffi.a
run: test -f lib/windows_amd64/libzerobus_ffi.a
shell: bash

- name: Test go build works (user step 2)
Expand Down
107 changes: 107 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
name: Seamless Release (CI-Augmented Tag)

on:
push:
tags:
- 'v*'

jobs:
build-rust:
name: Build Rust (${{ matrix.os }}-${{ matrix.arch }})
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
arch: amd64
target: x86_64-unknown-linux-gnu
- os: ubuntu-latest
arch: arm64
target: aarch64-unknown-linux-gnu
- os: macos-latest
arch: amd64
target: x86_64-apple-darwin
- os: macos-latest
arch: arm64
target: aarch64-apple-darwin
- os: windows-latest
arch: amd64
target: x86_64-pc-windows-gnu

steps:
- uses: actions/checkout@v4

- name: Install Rust
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}

- name: Install Cross-Compilation Tools (Linux ARM64)
if: matrix.target == 'aarch64-unknown-linux-gnu'
run: |
sudo apt-get update
sudo apt-get install -y gcc-aarch64-linux-gnu

- name: Build Rust Library
working-directory: zerobus-ffi
run: |
cargo build --release --target ${{ matrix.target }}
env:
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc

- name: Prepare Artifact
shell: bash
run: |
mkdir -p dist/${{ runner.os == 'macOS' && 'darwin' || (runner.os == 'Windows' && 'windows' || 'linux') }}_${{ matrix.arch }}
if [ "${{ runner.os }}" = "Windows" ]; then
cp zerobus-ffi/target/${{ matrix.target }}/release/libzerobus_ffi.a dist/windows_${{ matrix.arch }}/libzerobus_ffi.a || \
cp zerobus-ffi/target/${{ matrix.target }}/release/zerobus_ffi.lib dist/windows_${{ matrix.arch }}/libzerobus_ffi.a
else
cp zerobus-ffi/target/${{ matrix.target }}/release/libzerobus_ffi.a dist/${{ runner.os == 'macOS' && 'darwin' || 'linux' }}_${{ matrix.arch }}/libzerobus_ffi.a
fi

- name: Upload Artifact
uses: actions/upload-artifact@v4
with:
name: lib-${{ matrix.os }}-${{ matrix.arch }}
path: dist/

update-tag:
name: Update Tag with Binaries
needs: build-rust
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0

- name: Download all artifacts
uses: actions/download-artifact@v4
with:
path: lib-artifacts
merge-multiple: true

- name: Prepare lib directory
run: |
mkdir -p lib
cp -r lib-artifacts/* lib/
rm -rf lib-artifacts

- name: Commit and Force-Push Tag
run: |
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"

# Add the lib directory even though it's gitignored
git add -f lib/

# Create an augmented commit
git commit -m "Release ${{ github.ref_name }} with pre-built binaries"

# Force update the tag to this new commit
git tag -f ${{ github.ref_name }}
git push origin ${{ github.ref_name }} --force

1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Build artifacts - DO NOT COMMIT
zerobus-ffi/target/
lib/
libzerobus_ffi.a
*.dylib
*.so
Expand Down
42 changes: 28 additions & 14 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,24 +25,37 @@ build: build-rust build-go

build-rust:
@echo "Building Rust FFI layer..."
@# On Windows with MinGW, we need to build for the GNU target
@if [ "$$OS" = "Windows_NT" ]; then \
@# Detect OS and Arch for the target directory
@OS=$$(uname -s | tr '[:upper:]' '[:lower:]'); \
ARCH=$$(uname -m); \
case "$$OS" in \
darwin*) GOOS="darwin" ;; \
linux*) GOOS="linux" ;; \
msys*|mingw*|cygwin*) GOOS="windows" ;; \
*) GOOS="$$OS" ;; \
esac; \
case "$$ARCH" in \
x86_64) GOARCH="amd64" ;; \
aarch64|arm64) GOARCH="arm64" ;; \
*) GOARCH="$$ARCH" ;; \
esac; \
LIB_DIR="lib/$${GOOS}_$${GOARCH}"; \
echo "Target directory: $$LIB_DIR"; \
mkdir -p $$LIB_DIR; \
if [ "$$OS" = "Windows_NT" ] || [[ "$$OS" == *"mingw"* ]] || [[ "$$OS" == *"msys"* ]]; then \
echo "Detected Windows - building for x86_64-pc-windows-gnu target"; \
cd zerobus-ffi && cargo build --release --target x86_64-pc-windows-gnu; \
cd ..; \
if [ -f zerobus-ffi/target/x86_64-pc-windows-gnu/release/libzerobus_ffi.a ]; then \
cp zerobus-ffi/target/x86_64-pc-windows-gnu/release/libzerobus_ffi.a $$LIB_DIR/; \
elif [ -f zerobus-ffi/target/release/zerobus_ffi.lib ]; then \
cp zerobus-ffi/target/release/zerobus_ffi.lib $$LIB_DIR/libzerobus_ffi.a; \
fi; \
else \
cd zerobus-ffi && cargo build --release; \
fi
@echo "Copying static library and header..."
@if [ -f zerobus-ffi/target/release/libzerobus_ffi.a ]; then \
cp zerobus-ffi/target/release/libzerobus_ffi.a .; \
elif [ -f zerobus-ffi/target/x86_64-pc-windows-gnu/release/libzerobus_ffi.a ]; then \
cp zerobus-ffi/target/x86_64-pc-windows-gnu/release/libzerobus_ffi.a .; \
elif [ -f zerobus-ffi/target/release/zerobus_ffi.lib ]; then \
cp zerobus-ffi/target/release/zerobus_ffi.lib libzerobus_ffi.a; \
else \
echo "Error: Could not find Rust library"; \
exit 1; \
fi
cd ..; \
cp zerobus-ffi/target/release/libzerobus_ffi.a $$LIB_DIR/; \
fi; \
cp zerobus-ffi/zerobus.h .
@echo "✓ Rust FFI layer built successfully"

Expand All @@ -54,6 +67,7 @@ build-go: build-rust
clean:
@echo "Cleaning build artifacts..."
cd zerobus-ffi && cargo clean
rm -rf lib/
rm -f libzerobus_ffi.a
rm -rf releases
@echo "✓ Clean complete"
Expand Down
38 changes: 12 additions & 26 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,42 +85,28 @@ Choose your installation path:

| Path | When to Use |
|------|-------------|
| **[For SDK Users](#for-sdk-users-install-from-pkggodev)** | You want to use the SDK in your project (via `go get`) |
| **[For Contributors](#for-contributors-build-from-source)** | You want to contribute or build from source (via `git clone`) |
| **[Standard Installation](#installation)** | You want to use the SDK in your project (via `go get`) |
| **[Development Setup](#development-setup)** | You want to contribute or build from source (via `git clone`) |

### For SDK Users (Install from pkg.go.dev)
### Installation

The Zerobus Go SDK is a CGO-based wrapper around a high-performance Rust core. For the best experience, use a tagged release which includes pre-built binaries.

**Prerequisites:**

*System Requirements:*
- **Go 1.21+**
- **CGO enabled** (enabled by default)
- **Rust 1.75+** ([install from rustup.rs](https://rustup.rs))
- **C compiler** (gcc on Linux, clang on macOS, MinGW-w64 on Windows)

*Supported Platforms:*
- **Linux** (x86_64, ARM64)
- **macOS** (Intel, Apple Silicon)
- **Windows** (x86_64 with MinGW-w64)

*Databricks Requirements:*
- **Databricks workspace** with Zerobus access enabled (AWS, Azure, or GCP)
- **OAuth 2.0 client credentials** (client ID and secret)
- **Unity Catalog endpoint** access
- **C compiler** (gcc or clang)

**Installation:**
**Installation Steps:**

```bash
# 1. Add the SDK to your project
go get github.com/databricks/zerobus-sdk-go

# 2. Build the Rust FFI library (one-time setup, takes 2-5 minutes)
go generate github.com/databricks/zerobus-sdk-go

# 3. Build your project normally
go build
# Add the SDK to your project (use a tagged version for pre-built binaries)
go get github.com/databricks/zerobus-sdk-go@latest
```

> **Note:** Tagged releases (e.g., `v1.0.0`) come with pre-built Rust libraries for Linux, macOS, and Windows. If you use `@main` or a commit hash, you will need to have Rust installed and run `go generate` to build the library yourself.

**In your code:**

```go
Expand All @@ -141,7 +127,7 @@ func main() {

> **Note:** After the initial `go generate` step, regular `go build` works normally. The Rust library is statically linked into your binary.

### For Contributors (Build from Source)
### Development Setup

**Prerequisites:**
- Same as above (Go, CGO, Rust, C compiler, Databricks workspace)
Expand Down
10 changes: 7 additions & 3 deletions build.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ func init() {
return
}
sdkDir := filepath.Dir(filename)
libPath := filepath.Join(sdkDir, "libzerobus_ffi.a")
libPath := filepath.Join(sdkDir, "lib", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH), "libzerobus_ffi.a")

if _, err := os.Stat(libPath); os.IsNotExist(err) {
fmt.Fprintf(os.Stderr, "\n"+
Expand Down Expand Up @@ -56,7 +56,7 @@ func ensureRustLibrary() error {
return fmt.Errorf("failed to determine source directory")
}
sdkDir := filepath.Dir(filename)
libPath := filepath.Join(sdkDir, "libzerobus_ffi.a")
libPath := filepath.Join(sdkDir, "lib", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH), "libzerobus_ffi.a")

// Check if library already exists
if _, err := os.Stat(libPath); err == nil {
Expand Down Expand Up @@ -128,7 +128,11 @@ func buildRustLibrary(sdkDir string) error {
}

// Copy library to SDK directory (handle multiple possible locations)
dstLib := filepath.Join(sdkDir, "libzerobus_ffi.a")
dstDir := filepath.Join(sdkDir, "lib", fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH))
if err := os.MkdirAll(dstDir, 0755); err != nil {
return fmt.Errorf("failed to create library directory: %w", err)
}
dstLib := filepath.Join(dstDir, "libzerobus_ffi.a")

// Try different possible locations
possiblePaths := []string{
Expand Down
Loading
Loading