Skip to content

Nightly

Nightly #11

Workflow file for this run

# yaml-language-server: $schema=https://www.schemastore.org/github-workflow.json
name: Nightly
on:
schedule:
# Run at 2am UTC daily
- cron: "0 2 * * *"
workflow_dispatch:
# Cancel in-progress runs when new runs are triggered
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: write
packages: write
env:
CARGO_TERM_COLOR: always
# GitHub Container Registry
GHCR_REGISTRY: ghcr.io
GHCR_IMAGE: ${{ github.repository }}
# Docker Hub
DOCKERHUB_IMAGE: inferadb/inferadb-ledger
jobs:
# Check if there are changes since the last nightly
check-changes:
name: Changes
runs-on: ubuntu-latest
outputs:
should_run: ${{ steps.check.outputs.should_run }}
libraries_changed: ${{ steps.libraries.outputs.changed }}
server_changed: ${{ steps.server.outputs.changed }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@e3f713f2d8f53843e71c69a996d56f51aa9adfb9 # v2.14.1
with:
egress-policy: audit
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- name: Check for changes since last nightly
id: check
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Find the last nightly release
LAST_NIGHTLY=$(gh release list --limit 50 | grep -E 'nightly\.' | head -1 | awk '{print $1}' || echo "")
if [ -z "$LAST_NIGHTLY" ]; then
echo "No previous nightly release found, will run"
echo "should_run=true" >> $GITHUB_OUTPUT
exit 0
fi
# Get the commit SHA for the last nightly tag
LAST_SHA=$(git rev-list -n 1 "$LAST_NIGHTLY" 2>/dev/null || echo "")
if [ -z "$LAST_SHA" ]; then
echo "Could not find commit for last nightly, will run"
echo "should_run=true" >> $GITHUB_OUTPUT
exit 0
fi
# Check if there are commits since the last nightly
COMMITS_SINCE=$(git rev-list "$LAST_SHA"..HEAD --count)
if [ "$COMMITS_SINCE" -gt 0 ]; then
echo "Found $COMMITS_SINCE commits since last nightly, will run"
echo "should_run=true" >> $GITHUB_OUTPUT
else
echo "No commits since last nightly, skipping"
echo "should_run=false" >> $GITHUB_OUTPUT
fi
- name: Check for library changes since last nightly
id: libraries
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Find the last nightly release
LAST_NIGHTLY=$(gh release list --limit 50 | grep -E 'nightly\.' | head -1 | awk '{print $1}' || echo "")
if [ -z "$LAST_NIGHTLY" ]; then
echo "No previous nightly, assuming library changes"
echo "changed=true" >> $GITHUB_OUTPUT
exit 0
fi
LAST_SHA=$(git rev-list -n 1 "$LAST_NIGHTLY" 2>/dev/null || echo "")
if [ -z "$LAST_SHA" ]; then
echo "Could not find commit, assuming library changes"
echo "changed=true" >> $GITHUB_OUTPUT
exit 0
fi
# Check if library files changed since last nightly
LIBRARY_CHANGES=$(git diff --name-only "$LAST_SHA"..HEAD -- \
crates/types crates/store crates/state crates/raft crates/sdk \
proto Cargo.toml Cargo.lock | wc -l)
if [ "$LIBRARY_CHANGES" -gt 0 ]; then
echo "Found $LIBRARY_CHANGES library file changes"
echo "changed=true" >> $GITHUB_OUTPUT
else
echo "No library changes, only server/CI changes"
echo "changed=false" >> $GITHUB_OUTPUT
fi
- name: Check for server changes since last nightly
id: server
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# Find the last nightly release
LAST_NIGHTLY=$(gh release list --limit 50 | grep -E 'nightly\.' | head -1 | awk '{print $1}' || echo "")
if [ -z "$LAST_NIGHTLY" ]; then
echo "No previous nightly, assuming server changes"
echo "changed=true" >> $GITHUB_OUTPUT
exit 0
fi
LAST_SHA=$(git rev-list -n 1 "$LAST_NIGHTLY" 2>/dev/null || echo "")
if [ -z "$LAST_SHA" ]; then
echo "Could not find commit, assuming server changes"
echo "changed=true" >> $GITHUB_OUTPUT
exit 0
fi
# Check if server files changed since last nightly
SERVER_CHANGES=$(git diff --name-only "$LAST_SHA"..HEAD -- \
crates/server Dockerfile | wc -l)
if [ "$SERVER_CHANGES" -gt 0 ]; then
echo "Found $SERVER_CHANGES server file changes"
echo "changed=true" >> $GITHUB_OUTPUT
else
echo "No server changes"
echo "changed=false" >> $GITHUB_OUTPUT
fi
# Compute nightly version from release-please manifest
version:
name: Version
needs: check-changes
if: needs.check-changes.outputs.should_run == 'true'
runs-on: ubuntu-latest
outputs:
version: ${{ steps.version.outputs.version }}
date: ${{ steps.version.outputs.date }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@e3f713f2d8f53843e71c69a996d56f51aa9adfb9 # v2.14.1
with:
egress-policy: audit
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Compute nightly version
id: version
run: |
# Read next version from release-please manifest
if [ -f ".release-please-manifest.json" ]; then
NEXT_VERSION=$(jq -r '.["." ] // .[] | select(. != null)' .release-please-manifest.json | head -1)
else
NEXT_VERSION="0.1.0"
fi
DATE=$(date -u +%Y%m%d)
# Use dev.YYYYMMDD format for chronological semver sorting
NIGHTLY_VERSION="${NEXT_VERSION}-dev.${DATE}"
echo "version=${NIGHTLY_VERSION}" >> $GITHUB_OUTPUT
echo "date=${DATE}" >> $GITHUB_OUTPUT
echo "Nightly version: ${NIGHTLY_VERSION}"
# Create GitHub pre-release
create-release:
name: Release
needs: version
runs-on: ubuntu-latest
permissions:
contents: write
outputs:
version: ${{ needs.version.outputs.version }}
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@e3f713f2d8f53843e71c69a996d56f51aa9adfb9 # v2.14.1
with:
egress-policy: audit
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Create Pre-Release
uses: step-security/action-gh-release@d45511d7589f080cf54961ff056b9705a74fd160 # v2.5.0
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: v${{ needs.version.outputs.version }}
name: Nightly v${{ needs.version.outputs.version }}
draft: false
prerelease: true
body: |
🌙 **This is a nightly build for testing purposes.**
Built from commit: ${{ github.sha }}
Build date: ${{ needs.version.outputs.date }}
Install with Docker:
```bash
docker pull ${{ env.DOCKERHUB_IMAGE }}:nightly
# or
docker pull ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE }}:nightly
```
- name: Update nightly tag
run: |
git tag -f nightly
git push -f origin nightly
# Build release binaries for multiple platforms (only if server changed)
build-binaries:
name: Build (${{ matrix.name }})
needs: [check-changes, version, create-release]
if: needs.check-changes.outputs.server_changed == 'true'
runs-on: ${{ matrix.os }}
permissions:
contents: write
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
name: Linux AMD64
target: x86_64-unknown-linux-gnu
artifact_name: inferadb-ledger
asset_name: inferadb-ledger-linux-x86_64
- os: ubuntu-24.04-arm
name: Linux ARM64
target: aarch64-unknown-linux-gnu
artifact_name: inferadb-ledger
asset_name: inferadb-ledger-linux-aarch64
- os: macos-latest
name: macOS Apple Silicon
target: aarch64-apple-darwin
artifact_name: inferadb-ledger
asset_name: inferadb-ledger-macos-aarch64
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@e3f713f2d8f53843e71c69a996d56f51aa9adfb9 # v2.14.1
with:
egress-policy: audit
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@e97e2d8cc328f1b50210efc529dca0028893a2d9 # v1
with:
toolchain: stable
targets: ${{ matrix.target }}
- name: Install build dependencies (Linux)
if: runner.os == 'Linux'
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq mold protobuf-compiler
- name: Install build dependencies (macOS)
if: runner.os == 'macOS'
run: brew install protobuf
- name: Setup sccache
uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
- name: Configure build environment
run: |
echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV
echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
echo "CARGO_INCREMENTAL=0" >> $GITHUB_ENV
if [[ "${{ runner.os }}" == "Linux" ]]; then
echo "RUSTFLAGS=-C codegen-units=16 -C link-arg=-fuse-ld=mold" >> $GITHUB_ENV
elif [[ "${{ runner.os }}" == "macOS" ]]; then
echo "RUSTFLAGS=-C codegen-units=16" >> $GITHUB_ENV
echo "MACOSX_DEPLOYMENT_TARGET=11.0" >> $GITHUB_ENV
fi
- name: Cache Rust dependencies
uses: step-security/rust-cache@9be15b830520fab0ec3939586e917e4855cf76bd # v2.8.3
with:
shared-key: nightly-${{ matrix.os }}-${{ matrix.target }}
- name: Build release binary
run: cargo build --release --target ${{ matrix.target }} --package inferadb-ledger-server
- name: Strip binary
run: strip target/${{ matrix.target }}/release/${{ matrix.artifact_name }}
- name: Generate SBOM
run: |
cargo install cargo-sbom
cargo sbom --output-format spdx_json_2_3 > sbom-${{ matrix.target }}.spdx.json
- name: Compress binary
run: |
cd target/${{ matrix.target }}/release
tar czf ${{ matrix.asset_name }}.tar.gz ${{ matrix.artifact_name }}
- name: Upload binary to release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release upload "v${{ needs.version.outputs.version }}" \
"target/${{ matrix.target }}/release/${{ matrix.asset_name }}.tar.gz"
- name: Upload SBOM to release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release upload "v${{ needs.version.outputs.version }}" \
"sbom-${{ matrix.target }}.spdx.json"
# Build and push Docker images (only if server changed)
build-docker:
name: Docker (${{ matrix.name }})
needs: [check-changes, version, create-release]
if: needs.check-changes.outputs.server_changed == 'true'
strategy:
fail-fast: false
matrix:
include:
- platform: linux/amd64
name: Linux AMD64
runner: ubuntu-latest
suffix: amd64
- platform: linux/arm64
name: Linux ARM64
runner: ubuntu-24.04-arm
suffix: arm64
runs-on: ${{ matrix.runner }}
permissions:
contents: read
packages: write
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@e3f713f2d8f53843e71c69a996d56f51aa9adfb9 # v2.14.1
with:
egress-policy: audit
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up Docker Buildx
uses: step-security/setup-buildx-action@8c8aef2d414c0b66518fee2b7084e0986f82d7ac # v3.11.1
- name: Log in to GitHub Container Registry
uses: step-security/docker-login-action@c3e677aae8393bc9c81cfdf9709648720ea4bd4d # v3.6.0
with:
registry: ${{ env.GHCR_REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata
id: meta
uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051 # v5.10.0
with:
images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE }}
tags: |
type=raw,value=nightly
type=raw,value=nightly-${{ needs.version.outputs.date }}
- name: Build and push by digest
id: build
uses: step-security/docker-build-push-action@a8c3d08b23f8be6aeed43eb1a14ce6fe51284438 # v6.18.0
with:
context: .
platforms: ${{ matrix.platform }}
labels: ${{ steps.meta.outputs.labels }}
outputs: type=image,name=${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE }},push-by-digest=true,name-canonical=true,push=true
cache-from: type=gha,scope=nightly-${{ matrix.suffix }}
cache-to: type=gha,mode=max,scope=nightly-${{ matrix.suffix }}
- name: Export digest
run: |
mkdir -p /tmp/digests
digest="${{ steps.build.outputs.digest }}"
touch "/tmp/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
with:
name: digests-nightly-${{ matrix.suffix }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
# Merge platform-specific images into multi-arch manifest (only if server changed)
merge-docker:
name: Docker Manifest
runs-on: ubuntu-latest
needs: [check-changes, version, build-docker]
if: needs.check-changes.outputs.server_changed == 'true'
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@e3f713f2d8f53843e71c69a996d56f51aa9adfb9 # v2.14.1
with:
egress-policy: audit
- name: Download digests
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
with:
path: /tmp/digests
pattern: digests-nightly-*
merge-multiple: true
- name: Set up Docker Buildx
uses: step-security/setup-buildx-action@8c8aef2d414c0b66518fee2b7084e0986f82d7ac # v3.11.1
- name: Log in to GitHub Container Registry
uses: step-security/docker-login-action@c3e677aae8393bc9c81cfdf9709648720ea4bd4d # v3.6.0
with:
registry: ${{ env.GHCR_REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Log in to Docker Hub
uses: step-security/docker-login-action@c3e677aae8393bc9c81cfdf9709648720ea4bd4d # v3.6.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Create and push manifest to ghcr.io
working-directory: /tmp/digests
run: |
docker buildx imagetools create \
-t ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE }}:nightly \
-t ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE }}:nightly-${{ needs.version.outputs.date }} \
$(printf '${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE }}@sha256:%s ' *)
- name: Copy manifest to Docker Hub
run: |
docker buildx imagetools create \
-t ${{ env.DOCKERHUB_IMAGE }}:nightly \
-t ${{ env.DOCKERHUB_IMAGE }}:nightly-${{ needs.version.outputs.date }} \
${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE }}:nightly
- name: Inspect images
run: |
echo "=== ghcr.io ==="
docker buildx imagetools inspect ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE }}:nightly
echo "=== Docker Hub ==="
docker buildx imagetools inspect ${{ env.DOCKERHUB_IMAGE }}:nightly
# Generate SBOM for Docker image (only if server changed)
sbom-docker:
name: Docker SBOM
runs-on: ubuntu-latest
needs: [check-changes, version, merge-docker]
if: needs.check-changes.outputs.server_changed == 'true'
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@e3f713f2d8f53843e71c69a996d56f51aa9adfb9 # v2.14.1
with:
egress-policy: audit
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Log in to GitHub Container Registry
uses: step-security/docker-login-action@c3e677aae8393bc9c81cfdf9709648720ea4bd4d # v3.6.0
with:
registry: ${{ env.GHCR_REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Generate SBOM
uses: anchore/sbom-action@deef08a0db64bfad603422135db61477b16cef56 # v0.22.1
with:
image: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE }}:nightly
artifact-name: docker-sbom.spdx.json
output-file: docker-sbom.spdx.json
- name: Upload SBOM to release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh release upload "v${{ needs.version.outputs.version }}" \
docker-sbom.spdx.json --clobber || true
# Publish crates to crates.io with nightly version (only if libraries changed)
publish-crates:
name: Crates
needs: [check-changes, version, create-release]
if: needs.check-changes.outputs.libraries_changed == 'true'
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@e3f713f2d8f53843e71c69a996d56f51aa9adfb9 # v2.14.1
with:
egress-policy: audit
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Install Rust toolchain
uses: dtolnay/rust-toolchain@881ba7bf39a41cda34ac9e123fb41b44ed08232f # stable
- name: Install protobuf compiler
run: |
sudo apt-get update -qq
sudo apt-get install -y -qq protobuf-compiler
- name: Update crate versions
env:
VERSION: ${{ needs.version.outputs.version }}
run: |
# Update workspace package version
sed -i "s/^version = \".*\"/version = \"${VERSION}\"/" Cargo.toml
# Update inter-crate dependency versions to use exact prerelease version
# (semver ranges like ^0.1.0 don't match prereleases)
for crate in inferadb-ledger-types inferadb-ledger-store inferadb-ledger-state inferadb-ledger-raft inferadb-ledger-sdk; do
sed -i "s/\(${crate} = { version = \)\"[^\"]*\"/\1\"=${VERSION}\"/" Cargo.toml
done
- name: Cache Rust dependencies
uses: step-security/rust-cache@9be15b830520fab0ec3939586e917e4855cf76bd # v2.8.3
- name: Authenticate with crates.io
uses: rust-lang/crates-io-auth-action@b7e9a28eded4986ec6b1fa40eeee8f8f165559ec # v1.0.3
id: auth
- name: Publish crates
env:
CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
run: |
for crate in inferadb-ledger-types inferadb-ledger-store inferadb-ledger-state inferadb-ledger-raft inferadb-ledger-sdk; do
echo "Publishing ${crate}..."
cargo publish -p "${crate}" --token "$CARGO_REGISTRY_TOKEN" --allow-dirty
echo "Waiting for crates.io to index..."
sleep 45
done
# Cleanup old nightly releases (keep last 7)
cleanup:
name: Cleanup
runs-on: ubuntu-latest
needs: [sbom-docker]
steps:
- name: Harden the runner (Audit all outbound calls)
uses: step-security/harden-runner@e3f713f2d8f53843e71c69a996d56f51aa9adfb9 # v2.14.1
with:
egress-policy: audit
- name: Checkout code
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Delete old nightly releases
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
# List all nightly releases, sorted by date (newest first)
NIGHTLIES=$(gh release list --limit 100 | grep -E 'nightly\.' | awk '{print $1}')
# Keep the 7 most recent, delete the rest
COUNT=0
for TAG in $NIGHTLIES; do
COUNT=$((COUNT + 1))
if [ $COUNT -gt 7 ]; then
echo "Deleting old nightly release: $TAG"
gh release delete "$TAG" --yes --cleanup-tag || true
fi
done