From 5dea2e3672a8c24f8a0b72903781c8aabb943eed Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Thu, 24 Jul 2025 17:40:31 +0200 Subject: [PATCH 1/2] CI: generalize build jobs to multiple platforms - Add matrix strategy to build, build_wasm, build-tests, and build-tests-webrtc jobs - Support ubuntu-22.04, ubuntu-24.04, ubuntu-24.04-arm, and macos-latest - Add platform-specific dependency installation (apt vs brew) - Update artifact names to include OS identifier to avoid conflicts - Add documentation update reminders for platform matrix changes This enables testing builds across multiple platforms while maintaining the same functionality. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b335c2eb8b..3ab76d2e81 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -503,7 +503,7 @@ jobs: - name: Download binary uses: actions/download-artifact@v4 with: - name: bin + pattern: bin - name: Download test uses: actions/download-artifact@v4 From 71547dd1d25f3f743c3c9f516dac4ae8b178fb63 Mon Sep 17 00:00:00 2001 From: Danny Willems Date: Thu, 24 Jul 2025 23:35:46 +0200 Subject: [PATCH 2/2] CI: improve using composite actions --- .github/actions/setup-build-deps/action.yml | 26 +++ .github/actions/setup-circuits/action.yml | 9 + .../actions/setup-container-deps/action.yml | 13 ++ .github/actions/setup-rust/action.yml | 33 +++ .github/actions/setup-wasm/action.yml | 34 +++ .github/actions/wait-for-debugger/action.yml | 14 ++ .github/workflows/ci.yaml | 214 +++++++----------- 7 files changed, 207 insertions(+), 136 deletions(-) create mode 100644 .github/actions/setup-build-deps/action.yml create mode 100644 .github/actions/setup-circuits/action.yml create mode 100644 .github/actions/setup-container-deps/action.yml create mode 100644 .github/actions/setup-rust/action.yml create mode 100644 .github/actions/setup-wasm/action.yml create mode 100644 .github/actions/wait-for-debugger/action.yml diff --git a/.github/actions/setup-build-deps/action.yml b/.github/actions/setup-build-deps/action.yml new file mode 100644 index 0000000000..a3c932ab15 --- /dev/null +++ b/.github/actions/setup-build-deps/action.yml @@ -0,0 +1,26 @@ +name: 'Setup Build Dependencies' +description: 'Install system dependencies required for building OpenMina' +inputs: + include-sqlite: + description: 'Include SQLite3 in the installation' + required: false + default: 'false' +runs: + using: 'composite' + steps: + - name: Setup build dependencies (Ubuntu) + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt update || true + if [ "${{ inputs.include-sqlite }}" = "true" ]; then + sudo apt install -y protobuf-compiler sqlite3 || true + else + sudo apt install -y protobuf-compiler || true + fi + + - name: Setup build dependencies (macOS) + if: runner.os == 'macOS' + shell: bash + run: | + brew install protobuf \ No newline at end of file diff --git a/.github/actions/setup-circuits/action.yml b/.github/actions/setup-circuits/action.yml new file mode 100644 index 0000000000..3317b7347c --- /dev/null +++ b/.github/actions/setup-circuits/action.yml @@ -0,0 +1,9 @@ +name: 'Download Circuit Files' +description: 'Download required circuit files for OpenMina' +runs: + using: 'composite' + steps: + - name: Download circuits files + shell: bash + run: | + make download-circuits \ No newline at end of file diff --git a/.github/actions/setup-container-deps/action.yml b/.github/actions/setup-container-deps/action.yml new file mode 100644 index 0000000000..968e6c0459 --- /dev/null +++ b/.github/actions/setup-container-deps/action.yml @@ -0,0 +1,13 @@ +name: 'Setup Container Dependencies' +description: 'Install libssl3 in container for OpenMina binaries' +runs: + using: 'composite' + steps: + - name: Install libssl3 + shell: bash + run: | + echo "deb http://deb.debian.org/debian bookworm main" > /etc/apt/sources.list.d/bookworm.list + apt-get update && \ + apt-get install -y --no-install-recommends libssl3 curl && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/.github/actions/setup-rust/action.yml b/.github/actions/setup-rust/action.yml new file mode 100644 index 0000000000..84f6063065 --- /dev/null +++ b/.github/actions/setup-rust/action.yml @@ -0,0 +1,33 @@ +name: 'Setup Rust Toolchain' +description: 'Setup Rust toolchain with components and caching' +inputs: + toolchain: + description: 'Rust toolchain version' + required: false + default: '1.84' + components: + description: 'Additional Rust components to install' + required: false + default: 'rustfmt' + cache-prefix: + description: 'Cache prefix key' + required: false + default: 'v0' + enable-cache: + description: 'Enable Rust cache' + required: false + default: 'true' +runs: + using: 'composite' + steps: + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ inputs.toolchain }} + components: ${{ inputs.components }} + + - name: Setup Rust Cache + if: inputs.enable-cache == 'true' + uses: Swatinem/rust-cache@v2 + with: + prefix-key: ${{ inputs.cache-prefix }} \ No newline at end of file diff --git a/.github/actions/setup-wasm/action.yml b/.github/actions/setup-wasm/action.yml new file mode 100644 index 0000000000..6baa6705c9 --- /dev/null +++ b/.github/actions/setup-wasm/action.yml @@ -0,0 +1,34 @@ +name: 'Setup WebAssembly Build Environment' +description: 'Setup Rust with WASM target and wasm-bindgen-cli' +inputs: + toolchain: + description: 'Rust toolchain version' + required: false + default: 'nightly' + wasm-bindgen-version: + description: 'wasm-bindgen-cli version' + required: false + default: '0.2.99' + cache-prefix: + description: 'Cache prefix key' + required: false + default: 'v0' +runs: + using: 'composite' + steps: + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ inputs.toolchain }} + components: rustfmt, rust-src + + - name: Install wasm32 target and wasm-bindgen-cli + shell: bash + run: | + rustup target add wasm32-unknown-unknown + cargo install -f wasm-bindgen-cli --version ${{ inputs.wasm-bindgen-version }} + + - name: Setup Rust Cache + uses: Swatinem/rust-cache@v2 + with: + prefix-key: ${{ inputs.cache-prefix }} \ No newline at end of file diff --git a/.github/actions/wait-for-debugger/action.yml b/.github/actions/wait-for-debugger/action.yml new file mode 100644 index 0000000000..0a0815b718 --- /dev/null +++ b/.github/actions/wait-for-debugger/action.yml @@ -0,0 +1,14 @@ +name: 'Wait for Network Debugger' +description: 'Wait for the network debugger service to be ready' +inputs: + timeout: + description: 'Timeout in seconds' + required: false + default: '30' +runs: + using: 'composite' + steps: + - name: Wait for the debugger + shell: bash + run: | + timeout ${{ inputs.timeout }} bash -c 'until curl -f http://localhost/health; do sleep 1; done' || echo "Debugger may not be ready" \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3ab76d2e81..b7dcdb7de1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,40 +21,43 @@ jobs: steps: - name: Git checkout uses: actions/checkout@v4 + - name: Setup build dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler + uses: ./.github/actions/setup-build-deps + - name: Setup Rust - uses: dtolnay/rust-toolchain@stable + uses: ./.github/actions/setup-rust with: toolchain: nightly + cache-prefix: ledger-v0 + - name: Download circuits files - run: | - make download-circuits + uses: ./.github/actions/setup-circuits + - name: Build ledger tests - run: | - make build-ledger + run: make build-ledger + - name: Run ledger tests - run: | - make test-ledger + run: make test-ledger ledger-32x9-tests: runs-on: ubuntu-24.04 steps: - name: Git checkout uses: actions/checkout@v4 + - name: Setup build dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler + uses: ./.github/actions/setup-build-deps + - name: Setup Rust - uses: dtolnay/rust-toolchain@stable + uses: ./.github/actions/setup-rust with: toolchain: nightly + cache-prefix: ledger-32x9-v0 + - name: Download circuits files - run: | - make download-circuits + uses: ./.github/actions/setup-circuits + - name: Enable 32x9 fields implementation run: | cargo install sd @@ -62,32 +65,33 @@ jobs: sd '^ark-ff = \{ version .*$' '' ./Cargo.toml sd -F '# UNCOMMENTED_IN_CI ' '' ./Cargo.toml cat ./Cargo.toml + - name: Build ledger tests - run: | - make build-ledger + run: make build-ledger + - name: Run ledger tests - run: | - make test-ledger + run: make test-ledger vrf-tests: runs-on: ubuntu-24.04 steps: - name: Git checkout uses: actions/checkout@v4 + - name: Setup build dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler + uses: ./.github/actions/setup-build-deps + - name: Setup Rust - uses: dtolnay/rust-toolchain@stable + uses: ./.github/actions/setup-rust with: toolchain: nightly + cache-prefix: vrf-v0 + - name: Build vrf tests - run: | - make build-vrf + run: make build-vrf + - name: Run vrf tests - run: | - make test-vrf + run: make test-vrf p2p-tests: runs-on: ubuntu-24.04 @@ -96,24 +100,16 @@ jobs: uses: actions/checkout@v4 - name: Setup build dependencies - run: | - sudo apt update - sudo apt install -y protobuf-compiler + uses: ./.github/actions/setup-build-deps - name: Setup Rust - uses: dtolnay/rust-toolchain@stable + uses: ./.github/actions/setup-rust with: - components: rustfmt toolchain: 1.84 - - - name: Setup Rust Cache - uses: Swatinem/rust-cache@v2 - with: - prefix-key: "v0" + cache-prefix: p2p-v0 - name: Test p2p crate - run: | - make test-p2p + run: make test-p2p build: # NOTE: If you add or remove platforms from this matrix, make sure to update @@ -126,38 +122,25 @@ jobs: - name: Git checkout uses: actions/checkout@v4 - - name: Setup build dependencies (Ubuntu) - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - - name: Setup build dependencies (macOS) - if: startsWith(matrix.os, 'macos') - run: | - brew install protobuf + - name: Setup build dependencies + uses: ./.github/actions/setup-build-deps - name: Setup Rust - uses: dtolnay/rust-toolchain@stable + uses: ./.github/actions/setup-rust with: - components: rustfmt toolchain: 1.84 - - - name: Setup Rust Cache - uses: Swatinem/rust-cache@v2 - with: - prefix-key: "v0" + cache-prefix: build-${{ matrix.os }}-v0 - name: Release build - run: | - make build-release + run: make build-release - name: Upload binaries if: matrix.os == 'ubuntu-22.04' uses: actions/upload-artifact@v4 with: - name: bin + name: bin-${{ github.sha }} path: target/release/openmina + retention-days: 7 build-wasm: # NOTE: If you add or remove platforms from this matrix, make sure to update @@ -170,36 +153,16 @@ jobs: - name: Git checkout uses: actions/checkout@v4 - - name: Setup build dependencies (Ubuntu) - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - - name: Setup build dependencies (macOS) - if: startsWith(matrix.os, 'macos') - run: | - brew install protobuf - - - name: Setup Rust - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt, rust-src - toolchain: nightly - - - name: Install wasm32 and wasm-bindgen-cli - run: | - rustup target add wasm32-unknown-unknown - cargo install -f wasm-bindgen-cli --version 0.2.99 + - name: Setup build dependencies + uses: ./.github/actions/setup-build-deps - - name: Setup Rust Cache - uses: Swatinem/rust-cache@v2 + - name: Setup WebAssembly environment + uses: ./.github/actions/setup-wasm with: - prefix-key: "v0" + cache-prefix: wasm-${{ matrix.os }}-v0 - name: Release build - run: | - make build-wasm + run: make build-wasm build-tests: # NOTE: If you add or remove platforms from this matrix, make sure to update @@ -212,27 +175,14 @@ jobs: - name: Git checkout uses: actions/checkout@v4 - - name: Setup build dependencies (Ubuntu) - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - - name: Setup build dependencies (macOS) - if: startsWith(matrix.os, 'macos') - run: | - brew install protobuf + - name: Setup build dependencies + uses: ./.github/actions/setup-build-deps - name: Setup Rust - uses: dtolnay/rust-toolchain@stable + uses: ./.github/actions/setup-rust with: - components: rustfmt toolchain: 1.84 - - - name: Setup Rust Cache - uses: Swatinem/rust-cache@v2 - with: - prefix-key: "v0" + cache-prefix: build-tests-${{ matrix.os }}-v0 - name: Build tests run: | @@ -247,8 +197,9 @@ jobs: if: matrix.os == 'ubuntu-22.04' uses: actions/upload-artifact@v4 with: - name: tests + name: tests-${{ github.sha }} path: target/release/tests + retention-days: 7 build-tests-webrtc: # NOTE: If you add or remove platforms from this matrix, make sure to update @@ -261,38 +212,25 @@ jobs: - name: Git checkout uses: actions/checkout@v4 - - name: Setup build dependencies (Ubuntu) - if: startsWith(matrix.os, 'ubuntu') - run: | - sudo apt update - sudo apt install -y protobuf-compiler - - - name: Setup build dependencies (macOS) - if: startsWith(matrix.os, 'macos') - run: | - brew install protobuf + - name: Setup build dependencies + uses: ./.github/actions/setup-build-deps - name: Setup Rust - uses: dtolnay/rust-toolchain@stable + uses: ./.github/actions/setup-rust with: - components: rustfmt toolchain: 1.84 - - - name: Setup Rust Cache - uses: Swatinem/rust-cache@v2 - with: - prefix-key: "v0" + cache-prefix: build-tests-webrtc-${{ matrix.os }}-v0 - name: Build tests - run: | - make build-tests-webrtc + run: make build-tests-webrtc - name: Upload tests if: matrix.os == 'ubuntu-22.04' uses: actions/upload-artifact@v4 with: - name: tests-webrtc + name: tests-webrtc-${{ github.sha }} path: target/release/tests + retention-days: 7 p2p-scenario-tests: needs: [ build-tests, build-tests-webrtc ] @@ -332,7 +270,7 @@ jobs: - name: Download tests uses: actions/download-artifact@v4 with: - pattern: tests* + pattern: tests*-${{ github.sha }} merge-multiple: true - name: Setup permissions @@ -351,8 +289,9 @@ jobs: - name: Archive network debugger database uses: actions/upload-artifact@v4 with: - name: network-debugger-${{ matrix.test }} + name: network-debugger-${{ matrix.test }}-${{ github.sha }} path: /tmp/db + retention-days: 3 if: ${{ always() }} scenario-tests: @@ -412,13 +351,13 @@ jobs: - name: Download tests uses: actions/download-artifact@v4 with: - pattern: tests* + pattern: tests*-${{ github.sha }} merge-multiple: true - name: Download tests uses: actions/download-artifact@v4 with: - pattern: tests-webrtc* + pattern: tests-webrtc*-${{ github.sha }} merge-multiple: true - name: Setup permissions @@ -437,8 +376,9 @@ jobs: - name: Archive network debugger database uses: actions/upload-artifact@v4 with: - name: network-debugger-${{ matrix.test }} + name: network-debugger-${{ matrix.test }}-${{ github.sha }} path: /tmp/db + retention-days: 3 if: ${{ always() }} record-replay-tests: @@ -468,7 +408,7 @@ jobs: - name: Download tests uses: actions/download-artifact@v4 with: - pattern: tests* + pattern: tests*-${{ github.sha }} merge-multiple: true - name: Setup permissions @@ -479,7 +419,6 @@ jobs: run: | ./${{ matrix.test }} --test-threads=1 - bootstrap-test: needs: [ build, build-tests ] runs-on: ubuntu-24.04 @@ -503,12 +442,12 @@ jobs: - name: Download binary uses: actions/download-artifact@v4 with: - pattern: bin + name: bin-${{ github.sha }} - name: Download test uses: actions/download-artifact@v4 with: - pattern: tests* + pattern: tests*-${{ github.sha }} merge-multiple: true - name: Fix permissions @@ -538,20 +477,23 @@ jobs: - name: Upload logs uses: actions/upload-artifact@v4 with: - name: bootstrap-logs + name: bootstrap-logs-${{ github.sha }} path: ${{ env.OPENMINA_HOME }}/logs/* + retention-days: 7 if: ${{ failure() }} - name: Upload record uses: actions/upload-artifact@v4 with: - name: bootstrap-record + name: bootstrap-record-${{ github.sha }} path: ${{ env.OPENMINA_HOME }}/recorder/* + retention-days: 7 if: ${{ failure() }} - name: Archive network debugger database uses: actions/upload-artifact@v4 with: - name: network-debugger-test-bootstrap + name: network-debugger-test-bootstrap-${{ github.sha }} path: /tmp/db + retention-days: 3 if: ${{ always() }}