BM-244: Docker compose builds in CI (github#68)

This PR:
* Runs docker compose builds in CI to validate our docker builds work
before merging
* Passes through the sccache s3 caching creds into the docker build
context to make sure docker builds are not slow in CI
* Only triggers docker builds if compose / dockerfiles change to improve
CI perf

Author: mothran

.github/actions/bininstall-risc0/action.yml

@@ -20,7 +20,7 @@ runs:
     - name: install rust
       uses: risc0/risc0/.github/actions/[email protected]
 
-    - uses: risc0/cargo-install@v3
+    - uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3
       with:
         crate: cargo-binstall
         version: '=1.10.8'

.github/workflows/main.yml

@@ -12,6 +12,7 @@ concurrency:
 permissions:
   id-token: write
   contents: read
+  pull-requests: read
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -58,7 +59,7 @@ jobs:
           python-version: "3.10"
 
       - name: install foundry
-        uses: risc0/foundry-toolchain@2fe7e70b520f62368a0e3c464f997df07ede420f
+        uses: foundry-rs/foundry-toolchain@v1
 
       - name: install rust
         uses: risc0/risc0/.github/actions/[email protected]
@@ -75,7 +76,7 @@ jobs:
           toolchain-version: 'r0.1.79.0-2'
 
       - name: install cargo-sort
-        uses: risc0/cargo-install@v3
+        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3
         with:
           crate: cargo-sort
           version: "=1.0.9"
@@ -98,7 +99,7 @@ jobs:
         run: cargo check
 
       - name: Install sqlx-cli
-        uses: risc0/cargo-install@v3
+        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3
         with:
           crate: sqlx-cli
           version: '=0.8.2'
@@ -133,7 +134,7 @@ jobs:
           restore-keys: cache-lychee-
 
       - name: install cargo-binstall
-        uses: risc0/cargo-install@v3
+        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3
         with:
           crate: cargo-binstall
           version: '=1.10.8'
@@ -159,7 +160,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: install cargo-binstall
-        uses: risc0/cargo-install@v3
+        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3
         with:
           crate: cargo-binstall
           version: '=1.10.8'
@@ -178,7 +179,7 @@ jobs:
           submodules: recursive
 
       - name: install foundry
-        uses: risc0/foundry-toolchain@2fe7e70b520f62368a0e3c464f997df07ede420f
+        uses: foundry-rs/foundry-toolchain@v1
 
       - name: forge fmt
         run: forge fmt --check
@@ -203,7 +204,7 @@ jobs:
           submodules: recursive
 
       - name: install foundry
-        uses: risc0/foundry-toolchain@2fe7e70b520f62368a0e3c464f997df07ede420f
+        uses: foundry-rs/foundry-toolchain@v1
 
       - name: install rust
         uses: risc0/risc0/.github/actions/[email protected]
@@ -220,7 +221,7 @@ jobs:
           toolchain-version: 'r0.1.79.0-2'
 
       - name: install cargo-sort
-        uses: risc0/cargo-install@v3
+        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3
         with:
           crate: cargo-sort
           version: "=1.0.9"
@@ -265,8 +266,27 @@ jobs:
       - name: sccache stats
         run: sccache --show-stats
 
-  docker:
+  files-changed:
     runs-on: ubuntu-latest
+    outputs:
+      docker: ${{ steps.changes.outputs.docker }}
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v4
+
+      - uses: dorny/[email protected]
+        id: changes
+        with:
+          filters: |
+            src:
+              - 'dockerfiles/**'
+              - 'compose.yml'
+              - '.env-compose'
+
+  docker:
+    runs-on: [self-hosted, Linux, X64, prod, cpu]
+    needs: files-changed
+    if: needs.files-changed.outputs.docker == 'true'
     steps:
       - name: checkout code
         uses: actions/checkout@v4
@@ -275,3 +295,24 @@ jobs:
 
       - name: docker-compose lint
         run: docker compose --profile broker --env-file ./.env-compose config
+
+      - name: Fetch CI AWS Creds
+        id: aws-creds
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: 'us-west-2'
+          role-to-assume: 'arn:aws:iam::083632199359:role/gha_oidc_risc0_cache_shared_access'
+          output-credentials: true
+
+      - name: create ci creds file
+        run: |
+          echo "[default]" > ./dockerfiles/ci-cache-creds.txt
+          echo "aws_access_key_id=${{ steps.aws-creds.outputs.aws-access-key-id }}" >> ./dockerfiles/ci-cache-creds.txt && \
+          echo "aws_secret_access_key=${{ steps.aws-creds.outputs.aws-secret-access-key }}" >> ./dockerfiles/ci-cache-creds.txt && \
+          echo "aws_session_token=${{ steps.aws-creds.outputs.aws-session-token }}" >> ./dockerfiles/ci-cache-creds.txt
+
+      - name: Setup docker builder
+        run: docker buildx create --driver docker-container --use
+
+      - name: docker compose build
+        run: docker compose --profile broker --env-file ./.env-compose -f compose.yml -f ./dockerfiles/compose.ci.yml build

crates/boundless-market/src/contracts/mod.rs

@@ -27,6 +27,9 @@ use url::Url;
 
 use risc0_zkvm::sha::Digest;
 
+#[cfg(not(target_os = "zkvm"))]
+pub use risc0_ethereum_contracts::encode_seal;
+
 #[cfg(not(target_os = "zkvm"))]
 const TXN_CONFIRM_TIMEOUT: Duration = Duration::from_secs(45);
 
@@ -412,33 +415,6 @@ pub fn eip712_domain(addr: Address, chain_id: u64) -> EIP721DomainSaltless {
     }
 }
 
-// TODO: when upgrading to risc0-ethereum-contracts 1.1.0 this function will be removed.
-pub fn encode_seal(receipt: &risc0_zkvm::Receipt) -> anyhow::Result<Vec<u8>> {
-    use risc0_zkvm::sha::Digestible;
-
-    let seal = match receipt.inner.clone() {
-        risc0_zkvm::InnerReceipt::Fake(receipt) => {
-            let seal = receipt.claim.digest().as_bytes().to_vec();
-            let selector = &[0u8; 4];
-            // Create a new vector with the capacity to hold both selector and seal
-            let mut selector_seal = Vec::with_capacity(selector.len() + seal.len());
-            selector_seal.extend_from_slice(selector);
-            selector_seal.extend_from_slice(&seal);
-            selector_seal
-        }
-        risc0_zkvm::InnerReceipt::Groth16(receipt) => {
-            let selector = &receipt.verifier_parameters.as_bytes()[..4];
-            // Create a new vector with the capacity to hold both selector and seal
-            let mut selector_seal = Vec::with_capacity(selector.len() + receipt.seal.len());
-            selector_seal.extend_from_slice(selector);
-            selector_seal.extend_from_slice(receipt.seal.as_ref());
-            selector_seal
-        }
-        _ => anyhow::bail!("Unsupported receipt type"),
-    };
-    Ok(seal)
-}
-
 #[cfg(feature = "test-utils")]
 pub mod test_utils {
     use aggregation_set::SET_BUILDER_GUEST_ID;

crates/boundless-market/src/contracts/proof_market.rs

@@ -623,8 +623,8 @@ mod tests {
 
     use super::ProofMarketService;
     use crate::contracts::{
-        encode_seal, test_utils::TestCtx, AssessorJournal, Fulfillment, IProofMarket, Input,
-        InputType, Offer, Predicate, PredicateType, ProofStatus, ProvingRequest, Requirements,
+        test_utils::TestCtx, AssessorJournal, Fulfillment, IProofMarket, Input, InputType, Offer,
+        Predicate, PredicateType, ProofStatus, ProvingRequest, Requirements,
     };
     use aggregation_set::{merkle_root, GuestOutput, SetInclusionReceipt, SET_BUILDER_GUEST_ID};
     use alloy::{
@@ -639,6 +639,7 @@ mod tests {
     };
     use guest_assessor::ASSESSOR_GUEST_ID;
     use guest_util::ECHO_ID;
+    use risc0_ethereum_contracts::encode_seal;
     use risc0_zkvm::{
         sha::{Digest, Digestible},
         FakeReceipt, InnerReceipt, Journal, MaybePruned, Receipt, ReceiptClaim,

crates/broker/src/config.rs

@@ -231,7 +231,6 @@ impl ConfigWatcher {
             watcher
                 .watch(&config_path_copy, notify::RecursiveMode::NonRecursive)
                 .context("Failed to start watcher")?;
-
             startup_notification_copy.notify_one();
 
             while let Some(event) = rx.recv().await {

dockerfiles/agent.dockerfile

@@ -18,7 +18,7 @@ ENV RUSTUP_HOME=/usr/local/rustup \
 # Install rust and a target rust version (should match rust-toolchain.toml for best speed)
 RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
 RUN chmod -R a+w $RUSTUP_HOME $CARGO_HOME
-RUN rustup install 1.76
+RUN rustup install 1.79
 
 FROM rust-builder AS builder
 
@@ -46,16 +46,14 @@ SHELL ["/bin/bash", "-c"]
 # Prevent sccache collision in compose-builds
 ENV SCCACHE_SERVER_PORT=4226
 
-# This downloads and setups the rust-toolchain so docker can cache the layer
-RUN cargo
-
-
 RUN \
+    --mount=type=secret,id=ci_cache_creds,target=/root/.aws/credentials \
     --mount=type=cache,target=/root/.cache/sccache/,id=bndlss_agent_sc \
     source ./sccache-config.sh && \
     ls /root/.cache/sccache/ && \
     cargo build --release -F cuda -p workflow --bin agent && \
-    cp /src/target/release/agent /src/agent
+    cp /src/target/release/agent /src/agent && \
+    sccache --show-stats
 
 FROM risczero/risc0-groth16-prover:v2024-05-17.1 AS binaries
 

dockerfiles/broker.dockerfile

@@ -15,15 +15,15 @@ RUN curl -L https://foundry.paradigm.xyz | bash && \
 # Prevent sccache collision in compose-builds
 ENV SCCACHE_SERVER_PORT=4227
 
-RUN cargo
-
 RUN \
+    --mount=type=secret,id=ci_cache_creds,target=/root/.aws/credentials \
     --mount=type=cache,target=/root/.cache/sccache/,id=bndlss_broker_sc \
     source ./sccache-config.sh && \
     ls /root/.cache/sccache/ && \
     cargo install --version 1.6.9 cargo-binstall && \
     cargo binstall -y --force cargo-risczero --version 1.1 && \
-    cargo risczero install
+    cargo risczero install && \
+    sccache --show-stats
 
 FROM init AS builder
 
@@ -41,17 +41,17 @@ COPY foundry.toml .
 ENV PATH="$PATH:/root/.foundry/bin"
 RUN forge build
 
-RUN cargo
-
 # Prevent sccache collision in compose-builds
 ENV SCCACHE_SERVER_PORT=4227
 
 RUN \
+    --mount=type=secret,id=ci_cache_creds,target=/root/.aws/credentials \
     --mount=type=cache,target=/root/.cache/sccache/,id=bndlss_broker_sc \
     source /sccache-config.sh && \
     ls /root/.cache/sccache/ && \
     cargo build --release --bin broker && \
-    cp /src/target/release/broker /src/broker
+    cp /src/target/release/broker /src/broker && \
+    sccache --show-stats
 
 FROM rust:1.79.0-bookworm AS runtime
 

dockerfiles/ci-cache-creds.txt

@@ -0,0 +1,33 @@
+services:
+  exec_agent:
+    build:
+      secrets:
+        - ci_cache_creds
+      cache_to:
+        - type=s3,name=agent-cache,manifests_prefix=shared/boundless/docker/manifests/,blobs_prefix=shared/boundless/docker/blobs/,region=us-west-2,bucket=risc0-ci-cache,mode=max
+      cache_from:
+        - type=s3,name=agent-cache,manifests_prefix=shared/boundless/docker/manifests/,blobs_prefix=shared/boundless/docker/blobs/,region=us-west-2,bucket=risc0-ci-cache
+
+  rest_api:
+    build:
+      secrets:
+        - ci_cache_creds
+      cache_to:
+        - type=s3,name=rest-cache,manifests_prefix=shared/boundless/docker/manifests/,blobs_prefix=shared/boundless/docker/blobs/,region=us-west-2,bucket=risc0-ci-cache,mode=max
+      cache_from:
+        - type=s3,name=rest-cache,manifests_prefix=shared/boundless/docker/manifests/,blobs_prefix=shared/boundless/docker/blobs/,region=us-west-2,bucket=risc0-ci-cache
+
+  broker:
+    build:
+      secrets:
+        - ci_cache_creds
+      cache_to:
+        - type=s3,name=broker-cache,manifests_prefix=shared/boundless/docker/manifests/,blobs_prefix=shared/boundless/docker/blobs/,region=us-west-2,bucket=risc0-ci-cache,mode=max
+      cache_from:
+        - type=s3,name=broker-cache,manifests_prefix=shared/boundless/docker/manifests/,blobs_prefix=shared/boundless/docker/blobs/,region=us-west-2,bucket=risc0-ci-cache
+
+secrets:
+  # Optional s3 credentials file, used in risc0 CI for sccache'ing inside docker builds.
+  # unused in local builds, see sccache-config.sh for details.
+  ci_cache_creds:
+    file: ./dockerfiles/ci-cache-creds.txt

dockerfiles/compose.ci.yml

@@ -19,14 +19,13 @@ SHELL ["/bin/bash", "-c"]
 # Prevent sccache collision in compose-builds
 ENV SCCACHE_SERVER_PORT=4228
 
-RUN cargo
-
 RUN \
+    --mount=type=secret,id=ci_cache_creds,target=/root/.aws/credentials \
     --mount=type=cache,target=/root/.cache/sccache/,id=bndlss_api_sccache \
     source ./sccache-config.sh && \
     cargo build --release -p api --bin rest_api && \
-    cp /src/target/release/rest_api /src/rest_api
-
+    cp /src/target/release/rest_api /src/rest_api && \
+    sccache --show-stats
 
 FROM rust:1.79.0-bookworm AS runtime