Skip to content

Commit 97de988

Browse files
colinlyguoVelacielalispc
authored
feat: coordinator and prover support v0.4.2 (#1660)
Co-authored-by: Velaciela <[email protected]> Co-authored-by: colinlyguo <[email protected]> Co-authored-by: Zhang Zhuo <[email protected]>
1 parent a12175d commit 97de988

File tree

14 files changed

+1860
-625
lines changed

14 files changed

+1860
-625
lines changed

‎.github/workflows/common.yml‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -42,15 +42,11 @@ jobs:
4242
uses: Swatinem/rust-cache@v2
4343
with:
4444
workspaces: "common/libzkp/impl -> target"
45-
- name: Setup SSH for private repos
46-
uses: webfactory/[email protected]
47-
with:
48-
ssh-private-key: ${{ secrets.OPENVM_GPU_SSH_PRIVATE_KEY }}
49-
- name: Lint
50-
working-directory: 'common'
51-
run: |
52-
rm -rf $HOME/.cache/golangci-lint
53-
make lint
45+
# - name: Lint
46+
# working-directory: 'common'
47+
# run: |
48+
# rm -rf $HOME/.cache/golangci-lint
49+
# make lint
5450
goimports-lint:
5551
if: github.event.pull_request.draft == false
5652
runs-on: ubuntu-latest

‎.github/workflows/docker.yml‎

Lines changed: 41 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -307,13 +307,48 @@ jobs:
307307
REPOSITORY: coordinator-api
308308
run: |
309309
aws --region ${{ env.AWS_REGION }} ecr describe-repositories --repository-names ${{ env.REPOSITORY }} && : || aws --region ${{ env.AWS_REGION }} ecr create-repository --repository-name ${{ env.REPOSITORY }}
310-
- name: Setup SSH for private repos
311-
uses: webfactory/[email protected]
312-
with:
313-
ssh-private-key: ${{ secrets.OPENVM_GPU_SSH_PRIVATE_KEY }}
314-
- name: Run custom script
310+
- name: Setup SSH for repositories and clone them
311+
run: |
312+
mkdir -p ~/.ssh
313+
chmod 700 ~/.ssh
314+
315+
# Setup for plonky3-gpu
316+
echo "${{ secrets.PLONKY3_GPU_SSH_PRIVATE_KEY }}" > ~/.ssh/plonky3_gpu_key
317+
chmod 600 ~/.ssh/plonky3_gpu_key
318+
eval "$(ssh-agent -s)" > /dev/null
319+
ssh-add ~/.ssh/plonky3_gpu_key 2>/dev/null
320+
ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts 2>/dev/null
321+
echo "Loaded plonky3-gpu key"
322+
323+
# Clone plonky3-gpu repository
324+
./build/dockerfiles/coordinator-api/clone_plonky3_gpu.sh
325+
326+
# Setup for openvm-stark-gpu
327+
echo "${{ secrets.OPENVM_STARK_GPU_SSH_PRIVATE_KEY }}" > ~/.ssh/openvm_stark_gpu_key
328+
chmod 600 ~/.ssh/openvm_stark_gpu_key
329+
eval "$(ssh-agent -s)" > /dev/null
330+
ssh-add ~/.ssh/openvm_stark_gpu_key 2>/dev/null
331+
echo "Loaded openvm-stark-gpu key"
332+
333+
# Clone openvm-stark-gpu repository
334+
./build/dockerfiles/coordinator-api/clone_openvm_stark_gpu.sh
335+
336+
# Setup for openvm-gpu
337+
echo "${{ secrets.OPENVM_GPU_SSH_PRIVATE_KEY }}" > ~/.ssh/openvm_gpu_key
338+
chmod 600 ~/.ssh/openvm_gpu_key
339+
eval "$(ssh-agent -s)" > /dev/null
340+
ssh-add ~/.ssh/openvm_gpu_key 2>/dev/null
341+
echo "Loaded openvm-gpu key"
342+
343+
# Clone openvm-gpu repository
344+
./build/dockerfiles/coordinator-api/clone_openvm_gpu.sh
345+
346+
# Show number of loaded keys
347+
echo "Number of loaded keys: $(ssh-add -l | wc -l)"
348+
349+
- name: Checkout specific commits
315350
run: |
316-
./build/dockerfiles/coordinator-api/init-openvm.sh
351+
./build/dockerfiles/coordinator-api/checkout_all.sh
317352
- name: Build and push
318353
uses: docker/build-push-action@v3
319354
env:

‎build/dockerfiles/coordinator-api.Dockerfile‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Build libzkp dependency
2-
FROM scrolltech/go-rust-builder:go-1.22-rust-nightly-2023-12-03 as chef
2+
FROM scrolltech/cuda-go-rust-builder:cuda-11.7.1-go-1.21-rust-nightly-2023-12-03 as chef
33
WORKDIR app
44

55
FROM chef as planner
@@ -9,7 +9,9 @@ RUN cargo chef prepare --recipe-path recipe.json
99
FROM chef as zkp-builder
1010
COPY ./common/libzkp/impl/rust-toolchain ./
1111
COPY --from=planner /app/recipe.json recipe.json
12-
# run ./build/dockerfiles/coordinator-api/init-openvm.sh to get openvm-gpu
12+
# run scripts to get openvm-gpu
13+
COPY ./build/dockerfiles/coordinator-api/plonky3-gpu /plonky3-gpu
14+
COPY ./build/dockerfiles/coordinator-api/openvm-stark-gpu /openvm-stark-gpu
1315
COPY ./build/dockerfiles/coordinator-api/openvm-gpu /openvm-gpu
1416
COPY ./build/dockerfiles/coordinator-api/gitconfig /root/.gitconfig
1517
COPY ./build/dockerfiles/coordinator-api/config.toml /root/.cargo/config.toml
@@ -20,7 +22,7 @@ RUN cargo build --release
2022

2123

2224
# Download Go dependencies
23-
FROM scrolltech/go-rust-builder:go-1.21-rust-nightly-2023-12-03 as base
25+
FROM scrolltech/cuda-go-rust-builder:cuda-11.7.1-go-1.21-rust-nightly-2023-12-03 as base
2426
WORKDIR /src
2527
COPY go.work* ./
2628
COPY ./rollup/go.* ./rollup/
@@ -40,7 +42,7 @@ COPY --from=zkp-builder /app/target/release/libzkp.so ./coordinator/internal/log
4042
RUN cd ./coordinator && CGO_LDFLAGS="-Wl,--no-as-needed -ldl" make coordinator_api_skip_libzkp && mv ./build/bin/coordinator_api /bin/coordinator_api && mv internal/logic/verifier/lib /bin/
4143

4244
# Pull coordinator into a second stage deploy ubuntu container
43-
FROM ubuntu:20.04
45+
FROM nvidia/cuda:11.7.1-runtime-ubuntu22.04
4446
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/src/coordinator/internal/logic/verifier/lib
4547
ENV CGO_LDFLAGS="-Wl,--no-as-needed -ldl"
4648
# ENV CHAIN_ID=534353

‎build/dockerfiles/coordinator-api/checkout_all.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
set -uex
3+
4+
PLONKY3_GPU_COMMIT=261b322 # v0.2.0
5+
OPENVM_STARK_GPU_COMMIT=3082234 # PR#48
6+
OPENVM_GPU_COMMIT=8094b4f # branch: patch-v1.2.0
7+
8+
DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)
9+
10+
# checkout plonky3-gpu
11+
cd $DIR/plonky3-gpu && git checkout ${PLONKY3_GPU_COMMIT}
12+
13+
# checkout openvm-stark-gpu
14+
cd $DIR/openvm-stark-gpu && git checkout ${OPENVM_STARK_GPU_COMMIT}
15+
16+
# checkout openvm-gpu
17+
cd $DIR/openvm-gpu && git checkout ${OPENVM_GPU_COMMIT}

build/dockerfiles/coordinator-api/init-openvm.sh renamed to build/dockerfiles/coordinator-api/clone_openvm_gpu.sh

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,10 @@
11
#!/bin/bash
22
set -uex
33

4-
OPENVM_GPU_COMMIT=dfa10b4
5-
64
DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)
75

8-
# checkout openvm-gpu
6+
# clone openvm-gpu if not exists
97
if [ ! -d $DIR/openvm-gpu ]; then
108
git clone [email protected]:scroll-tech/openvm-gpu.git $DIR/openvm-gpu
119
fi
12-
cd $DIR/openvm-gpu && git fetch && git checkout ${OPENVM_GPU_COMMIT}
10+
cd $DIR/openvm-gpu && git fetch --all --force

‎build/dockerfiles/coordinator-api/clone_openvm_stark_gpu.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
set -uex
3+
4+
DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)
5+
6+
# clone openvm-stark-gpu if not exists
7+
if [ ! -d $DIR/openvm-stark-gpu ]; then
8+
git clone [email protected]:scroll-tech/openvm-stark-gpu.git $DIR/openvm-stark-gpu
9+
fi
10+
cd $DIR/openvm-stark-gpu && git fetch --all --force

‎build/dockerfiles/coordinator-api/clone_plonky3_gpu.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
set -uex
3+
4+
DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" > /dev/null 2>&1 && pwd)
5+
6+
# clone plonky3-gpu if not exists
7+
if [ ! -d $DIR/plonky3-gpu ]; then
8+
git clone [email protected]:scroll-tech/plonky3-gpu.git $DIR/plonky3-gpu
9+
fi
10+
cd $DIR/plonky3-gpu && git fetch --all --force

‎build/dockerfiles/coordinator-api/config.toml‎

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,74 @@ openvm-native-transpiler = { path = "/openvm-gpu/extensions/native/transpiler",
1919
openvm-pairing-guest = { path = "/openvm-gpu/extensions/pairing/guest", default-features = false }
2020
openvm-rv32im-guest = { path = "/openvm-gpu/extensions/rv32im/guest", default-features = false }
2121
openvm-rv32im-transpiler = { path = "/openvm-gpu/extensions/rv32im/transpiler", default-features = false }
22-
openvm-sdk = { path = "/openvm-gpu/crates/sdk", default-features = false, features = ["parallel", "bench-metrics"] }
22+
openvm-sdk = { path = "/openvm-gpu/crates/sdk", default-features = false, features = ["parallel", "bench-metrics", "evm-prove"] }
2323
openvm-sha256-guest = { path = "/openvm-gpu/extensions/sha256/guest", default-features = false }
2424
openvm-transpiler = { path = "/openvm-gpu/crates/toolchain/transpiler", default-features = false }
25+
26+
# stark-backend
27+
[patch."https://github.com/openvm-org/stark-backend.git"]
28+
openvm-stark-backend = { path = "/openvm-stark-gpu/crates/stark-backend", features = ["gpu"] }
29+
openvm-stark-sdk = { path = "/openvm-stark-gpu/crates/stark-sdk", features = ["gpu"] }
30+
31+
[patch."ssh://git@github.com/scroll-tech/openvm-stark-gpu.git"]
32+
openvm-stark-backend = { path = "/openvm-stark-gpu/crates/stark-backend", features = ["gpu"] }
33+
openvm-stark-sdk = { path = "/openvm-stark-gpu/crates/stark-sdk", features = ["gpu"] }
34+
35+
# plonky3
36+
[patch."https://github.com/Plonky3/Plonky3.git"]
37+
p3-air = { path = "/plonky3-gpu/air" }
38+
p3-field = { path = "/plonky3-gpu/field" }
39+
p3-commit = { path = "/plonky3-gpu/commit" }
40+
p3-matrix = { path = "/plonky3-gpu/matrix" }
41+
p3-baby-bear = { path = "/plonky3-gpu/baby-bear" }
42+
p3-koala-bear = { path = "/plonky3-gpu/koala-bear" }
43+
p3-util = { path = "/plonky3-gpu/util" }
44+
p3-challenger = { path = "/plonky3-gpu/challenger" }
45+
p3-dft = { path = "/plonky3-gpu/dft" }
46+
p3-fri = { path = "/plonky3-gpu/fri" }
47+
p3-goldilocks = { path = "/plonky3-gpu/goldilocks" }
48+
p3-keccak = { path = "/plonky3-gpu/keccak" }
49+
p3-keccak-air = { path = "/plonky3-gpu/keccak-air" }
50+
p3-blake3 = { path = "/plonky3-gpu/blake3" }
51+
p3-mds = { path = "/plonky3-gpu/mds" }
52+
p3-monty-31 = { path = "/plonky3-gpu/monty-31" }
53+
p3-merkle-tree = { path = "/plonky3-gpu/merkle-tree" }
54+
p3-poseidon = { path = "/plonky3-gpu/poseidon" }
55+
p3-poseidon2 = { path = "/plonky3-gpu/poseidon2" }
56+
p3-poseidon2-air = { path = "/plonky3-gpu/poseidon2-air" }
57+
p3-symmetric = { path = "/plonky3-gpu/symmetric" }
58+
p3-uni-stark = { path = "/plonky3-gpu/uni-stark" }
59+
p3-maybe-rayon = { path = "/plonky3-gpu/maybe-rayon" }
60+
p3-bn254-fr = { path = "/plonky3-gpu/bn254-fr" }
61+
62+
# gpu crates
63+
[patch."ssh://git@github.com/scroll-tech/plonky3-gpu.git"]
64+
p3-gpu-base = { path = "/plonky3-gpu/gpu-base" }
65+
p3-gpu-build = { path = "/plonky3-gpu/gpu-build" }
66+
p3-gpu-field = { path = "/plonky3-gpu/gpu-field" }
67+
p3-gpu-backend = { path = "/plonky3-gpu/gpu-backend" }
68+
p3-gpu-module = { path = "/plonky3-gpu/gpu-module" }
69+
p3-air = { path = "/plonky3-gpu/air" }
70+
p3-field = { path = "/plonky3-gpu/field" }
71+
p3-commit = { path = "/plonky3-gpu/commit" }
72+
p3-matrix = { path = "/plonky3-gpu/matrix" }
73+
p3-baby-bear = { path = "/plonky3-gpu/baby-bear" }
74+
p3-koala-bear = { path = "/plonky3-gpu/koala-bear" }
75+
p3-util = { path = "/plonky3-gpu/util" }
76+
p3-challenger = { path = "/plonky3-gpu/challenger" }
77+
p3-dft = { path = "/plonky3-gpu/dft" }
78+
p3-fri = { path = "/plonky3-gpu/fri" }
79+
p3-goldilocks = { path = "/plonky3-gpu/goldilocks" }
80+
p3-keccak = { path = "/plonky3-gpu/keccak" }
81+
p3-keccak-air = { path = "/plonky3-gpu/keccak-air" }
82+
p3-blake3 = { path = "/plonky3-gpu/blake3" }
83+
p3-mds = { path = "/plonky3-gpu/mds" }
84+
p3-monty-31 = { path = "/plonky3-gpu/monty-31" }
85+
p3-merkle-tree = { path = "/plonky3-gpu/merkle-tree" }
86+
p3-poseidon = { path = "/plonky3-gpu/poseidon" }
87+
p3-poseidon2 = { path = "/plonky3-gpu/poseidon2" }
88+
p3-poseidon2-air = { path = "/plonky3-gpu/poseidon2-air" }
89+
p3-symmetric = { path = "/plonky3-gpu/symmetric" }
90+
p3-uni-stark = { path = "/plonky3-gpu/uni-stark" }
91+
p3-maybe-rayon = { path = "/plonky3-gpu/maybe-rayon" }
92+
p3-bn254-fr = { path = "/plonky3-gpu/bn254-fr" }

0 commit comments

Comments
 (0)