Skip to content

Commit b1e4ba3

Browse files
kariykariy
committed
release: vendor cargo deps for reproducible katana binaries
1 parent 8b71ea9 commit b1e4ba3

26 files changed

+620
-5
lines changed

‎.github/workflows/release.yml‎

Lines changed: 81 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,77 @@ jobs:
5959
path: ./crates/contracts/build
6060
retention-days: 1
6161

62+
reproducibility-check:
63+
name: reproducibility-check (linux-amd64)
64+
needs: [prepare, build-contracts]
65+
runs-on: ubuntu-latest-8-cores
66+
env:
67+
TARGET: x86_64-unknown-linux-gnu
68+
steps:
69+
- uses: actions/checkout@v4
70+
71+
- name: Set SOURCE_DATE_EPOCH
72+
shell: bash
73+
run: |
74+
echo "SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)" >> $GITHUB_ENV
75+
76+
- name: Download contract artifacts
77+
uses: actions/download-artifact@v4
78+
with:
79+
name: contract-artifacts
80+
path: ./crates/contracts/build
81+
82+
- uses: actions-rust-lang/setup-rust-toolchain@v1
83+
name: Rust Toolchain Setup
84+
with:
85+
toolchain: ${{ env.RUST_VERSION }}
86+
target: ${{ env.TARGET }}
87+
cache-on-failure: true
88+
cache-key: reproducibility-${{ env.TARGET }}
89+
90+
- name: Verify vendored dependency archive
91+
shell: bash
92+
run: |
93+
./scripts/release/verify-vendor-archive.sh --extract-check
94+
95+
- name: Build (pass 1)
96+
shell: bash
97+
run: |
98+
./scripts/release/build-katana-vendored.sh --target "$TARGET"
99+
SHA_ONE=$(sha256sum "./target/$TARGET/performance/katana" | awk '{ print $1 }')
100+
echo "sha_one=$SHA_ONE" >> $GITHUB_ENV
101+
102+
- name: Build (pass 2)
103+
shell: bash
104+
run: |
105+
rm -rf ./target
106+
./scripts/release/build-katana-vendored.sh --target "$TARGET"
107+
SHA_TWO=$(sha256sum "./target/$TARGET/performance/katana" | awk '{ print $1 }')
108+
echo "sha_two=$SHA_TWO" >> $GITHUB_ENV
109+
110+
- name: Compare hashes
111+
shell: bash
112+
run: |
113+
echo "Build #1 SHA256: $sha_one"
114+
echo "Build #2 SHA256: $sha_two"
115+
if [[ "$sha_one" != "$sha_two" ]]; then
116+
echo "Reproducibility check failed: hashes differ."
117+
exit 1
118+
fi
119+
120+
- name: Summary
121+
shell: bash
122+
run: |
123+
echo "## Reproducibility Check" >> $GITHUB_STEP_SUMMARY
124+
echo "" >> $GITHUB_STEP_SUMMARY
125+
echo "- target: \`$TARGET\`" >> $GITHUB_STEP_SUMMARY
126+
echo "- SOURCE_DATE_EPOCH: \`$SOURCE_DATE_EPOCH\`" >> $GITHUB_STEP_SUMMARY
127+
echo "- sha256(build #1): \`$sha_one\`" >> $GITHUB_STEP_SUMMARY
128+
echo "- sha256(build #2): \`$sha_two\`" >> $GITHUB_STEP_SUMMARY
129+
62130
release:
63131
name: ${{ matrix.job.target }} (${{ matrix.job.os }}${{ matrix.job.native_build == true && ', native' || '' }})
64-
needs: [prepare, build-contracts]
132+
needs: [prepare, build-contracts, reproducibility-check]
65133
runs-on: ${{ matrix.job.os }}
66134
env:
67135
PLATFORM_NAME: ${{ matrix.job.platform }}
@@ -118,6 +186,11 @@ jobs:
118186
steps:
119187
- uses: actions/checkout@v4
120188

189+
- name: Set SOURCE_DATE_EPOCH
190+
shell: bash
191+
run: |
192+
echo "SOURCE_DATE_EPOCH=$(git log -1 --format=%ct)" >> $GITHUB_ENV
193+
121194
- name: Download contract artifacts
122195
uses: actions/download-artifact@v4
123196
with:
@@ -204,17 +277,22 @@ jobs:
204277
# See: https://github.com/jemalloc/jemalloc/issues/467
205278
echo "JEMALLOC_SYS_WITH_LG_PAGE=16" >> $GITHUB_ENV
206279
280+
- name: Verify vendored dependency archive
281+
shell: bash
282+
run: |
283+
./scripts/release/verify-vendor-archive.sh --extract-check
284+
207285
- name: Build binary
208286
if: ${{ matrix.job.native_build == false }}
209287
shell: bash
210288
run: |
211-
cargo build -p katana --bin katana --profile performance --target ${{ matrix.job.target }}
289+
./scripts/release/build-katana-vendored.sh --target "${{ matrix.job.target }}"
212290
213291
- name: Build binary ( w/ cairo-native )
214292
if: ${{ matrix.job.native_build == true }}
215293
shell: bash
216294
run: |
217-
cargo build -p katana --bin katana --profile performance --features native --target ${{ matrix.job.target }}
295+
./scripts/release/build-katana-vendored.sh --target "${{ matrix.job.target }}" --native
218296
219297
- name: Archive binaries
220298
id: artifacts

‎.github/workflows/test.yml‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,26 @@ on:
77
- "release/**"
88
paths:
99
- "Cargo.toml"
10+
- "Cargo.lock"
1011
- "bin/**/*.rs"
1112
- "bin/**/Cargo.toml"
1213
- "crates/**/*.rs"
1314
- "crates/**/Cargo.toml"
15+
- "scripts/release/*.sh"
16+
- "third_party/cargo/**"
1417
- ".github/workflows/test.yml"
1518

1619
pull_request:
1720
types: [opened, synchronize, ready_for_review]
1821
paths:
1922
- "Cargo.toml"
23+
- "Cargo.lock"
2024
- "bin/**/*.rs"
2125
- "bin/**/Cargo.toml"
2226
- "crates/**/*.rs"
2327
- "crates/**/Cargo.toml"
28+
- "scripts/release/*.sh"
29+
- "third_party/cargo/**"
2430
- ".github/workflows/test.yml"
2531

2632
# Cancel in progress workflow when a new one is triggered by running in a concurrency group
@@ -42,8 +48,38 @@ jobs:
4248
- uses: actions/checkout@v3
4349
- run: scripts/rust_fmt.sh --check
4450

51+
vendor-consistency:
52+
runs-on: ubuntu-latest
53+
if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
54+
container:
55+
image: ghcr.io/dojoengine/katana-dev:latest
56+
steps:
57+
- uses: actions/checkout@v3
58+
with:
59+
fetch-depth: 0
60+
# Workaround for https://github.com/actions/runner-images/issues/6775
61+
- run: git config --global --add safe.directory "*"
62+
63+
- name: Verify vendored archive and manifest
64+
run: make vendor-verify
65+
66+
- name: Enforce vendor refresh when Cargo.lock changes
67+
if: github.event_name == 'pull_request'
68+
run: |
69+
BASE_SHA="${{ github.event.pull_request.base.sha }}"
70+
HEAD_SHA="${{ github.event.pull_request.head.sha }}"
71+
CHANGED_FILES="$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")"
72+
73+
if echo "$CHANGED_FILES" | grep -q "^Cargo.lock$"; then
74+
if ! echo "$CHANGED_FILES" | grep -Eq "^third_party/cargo/(vendor\.tar\.gz\.part-[0-9]+|vendor\.tar\.gz\.sha256|VENDOR_MANIFEST\.lock)$"; then
75+
echo "Cargo.lock changed but vendored dependency artifacts were not refreshed."
76+
echo "Run: make vendor-refresh"
77+
exit 1
78+
fi
79+
fi
80+
4581
generate-test-artifacts:
46-
needs: [fmt]
82+
needs: [fmt, vendor-consistency]
4783
runs-on: ubuntu-latest
4884
if: github.event_name == 'push' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false)
4985
container:

‎Makefile‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ SCARB_REQUIRED_VERSIONS := $(sort $(SCARB_VERSION) $(AVNU_SCARB_VERSION) $(VRF_S
4242

4343
.DEFAULT_GOAL := usage
4444
.SILENT: clean
45-
.PHONY: usage help check-llvm native-deps native-deps-macos native-deps-linux native-deps-windows build-explorer contracts tee-sev-snp clean deps install-scarb fixtures snos-artifacts db-compat-artifacts generate-db-fixtures install-pyenv
45+
.PHONY: usage help check-llvm native-deps native-deps-macos native-deps-linux native-deps-windows build-explorer contracts tee-sev-snp clean deps install-scarb fixtures snos-artifacts db-compat-artifacts generate-db-fixtures install-pyenv vendor-refresh vendor-verify
4646

4747
usage help:
4848
@echo "Usage:"
@@ -55,6 +55,8 @@ usage help:
5555
@echo " snos-artifacts: Prepare SNOS tests artifacts."
5656
@echo " db-compat-artifacts: Prepare database compatibility test artifacts."
5757
@echo " generate-db-fixtures: Generate spawn-and-move and simple DB fixtures (requires scarb + sozo)."
58+
@echo " vendor-refresh: Refresh the vendored Cargo dependency archive and manifest."
59+
@echo " vendor-verify: Verify vendored Cargo archive and Cargo.lock consistency."
5860
@echo " native-deps-macos: Install cairo-native dependencies for macOS."
5961
@echo " native-deps-linux: Install cairo-native dependencies for Linux."
6062
@echo " native-deps-windows: Install cairo-native dependencies for Windows."
@@ -96,6 +98,12 @@ build-explorer:
9698

9799
contracts: install-scarb $(CONTRACTS_BUILD_DIR)
98100

101+
vendor-refresh:
102+
@./scripts/release/update-vendor-archive.sh
103+
104+
vendor-verify:
105+
@./scripts/release/verify-vendor-archive.sh --extract-check
106+
99107
tee-sev-snp:
100108
@echo "Building AMD SEV-SNP TEE VM components..."
101109
@if [ -n "$(KATANA_BINARY)" ]; then \

‎README.md‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
- [Cairo Native](#cairo-native)
77
- [Explorer](#explorer)
88
- [Testing](#testing)
9+
- [Release Reproducibility](#release-reproducibility)
910

1011
## Development Setup
1112

@@ -102,3 +103,25 @@ Once setup is complete, you can run the tests using:
102103
```bash
103104
cargo nextest run
104105
```
106+
107+
## Release Reproducibility
108+
109+
Katana release binaries are built from a vendored Cargo dependency archive committed in this repository:
110+
111+
- `third_party/cargo/vendor.tar.gz.part-*`
112+
- `third_party/cargo/vendor.tar.gz.sha256`
113+
- `third_party/cargo/VENDOR_MANIFEST.lock`
114+
115+
The archive is committed as split parts to stay under GitHub's per-file size limit.
116+
117+
When dependencies change, refresh these artifacts in the same PR:
118+
119+
```bash
120+
make vendor-refresh
121+
```
122+
123+
You can verify integrity and lockfile consistency locally with:
124+
125+
```bash
126+
make vendor-verify
127+
```

‎docs/release-reproducibility.md‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
# Release Reproducibility
2+
3+
Katana release binaries use a vendored Cargo dependency archive to reduce non-determinism from network/package source drift.
4+
5+
## Vendored Artifacts
6+
7+
- `third_party/cargo/vendor.tar.gz.part-*`
8+
- `third_party/cargo/vendor.tar.gz.sha256`
9+
- `third_party/cargo/VENDOR_MANIFEST.lock`
10+
11+
The archive is stored as split parts to remain below GitHub's per-file blob limit.
12+
13+
The manifest records:
14+
15+
- SHA-256 of `Cargo.lock`
16+
- SHA-256 of the dependency archive
17+
- archive path metadata used by CI/release validation
18+
19+
## Updating Vendored Dependencies
20+
21+
After any dependency update (`cargo add`, `cargo update`, manifest edits), refresh vendored artifacts:
22+
23+
```bash
24+
make vendor-refresh
25+
```
26+
27+
Commit all of the following in the same PR:
28+
29+
- `Cargo.lock`
30+
- `third_party/cargo/vendor.tar.gz.part-*`
31+
- `third_party/cargo/vendor.tar.gz.sha256`
32+
- `third_party/cargo/VENDOR_MANIFEST.lock`
33+
34+
## Validation
35+
36+
Run local verification:
37+
38+
```bash
39+
make vendor-verify
40+
```
41+
42+
Verification checks:
43+
44+
- archive checksum matches `vendor.tar.gz.sha256`
45+
- manifest archive hash matches the archive
46+
- manifest lock hash matches `Cargo.lock`
47+
- archive extract sanity checks (`cargo-home` layout)
48+
49+
CI enforces this for pull requests and blocks drift.
50+
51+
## Release Workflow Behavior
52+
53+
`release.yml` now:
54+
55+
- validates vendored artifacts before building
56+
- builds Katana with `--locked --offline --frozen` via `scripts/release/build-katana-vendored.sh`
57+
- runs a reproducibility gate (two clean Linux amd64 builds with matching SHA-256)

0 commit comments

Comments
 (0)