Skip to content

Commit aac60d0

Browse files
Geometer1729Geometer1729
authored
Merge pull request #1923 from o1-labs/brian/pure-bindings
Build bindings in a nix derivation
2 parents 45e1a89 + 20690d1 commit aac60d0

File tree

5 files changed

+181
-41
lines changed

5 files changed

+181
-41
lines changed

‎.github/workflows/build-bindings.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,15 @@ jobs:
2727
run: |
2828
set -Eeu
2929
./pin.sh
30-
nix develop o1js --command bash -c "npm run build:update-bindings"
30+
nix run o1js#update-bindings --max-jobs 4
31+
#fail if this changes any files
32+
cd src/bindings
33+
git diff --exit-code
34+
- name: add build to gc-root if on main
35+
if: github.ref == 'refs/heads/main'
36+
run: |
37+
nix build o1js#o1js-bindings --out-link /home/app/actions-runner/nix-cache/main-bindings-gcroot
3138
- name: Cleanup the Nix store
3239
run: |
33-
nix-env --delete-generations old
34-
nix-collect-garbage -d --quiet
3540
nix-store --gc --print-dead
3641
nix-store --optimise

‎flake.lock‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎flake.nix‎

Lines changed: 169 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,42 @@
8484
{ targets = ["wasm32-unknown-unknown" "x86_64-unknown-linux-gnu" ];
8585
extensions = [ "rust-src" ];
8686
});
87+
rust-platform = pkgs.makeRustPlatform
88+
{ cargo = rust-channel;
89+
rustc = rust-channel;
90+
};
91+
92+
bindings-pkgs = with pkgs;
93+
[ nodejs
94+
nodePackages.npm
95+
typescript
96+
nodePackages.typescript-language-server
97+
98+
#Rustup doesn't allow local toolchains to contain 'nightly' in the name
99+
#so the toolchain is linked with the name nix and rustup is wrapped in a shellscript
100+
#which calls the nix toolchain instead of the nightly one
101+
(writeShellApplication
102+
{ name = "rustup";
103+
text =
104+
''
105+
if [ "$1" = run ] && { [ "$2" = nightly-2023-09-01 ] || [ "$2" = 1.72-x86_64-unknowl-linux-gnu ]; }
106+
then
107+
echo using nix toolchain
108+
${rustup}/bin/rustup run nix "''${@:3}"
109+
else
110+
echo using plain rustup "$@"
111+
${rustup}/bin/rustup "$@"
112+
fi
113+
'';
114+
}
115+
)
116+
rustup
117+
wasm-pack
118+
binaryen # provides wasm-opt
119+
120+
dune_3
121+
] ++ commonOverrides.buildInputs ;
122+
87123
inherit (nixpkgs) lib;
88124
# All the submodules required by .gitmodules
89125
submodules = map builtins.head (builtins.filter lib.isList
@@ -111,6 +147,56 @@
111147
command "nix-shell"
112148
}.
113149
'';
150+
151+
o1js-npm-deps = pkgs.buildNpmPackage
152+
{ name = "o1js";
153+
src = with pkgs.lib.fileset;
154+
(toSource {
155+
root = ./.;
156+
fileset = unions [
157+
./package.json
158+
./package-lock.json
159+
];
160+
});
161+
# If you see 'ERROR: npmDepsHash is out of date' in ci
162+
# set this to blank run ``nix build o1js#o1js-bindings`
163+
# If you don't want to install nix you can also set it to "" and run ci to get the new hash
164+
# You should get an output like this:
165+
166+
# error: hash mismatch in fixed-output derivation '/nix/store/a03cg2az0b2cvjsp1wnr89clf31i79c1-o1js-npm-deps.drv':
167+
# specified: sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
168+
# got: sha256-8EPvXpOgn0nvm/pFKN3h6EMjabOeBqfy5optIfe8E8Q=
169+
# replace npmDepsHash bellow with the new hash
170+
171+
npmDepsHash = "sha256-QLnSfX6JwYQXyHGNSxXdzqbhkbFl67sDrmlW/F6D/pw=";
172+
# The prepack script runs the build script, which we'd rather do in the build phase.
173+
npmPackFlags = [ "--ignore-scripts" ];
174+
dontNpmBuild = true;
175+
installPhase = ''
176+
runHook preInstall
177+
178+
mkdir -p $out/lib
179+
cp -r node_modules $out/lib
180+
181+
runHook postInstall
182+
'';
183+
};
184+
test-vectors = rust-platform.buildRustPackage {
185+
src = pkgs.lib.sourceByRegex ./src/mina/src
186+
[
187+
"^lib(/crypto(/proof-systems(/.*)?)?)?$"
188+
];
189+
sourceRoot = "source/lib/crypto/proof-systems/poseidon/export_test_vectors";
190+
patchPhase =
191+
''
192+
cp ${./src/mina/src/lib/crypto/proof-systems/Cargo.lock} .
193+
'';
194+
name = "export_test_vectors";
195+
version = "0.1.0";
196+
cargoSha256 = "";
197+
CARGO_TARGET_DIR = "./target";
198+
cargoLock = { lockFile = ./src/mina/src/lib/crypto/proof-systems/Cargo.lock ; };
199+
};
114200
in {
115201
formatter = pkgs.nixfmt;
116202
inherit mina;
@@ -124,48 +210,96 @@
124210
export RUSTUP_HOME
125211
rustup toolchain link nix ${rust-channel}
126212
'';
127-
packages = with pkgs;
128-
[ nodejs
129-
nodePackages.npm
130-
typescript
131-
nodePackages.typescript-language-server
213+
packages = bindings-pkgs;
214+
});
132215

133-
#Rustup doesn't allow local toolchains to contain 'nightly' in the name
134-
#so the toolchain is linked with the name nix and rustup is wrapped in a shellscript
135-
#which calls the nix toolchain instead of the nightly one
136-
(writeShellApplication
137-
{ name = "rustup";
138-
text =
139-
''
140-
if [ "$1" = run ] && [ "$2" = nightly-2023-09-01 ]
141-
then
142-
${rustup}/bin/rustup run nix "''${@:3}"
143-
else
144-
${rustup}/bin/rustup "$@"
145-
fi
146-
'';
147-
}
148-
)
149-
rustup
150-
wasm-pack
151-
binaryen # provides wasm-opt
152216

153-
dune_3
154-
] ++ commonOverrides.buildInputs ;
155-
});
156217
};
157218
# TODO build from ./ocaml root, not ./. (after fixing a bug in dune-nix)
158219
packages = {
159-
kim = pkgs.kimchi-rust-wasm;
160220
inherit dune-description;
161-
bindings = prj.pkgs.o1js_bindings;
221+
o1js-bindings = pkgs.stdenv.mkDerivation {
222+
name = "o1js_bindings";
223+
src = with pkgs.lib.fileset;
224+
(toSource {
225+
root = ./.;
226+
fileset = unions [
227+
./src/mina
228+
./src/bindings/scripts
229+
./src/bindings/js
230+
./src/bindings/crypto
231+
./src/bindings/lib
232+
./src/bindings/mina-transaction/gen/dune
233+
(fileFilter (file: file.hasExt "js") ./src/bindings/mina-transaction)
234+
./src/bindings/ocaml/lib
235+
./src/bindings/ocaml/dune
236+
./src/bindings/ocaml/dune-project
237+
(fileFilter (file: file.hasExt "ml") ./src/bindings/ocaml)
238+
./package.json
239+
./package-lock.json
240+
./src/bindings/ocaml/jsoo_exports
241+
./dune-project
242+
./.prettierrc.cjs
243+
./src/build
244+
./src/snarky.d.ts
245+
];
246+
});
247+
inherit (inputs.mina.devShells."${system}".default)
248+
PLONK_WASM_NODEJS
249+
PLONK_WASM_WEB
250+
MARLIN_PLONK_STUBS
251+
;
252+
PREBUILT_KIMCHI_BINDINGS_JS_WEB =
253+
"${mina.files.src-lib-crypto-kimchi_bindings-js-web}/src/lib/crypto/kimchi_bindings/js/web";
254+
PREBUILT_KIMCHI_BINDINGS_JS_NODE_JS =
255+
"${mina.files.src-lib-crypto-kimchi_bindings-js-node_js}/src/lib/crypto/kimchi_bindings/js/node_js";
256+
EXPORT_TEST_VECTORS = "${test-vectors}/bin/export_test_vectors";
257+
buildInputs = bindings-pkgs ++ [ pkgs.bash ];
258+
SKIP_MINA_COMMIT = true;
259+
JUST_BINDINGS = true;
260+
patchPhase = ''
261+
patchShebangs ./src/bindings/scripts/
262+
patchShebangs ./src/bindings/crypto/test-vectors/
263+
'';
264+
buildPhase =
265+
''
266+
RUSTUP_HOME=$(pwd)/.rustup
267+
export RUSTUP_HOME
268+
rustup toolchain link nix ${rust-channel}
269+
cp -r ${o1js-npm-deps}/lib/node_modules/ .
270+
271+
mkdir -p src/bindings/compiled/node_bindings
272+
echo '// this file exists to prevent TS from type-checking `o1js_node.bc.cjs`' \
273+
> src/bindings/compiled/node_bindings/o1js_node.bc.d.cts
274+
275+
npm run build:update-bindings
276+
277+
mkdir -p $out/mina-transaction
278+
pushd ./src/bindings
279+
rm -rf ./compiled/_node_bindings
280+
cp -Lr ./compiled $out
281+
cp -Lr ./mina-transaction/gen $out/mina-transaction/
282+
popd
283+
'';
284+
};
285+
kimchi = pkgs.kimchi-rust-wasm;
162286
ocaml-js = prj.pkgs.__ocaml-js__;
163-
default = pkgs.buildNpmPackage
164-
{ name = "o1js";
165-
src = ./.;
166-
npmDepsHash = "sha256-++MTGDUVBccYN8LA2Xb0FkbrZ14ZyVCrDPESXa52AwQ=";
167-
# TODO ideally re-build bindings here
168-
};
287+
};
288+
apps = {
289+
update-bindings = {
290+
type = "app";
291+
program = "${pkgs.writeShellApplication
292+
{ name = "update-bindings";
293+
text =
294+
''
295+
cp -r ${self.packages."${system}".o1js-bindings}/* ./src/bindings
296+
chmod +w -R src/bindings/compiled
297+
MINA_COMMIT=$(git -C src/mina rev-parse HEAD)
298+
echo "The mina commit used to generate the backends for node and web is" "$MINA_COMMIT" \
299+
> src/bindings/MINA_COMMIT
300+
'';
301+
}}/bin/update-bindings";
302+
};
169303
};
170304
});
171305
}

‎pin.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,5 +9,6 @@ git submodule sync && git submodule update --init --recursive
99
# Add the flake registry entry
1010
nix registry add o1js "git+file://$ROOT?submodules=1"
1111
# update mina input to local submodule
12+
# --override-input seems redundant but fixes a caching issue with local paths
1213
nix flake update mina --override-input mina 'path:src/mina' --flake '.?submodules=1'
1314
popd

‎src/bindings‎

0 commit comments

Comments
 (0)