Merge pull request #2187 from o1-labs/brian/nix-cache

Push to the nix cache in ci

Author: Geometer1729

.github/actions/build/action.yml

@@ -1,16 +1,34 @@
 name: Build o1js
 description: 'All of the building steps for o1js'
 
-
 permissions:
   contents: write
 
+inputs:
+  cache_id:
+    required: true
+  nar_secret:
+    required: true
+  gcp_secret:
+    required: true
+
 runs:
   using: "composite"
   steps:
-    - name: Set up Nix
+    - name: Install zstd if needed
       shell: bash
-      run: echo "PATH=$PATH:/nix/var/nix/profiles/default/bin" >> $GITHUB_ENV
+      run: command -v unzstd || (sudo apt-get update && sudo apt-get install -y zstd)
+    - uses: nixbuild/nix-quick-install-action@v30
+      continue-on-error: true
+      with:
+        nix_conf: |
+          keep-env-derivations = true
+          keep-outputs = true
+    - name: Fallback to persistant nix
+      # only available on some runners
+      shell: bash
+      run: command -v nix || echo "PATH=$PATH:/nix/var/nix/profiles/default/bin" >> $GITHUB_ENV
+
     - name: Disable smudging
       shell: bash
       run: echo "GIT_LFS_SKIP_SMUDGE=1" >> $GITHUB_ENV
@@ -30,11 +48,20 @@ runs:
 
     - name: Build the o1js bindings
       if: steps.bindings-cache.outputs.cache-hit != 'true'
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.cache_id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.gcp_secret }}
       shell: bash
       run: |
         set -Eeu
         ./pin.sh
+        nix build o1js#bindings --accept-flake-config
+        temp_key=$(mktemp)
+        echo ${{ inputs.nar_secret }} > "$temp_key"
+        nix store sign --key-file "$temp_key" --recursive ./result
+        nix copy --to "s3://mina-nix-cache?endpoint=https://storage.googleapis.com" $(nix path-info ./result)
         nix run o1js#generate-bindings --max-jobs auto
+        rm "$temp_key"
 
     - name: Cache dependencies and build
       uses: actions/cache@v4

.github/actions/upload/action.yml

@@ -3,11 +3,24 @@ on:
   workflow_dispatch:
   workflow_call:
 
+inputs:
+  cache_id:
+    required: true
+  nar_secret:
+    required: true
+  gcp_secret:
+    required: true
+
 runs:
   using: "composite"
   steps:
     - name: Build
       uses: ./.github/actions/build
+      with:
+          cache_id: ${{ inputs.cache_id }}
+          nar_secret: ${{ inputs.nar_secret }}
+          gcp_secret: ${{ inputs.gcp_secret }}
+
     - name: generate tar
       shell: bash
       run: |

.github/workflows/checks.yml

@@ -18,14 +18,18 @@ permissions:
 
 jobs:
   Prepare:
-    runs-on: [sdk-self-hosted-linux-amd64-build-system]
+    runs-on: self-hosted
     steps:
       - name: Checkout repository with submodules
         uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Build
         uses: ./.github/actions/build
+        with:
+          cache_id: ${{ secrets.NIX_CACHE_GCP_ID }}
+          nar_secret: ${{ secrets.NIX_CACHE_NAR_SECRET }}
+          gcp_secret: ${{ secrets.NIX_CACHE_GCP_SECRET }}
 
   Lint-Format-and-TypoCheck:
     strategy:
@@ -68,7 +72,7 @@ jobs:
       - name: Run Oxlint
         if: steps.get_changed_files.outputs.files_changed == 'true'
         run: xargs npm run lint:strict < changed_files.txt
-      
+
       - name: Run codespell
         if: steps.get_changed_files.outputs.files_changed == 'true' && github.event.pull_request.labels.*.name != 'no-typo-check'
         uses: codespell-project/actions-codespell@master
@@ -82,13 +86,17 @@ jobs:
   Upload-bindings:
     name: upload bindings artifact
     needs: [Prepare]
-    runs-on: [sdk-self-hosted-linux-amd64-build-system]
+    runs-on: self-hosted
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: upload
         uses: ./.github/actions/upload
+        with:
+          cache_id: ${{ secrets.NIX_CACHE_ID }}
+          nar_secret: ${{ secrets.NIX_CACHE_NAR_SECRET }}
+          gcp_secret: ${{ secrets.NIX_CACHE_GCP_SECRET }}
 
   Build-And-Test-Server:
     needs: [Prepare]
@@ -116,6 +124,10 @@ jobs:
 
       - name: build
         uses: ./.github/actions/build
+        with:
+          cache_id: ${{ secrets.NIX_CACHE_ID }}
+          nar_secret: ${{ secrets.NIX_CACHE_NAR_SECRET }}
+          gcp_secret: ${{ secrets.NIX_CACHE_GCP_SECRET }}
       - name: Prepare for tests
         run: touch profiling.md
       - name: Execute tests
@@ -144,6 +156,10 @@ jobs:
           submodules: recursive
       - name: build
         uses: ./.github/actions/build
+        with:
+          cache_id: ${{ secrets.NIX_CACHE_ID }}
+          nar_secret: ${{ secrets.NIX_CACHE_NAR_SECRET }}
+          gcp_secret: ${{ secrets.NIX_CACHE_GCP_SECRET }}
       - name: Count tests
         id: count_tests
         run: |
@@ -277,6 +293,10 @@ jobs:
         uses: actions/checkout@v4
       - name: build
         uses: ./.github/actions/build
+        with:
+          cache_id: ${{ secrets.NIX_CACHE_ID }}
+          nar_secret: ${{ secrets.NIX_CACHE_NAR_SECRET }}
+          gcp_secret: ${{ secrets.NIX_CACHE_GCP_SECRET }}
       - name: Use shared steps for live testing jobs
         uses: ./.github/actions/live-tests-shared
         with:
@@ -305,6 +325,10 @@ jobs:
         uses: actions/checkout@v4
       - name: build
         uses: ./.github/actions/build
+        with:
+          cache_id: ${{ secrets.NIX_CACHE_ID }}
+          nar_secret: ${{ secrets.NIX_CACHE_NAR_SECRET }}
+          gcp_secret: ${{ secrets.NIX_CACHE_GCP_SECRET }}
       - name: Use shared steps for live testing jobs
         uses: ./.github/actions/live-tests-shared
         with:

.github/workflows/push_main.yml

@@ -85,6 +85,10 @@ jobs:
           node-version: ${{ matrix.node }}
       - name: build
         uses: ./.github/actions/build
+        with:
+          cache_id: ${{ secrets.NIX_CACHE_ID }}
+          nar_secret: ${{ secrets.NIX_CACHE_NAR_SECRET }}
+          gcp_secret: ${{ secrets.NIX_CACHE_GCP_SECRET }}
       - name: Build o1js and mina-signer
         run: |
           npm run prepublishOnly
@@ -116,6 +120,10 @@ jobs:
 
       - name: build
         uses: ./.github/actions/build
+        with:
+          cache_id: ${{ secrets.NIX_CACHE_ID }}
+          nar_secret: ${{ secrets.NIX_CACHE_NAR_SECRET }}
+          gcp_secret: ${{ secrets.NIX_CACHE_GCP_SECRET }}
       - name: add build to gc-root if on main
         run: |
           nix build o1js#o1js-bindings --out-link /home/app/actions-runner/nix-cache/main-bindings-gcroot

.github/workflows/remote_bindings.yml

@@ -13,4 +13,8 @@ jobs:
           submodules: recursive
       - name: upload
         uses: ./.github/actions/upload
+        with:
+          cache_id: ${{ secrets.NIX_CACHE_ID }}
+          nar_secret: ${{ secrets.NIX_CACHE_NAR_SECRET }}
+          gcp_secret: ${{ secrets.NIX_CACHE_GCP_SECRET }}
 

README-nix.md

@@ -88,9 +88,11 @@ with all the dependencies required executing `nix develop o1js#default`.
 nix develop o1js#default
 ```
 
-The first time you run this command, you can expect it to take hours (or even a full day) to complete. Then, you will observe that the current devshell becomes a Nix shell with the right
+On macos the first time you run this command, you can expect it to take hours (or even a full day) to complete, due to the lack of cached builds.
+Then, you will observe that the current devshell becomes a Nix shell with the right
 configuration for `o1js` and `mina`.
 
+
 From within the shell, you can build o1js and update the bindings.
 
 ```console
@@ -114,28 +116,31 @@ nix develop mina
 
 Using Nix can take up a lot of disk space if not optimized. Every time you run `nix develop {SOMETHING}`, Nix will create new generations taking gigabytes of data instead of replacing the old ones. This can soon become a problem in your hard disk if you don't handle it carefully. Here are a few indications that can help with this.
 
-Nix has a garbage collector that **is not used by default** after every run. Instead, artifacts get accumulated in your disk unless configured otherwise. But if the full gargabe collector is executed (`nix-store --gc`), it will get the dependencies removed completely, and you can expect that the next time executing the Nix build will take hours to complete.
-
-Instead, you can try to run `nix-env --delete-generations old` or any other time bound like `7d`. This will not have any effect on MacOS though. Alternatively, the [direnv](https://github.com/direnv/direnv) / [nix-direnv](https://github.com/nix-community/nix-direnv) tool can create garbage collector roots that won't be collected for removal. It just keeps one gc-root to the latest build of the dev shell so that `nix-store --gc` only removes older generations.
+Nix has a garbage collector that **is not used by default** after every run. Instead, artifacts get accumulated in your disk unless configured otherwise.
+This is why we recomend `auto-optimise-store = true` (you will be prompted to accept this). You can also run `nix-store --optimize` retroactively.
 
-On top of that, adding `auto-optimise-store = true` to `/etc/nix/nix.conf` and running `nix-store --optimize` should help with disk usage, as it replaces duplicated files with symlinks.
+If you still need to free up space you can run `nix-store --gc`, unfortunately this can slow down futurue nix builds by forcing you to rebuild dependencies.
+This can be mitigated with [direnv](https://github.com/direnv/direnv) and [nix-direnv](https://github.com/nix-community/nix-direnv) which can create garbage collector roots,
+keeping one gc-root to the latest build of the dev shell so that `nix-store --gc` won't remove it.
+You can also create a gc root any time you run `nix build` (until you remove `./result`) so running `nix build o1js#bindings` before `nix-store --gc` may also help.
 
 ### Runtime optimization
 
-Other configurations are worth adding into your `/etc/nix/nix.conf`:
-
-```bash
-keep-outputs = true
-max-jobs = 20
-extra-substituters = https://storage.googleapis.com/mina-nix-cache
-extra-trusted-public-keys = nix-cache.minaprotocol.org:fdcuDzmnM0Kbf7yU4yywBuUEJWClySc1WIF6t6Mm8h4= nix-cache.minaprotocol.org:D3B1W+V7ND1Fmfii8EhbAbF1JXoe2Ct4N34OKChwk2c= mina-nix-cache-1:djtioLfv2oxuK2lqPUgmZbf8bY8sK/BnYZCU2iU5Q10=
-```
+We suggest a few settings in `flake.nix`.
+You will be prompted to accept or reject these the first time you use nix in this repo.
+You can also use `--accept-flake-config` to accept all of them.
 
-The first of those flags tells the garbage collector to keep build time dependencies of current gc-roots, which should help reduce the amount of data that gets removed and rebuilt.
+`max-jobs = auto`
+For some reason the default is `1`.
 
-The second flag increases the default number of jobs being 1, so that rebuilding from scratch will take shorter time.
+`auto-optimize-store = true;`
+When building slightly different versions of the same repo your nix store can fill up with coppies of the same files.
+This saves space by replacing them with symlinks.
 
-The last two lines tell Nix to use the Mina Foundation's cache whenever possible, which should as well speed things up when building code that has been build in Mina's CI before.
+`substituters = ...`
+`trusted-public-keys = ...`
+These make sure you are using the mina-nix-cache which will save time by downloading any derivations already available.
+Anything built in CI is added to this nix-cache, so it should make a big difference in build times.
 
 ## Common Issues
 

flake.lock

@@ -14,6 +14,20 @@
     dune-nix.inputs.flake-utils.follows = "flake-utils";
     flake-utils.url = "github:numtide/flake-utils";
   };
+  nixConfig = {
+    auto-optimize-store = true;
+    max-jobs = "auto";
+    #coppied from flake.nix in mina
+    allow-import-from-derivation = "true";
+    substituters =
+      [ "https://storage.googleapis.com/mina-nix-cache"
+      ];
+    trusted-public-keys =
+      [ "mina-nix-cache-1:djtioLfv2oxuK2lqPUgmZbf8bY8sK/BnYZCU2iU5Q10="
+        "nix-cache.minaprotocol.org:fdcuDzmnM0Kbf7yU4yywBuUEJWClySc1WIF6t6Mm8h4="
+        "nix-cache.minaprotocol.org:D3B1W+V7ND1Fmfii8EhbAbF1JXoe2Ct4N34OKChwk2c="
+      ];
+  };
   outputs = { self, nixpkgs-mina, nixpkgs-newer, flake-utils, ... }@inputs:
     flake-utils.lib.eachDefaultSystem (system:
       let