diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index a19452d..c352dae 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -25,13 +25,8 @@ jobs:
       - run: echo NAME=dune-${{ github.ref_name }}-${{ matrix.name }} >> $GITHUB_ENV
       - uses: actions/checkout@v4
       - uses: cachix/install-nix-action@v31
-      - name: Build dune
-        run: nix build ${{ matrix.output }}
-      - name: Create dune tarball
-        run: |
-          cp -rL result $NAME
-          chmod -R u+w $NAME
-          tar czf $NAME.tar.gz $NAME
+      - name: Build the binary distro and package it as a tarball
+      - run: ./make_tarball.sh $NAME ${{ matrix.output }}
       - uses: ncipollo/release-action@v1
         with:
           allowUpdates: true
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 0e9cfb2..bc11cb7 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -2,16 +2,68 @@ name: test
 on: [push, pull_request]
 jobs:
 
-  shellcheck:
+  test-make-tarball:
+    name: "Test that we can build the binary distro and package it as a tarball"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        include:
+          - os: macos-13
+            name: x86_64-apple-darwin
+            output: .#dune.dynamic
+          - os: macos-14
+            name: aarch64-apple-darwin
+            output: .#dune.dynamic
+          - os: ubuntu-latest
+            name: x86_64-unknown-linux-musl
+            output: .#dune.static
+    steps:
+      - run: echo NAME=dune-${{ matrix.name }} >> $GITHUB_ENV
+      - uses: actions/checkout@v4
+      - if: matrix.os == 'ubuntu-latest'
+        name: "Install shells on ubuntu"
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y zsh fish
+      - if: matrix.os != 'ubuntu-latest'
+        name: "Install shells on macos"
+        run: brew install fish
+      - uses: cachix/install-nix-action@v31
+      - name: "Build the binary distro and package it as a tarball"
+        run: ./make_tarball.sh $NAME ${{ matrix.output }}
+      - name: "Extract the tarball"
+        run: tar xf $NAME.tar.gz
+      - name: "Test that we can env scripts and the dune executable is in the expected place"
+        run: |
+          bash -c '. $NAME/share/dune/env/env.bash; __dune_env $PWD/$NAME; test $(which dune) = $PWD/$NAME/bin/dune'
+          zsh -c  '. $NAME/share/dune/env/env.zsh;  __dune_env $PWD/$NAME; test $(which dune) = $PWD/$NAME/bin/dune'
+          fish -c '. $NAME/share/dune/env/env.fish; __dune_env $PWD/$NAME; test $(which dune) = $PWD/$NAME/bin/dune'
+          sh -c   '. $NAME/share/dune/env/env.sh;   __dune_env $PWD/$NAME; test $(which dune) = $PWD/$NAME/bin/dune'
+
+
+  test-make-tarball-shellcheck:
+    name: "Run shellcheck on the build script"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: "Install shellcheck"
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+      - name: "Run shellcheck on the build script"
+        run: shellcheck make_tarball.sh
+
+
+  test-env-script-shellcheck:
     name: "Run shellcheck on the shell env script"
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - name: "Install shellcheck"
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y shellcheck
-    - name: "Run shellcheck on the shell env script"
-      run: |
-        # exclude warning for sourcing missing external file
-        shellcheck extra/share/dune/env/env.sh extra/share/dune/env/env.bash
+      - uses: actions/checkout@v4
+      - name: "Install shellcheck"
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y shellcheck
+      - name: "Run shellcheck on the shell env script"
+        run: |
+          # exclude warning for sourcing missing external file
+          shellcheck extra/share/dune/env/env.sh extra/share/dune/env/env.bash
diff --git a/.gitignore b/.gitignore
index b2be92b..0c7ecb2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 result
+*.tar.gz
diff --git a/make_tarball.sh b/make_tarball.sh
new file mode 100755
index 0000000..71d73ba
--- /dev/null
+++ b/make_tarball.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -eu
+
+if [ "$#" -ne "2" ]; then
+    echo "Usage: $0 NAME TARGET"
+    echo
+    echo "Creates a file NAME.tar.gz containing the dune binary distro."
+    echo "TARGET is the nix target that gets built. It should be one of:"
+    echo "  - .#dune.dynamic (builds a dynamically-linked dune executable)"
+    echo "  - .#dune.static (builds a statically-linked dune executable)"
+    exit 1
+fi
+
+NAME="$1"
+TARGET="$2"
+
+set -x
+
+tmp_dir="$(mktemp -d)"
+trap 'rm -rf "$tmp_dir"' EXIT
+nix build "$TARGET"
+cp -rL result "$tmp_dir/$NAME"
+chmod -R u+w "$tmp_dir/$NAME"
+pushd "$tmp_dir"
+tar czf "$NAME.tar.gz" "$NAME"
+popd
+mv "$tmp_dir/$NAME.tar.gz" .