diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
index 0ff2847..eb41fc7 100644
--- a/.github/workflows/publish.yaml
+++ b/.github/workflows/publish.yaml
@@ -33,25 +33,24 @@ jobs:
         if: ${{ github.ref_type == 'tag' }}
         run: sed -i 's/^version = ".*"$/version = "${{ steps.meta.outputs.version }}"/' Cargo.toml
 
+      - name: Login to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io/diamondlightsource
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Build image
         uses: docker/build-push-action@v6
         env:
           DOCKER_BUILD_RECORD_UPLOAD: false
         with:
           context: .
-          cache-from: type=gha
-          cache-to: type=gha
-          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=registry,ref=ghcr.io/diamondlightsource/numtracker:buildcache
+          cache-to: type=registry,ref=ghcr.io/diamondlightsource/numtracker:buildcache,mode=max
+          tags: numtracker-build-cache
           load: true
 
-      - name: Login to GHCR
-        if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io/${{ github.repository_owner }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Publish image
         uses: docker/build-push-action@v6
         env:
diff --git a/Dockerfile b/Dockerfile
index 893e4bb..ffdf75d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -7,15 +7,29 @@ RUN rustup target add x86_64-unknown-linux-musl && \
 
 WORKDIR /build
 
+# Jump through hoops here to improve the caching performance.
+# As each command creates a new layer that can be re-used as long as nothing
+# has changed, maximise the amount of work done before the frequently updated
+# files (.git and src) are included
+
+# Create an empty project
+RUN cargo init --name empty-build-init
+
+# Copy the cargo config but no source
 COPY ./Cargo.toml ./Cargo.toml
 COPY ./Cargo.lock ./Cargo.lock
+
+# Build the empty project with all the dependencies
+RUN cargo build --release --target x86_64-unknown-linux-musl
+
+# Copy the rest of the project in and build again
 COPY ./build.rs ./build.rs
 COPY ./.env ./.env
-COPY ./src ./src
 COPY ./.sqlx ./.sqlx
 COPY ./migrations ./migrations
 COPY ./queries ./queries
 COPY ./static ./static
+COPY ./src ./src
 # Copy the git directory purely so that the commit information can be included
 # build details provided by the 'built' library. See #99
 COPY ./.git ./.git