diff --git a/.github/workflows/_container.yml b/.github/workflows/_container.yml
index 97fefb0..38f2edd 100644
--- a/.github/workflows/_container.yml
+++ b/.github/workflows/_container.yml
@@ -21,14 +21,20 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
             type=semver,pattern={{major}}
 
+      - name: Setup buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Build and export to Docker local cache
         uses: docker/build-push-action@v6
         env:
           DOCKER_BUILD_RECORD_UPLOAD: false
         with:
           # Need load and tags so we can test it below
+          context: .
           load: true
           tags: test_tag
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
 
       - name: Test cli works in cached runtime image
         run: docker run --rm test_tag --version
@@ -47,6 +53,7 @@ jobs:
         env:
           DOCKER_BUILD_RECORD_UPLOAD: false
         with:
+          context: .
           push: ${{ github.event_name == 'push' && github.ref_type == 'tag' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
diff --git a/Dockerfile b/Dockerfile
index 353222b..3bfeb04 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,8 +6,16 @@ RUN rustup target add x86_64-unknown-linux-musl && \
     apt-get install -y musl-tools musl-dev && \
     update-ca-certificates
 
+# Build an empty project with only the Cargo files to improve the cache
+# performance of the container build. The src directory is expected to change
+# most frequently invalidating later caches.
+# This downloads and builds the dependencies early allowing built dependencies
+# to be cached.
+RUN mkdir src && echo 'fn main() {}' > src/main.rs
 COPY ./Cargo.toml ./Cargo.toml
 COPY ./Cargo.lock ./Cargo.lock
+RUN cargo build --release --target x86_64-unknown-linux-musl
+
 COPY ./src ./src
 
 RUN cargo build --release --target x86_64-unknown-linux-musl