diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
index bdf7afc5..06cadc3b 100644
--- a/.github/workflows/build.yaml
+++ b/.github/workflows/build.yaml
@@ -55,6 +55,15 @@ jobs:
       run: |
         echo -n ${{ github.event.pull_request.head.sha }} > VERSION
 
+    - name: Create cache folder
+      run: mkdir ./cache
+    
+    - name: Setup cache
+      uses: namespacelabs/nscloud-cache-action@v1
+      with:
+        path: |
+          ./cache
+
     - name: Build system
       run: ./build_system.sh
 
diff --git a/Dockerfile.agnos b/Dockerfile.agnos
index e19db364..26143fc4 100644
--- a/Dockerfile.agnos
+++ b/Dockerfile.agnos
@@ -14,6 +14,10 @@ RUN set -xe
 
 ARG USERNAME=comma
 
+# Restore apt cache
+COPY ./cache/var_lib_apt/lists /var/lib/apt/lists
+COPY ./cache/var_cache_apt/archives /var/cache/apt/archives
+
 # Base system setup
 RUN echo "resolvconf resolvconf/linkify-resolvconf boolean false" | debconf-set-selections
 COPY ./userspace/base_setup.sh /tmp/agnos
@@ -39,7 +43,7 @@ RUN apt-get -o Dpkg::Options::="--force-overwrite" install -yq \
 # ################## #
 FROM agnos-base AS agnos-compiler
 
-RUN apt-fast update && apt-fast install --no-install-recommends -yq checkinstall
+RUN apt-fast install --no-install-recommends -yq checkinstall
 
 # Individual compiling images
 FROM agnos-compiler AS agnos-compiler-pyqt5
@@ -62,7 +66,7 @@ RUN /tmp/agnos/compile-qtwayland5.sh
 # ################### #
 # ###### AGNOS ###### #
 # ################### #
-FROM agnos-base
+FROM agnos-base AS agnos
 
 # Hardware setup
 RUN mkdir -p /tmp/agnos/debs
@@ -201,6 +205,8 @@ RUN cd /usr/lib/gcc/arm-none-eabi/* && \
 # keep this last
 RUN ldconfig
 
+FROM agnos AS agnos-final
+
 # Setup RO rootfs
 RUN mkdir -p /tmptmp
 COPY ./userspace/files/comma.conf /usr/lib/tmpfiles.d/
diff --git a/build_system.sh b/build_system.sh
index a899588a..8b8473f6 100755
--- a/build_system.sh
+++ b/build_system.sh
@@ -12,6 +12,7 @@ ARCH=$(uname -m)
 
 BUILD_DIR="$DIR/build"
 OUTPUT_DIR="$DIR/output"
+CACHE_DIR="$DIR/cache"
 
 ROOTFS_DIR="$BUILD_DIR/agnos-rootfs"
 ROOTFS_IMAGE="$BUILD_DIR/system.img"
@@ -23,7 +24,7 @@ OUT_SKIP_CHUNKS_IMAGE="$OUTPUT_DIR/system-skip-chunks.img"
 ROOTFS_IMAGE_SIZE=5G
 
 # Create temp dir if non-existent
-mkdir -p $BUILD_DIR $OUTPUT_DIR
+mkdir -p $BUILD_DIR $OUTPUT_DIR $CACHE_DIR/var_lib_apt/lists $CACHE_DIR/var_cache_apt/archives
 
 # Copy kernel modules
 if ! ls $OUTPUT_DIR/*.ko >/dev/null 2>&1; then
@@ -50,9 +51,13 @@ docker build -f Dockerfile.agnos --check $DIR
 
 # Start agnos-builder docker build and create container
 echo "Building agnos-builder docker image"
+# Docker image for agnos
 docker build -f Dockerfile.agnos -t agnos-builder $DIR
+# Cache image that skips the last cache deletion step
+docker build -f Dockerfile.agnos --target agnos -t agnos-builder-cache $DIR
 echo "Creating agnos-builder container"
 CONTAINER_ID=$(docker container create --entrypoint /bin/bash agnos-builder:latest)
+CACHE_CONTAINER_ID=$(docker container create --entrypoint /bin/bash agnos-builder-cache:latest)
 
 # Check agnos-meta-builder Dockerfile
 docker build -f Dockerfile.builder --check $DIR \
@@ -95,13 +100,18 @@ exec_as_root mount $ROOTFS_IMAGE $ROOTFS_DIR
 # Also unmount filesystem (overwrite previous trap)
 trap "exec_as_root umount -l $ROOTFS_DIR &> /dev/null || true; \
 echo \"Cleaning up containers:\"; \
-docker container rm -f $CONTAINER_ID $MOUNT_CONTAINER_ID" EXIT
+docker container rm -f $CONTAINER_ID $MOUNT_CONTAINER_ID $CACHE_CONTAINER_ID" EXIT
 
 # Extract image
 echo "Extracting docker image"
 docker container export -o $BUILD_DIR/filesystem.tar $CONTAINER_ID
 exec_as_root tar -xf $BUILD_DIR/filesystem.tar -C $ROOTFS_DIR > /dev/null
 
+# Extract cache
+echo "Extracting cache"
+docker cp $CACHE_CONTAINER_ID:/var/cache/apt/archives $CACHE_DIR/var_cache_apt/
+docker cp $CACHE_CONTAINER_ID:/var/lib/apt/lists/ $CACHE_DIR/var_lib_apt/
+
 # Avoid detecting as container
 echo "Removing .dockerenv file"
 exec_as_root rm -f $ROOTFS_DIR/.dockerenv
diff --git a/userspace/base_setup.sh b/userspace/base_setup.sh
index 659b2d0c..9baf7590 100755
--- a/userspace/base_setup.sh
+++ b/userspace/base_setup.sh
@@ -107,8 +107,6 @@ apt-fast install --no-install-recommends -yq \
     wireless-tools \
     zlib1g-dev
 
-rm -rf /var/lib/apt/lists/*
-
 # Allow chrony to make a big adjustment to system time on boot
 echo "makestep 0.1 3" >> /etc/chrony/chrony.conf
 
@@ -126,7 +124,6 @@ echo "comma ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
 ln -sf /bin/bash /bin/sh
 
 # Install necessary libs
-apt-fast update -yq
 apt-fast install --no-install-recommends -yq \
     libacl1:armhf \
     libasan6-armhf-cross \
diff --git a/userspace/hardware_setup.sh b/userspace/hardware_setup.sh
index 26907692..0dac9644 100755
--- a/userspace/hardware_setup.sh
+++ b/userspace/hardware_setup.sh
@@ -19,8 +19,5 @@ cd /tmp
 wget http://ports.ubuntu.com/pool/main/j/json-c/libjson-c2_0.11-4ubuntu2.6_arm64.deb -O /tmp/libjson-c2_0.11-4ubuntu2.6_arm64.deb
 apt install -yq /tmp/libjson-c2_0.11-4ubuntu2.6_arm64.deb
 
-# Remove apt cache
-rm -rf /var/lib/apt/lists/*
-
 USERNAME=comma
 adduser $USERNAME netdev
diff --git a/userspace/openpilot_dependencies.sh b/userspace/openpilot_dependencies.sh
index 1d006109..800abc27 100755
--- a/userspace/openpilot_dependencies.sh
+++ b/userspace/openpilot_dependencies.sh
@@ -3,7 +3,6 @@
 echo "Installing openpilot dependencies"
 
 # Install necessary libs
-apt-fast update
 apt-fast install --no-install-recommends -yq \
     autoconf \
     automake \