Add ccache for cmake caching + optimise dockerfile ordering for cache usage

.github/workflows/publish-docker.yaml

@@ -13,41 +13,47 @@ jobs:
     timeout-minutes: 2880
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
+
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v3
+
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v3
       with:
         config-inline: |
           [worker.oci]
             max-parallelism = 1
+
     - name: Login to DockerHub
-      uses: docker/login-action@v1 
+      uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}
-    # Use the repository information of the checked-out code to format docker tags
+
     - name: Docker meta
       id: docker_meta
-      uses: crazy-max/ghaction-docker-meta@v1
+      uses: docker/metadata-action@v5
       with:
         images: opendronemap/odm
-        tag-semver: |
-          {{version}}
+        tags: |
+          type=semver,pattern={{version}}
+          type=ref,event=branch
+
     - name: Build and push Docker image
       id: docker_build
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v6
       with:
         file: ./portable.Dockerfile
         platforms: linux/amd64,linux/arm64
         push: true
-        no-cache: true
         tags: |
           ${{ steps.docker_meta.outputs.tags }}
           opendronemap/odm:latest
-    # Trigger NodeODM build
+        cache-from: type=registry,ref=opendronemap/odm:buildcache
+        cache-to: type=registry,ref=opendronemap/odm:buildcache,mode=max
+
     - name: Dispatch NodeODM Build Event
       id: nodeodm_dispatch
       run: |
-        curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/OpenDroneMap/NodeODM/actions/workflows/publish-docker.yaml/dispatches --data '{"ref": "master"}'
+        curl -X POST -u "${{secrets.PAT_USERNAME}}:${{secrets.PAT_TOKEN}}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" https://api.github.com/repos/OpenDroneMap/NodeODM/actions/workflows/publish-docker.yaml/dispatches --data '{"ref": "master"}'

.github/workflows/test-build-prs.yaml

@@ -8,21 +8,27 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
+
     - name: Set Swap Space
       uses: pierotofy/set-swap-space@master
       with:
         swap-size-gb: 12
+
     - name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v3
+
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v3
+
     - name: Build
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v6
       with:
         file: ./portable.Dockerfile
         platforms: linux/amd64
         push: false
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
 
   # snapcraft:
   #   runs-on: ubuntu-latest
@@ -32,7 +38,7 @@ jobs:
   #       - amd64
   #   steps:
   #   - name: Checkout
-  #     uses: actions/checkout@v2
+  #     uses: actions/checkout@v4
   #   - name: Set Swap Space
   #     uses: pierotofy/set-swap-space@master
   #     with:
@@ -52,34 +58,42 @@ jobs:
     runs-on: windows-2022
     steps:
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
+
     - name: Setup Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: '3.12.10'
         architecture: 'x64'
+
     - uses: Jimver/[email protected]
       id: cuda-toolkit
       with:
         cuda: '12.8.1'
+
     - name: Setup cmake
-      uses: jwlawson/actions-setup-cmake@v1.13
+      uses: jwlawson/actions-setup-cmake@v2
       with:
         cmake-version: '3.24.x'
+
     - name: Install venv
       run: |
         python -m pip install virtualenv
+
     - name: Build sources
       run: |
         python configure.py build
+
     - name: Free up space
       run: |
         rmdir SuperBuild\download /s /q
         rmdir SuperBuild\build /s /q
       shell: cmd
+
     - name: Create setup
       run: |
         python configure.py dist
+
     - name: Upload Setup File
       uses: actions/upload-artifact@v4
       with:

Dockerfile

@@ -3,22 +3,49 @@ FROM ubuntu:24.04 AS builder
 # Env variables
 ENV DEBIAN_FRONTEND=noninteractive \
     PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
-    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib"
+    LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" \
+    CC="ccache gcc" \
+    CXX="ccache g++" \
+    CCACHE_DIR=/ccache
 
 # Prepare directories
 WORKDIR /code
 
-# Copy everything
-COPY . ./
+# Install ccache first for all subsequent builds
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ccache && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Copy only what's needed for dep installation
+COPY snap/ ./snap/
+COPY configure.sh ./
+COPY requirements.txt ./
+
+# Install system deps and Python requirements
+# Layer is cached unless these files change
+RUN bash configure.sh installreqs
 
-# Run the build
-RUN bash configure.sh install
+# Copy SuperBuild config
+COPY SuperBuild/ ./SuperBuild/
+
+# Build SuperBuild (heavy compilation step),
+# with cache for faster rebuilds
+RUN --mount=type=cache,target=/ccache \
+    cd SuperBuild && \
+    mkdir -p build && \
+    cd build && \
+    cmake .. && \
+    make -j$(nproc)
+
+# Copy app code late, to avoid cache busting
+COPY . ./
 
 # Run the tests
 ENV PATH="/code/venv/bin:$PATH"
 RUN bash test.sh
 
-# Clean Superbuild
+# Clean SuperBuild temp files
 RUN bash configure.sh clean
 
 ### END Builder

configure.sh

@@ -110,6 +110,26 @@ installruntimedepsonly() {
     installdepsfromsnapcraft runtime openmvs
 }
 
+setup_ccache() {
+    # Setup ccache if available to speed up rebuilds
+    if command -v ccache &> /dev/null; then
+        echo "Setting up ccache for faster compilation"
+        export CC="ccache gcc"
+        export CXX="ccache g++"
+        export CCACHE_DIR="${CCACHE_DIR:-$HOME/.ccache}"
+
+        # Configure ccache settings for optimal performance
+        ccache --max-size=5G 2>/dev/null || true
+        ccache --set-config=compression=true 2>/dev/null || true
+        ccache --set-config=compression_level=6 2>/dev/null || true
+
+        echo "ccache statistics:"
+        ccache -s 2>/dev/null || true
+    else
+        echo "ccache not found, compiling without cache"
+    fi
+}
+
 installreqs() {
     cd /code
 
@@ -121,6 +141,10 @@ installreqs() {
     ensure_prereqs
     check_version
 
+    # Install ccache for faster rebuilds
+    echo "Installing ccache for build optimization"
+    sudo $APT_GET install -y -qq --no-install-recommends ccache
+
     echo "Installing Required Requisites"
     installdepsfromsnapcraft build prereqs
     echo "Installing OpenCV Dependencies"
@@ -155,12 +179,21 @@ install() {
         fi
     fi
 
+    # Setup ccache before building
+    setup_ccache
+
     set -eo pipefail
 
     echo "Compiling SuperBuild"
     cd ${RUNPATH}/SuperBuild
     mkdir -p build && cd build
     cmake .. && make -j$processes
+
+    # Show final ccache stats
+    if command -v ccache &> /dev/null; then
+        echo "Final ccache statistics:"
+        ccache -s
+    fi
 
     echo "Configuration Finished"
 }