From ad61aff560676a9867bcb094f4123a531fd6d9b3 Mon Sep 17 00:00:00 2001 From: Pratish Neerputh Date: Wed, 17 Sep 2025 07:27:36 +0200 Subject: [PATCH 1/6] added arm64 Dockerfile and workflow entry --- .devops/arm64.Dockerfile | 127 +++++++++++++++++++++++++++++++++++ .github/workflows/docker.yml | 22 ++++-- 2 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 .devops/arm64.Dockerfile diff --git a/.devops/arm64.Dockerfile b/.devops/arm64.Dockerfile new file mode 100644 index 0000000000000..e1622877bc743 --- /dev/null +++ b/.devops/arm64.Dockerfile @@ -0,0 +1,127 @@ +# Args for Build actions +ARG BUILDPLATFORM_builder=linux/amd64 +ARG BUILDPLATFORM_runner=linux/arm64 +ARG TARGETARCH + +# Stage 1: Builder Docker +FROM --platform=$BUILDPLATFORM_builder debian:bookworm-slim AS builder + +# FROM ubuntu:$UBUNTU_VERSION AS builder + +# add arm64 deps +RUN dpkg --add-architecture arm64 + +# Install build dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + git \ + curl \ + ninja-build \ + ca-certificates \ + libopenblas-dev \ + libgomp1 \ + libcurl4-openssl-dev \ + gcc-aarch64-linux-gnu \ + g++-aarch64-linux-gnu \ + libc6-dev-arm64-cross \ + libcurl4-openssl-dev:arm64 \ + libssl-dev:arm64 \ + && update-ca-certificates + +WORKDIR /workspace + +COPY . . + +# Set your cross compilers environment variables (adjust if needed) +ENV CC64=aarch64-linux-gnu-gcc +ENV CXX64=aarch64-linux-gnu-g++ + +# remove 'armv9' since gcc-12 doesn't support it +RUN sed -i '/armv9/d' "ggml/src/CMakeLists.txt" + +# Run CMake configure and build + +RUN cmake -S . -B build \ + -DCMAKE_SYSTEM_NAME=Linux \ + -DCMAKE_SYSTEM_PROCESSOR=aarch64 \ + -DCMAKE_C_COMPILER=$CC64 \ + -DCMAKE_CXX_COMPILER=$CXX64 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCURL_INCLUDE_DIR=/usr/aarch64-linux-gnu/include \ + -DCMAKE_BUILD_TYPE=Release \ + -DGGML_NATIVE=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DGGML_BACKEND_DL=ON \ + -DGGML_CPU_ALL_VARIANTS=ON +RUN cmake --build build -j $(nproc) + +RUN mkdir -p /app/lib && \ + find build -name "*.so" -exec cp {} /app/lib \; + +RUN mkdir -p /app/full \ + && cp build/bin/* /app/full \ + && cp *.py /app/full \ + && cp -r gguf-py /app/full \ + && cp -r requirements /app/full \ + && cp requirements.txt /app/full \ + && cp .devops/tools.sh /app/full/tools.sh + +# Stage 2: Runtime +FROM --platform=$BUILDPLATFORM_runner debian:bookworm-slim As base + +#FROM ubuntu:$UBUNTU_VERSION AS base + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + libopenblas-dev \ + libgomp1 \ + curl \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Copy built binaries from builder +COPY --from=builder /app/full /app + +# Full +FROM base AS full + +COPY --from=builder /app/full /app + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y \ + git \ + python3 \ + python3-pip + +RUN pip install --upgrade pip setuptools wheel --break-system-packages +RUN pip install -r requirements.txt --break-system-packages +RUN apt autoremove -y +RUN apt clean -y +RUN rm -rf /tmp/* /var/tmp/* +RUN find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete +RUN find /var/cache -type f -delete + +ENTRYPOINT ["/app/tools.sh"] + +# Light +FROM base AS light + +COPY --from=builder /app/full/llama-cli /app + +WORKDIR /app + +ENTRYPOINT [ "/app/llama-cli" ] + +# Server +FROM base AS server + +COPY --from=builder /app/full/llama-server /app + +WORKDIR /app + +HEALTHCHECK CMD [ "curl", "-f", "http://localhost:8080/health" ] + +ENTRYPOINT [ "/app/llama-server" ] diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2067927be56ca..fffeb9d2ce99a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,11 +39,12 @@ jobs: # Note: the arm64 images are failing, which prevents the amd64 images from being built # https://github.com/ggml-org/llama.cpp/issues/11888 #- { tag: "cpu", dockerfile: ".devops/cpu.Dockerfile", platforms: "linux/amd64,linux/arm64", full: true, light: true, server: true, free_disk_space: false } - - { tag: "cpu", dockerfile: ".devops/cpu.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false } - - { tag: "cuda", dockerfile: ".devops/cuda.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false } - - { tag: "musa", dockerfile: ".devops/musa.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: true } - - { tag: "intel", dockerfile: ".devops/intel.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: true } - - { tag: "vulkan", dockerfile: ".devops/vulkan.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false } + #- { tag: "cpu", dockerfile: ".devops/cpu.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false } + #- { tag: "cuda", dockerfile: ".devops/cuda.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false } + #- { tag: "musa", dockerfile: ".devops/musa.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: true } + #- { tag: "intel", dockerfile: ".devops/intel.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: true } + #- { tag: "vulkan", dockerfile: ".devops/vulkan.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false } + - { tag: "arm64", dockerfile: ".devops/arm64.Dockerfile", platforms: "linux/arm64", full: true, light: true, server: true, free_disk_space: false } # Note: the rocm images are failing due to a compiler error and are disabled until this is fixed to allow the workflow to complete #- {tag: "rocm", dockerfile: ".devops/rocm.Dockerfile", platforms: "linux/amd64,linux/arm64", full: true, light: true, server: true, free_disk_space: true } steps: @@ -52,14 +53,25 @@ jobs: with: fetch-depth: 0 # preserve git history, so we can determine the build number + #- name: Set up QEMU + # uses: docker/setup-qemu-action@v3 + # with: + # image: tonistiigi/binfmt:qemu-v7.0.0-28 + + #- name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + - name: Set up QEMU + if: matrix.config.platforms != 'linux/arm64' uses: docker/setup-qemu-action@v3 with: image: tonistiigi/binfmt:qemu-v7.0.0-28 - name: Set up Docker Buildx + if: matrix.config.platforms != 'linux/arm64' uses: docker/setup-buildx-action@v3 + - name: Log in to Docker Hub uses: docker/login-action@v2 with: From 5553e22afe7b941538cdc87872df74e62e2d097a Mon Sep 17 00:00:00 2001 From: Pratish Neerputh Date: Wed, 17 Sep 2025 07:46:36 +0200 Subject: [PATCH 2/6] added buildx --- .github/workflows/docker.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fffeb9d2ce99a..5cae36c6bf9e8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -57,10 +57,7 @@ jobs: # uses: docker/setup-qemu-action@v3 # with: # image: tonistiigi/binfmt:qemu-v7.0.0-28 - - #- name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - + - name: Set up QEMU if: matrix.config.platforms != 'linux/arm64' uses: docker/setup-qemu-action@v3 @@ -68,7 +65,6 @@ jobs: image: tonistiigi/binfmt:qemu-v7.0.0-28 - name: Set up Docker Buildx - if: matrix.config.platforms != 'linux/arm64' uses: docker/setup-buildx-action@v3 From 8a42219bf5d33f7d8ce9ac278e59309ed3cadc2e Mon Sep 17 00:00:00 2001 From: Pratish Neerputh Date: Wed, 17 Sep 2025 08:32:24 +0200 Subject: [PATCH 3/6] restored other builds --- .github/workflows/docker.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5cae36c6bf9e8..ffa1293eed26f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,11 +39,11 @@ jobs: # Note: the arm64 images are failing, which prevents the amd64 images from being built # https://github.com/ggml-org/llama.cpp/issues/11888 #- { tag: "cpu", dockerfile: ".devops/cpu.Dockerfile", platforms: "linux/amd64,linux/arm64", full: true, light: true, server: true, free_disk_space: false } - #- { tag: "cpu", dockerfile: ".devops/cpu.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false } - #- { tag: "cuda", dockerfile: ".devops/cuda.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false } - #- { tag: "musa", dockerfile: ".devops/musa.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: true } - #- { tag: "intel", dockerfile: ".devops/intel.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: true } - #- { tag: "vulkan", dockerfile: ".devops/vulkan.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false } + - { tag: "cpu", dockerfile: ".devops/cpu.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false } + - { tag: "cuda", dockerfile: ".devops/cuda.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false } + - { tag: "musa", dockerfile: ".devops/musa.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: true } + - { tag: "intel", dockerfile: ".devops/intel.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: true } + - { tag: "vulkan", dockerfile: ".devops/vulkan.Dockerfile", platforms: "linux/amd64", full: true, light: true, server: true, free_disk_space: false } - { tag: "arm64", dockerfile: ".devops/arm64.Dockerfile", platforms: "linux/arm64", full: true, light: true, server: true, free_disk_space: false } # Note: the rocm images are failing due to a compiler error and are disabled until this is fixed to allow the workflow to complete #- {tag: "rocm", dockerfile: ".devops/rocm.Dockerfile", platforms: "linux/amd64,linux/arm64", full: true, light: true, server: true, free_disk_space: true } @@ -57,7 +57,7 @@ jobs: # uses: docker/setup-qemu-action@v3 # with: # image: tonistiigi/binfmt:qemu-v7.0.0-28 - + - name: Set up QEMU if: matrix.config.platforms != 'linux/arm64' uses: docker/setup-qemu-action@v3 From 21822d8433deb2a6a80694941b453a0e0064a4b2 Mon Sep 17 00:00:00 2001 From: Pratish Neerputh Date: Wed, 17 Sep 2025 08:44:19 +0200 Subject: [PATCH 4/6] added clean up steps after pip installs to reduce image size --- .devops/arm64.Dockerfile | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.devops/arm64.Dockerfile b/.devops/arm64.Dockerfile index e1622877bc743..a2489e41d5beb 100644 --- a/.devops/arm64.Dockerfile +++ b/.devops/arm64.Dockerfile @@ -96,11 +96,12 @@ RUN apt-get update \ python3 \ python3-pip -RUN pip install --upgrade pip setuptools wheel --break-system-packages -RUN pip install -r requirements.txt --break-system-packages -RUN apt autoremove -y -RUN apt clean -y -RUN rm -rf /tmp/* /var/tmp/* +RUN pip install --no-cache-dir --upgrade pip setuptools wheel --break-system-packages +RUN pip install --no-cache-dir -r requirements.txt --break-system-packages + +# Clean up unnecessary files to reduce image size +RUN rm -rf /root/.cache/pip && rm -rf /root/.cache/build && rm -rf /tmp/* && rm -rf /var/tmp/* +RUN apt autoremove -y && apt clean -y RUN find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete RUN find /var/cache -type f -delete From c00ea04cd37312e31c92440653acc1593c76e3c7 Mon Sep 17 00:00:00 2001 From: Pratish Neerputh Date: Wed, 17 Sep 2025 09:32:17 +0200 Subject: [PATCH 5/6] annotation fix As to AS --- .devops/arm64.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devops/arm64.Dockerfile b/.devops/arm64.Dockerfile index a2489e41d5beb..3e604f9019042 100644 --- a/.devops/arm64.Dockerfile +++ b/.devops/arm64.Dockerfile @@ -68,7 +68,7 @@ RUN mkdir -p /app/full \ && cp .devops/tools.sh /app/full/tools.sh # Stage 2: Runtime -FROM --platform=$BUILDPLATFORM_runner debian:bookworm-slim As base +FROM --platform=$BUILDPLATFORM_runner debian:bookworm-slim AS base #FROM ubuntu:$UBUNTU_VERSION AS base From 7b4298a22c54c524999c582570545cfc55b42646 Mon Sep 17 00:00:00 2001 From: Pratish Surendra Neerputh <53964950+kyroninja@users.noreply.github.com> Date: Thu, 18 Sep 2025 16:59:59 +0200 Subject: [PATCH 6/6] Fix ENTRYPOINT formatting in arm64 Dockerfile