diff --git a/Dockerfile b/Dockerfile index 66a09ebb9..840391820 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,11 @@ ARG GO_VERSION=1.24.2 ARG LLAMA_SERVER_VERSION=latest ARG LLAMA_SERVER_VARIANT=cpu +ARG TARGETARCH=${BUILDARCH} ARG LLAMA_BINARY_PATH=/com.docker.llama-server.native.linux.${LLAMA_SERVER_VARIANT}.${TARGETARCH} ARG BASE_IMAGE=ubuntu:24.04 -FROM golang:${GO_VERSION}-bookworm AS builder +FROM docker.io/library/golang:${GO_VERSION}-bookworm AS builder # Install git for go mod download if needed RUN apt-get update && apt-get install -y --no-install-recommends git && rm -rf /var/lib/apt/lists/* @@ -33,7 +34,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \ FROM docker/docker-model-backend-llamacpp:${LLAMA_SERVER_VERSION}-${LLAMA_SERVER_VARIANT} AS llama-server # --- Final image --- -FROM ${BASE_IMAGE} AS final +FROM docker.io/${BASE_IMAGE} AS final ARG LLAMA_SERVER_VARIANT diff --git a/Makefile b/Makefile index 3ced13410..661a2f9f8 100644 --- a/Makefile +++ b/Makefile @@ -70,18 +70,14 @@ docker-run: docker-build @echo "Service will be available at: http://localhost:$(PORT)" @echo "Example usage: curl http://localhost:$(PORT)/models" @echo "" - mkdir -p $(MODELS_PATH) - docker run --rm \ - -p $(PORT):$(PORT) \ - -v "$(MODELS_PATH):/models" \ - -e MODEL_RUNNER_PORT=$(PORT) \ - -e LLAMA_SERVER_PATH=/app/bin \ - -e MODELS_PATH=/models \ - -e LLAMA_ARGS="$(LLAMA_ARGS)" \ - -e DMR_ORIGINS="$(DMR_ORIGINS)" \ - -e DO_NOT_TRACK=${DO_NOT_TRACK} \ - -e DEBUG=${DEBUG} \ - $(DOCKER_IMAGE) + PORT="$(PORT)" \ + MODELS_PATH="$(MODELS_PATH)" \ + DOCKER_IMAGE="$(DOCKER_IMAGE)" \ + LLAMA_ARGS="$(LLAMA_ARGS)" \ + DMR_ORIGINS="$(DMR_ORIGINS)" \ + DO_NOT_TRACK="${DO_NOT_TRACK}" \ + DEBUG="${DEBUG}" \ + scripts/docker-run.sh # Model distribution tool operations mdl-pull: model-distribution-tool diff --git a/scripts/apt-install.sh b/scripts/apt-install.sh index de376538c..908515047 100755 --- a/scripts/apt-install.sh +++ b/scripts/apt-install.sh @@ -9,7 +9,7 @@ main() { packages+=("libvulkan1") fi - apt-get install -y --no-install-recommends "${packages[@]}" + apt-get install -y "${packages[@]}" rm -rf /var/lib/apt/lists/* } diff --git a/scripts/docker-run.sh b/scripts/docker-run.sh new file mode 100755 index 000000000..dd17277fd --- /dev/null +++ b/scripts/docker-run.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +main() { + set -eux -o pipefail + + local gpu_device_flag=() + for i in /dev/dri /dev/kfd /dev/accel /dev/davinci* /dev/devmm_svm /dev/hisi_hdc; do + if [ -e "$i" ]; then + gpu_device_flag+=("--device" "$i") + fi + done + + mkdir -p "$MODELS_PATH" + chmod a+rx "$MODELS_PATH" + docker run --rm \ + -p "$PORT:$PORT" \ + -v "$MODELS_PATH:/models" \ + -e MODEL_RUNNER_PORT="$PORT" \ + -e LLAMA_SERVER_PATH=/app/bin \ + -e MODELS_PATH=/models \ + -e LLAMA_ARGS="$LLAMA_ARGS" \ + -e DMR_ORIGINS="$DMR_ORIGINS" \ + -e DO_NOT_TRACK="$DO_NOT_TRACK" \ + -e DEBUG="$DEBUG" \ + "${gpu_device_flag[@]}" \ + "$DOCKER_IMAGE" +} + +main "$@" +