feat: Replace mock LLM with local SmolLM2-135M inference

.env

@@ -159,11 +159,12 @@ LLM_DOCKERFILE=./src/llm/Dockerfile
 LLM_HOST=llm
 LLM_PORT=8000
 
-# Mock LLM - to use a real OpenAI API-compatible LLM,
-# configure the .env.override file
+# LLM Service - uses local SmolLM2-135M by default
+# To use a real OpenAI API-compatible LLM, configure .env.override
 LLM_BASE_URL=http://${LLM_HOST}:${LLM_PORT}/v1
-LLM_MODEL=astronomy-llm
+LLM_MODEL=smollm2-135m
 OPENAI_API_KEY=dummy
+SMOLLM_MODEL_PATH=/app/models/SmolLM2-135M-Instruct-Q4_K_M.gguf
 
 # Valkey
 VALKEY_PORT=6379

.env.override

@@ -2,10 +2,8 @@
 # PLACE YOUR .env ENVIRONMENT VARIABLES OVERRIDES IN THIS FILE
 
 
-# To use a real OpenAI API-compatible LLM,
-# set the appropriate values for the target LLM
-# Required permissions:
-# [Models and Model capabilities]
+# To use a real OpenAI API-compatible LLM instead of the built-in SmolLM2,
+# uncomment and set ALL three values below:
 
 #LLM_BASE_URL=https://api.openai.com/v1
 #LLM_MODEL=gpt-4o-mini

CHANGELOG.md

@@ -7,6 +7,8 @@ the release.
 
 ## Unreleased
 
+* [llm] Replace mock LLM with local SmolLM2-135M inference via llama-cpp-python
+  ([#3014](https://github.com/open-telemetry/opentelemetry-demo/pull/3014))
 * [accounting] fix memory leak with dbcontext
   ([#2876](https://github.com/open-telemetry/opentelemetry-demo/pull/2876))
 * [chore] Upgrade OTel Collector to v0.145.0 with :warning: breaking change:

docker-compose.minimal.yml

@@ -612,11 +612,18 @@ services:
     deploy:
       resources:
         limits:
-          memory: 50M
+          memory: 150M
     restart: unless-stopped
     environment:
       - FLAGD_HOST
       - FLAGD_PORT
+      - LLM_PORT
+      - LLM_MODEL
+      - SMOLLM_MODEL_PATH
+      - OTEL_EXPORTER_OTLP_ENDPOINT
+      - OTEL_SERVICE_NAME=llm
+      - OTEL_RESOURCE_ATTRIBUTES
+      - OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true
     ports:
       - "${LLM_PORT}"
     logging: *logging

docker-compose.yml

@@ -763,16 +763,25 @@ services:
     deploy:
       resources:
         limits:
-          memory: 50M
+          memory: 150M
     restart: unless-stopped
     environment:
       - FLAGD_HOST
       - FLAGD_PORT
+      - LLM_PORT
+      - LLM_MODEL
+      - SMOLLM_MODEL_PATH
+      - OTEL_EXPORTER_OTLP_ENDPOINT
+      - OTEL_SERVICE_NAME=llm
+      - OTEL_RESOURCE_ATTRIBUTES
+      - OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true
     ports:
       - "${LLM_PORT}"
     depends_on:
       flagd:
         condition: service_started
+      otel-collector:
+        condition: service_started
     logging: *logging
 
   # Postgresql used by Accounting service

src/llm/Dockerfile

@@ -1,26 +1,37 @@
 # Copyright The OpenTelemetry Authors
 # SPDX-License-Identifier: Apache-2.0
 
+# Build stage: compile llama-cpp-python and download model
+FROM docker.io/library/python:3.12-slim-bookworm AS build
 
-FROM docker.io/library/python:3.12-alpine3.22 AS build-venv
-
-RUN apk update && \
-    apk add gcc g++ linux-headers
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends build-essential g++ curl git && \
+    rm -rf /var/lib/apt/lists/*
 
 COPY ./src/llm/requirements.txt requirements.txt
 
-RUN python -m venv venv && \
-    venv/bin/pip install --no-cache-dir -r requirements.txt
+ENV CMAKE_ARGS="-DGGML_NATIVE=OFF" CMAKE_BUILD_PARALLEL_LEVEL=2
+RUN python -m venv /venv && \
+    /venv/bin/pip install --no-cache-dir -r requirements.txt
+
+# Download SmolLM2-135M-Instruct Q4_K_M GGUF (~105MB)
+RUN mkdir -p /models && \
+    curl -L -o /models/SmolLM2-135M-Instruct-Q4_K_M.gguf \
+    "https://huggingface.co/lmstudio-community/SmolLM2-135M-Instruct-GGUF/resolve/main/SmolLM2-135M-Instruct-Q4_K_M.gguf"
+
+# Runtime stage
+FROM docker.io/library/python:3.12-slim-bookworm
 
-FROM docker.io/library/python:3.12-alpine3.22
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends libgomp1 && \
+    rm -rf /var/lib/apt/lists/*
 
-COPY --from=build-venv /venv/ /venv/
+COPY --from=build /venv/ /venv/
+COPY --from=build /models/ /app/models/
 
 WORKDIR /app
 
 COPY ./src/llm/app.py app.py
-COPY ./src/llm/product-review-summaries/product-review-summaries.json product-review-summaries.json
-COPY ./src/llm/product-review-summaries/inaccurate-product-review-summaries.json inaccurate-product-review-summaries.json
 
 EXPOSE ${LLM_PORT}
 ENTRYPOINT [ "/venv/bin/python", "app.py" ]