cutting a release to update quay prod image in llm-d

Author: clubanderson

.tekton/README.md

@@ -20,9 +20,9 @@ Each repo includes a `.version.json` file at its root. This file controls:
 ```json
 {
   "dev-version": "0.0.5",
-  "dev-registry": "quay.io/vllm-d/<your project name>-dev",
+  "dev-registry": "quay.io/llm-d/<your project name>-dev",
   "prod-version": "0.0.4",
-  "prod-registry": "quay.io/vllm-d/<your project name>"
+  "prod-registry": "quay.io/llm-d/<your project name>"
 }
 ```
 
@@ -43,8 +43,8 @@ The pipeline reads this file to:
 
 This pipeline maintains two container repositories for this GitHub repository, as follows.
 
-- `quay.io/vllm-d/<repoName>-dev`. Hold builds from the `dev` branch as described below.
-- `quay.io/vllm-d/<repoName>`. Holds promotions to prod, as described below.
+- `quay.io/llm-d/<repoName>-dev`. Hold builds from the `dev` branch as described below.
+- `quay.io/llm-d/<repoName>`. Holds promotions to prod, as described below.
 
 ---
 

.tekton/buildah-build.yaml

@@ -14,17 +14,9 @@ spec:
   workspaces:
     - name: source
     - name: registry
+    - name: container-storage
+      mountPath: /var/lib/containers
   steps:
-    # - name: setup-qemu
-    #   image: us.icr.io/ibm-hc4ai-operator/qemu-user-static:latest
-    #   imagePullPolicy: IfNotPresent
-    #   securityContext:
-    #     privileged: true
-    #   script: |
-    #     #!/bin/sh
-    #     echo "⚙️ Attempting QEMU registration..."
-    #     /register || echo "ℹ️ Skipping errors – QEMU might already be registered"
-
     - name: build
       image: quay.io/buildah/stable:latest
       imagePullPolicy: IfNotPresent
@@ -33,7 +25,7 @@ spec:
         privileged: true
       env:
         - name: STORAGE_DRIVER
-          value: vfs
+          value: overlay
       script: |
         #!/bin/sh
         set -e

.tekton/pipelinerun.yaml

@@ -326,6 +326,8 @@ spec:
             workspace: source
           - name: registry
             workspace: registry-secret
+          - name: container-storage
+            workspace: container-storage
 
       - name: vulnerability-scan
         when:
@@ -606,6 +608,9 @@ spec:
           name: noop-task
 
   workspaces:
+    - name: container-storage
+      persistentVolumeClaim:
+        claimName: buildah-cache6
     - name: source
       volumeClaimTemplate:
         spec:
@@ -630,4 +635,4 @@ spec:
         secretName: "git-auth-secret-neuralmagic"
     - name: registry-secret
       secret:
-        secretName: quay-secret
+        secretName: quay-secret-llm-d

.version.json

@@ -1,6 +1,6 @@
 {
   "dev-version": "0.0.4",
-  "dev-registry": "quay.io/vllm-d/vllm-sim-dev",
+  "dev-registry": "quay.io/llm-d/vllm-sim-dev",
   "prod-version": "0.0.3",
-  "prod-registry": "quay.io/vllm-d/vllm-sim"
+  "prod-registry": "quay.io/llm-d/vllm-sim"
 }

Makefile

@@ -39,7 +39,7 @@ SHELL := /usr/bin/env bash
 PROJECT_NAME ?= vllm-sim
 DEV_VERSION ?= 0.0.1
 PROD_VERSION ?= 0.0.0
-IMAGE_TAG_BASE ?= quay.io/vllm-d/$(PROJECT_NAME)
+IMAGE_TAG_BASE ?= quay.io/llm-d/$(PROJECT_NAME)
 IMG = $(IMAGE_TAG_BASE):$(DEV_VERSION)
 NAMESPACE ?= hc4ai-operator
 
@@ -92,16 +92,23 @@ buildah-build: check-builder load-version-json ## Build and push image (multi-ar
 	@echo "✅ Using builder: $(BUILDER)"
 	@if [ "$(BUILDER)" = "buildah" ]; then \
 	  echo "🔧 Buildah detected: Performing multi-arch build..."; \
+	  FINAL_TAG=$(IMG); \
 	  for arch in amd64; do \
+	    ARCH_TAG=$$FINAL_TAG-$$arch; \
 	    echo "📦 Building for architecture: $$arch"; \
-	    buildah build --arch=$$arch --os=linux -t $(IMG)-$$arch . || exit 1; \
-	    echo "🚀 Pushing image: $(IMG)-$$arch"; \
-	    buildah push $(IMG)-$$arch docker://$(IMG)-$$arch || exit 1; \
+		buildah build --arch=$$arch --os=linux --layers -t $(IMG)-$$arch . || exit 1; \
+	    echo "🚀 Pushing image: $$ARCH_TAG"; \
+	    buildah push $$ARCH_TAG docker://$$ARCH_TAG || exit 1; \
 	  done; \
-	  echo "🧱 Creating and pushing manifest list: $(IMG)"; \
-	  buildah manifest create $(IMG); \
-	  buildah manifest add $(IMG) $(IMG)-amd64; \
-	  buildah manifest push --all $(IMG) docker://$(IMG); \
+	  echo "🧼 Removing existing manifest (if any)..."; \
+	  buildah manifest rm $$FINAL_TAG || true; \
+	  echo "🧱 Creating and pushing manifest list: $$FINAL_TAG"; \
+	  buildah manifest create $$FINAL_TAG; \
+	  for arch in amd64; do \
+	    ARCH_TAG=$$FINAL_TAG-$$arch; \
+	    buildah manifest add $$FINAL_TAG $$ARCH_TAG; \
+	  done; \
+	  buildah manifest push --all $$FINAL_TAG docker://$$FINAL_TAG; \
 	elif [ "$(BUILDER)" = "docker" ]; then \
 	  echo "🐳 Docker detected: Building with buildx..."; \
 	  sed -e '1 s/\(^FROM\)/FROM --platform=$${BUILDPLATFORM}/' Dockerfile > Dockerfile.cross; \
@@ -259,7 +266,7 @@ load-version-json: check-jq
 	  export DEV_VERSION; \
 	  export PROD_VERSION; \
 	fi && \
-	CURRENT_DEFAULT="quay.io/vllm-d/$(PROJECT_NAME)"; \
+	CURRENT_DEFAULT="quay.io/llm-d/$(PROJECT_NAME)"; \
 	if [ "$(IMAGE_TAG_BASE)" = "$$CURRENT_DEFAULT" ]; then \
 	  IMAGE_TAG_BASE=$$(jq -r '."dev-registry"' .version.json); \
 	  echo "✔ Loaded IMAGE_TAG_BASE from .version.json: $$IMAGE_TAG_BASE"; \

README.md

@@ -81,6 +81,7 @@ For more details see the <a href="https://docs.vllm.ai/en/stable/getting_started
 - `inter-token-latency`: the time to 'generate' each additional token (in milliseconds), optional, by default zero
 - `max-loras`: maximum number of LoRAs in a single batch, optional, default is one
 - `max-cpu-loras`: maximum number of LoRAs to store in CPU memory, optional, must be >= than max_loras, default is max_loras
+- `max-running-requests`: maximum number of inference requests that could be processed at the same time
 
 
 ## Working with docker image
@@ -113,4 +114,3 @@ To run the router in a standalone test environment, run:
 ```
 
 
-

cmd/vllm-sim/main.go

@@ -31,12 +31,12 @@ func main() {
 	// setup logger and context with graceful shutdown
 	logger := klog.Background()
 	ctx := klog.NewContext(context.Background(), logger)
-	_ = signals.SetupSignalHandler(ctx)
+	ctx = signals.SetupSignalHandler(ctx)
 
 	logger.Info("Start vllm simulator")
 
 	vllmSim := vllmsim.New(logger)
-	err := vllmSim.Start()
+	err := vllmSim.Start(ctx)
 
 	if err != nil {
 		logger.Error(err, "VLLM simulator failed")

deploy/common/statefulset.yaml

@@ -16,5 +16,5 @@ spec:
       serviceAccountName: operator-controller-manager
       containers:
         - name: cmd
-          image: quay.io/vllm-d/placeholder:placeholder
+          image: quay.io/llm-d/placeholder:placeholder
           imagePullPolicy: Always

deploy/kustomization.yaml

@@ -25,8 +25,8 @@ configMapGenerator:
 
 # Define the image to be updated.
 # images:
-# - name: quay.io/vllm-d/placeholder
-#   newName: quay.io/vllm-d/${IMAGE_TAG_BASE}
+# - name: quay.io/llm-d/placeholder
+#   newName: quay.io/llm-d/${IMAGE_TAG_BASE}
 #   newTag: ${VERSION}
 patches:
 - path: common/patch-service.yaml

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/onsi/ginkgo/v2 v2.23.4
 	github.com/onsi/gomega v1.37.0
+	github.com/openai/openai-go v0.1.0-beta.10
 	github.com/prometheus/client_golang v1.21.1
 	github.com/spf13/pflag v1.0.6
 	github.com/valyala/fasthttp v1.59.0
@@ -30,6 +31,10 @@ require (
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.62.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
+	github.com/tidwall/gjson v1.18.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.1 // indirect
+	github.com/tidwall/sjson v1.2.5 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	go.uber.org/automaxprocs v1.6.0 // indirect
 	golang.org/x/net v0.38.0 // indirect