From 17e649350df4e7ff1dd8529849e28a451a97026f Mon Sep 17 00:00:00 2001 From: Ronald Eytchison Date: Tue, 10 Feb 2026 16:10:41 -0500 Subject: [PATCH 1/3] Create minikube config for a large machine (16 CPU, 128 GB) --- deployment/env.template | 1 + ...ues-upstream-minikube-16cpu-128gb.template | 144 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 deployment/k8s/values-upstream-minikube-16cpu-128gb.template diff --git a/deployment/env.template b/deployment/env.template index 74347da2..64d7ae79 100644 --- a/deployment/env.template +++ b/deployment/env.template @@ -1,6 +1,7 @@ # Reference a values file for kubernetes deployment. This should be a template # file, filled by crs-architecture.sh with the correct variables. # See values-upstream-minikube.template, values-minikube.template, values-aks.template, values-prod.template +# For a large machine, see values-upstream-minikube-16cpu-128gb.template export BUTTERCUP_K8S_VALUES_TEMPLATE="k8s/values-upstream-minikube.template" # Namespace used to install the whole CRS in. diff --git a/deployment/k8s/values-upstream-minikube-16cpu-128gb.template b/deployment/k8s/values-upstream-minikube-16cpu-128gb.template new file mode 100644 index 00000000..b7729390 --- /dev/null +++ b/deployment/k8s/values-upstream-minikube-16cpu-128gb.template @@ -0,0 +1,144 @@ +# Config for a 16 CPU (32 thread), 128 GB machine without scaling LLM usage. +# To use, update the following values in deployment/env: +# export BUTTERCUP_K8S_VALUES_TEMPLATE="k8s/values-upstream-minikube-16cpu-128gb.template" +# export MINIKUBE_CPU=30 +# export MINIKUBE_MEMORY_GB=120 +# export MINIKUBE_DISK_GB=160 +global: + environment: "minikube" + orchestratorImage: + repository: localhost/orchestrator + tag: "latest" + pullPolicy: Never + fuzzerImage: + repository: localhost/fuzzer + tag: "latest" + pullPolicy: Never + seedGenImage: + repository: localhost/seed-gen + tag: "latest" + pullPolicy: Never + patcherImage: + repository: localhost/patcher + tag: "latest" + pullPolicy: Never + programModelImage: + repository: localhost/program-model + tag: "latest" + pullPolicy: Never + + langfuse: + enabled: ${LANGFUSE_ENABLED} + host: "${LANGFUSE_HOST}" + publicKey: "${LANGFUSE_PUBLIC_KEY}" + secretKey: "${LANGFUSE_SECRET_KEY}" + + signoz: + deployed: ${DEPLOY_SIGNOZ} + + volumes: + corpusTmpfs: + enabled: true + + crs: + api_key_id: ${CRS_KEY_ID} + api_key_token: ${CRS_KEY_TOKEN} + api_key_token_hash: "${CRS_KEY_TOKEN_HASH}" + api_url: "${CRS_URL}" + hostname: "${CRS_API_HOSTNAME}" + competition_api_key_id: ${COMPETITION_API_KEY_ID} + competition_api_key_token: ${COMPETITION_API_KEY_TOKEN} + competition_api_url: "ui" + + otel: + endpoint: "${OTEL_ENDPOINT}" + token: "${OTEL_TOKEN}" + protocol: "${OTEL_PROTOCOL}" + +redis: + master: + persistence: + enabled: false + medium: "Memory" # prevent redis failures if Disk I/O is saturated + resources: + limits: + cpu: 1000m + memory: 1Gi + requests: + cpu: 250m + memory: 512Mi + +build-bot: + replicaCount: 2 + +litellm: + masterKey: "${LITELLM_MASTER_KEY}" + maxBudget: "${LITELLM_MAX_BUDGET}" + azure: + apiBase: "${AZURE_API_BASE}" + apiKey: "${AZURE_API_KEY}" + openai: + apiKey: "${OPENAI_API_KEY}" + anthropic: + apiKey: "${ANTHROPIC_API_KEY}" + gemini: + apiKey: "${GEMINI_API_KEY}" + +ui: + enabled: true + +competition-api: + enabled: false + +image-preloader: + baseImages: + - gcr.io/oss-fuzz/base-runner + - gcr.io/oss-fuzz/base-builder + - gcr.io/oss-fuzz/base-image + - gcr.io/oss-fuzz/base-clang + - gcr.io/oss-fuzz/base-runner-debug + - gcr.io/oss-fuzz-base/base-builder-jvm + - gcr.io/oss-fuzz-base/base-builder-ruby + - gcr.io/oss-fuzz-base/base-builder-go + - gcr.io/oss-fuzz-base/base-builder-python + - gcr.io/oss-fuzz-base/base-builder-rust + versions: + - latest + +signoz: + enabled: ${DEPLOY_SIGNOZ} + + + +coverage-bot: + enabled: true + replicaCount: 1 + resources: + limits: + cpu: 1000m + memory: 8Gi + requests: + cpu: 250m + memory: 2Gi + +fuzzer-bot: + enabled: true + replicaCount: 8 + resources: + limits: + cpu: 1000m + memory: 4Gi + requests: + cpu: 1000m + memory: 1Gi + +seed-gen: + enabled: true + replicaCount: 1 + resources: + limits: + cpu: 1000m + memory: 4Gi + requests: + cpu: 500m + memory: 1Gi From 271cb91c3e64fab624b29ce116c9e9ef572796ca Mon Sep 17 00:00:00 2001 From: Ronald Eytchison Date: Wed, 11 Feb 2026 11:24:11 -0500 Subject: [PATCH 2/3] Address PR feedback and scale services - Increase redis memory - Disable redis in-memory backups to conserve memory - Increase build bot replicas - Increase dind CPU/Memory given build bot scaling - Set reasonable limits for merger-bot and pov-reproducer - increase coverage bot memory --- ...ues-upstream-minikube-16cpu-128gb.template | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/deployment/k8s/values-upstream-minikube-16cpu-128gb.template b/deployment/k8s/values-upstream-minikube-16cpu-128gb.template index b7729390..f0e588c8 100644 --- a/deployment/k8s/values-upstream-minikube-16cpu-128gb.template +++ b/deployment/k8s/values-upstream-minikube-16cpu-128gb.template @@ -59,17 +59,16 @@ redis: master: persistence: enabled: false - medium: "Memory" # prevent redis failures if Disk I/O is saturated resources: limits: cpu: 1000m - memory: 1Gi + memory: 4Gi requests: cpu: 250m memory: 512Mi build-bot: - replicaCount: 2 + replicaCount: 3 litellm: masterKey: "${LITELLM_MASTER_KEY}" @@ -108,7 +107,15 @@ image-preloader: signoz: enabled: ${DEPLOY_SIGNOZ} - +dind-daemon: + enabled: true + resources: + limits: + cpu: 8000m + memory: 16Gi + requests: + cpu: 1000m + memory: 4Gi coverage-bot: enabled: true @@ -116,7 +123,7 @@ coverage-bot: resources: limits: cpu: 1000m - memory: 8Gi + memory: 16Gi requests: cpu: 250m memory: 2Gi @@ -142,3 +149,23 @@ seed-gen: requests: cpu: 500m memory: 1Gi + +merger-bot: + enabled: true + resources: + limits: + cpu: 1000m + memory: 4Gi + requests: + cpu: 500m + memory: 512Mi + +pov-reproducer: + enabled: true + resources: + limits: + cpu: 1000m + memory: 4Gi + requests: + cpu: 100m + memory: 1Gi From 1676071e798e46a393cfe0fc468846a446b07bc1 Mon Sep 17 00:00:00 2001 From: Ronald Eytchison Date: Wed, 11 Feb 2026 23:43:33 -0500 Subject: [PATCH 3/3] Configure /dev/shm size for corpus tmpfs --- deployment/crs-architecture.sh | 7 +++++++ deployment/env.template | 2 ++ .../k8s/values-upstream-minikube-16cpu-128gb.template | 1 + 3 files changed, 10 insertions(+) diff --git a/deployment/crs-architecture.sh b/deployment/crs-architecture.sh index d4d3be39..654f1120 100755 --- a/deployment/crs-architecture.sh +++ b/deployment/crs-architecture.sh @@ -106,6 +106,13 @@ up() { echo -e "${GRN}Minikube cluster status:${NC}" minikube status + # Resize /dev/shm inside minikube for corpus tmpfs storage. + # Minikube docker driver defaults to 64MB, which is too small for fuzzing corpus data + if [ -n "${MINIKUBE_SHM_SIZE_GB:-}" ]; then + echo -e "${BLU}Resizing minikube /dev/shm to ${MINIKUBE_SHM_SIZE_GB}G${NC}" + docker exec minikube mount -o remount,size="${MINIKUBE_SHM_SIZE_GB}G" /dev/shm + fi + echo -e "${BLU}Building local docker images${NC}" eval "$(minikube docker-env --shell bash)" diff --git a/deployment/env.template b/deployment/env.template index 64d7ae79..31d7ed18 100644 --- a/deployment/env.template +++ b/deployment/env.template @@ -103,3 +103,5 @@ export FUZZER_BASE_IMAGE="gcr.io/oss-fuzz-base/base-runner" export MINIKUBE_CPU=6 export MINIKUBE_MEMORY_GB=10 export MINIKUBE_DISK_GB=80 +# Increase /dev/shm size if corpus tmpfs is enabled +# export MINIKUBE_SHM_SIZE_GB=60 diff --git a/deployment/k8s/values-upstream-minikube-16cpu-128gb.template b/deployment/k8s/values-upstream-minikube-16cpu-128gb.template index f0e588c8..5c6f0394 100644 --- a/deployment/k8s/values-upstream-minikube-16cpu-128gb.template +++ b/deployment/k8s/values-upstream-minikube-16cpu-128gb.template @@ -4,6 +4,7 @@ # export MINIKUBE_CPU=30 # export MINIKUBE_MEMORY_GB=120 # export MINIKUBE_DISK_GB=160 +# export MINIKUBE_SHM_SIZE_GB=60 global: environment: "minikube" orchestratorImage: