diff --git a/deployment/crs-architecture.sh b/deployment/crs-architecture.sh index d4d3be39..654f1120 100755 --- a/deployment/crs-architecture.sh +++ b/deployment/crs-architecture.sh @@ -106,6 +106,13 @@ up() { echo -e "${GRN}Minikube cluster status:${NC}" minikube status + # Resize /dev/shm inside minikube for corpus tmpfs storage. + # Minikube docker driver defaults to 64MB, which is too small for fuzzing corpus data + if [ -n "${MINIKUBE_SHM_SIZE_GB:-}" ]; then + echo -e "${BLU}Resizing minikube /dev/shm to ${MINIKUBE_SHM_SIZE_GB}G${NC}" + docker exec minikube mount -o remount,size="${MINIKUBE_SHM_SIZE_GB}G" /dev/shm + fi + echo -e "${BLU}Building local docker images${NC}" eval "$(minikube docker-env --shell bash)" diff --git a/deployment/env.template b/deployment/env.template index 74347da2..31d7ed18 100644 --- a/deployment/env.template +++ b/deployment/env.template @@ -1,6 +1,7 @@ # Reference a values file for kubernetes deployment. This should be a template # file, filled by crs-architecture.sh with the correct variables. # See values-upstream-minikube.template, values-minikube.template, values-aks.template, values-prod.template +# For a large machine, see values-upstream-minikube-16cpu-128gb.template export BUTTERCUP_K8S_VALUES_TEMPLATE="k8s/values-upstream-minikube.template" # Namespace used to install the whole CRS in. @@ -102,3 +103,5 @@ export FUZZER_BASE_IMAGE="gcr.io/oss-fuzz-base/base-runner" export MINIKUBE_CPU=6 export MINIKUBE_MEMORY_GB=10 export MINIKUBE_DISK_GB=80 +# Increase /dev/shm size if corpus tmpfs is enabled +# export MINIKUBE_SHM_SIZE_GB=60 diff --git a/deployment/k8s/values-upstream-minikube-16cpu-128gb.template b/deployment/k8s/values-upstream-minikube-16cpu-128gb.template new file mode 100644 index 00000000..5c6f0394 --- /dev/null +++ b/deployment/k8s/values-upstream-minikube-16cpu-128gb.template @@ -0,0 +1,172 @@ +# Config for a 16 CPU (32 thread), 128 GB machine without scaling LLM usage. +# To use, update the following values in deployment/env: +# export BUTTERCUP_K8S_VALUES_TEMPLATE="k8s/values-upstream-minikube-16cpu-128gb.template" +# export MINIKUBE_CPU=30 +# export MINIKUBE_MEMORY_GB=120 +# export MINIKUBE_DISK_GB=160 +# export MINIKUBE_SHM_SIZE_GB=60 +global: + environment: "minikube" + orchestratorImage: + repository: localhost/orchestrator + tag: "latest" + pullPolicy: Never + fuzzerImage: + repository: localhost/fuzzer + tag: "latest" + pullPolicy: Never + seedGenImage: + repository: localhost/seed-gen + tag: "latest" + pullPolicy: Never + patcherImage: + repository: localhost/patcher + tag: "latest" + pullPolicy: Never + programModelImage: + repository: localhost/program-model + tag: "latest" + pullPolicy: Never + + langfuse: + enabled: ${LANGFUSE_ENABLED} + host: "${LANGFUSE_HOST}" + publicKey: "${LANGFUSE_PUBLIC_KEY}" + secretKey: "${LANGFUSE_SECRET_KEY}" + + signoz: + deployed: ${DEPLOY_SIGNOZ} + + volumes: + corpusTmpfs: + enabled: true + + crs: + api_key_id: ${CRS_KEY_ID} + api_key_token: ${CRS_KEY_TOKEN} + api_key_token_hash: "${CRS_KEY_TOKEN_HASH}" + api_url: "${CRS_URL}" + hostname: "${CRS_API_HOSTNAME}" + competition_api_key_id: ${COMPETITION_API_KEY_ID} + competition_api_key_token: ${COMPETITION_API_KEY_TOKEN} + competition_api_url: "ui" + + otel: + endpoint: "${OTEL_ENDPOINT}" + token: "${OTEL_TOKEN}" + protocol: "${OTEL_PROTOCOL}" + +redis: + master: + persistence: + enabled: false + resources: + limits: + cpu: 1000m + memory: 4Gi + requests: + cpu: 250m + memory: 512Mi + +build-bot: + replicaCount: 3 + +litellm: + masterKey: "${LITELLM_MASTER_KEY}" + maxBudget: "${LITELLM_MAX_BUDGET}" + azure: + apiBase: "${AZURE_API_BASE}" + apiKey: "${AZURE_API_KEY}" + openai: + apiKey: "${OPENAI_API_KEY}" + anthropic: + apiKey: "${ANTHROPIC_API_KEY}" + gemini: + apiKey: "${GEMINI_API_KEY}" + +ui: + enabled: true + +competition-api: + enabled: false + +image-preloader: + baseImages: + - gcr.io/oss-fuzz/base-runner + - gcr.io/oss-fuzz/base-builder + - gcr.io/oss-fuzz/base-image + - gcr.io/oss-fuzz/base-clang + - gcr.io/oss-fuzz/base-runner-debug + - gcr.io/oss-fuzz-base/base-builder-jvm + - gcr.io/oss-fuzz-base/base-builder-ruby + - gcr.io/oss-fuzz-base/base-builder-go + - gcr.io/oss-fuzz-base/base-builder-python + - gcr.io/oss-fuzz-base/base-builder-rust + versions: + - latest + +signoz: + enabled: ${DEPLOY_SIGNOZ} + +dind-daemon: + enabled: true + resources: + limits: + cpu: 8000m + memory: 16Gi + requests: + cpu: 1000m + memory: 4Gi + +coverage-bot: + enabled: true + replicaCount: 1 + resources: + limits: + cpu: 1000m + memory: 16Gi + requests: + cpu: 250m + memory: 2Gi + +fuzzer-bot: + enabled: true + replicaCount: 8 + resources: + limits: + cpu: 1000m + memory: 4Gi + requests: + cpu: 1000m + memory: 1Gi + +seed-gen: + enabled: true + replicaCount: 1 + resources: + limits: + cpu: 1000m + memory: 4Gi + requests: + cpu: 500m + memory: 1Gi + +merger-bot: + enabled: true + resources: + limits: + cpu: 1000m + memory: 4Gi + requests: + cpu: 500m + memory: 512Mi + +pov-reproducer: + enabled: true + resources: + limits: + cpu: 1000m + memory: 4Gi + requests: + cpu: 100m + memory: 1Gi