Skip to content

Commit 77a3e77

Browse files
reytchisonreytchison
authored
Create minikube config for a large machine (16 CPU, 128 GB) (#469)
* Create minikube config for a large machine (16 CPU, 128 GB) * Address PR feedback and scale services - Increase redis memory - Disable redis in-memory backups to conserve memory - Increase build bot replicas - Increase dind CPU/Memory given build bot scaling - Set reasonable limits for merger-bot and pov-reproducer - increase coverage bot memory * Configure /dev/shm size for corpus tmpfs
1 parent fb9a40c commit 77a3e77

File tree

3 files changed

+182
-0
lines changed

3 files changed

+182
-0
lines changed

‎deployment/crs-architecture.sh‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,13 @@ up() {
106106
echo -e "${GRN}Minikube cluster status:${NC}"
107107
minikube status
108108

109+
# Resize /dev/shm inside minikube for corpus tmpfs storage.
110+
# Minikube docker driver defaults to 64MB, which is too small for fuzzing corpus data
111+
if [ -n "${MINIKUBE_SHM_SIZE_GB:-}" ]; then
112+
echo -e "${BLU}Resizing minikube /dev/shm to ${MINIKUBE_SHM_SIZE_GB}G${NC}"
113+
docker exec minikube mount -o remount,size="${MINIKUBE_SHM_SIZE_GB}G" /dev/shm
114+
fi
115+
109116
echo -e "${BLU}Building local docker images${NC}"
110117
eval "$(minikube docker-env --shell bash)"
111118

‎deployment/env.template‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# Reference a values file for kubernetes deployment. This should be a template
22
# file, filled by crs-architecture.sh with the correct variables.
33
# See values-upstream-minikube.template, values-minikube.template, values-aks.template, values-prod.template
4+
# For a large machine, see values-upstream-minikube-16cpu-128gb.template
45
export BUTTERCUP_K8S_VALUES_TEMPLATE="k8s/values-upstream-minikube.template"
56

67
# Namespace used to install the whole CRS in.
@@ -102,3 +103,5 @@ export FUZZER_BASE_IMAGE="gcr.io/oss-fuzz-base/base-runner"
102103
export MINIKUBE_CPU=6
103104
export MINIKUBE_MEMORY_GB=10
104105
export MINIKUBE_DISK_GB=80
106+
# Increase /dev/shm size if corpus tmpfs is enabled
107+
# export MINIKUBE_SHM_SIZE_GB=60

‎deployment/k8s/values-upstream-minikube-16cpu-128gb.template‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
# Config for a 16 CPU (32 thread), 128 GB machine without scaling LLM usage.
2+
# To use, update the following values in deployment/env:
3+
# export BUTTERCUP_K8S_VALUES_TEMPLATE="k8s/values-upstream-minikube-16cpu-128gb.template"
4+
# export MINIKUBE_CPU=30
5+
# export MINIKUBE_MEMORY_GB=120
6+
# export MINIKUBE_DISK_GB=160
7+
# export MINIKUBE_SHM_SIZE_GB=60
8+
global:
9+
environment: "minikube"
10+
orchestratorImage:
11+
repository: localhost/orchestrator
12+
tag: "latest"
13+
pullPolicy: Never
14+
fuzzerImage:
15+
repository: localhost/fuzzer
16+
tag: "latest"
17+
pullPolicy: Never
18+
seedGenImage:
19+
repository: localhost/seed-gen
20+
tag: "latest"
21+
pullPolicy: Never
22+
patcherImage:
23+
repository: localhost/patcher
24+
tag: "latest"
25+
pullPolicy: Never
26+
programModelImage:
27+
repository: localhost/program-model
28+
tag: "latest"
29+
pullPolicy: Never
30+
31+
langfuse:
32+
enabled: ${LANGFUSE_ENABLED}
33+
host: "${LANGFUSE_HOST}"
34+
publicKey: "${LANGFUSE_PUBLIC_KEY}"
35+
secretKey: "${LANGFUSE_SECRET_KEY}"
36+
37+
signoz:
38+
deployed: ${DEPLOY_SIGNOZ}
39+
40+
volumes:
41+
corpusTmpfs:
42+
enabled: true
43+
44+
crs:
45+
api_key_id: ${CRS_KEY_ID}
46+
api_key_token: ${CRS_KEY_TOKEN}
47+
api_key_token_hash: "${CRS_KEY_TOKEN_HASH}"
48+
api_url: "${CRS_URL}"
49+
hostname: "${CRS_API_HOSTNAME}"
50+
competition_api_key_id: ${COMPETITION_API_KEY_ID}
51+
competition_api_key_token: ${COMPETITION_API_KEY_TOKEN}
52+
competition_api_url: "ui"
53+
54+
otel:
55+
endpoint: "${OTEL_ENDPOINT}"
56+
token: "${OTEL_TOKEN}"
57+
protocol: "${OTEL_PROTOCOL}"
58+
59+
redis:
60+
master:
61+
persistence:
62+
enabled: false
63+
resources:
64+
limits:
65+
cpu: 1000m
66+
memory: 4Gi
67+
requests:
68+
cpu: 250m
69+
memory: 512Mi
70+
71+
build-bot:
72+
replicaCount: 3
73+
74+
litellm:
75+
masterKey: "${LITELLM_MASTER_KEY}"
76+
maxBudget: "${LITELLM_MAX_BUDGET}"
77+
azure:
78+
apiBase: "${AZURE_API_BASE}"
79+
apiKey: "${AZURE_API_KEY}"
80+
openai:
81+
apiKey: "${OPENAI_API_KEY}"
82+
anthropic:
83+
apiKey: "${ANTHROPIC_API_KEY}"
84+
gemini:
85+
apiKey: "${GEMINI_API_KEY}"
86+
87+
ui:
88+
enabled: true
89+
90+
competition-api:
91+
enabled: false
92+
93+
image-preloader:
94+
baseImages:
95+
- gcr.io/oss-fuzz/base-runner
96+
- gcr.io/oss-fuzz/base-builder
97+
- gcr.io/oss-fuzz/base-image
98+
- gcr.io/oss-fuzz/base-clang
99+
- gcr.io/oss-fuzz/base-runner-debug
100+
- gcr.io/oss-fuzz-base/base-builder-jvm
101+
- gcr.io/oss-fuzz-base/base-builder-ruby
102+
- gcr.io/oss-fuzz-base/base-builder-go
103+
- gcr.io/oss-fuzz-base/base-builder-python
104+
- gcr.io/oss-fuzz-base/base-builder-rust
105+
versions:
106+
- latest
107+
108+
signoz:
109+
enabled: ${DEPLOY_SIGNOZ}
110+
111+
dind-daemon:
112+
enabled: true
113+
resources:
114+
limits:
115+
cpu: 8000m
116+
memory: 16Gi
117+
requests:
118+
cpu: 1000m
119+
memory: 4Gi
120+
121+
coverage-bot:
122+
enabled: true
123+
replicaCount: 1
124+
resources:
125+
limits:
126+
cpu: 1000m
127+
memory: 16Gi
128+
requests:
129+
cpu: 250m
130+
memory: 2Gi
131+
132+
fuzzer-bot:
133+
enabled: true
134+
replicaCount: 8
135+
resources:
136+
limits:
137+
cpu: 1000m
138+
memory: 4Gi
139+
requests:
140+
cpu: 1000m
141+
memory: 1Gi
142+
143+
seed-gen:
144+
enabled: true
145+
replicaCount: 1
146+
resources:
147+
limits:
148+
cpu: 1000m
149+
memory: 4Gi
150+
requests:
151+
cpu: 500m
152+
memory: 1Gi
153+
154+
merger-bot:
155+
enabled: true
156+
resources:
157+
limits:
158+
cpu: 1000m
159+
memory: 4Gi
160+
requests:
161+
cpu: 500m
162+
memory: 512Mi
163+
164+
pov-reproducer:
165+
enabled: true
166+
resources:
167+
limits:
168+
cpu: 1000m
169+
memory: 4Gi
170+
requests:
171+
cpu: 100m
172+
memory: 1Gi

0 commit comments

Comments
 (0)