Skip to content

Commit 35fe5a6

Browse files
phil-scaleyunfeng-scale
phil-scale
and
yunfeng-scale
authored
Ensure successful helm installation in integration test (#224)
* update integration test * fix * add missing files * fix * fix * fix * try updating ImagePullPolicy * fix * add custom configmap * updates * fix * fix * fix * add registry creds * fix * fix * Update service template * fix * fix * fix docs building * fix * fix * fix * final fix hopefully --------- Co-authored-by: Yunfeng Bai <[email protected]>
1 parent b4e49ea commit 35fe5a6

18 files changed

+209
-54
lines changed

‎.circleci/config.yml‎

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -93,34 +93,63 @@ jobs:
9393
- run:
9494
name: Build Docker Image
9595
command: |
96-
docker build . -f model-engine/Dockerfile -t llm-engine:$CIRCLE_SHA1
96+
docker build . -f model-engine/Dockerfile -t model-engine:$CIRCLE_SHA1
9797
integration_tests:
9898
executor: ubuntu-large
9999
steps:
100100
- checkout
101+
- run:
102+
name: Build Docker Image
103+
command: |
104+
docker build . -f model-engine/Dockerfile -t model-engine:$CIRCLE_SHA1
101105
- run:
102106
name: Install minikube
103107
command: |
104108
cd $HOME
105109
curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb
106110
sudo dpkg -i minikube_latest_amd64.deb
107-
minikube start --vm-driver=docker --kubernetes-version=v1.23.0 --memory=14336 --cpus=8
111+
minikube start --vm-driver=docker --kubernetes-version=v1.23.0 --memory=49152 --cpus=14
108112
- run:
109-
name: Install helm
113+
name: Install kubectl, helm
110114
command: |
111-
cd $HOME
115+
cd $HOME/bin
112116
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
117+
curl -LO "https://dl.k8s.io/release/v1.23.0/bin/linux/amd64/kubectl"
118+
chmod +x kubectl
119+
- run:
120+
name: Install helm chart dependencies (Redis, Postgres, Istio)
121+
command: |
122+
sudo apt-get update && sudo apt-get install -y expect
123+
pushd $HOME/project/.circleci/resources
124+
kubectl apply -f redis-k8s.yaml
125+
kubectl apply -f postgres-k8s.yaml
126+
kubectl create secret generic model-engine-postgres-credentials --from-literal=database_url=postgresql://postgres:[email protected]:5432/circle_test
127+
export ISTIO_VERSION=1.15.0
128+
popd
129+
curl -L https://istio.io/downloadIstio | TARGET_ARCH=x86_64 sh -
130+
install istio-${ISTIO_VERSION}/bin/istioctl $HOME/bin
131+
$HOME/bin/istioctl install --set profile=demo -y
132+
kubectl create namespace model-engine
133+
kubectl create configmap default-config --from-literal=config="$(cat $HOME/project/.circleci/resources/.minikube-config-map | envsubst)"
134+
kubectl create configmap default-config --namespace model-engine --from-literal=config="$(cat $HOME/project/.circleci/resources/.minikube-config-map | envsubst)"
135+
cat $HOME/project/.circleci/resources/.minikube-registry-creds | envsubst | expect
136+
minikube addons enable registry-creds
137+
- run:
138+
name: Pre-load model-engine image to minikube
139+
command: |
140+
minikube --logtostderr -v 1 image load model-engine:$CIRCLE_SHA1
113141
- run:
114142
name: Install helm chart
115143
command: |
116-
cd $HOME/project/charts
117-
helm install llm-engine llm-engine --values llm-engine/values_circleci.yaml --set tag=$CIRCLE_SHA1
144+
pushd $HOME/project/charts
145+
cat llm-engine/values_circleci.yaml | envsubst > llm-engine/values_circleci_subst.yaml
146+
helm install llm-engine llm-engine --values llm-engine/values_circleci_subst.yaml --set tag=$CIRCLE_SHA1 --atomic --debug
118147
119148
executors:
120149
ubuntu-large:
121150
machine:
122151
image: "ubuntu-2004:202201-02"
123-
resource_class: xlarge
152+
resource_class: 2xlarge
124153

125154
commands:
126155
environment_setup:

‎.circleci/resources/.minikube-config-map‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Configmap for AWS credentials inside minikube.
2+
[default]
3+
aws_access_key_id = $CIRCLECI_AWS_ACCESS_KEY
4+
aws_secret_access_key = $CIRCLECI_AWS_SECRET_KEY

‎.circleci/resources/.minikube-registry-creds‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Script to send the registry-creds addon configuration to minikube
2+
# Source: https://github.com/kubernetes/minikube/issues/8283
3+
# See expect syntax here: https://manpages.ubuntu.com/manpages/trusty/man1/expect.1.html
4+
spawn minikube addons configure registry-creds
5+
expect "Do you want to enable AWS Elastic Container Registry?" { send "y\r" }
6+
expect "Enter AWS Access Key ID:" { send "$CIRCLECI_AWS_ACCESS_KEY\r" }
7+
expect "Enter AWS Secret Access Key:" { send "$CIRCLECI_AWS_SECRET_KEY\r" }
8+
expect "Enter AWS Session Token:" { send "\r" }
9+
expect "Enter AWS Region:" { send "us-west-2\r" }
10+
expect "Enter 12 digit AWS Account ID (Comma separated list):" { send "$CIRCLECI_AWS_ACCOUNT_ID\r" }
11+
expect "Enter ARN of AWS role to assume:" { send "\r" }
12+
expect "Do you want to enable Google Container Registry?" { send "n\r" }
13+
expect "Do you want to enable Docker Registry?" { send "n\r" }
14+
expect "Do you want to enable Azure Container Registry?" { send "n\r" }
15+
expect eof

‎.circleci/resources/postgres-k8s.yaml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: postgres
5+
labels:
6+
app: postgres
7+
spec:
8+
replicas: 1
9+
selector:
10+
matchLabels:
11+
app: postgres
12+
template:
13+
metadata:
14+
labels:
15+
app: postgres
16+
spec:
17+
containers:
18+
- name: main
19+
image: "cimg/postgres:12.8-postgis"
20+
imagePullPolicy: IfNotPresent
21+
resources:
22+
requests:
23+
memory: 1Gi
24+
cpu: 1
25+
ports:
26+
- containerPort: 5432
27+
env:
28+
- name: POSTGRES_USER
29+
value: postgres
30+
- name: POSTGRES_DB
31+
value: circle_test
32+
- name: POSTGRES_PASSWORD
33+
value: circle_test
34+
35+
---
36+
37+
kind: Service
38+
apiVersion: v1
39+
metadata:
40+
name: postgres
41+
labels:
42+
app: postgres
43+
spec:
44+
type: ClusterIP
45+
selector:
46+
app: postgres
47+
ports:
48+
- name: redis
49+
port: 5432
50+
targetPort: 5432

‎.circleci/resources/redis-k8s.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: redis-message-broker-master
5+
labels:
6+
app: redis-message-broker-master
7+
spec:
8+
replicas: 1
9+
selector:
10+
matchLabels:
11+
app: redis-message-broker-master
12+
template:
13+
metadata:
14+
labels:
15+
app: redis-message-broker-master
16+
spec:
17+
containers:
18+
- name: main
19+
image: redis
20+
imagePullPolicy: IfNotPresent
21+
resources:
22+
requests:
23+
memory: 1Gi
24+
cpu: 1
25+
ports:
26+
- containerPort: 6379
27+
28+
---
29+
30+
kind: Service
31+
apiVersion: v1
32+
metadata:
33+
name: redis-message-broker-master
34+
labels:
35+
app: redis-message-broker-master
36+
spec:
37+
type: ClusterIP
38+
selector:
39+
app: redis-message-broker-master
40+
ports:
41+
- name: redis
42+
port: 6379
43+
targetPort: 6379

‎charts/llm-engine/templates/_helpers.tpl‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -150,9 +150,9 @@ env:
150150
value: "${PREWARM}"
151151
- name: ML_INFRA_SERVICES_CONFIG_PATH
152152
{{- if .Values.config.file }}
153-
value: "${BASE_PATH}/ml_infra_core/llm_engine.core/llm_engine.core/configs/{{ .Values.config.file.infra }}"
153+
value: "${BASE_PATH}/model-engine/model_engine_server/core/configs/{{ .Values.config.file.infra }}"
154154
{{- else }}
155-
value: "${BASE_PATH}/ml_infra_core/llm_engine.core/llm_engine.core/configs/config.yaml"
155+
value: "${BASE_PATH}/model-engine/model_engine_server/core/configs/config.yaml"
156156
{{- end }}
157157
{{- end }}
158158

@@ -198,9 +198,9 @@ env:
198198
value: "/workspace"
199199
- name: ML_INFRA_SERVICES_CONFIG_PATH
200200
{{- if .Values.config.file }}
201-
value: "/workspace/ml_infra_core/llm_engine.core/llm_engine.core/configs/{{ .Values.config.file.infra }}"
201+
value: "/workspace/model-engine/model_engine_server/core/configs/{{ .Values.config.file.infra }}"
202202
{{- else }}
203-
value: "/workspace/ml_infra_core/llm_engine.core/llm_engine.core/configs/config.yaml"
203+
value: "/workspace/model-engine/model_engine_server/core/configs/config.yaml"
204204
{{- end }}
205205
{{- end }}
206206

@@ -262,12 +262,12 @@ env:
262262
- name: DEPLOY_SERVICE_CONFIG_PATH
263263
value: "/workspace/llm_engine/service_configs/{{ .Values.config.file.llm_engine }}"
264264
- name: ML_INFRA_SERVICES_CONFIG_PATH
265-
value: "/workspace/ml_infra_core/llm_engine.core/llm_engine.core/configs/{{ .Values.config.file.infra }}"
265+
value: "/workspace/model-engine/model_engine_server/core/configs/{{ .Values.config.file.infra }}"
266266
{{- else }}
267267
- name: DEPLOY_SERVICE_CONFIG_PATH
268268
value: "/workspace/llm_engine/service_configs/service_config.yaml"
269269
- name: ML_INFRA_SERVICES_CONFIG_PATH
270-
value: "/workspace/ml_infra_core/llm_engine.core/llm_engine.core/configs/config.yaml"
270+
value: "/workspace/model-engine/model_engine_server/core/configs/config.yaml"
271271
{{- end }}
272272
- name: CELERY_ELASTICACHE_ENABLED
273273
value: "true"

‎charts/llm-engine/templates/balloon_a100_deployment.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ spec:
3232
effect: "NoSchedule"
3333
containers:
3434
- image: public.ecr.aws/ubuntu/ubuntu:latest
35-
imagePullPolicy: IfNotPresent
35+
imagePullPolicy: {{ .Values.image.pullPolicy }}
3636
name: main
3737
resources:
3838
limits:

‎charts/llm-engine/templates/balloon_a10_deployment.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ spec:
3232
effect: "NoSchedule"
3333
containers:
3434
- image: public.ecr.aws/ubuntu/ubuntu:latest
35-
imagePullPolicy: IfNotPresent
35+
imagePullPolicy: {{ .Values.image.pullPolicy }}
3636
name: main
3737
resources:
3838
limits:

‎charts/llm-engine/templates/balloon_cpu_deployment.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ spec:
2727
node-lifecycle: normal
2828
containers:
2929
- image: public.ecr.aws/ubuntu/ubuntu:latest
30-
imagePullPolicy: IfNotPresent
30+
imagePullPolicy: {{ .Values.image.pullPolicy }}
3131
name: main
3232
resources:
3333
limits:

‎charts/llm-engine/templates/balloon_t4_deployment.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ spec:
3232
effect: "NoSchedule"
3333
containers:
3434
- image: public.ecr.aws/ubuntu/ubuntu:latest
35-
imagePullPolicy: IfNotPresent
35+
imagePullPolicy: {{ .Values.image.pullPolicy }}
3636
name: main
3737
resources:
3838
limits:

0 commit comments

Comments
 (0)