Load-balancing / auto-scaling for LLM serving on Google Cloud

my-llm-deployment.yaml

@@ -0,0 +1,17 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: my-llm-deployment
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: my-llm-service
+  template:
+    metadata:
+      labels:
+        app: my-llm-service
+    spec:
+      containers:
+      - name: my-llm-container
+        image: gcr.io/your-project/llm-image:tag

my-llm-hpa.yaml

@@ -0,0 +1,16 @@
+apiVersion: autoscaling/v2beta2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: my-llm-hpa
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: my-llm-deployment
+  minReplicas: 1
+  maxReplicas: 10
+  metrics:
+  - type: Resource
+    resource:
+      name: cpu
+      targetAverageUtilization: 50

my-llm-service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: my-llm-service
+spec:
+  selector:
+    app: my-llm-service
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: 80
+  type: LoadBalancer