switch from using versions to using tags, allowing us to lock to a stable version. Upgrade litellm to latest stable. Add delete fake llm load testing server script. (#108)

Co-authored-by: michael rodriguez <[email protected]>

Author: mirodrr2

.env.template

@@ -1,6 +1,6 @@
-# LITELLM_VERSION eg: v1.56.5
-# Get it from https://github.com/BerriAI/litellm/releases
-LITELLM_VERSION="v1.62.1-nightly"
+# LITELLM_VERSION eg: main-v1.56.5
+# Get it from https://github.com/berriai/litellm/pkgs/container/litellm/versions?filters%5Bversion_type%5D=tagged
+LITELLM_VERSION="litellm_stable_release_branch-v1.63.2-stable"
 TERRAFORM_S3_BUCKET_NAME="" #Must be globally unique
 BUILD_FROM_SOURCE="false"
 HOSTED_ZONE_NAME=""
@@ -50,7 +50,7 @@ EKS_ARM_AMI_TYPE="AL2_ARM_64"
 EKS_X86_AMI_TYPE="AL2_x86_64"
 CPU_ARCHITECTURE="" #If empty, defaults to the architecture of your deployment machine "x86" or "arm"
 PUBLIC_LOAD_BALANCER="true"
-RDS_INSTANCE_CLASS="db.t3.micro"
+RDS_INSTANCE_CLASS="db.t3.small"
 RDS_ALLOCATED_STORAGE_GB="20"
 REDIS_NODE_TYPE="cache.t3.micro"
 REDIS_NUM_CACHE_CLUSTERS="2" #Number of cache clusters (primary and replicas) the replication group will have

Dockerfile

@@ -1,2 +1,2 @@
 ARG LITELLM_VERSION=latest
-FROM ghcr.io/berriai/litellm:main-${LITELLM_VERSION}
+FROM ghcr.io/berriai/litellm:${LITELLM_VERSION}

create-fake-llm-load-testing-server.sh

@@ -81,7 +81,7 @@ if [ $? -eq 0 ]; then
         --cluster $LITELLM_ECS_CLUSTER \
         --service $LITELLM_ECS_TASK \
         --force-new-deployment \
-        --desired-count 1 \
+        --desired-count 3 \
         --no-cli-pager
 else
     echo "Deployment failed"

delete-fake-llm-load-testing-server.sh

@@ -0,0 +1,69 @@
+
+#!/bin/bash
+set -aeuo pipefail
+
+aws_region=$(aws ec2 describe-availability-zones --output text --query 'AvailabilityZones[0].[RegionName]')
+echo $aws_region
+
+APP_NAME=fakeserver
+
+source .env
+
+cd litellm-terraform-stack
+VPC_ID=$(terraform output -raw vpc_id)
+cd ..
+
+cd litellm-fake-llm-load-testing-server-terraform
+
+if [ -n "$CPU_ARCHITECTURE" ]; then
+    # Check if CPU_ARCHITECTURE is either "x86" or "arm"
+    case "$CPU_ARCHITECTURE" in
+        "x86"|"arm")
+            ARCH="$CPU_ARCHITECTURE"
+            ;;
+        *)
+            echo "Error: CPU_ARCHITECTURE must be either 'x86' or 'arm'"
+            exit 1
+            ;;
+    esac
+else
+    # Determine architecture from system
+    ARCH=$(uname -m)
+    case $ARCH in
+        x86_64)
+            ARCH="x86"
+            ;;
+        arm64)
+            ARCH="arm"
+            ;;
+        *)
+            echo "Unsupported architecture: $ARCH"
+            exit 1
+            ;;
+    esac
+fi
+
+echo $ARCH
+
+echo "about to destroy"
+
+export TF_VAR_vpc_id="vpc-02b681fa786fa8292"
+export TF_VAR_ecr_fake_server_repository=$APP_NAME
+export TF_VAR_architecture=$ARCH
+export TF_VAR_fake_llm_load_testing_endpoint_certifiacte_arn=$FAKE_LLM_LOAD_TESTING_ENDPOINT_CERTIFICATE_ARN
+export TF_VAR_fake_llm_load_testing_endpoint_hosted_zone_name=$FAKE_LLM_LOAD_TESTING_ENDPOINT_HOSTED_ZONE_NAME
+export TF_VAR_fake_llm_load_testing_endpoint_record_name=$FAKE_LLM_LOAD_TESTING_ENDPOINT_RECORD_NAME
+
+
+cat > backend.hcl << EOF
+bucket  = "${TERRAFORM_S3_BUCKET_NAME}"
+key     = "terraform-fake-llm-server.tfstate"
+region  = "${aws_region}"
+encrypt = true
+EOF
+echo "Generated backend.hcl configuration"
+
+terraform init -backend-config=backend.hcl -reconfigure
+terraform destroy -auto-approve
+
+echo "destroyed"

deploy.sh

@@ -139,7 +139,7 @@ echo "EKS_X86_INSTANCE_TYPE: $EKS_X86_INSTANCE_TYPE"
 echo "EKS_ARM_AMI_TYPE: $EKS_ARM_AMI_TYPE"
 echo "EKS_X86_AMI_TYPE: $EKS_X86_AMI_TYPE"
 echo "PUBLIC_LOAD_BALANCER: $PUBLIC_LOAD_BALANCER"
-echo "RDS_INSTANCE_CLASS: $PUBLIC_LOAD_BALANCER"
+echo "RDS_INSTANCE_CLASS: $RDS_INSTANCE_CLASS"
 echo "RDS_ALLOCATED_STORAGE_GB: $RDS_ALLOCATED_STORAGE_GB"
 echo "REDIS_NODE_TYPE: $REDIS_NODE_TYPE"
 echo "REDIS_NUM_CACHE_CLUSTERS: $REDIS_NUM_CACHE_CLUSTERS"

litellm-fake-llm-load-testing-server-terraform/main.tf

@@ -215,7 +215,7 @@ resource "aws_ecs_service" "fake_server_service" {
   name                               = "FakeServer"
   cluster                            = aws_ecs_cluster.fake_llm_cluster.id
   task_definition                    = aws_ecs_task_definition.fake_server_task_def.arn
-  desired_count                      = 1
+  desired_count                      = 3
   launch_type                        = "FARGATE"
   health_check_grace_period_seconds  = 300
 

litellm-terraform-stack/modules/base/rds.tf

@@ -90,4 +90,5 @@ resource "aws_db_instance" "database" {
   monitoring_role_arn      = aws_iam_role.rds_enhanced_monitoring.arn
   parameter_group_name = aws_db_parameter_group.example_pg.name
   copy_tags_to_snapshot     = true
+  apply_immediately = true
 }