Merge pull request #93 from Azure-Samples/fix-load-balancing-priority-weight

Fix missing priority & weight properties

Author: vieiraae

labs/backend-pool-load-balancing/backend-pool-load-balancing.ipynb

@@ -67,10 +67,11 @@
     "\n",
     "apim_sku = 'Basicv2'\n",
     "\n",
+    "# Prioritize UK South until exhaustion (simulate PTU with TPM), then equally distribute between Sweden and France (consumption fallback)\n",
     "openai_resources = [\n",
-    "    {\"name\": \"openai1\", \"location\": \"uksouth\", \"priority\": 1, \"weight\": 80},\n",
-    "    {\"name\": \"openai2\", \"location\": \"swedencentral\", \"priority\": 1, \"weight\": 10},\n",
-    "    {\"name\": \"openai3\", \"location\": \"francecentral\", \"priority\": 1, \"weight\": 10}\n",
+    "    {\"name\": \"openai1\", \"location\": \"uksouth\", \"priority\": 1},\n",
+    "    {\"name\": \"openai2\", \"location\": \"swedencentral\", \"priority\": 2, \"weight\": 50},\n",
+    "    {\"name\": \"openai3\", \"location\": \"francecentral\", \"priority\": 2, \"weight\": 50}\n",
     "]\n",
     "\n",
     "openai_deployment_name = \"gpt-35-turbo\"\n",
@@ -79,7 +80,7 @@
     "openai_model_capacity = 8\n",
     "openai_api_version = \"2024-02-01\"\n",
     "\n",
-    "utils.print_ok('Notebook initiaized')"
+    "utils.print_ok('Notebook initialized')"
    ]
   },
   {
@@ -208,7 +209,7 @@
    "source": [
     "import requests, time\n",
     "\n",
-    "runs = 10\n",
+    "runs = 20\n",
     "sleep_time_ms = 100\n",
     "url = f\"{apim_resource_gateway_url}/openai/deployments/{openai_deployment_name}/chat/completions?api-version={openai_api_version}\"\n",
     "api_runs = []\n",
@@ -265,7 +266,9 @@
    "metadata": {},
    "source": [
     "<a id='plot'></a>\n",
-    "### 🔍 Analyze Load Balancing results\n"
+    "### 🔍 Analyze Load Balancing results\n",
+    "\n",
+    "The priority 1 backend will be used until TPM exhaustion sets in, then distribution will occur near equally across the two priority 2 backends with 50/50 weights."
    ]
   },
   {
@@ -323,7 +326,7 @@
     "import time\n",
     "from openai import AzureOpenAI\n",
     "\n",
-    "runs = 10\n",
+    "runs = 20\n",
     "sleep_time_ms = 100\n",
     "\n",
     "client = AzureOpenAI(\n",

labs/backend-pool-load-balancing/result.png

@@ -153,9 +153,10 @@ resource backendPoolOpenAI 'Microsoft.ApiManagement/service/backends@2024-06-01-
     type: 'Pool'
     pool: {
       services: [for (config, i) in openAIConfig: {
-          id: '/backends/${backendOpenAI[i].name}'
-        }
-      ]
+        id: '/backends/${backendOpenAI[i].name}'
+        priority: config.?priority
+        weight: config.?weight
+      }]
     }
   }
 }

modules/apim/v1/openai-api.bicep

@@ -112,8 +112,12 @@ resource roleAssignment 'Microsoft.Authorization/roleAssignments@2022-04-01' = [
 // ------------------
 
 output extendedOpenAIConfig array = [for (config, i) in openAIConfig: {
+  // Original openAIConfig properties
   name: config.name
   location: config.location
+  priority: config.?priority
+  weight: config.?weight
+  // Additional properties
   sku: openAISku
   deploymentName: openAIDeploymentName
   modelName: openAIModelName