fix: Set Bedrock client backoff to handle per-minute throttles

Author: clareliguori

e2e_tests/python/chat_session.py

@@ -68,10 +68,6 @@ async def start(self) -> None:
             messages = []
 
             for i, user_input in enumerate(self.user_utterances):
-                if i != 0:
-                    print("\n**Pausing 30 seconds to avoid Bedrock throttling**")
-                    await asyncio.sleep(30)
-
                 print(f"\nYou: {user_input}")
 
                 messages.append({"role": "user", "content": [{"text": user_input}]})

e2e_tests/python/main.py

@@ -1,5 +1,6 @@
 import asyncio
 import boto3
+from botocore.config import Config
 import json
 import logging
 import os
@@ -57,7 +58,15 @@ def bedrock_client(self) -> Any:
         Returns:
             The Bedrock client.
         """
-        return boto3.client("bedrock-runtime", region_name=self.region)
+        retry_config = Config(
+            retries={
+                "max_attempts": 10,
+                "mode": "standard",
+            }
+        )
+        return boto3.client(
+            "bedrock-runtime", region_name=self.region, config=retry_config
+        )
 
 
 async def main() -> None:

e2e_tests/typescript/src/chat_session.ts

@@ -82,11 +82,6 @@ export class ChatSession {
       const messages: Message[] = [];
 
       for (const [i, userInput] of this.userUtterances.entries()) {
-        if (i != 0) {
-          console.log("\n**Pausing 30 seconds to avoid Bedrock throttling**");
-          await new Promise((resolve) => setTimeout(resolve, 30 * 1000));
-        }
-
         console.log(`\You: ${userInput}`);
 
         messages.push({ role: "user", content: [{ text: userInput }] });

e2e_tests/typescript/src/configuration.ts

@@ -43,6 +43,10 @@ export class Configuration {
    * @returns The Bedrock client.
    */
   get bedrockClient(): BedrockRuntimeClient {
-    return new BedrockRuntimeClient({ region: this.region });
+    return new BedrockRuntimeClient({
+      region: this.region,
+      retryMode: "standard",
+      maxAttempts: 10,
+    });
   }
 }

examples/chatbots/python/chat_session.py

@@ -66,7 +66,7 @@ async def start(self) -> None:
             while True:
                 try:
                     user_input = input("\nYou: ").strip().lower()
-                    if user_input in ["quit", "exit"]:
+                    if user_input in ["quit", "exit", "/quit", "/exit"]:
                         logging.info("\nExiting...")
                         break
 

examples/chatbots/python/main.py

@@ -1,5 +1,6 @@
 import asyncio
 import boto3
+from botocore.config import Config
 import json
 import logging
 import os
@@ -57,7 +58,15 @@ def bedrock_client(self) -> Any:
         Returns:
             The Bedrock client.
         """
-        return boto3.client("bedrock-runtime", region_name=self.region)
+        retry_config = Config(
+            retries={
+                "max_attempts": 10,
+                "mode": "standard",
+            }
+        )
+        return boto3.client(
+            "bedrock-runtime", region_name=self.region, config=retry_config
+        )
 
 
 async def main() -> None:

examples/chatbots/typescript/src/chat_session.ts

@@ -78,7 +78,12 @@ export class ChatSession {
 
       while (true) {
         const userInput = readline.question("\nYou: ").trim().toLowerCase();
-        if (userInput === "quit" || userInput === "exit") {
+        if (
+          userInput === "quit" ||
+          userInput === "exit" ||
+          userInput === "/quit" ||
+          userInput === "/exit"
+        ) {
           logger.info("\nExiting...");
           break;
         }

examples/chatbots/typescript/src/configuration.ts

@@ -43,6 +43,10 @@ export class Configuration {
    * @returns The Bedrock client.
    */
   get bedrockClient(): BedrockRuntimeClient {
-    return new BedrockRuntimeClient({ region: this.region });
+    return new BedrockRuntimeClient({
+      region: this.region,
+      retryMode: "standard",
+      maxAttempts: 10,
+    });
   }
 }