chore: hard codes inference config for test purposes

Author: bencegadanyi1-nhs

packages/cdk/resources/BedrockPromptResources.ts

@@ -37,6 +37,7 @@ User Query: {{user_query}}`
       variants: [queryReformulationPromptVariant]
     })
 
+    // TODO: add inference settings
     const ragResponsePromptVariant = PromptVariant.chat({
       variantName: "default",
       model: claudeSonnetModel,

packages/slackBotFunction/app/services/bedrock.py

@@ -47,7 +47,17 @@ def query_bedrock(user_query: str, session_id: str = None) -> RetrieveAndGenerat
                     "guardrailConfiguration": {
                         "guardrailId": GUARD_RAIL_ID,
                         "guardrailVersion": GUARD_VERSION,
-                    }
+                    },
+                    "inferenceConfig": {
+                        "textInferenceConfig": {
+                            "temperature": 0,
+                            "topP": 0.9,
+                            "maxTokens": 512,
+                            "stopSequences": [
+                                "Human:",
+                            ],
+                        }
+                    },
                 },
             },
         },

packages/slackBotFunction/app/services/prompt_loader.py

@@ -97,6 +97,10 @@ def load_prompt(prompt_name: str, prompt_version: str = None) -> str:
             response = client.get_prompt(promptIdentifier=prompt_id)
 
         template_config = response["variants"][0]["templateConfiguration"]
+        # TODO: derive actual inference config then pass it along with prompt text to the retrieve_and_generate call
+        # so that all settings from the prompt management are applied directly from the cdk
+        # inference_config = response["variants"][0]["inferenceConfiguration"]
+
         prompt_text = _render_prompt(template_config)
         actual_version = response.get("version", "DRAFT")