feat: Add tests back in

kieran-wilkinson-4 · kieran-wilkinson-4 · commit e5868555d9c6 · 2025-12-18T15:54:14.000Z
diff --git a/packages/cdk/prompts/systemPrompt.txt b/packages/cdk/prompts/systemPrompt.txt
@@ -1,27 +1,29 @@
-# 1. ROLE & OBJECTIVE
+# 1. Persona
 You are an AI assistant designed to provide guidance and references from your knowledge base to help users make decisions during onboarding.
 
-**CRITICAL RULE:** It is **VERY** important that you return **ALL** references found in the context for user examination.
+It is **VERY** important that you return **ALL** references found in the context for user examination.
 
 ---
 
 # 2. THINKING PROCESS & LOGIC
 Before generating a response, adhere to these processing rules:
 
-## Question Analysis
+## A. Question Analysis
 1.  **Detection:** Determine if the query contains one or multiple questions.
 2.  **Decomposition:** Split complex queries into individual sub-questions.
 3.  **Classification:** Identify if the question is Factual, Procedural, Diagnostic, Troubleshooting, or Clarification-seeking.
 4.  **Multi-Question Strategy:** Number sub-questions clearly (Q1, Q2, etc).
 
-## Entity Correction
+## B. Entity Correction
 - If you encounter "National Health Service Digital (NHSD)", automatically treat and output it as **"National Health Service England (NHSE)"**.
 
-## RAG Confidence Scoring
-Evaluate retrieved context using these thresholds:
+## C. RAG Confidence Scoring
+```
+Evaluate retrieved context using these relevance score thresholds:
 - `Score > 0.85` : **High confidence**
 - `Score 0.70 - 0.85` : **Medium confidence**
 - `Score < 0.70` : **Low confidence**
+```
 
 ---
 
@@ -52,14 +54,14 @@ You must use a specific variation of markdown. Follow this table strictly:
 ---
 
 # 5. BIBLIOGRAPHY GENERATOR
-## Requirements:
+**Requirements:**
 - Return **ALL** retrieved documents from the context.
 - Title length must be **< 50 characters**.
-- Text excerpt should be a long as possible and include text used for reasoning
+- Use the exact string format below (do not render it as a table or list).
 
-## Format
-Use the following format when creating bibliography entries:
-<cit>source number||summary title||file contents||relevance score</cit>
+**Template:**
+```text
+<cit>source number||summary title||excerpt||relevance score||source name</cit>
 
 # 6. Example
 """
@@ -70,6 +72,7 @@ Short summary text
 A longer answer, going into more detail gained from the knowledge base and using critical thinking.
 
 ------
-<cit>1||Some PDF file||This is the precise snippet of the pdf file which answers the question.||0.98</cit>
-<cit>1||Some TXT file||A 500 word text excerpt which gives some inference to the answer, but the long citation helps fill in the information for the user, so it's worth the tokens.||0.76</cit>
+<cit>1||A document||This is the precise snippet of the pdf file which answers the question.||0.98||very_helpful_doc.pdf</cit>
+<cit>2||Another file||A 500 word text excerpt which gives some inference to the answer, but the long citation helps fill in the information for the user, so it's worth the tokens.||0.76||something_interesting.txt</cit>
+<cit>3||A useless file||This file doesn't contain anything that useful||0.05||folder/another/some_file.txt</cit>
 """
diff --git a/packages/cdk/resources/BedrockPromptResources.ts b/packages/cdk/resources/BedrockPromptResources.ts
@@ -20,14 +20,12 @@ export class BedrockPromptResources extends Construct {
   constructor(scope: Construct, id: string, props: BedrockPromptResourcesProps) {
     super(scope, id)
 
-    // Nova Pro is recommended for text generation tasks requiring high accuracy and complex understanding.
-    const novaProModel = BedrockFoundationModel.AMAZON_NOVA_PRO_V1
-    // Nova Lite is recommended for tasks
-    const novaLiteModel = BedrockFoundationModel.AMAZON_NOVA_LITE_V1
+    const ragModel = new BedrockFoundationModel("meta.llama3-70b-instruct-v1:0")
+    const reformulationModel = BedrockFoundationModel.AMAZON_NOVA_LITE_V1
 
     const queryReformulationPromptVariant = PromptVariant.text({
       variantName: "default",
-      model: novaLiteModel,
+      model: reformulationModel,
       promptVariables: ["topic"],
       promptText: props.settings.reformulationPrompt.text
     })
@@ -41,7 +39,7 @@ export class BedrockPromptResources extends Construct {
 
     const ragResponsePromptVariant = PromptVariant.chat({
       variantName: "default",
-      model: novaProModel,
+      model: ragModel,
       promptVariables: ["query", "search_results"],
       system: props.settings.systemPrompt.text,
       messages: [props.settings.userPrompt]
@@ -59,8 +57,8 @@ export class BedrockPromptResources extends Construct {
     })
 
     // expose model IDs for use in Lambda environment variables
-    this.ragModelId = novaProModel.modelId
-    this.queryReformulationModelId = novaLiteModel.modelId
+    this.ragModelId = ragModel.modelId
+    this.queryReformulationModelId = reformulationModel.modelId
 
     this.queryReformulationPrompt = queryReformulationPrompt
     this.ragResponsePrompt = ragPrompt
diff --git a/packages/slackBotFunction/app/slack/slack_events.py b/packages/slackBotFunction/app/slack/slack_events.py
@@ -222,7 +222,6 @@ def _create_response_body(citations: list[dict[str, str]], feedback_data: dict[s
 
 
 def _create_citation(citation: dict[str, str], feedback_data: dict, response_text: str):
-    logger.info("Creating citation", extra={"Citation": citation})
     invalid_body = "No document excerpt available."
     action_buttons = []
 
diff --git a/packages/slackBotFunction/tests/test_slack_events/test_slack_events_actions.py b/packages/slackBotFunction/tests/test_slack_events/test_slack_events_actions.py
@@ -0,0 +1,276 @@
+import sys
+import pytest
+from unittest.mock import Mock, patch, MagicMock
+
+
+@pytest.fixture
+def mock_logger():
+    return MagicMock()
+
+
+@patch("app.utils.handler_utils.forward_event_to_pull_request_lambda")
+def test_process_async_slack_event_feedback(
+    mock_forward_event_to_pull_request_lambda: Mock,
+    mock_get_parameter: Mock,
+    mock_env: Mock,
+):
+    """Test successful async event processing"""
+    # set up mocks
+    mock_client = Mock()
+
+    # delete and import module to test
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+    from app.slack.slack_events import process_async_slack_event
+
+    # perform operation
+    slack_event_data = {
+        "text": "feedback: this is some feedback",
+        "user": "U456",
+        "channel": "C789",
+        "ts": "1234567890.123",
+    }
+    with patch("app.slack.slack_events.process_feedback_event") as mock_process_feedback_event, patch(
+        "app.slack.slack_events.process_slack_message"
+    ) as mock_process_slack_message:
+        process_async_slack_event(event=slack_event_data, event_id="evt123", client=mock_client)
+        mock_forward_event_to_pull_request_lambda.assert_not_called()
+        mock_process_feedback_event.assert_called_once_with(
+            message_text="feedback: this is some feedback",
+            conversation_key="thread#C789#1234567890.123",
+            user_id="U456",
+            channel_id="C789",
+            thread_root="1234567890.123",
+            client=mock_client,
+            event=slack_event_data,
+        )
+        mock_process_slack_message.assert_not_called()
+
+
+@patch("app.utils.handler_utils.is_latest_message")
+def test_process_async_slack_action_positive(
+    mock_is_latest_message: Mock,
+    mock_get_parameter: Mock,
+    mock_env: Mock,
+):
+    """Test successful async action processing"""
+    # set up mocks
+    mock_client = Mock()
+    mock_is_latest_message.return_value = True
+
+    # delete and import module to test
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+    from app.slack.slack_events import process_async_slack_action
+
+    feedback_value = '{"ck":"thread#C123#123","ch":"C123","mt":"1759845126.972219","tt":"1759845114.407989"}'
+
+    # perform operation
+    slack_action_data = {
+        "type": "block_actions",
+        "user": {"id": "U123"},
+        "channel": {"id": "C123"},
+        "message": {"ts": "1759845126.972219"},
+        "actions": [{"action_id": "feedback_yes", "value": feedback_value}],
+    }
+    with patch("app.slack.slack_events.store_feedback") as mock_store_feedback:
+        process_async_slack_action(body=slack_action_data, client=mock_client)
+
+        # assertions
+        mock_store_feedback.assert_called_once_with(
+            conversation_key="thread#C123#123",
+            feedback_type="positive",
+            user_id="U123",
+            channel_id="C123",
+            thread_ts="1759845114.407989",
+            message_ts="1759845126.972219",
+            client=mock_client,
+        )
+        mock_client.chat_postMessage.assert_called_once_with(
+            channel="C123",
+            text="Thank you for your feedback.",
+            thread_ts="1759845114.407989",
+        )
+
+
+@patch("app.utils.handler_utils.is_latest_message")
+def test_process_async_slack_action_negative(
+    mock_is_latest_message: Mock,
+    mock_get_parameter: Mock,
+    mock_env: Mock,
+):
+    """Test successful async action processing"""
+    # set up mocks
+    mock_client = Mock()
+    mock_is_latest_message.return_value = True
+
+    # delete and import module to test
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+    from app.slack.slack_events import process_async_slack_action
+
+    feedback_value = '{"ck":"thread#C123#123","ch":"C123","mt":"1759845126.972219","tt":"1759845114.407989"}'
+
+    # perform operation
+    slack_action_data = {
+        "type": "block_actions",
+        "user": {"id": "U123"},
+        "channel": {"id": "C123"},
+        "message": {"ts": "1759845126.972219"},
+        "actions": [{"action_id": "feedback_no", "value": feedback_value}],
+    }
+    with patch("app.slack.slack_events.store_feedback") as mock_store_feedback:
+        process_async_slack_action(body=slack_action_data, client=mock_client)
+
+        # assertions
+        mock_store_feedback.assert_called_once_with(
+            conversation_key="thread#C123#123",
+            feedback_type="negative",
+            user_id="U123",
+            channel_id="C123",
+            thread_ts="1759845114.407989",
+            message_ts="1759845126.972219",
+            client=mock_client,
+        )
+        mock_client.chat_postMessage.assert_called_once_with(
+            channel="C123",
+            text='Please let us know how the answer could be improved. Start your message with "feedback:"',
+            thread_ts="1759845114.407989",
+        )
+
+
+@patch("app.utils.handler_utils.is_latest_message")
+def test_process_async_slack_action_not_latest(
+    mock_is_latest_message: Mock,
+    mock_get_parameter: Mock,
+    mock_env: Mock,
+):
+    """Test successful async action processing"""
+    # set up mocks
+    mock_client = Mock()
+    mock_is_latest_message.return_value = False
+
+    # delete and import module to test
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+    from app.slack.slack_events import process_async_slack_action
+
+    feedback_value = '{"ck":"thread#C123#123","ch":"C123","mt":"1759845126.972219","tt":"1759845114.407989"}'
+
+    # perform operation
+    slack_action_data = {
+        "type": "block_actions",
+        "user": {"id": "U123"},
+        "channel": {"id": "C123"},
+        "actions": [{"action_id": "feedback_no", "value": feedback_value}],
+    }
+    with patch("app.slack.slack_events.store_feedback") as mock_store_feedback:
+        process_async_slack_action(body=slack_action_data, client=mock_client)
+
+        # assertions
+        mock_store_feedback.assert_not_called()
+        mock_client.chat_postMessage.assert_not_called()
+
+
+@patch("app.utils.handler_utils.is_latest_message")
+def test_process_async_slack_action_unknown_action(
+    mock_is_latest_message: Mock,
+    mock_get_parameter: Mock,
+    mock_env: Mock,
+):
+    """Test successful async action processing"""
+    # set up mocks
+    mock_client = Mock()
+    mock_is_latest_message.return_value = True
+
+    # delete and import module to test
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+    from app.slack.slack_events import process_async_slack_action
+
+    feedback_value = '{"ck":"thread#C123#123","ch":"C123","mt":"1759845126.972219","tt":"1759845114.407989"}'
+
+    # perform operation
+    slack_action_data = {
+        "type": "block_actions",
+        "user": {"id": "U123"},
+        "channel": {"id": "C123"},
+        "actions": [{"action_id": "I_Do_Not_Know_This_Action", "value": feedback_value}],
+    }
+    with patch("app.slack.slack_events.store_feedback") as mock_store_feedback:
+        process_async_slack_action(body=slack_action_data, client=mock_client)
+
+        # assertions
+        mock_store_feedback.assert_not_called()
+        mock_client.chat_postMessage.assert_not_called()
+
+
+def test_process_feedback_event():
+    # set up mocks
+    mock_client = Mock()
+
+    # delete and import module to test
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+    from app.slack.slack_events import process_feedback_event
+
+    # perform operation
+    mock_event = {}
+    with patch("app.slack.slack_events.store_feedback") as mock_store_feedback:
+        process_feedback_event(
+            message_text="feedback: this is some feedback",
+            conversation_key="thread#C123#123",
+            user_id="U123",
+            channel_id="C123",
+            thread_root="1759845114.407989",
+            event=mock_event,
+            client=mock_client,
+        )
+
+        # assertions
+        mock_store_feedback.assert_called_once_with(
+            conversation_key="thread#C123#123",
+            feedback_type="additional",
+            user_id="U123",
+            channel_id="C123",
+            thread_ts="1759845114.407989",
+            message_ts=None,
+            feedback_text="this is some feedback",
+            client=mock_client,
+        )
+        mock_client.chat_postMessage.assert_called_once_with(
+            channel="C123", text="Thank you for your feedback.", thread_ts="1759845114.407989"
+        )
+
+
+@patch("app.services.slack.post_error_message")
+def test_process_feedback_event_error(
+    mock_post_error_message: Mock,
+):
+    # set up mocks
+    mock_client = Mock()
+
+    # delete and import module to test
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+    from app.slack.slack_events import process_feedback_event
+
+    # perform operation
+    mock_event = {
+        "channel": "C123",
+        "thread_ts": "123",
+    }
+    with patch("app.slack.slack_events.store_feedback") as mock_store_feedback:
+        mock_store_feedback.side_effect = Exception("There was an error")
+        process_feedback_event(
+            message_text="feedback: this is some feedback",
+            conversation_key="thread#C123#123",
+            user_id="U123",
+            channel_id="C123",
+            thread_root="1759845114.407989",
+            event=mock_event,
+            client=mock_client,
+        )
+
+        # assertions
+        mock_post_error_message.assert_called_once_with(channel="C123", thread_ts="123", client=mock_client)
diff --git a/packages/slackBotFunction/tests/test_slack_events/test_slack_events_citations.py b/packages/slackBotFunction/tests/test_slack_events/test_slack_events_citations.py
diff --git a/packages/slackBotFunction/tests/test_slack_events/test_slack_events_messages.py b/packages/slackBotFunction/tests/test_slack_events/test_slack_events_messages.py
