feat: Add tests back for citations

Author: kieran-wilkinson-4

packages/cdk/prompts/systemPrompt.txt

@@ -8,21 +8,32 @@ It is **VERY** important that you return **ALL** references found in the context
 # 2. THINKING PROCESS & LOGIC
 Before generating a response, adhere to these processing rules:
 
-## A. Question Analysis
+## A. Context Verification
+Scan the retrieved context for the specific answer
+1. **No information found**: If the information is not present in the context:
+   - Do NOT formulate a general answer.
+   - Do NOT user external resources (i.e., websites, etc) to get an answer.
+   - Do NOT infer an answer from the users question.
+
+## B. Question Analysis
 1.  **Detection:** Determine if the query contains one or multiple questions.
 2.  **Decomposition:** Split complex queries into individual sub-questions.
 3.  **Classification:** Identify if the question is Factual, Procedural, Diagnostic, Troubleshooting, or Clarification-seeking.
 4.  **Multi-Question Strategy:** Number sub-questions clearly (Q1, Q2, etc).
+5.  **No Information:** If there is no information supporting an answer to the query, do not try and fill in the information
+6. **Strictness:** Do not infer information, be strict on evidence.
 
-## B. Entity Correction
+## C. Entity Correction
 - If you encounter "National Health Service Digital (NHSD)", automatically treat and output it as **"National Health Service England (NHSE)"**.
 
-## C. RAG Confidence Scoring
+## D. RAG Confidence Scoring
 ```
 Evaluate retrieved context using these relevance score thresholds:
-- `Score > 0.85` : **High confidence**
-- `Score 0.70 - 0.85` : **Medium confidence**
-- `Score < 0.70` : **Low confidence**
+- `Score > 0.9`     : **Diamond** (Definitive source)
+- `Score 0.8 - 0.9` : **Gold** (Strong evidence)
+- `Score 0.7 - 0.8` : **Silver** (Partial context)
+- `Score 0.6 - 0.7` : **Bronze** (Weak relevance)
+- `Score < 0.6`     : **Scrap** (Ignore completely)
 ```
 
 ---

packages/slackBotFunction/app/slack/slack_events.py

@@ -203,6 +203,7 @@ def _create_response_body(citations: list[dict[str, str]], feedback_data: dict[s
             response_text = result.get("response_text", response_text)
 
     # Remove any citations that have not been returned
+    response_text = convert_markdown_to_slack(response_text)
     response_text = response_text.replace("cit_", "")
 
     # Main body
@@ -226,35 +227,56 @@ def _create_citation(citation: dict[str, str], feedback_data: dict, response_tex
     action_buttons = []
 
     # Create citation blocks ["source_number", "title", "excerpt", "relevance_score"]
-    source_number = (citation.get("source_number", "0")).replace("\n", "")
-    title = citation.get("title") or citation.get("filename") or "Source"
-    body = citation.get("excerpt") or invalid_body
-    score = citation.get("relevance_score") or "0"
-
-    # Buttons can only be 75 characters long, truncate to be safe
-    button_text = f"[{source_number}] {title}"
-    button_value = {**feedback_data, "source_number": source_number, "title": title, "body": body, "score": score}
-    button = {
-        "type": "button",
-        "text": {
-            "type": "plain_text",
-            "text": button_text if len(button_text) < 75 else f"{button_text[:70]}...",
-        },
-        "action_id": f"cite_{source_number}",
-        "value": json.dumps(
-            button_value,
-            separators=(",", ":"),
-        ),
-    }
-    action_buttons.append(button)
+    source_number: str = (citation.get("source_number", "0")).replace("\n", "")
+    title: str = citation.get("title") or citation.get("filename") or "Source"
+    body: str = citation.get("excerpt") or invalid_body
+    score: float = float(citation.get("relevance_score") or "0")
+
+    # Format body
+    body = convert_markdown_to_slack(body)
 
-    # Update inline citations to remove "cit_" prefix
-    response_text = response_text.replace(f"[cit_{source_number}]", f"[{source_number}]")
+    if score < 60:  # low relevance score, skip citation
+        logger.info("Skipping low relevance citation", extra={"source_number": source_number, "score": score})
+    else:
+        # Buttons can only be 75 characters long, truncate to be safe
+        button_text = f"[{source_number}] {title}"
+        button_value = {**feedback_data, "source_number": source_number, "title": title, "body": body, "score": score}
+        button = {
+            "type": "button",
+            "text": {
+                "type": "plain_text",
+                "text": button_text if len(button_text) < 75 else f"{button_text[:70]}...",
+            },
+            "action_id": f"cite_{source_number}",
+            "value": json.dumps(
+                button_value,
+                separators=(",", ":"),
+            ),
+        }
+        action_buttons.append(button)
+
+        # Update inline citations to remove "cit_" prefix
+        response_text = response_text.replace(f"[cit_{source_number}]", f"[{source_number}]")
+        logger.info("Created citation", extra=button_value)
 
-    logger.info("Created citation", extra=button_value)
     return {"action_buttons": action_buttons, "response_text": response_text}
 
 
+def convert_markdown_to_slack(body: str) -> str:
+    """Convert basic markdown to Slack formatting"""
+    # Fix common encoding issues
+    body = body.replace("»", "")  # Remove double chevrons
+    body = body.replace("â¢", "-")  # Replace bullet points with encoding issues
+
+    # Simple markdown conversions
+    body = re.sub(r"(\*{1,2}|_{1,2})([^\*_]+)\1", r"_\2_", body)  # Italic (Do this first to avoid conflict with bold)
+    body = body.replace("**", "*")  # Bold
+
+    body = re.sub(r"(\u2022|-)\s", r"\n\g<0>", body)  # Ensure bullet points on new lines
+    body = re.sub(r"\[([^\]]+)\]\(([^\)]+)\)", r"<\1|\2>", body)  # Convert links
+    return body
+
+
 # ================================================================
 # Main async event processing
 # ================================================================
@@ -666,26 +688,35 @@ def open_citation(channel: str, timestamp: str, message: Any, params: Dict[str,
 
 
 def format_blocks(blocks: Any, current_id: str):
+    """Format blocks by styling the selected citation button and unstyle others"""
     selected = False
+
     for block in blocks:
-        if block.get("type") == "actions":
-            for element in block.get("elements", []):
-                if element.get("type") == "button":
-                    action_id = element.get("action_id")
-                    if action_id == current_id:
-                        # Toggle: if already styled, unselect; else select
-                        if element.get("style") == "primary":
-                            element.pop("style", None)
-                            selected = False
-                        else:
-                            element["style"] = "primary"
-                            selected = True
-                    else:
-                        # Unselect all other buttons
-                        element.pop("style", None)
+        if block.get("type") != "actions":
+            continue
+
+        for element in block.get("elements", []):
+            if element.get("type") != "button":
+                continue
+
+            if element.get("action_id") == current_id:
+                selected = _toggle_button_style(element)
+            else:
+                element.pop("style", None)
+
     return {"selected": selected, "blocks": blocks}
 
 
+def _toggle_button_style(element: dict) -> bool:
+    """Toggle button style and return whether it's now selected"""
+    if element.get("style") == "primary":
+        element.pop("style", None)
+        return False
+    else:
+        element["style"] = "primary"
+        return True
+
+
 # ================================================================
 # Session management
 # ================================================================

packages/slackBotFunction/tests/test_slack_events/test_slack_events_citations.py

@@ -322,3 +322,77 @@ def test_process_citation_events_update_chat_message_change_close_citation():
     expected_blocks = [citations, second_citation_body]
     mock_client.chat_update.assert_called()
     mock_client.chat_update.assert_called_with(channel="ABC", ts="123", blocks=expected_blocks)
+
+
+def test_create_response_body_no_error_without_citations(
+    mock_get_parameter: Mock,
+    mock_env: Mock,
+):
+    """Test regex text processing functionality within process_async_slack_event"""
+    # delete and import module to test
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+    from app.slack.slack_events import _create_response_body
+
+    # perform operation
+    _create_response_body(
+        citations=[],
+        feedback_data={},
+        response_text="This is a response without a citation.[1]",
+    )
+
+    # assertions
+    # no assertions as we are just checking it does not throw an error
+
+
+def test_create_response_body_creates_body_without_citations(
+    mock_get_parameter: Mock,
+    mock_env: Mock,
+):
+    """Test regex text processing functionality within process_async_slack_event"""
+    # delete and import module to test
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+    from app.slack.slack_events import _create_response_body
+
+    # perform operation
+    response = _create_response_body(
+        citations=[],
+        feedback_data={},
+        response_text="This is a response without a citation.",
+    )
+
+    # assertions
+    assert len(response) > 0
+    assert response[0]["type"] == "section"
+    assert "This is a response without a citation." in response[0]["text"]["text"]
+
+
+def test_create_response_body_creates_body_with_citations(
+    mock_get_parameter: Mock,
+    mock_env: Mock,
+):
+    """Test regex text processing functionality within process_async_slack_event"""
+    # delete and import module to test
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+    from app.slack.slack_events import _create_response_body
+
+    # perform operation
+    response = _create_response_body(
+        citations=[
+            {
+                "source_number": "1",
+                "title": "Citation Title",
+                "body": "Citation Body",
+                "relevance_score": "0.95",
+            }
+        ],
+        feedback_data={},
+        response_text="This is a response with a citation.[1]",
+    )
+
+    # assertions
+    assert len(response) > 1
+    assert response[0]["type"] == "section"
+    assert "This is a response with a citation.[1]" in response[0]["text"]["text"]

packages/slackBotFunction/tests/test_slack_events/test_slack_events_messages.py

@@ -259,7 +259,7 @@ def test_process_slack_message_with_session_storage(
 @patch("app.services.dynamo.get_state_information")
 @patch("app.services.ai_processor.process_ai_query")
 @patch("app.slack.slack_events.get_conversation_session")
-def test_process_slack_message_chat_update_error(
+def test_process_slack_message_chat_update_no_error(
     mock_get_session: Mock,
     mock_process_ai_query: Mock,
     mock_get_state_information: Mock,
@@ -292,6 +292,48 @@ def test_process_slack_message_chat_update_error(
     # no assertions as we are just checking it does not throw an error
 
 
+@patch("app.slack.slack_events.get_conversation_session")
+@patch("app.slack.slack_events.get_conversation_session_data")
+@patch("app.slack.slack_events.cleanup_previous_unfeedback_qa")
+@patch("app.slack.slack_events.update_session_latest_message")
+@patch("app.services.ai_processor.process_ai_query")
+def test_process_slack_message_chat_update_cleanup(
+    mock_process_ai_query: Mock,
+    mock_update_session_latest_message: Mock,
+    mock_cleanup_previous_unfeedback_qa: Mock,
+    mock_get_conversation_session_data: Mock,
+    mock_get_session: Mock,
+    mock_get_parameter: Mock,
+    mock_env: Mock,
+):
+    """Test process_async_slack_event with chat_update error"""
+    # set up mocks
+    mock_client = Mock()
+    mock_client.chat_postMessage.return_value = {"ts": "1234567890.124"}
+    mock_client.chat_update.side_effect = Exception("Update failed")
+    mock_process_ai_query.return_value = {
+        "text": "AI response",
+        "session_id": "session-123",
+        "citations": [],
+        "kb_response": {"output": {"text": "AI response"}},
+    }
+    mock_get_conversation_session_data.return_value = {"session_id": "session-123"}
+    mock_get_session.return_value = None  # No existing session
+    mock_cleanup_previous_unfeedback_qa.return_value = {"test": "123"}
+
+    # delete and import module to test
+    from app.slack.slack_events import process_slack_message
+
+    # perform operation
+    slack_event_data = {"text": "<@U123> test question", "user": "U456", "channel": "C789", "ts": "1234567890.123"}
+    with patch("app.slack.slack_events.get_conversation_session_data", mock_get_conversation_session_data):
+        process_slack_message(event=slack_event_data, event_id="evt123", client=mock_client)
+
+        # assertions
+        mock_cleanup_previous_unfeedback_qa.assert_called_once()
+        mock_update_session_latest_message.assert_called_once()
+
+
 @patch("app.services.dynamo.get_state_information")
 @patch("app.services.ai_processor.process_ai_query")
 @patch("app.slack.slack_events.get_conversation_session")
@@ -331,3 +373,46 @@ def test_process_slack_message_dm_context(
 
     # assertions
     # no assertions as we are just checking it does not throw an error
+
+
+@patch("app.services.dynamo.delete_state_information")
+def test_cleanup_previous_unfeedback_qa_no_previous_message(
+    mock_delete_state_information: Mock,
+):
+    """Test cleanup skipped when no previous message exists"""
+    conversation_key = "conv-123"
+    current_message_ts = "1234567890.124"
+    session_data = {}
+
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+    from app.slack.slack_events import cleanup_previous_unfeedback_qa
+
+    # perform operation
+    cleanup_previous_unfeedback_qa(conversation_key, current_message_ts, session_data)
+
+    # assertions
+    mock_delete_state_information.assert_not_called()
+
+
+@patch("app.services.dynamo.delete_state_information")
+def test_cleanup_previous_unfeedback_qa_same_message(
+    mock_delete_state_information: Mock,
+):
+    """Test cleanup skipped when previous message is same as current"""
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+
+    conversation_key = "conv-123"
+    current_message_ts = "1234567890.123"
+    session_data = {"latest_message_ts": "1234567890.123"}
+
+    if "app.slack.slack_events" in sys.modules:
+        del sys.modules["app.slack.slack_events"]
+    from app.slack.slack_events import cleanup_previous_unfeedback_qa
+
+    # perform operation
+    cleanup_previous_unfeedback_qa(conversation_key, current_message_ts, session_data)
+
+    # assertions
+    mock_delete_state_information.assert_not_called()