feat(scribe): Log user interactions to #scribe-qa. (#5942)

Author: eyw520

servers/fai/src/fai/routes/settings.py

@@ -388,6 +388,7 @@ async def get_toggle_status(
         LOGGER.exception("Failed to get toggle status")
         return JSONResponse(content=jsonable_encoder(ToggleStatusResponse(status="failed", ask_ai_enabled=False)))
 
+
 @fai_app.post(
     "/settings/ask-ai/set-job-id",
     response_model=SetJobIdResponse,

servers/fai/src/fai/utils/scribe/pr_qa_logger.py

@@ -9,10 +9,74 @@
 )
 from fai.utils.slack.client import send_slack_message
 
+QA_CHANNEL_ID = "C0A0YHMKJUT"
+
+
+async def log_install_for_qa(integration: ScribeIntegrationDb) -> None:
+    try:
+        qa_bot_token = VARIABLES.SCRIBE_SLACK_BOT_TOKEN
+
+        message_text = f"""*SCRIBE APP INSTALLED* 🎉
+
+*Org:* {integration.slack_team_name or 'Unknown'}
+*Repository:* `{integration.github_repo}`
+"""
+
+        message_key = f"scribe_install_{integration.integration_id}"
+        await send_slack_message(
+            channel=QA_CHANNEL_ID,
+            text=message_text,
+            bot_token=qa_bot_token,
+            message_key=message_key,
+        )
+
+        LOGGER.info(f"[SCRIBE] Sent install notification to QA channel for {integration.slack_team_name}")
+
+    except Exception as e:
+        LOGGER.error(f"[SCRIBE] Failed to send install notification: {e}")
+
+
+async def log_pr_created_for_qa(session_db: ScribeSessionDb) -> None:
+    try:
+        qa_bot_token = VARIABLES.SCRIBE_SLACK_BOT_TOKEN
+
+        async with async_session_maker() as session:
+            result = await session.execute(
+                select(ScribeIntegrationDb).where(ScribeIntegrationDb.integration_id == session_db.integration_id)
+            )
+            integration = result.scalar_one_or_none()
+
+            if not integration:
+                LOGGER.warning(f"[SCRIBE] No integration found for session {session_db.id}")
+                return
+
+        message_text = f"""*SCRIBE PR CREATED* 🚀
+
+*Org:* {integration.slack_team_name or 'Unknown'}
+*Pull request:* {session_db.pr_url}
+"""
+
+        if session_db.devin_session_url:
+            message_text += f"*Devin Session:* {session_db.devin_session_url}\n"
+        if session_db.slack_thread_ts:
+            message_text += f"*Slack Thread:* https://slack.com/app_redirect?channel={session_db.slack_channel}&message_ts={session_db.slack_thread_ts}\n"
+
+        message_key = f"scribe_pr_created_{session_db.id}"
+        await send_slack_message(
+            channel=QA_CHANNEL_ID,
+            text=message_text,
+            bot_token=qa_bot_token,
+            message_key=message_key,
+        )
+
+        LOGGER.info(f"[SCRIBE] Sent PR created notification to QA channel for {session_db.pr_url}")
+
+    except Exception as e:
+        LOGGER.error(f"[SCRIBE] Failed to send PR created notification: {e}")
+
 
 async def log_merged_pr_for_qa(session_db: ScribeSessionDb, status: str) -> None:
     try:
-        qa_channel_id = "C0A0YHMKJUT"
         qa_bot_token = VARIABLES.SCRIBE_SLACK_BOT_TOKEN
 
         async with async_session_maker() as session:
@@ -34,9 +98,8 @@ async def log_merged_pr_for_qa(session_db: ScribeSessionDb, status: str) -> None
 
         message_text = f"""{title}
 
-*Repository:* `{integration.github_repo}`
-*PR URL:* {session_db.pr_url}
-*Team:* {integration.slack_team_name or 'Unknown'}
+*Org:* {integration.slack_team_name or 'Unknown'}
+*Pull request:* {session_db.pr_url}
 """
 
         if session_db.devin_session_url:
@@ -46,7 +109,7 @@ async def log_merged_pr_for_qa(session_db: ScribeSessionDb, status: str) -> None
 
         message_key = f"scribe_pr_{session_db.id}_{status}"
         await send_slack_message(
-            channel=qa_channel_id,
+            channel=QA_CHANNEL_ID,
             text=message_text,
             bot_token=qa_bot_token,
             message_key=message_key,

servers/fai/src/fai/utils/scribe/session_poller.py

@@ -130,6 +130,7 @@ async def poll_devin_session(
                         except Exception as e:
                             LOGGER.error(f"[SCRIBE] Failed to post message to Slack: {e}")
 
+            pr_was_created = False
             async with async_session_maker() as db_session:
                 result = await db_session.execute(select(ScribeSessionDb).where(ScribeSessionDb.id == session_id))
                 session_record = result.scalar_one_or_none()
@@ -144,10 +145,19 @@ async def poll_devin_session(
                         if pr_url:
                             session_record.pr_url = pr_url
                             session_record.pr_status = "open"
+                            pr_was_created = True
                             LOGGER.info(f"[SCRIBE] Stored PR URL for session {session_id}: {pr_url}")
 
                     await db_session.commit()
 
+            if pr_was_created and session_record:
+                from fai.utils.scribe.pr_qa_logger import log_pr_created_for_qa
+
+                try:
+                    await log_pr_created_for_qa(session_record)
+                except Exception as e:
+                    LOGGER.error(f"[SCRIBE] Failed to send PR created notification to QA channel: {e}")
+
             if status_enum in ["blocked", "stopped"]:
                 LOGGER.info(f"[SCRIBE] Devin session {devin_session_id} reached terminal state: {status_enum}")
                 break

servers/fai/src/fai/utils/slack/integration_common.py

@@ -216,6 +216,14 @@ async def handle_oauth_callback(
 
             LOGGER.info(f"{log_prefix} Successfully installed Slack app for team: {integration.slack_team_id}")
 
+            if log_prefix == "[SCRIBE]":
+                from fai.utils.scribe.pr_qa_logger import log_install_for_qa
+
+                try:
+                    await log_install_for_qa(integration)
+                except Exception as e:
+                    LOGGER.error(f"{log_prefix} Failed to send install notification to QA channel: {e}")
+
         metadata = {}
         if hasattr(integration, "domain"):
             metadata["domain"] = integration.domain

servers/fai/tests/utils/test_scribe_pr_qa_logger.py

@@ -0,0 +1,228 @@
+from unittest.mock import (
+    AsyncMock,
+    MagicMock,
+    patch,
+)
+
+import pytest
+
+from fai.models.db.scribe_integration_db import ScribeIntegrationDb
+from fai.models.db.scribe_session_db import ScribeSessionDb
+from fai.utils.scribe.pr_qa_logger import (
+    QA_CHANNEL_ID,
+    log_install_for_qa,
+    log_merged_pr_for_qa,
+    log_pr_created_for_qa,
+)
+
+
+@pytest.fixture
+def mock_integration() -> ScribeIntegrationDb:
+    integration = MagicMock(spec=ScribeIntegrationDb)
+    integration.integration_id = "test-integration-123"
+    integration.slack_team_name = "Test Org"
+    integration.github_repo = "test-org/test-repo"
+    return integration
+
+
+@pytest.fixture
+def mock_session() -> ScribeSessionDb:
+    session = MagicMock(spec=ScribeSessionDb)
+    session.id = "test-session-456"
+    session.integration_id = "test-integration-123"
+    session.pr_url = "https://github.com/test-org/test-repo/pull/42"
+    session.devin_session_url = "https://app.devin.ai/sessions/test-session"
+    session.slack_thread_ts = "1234567890.123456"
+    session.slack_channel = "C0123456789"
+    return session
+
+
+class TestLogInstallForQA:
+    @pytest.mark.asyncio
+    async def test_sends_install_notification(self, mock_integration: ScribeIntegrationDb) -> None:
+        with (
+            patch("fai.utils.scribe.pr_qa_logger.send_slack_message", new_callable=AsyncMock) as mock_send,
+            patch("fai.utils.scribe.pr_qa_logger.VARIABLES") as mock_vars,
+        ):
+            mock_vars.SCRIBE_SLACK_BOT_TOKEN = "test-bot-token"
+
+            await log_install_for_qa(mock_integration)
+
+            mock_send.assert_called_once()
+            call_args = mock_send.call_args
+            assert call_args.kwargs["channel"] == QA_CHANNEL_ID
+            assert call_args.kwargs["bot_token"] == "test-bot-token"
+            assert call_args.kwargs["message_key"] == "scribe_install_test-integration-123"
+
+            message_text = call_args.kwargs["text"]
+            assert "*SCRIBE APP INSTALLED* 🎉" in message_text
+            assert "*Org:* Test Org" in message_text
+            assert "*Repository:* `test-org/test-repo`" in message_text
+
+    @pytest.mark.asyncio
+    async def test_handles_unknown_org_name(self, mock_integration: ScribeIntegrationDb) -> None:
+        mock_integration.slack_team_name = None
+
+        with (
+            patch("fai.utils.scribe.pr_qa_logger.send_slack_message", new_callable=AsyncMock) as mock_send,
+            patch("fai.utils.scribe.pr_qa_logger.VARIABLES") as mock_vars,
+        ):
+            mock_vars.SCRIBE_SLACK_BOT_TOKEN = "test-bot-token"
+
+            await log_install_for_qa(mock_integration)
+
+            message_text = mock_send.call_args.kwargs["text"]
+            assert "*Org:* Unknown" in message_text
+
+
+class TestLogPRCreatedForQA:
+    @pytest.mark.asyncio
+    async def test_sends_pr_created_notification(
+        self, mock_session: ScribeSessionDb, mock_integration: ScribeIntegrationDb
+    ) -> None:
+        mock_db_session = AsyncMock()
+        mock_result = MagicMock()
+        mock_result.scalar_one_or_none.return_value = mock_integration
+        mock_db_session.execute = AsyncMock(return_value=mock_result)
+        mock_db_session.__aenter__ = AsyncMock(return_value=mock_db_session)
+        mock_db_session.__aexit__ = AsyncMock()
+
+        with (
+            patch("fai.utils.scribe.pr_qa_logger.async_session_maker", return_value=mock_db_session),
+            patch("fai.utils.scribe.pr_qa_logger.send_slack_message", new_callable=AsyncMock) as mock_send,
+            patch("fai.utils.scribe.pr_qa_logger.VARIABLES") as mock_vars,
+        ):
+            mock_vars.SCRIBE_SLACK_BOT_TOKEN = "test-bot-token"
+
+            await log_pr_created_for_qa(mock_session)
+
+            mock_send.assert_called_once()
+            call_args = mock_send.call_args
+            assert call_args.kwargs["channel"] == QA_CHANNEL_ID
+            assert call_args.kwargs["bot_token"] == "test-bot-token"
+            assert call_args.kwargs["message_key"] == "scribe_pr_created_test-session-456"
+
+            message_text = call_args.kwargs["text"]
+            assert "*SCRIBE PR CREATED* 🚀" in message_text
+            assert "*Org:* Test Org" in message_text
+            assert "*Pull request:* https://github.com/test-org/test-repo/pull/42" in message_text
+            assert "*Devin Session:* https://app.devin.ai/sessions/test-session" in message_text
+            assert (
+                "*Slack Thread:* https://slack.com/app_redirect?channel=C0123456789&message_ts=1234567890.123456"
+                in message_text
+            )
+
+    @pytest.mark.asyncio
+    async def test_handles_missing_optional_fields(
+        self, mock_session: ScribeSessionDb, mock_integration: ScribeIntegrationDb
+    ) -> None:
+        mock_session.devin_session_url = None
+        mock_session.slack_thread_ts = None
+
+        mock_db_session = AsyncMock()
+        mock_result = MagicMock()
+        mock_result.scalar_one_or_none.return_value = mock_integration
+        mock_db_session.execute = AsyncMock(return_value=mock_result)
+        mock_db_session.__aenter__ = AsyncMock(return_value=mock_db_session)
+        mock_db_session.__aexit__ = AsyncMock()
+
+        with (
+            patch("fai.utils.scribe.pr_qa_logger.async_session_maker", return_value=mock_db_session),
+            patch("fai.utils.scribe.pr_qa_logger.send_slack_message", new_callable=AsyncMock) as mock_send,
+            patch("fai.utils.scribe.pr_qa_logger.VARIABLES") as mock_vars,
+        ):
+            mock_vars.SCRIBE_SLACK_BOT_TOKEN = "test-bot-token"
+
+            await log_pr_created_for_qa(mock_session)
+
+            message_text = mock_send.call_args.kwargs["text"]
+            assert "*Devin Session:*" not in message_text
+            assert "*Slack Thread:*" not in message_text
+
+
+class TestLogMergedPRForQA:
+    @pytest.mark.asyncio
+    async def test_sends_merged_pr_notification(
+        self, mock_session: ScribeSessionDb, mock_integration: ScribeIntegrationDb
+    ) -> None:
+        mock_db_session = AsyncMock()
+        mock_result = MagicMock()
+        mock_result.scalar_one_or_none.return_value = mock_integration
+        mock_db_session.execute = AsyncMock(return_value=mock_result)
+        mock_db_session.__aenter__ = AsyncMock(return_value=mock_db_session)
+        mock_db_session.__aexit__ = AsyncMock()
+
+        with (
+            patch("fai.utils.scribe.pr_qa_logger.async_session_maker", return_value=mock_db_session),
+            patch("fai.utils.scribe.pr_qa_logger.send_slack_message", new_callable=AsyncMock) as mock_send,
+            patch("fai.utils.scribe.pr_qa_logger.VARIABLES") as mock_vars,
+        ):
+            mock_vars.SCRIBE_SLACK_BOT_TOKEN = "test-bot-token"
+
+            await log_merged_pr_for_qa(mock_session, "merged")
+
+            mock_send.assert_called_once()
+            call_args = mock_send.call_args
+            assert call_args.kwargs["channel"] == QA_CHANNEL_ID
+            assert call_args.kwargs["bot_token"] == "test-bot-token"
+            assert call_args.kwargs["message_key"] == "scribe_pr_test-session-456_merged"
+
+            message_text = call_args.kwargs["text"]
+            assert "*SCRIBE PR MERGED* ✅" in message_text
+            assert "*Org:* Test Org" in message_text
+            assert "*Pull request:* https://github.com/test-org/test-repo/pull/42" in message_text
+            assert "*Devin Session:* https://app.devin.ai/sessions/test-session" in message_text
+            assert (
+                "*Slack Thread:* https://slack.com/app_redirect?channel=C0123456789&message_ts=1234567890.123456"
+                in message_text
+            )
+
+    @pytest.mark.asyncio
+    async def test_sends_closed_pr_notification(
+        self, mock_session: ScribeSessionDb, mock_integration: ScribeIntegrationDb
+    ) -> None:
+        mock_db_session = AsyncMock()
+        mock_result = MagicMock()
+        mock_result.scalar_one_or_none.return_value = mock_integration
+        mock_db_session.execute = AsyncMock(return_value=mock_result)
+        mock_db_session.__aenter__ = AsyncMock(return_value=mock_db_session)
+        mock_db_session.__aexit__ = AsyncMock()
+
+        with (
+            patch("fai.utils.scribe.pr_qa_logger.async_session_maker", return_value=mock_db_session),
+            patch("fai.utils.scribe.pr_qa_logger.send_slack_message", new_callable=AsyncMock) as mock_send,
+            patch("fai.utils.scribe.pr_qa_logger.VARIABLES") as mock_vars,
+        ):
+            mock_vars.SCRIBE_SLACK_BOT_TOKEN = "test-bot-token"
+
+            await log_merged_pr_for_qa(mock_session, "closed")
+
+            message_text = mock_send.call_args.kwargs["text"]
+            assert "*SCRIBE PR CLOSED* ❌" in message_text
+
+    @pytest.mark.asyncio
+    async def test_updated_format_uses_org_not_team(
+        self, mock_session: ScribeSessionDb, mock_integration: ScribeIntegrationDb
+    ) -> None:
+        mock_db_session = AsyncMock()
+        mock_result = MagicMock()
+        mock_result.scalar_one_or_none.return_value = mock_integration
+        mock_db_session.execute = AsyncMock(return_value=mock_result)
+        mock_db_session.__aenter__ = AsyncMock(return_value=mock_db_session)
+        mock_db_session.__aexit__ = AsyncMock()
+
+        with (
+            patch("fai.utils.scribe.pr_qa_logger.async_session_maker", return_value=mock_db_session),
+            patch("fai.utils.scribe.pr_qa_logger.send_slack_message", new_callable=AsyncMock) as mock_send,
+            patch("fai.utils.scribe.pr_qa_logger.VARIABLES") as mock_vars,
+        ):
+            mock_vars.SCRIBE_SLACK_BOT_TOKEN = "test-bot-token"
+
+            await log_merged_pr_for_qa(mock_session, "merged")
+
+            message_text = mock_send.call_args.kwargs["text"]
+            assert "*Org:*" in message_text
+            assert "*Pull request:*" in message_text
+            assert "*Team:*" not in message_text
+            assert "*Repository:*" not in message_text
+            assert "*PR URL:*" not in message_text