Merge pull request #67 from CMU-17313Q/p4-ci-workflow

P4 CI LLM Workflow

Author: FaizFarooqMoazam

.github/workflows/P4ci.yml

@@ -0,0 +1,29 @@
+name: P4CI Pipeline
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Run tests with PYTHONPATH
+      run: |
+        export PYTHONPATH="$PYTHONPATH:$(pwd)/src"
+        pytest -q

requirements.txt

@@ -0,0 +1,10 @@
+pytest
+pytest-mock
+
+# Only include ollama if your team actually needs it
+# but keep it optional-friendly for CI
+ollama
+
+# If your notebook / code uses any of these, include them:
+requests
+python-dotenv

src/llm_experiment.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+from typing import Tuple
+import os
+# LLM client setup copied from notebook
+try:
+    from ollama import Client
+except Exception:  # if ollama isn't installed in some env, don't crash imports
+    Client = None  # type: ignore
+MODEL_NAME = os.getenv("MODEL_NAME", "Llama3.1:8b")
+OLLAMA_URL = os.getenv("OLLAMA_HOST", "localhost:11434")
+class _DummyClient:
+    """Fallback so that importing this module doesn't explode in CI."""
+    def chat(self, *args, **kwargs):
+        class Message:
+            def __init__(self, content):
+                self.content = content
+        class Response:
+            def __init__(self, content):
+                self.message = Message(content)
+        return Response("dummy response")
+if Client:
+    client = Client(host=OLLAMA_URL)
+else:
+    client = _DummyClient()
+def get_translation(post: str) -> str:
+    """
+    Translate a non-English post into English using the local Ollama model.
+    (Copied from notebook.)
+    """
+    context = (
+        "You are a translation assistant. Translate the following text into natural English. "
+        "Only output the translated text, with no explanations or commentary."
+    )
+    try:
+        prompt = f"{context}\n\nText to translate:\n{post}"
+        response = client.chat(
+            model=MODEL_NAME,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        return response.message.content.strip()
+    except Exception as e:
+        return f"[Error: {type(e).__name__} - {e}]"
+def get_language(post: str) -> str:
+    """
+    Detects the language of a given post using the LLM.
+    Returns the name of the language in English (e.g. 'German', 'Spanish', 'Chinese', etc.)
+    (Copied from notebook, with a small robustness tweak.)
+    """
+    prompt = (
+        "Identify the language of the following text. "
+        "Respond with only the language name in English (for example, 'German', 'Spanish', 'Chinese'). "
+        "Do not answer in the language itself.\n\n"
+        f"Text:\n{post}"
+    )
+    try:
+        response = client.chat(
+            model=MODEL_NAME,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        if isinstance(response, dict):
+            content = response.get("message", {}).get("content", "")
+        else:
+            content = getattr(getattr(response, "message", None), "content", "")
+        return str(content).strip()
+    except Exception as e:
+        return f"[Error: {type(e).__name__} - {e}]"
+def query_llm_robust(post: str) -> tuple[bool, str]:
+    """
+    A robust version of query_llm that safely handles unexpected model responses or errors.
+    Ensures output is always in the correct format (bool, str).
+    (Exactly your notebook code.)
+    """
+    try:
+        # Try language detection
+        lang = get_language(post)
+        # Validate language detection output
+        if not isinstance(lang, str) or len(lang.strip()) == 0:
+            # Model gave empty or invalid response
+            return False, "[Invalid language detection output]"
+        # Determine if English
+        is_english = lang.strip().lower() in ["english", "en"]
+        # If English, return original post
+        if is_english:
+            return True, post.strip()
+        # If not English, attempt translation
+        translation = get_translation(post)
+        # Validate translation output
+        if not isinstance(translation, str) or len(translation.strip()) == 0:
+            return False, "[Invalid translation output]"
+        return False, translation.strip()
+    except Exception as e:
+        return False, f"[Error: {type(e).__name__} - {e}]"

test/test_llm_experiment.py

@@ -0,0 +1,59 @@
+
+from unittest.mock import patch
+from src.llm_experiment import query_llm_robust, client
+
+
+@patch.object(client, "chat")
+def test_unexpected_language(mock_chat):
+    # we mock the model's response to return a random message
+    mock_chat.return_value.message.content = "I don't understand your request"
+
+    # just check it returns a valid tuple and doesn't crash
+    result = query_llm_robust("Hier ist dein erstes Beispiel.")
+    assert isinstance(result, tuple)
+    assert isinstance(result[0], bool)
+    assert isinstance(result[1], str)
+
+
+@patch.object(client, "chat")
+def test_empty_response(mock_chat):
+    mock_chat.return_value.message.content = ""
+    result = query_llm_robust("Bonjour le monde")
+    assert isinstance(result, tuple)
+    assert result[0] is False
+
+
+@patch.object(client, "chat")
+def test_nonstring_response(mock_chat):
+    mock_chat.return_value.message.content = {"text": "Hello"}
+    result = query_llm_robust("Hola amigo")
+    assert result[0] is False
+
+
+@patch.object(client, "chat", side_effect=Exception("Network error"))
+def test_model_exception(mock_chat):
+    result = query_llm_robust("Ciao amico")
+    assert result[0] is False
+
+
+@patch.object(client, "chat")
+def test_none_response(mock_chat):
+    mock_chat.return_value.message.content = None
+    result = query_llm_robust("こんにちは")
+    assert result[0] is False
+    assert "Invalid" in result[1] or "Error" in result[1]
+
+
+@patch.object(client, "chat")
+def test_very_long_response(mock_chat):
+    mock_chat.return_value.message.content = "Hello" * 10000
+    result = query_llm_robust("Привет")
+    assert isinstance(result, tuple)
+    assert len(result[1]) < 60000
+
+
+@patch.object(client, "chat")
+def test_gibberish_response(mock_chat):
+    mock_chat.return_value.message.content = "�#@!∂ƒ©˙∆˚¬…æ≈ç√"
+    result = query_llm_robust("안녕하세요")
+    assert result[0] is False