feat: implement extractive text summarization with MMR

Author: prithaxdev

CLAUDE.md

@@ -512,7 +512,7 @@ data: create seed dataset of 50 code snippets
 
 #### Week 3 (Apr 27 – May 2): Summarizer + FastAPI + Proposal Defense
 
-- [ ] **Day 10 — Mon, Apr 27:** Implement `summarizer.py` — `extractive_summarize(snippets, max_points)` with TF-IDF scoring + MMR diversity selection. Write `test_summarizer.py`. Commit: `feat: implement extractive text summarization with MMR`
+- [x] **Day 10 — Mon, Apr 27:** Implement `summarizer.py` — `extractive_summarize(snippets, max_points)` with TF-IDF scoring + MMR diversity selection. Write `test_summarizer.py`. Commit: `feat: implement extractive text summarization with MMR`
 
 - [ ] **Day 11 — Tue, Apr 28:** Create Pydantic models in `models/schemas.py`. Create `routes/` for all 5 endpoints. Wire into `main.py` with CORS. Test with Swagger UI at /docs. Commit: `feat: create FastAPI routes and Pydantic schemas`
 
@@ -720,7 +720,7 @@ NEXT_PUBLIC_API_URL=http://localhost:8000
 ## Status
 
 **Current Phase:** Phase 1 — Algorithm Core
-**Current Day:** Day 10 (Mon, Apr 27, 2026)
-**Last Updated:** Sat, Apr 25, 2026
-**Total Progress:** 9/68 tasks completed
+**Current Day:** Day 11 (Tue, Apr 28, 2026)
+**Last Updated:** Mon, Apr 27, 2026
+**Total Progress:** 10/68 tasks completed
 **Next milestone:** Proposal Defense (~early May 2026)

apps/api/app/algorithms/summarizer.py

@@ -0,0 +1,141 @@
+"""Extractive text summarization with Maximal Marginal Relevance (MMR).
+
+Scores sentences by TF-IDF weight, then selects diverse points using MMR
+to avoid redundancy across the generated thread draft.
+
+Reference: Carbonell, J. & Goldstein, J. (1998). The use of MMR, diversity-based
+reranking for reordering documents and producing summaries. SIGIR '98.
+"""
+
+from .similarity import cosine_similarity
+from .tfidf import TFIDFEngine, tokenize, compute_tf, tfidf_vector
+
+
+def _sentence_tfidf_score(sentence: str, idf: dict[str, float]) -> float:
+    """Score a sentence by summing its TF-IDF weights.
+
+    Args:
+        sentence: Raw sentence string.
+        idf: Fitted IDF table from the corpus.
+
+    Returns:
+        Sum of TF-IDF weights for all terms in the sentence.
+    """
+    tokens = tokenize(sentence)
+    if not tokens:
+        return 0.0
+    tf = compute_tf(tokens)
+    vec = tfidf_vector(tf, idf)
+    return sum(vec.values())
+
+
+def _sentence_vector(sentence: str, idf: dict[str, float]) -> dict[str, float]:
+    """Produce a TF-IDF vector for a single sentence.
+
+    Args:
+        sentence: Raw sentence string.
+        idf: Fitted IDF table.
+
+    Returns:
+        Sparse TF-IDF vector.
+    """
+    tokens = tokenize(sentence)
+    if not tokens:
+        return {}
+    tf = compute_tf(tokens)
+    return tfidf_vector(tf, idf)
+
+
+def _split_sentences(text: str) -> list[str]:
+    """Split text into sentences on period, newline, or semicolon boundaries.
+
+    Args:
+        text: Raw input text.
+
+    Returns:
+        List of non-empty sentence strings.
+    """
+    import re
+    parts = re.split(r"(?<=[.!?])\s+|\n+|;\s*", text)
+    return [s.strip() for s in parts if s.strip()]
+
+
+def extractive_summarize(
+    snippets: list[dict],
+    max_points: int = 5,
+    mmr_lambda: float = 0.6,
+) -> list[dict]:
+    """Summarize a cluster of snippets using TF-IDF scoring + MMR diversity.
+
+    For each snippet, picks the highest-scoring sentence that is least similar
+    to already-selected sentences (Maximal Marginal Relevance).
+
+    Args:
+        snippets: List of snippet dicts, each with at least a 'description'
+            or 'notes' key and an 'id' key. Example:
+            [{"id": "abc", "description": "...", "notes": "..."}]
+        max_points: Maximum number of thread points to return.
+        mmr_lambda: Trade-off between relevance (1.0) and diversity (0.0).
+            Higher → more relevant, lower → more diverse.
+
+    Returns:
+        Ordered list of dicts: [{"point": str, "source_snippet_id": str}]
+        Returns empty list if snippets is empty.
+    """
+    if not snippets:
+        return []
+
+    # Build corpus: one document per snippet (description + notes combined)
+    def _snippet_text(s: dict) -> str:
+        parts = [s.get("description", ""), s.get("notes", "")]
+        return " ".join(p for p in parts if p)
+
+    corpus_texts = [_snippet_text(s) for s in snippets]
+
+    # Fit IDF on the full cluster corpus
+    engine = TFIDFEngine()
+    engine.fit(corpus_texts)
+    idf = engine._idf
+
+    selected: list[dict] = []        # final thread points
+    selected_vecs: list[dict[str, float]] = []
+
+    for snippet in snippets:
+        if len(selected) >= max_points:
+            break
+
+        text = _snippet_text(snippet)
+        sentences = _split_sentences(text)
+
+        if not sentences:
+            continue
+
+        # Score each sentence in this snippet using MMR
+        best_sentence: str | None = None
+        best_mmr: float = -1.0
+
+        for sent in sentences:
+            relevance = _sentence_tfidf_score(sent, idf)
+
+            if not selected_vecs:
+                # First selection — pick purely by relevance
+                mmr_score = relevance
+            else:
+                sent_vec = _sentence_vector(sent, idf)
+                max_sim = max(
+                    cosine_similarity(sent_vec, sv) for sv in selected_vecs
+                )
+                mmr_score = mmr_lambda * relevance - (1 - mmr_lambda) * max_sim
+
+            if mmr_score > best_mmr:
+                best_mmr = mmr_score
+                best_sentence = sent
+
+        if best_sentence:
+            selected.append({
+                "point": best_sentence,
+                "source_snippet_id": snippet["id"],
+            })
+            selected_vecs.append(_sentence_vector(best_sentence, idf))
+
+    return selected

apps/api/tests/test_summarizer.py

@@ -0,0 +1,207 @@
+"""Tests for extractive text summarization with MMR."""
+
+import pytest
+
+from app.algorithms.summarizer import extractive_summarize, _split_sentences
+
+
+# ---------------------------------------------------------------------------
+# _split_sentences
+# ---------------------------------------------------------------------------
+
+def test_split_sentences_basic():
+    text = "First sentence. Second sentence. Third sentence."
+    parts = _split_sentences(text)
+    assert len(parts) == 3
+    assert parts[0] == "First sentence."
+
+
+def test_split_sentences_newlines():
+    text = "Line one\nLine two\nLine three"
+    parts = _split_sentences(text)
+    assert len(parts) == 3
+
+
+def test_split_sentences_empty_string():
+    assert _split_sentences("") == []
+
+
+def test_split_sentences_ignores_blank_parts():
+    text = "Hello.\n\n\nWorld."
+    parts = _split_sentences(text)
+    assert "" not in parts
+
+
+# ---------------------------------------------------------------------------
+# extractive_summarize — edge cases
+# ---------------------------------------------------------------------------
+
+def test_empty_snippets_returns_empty():
+    assert extractive_summarize([]) == []
+
+
+def test_single_snippet_returns_one_point():
+    snippets = [{"id": "a1", "description": "React hooks simplify state. Use useState for local state management."}]
+    result = extractive_summarize(snippets, max_points=5)
+    assert len(result) == 1
+    assert result[0]["source_snippet_id"] == "a1"
+    assert isinstance(result[0]["point"], str)
+    assert len(result[0]["point"]) > 0
+
+
+def test_snippet_with_no_description_is_skipped():
+    snippets = [
+        {"id": "x1", "description": ""},
+        {"id": "x2", "description": "Python list comprehensions are concise and fast."},
+    ]
+    result = extractive_summarize(snippets, max_points=5)
+    # Only the non-empty snippet should produce a point
+    ids = [r["source_snippet_id"] for r in result]
+    assert "x2" in ids
+
+
+def test_max_points_is_respected():
+    snippets = [
+        {"id": str(i), "description": f"Snippet {i} talks about topic {i} in detail."}
+        for i in range(10)
+    ]
+    result = extractive_summarize(snippets, max_points=3)
+    assert len(result) <= 3
+
+
+def test_result_structure_has_required_keys():
+    snippets = [{"id": "s1", "description": "TypeScript generics enable reusable typed functions."}]
+    result = extractive_summarize(snippets)
+    assert "point" in result[0]
+    assert "source_snippet_id" in result[0]
+
+
+def test_notes_field_used_when_description_missing():
+    snippets = [{"id": "n1", "notes": "Tailwind utility classes reduce custom CSS overhead."}]
+    result = extractive_summarize(snippets, max_points=5)
+    assert len(result) == 1
+    assert result[0]["source_snippet_id"] == "n1"
+
+
+def test_both_description_and_notes_combined():
+    snippets = [{
+        "id": "b1",
+        "description": "useEffect runs after render.",
+        "notes": "Cleanup function prevents memory leaks.",
+    }]
+    result = extractive_summarize(snippets, max_points=5)
+    assert len(result) == 1
+
+
+# ---------------------------------------------------------------------------
+# MMR diversity — selected points should not be identical
+# ---------------------------------------------------------------------------
+
+def test_selected_points_are_diverse():
+    """Three snippets with different topics should yield 3 different sentences."""
+    snippets = [
+        {"id": "r1", "description": "React useState hook manages component state. Call it at top level only."},
+        {"id": "p1", "description": "Python generators use yield keyword. They are memory efficient for large data."},
+        {"id": "s1", "description": "SQL JOIN merges rows from two tables. Use ON clause to specify condition."},
+    ]
+    result = extractive_summarize(snippets, max_points=5)
+    points = [r["point"] for r in result]
+    # All three selected sentences should be distinct
+    assert len(set(points)) == len(points)
+
+
+def test_source_ids_map_correctly():
+    """Each result point must trace back to the snippet it came from."""
+    snippets = [
+        {"id": "alpha", "description": "Git rebase rewrites commit history cleanly."},
+        {"id": "beta", "description": "Docker containers isolate application environments."},
+    ]
+    result = extractive_summarize(snippets, max_points=5)
+    ids = {r["source_snippet_id"] for r in result}
+    assert ids == {"alpha", "beta"}
+
+
+# ---------------------------------------------------------------------------
+# MMR lambda boundary values
+# ---------------------------------------------------------------------------
+
+def test_mmr_lambda_zero_still_runs():
+    """lambda=0.0 is purely diversity-based — should not crash."""
+    snippets = [
+        {"id": "1", "description": "TypeScript interfaces define object shapes."},
+        {"id": "2", "description": "TypeScript generics allow reusable typed code."},
+    ]
+    result = extractive_summarize(snippets, max_points=5, mmr_lambda=0.0)
+    assert len(result) >= 1
+
+
+def test_mmr_lambda_one_still_runs():
+    """lambda=1.0 is purely relevance-based — should not crash."""
+    snippets = [
+        {"id": "1", "description": "Next.js app router uses file-based routing."},
+        {"id": "2", "description": "Next.js server components reduce client bundle size."},
+    ]
+    result = extractive_summarize(snippets, max_points=5, mmr_lambda=1.0)
+    assert len(result) >= 1
+
+
+# ---------------------------------------------------------------------------
+# Real-world-like cluster: 5 related React snippets
+# ---------------------------------------------------------------------------
+
+REACT_SNIPPETS = [
+    {
+        "id": "rc1",
+        "description": (
+            "useState returns a state value and a setter. "
+            "Calling the setter triggers a re-render with the new value."
+        ),
+    },
+    {
+        "id": "rc2",
+        "description": (
+            "useEffect runs after every render by default. "
+            "Pass a dependency array to control when it fires."
+        ),
+    },
+    {
+        "id": "rc3",
+        "description": (
+            "useContext provides global state without prop drilling. "
+            "Wrap your tree in a Provider to supply the value."
+        ),
+    },
+    {
+        "id": "rc4",
+        "description": (
+            "useReducer is better than useState for complex state logic. "
+            "Dispatch actions to a reducer function instead of calling setters."
+        ),
+    },
+    {
+        "id": "rc5",
+        "description": (
+            "useMemo caches expensive computed values between renders. "
+            "Only recomputes when its dependency array changes."
+        ),
+    },
+]
+
+
+def test_react_cluster_returns_one_point_per_snippet():
+    result = extractive_summarize(REACT_SNIPPETS, max_points=5)
+    assert len(result) == 5
+
+
+def test_react_cluster_all_source_ids_present():
+    result = extractive_summarize(REACT_SNIPPETS, max_points=5)
+    ids = {r["source_snippet_id"] for r in result}
+    expected = {"rc1", "rc2", "rc3", "rc4", "rc5"}
+    assert ids == expected
+
+
+def test_react_cluster_points_are_non_empty_strings():
+    result = extractive_summarize(REACT_SNIPPETS, max_points=5)
+    for item in result:
+        assert isinstance(item["point"], str)
+        assert len(item["point"]) > 0