update to use new assisted facilitation approach

Author: forrestmurray-db

doc/DISCOVERY.md

@@ -18,3 +18,20 @@ The process can be more art than science. It's often messy, and can suffer from
     - The application generates follow up questions specific to participants / examples. Observations from other participants may be included to specifically probe for interesting disagreements.
     - The findings aggregated by participant/example are presented to the facilitator to _inform_ discussion (not to replace it). Since facilitators often don't know about the domain, this helps reduce cognitive load.
     - Participants autonomously own the process with the faciliator and workshop just providing the framework. They will organically identify common clusters of findings and themes. 
+
+## How assisted facilitation works (high level)
+
+Assisted facilitation helps participants go deeper on each example and helps facilitators guide discussion without needing to be a domain expert.
+
+### During participant review (per example)
+
+- **Start simple, then go deeper**: each example begins with a baseline prompt (“what makes this effective or ineffective?”). As a participant responds, the application can propose a small number of follow-up questions that encourage deeper thinking (edge cases, missing info, boundary conditions, failure modes).
+- **Probe disagreements intentionally**: when different participants notice different things about the same example, follow-up questions can be tailored to surface the disagreement and clarify the underlying definition of “good” vs “bad”.
+- **Stop when coverage is good**: follow-up questions aren’t infinite; once the key angles have been explored (or a sensible limit is reached), the application stops proposing more so the group can move on.
+
+### For the facilitator (across participants and examples)
+
+- **Theme extraction and synthesis**: participant observations are summarized into a small set of themes and recurring patterns, both overall and broken down by participant and by example.
+- **Discussion-ready outputs**: the app surfaces key disagreements and provides short discussion prompts that help the facilitator run a productive conversation.
+- **Bridge to rubric creation**: the system can suggest candidate rubric questions (concrete “quality dimensions”) derived from the themes so the group can turn discovery insights into a rubric more quickly.
+- **Progress signals**: simple convergence indicators (how consistently themes appear across participants) help the facilitator judge when the group has enough shared understanding to move into rubric definition and annotation.

migrations/versions/0007_discovery_question_category.py

@@ -0,0 +1,44 @@
+"""Add category column to discovery_questions table.
+
+Adds:
+- category column to discovery_questions for coverage tracking
+"""
+
+from __future__ import annotations
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "0007_discovery_question_category"
+down_revision = "0006_discovery_summaries_table"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+
+    if not inspector.has_table("discovery_questions"):
+        return
+
+    columns = [c["name"] for c in inspector.get_columns("discovery_questions")]
+    if "category" not in columns:
+        op.add_column(
+            "discovery_questions",
+            sa.Column("category", sa.String(), nullable=True),
+        )
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+
+    if not inspector.has_table("discovery_questions"):
+        return
+
+    columns = [c["name"] for c in inspector.get_columns("discovery_questions")]
+    if "category" in columns:
+        op.drop_column("discovery_questions", "category")
+

server/database.py

@@ -214,6 +214,7 @@ class DiscoveryQuestionDB(Base):
     question_id = Column(String, nullable=False)  # Stable ID per (user, trace), e.g. "q_1"
     prompt = Column(Text, nullable=False)
     placeholder = Column(Text, nullable=True)
+    category = Column(String, nullable=True)  # Coverage category: themes, edge_cases, boundary_conditions, failure_modes, missing_info, disagreements
     created_at = Column(DateTime, default=func.now())
 
 

server/routers/discovery.py

@@ -25,12 +25,23 @@ class DiscoveryQuestion(BaseModel):
     id: str
     prompt: str
     placeholder: Optional[str] = None
+    category: Optional[str] = None
+
+
+class DiscoveryCoverage(BaseModel):
+    """Coverage state for discovery questions."""
+
+    covered: List[str]
+    missing: List[str]
 
 
 class DiscoveryQuestionsResponse(BaseModel):
-    """Response model for discovery questions."""
+    """Response model for discovery questions with coverage metadata."""
 
     questions: List[DiscoveryQuestion]
+    can_generate_more: bool = True
+    stop_reason: Optional[str] = None
+    coverage: DiscoveryCoverage
 
 
 class DiscoveryQuestionsModelConfig(BaseModel):
@@ -39,12 +50,39 @@ class DiscoveryQuestionsModelConfig(BaseModel):
     model_name: str
 
 
+class KeyDisagreementResponse(BaseModel):
+    """A disagreement between participants."""
+
+    theme: str
+    trace_ids: List[str] = []
+    viewpoints: List[str] = []
+
+
+class DiscussionPromptResponse(BaseModel):
+    """A facilitator discussion prompt."""
+
+    theme: str
+    prompt: str
+
+
+class ConvergenceMetricsResponse(BaseModel):
+    """Cross-participant agreement metrics."""
+
+    theme_agreement: Dict[str, float] = {}
+    overall_alignment_score: float = 0.0
+
+
 class DiscoverySummariesResponse(BaseModel):
     """LLM-generated summaries of discovery findings for facilitators."""
 
     overall: Dict[str, Any]
     by_user: List[Dict[str, Any]]
     by_trace: List[Dict[str, Any]]
+    candidate_rubric_questions: List[str] = []
+    key_disagreements: List[KeyDisagreementResponse] = []
+    discussion_prompts: List[DiscussionPromptResponse] = []
+    convergence: ConvergenceMetricsResponse = ConvergenceMetricsResponse()
+    ready_for_rubric: bool = False
 
 
 @router.get(
@@ -59,8 +97,13 @@ async def get_discovery_questions(
     db: Session = Depends(get_db),
 ) -> DiscoveryQuestionsResponse:
     svc = DiscoveryService(db)
-    questions = svc.get_discovery_questions(workshop_id=workshop_id, trace_id=trace_id, user_id=user_id, append=append)
-    return DiscoveryQuestionsResponse(questions=[DiscoveryQuestion(**q) for q in questions])
+    result = svc.get_discovery_questions(workshop_id=workshop_id, trace_id=trace_id, user_id=user_id, append=append)
+    return DiscoveryQuestionsResponse(
+        questions=[DiscoveryQuestion(**q) for q in result["questions"]],
+        can_generate_more=result.get("can_generate_more", True),
+        stop_reason=result.get("stop_reason"),
+        coverage=DiscoveryCoverage(**result.get("coverage", {"covered": [], "missing": []})),
+    )
 
 
 @router.put("/{workshop_id}/discovery-questions-model")
@@ -74,28 +117,53 @@ async def update_discovery_questions_model(
     return {"message": "Discovery questions model updated", "model_name": model_name}
 
 
+def _build_summaries_response(payload: Dict[str, Any]) -> DiscoverySummariesResponse:
+    """Build a DiscoverySummariesResponse from a payload dict."""
+    # Parse key_disagreements
+    key_disagreements = []
+    for d in payload.get("key_disagreements") or []:
+        if isinstance(d, dict):
+            key_disagreements.append(KeyDisagreementResponse(**d))
+
+    # Parse discussion_prompts
+    discussion_prompts = []
+    for p in payload.get("discussion_prompts") or []:
+        if isinstance(p, dict):
+            discussion_prompts.append(DiscussionPromptResponse(**p))
+
+    # Parse convergence
+    convergence_data = payload.get("convergence") or {}
+    if isinstance(convergence_data, dict):
+        convergence = ConvergenceMetricsResponse(**convergence_data)
+    else:
+        convergence = ConvergenceMetricsResponse()
+
+    return DiscoverySummariesResponse(
+        overall=payload.get("overall") or {},
+        by_user=payload.get("by_user") or [],
+        by_trace=payload.get("by_trace") or [],
+        candidate_rubric_questions=payload.get("candidate_rubric_questions") or [],
+        key_disagreements=key_disagreements,
+        discussion_prompts=discussion_prompts,
+        convergence=convergence,
+        ready_for_rubric=payload.get("ready_for_rubric", False),
+    )
+
+
 @router.post("/{workshop_id}/discovery-summaries", response_model=DiscoverySummariesResponse)
 async def generate_discovery_summaries(
     workshop_id: str, refresh: bool = False, db: Session = Depends(get_db)
 ) -> DiscoverySummariesResponse:
     svc = DiscoveryService(db)
     payload = svc.generate_discovery_summaries(workshop_id=workshop_id, refresh=refresh)
-    return DiscoverySummariesResponse(
-        overall=payload.get("overall") or {},
-        by_user=payload.get("by_user") or [],
-        by_trace=payload.get("by_trace") or [],
-    )
+    return _build_summaries_response(payload)
 
 
 @router.get("/{workshop_id}/discovery-summaries", response_model=DiscoverySummariesResponse)
 async def get_discovery_summaries(workshop_id: str, db: Session = Depends(get_db)) -> DiscoverySummariesResponse:
     svc = DiscoveryService(db)
     payload = svc.get_discovery_summaries(workshop_id=workshop_id)
-    return DiscoverySummariesResponse(
-        overall=payload.get("overall") or {},
-        by_user=payload.get("by_user") or [],
-        by_trace=payload.get("by_trace") or [],
-    )
+    return _build_summaries_response(payload)
 
 
 @router.post("/{workshop_id}/findings", response_model=DiscoveryFinding)

server/services/database_service.py

@@ -621,6 +621,7 @@ def get_discovery_questions(self, workshop_id: str, trace_id: str, user_id: str)
                 "id": r.question_id,
                 "prompt": r.prompt,
                 "placeholder": r.placeholder,
+                "category": getattr(r, "category", None),
                 "created_at": r.created_at,
             }
             for r in rows
@@ -633,6 +634,7 @@ def add_discovery_question(
         user_id: str,
         prompt: str,
         placeholder: Optional[str] = None,
+        category: Optional[str] = None,
     ) -> Dict[str, Any]:
         """Append a new generated discovery question for a specific (workshop, trace, user)."""
         # Compute next stable question_id, reserving q_1 for the fixed baseline question.
@@ -664,6 +666,7 @@ def add_discovery_question(
             question_id=question_id,
             prompt=prompt,
             placeholder=placeholder,
+            category=category,
         )
         self.db.add(row)
         self.db.commit()
@@ -673,6 +676,7 @@ def add_discovery_question(
             "id": row.question_id,
             "prompt": row.prompt,
             "placeholder": row.placeholder,
+            "category": row.category,
             "created_at": row.created_at,
         }