feat: MBW Monitoring Dashboard v3 — GPS, streaming, performance & approval filters

commcare_connect/labs/analysis/backends/sql/backend.py

@@ -387,6 +387,13 @@ def get_cached_visit_result(
                 if key == "entity_id":
                     computed_qs = computed_qs.filter(entity_id=value)
                     logger.info(f"[SQL] Applying entity_id filter: {value}")
+                # status is a column on ComputedVisitCache, not in computed_fields JSONB
+                elif key == "status":
+                    if isinstance(value, list):
+                        computed_qs = computed_qs.filter(status__in=value)
+                    else:
+                        computed_qs = computed_qs.filter(status=value)
+                    logger.info(f"[SQL] Applying status filter: {value}")
                 # All other filters are treated as computed field filters
                 # This enables linking by fields like beneficiary_case_id, rutf_case_id, etc.
                 else:

commcare_connect/labs/analysis/pipeline.py

@@ -375,8 +375,10 @@ def stream_analysis(
             tolerance = self.cache_tolerance_pct
             # For CCHQ form sources, don't validate against opportunity visit count
             # (CCHQ forms have far fewer rows than Connect visits)
+            # For filtered configs, skip strict tolerance — filter changes should reuse
+            # existing cache, not re-download. The expires_at TTL handles staleness.
             is_cchq = config.data_source.type == "cchq_forms"
-            expected_count = 0 if is_cchq else self.visit_count
+            expected_count = 0 if (is_cchq or has_filters) else self.visit_count
             if not force_refresh:
                 cached_result = None
                 if terminal_stage == CacheStage.AGGREGATED:
@@ -462,13 +464,14 @@ def stream_analysis(
                         logger.info(f"[Pipeline/{self.backend_name}] Reading cached data with filters applied")
                         yield (EVENT_STATUS, {"message": "Applying filters..."})
 
+                        # expected_count=0: we just wrote this cache, skip count validation
                         if terminal_stage == CacheStage.AGGREGATED:
                             filtered_result = self.backend.get_cached_flw_result(
-                                opp_id, config, expected_count, tolerance_pct=tolerance,
+                                opp_id, config, 0, tolerance_pct=tolerance,
                             )
                         else:
                             filtered_result = self.backend.get_cached_visit_result(
-                                opp_id, config, expected_count, tolerance_pct=tolerance,
+                                opp_id, config, 0, tolerance_pct=tolerance,
                             )
 
                         if filtered_result:
@@ -543,13 +546,14 @@ def stream_analysis(
                     logger.info(f"[Pipeline/{self.backend_name}] Reading cached data with filters applied")
                     yield (EVENT_STATUS, {"message": "Applying filters..."})
 
+                    # expected_count=0: we just wrote this cache, skip count validation
                     if terminal_stage == CacheStage.AGGREGATED:
                         filtered_result = self.backend.get_cached_flw_result(
-                            opp_id, config, expected_count, tolerance_pct=tolerance,
+                            opp_id, config, 0, tolerance_pct=tolerance,
                         )
                     else:
                         filtered_result = self.backend.get_cached_visit_result(
-                            opp_id, config, expected_count, tolerance_pct=tolerance,
+                            opp_id, config, 0, tolerance_pct=tolerance,
                         )
 
                     if filtered_result:

commcare_connect/workflow/templates/mbw_monitoring/DASHBOARD_GUIDE.md

@@ -30,6 +30,22 @@ The MBW Monitoring Dashboard has **4 tabs**, each providing a different lens on
 
 All data is loaded via a single streaming connection when you open the dashboard. After the first load, a snapshot is saved so subsequent visits load instantly (use "Refresh Data" to fetch fresh data).
 
+### Filter Bar
+
+The filter bar sits above all tabs and provides controls that affect the data shown across the dashboard:
+
+| Filter | Scope | Default | Description |
+|--------|-------|---------|-------------|
+| **Visit Status** | All tabs | Approved only | Filters by Connect visit approval status. Options: Approved, Pending, Rejected, Over Limit. Select one or more statuses to include. |
+| **App Version** (GPS only) | GPS tab | > 14 | Filters GPS visits by app build version. Configurable operator (>, >=, =, <=, <) and version number. |
+| **FLW filter** | All tabs | All | Multi-select list to filter by specific FLWs. |
+| **Mother filter** | Follow-Up tab | All | Multi-select list to filter by specific mothers. |
+
+- **Apply**: Click to apply any changes to Visit Status or App Version. These filters require a server reload.
+- **Reset**: Restores all filters to defaults (Approved only, App Version > 14, no FLW/mother selection).
+
+> **Note**: Changing Visit Status typically does not re-download data from Connect. The dashboard reuses its cached pipeline data and applies the filter server-side, so switching statuses is usually fast (seconds, not minutes). Exceptions include a cold or expired cache, or when a forced refresh (`?bust_cache=1`) triggers a full re-download.
+
 ---
 
 ## Tab 1: Overview
@@ -138,8 +154,8 @@ The column shows: *(still on track) / (total eligible) = percentage*
 - Missed visits: count of visits with status "Missed" (visit not completed and past `form.var_visit_N.visit_expiry_date`)
 
 **Color coding:**
-- Green: 70% or above
-- Yellow: 50-69%
+- Green: 85% or above
+- Yellow: 50-84%
 - Red: below 50%
 
 ---
@@ -606,42 +622,42 @@ Mothers marked as eligible for the full intervention bonus at the time of regist
 **(Still Eligible / Eligible at Reg) × 100**
 
 **Color coding:**
-- Green: 70% or above
-- Yellow: 50-69%
+- Green: 85% or above
+- Yellow: 50-84%
 - Red: below 50%
 
 #### % ≤1 Missed
-**What it shows:** Percentage of **all** mothers (not just eligible) with 0 or 1 missed visits.
+**What it shows:** Percentage of **eligible** mothers (`eligible_full_intervention_bonus = "1"`) with 0 or 1 missed visits.
 
-**How it's calculated:** Count mothers where total missed visits ≤ 1, divide by total mothers in this status group.
+**How it's calculated:** Count eligible mothers where total missed visits ≤ 1, divide by total eligible mothers in this status group.
 
 #### % 4 Visits On Track
-**What it shows:** Among mothers whose Month 1 visit is due (5-day grace), what percentage have 3 or more completed visits?
+**What it shows:** Among **eligible** mothers whose Month 1 visit is due (5-day grace), what percentage have 3 or more completed visits?
 
 **How it's calculated:**
-1. Filter to mothers whose Month 1 visit scheduled date is 5+ days ago (i.e., they should have completed their first 4 visits by now)
-2. Count those with 3+ total completed visits
-3. **(with 3+ completed / total with Month 1 due) × 100**
+1. Filter to eligible mothers whose Month 1 visit scheduled date is 5+ days ago (i.e., all 4 visits up to Month 1 should be completable by now)
+2. Count those with 3+ total completed visits (the "3-of-4" threshold — on track if at most 1 visit is incomplete)
+3. **(eligible with ≥3 completed / total eligible with Month 1 due) × 100**
 
-**Data path (denominator):** `form.var_visit_N.visit_date_scheduled` where `visit_type` = "1 Month Visit", filtered to dates ≤ (today - 5 days)
+**Data path (denominator):** `form.var_visit_N.visit_date_scheduled` where `visit_type` = "1 Month Visit", filtered to eligible mothers with dates ≤ (today - 5 days)
 
 #### % 5 Visits Complete
-**What it shows:** Among mothers whose Month 3 visit is due (5-day grace), what percentage have 4 or more completed visits?
+**What it shows:** Among **eligible** mothers whose Month 3 visit is due (5-day grace), what percentage have 4 or more completed visits?
 
 **How it's calculated:** Same logic as above, but:
-- Denominator: mothers whose Month 3 scheduled date is 5+ days ago
+- Denominator: eligible mothers whose Month 3 scheduled date is 5+ days ago
 - Numerator: those with 4+ completed visits
 
-**Data path (denominator):** `form.var_visit_N.visit_date_scheduled` where `visit_type` = "3 Month Visit", filtered to dates ≤ (today - 5 days)
+**Data path (denominator):** `form.var_visit_N.visit_date_scheduled` where `visit_type` = "3 Month Visit", filtered to eligible mothers with dates ≤ (today - 5 days)
 
 #### % 6 Visits Complete
-**What it shows:** Among mothers whose Month 6 visit is due (5-day grace), what percentage have 5 or more completed visits?
+**What it shows:** Among **eligible** mothers whose Month 6 visit is due (5-day grace), what percentage have 5 or more completed visits?
 
 **How it's calculated:** Same logic:
-- Denominator: mothers whose Month 6 scheduled date is 5+ days ago
+- Denominator: eligible mothers whose Month 6 scheduled date is 5+ days ago
 - Numerator: those with 5+ completed visits
 
-**Data path (denominator):** `form.var_visit_N.visit_date_scheduled` where `visit_type` = "6 Month Visit", filtered to dates ≤ (today - 5 days)
+**Data path (denominator):** `form.var_visit_N.visit_date_scheduled` where `visit_type` = "6 Month Visit", filtered to eligible mothers with dates ≤ (today - 5 days)
 
 ### Totals Row
 The bottom row aggregates all status groups together — total FLWs, total cases, and weighted percentages across all categories.
@@ -709,8 +725,8 @@ When creating a task for an FLW (via the OCS AI integration), the system automat
 
 | Color | Range |
 |-------|-------|
-| Green | ≥ 70% |
-| Yellow | 50-69% |
+| Green | ≥ 85% |
+| Yellow | 50-84% |
 | Red | < 50% |
 
 ### Last Active Colors

commcare_connect/workflow/templates/mbw_monitoring/DOCUMENTATION.md

@@ -40,7 +40,7 @@ The dashboard runs within the **Workflow module** of Connect Labs. The UI is a R
 
 - **Render code pattern**: The entire dashboard UI (~1,860 lines of JSX) lives in a Python string (`RENDER_CODE`), transpiled by Babel in the browser. Only `React` is available as a global - no imports. All declarations use `var` (not `const`/`let`) for Babel compatibility.
 - **Single SSE connection**: All three tabs load data from one streaming endpoint, avoiding redundant API calls
-- **Client-side filtering**: Raw data is sent once; FLW and mother filtering happens entirely in the browser via React state
+- **Hybrid filtering**: Some filters (Visit Status, App Version) are server-side and trigger an SSE reload with updated pipeline data; others (FLW, Mother, Date) are client-side and operate on already-fetched data via React state. Server-side filters require clicking "Apply" and state is persisted in `sessionStorage` to survive OAuth redirects.
 - **Two-layer caching**: Pipeline-level cache (Redis) for visit form data + Django cache for CCHQ form/case data
 - **Tolerance-based cache validation**: Caches are accepted if they meet count, percentage, or time-based tolerance thresholds
 - **No database writes**: All data is fetched from external APIs (Connect Production + CommCare HQ) and cached transiently. Workflow state is persisted via the LabsRecord API.
@@ -281,7 +281,7 @@ The stream view executes 7 steps, yielding progress messages at each stage:
 
 | Step | Data Source | What It Fetches | Cache |
 |------|-----------|-----------------|-------|
-| 1 | Connect API via AnalysisPipeline | Visit form data (13 FieldComputations: GPS, case IDs, form names, dates, parity, etc.) | Pipeline cache (Redis, config-hash based) |
+| 1 | Connect API via AnalysisPipeline | Visit form data (13 FieldComputations: GPS, case IDs, form names, dates, parity, etc.). Filtered by `status_filter` param (default: approved only) via SQL `WHERE status IN (...)` | Pipeline cache (Redis, config-hash based; filter changes reuse cache) |
 | 2 | Connect API | Active FLW usernames + display names | In-memory |
 | 3 | In-memory | GPS metrics (Haversine distances, daily travel) | None (computed) |
 | 4a | CCHQ Form API v1 | Registration forms -> mother metadata (name, age, phone, eligibility, EDD, etc.) | Django cache (1hr) |
@@ -538,11 +538,11 @@ Aggregated case metrics grouped by each FLW's latest known assessment status. Co
 | Total Cases | Total registered mothers across all FLWs in the group |
 | Eligible at Reg | Mothers marked eligible for full intervention bonus at registration |
 | Still Eligible | Mothers with 5+ completed visits OR <=1 missed visits |
-| % Still Eligible | Still Eligible / Eligible at Reg (color: green >=70%, yellow 50-69%, red <50%) |
-| % <=1 Missed | Cases with 0 or 1 missed visits / all cases |
-| % 4 Visits On Track | Cases with 3+ completed visits among those whose Month 1 visit is due (5-day buffer) |
-| % 5 Visits Complete | Cases with 4+ completed visits among those whose Month 3 visit is due (5-day buffer) |
-| % 6 Visits Complete | Cases with 5+ completed visits among those whose Month 6 visit is due (5-day buffer) |
+| % Still Eligible | Still Eligible / Eligible at Reg (color: green >=85%, yellow 50-84%, red <50%) |
+| % <=1 Missed | Eligible cases with 0 or 1 missed visits / eligible cases |
+| % 4 Visits On Track | Eligible cases with 3+ completed visits among those whose Month 1 visit is due (5-day buffer) |
+| % 5 Visits Complete | Eligible cases with 4+ completed visits among those whose Month 3 visit is due (5-day buffer) |
+| % 6 Visits Complete | Eligible cases with 5+ completed visits among those whose Month 6 visit is due (5-day buffer) |
 
 **Totals Row**: Aggregated totals across all status groups.
 
@@ -899,6 +899,10 @@ Note: "Suspended" is a **label only** - it does NOT trigger any action on Connec
 | Computed Visit Cache | Visit-level computed fields (GPS, case IDs, dates, etc.) | `opportunity_id` + `config_hash` | Configurable TTL | Keyed by opportunity_id + config_hash; username as secondary index for filtered queries |
 | Dashboard Snapshot | Computed dashboard metrics | `run.data["snapshot"]` | Permanent (updated on refresh) | Per run |
 
+### Filter-Aware Caching
+
+The pipeline config hash **excludes filters** (via `get_config_hash()` in `utils.py`), so one cached dataset serves multiple filtered queries. When a config has filters (e.g., `status_filter`), the cache tolerance check uses `expected_count=0`, accepting any non-expired cache. This means filter changes reuse existing cached data instantly — only the SQL `WHERE` clause changes. The `expires_at` TTL still guards against stale data. Force refresh (`?refresh=1`) bypasses the cache entirely regardless of filters.
+
 ### Tolerance-Based Cache Validation
 
 HQ case caches use a 3-tier validation system (implemented in `_validate_hq_cache()`):
@@ -968,9 +972,22 @@ Since the render code is a string transpiled by Babel in the browser:
 6. **Inline styles for dynamic values** - Tailwind classes work but dynamic CSS needs `style={{}}`
 7. **No TypeScript** - plain JavaScript only
 
-### Client-Side Filtering
+### Filtering
+
+The dashboard uses two types of filters:
+
+#### Server-Side Filters (trigger SSE reload)
+
+These filters modify the SSE stream URL and cause a new server-side query:
+
+- **Visit Status filter**: Filters by Connect visit approval status (Approved, Pending, Rejected, Over Limit). Default: Approved only. Applied server-side via pipeline SQL `WHERE status IN (...)`. Changing this filter triggers a new SSE stream but reuses the cached pipeline data (no re-download from Connect). State persisted in `sessionStorage` to survive OAuth redirects.
+- **App Version filter** (GPS only): Operator (>, >=, =, <=, <) + version number. Default: > 14. Applied server-side to GPS visit data.
+
+Both require clicking "Apply" to take effect. "Reset" restores defaults (Approved only, > 14).
+
+#### Client-Side Filters (no API calls)
 
-All filtering happens in the browser without additional API calls:
+These filters happen entirely in the browser:
 
 - **FLW filter**: Multi-select listbox. Filters all three tabs by `username` set membership
 - **Mother filter**: Multi-select listbox (populated from drilldown data). Filters follow-up tab
@@ -1072,6 +1089,7 @@ Django uses `CompressedManifestStaticFilesStorage` (whitenoise). The `{% static
 | 3-option FLW assessment | Overview | Eligible for Renewal / Probation / Suspended with progress |
 | Task creation | Overview | Create task for FLW with OCS integration |
 | Inline task management | Overview | Expand FLW row to view AI conversation, update status, close task with outcome |
+| Visit Status filter | Filter Bar | Server-side filter by Connect approval status (Approved, Pending, Rejected, Over Limit); default Approved only; reuses cached pipeline data on filter change; state persisted in `sessionStorage` to survive OAuth redirects |
 | App version filter (GPS) | Filter Bar | User-configurable operator (>, >=, =, <=, <) + version number for GPS data filtering; default > 14; persisted in run state |
 | Template sync | - | Sync render code from template.py to DB via `?sync=true` |
 | Template registry | - | Auto-discovery of workflow templates |

commcare_connect/workflow/templates/mbw_monitoring/followup_analysis.py

@@ -1120,9 +1120,9 @@ def compute_flw_performance_by_status(
             if completed >= 5 or missed <= 1:
                 still_eligible += 1
 
-        # --- pct missed ≤1 (all mothers, not just eligible) ---
+        # --- pct missed ≤1 (eligible mothers only) ---
         missed_1_or_less = 0
-        for m in all_mothers:
+        for m in eligible_mothers:
             missed = sum(1 for v in m["visits"] if v["status"] == "Missed")
             if missed <= 1:
                 missed_1_or_less += 1
@@ -1132,7 +1132,7 @@ def compute_flw_performance_by_status(
         for visit_display_type, min_completed, metric_key in _VISIT_MILESTONES:
             denominator = 0
             numerator = 0
-            for m in all_mothers:
+            for m in eligible_mothers:
                 # Find the specific visit type for this mother
                 milestone_visit = None
                 for v in m["visits"]:
@@ -1170,7 +1170,7 @@ def compute_flw_performance_by_status(
             "total_cases_eligible_at_registration": total_eligible,
             "total_cases_still_eligible": still_eligible,
             "pct_still_eligible": round(still_eligible / total_eligible * 100) if total_eligible > 0 else 0,
-            "pct_missed_1_or_less_visits": round(missed_1_or_less / total_cases * 100) if total_cases > 0 else 0,
+            "pct_missed_1_or_less_visits": round(missed_1_or_less / total_eligible * 100) if total_eligible > 0 else 0,
             **milestone_results,
         })