index.qmd

---
pagetitle: "MTTR Alone Misleads | MOVA"
format:
  revealjs:
    include-in-header:
      text: |
        <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.3/font/bootstrap-icons.css">
execute:
  echo: false
  warning: false
  message: false
jupyter: python3
---

```{python}
#| echo: false
import polars as pl
from great_tables import GT, loc, style
from plotnine import (
    aes,
    element_blank,
    element_line,
    element_rect,
    element_text,
    geom_line,
    geom_point,
    geom_text,
    ggplot,
    labs,
    scale_color_manual,
    scale_x_continuous,
    scale_y_continuous,
    theme,
    theme_minimal,
)

STRATEGY_LABELS = {
    "oldest_first": "Oldest-First",
    "newest_first": "Newest-First",
}


def add_strategy_label(dataframe):
    return dataframe.with_columns(
        pl.col("strategy").replace(STRATEGY_LABELS).alias("strategy_label")
    )


metrics = add_strategy_label(pl.read_parquet("data/metrics.parquet"))
summary = add_strategy_label(pl.read_parquet("data/summary.parquet"))

STRATEGY_COLORS = {
    "Oldest-First": "#1F6F78",
    "Newest-First": "#8B1E3F",
}
STRATEGY_ORDER = ["Oldest-First", "Newest-First"]
MONTH_BREAKS = [1, 6, 12, 18, 24]
LAST_MONTH = metrics.get_column("month_index").max()
if LAST_MONTH is None:
    raise ValueError("data/metrics.parquet does not contain any month_index values.")
ROOM_BG = "#f3ebde"
PANEL_BG = "#fffdf8"
GRID_COLOR = "#c8baa5"
LINE_SIZE = 2.8
ENDPOINT_SIZE = 5.0
ENDPOINT_LABEL_SIZE = 8
ENDPOINT_LABEL_X_NUDGE = 0.55
X_AXIS_EXPAND = (0.01, 0.85)
DEFAULT_Y_EXPAND = (0.02, 0)
TAIL_Y_EXPAND = (0.02, 12)
chart_theme = theme_minimal(base_size=22) + theme(
    figure_size=(11.35, 5.35),
    legend_position="top",
    legend_title=element_blank(),
    legend_text=element_text(color="#26333A", size=15),
    legend_background=element_blank(),
    legend_key=element_blank(),
    panel_grid_minor=element_blank(),
    panel_grid_major_x=element_blank(),
    panel_grid_major_y=element_line(color=GRID_COLOR, size=0.7),
    panel_background=element_rect(fill=PANEL_BG, color=None),
    plot_background=element_rect(fill=ROOM_BG, color=None),
    axis_title=element_text(color="#26333A", size=18),
    axis_text=element_text(color="#26333A", size=15),
    axis_text_x=element_text(margin={"t": 8}),
    axis_text_y=element_text(margin={"r": 8}),
    plot_title=element_text(weight="bold", color="#101828", size=22),
    plot_subtitle=element_text(color="#55606B", size=14, margin={"b": 10}),
    axis_title_x=element_text(margin={"t": 16}),
    axis_title_y=element_text(margin={"r": 18}),
)


def strategy_line_plot(
    dataframe,
    y_col,
    title,
    subtitle,
    y_axis_label,
    y_expand=DEFAULT_Y_EXPAND,
    y_limits=None,
):
    is_endpoint = pl.col("month_index") == LAST_MONTH
    endpoints = dataframe.filter(is_endpoint)
    y_min = dataframe.select(pl.col(y_col).min()).item()
    y_max = dataframe.select(pl.col(y_col).max()).item()
    if y_min is None or y_max is None:
        raise ValueError(f"{y_col} does not contain any values to plot.")

    label_floor_offset = max(3.0, (y_max - y_min) * 0.03)
    endpoint_labels = endpoints.with_columns(
        (pl.col("month_index") + ENDPOINT_LABEL_X_NUDGE).alias("label_month"),
        pl.col(y_col).round(0).cast(pl.Int64).cast(pl.String).alias("endpoint_label"),
        (
            pl.when(pl.col(y_col) <= label_floor_offset)
            .then(pl.col(y_col) + label_floor_offset)
            .otherwise(pl.col(y_col))
        ).alias("label_value"),
    )

    return (
        ggplot(mapping=aes(x="month_index", y=y_col, color="strategy_label"))
        + geom_line(data=dataframe, size=LINE_SIZE)
        + geom_point(data=endpoints, size=ENDPOINT_SIZE)
        + geom_text(
            data=endpoint_labels,
            mapping=aes(
                x="label_month",
                y="label_value",
                label="endpoint_label",
            ),
            size=ENDPOINT_LABEL_SIZE,
            ha="left",
            va="center",
            show_legend=False,
        )
        + scale_color_manual(values=STRATEGY_COLORS, breaks=STRATEGY_ORDER)
        + scale_x_continuous(breaks=MONTH_BREAKS, expand=X_AXIS_EXPAND)
        + scale_y_continuous(expand=y_expand, limits=y_limits)
        + labs(
            title=title,
            subtitle=subtitle,
            x="Month",
            y=y_axis_label,
        )
        + chart_theme
    )


card_lookup = {
    row["strategy_label"]: row
    for row in summary.select(
        "strategy_label",
        pl.col("mttr_days").round(1).alias("mttr_days"),
        pl.col("mova_days").round(1).alias("mova_days"),
    ).to_dicts()
}


def get_strategy_card(strategy_label):
    card = card_lookup.get(strategy_label)
    if card is None:
        available_labels = ", ".join(sorted(card_lookup)) or "none"
        raise ValueError(
            "data/summary.parquet is missing required strategy_label "
            f"{strategy_label!r}. Available labels: {available_labels}."
        )
    return card


oldest_card = get_strategy_card("Oldest-First")
newest_card = get_strategy_card("Newest-First")

oldest_mttr = f"{oldest_card['mttr_days']:.1f}"
oldest_mova = f"{oldest_card['mova_days']:.1f}"

newest_mttr = f"{newest_card['mttr_days']:.1f}"
newest_mova = f"{newest_card['mova_days']:.1f}"
```

## {#opening .hero-slide}

::: {.hero-block}
::: {.hero-eyebrow}
BSides Charm · 2026
:::

<div class="hero-title">Why Vulnerability<br>MTTR Alone<br>Misleads</div>
<div class="hero-subtitle">Use MTTR and MOVA to Measure Remediation Progress</div>
<div class="hero-divider" aria-hidden="true"></div>
<div class="hero-author">Caleb Kinney</div>
:::

---

## MTTR just doubled {.center .hook-slide}

:::: {.shock-card}
::: {.shock-label}
Dashboard metric
:::

:::: {.shock-row}
::: {.shock-panel}
::: {.shock-caption}
Last month
:::
::: {.shock-value}
15d
:::
:::

::: {.shock-arrow}
→
:::

::: {.shock-panel .danger}
::: {.shock-caption}
This month
:::
::: {.shock-value}
31d
:::
:::
::::

::: {.shock-subtitle}
Dashboards call this failure.

Better question: what actually changed?

What happened to backlog age?
What happened to open risk?
:::
::::

---

## This actually happened {.top-loaded .story-slide}

::: {.line-anchor .story-lead}
MTTR went up after we made it a KPI.
:::

::: {.story-grid}

::: {.story-steps}
- We made MTTR visible
- Teams started closing older backlog, not just recent work
- The metric got worse
:::

::: {.story-question}
The review question was:

**“Why are we getting worse?”**
:::

:::

::: {.kicker .story-joke}
That was a fun meeting.
:::

::: {.line-anchor .line-anchor-subtle .centered}
MTTR rose because we finally closed older backlog, not just recent work.
:::

---

## Two Metrics, Two Questions

::: {.definition-grid}
::: {.definition-card}
::: {.eyebrow}
Flow
:::
### MTTR — Mean Time to Remediate
How fast are we closing work?
:::

::: {.definition-card}
::: {.eyebrow}
Backlog Age
:::
### MOVA — Mean Open Vulnerability Age
How old is the open backlog today?
:::
:::

::: {.centered .muted}
Flow vs Backlog Age
:::

---

## Same records, different slices

::: {.definition-grid}
::: {.definition-card}
::: {.eyebrow}
Resolved records
:::
### MTTR
Closed findings only.
:::

::: {.definition-card}
::: {.eyebrow}
Open records
:::
### MOVA
Open findings only.
:::
:::

::: {.line-anchor .line-anchor-subtle .centered}
Same export. Different slice. Different signal.
:::

---

## MTTR measures closed work {auto-animate="true"}

```{python}
#| echo: true
#| eval: false
#| code-line-numbers: "2|3-5|6"
mttr = (
    vulns.filter(pl.col("resolved_at").is_not_null())
    .with_columns(
        (pl.col("resolved_at") - pl.col("created_at")).dt.total_days().alias("age_days")
    )
    .select(pl.col("age_days").mean().alias("mttr_days"))
)
```

::: {.kicker}
The flow signal comes from resolved records.
:::

---

## MOVA measures backlog age {auto-animate="true"}

```{python}
#| echo: true
#| eval: false
#| code-line-numbers: "2|3-5|6"
mova = (
    vulns.filter(pl.col("resolved_at").is_null())
    .with_columns(
        (pl.datetime_now() - pl.col("created_at")).dt.total_days().alias("age_days")
    )
    .select(pl.col("age_days").mean().alias("mova_days"))
)
```

::: {.kicker}
The backlog-age signal comes from open records.
:::

---

## You already have the data {.center .statement-slide}

::: {.metric-big .centered}
You do not need a new platform.
:::

::: {.compact-list}
- MTTR and MOVA use the same records
- A CSV export is often enough
- MOVA comes from open findings
- Older backlog often shows risk that averages hide
:::

::: {.line-anchor .centered .blood-line}
SLA views tell you if you hit a deadline, not the backlog age you still carry.
:::

---

## A simple simulation

::: {.definition-grid}
::: {.definition-card}
::: {.eyebrow}
Hold constant
:::
### Setup
Start with one backlog.

Keep arrivals and capacity fixed.
:::

::: {.definition-card}
::: {.eyebrow}
Change one thing
:::
### Closure pattern
Newest-First and Oldest-First are simplified closure patterns.

They are used only to reveal metric behavior.
:::
:::

::: {.line-anchor .line-anchor-subtle .centered}
Same team. Same capacity. Different simulated closure pattern.
:::

::: {.centered .muted}
This is not a remediation policy. It is a metric demonstration.

Prioritization is risk-based.
:::

---

## Real prioritization is risk-based {.center}

::: {.risk-list}

- **Severity**
- **Exploitability**
- **Exposure**

- **Asset criticality**
- **Threat intelligence**

:::

::: {.centered .risk-contrast}
Not newest-first. Not oldest-first.
:::

::: {.line-anchor .centered .blood-line}
Metrics do not drive the queue. They show the consequences.
:::

---

## MTTR reflects recent closures {.chart-slide}

```{python}
#| echo: false
#| fig-alt: "Line chart of MTTR over time under two simplified closure patterns. MTTR looks lower when recent findings close first because it reflects age at closure."
mttr_plot = metrics.select(
    "month_index",
    "strategy_label",
    pl.col("mttr_days").alias("days"),
)
(
    strategy_line_plot(
        dataframe=mttr_plot,
        y_col="days",
        title="MTTR reflects recent closures",
        subtitle="Closing recent findings first lowers the age of closed work in the time window.",
        y_axis_label="MTTR (days)",
    )
)
```

::: {.kicker .centered}
Newest-First looks better on MTTR alone.
:::

::: {.centered .muted}
MTTR is windowed. Label the window before calling a regression.
:::

---

## MOVA shows backlog age {.chart-slide}

```{python}
#| echo: false
#| fig-alt: "Line chart of MOVA over time under two simplified closure patterns. It shows the age of work still open today under each pattern."
mova_plot = metrics.select(
    "month_index",
    "strategy_label",
    pl.col("mova_days").alias("days"),
)
(
    strategy_line_plot(
        dataframe=mova_plot,
        y_col="days",
        title="MOVA shows backlog age",
        subtitle="How old is the work still open today?",
        y_axis_label="MOVA (days)",
        y_limits=(0, mova_plot.get_column("days").max() * 1.08),
    )
)
```

::: {.kicker .centered}
MOVA reveals the backlog you are carrying.
:::

::: {.centered .muted}
MOVA is not windowed. It reflects the backlog right now.
:::

---

## Paradox {.center .statement-slide}

::: {.metric-big .centered}
The MTTR Paradox
:::

::: {.paradox-cue}
::: {.paradox-pill}
<span class="paradox-name">MTTR</span>
<span class="paradox-trend">↑</span>
:::
::: {.paradox-connector}
while
:::
::: {.paradox-pill}
<span class="paradox-name">MOVA</span>
<span class="paradox-trend paradox-trend-down">↓</span>
:::
:::

::: {.line-anchor .line-anchor-subtle .centered}
MTTR can rise while backlog age falls.
:::

---

## When the dashboard shows only MTTR

::: {.dashboard-grid}

::: {.dashboard-card .dashboard-bad}
::: {.dashboard-title}
Oldest-First
:::
::: {.dashboard-value}
`{python} oldest_mttr`
:::
::: {.dashboard-label}
MTTR
:::
:::

::: {.dashboard-card .dashboard-good}
::: {.dashboard-title}
Newest-First
:::
::: {.dashboard-value}
`{python} newest_mttr`
:::
::: {.dashboard-label}
MTTR
:::
:::

:::

::: {.centered .muted}
A one-metric dashboard makes that the verdict.
:::

---

## What MOVA shows instead

::: {.dashboard-grid}

::: {.dashboard-card .dashboard-good}
::: {.dashboard-title}
Oldest-First
:::
::: {.dashboard-value}
`{python} oldest_mova`
:::
::: {.dashboard-label}
MOVA
:::
:::

::: {.dashboard-card .dashboard-bad}
::: {.dashboard-title}
Newest-First
:::
::: {.dashboard-value}
`{python} newest_mova`
:::
::: {.dashboard-label}
MOVA
:::
:::

:::

::: {.centered .muted}
On backlog age, the story changes.
:::

---

## Same data, different signals {.statement-slide .table-slide}

::: {.same-signals-gt}
```{python}
#| echo: false
decision_table = (
    summary.with_columns(
        pl.when(pl.col("strategy") == "oldest_first")
        .then(pl.lit("MOVA is lower"))
        .otherwise(pl.lit("MTTR is lower"))
        .alias("decision"),
        pl.when(pl.col("strategy") == "oldest_first")
        .then(pl.lit(0))
        .otherwise(pl.lit(1))
        .alias("sort_order"),
    )
    .sort("sort_order")
    .select(
        "strategy_label",
        pl.col("mttr_days").round(1).alias("mttr_days"),
        pl.col("mova_days").round(1).alias("mova_days"),
        "decision",
    )
)

(
    GT(decision_table, rowname_col="strategy_label")
    .tab_header(
        title="Same data, different signals",
        subtitle="Each metric answers one side of the earlier split.",
    )
    .tab_spanner(label="Flow", columns="mttr_days")
    .tab_spanner(label="Backlog Age", columns="mova_days")
    .tab_spanner(label="Interpretation", columns="decision")
    .cols_label(
        mttr_days="MTTR",
        mova_days="MOVA",
        decision="Interpretation",
    )
    .cols_align(align="left", columns=["decision"])
    .cols_align(
        align="center",
        columns=["mttr_days", "mova_days"],
    )
    .cols_width(
        strategy_label="24%",
        mttr_days="15%",
        mova_days="15%",
        decision="46%",
    )
    .fmt_number(columns=["mttr_days", "mova_days"], decimals=1)
    .tab_style(
        style=[style.fill("#f7e7ec"), style.text(color="#6b1230", weight="bold")],
        locations=loc.body(
            columns="mttr_days",
            rows=pl.col("strategy_label") == "Newest-First",
        ),
    )
    .tab_style(
        style=[style.fill("#fbf0f4"), style.text(color="#7a2941", weight="bold")],
        locations=loc.body(
            columns="decision",
            rows=pl.col("strategy_label") == "Newest-First",
        ),
    )
    .tab_style(
        style=[style.fill("#e4f0ee"), style.text(color="#134b52", weight="bold")],
        locations=loc.body(
            columns=["mova_days", "decision"],
            rows=pl.col("strategy_label") == "Oldest-First",
        ),
    )
    .tab_style(
        style=style.text(weight="bold"),
        locations=loc.stub(),
    )
    .tab_style(
        style=style.text(weight="bold"),
        locations=loc.column_labels(),
    )
    .tab_style(
        style=style.text(size="27px", weight="bold"),
        locations=loc.spanner_labels(ids=["Flow", "Backlog Age", "Interpretation"]),
    )
    .tab_style(
        style=style.text(size="46px", weight="bold"),
        locations=loc.body(columns=["mttr_days", "mova_days"]),
    )
    .opt_align_table_header(align="left")
    .opt_row_striping()
    .tab_options(
        table_width="100%",
        table_layout="fixed",
        table_font_size="30px",
        table_font_weight="500",
        heading_title_font_size="40px",
        heading_title_font_weight="700",
        heading_subtitle_font_size="24px",
        heading_padding="18px",
        column_labels_font_size="31px",
        column_labels_font_weight="800",
        column_labels_padding="18px",
        column_labels_padding_horizontal="14px",
        data_row_padding="22px",
        data_row_padding_horizontal="18px",
        stub_font_size="30px",
        stub_font_weight="800",
    )
)
```
:::

::: {.line-anchor .line-anchor-subtle .centered}
If you read only MTTR, you may misread progress.
:::

---

## Which signal are you reading? {.center .meme-slide}

![](assets/images/mttr-vs-mova-meme.png){width=100%}

---

## Both are partial signals {.center .statement-slide}

::: {.line-anchor .centered .blood-line}
Each signal is intentionally partial.

The mistake is reading either one alone.
:::

---

## When the dashboard shows both core signals {auto-animate="true"}

::: {.dashboard-grid}
::: {.dashboard-card .dashboard-detail}
::: {.dashboard-title}
Newest-First
:::
::: {.dashboard-stat}
<span>MTTR</span>
<strong>`{python} newest_mttr` days</strong>
:::
::: {.dashboard-stat}
<span>MOVA</span>
<strong>`{python} newest_mova` days</strong>
:::
:::

::: {.dashboard-card .dashboard-detail}
::: {.dashboard-title}
Oldest-First
:::
::: {.dashboard-stat}
<span>MTTR</span>
<strong>`{python} oldest_mttr` days</strong>
:::
::: {.dashboard-stat}
<span>MOVA</span>
<strong>`{python} oldest_mova` days</strong>
:::
:::
:::

::: {.centered .muted}
The disagreement is now visible.

Interpret it before judging progress.
:::

---

## Read the signals like a system {.center .signals-slide}

::: {.score-grid .signals-grid}

::: {.score-card}
::: {.score-title}
MTTR <i class="bi bi-arrow-down text-good"></i> + MOVA <i class="bi bi-arrow-down text-good"></i>
:::
::: {.score-label}
System improving
:::
:::

::: {.score-card}
::: {.score-title}
MTTR <i class="bi bi-arrow-up text-warn"></i> + MOVA <i class="bi bi-arrow-down text-good"></i>
:::
::: {.score-label}
Clearing older backlog
:::
:::

::: {.score-card}
::: {.score-title}
MTTR <i class="bi bi-arrow-down text-good"></i> + MOVA <i class="bi bi-arrow-up text-bad"></i>
:::
::: {.score-label}
Backlog is aging
:::
:::

:::

::: {.line-anchor .centered}
Read them together or you may misread progress.
:::

---

## What to report

::: {.definition-grid}
::: {.definition-card}
::: {.eyebrow}
Monthly view
:::
### MTTR + MOVA
Put both on the same monthly view.
:::

::: {.definition-card}
::: {.eyebrow}
Definitions
:::
### Flow + backlog age
Use the earlier split consistently.
:::

::: {.definition-card}
::: {.eyebrow}
Review rule
:::
### Read together
Review MTTR with MOVA visible.
:::

::: {.definition-card}
::: {.eyebrow}
Framing
:::
### Label the window
Label the MTTR time window before calling a regression.
:::
:::

---

## Add data science to your toolbox {.toolbox-slide}

::: {.toolbox-intro}
Once the metric is defined, make the analysis reproducible.

Define the metric in code before it reaches the dashboard.
:::

::: {.tool-grid}

::: {.tool-card .tool-define}
<div class="tool-head">
<div class="tool-icon" aria-hidden="true"><i class="bi bi-code-slash"></i></div>
<div class="tool-title">Define the metric</div>
</div>
<div class="tool-note">Define the logic in code before it hits the dashboard</div>
<div class="tool-method">Polars</div>
:::

::: {.tool-card .tool-explore}
<div class="tool-head">
<div class="tool-icon" aria-hidden="true"><i class="bi bi-search"></i></div>
<div class="tool-title">Inspect the raw data</div>
</div>
<div class="tool-note">Inspect raw records early to catch edge cases</div>
<div class="tool-method">Positron</div>
:::

::: {.tool-card .tool-show}
<div class="tool-head">
<div class="tool-icon" aria-hidden="true"><i class="bi bi-graph-up"></i></div>
<div class="tool-title">Visualize the trend</div>
</div>
<div class="tool-note">Visualize flow and backlog age over time</div>
<div class="tool-method">Plotnine</div>
:::

::: {.tool-card .tool-summarize}
<div class="tool-head">
<div class="tool-icon" aria-hidden="true"><i class="bi bi-table"></i></div>
<div class="tool-title">Summarize the backlog</div>
</div>
<div class="tool-note">Summarize the current state at a glance</div>
<div class="tool-method">Great Tables</div>
:::

::: {.tool-card .tool-explore-metric}
<div class="tool-head">
<div class="tool-icon" aria-hidden="true"><i class="bi bi-sliders"></i></div>
<div class="tool-title">Explore the metric</div>
</div>
<div class="tool-note">Turn the analysis into a simple app people can inspect</div>
<div class="tool-method">Shiny for Python</div>
:::

::: {.tool-card .tool-report}
<div class="tool-head">
<div class="tool-icon" aria-hidden="true"><i class="bi bi-journal-text"></i></div>
<div class="tool-title">Publish the analysis</div>
</div>
<div class="tool-note">Publish code, charts, and narrative together</div>
<div class="tool-method">Quarto</div>
:::

:::

::: {.toolbox-takeaway}
Start with an export. Define the metric in code first.
:::

---

## A simple policy {.center}

::: {.definition-grid .narrow-grid .decision-grid}

::: {.definition-card .decision-card .decision-card-observe}
::: {.eyebrow}
Policy
:::
### Track MTTR + MOVA
Review both together in the same view.
:::

::: {.definition-card .decision-card .decision-card-guardrail}
::: {.eyebrow}
Guardrail
:::
### Do not drive the queue
Prioritization stays risk-based.
:::

:::

::: {.line-anchor .centered .decision-close}
Metrics verify decisions. They do not decide them.
:::

---

## Read the system, not just the metric {.center .statement-slide}

::: {.metric-big .centered}
Read both signals.
:::

::: {.metric-big .centered .muted}
Question the disagreement.
:::

::: {.line-anchor .centered .blood-line}
If they disagree, the system is telling you something.
:::

::: {.centered .muted}
Read both or you may misread progress.
:::

::: {.cta-link}
typeerror.com/mova
:::