Percentage-mode decoder: anchor on probability<...>: instead of prob mass

The previous decoder picked the "best answer position" by scanning for
the position with the most numeric mass, and skipped range-label
positions via a `max_single_value_prob > 0.95 * total_mass` heuristic.
That worked but was implicit: correctness depended on the numeric
argmax at the answer position being ≤ 0.95 (empirical on gpt-5.4-nano),
not on a property of the response format.

Rewrite as a deterministic walk over the reconstructed chosen-token
stream (argmax per position; temperature=0 guarantees chosen==argmax):

  1. Accumulate text token-by-token.
  2. Fire an anchor once the running text matches
     `re.compile(r"probability[^:]*:", re.IGNORECASE)` — this catches
     both `Probability (0-100):` (label preserved) and paraphrases
     like `Estimated probability:` (label dropped) that the model
     emits roughly evenly on gpt-5.4-nano.
  3. Return the mass-weighted expected value at the first post-anchor
     position with ≥0.1 numeric mass.

Pre-anchor digit positions (the `0` and `100` from `(0-100)`) are prompt
echo — they precede the colon, so `[^:]*` correctly stops before them
and they never get considered as answers. No probability threshold
needed.

Live n=100 ACSIncome / gpt-5.4-nano: AUC 0.803 (previous heuristic
approach: 0.805 — a 0.002 rounding-noise difference; the anchor loses
one edge case where the model malformed its response as
`**Estimated probability ( 62%**` with no colon). Full test suite
still passes (296 tests). Percentage-mode unit tests rewritten to
mirror real response shapes (echoed label; paraphrased anchor;
preamble-only rejection).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: AndreFCruz

folktexts/qa_interface.py

@@ -263,7 +263,7 @@ def get_answer_from_model_output(
         # violates.
         if self.percentage:
             return self._decode_percentage_expected_value(
-                last_token_probs, numeric_tokens_vocab
+                last_token_probs, numeric_tokens_vocab, tokenizer_vocab
             )
 
         answer_text = ""
@@ -294,19 +294,49 @@ def get_answer_from_model_output(
         else:
             return float(numeric_answer_text)
 
+    # Preamble anchor for percentage-mode decoding. Matches the `probability`
+    # keyword (case-insensitive) followed by a colon at any distance, which
+    # is where all gpt-5-style responses (and the underlying prompt
+    # `NUMERIC_PERCENTAGE_CHAT_PROMPT`) place their answer delimiter:
+    #   - `**Probability (0-100): 28%**`      -> `Probability (0-100):`
+    #   - `**Estimated probability: 18%**`    -> `probability:`
+    #   - `**Probability (above $50,000): ~62%**` -> `Probability (above $50,000):`
+    # `[^:]*` deliberately stops at the FIRST colon after `probability`,
+    # so any digit tokens the model emits as prompt echo (the `0` and `100`
+    # in `(0-100)`) precede the anchor and are correctly skipped.
+    _ANSWER_ANCHOR: ClassVar[re.Pattern] = re.compile(
+        r"probability[^:]*:", re.IGNORECASE
+    )
+
     def _decode_percentage_expected_value(
         self,
         last_token_probs: np.ndarray,
         numeric_tokens_vocab: dict[str, int],
+        tokenizer_vocab: dict[str, int],
     ) -> float:
-        """Decode a percentage-mode response.
-
-        Real API responses often prefix the digit with markdown echo (e.g.
-        `'**Probability (0-100): 28**'`), so we scan every position for the
-        one where numeric-token mass concentrates on plausible percentage
-        values (integers in [0, 100]) and compute a mass-weighted expected
-        value at that position. Falls back to 0.0 if no such position
-        exists (model emitted no digits at all).
+        """Decode a percentage-mode response deterministically.
+
+        Real gpt-5 responses either echo the prompt's `(0-100)` range
+        label (`'**Probability (0-100): 28%**'`) or paraphrase it
+        (`'**Estimated probability: 18%**'`) before the answer digit.
+        Argmax-per-position over the whole response is noisy: the `0`
+        and `100` from the range label carry high probability and can
+        hijack the "best answer position" selection.
+
+        Algorithm:
+          1. Reconstruct the chosen-token stream via argmax per
+             position (temperature=0 guarantees chosen == argmax).
+          2. Walk positions accumulating text; once the accumulated
+             text matches `_ANSWER_ANCHOR` (`probability<...>:`), mark
+             the anchor as consumed.
+          3. Return the mass-weighted expected value of integer-percent
+             tokens at the first *post-anchor* position that carries
+             meaningful numeric mass.
+
+        Falls back to 0.0 when either no anchor appears (the model
+        never framed the answer as a probability — usually a preamble
+        that never reached the digit) or no post-anchor position
+        carries digit mass.
         """
         # Restrict to tokens that could be an integer percentage answer.
         # Rejects: multi-digit tokens with a `.`, values >100, non-digit
@@ -325,10 +355,28 @@ def _decode_percentage_expected_value(
             )
             return 0.0
 
-        best_pos_idx = -1
-        best_pos_mass = -1.0
-        best_pos_expected = 0.0
+        # Reconstruct chosen tokens per position to walk the response text
+        # deterministically. `tokenizer_vocab` on the WebAPI backend is a
+        # synthetic map (token string → sequential id) built from the top-K
+        # entries actually returned for this row, so the inverse is total
+        # on the ids that appear as argmax.
+        inverse_vocab = {tid: tok for tok, tid in tokenizer_vocab.items()}
+
+        passed_anchor = False
+        accumulated_text = ""
         for pos_idx, ltp in enumerate(last_token_probs):
+            chosen_id = int(np.argmax(ltp))
+            chosen_tok = inverse_vocab.get(chosen_id, "")
+            accumulated_text += chosen_tok
+
+            if not passed_anchor:
+                if self._ANSWER_ANCHOR.search(accumulated_text):
+                    passed_anchor = True
+                # Everything before/at the anchor is prompt echo (label
+                # digits like `0`/`100`, punctuation, preamble words) —
+                # never the answer, even when it's a valid percentage.
+                continue
+
             probs_by_value: dict[int, float] = {}
             for tok, tid in valid_percent_tokens.items():
                 p = ltp[tid]
@@ -337,32 +385,20 @@ def _decode_percentage_expected_value(
                 if p > 0:
                     probs_by_value[int(tok)] = probs_by_value.get(int(tok), 0.0) + p
             total_mass = sum(probs_by_value.values())
-            if total_mass <= 0:
+            # Positions between the label and the answer (whitespace,
+            # `):`, `%`, etc.) carry no numeric mass; skip them.
+            if total_mass < 0.1:
                 continue
-            # Skip positions where a single value dominates (>95% of mass)
-            # — those are almost always range-label positions like the `0`
-            # or `100` in `(0-100)`, not the answer. Real answer positions
-            # have a spread of plausible percentages (probe on gpt-5.4-nano:
-            # top value ≈0.3-0.5 with 4-5 alternatives sharing the rest).
-            max_single_value_prob = max(probs_by_value.values())
-            if max_single_value_prob > 0.95 * total_mass:
-                continue
-            if total_mass > best_pos_mass:
-                best_pos_mass = total_mass
-                best_pos_idx = pos_idx
-                # Mass-weighted expected value over the numeric tokens.
-                best_pos_expected = (
-                    sum(v * p for v, p in probs_by_value.items()) / total_mass
-                )
 
-        if best_pos_idx < 0 or best_pos_mass < 0.1:
-            logging.warning(
-                f"No confident percentage answer found "
-                f"(best_pos_mass={best_pos_mass:.3f}); returning 0.0."
-            )
-            return 0.0
+            expected = sum(v * p for v, p in probs_by_value.items()) / total_mass
+            return min(expected / 100.0, 1.0)
 
-        return min(best_pos_expected / 100.0, 1.0)
+        logging.warning(
+            "No post-anchor digit position found in percentage-mode "
+            "response (accumulated_text=%r); returning 0.0.",
+            accumulated_text,
+        )
+        return 0.0
 
 
 @dataclass(frozen=True, eq=True)

tests/test_web_api_classifier.py

@@ -421,41 +421,104 @@ def test_directnumericqa_percentage_prefix_has_no_decimal_prefill():
 
 def test_directnumericqa_percentage_decode_expected_value():
     """`percentage=True` returns a mass-weighted expected value over the
-    numeric tokens at the highest-confidence position, divided by 100."""
+    numeric tokens at the first post-`probability<...>:` position with
+    meaningful numeric mass, divided by 100."""
     import numpy as np
     from folktexts.qa_interface import DirectNumericQA
     q = DirectNumericQA(
-        column="test", text="P?", num_forward_passes=1, percentage=True,
+        column="test", text="P?", num_forward_passes=4, percentage=True,
     )
-    # Position 0: only '92' has meaningful mass → expected value = 92.
-    vocab = {"92": 0, "88": 1, "12": 2}
-    probs = np.zeros((1, 3))
-    probs[0, 0] = 0.9   # '92'
-    probs[0, 1] = 0.05  # '88' → expected = (92*0.9 + 88*0.05 + 12*0.05) / 1.0 = 87.8
-    probs[0, 2] = 0.05  # '12'
+    # Simulate `'**Probability (0-100): 92%**'`-style response: the
+    # accumulated text hits `probability(0-100):` by pos 2, then the
+    # answer arrives at pos 3.
+    vocab = {
+        "92": 0, "88": 1, "12": 2,           # answer tokens
+        "**Probability ": 3, "(0-100):": 4, " ": 5,
+    }
+    probs = np.zeros((4, 6))
+    probs[0, 3] = 1.0   # '**Probability '
+    probs[1, 4] = 1.0   # '(0-100):'    (anchor `Probability (0-100):` fires)
+    probs[2, 5] = 1.0   # ' '           (whitespace, no numeric mass → skip)
+    probs[3, 0] = 0.9   # '92'   → expected = (92*0.9 + 88*0.05 + 12*0.05) / 1.0 = 87.8
+    probs[3, 1] = 0.05  # '88'
+    probs[3, 2] = 0.05  # '12'
     decoded = q.get_answer_from_model_output(probs, vocab)
     assert decoded == pytest.approx(0.878)
 
 
-def test_directnumericqa_percentage_finds_answer_across_positions():
-    """When the model wraps the digit in markdown (early positions have no
-    digit mass, digit appears at a later position), the decoder should
-    lock onto the answer position rather than concatenate noise."""
+def test_directnumericqa_percentage_skips_range_label_digits():
+    """The `0` and `100` from an echoed `(0-100)` range label are valid
+    percentages, but appear BEFORE the anchor colon and must be skipped.
+    The decoder should walk to the first post-anchor position that
+    carries digit mass."""
+    import numpy as np
+    from folktexts.qa_interface import DirectNumericQA
+    q = DirectNumericQA(
+        column="test", text="P?", num_forward_passes=6, percentage=True,
+    )
+    # Response shape: '**Probability 0 - 100 ): 35'
+    # Prefix tokens produce accumulated `**Probability 0-100):` by
+    # position 5, so the anchor `probability[^:]*:` matches. Answer at
+    # position 5 must NOT be the `100` (still pre-anchor) — verify by
+    # placing the real answer at position 6.
+    vocab = {
+        "35": 0, "100": 1, "0": 2, "-": 3, "):": 4,
+        "**Probability ": 5,
+    }
+    probs = np.zeros((7, 6))
+    probs[0, 5] = 1.0    # '**Probability '  (no anchor yet — no colon)
+    probs[1, 2] = 1.0    # '0'               (range label starts)
+    probs[2, 3] = 1.0    # '-'
+    probs[3, 1] = 1.0    # '100'             (still pre-anchor — no colon yet)
+    probs[4, 4] = 1.0    # '):'              (accumulated hits `Probability 0-100):`)
+    probs[5, 0] = 0.0    # (empty — skipped, no mass)
+    probs[6, 0] = 0.6    # '35'              (post-anchor answer)
+    probs[6, 2] = 0.3    # '0'               (contributes 0)
+    decoded = q.get_answer_from_model_output(probs, vocab)
+    # Expected = (35*0.6 + 0*0.3 + 100*0) / 0.9 = 21 / 0.9 = 23.333...
+    # `100` doesn't appear at pos 6 → not counted.
+    assert decoded == pytest.approx(23.333 / 100.0, abs=0.001)
+
+
+def test_directnumericqa_percentage_returns_zero_without_anchor():
+    """When the model preambles for the entire token budget without
+    ever emitting `probability<...>:`, the decoder should return 0.0 —
+    those responses (~4% on gpt-5.4-nano) contain no answer to decode,
+    and inventing one would harm calibration."""
     import numpy as np
     from folktexts.qa_interface import DirectNumericQA
     q = DirectNumericQA(
         column="test", text="P?", num_forward_passes=3, percentage=True,
     )
-    # 3 positions × 3 numeric tokens. Position 0: no numeric mass.
-    # Position 1: dominated by literal `100` (the range label — should be
-    # rejected). Position 2: real answer at 35 with some spread.
-    vocab = {"35": 0, "100": 1, "0": 2}
+    # Preamble that never reaches the anchor `probability:`, but
+    # includes a numeric-looking token (`42` from `**42-year-old`).
+    vocab = {"42": 0, "Doctorate": 1, "**": 2}
     probs = np.zeros((3, 3))
-    probs[0, :] = [0.0, 0.0, 0.0]        # pos 0: no digits (e.g. '**')
-    probs[1, :] = [0.0, 0.99, 0.0]       # pos 1: only '100' — must be skipped
-    probs[2, :] = [0.6, 0.0, 0.3]        # pos 2: '35' dominates; expected = 35*0.6/0.9 = 23.33...
+    probs[0, 2] = 1.0   # '**'
+    probs[1, 0] = 1.0   # '42' — a numeric token but before any anchor
+    probs[2, 1] = 1.0   # 'Doctorate'
     decoded = q.get_answer_from_model_output(probs, vocab)
-    assert decoded == pytest.approx(23.333 / 100.0, abs=0.001)
+    assert decoded == 0.0
+
+
+def test_directnumericqa_percentage_accepts_paraphrased_anchor():
+    """Real gpt-5 responses often drop the `(0-100)` range label and
+    paraphrase (e.g. `'**Estimated probability: 18%**'`). The anchor
+    matches `probability<anything>:` so paraphrases still decode."""
+    import numpy as np
+    from folktexts.qa_interface import DirectNumericQA
+    q = DirectNumericQA(
+        column="test", text="P?", num_forward_passes=4, percentage=True,
+    )
+    # Response: '**Estimated probability: 18%**'
+    vocab = {"18": 0, "**Estimated probability": 1, ":": 2, " ": 3}
+    probs = np.zeros((4, 4))
+    probs[0, 1] = 1.0   # '**Estimated probability'  (no anchor yet)
+    probs[1, 2] = 1.0   # ':'                        (anchor fires here)
+    probs[2, 3] = 1.0   # ' '                        (whitespace — skip)
+    probs[3, 0] = 1.0   # '18'                       (answer)
+    decoded = q.get_answer_from_model_output(probs, vocab)
+    assert decoded == pytest.approx(0.18)
 
 
 def test_directnumericqa_percentage_clamps_above_100():