Validate when explanations logging is supported or not (#124)

Co-authored-by: huiwengoh <45724323+huiwengoh@users.noreply.github.com>

Author: jwmueller

CHANGELOG.md

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.1.34] - 2025-09-24
+
+### Added
+
+- Validate when explanations logging is supported or not
+
 ## [1.1.33] - 2025-09-23
 
 ### Fixed
@@ -355,7 +361,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Release of the Cleanlab TLM Python client.
 
-[Unreleased]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.33...HEAD
+[Unreleased]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.34...HEAD
+[1.1.34]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.33...v1.1.34
 [1.1.33]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.32...v1.1.33
 [1.1.32]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.31...v1.1.32
 [1.1.31]: https://github.com/cleanlab/cleanlab-tlm/compare/v1.1.30...v1.1.31

src/cleanlab_tlm/__about__.py

@@ -1,2 +1,2 @@
 # SPDX-License-Identifier: MIT
-__version__ = "1.1.33"
+__version__ = "1.1.34"

src/cleanlab_tlm/internal/constants.py

@@ -5,6 +5,7 @@
 _VALID_TLM_QUALITY_PRESETS: list[str] = ["best", "high", "medium", "low", "base"]
 _VALID_TLM_QUALITY_PRESETS_CHAT_COMPLETIONS: list[str] = ["medium", "low", "base"]
 _DEFAULT_TLM_QUALITY_PRESET: TLMQualityPreset = "medium"
+_QUALITY_PRESETS_W_CONSISTENCY_SAMPLES: set[str] = {"best", "high"}  # Must also apply to TrustworthyRAG
 _DEFAULT_TLM_MAX_TOKENS: int = 512
 _VALID_TLM_MODELS: list[str] = [
     "gpt-3.5-turbo-16k",
@@ -38,6 +39,17 @@
     "nova-pro",
 ]
 _TLM_DEFAULT_MODEL: str = "gpt-4.1-mini"
+_HIDDEN_REASONING_MODELS: set[str] = {
+    "o1-preview",
+    "o1",
+    "o1-mini",
+    "o3",
+    "o3-mini",
+    "o4-mini",
+    "gpt-5",
+    "gpt-5-mini",
+    "gpt-5-nano",
+}
 _TLM_DEFAULT_CONTEXT_LIMIT: int = 70000
 _VALID_TLM_TASKS: set[str] = {task.value for task in Task}
 TLM_TASK_SUPPORTING_CONSTRAIN_OUTPUTS: set[Task] = {
@@ -95,3 +107,7 @@
 _TLM_EVAL_QUERY_IDENTIFIER_KEY: str = "query_identifier"
 _TLM_EVAL_CONTEXT_IDENTIFIER_KEY: str = "context_identifier"
 _TLM_EVAL_RESPONSE_IDENTIFIER_KEY: str = "response_identifier"
+
+# Values that wont support logging explanation by default
+_REASONING_EFFORT_UNSUPPORTED_EXPLANATION_LOGGING: set[str] = {"none", "minimal"}
+_QUALITY_PRESETS_UNSUPPORTED_EXPLANATION_LOGGING: set[str] = {"low", "base"}  # For regular TLM not TrustworthyRAG

src/cleanlab_tlm/internal/validation.py

@@ -7,6 +7,10 @@
 
 from cleanlab_tlm.errors import ValidationError
 from cleanlab_tlm.internal.constants import (
+    _HIDDEN_REASONING_MODELS,
+    _QUALITY_PRESETS_UNSUPPORTED_EXPLANATION_LOGGING,
+    _QUALITY_PRESETS_W_CONSISTENCY_SAMPLES,
+    _REASONING_EFFORT_UNSUPPORTED_EXPLANATION_LOGGING,
     _TLM_CONSTRAIN_OUTPUTS_KEY,
     _TLM_DEFAULT_MODEL,
     _TLM_MAX_TOKEN_RANGE,
@@ -143,6 +147,12 @@ def validate_tlm_options(
                 )
 
         elif option == "use_self_reflection":
+            if "num_self_reflections" in options:
+                raise ValidationError(
+                    "`use_self_reflection` and `num_self_reflections` cannot be specified together. "
+                    "`use_self_reflection` is deprecated. Use `num_self_reflections` instead."
+                )
+
             if not isinstance(val, bool):
                 raise ValidationError(f"Invalid type {type(val)}, use_self_reflection must be a boolean")
 
@@ -169,9 +179,6 @@ def validate_tlm_options(
                 raise ValidationError(f"Invalid type {type(val)}, log must be a list of strings.")
 
             invalid_log_options = set(val) - TLM_VALID_LOG_OPTIONS
-
-            model = options.get("model", _TLM_DEFAULT_MODEL)
-
             if invalid_log_options:
                 raise ValidationError(
                     f"Invalid options for log: {invalid_log_options}. Valid options include: {TLM_VALID_LOG_OPTIONS}"
@@ -248,6 +255,75 @@ def _validate_trustworthy_rag_options(options: Optional[TLMOptions], initialized
         )
 
 
+def validate_logging(options: Optional[TLMOptions], quality_preset: str, subclass: str) -> None:
+    """If user asks to log explanation, then either:
+    ensure the specified TLM configuration supports this (return early), or otherwise raise informative error.
+
+    subclass: str
+        Either "TLM" or "TrustworthyRAG".
+        Indicates which type of TLM subclass object we are validating, different types have different quality_preset -> base options mappings.
+    """
+    if not options:
+        return
+    if "log" not in options:
+        return
+    if "explanation" not in options["log"]:
+        return
+
+    # Otherwise ensure we're using TLM configuration that supports logging explanations:
+    unsupported_error = ValueError(
+        "Your TLM configuration does not support logged explanations.  "
+        "Please remove 'explanation' from your specified `log`, and instead use the `get_explanation()` method after computing trust scores."
+    )
+
+    disable_trustworthiness = options.get("disable_trustworthiness", False)
+    if disable_trustworthiness:
+        raise unsupported_error
+
+    model = options.get("model")
+    num_consistency_samples = options.get("num_consistency_samples")
+    reasoning_effort = options.get("reasoning_effort")
+
+    num_self_reflections = options.get("num_self_reflections")
+    use_self_reflection = options.get("use_self_reflection")
+    if use_self_reflection is False:
+        # use_self_reflection is deprecated, consolidating to one parameter
+        num_self_reflections = 0
+
+    if num_consistency_samples == 0 and num_self_reflections == 0:
+        raise unsupported_error
+
+    if (num_consistency_samples is not None) and (num_consistency_samples > 0):
+        return
+    if (
+        (reasoning_effort is not None)
+        and (reasoning_effort not in _REASONING_EFFORT_UNSUPPORTED_EXPLANATION_LOGGING)
+        and (num_self_reflections is None or num_self_reflections > 0)
+    ):
+        return
+    if (num_consistency_samples == 0) and (reasoning_effort in _REASONING_EFFORT_UNSUPPORTED_EXPLANATION_LOGGING):
+        raise unsupported_error
+
+    if model in _HIDDEN_REASONING_MODELS:
+        raise unsupported_error
+
+    # Otherwise we can assume relevant TLMOptions were left unspecified by user
+    if subclass == "TLM":
+        if quality_preset in _QUALITY_PRESETS_UNSUPPORTED_EXPLANATION_LOGGING:
+            raise unsupported_error
+        if quality_preset not in _QUALITY_PRESETS_W_CONSISTENCY_SAMPLES:
+            if reasoning_effort in _REASONING_EFFORT_UNSUPPORTED_EXPLANATION_LOGGING:
+                raise unsupported_error
+            if num_self_reflections == 0 and num_consistency_samples is None:
+                raise unsupported_error
+
+    if subclass == "TrustworthyRAG":
+        if quality_preset not in _QUALITY_PRESETS_W_CONSISTENCY_SAMPLES:
+            raise unsupported_error
+        if num_consistency_samples == 0:
+            raise unsupported_error
+
+
 def process_and_validate_kwargs_constrain_outputs(
     prompt: Union[str, Sequence[str]],
     task: Optional[Task],

src/cleanlab_tlm/tlm.py

@@ -41,6 +41,7 @@
     tlm_explanation_format_tlm_result,
     tlm_prompt_process_and_validate_kwargs,
     tlm_score_process_response_and_kwargs,
+    validate_logging,
     validate_tlm_prompt,
     validate_tlm_prompt_response,
 )
@@ -117,6 +118,7 @@ def __init__(
         )
 
         # TLM-specific initialization
+        validate_logging(options=options, quality_preset=quality_preset, subclass="TLM")
         if task not in _VALID_TLM_TASKS:
             raise ValidationError(f"Invalid task {task} -- must be one of {_VALID_TLM_TASKS}")
 

src/cleanlab_tlm/utils/rag.py

@@ -44,6 +44,7 @@
     _validate_trustworthy_rag_options,
     tlm_explanation_format_trustworthy_rag_result,
     tlm_score_process_response_and_kwargs,
+    validate_logging,
     validate_rag_inputs,
 )
 
@@ -134,6 +135,7 @@ def __init__(
             self._evals = evals
 
         _validate_trustworthy_rag_options(options=options, initialized_evals=self._evals)
+        validate_logging(options=options, quality_preset=quality_preset, subclass="TrustworthyRAG")
 
         # Optional per-eval tool call overrides
         # These are name-based include/exclude sets used only in the _handle_tool_call_filtering decorator

tests/conftest.py

@@ -19,6 +19,7 @@
     TLM_SIMILARITY_MEASURES,
 )
 from cleanlab_tlm.internal.types import TLMQualityPreset
+from cleanlab_tlm.internal.validation import validate_logging
 from cleanlab_tlm.tlm import TLM, TLMOptions
 from cleanlab_tlm.utils.chat_completions import TLMChatCompletion
 from cleanlab_tlm.utils.rag import TrustworthyRAG
@@ -83,6 +84,16 @@ def tlm_dict(tlm_api_key: str) -> dict[str, Any]:
             tlm_dict[quality_preset][model] = {}
             task = random.choice(list(_VALID_TLM_TASKS))
             options = _get_options_dictionary(model)
+            try:  # ensure valid options/preset/model configuration for logging
+                validate_logging(options=options, quality_preset=quality_preset, subclass="TLM")
+            except ValueError as e:
+                if "does not support logged explanations" in str(e):
+                    options["log"].remove("explanation")
+                    if len(options["log"]) == 0:
+                        del options["log"]  # log cannot be empty list
+                else:
+                    raise ValueError(e)
+
             tlm_dict[quality_preset][model]["tlm"] = TLM(
                 quality_preset=quality_preset,
                 task=task,

tests/test_chat_completions.py

@@ -283,7 +283,7 @@ def test_tlm_chat_completion_structured_output_per_field_scoring() -> None:
     # test per_field_score
     assert len(score["log"]["per_field_score"]) == 2  # noqa: PLR2004
     assert {"steps", "final_answer"} == set(score["log"]["per_field_score"].keys())
-    assert tlm_chat.get_untrustworthy_fields(response=response, tlm_result=score) == ["final_answer"]
+    assert "final_answer" in tlm_chat.get_untrustworthy_fields(response=response, tlm_result=score)
 
 
 def test_tlm_chat_completion_score_invalid_response() -> None:

tests/test_validation.py

@@ -875,14 +875,114 @@ def test_disable_trustworthiness_with_custom_criteria_works(tlm_api_key: str) ->
 
 def test_disable_trustworthiness_without_custom_criteria_raises_error_rag(tlm_api_key: str) -> None:
     """Test that disable_trustworthiness=True without custom_eval_criteria raises ValueError for TrustworthyRAG."""
-    from cleanlab_tlm.utils.rag import TrustworthyRAG
-
     with pytest.raises(ValidationError, match="^When disable_trustworthiness=True in TrustworthyRAG"):
         TrustworthyRAG(evals=[], api_key=tlm_api_key, options={"disable_trustworthiness": True})
 
 
 def test_disable_trustworthiness_with_custom_criteria_works_rag(tlm_api_key: str) -> None:
     """Test that disable_trustworthiness=True with custom_eval_criteria works normally for TrustworthyRAG."""
-    from cleanlab_tlm.utils.rag import TrustworthyRAG
-
     TrustworthyRAG(api_key=tlm_api_key, options={"disable_trustworthiness": True})
+
+
+@pytest.mark.filterwarnings("ignore::DeprecationWarning")
+def test_validate_logging(tlm_api_key: str) -> None:
+    """Test validate_logging() method errors at the right times."""
+    # Settings that should not raise error:
+    TLM(api_key=tlm_api_key)
+    TLM(api_key=tlm_api_key, options={"log": ["explanation"]})
+    TLM(api_key=tlm_api_key, quality_preset="best", options={"log": ["explanation"], "reasoning_effort": "none"})
+    TLM(api_key=tlm_api_key, quality_preset="high", options={"log": ["explanation"], "reasoning_effort": "none"})
+    TLM(api_key=tlm_api_key, quality_preset="base", options={"log": ["explanation"], "num_consistency_samples": 8})
+    TLM(
+        api_key=tlm_api_key,
+        quality_preset="best",
+        options={"log": ["explanation"], "num_self_reflections": 0},
+    )
+    TLM(
+        api_key=tlm_api_key,
+        quality_preset="low",
+        options={
+            "log": ["explanation"],
+            "num_self_reflections": 0,
+            "num_consistency_samples": 4,
+        },
+    )
+    TLM(api_key=tlm_api_key, options={"model": "gpt-5-mini"})
+
+    # Settings that should error:
+    with pytest.raises(ValueError, match="does not support logged explanations"):
+        TLM(api_key=tlm_api_key, quality_preset="low", options={"log": ["explanation"]})
+    with pytest.raises(ValueError, match="does not support logged explanations"):
+        TLM(api_key=tlm_api_key, quality_preset="base", options={"log": ["explanation"]})
+    with pytest.raises(ValueError, match="does not support logged explanations"):
+        TLM(
+            api_key=tlm_api_key,
+            quality_preset="best",
+            options={"log": ["explanation"], "reasoning_effort": "none", "num_consistency_samples": 0},
+        )
+    with pytest.raises(ValueError, match="does not support logged explanations"):
+        TLM(
+            api_key=tlm_api_key,
+            options={"log": ["explanation"], "num_self_reflections": 0},
+        )
+
+    with pytest.raises(ValueError, match="does not support logged explanations"):
+        TLM(
+            api_key=tlm_api_key,
+            options={"log": ["explanation"], "use_self_reflection": False},
+        )
+    with pytest.raises(ValueError, match="does not support logged explanations"):
+        TLM(
+            api_key=tlm_api_key,
+            quality_preset="best",
+            options={
+                "log": ["explanation"],
+                "num_self_reflections": 0,
+                "num_consistency_samples": 0,
+            },
+        )
+    with pytest.raises(ValueError, match="does not support logged explanations"):
+        TLM(
+            api_key=tlm_api_key,
+            options={
+                "log": ["explanation"],
+                "reasoning_effort": "high",
+                "num_self_reflections": 0,
+            },
+        )
+    with pytest.raises(ValueError, match="does not support logged explanations"):
+        TLM(api_key=tlm_api_key, options={"log": ["explanation"], "model": "gpt-5-mini"})
+
+    # Settings that should not raise error:
+    TrustworthyRAG(api_key=tlm_api_key)
+    TrustworthyRAG(api_key=tlm_api_key, options={"log": ["explanation"], "num_consistency_samples": 5})
+    TrustworthyRAG(api_key=tlm_api_key, options={"log": ["explanation"], "reasoning_effort": "high"})
+    TrustworthyRAG(api_key=tlm_api_key, quality_preset="best", options={"log": ["explanation"]})
+
+    # Settings that should error:
+    with pytest.raises(ValueError, match="does not support logged explanations"):
+        TrustworthyRAG(api_key=tlm_api_key, options={"log": ["explanation"]})
+    with pytest.raises(ValueError, match="does not support logged explanations"):
+        TrustworthyRAG(
+            api_key=tlm_api_key, quality_preset="best", options={"log": ["explanation"], "num_consistency_samples": 0}
+        )
+    with pytest.raises(ValueError, match="does not support logged explanations"):
+        TrustworthyRAG(
+            api_key=tlm_api_key,
+            options={
+                "log": ["explanation"],
+                "reasoning_effort": "high",
+                "num_self_reflections": 0,
+            },
+        )
+    with pytest.raises(ValueError, match="does not support logged explanations"):
+        TrustworthyRAG(
+            api_key=tlm_api_key,
+            quality_preset="best",
+            options={
+                "log": ["explanation"],
+                "reasoning_effort": "high",
+                "num_self_reflections": 0,
+                "num_consistency_samples": 0,
+            },
+        )