Merge branch 'master' into feat/ph-ai/posthogai-pricing-metadata

Author: tatoalo

.github/workflows/ci.yml

@@ -36,6 +36,10 @@ jobs:
               run: |
                   ruff format --check .
 
+            - name: Lint with ruff
+              run: |
+                  ruff check .
+
             - name: Check types with mypy
               run: |
                   mypy --no-site-packages --config-file mypy.ini  . | mypy-baseline filter

CHANGELOG.md

@@ -1,6 +1,6 @@
 # 6.9.2 - 2025-11-10
 
-- feat(ph-ai): PostHog properties dict in GenerationMetadata
+- fix(llma): fix cache token double subtraction in Langchain for non-Anthropic providers causing negative costs
 
 # 6.9.1 - 2025-11-07
 

posthog/__init__.py

@@ -18,8 +18,15 @@
     DEFAULT_CODE_VARIABLES_IGNORE_PATTERNS,
     DEFAULT_CODE_VARIABLES_MASK_PATTERNS,
 )
-from posthog.feature_flags import InconclusiveMatchError, RequiresServerEvaluation
-from posthog.types import FeatureFlag, FlagsAndPayloads, FeatureFlagResult
+from posthog.feature_flags import (
+    InconclusiveMatchError as InconclusiveMatchError,
+    RequiresServerEvaluation as RequiresServerEvaluation,
+)
+from posthog.types import (
+    FeatureFlag,
+    FlagsAndPayloads,
+    FeatureFlagResult as FeatureFlagResult,
+)
 from posthog.version import VERSION
 
 __version__ = VERSION

posthog/ai/langchain/callbacks.py

@@ -582,7 +582,7 @@ def _capture_generation(
             event_properties["$ai_is_error"] = True
         else:
             # Add usage
-            usage = _parse_usage(output)
+            usage = _parse_usage(output, run.provider, run.model)
             event_properties["$ai_input_tokens"] = usage.input_tokens
             event_properties["$ai_output_tokens"] = usage.output_tokens
             event_properties["$ai_cache_creation_input_tokens"] = (
@@ -703,6 +703,8 @@ class ModelUsage:
 
 def _parse_usage_model(
     usage: Union[BaseModel, dict],
+    provider: Optional[str] = None,
+    model: Optional[str] = None,
 ) -> ModelUsage:
     if isinstance(usage, BaseModel):
         usage = usage.__dict__
@@ -771,16 +773,30 @@ def _parse_usage_model(
             for mapped_key, dataclass_key in field_mapping.items()
         },
     )
-    # In LangChain, input_tokens is the sum of input and cache read tokens.
-    # Our cost calculation expects them to be separate, for Anthropic.
-    if normalized_usage.input_tokens and normalized_usage.cache_read_tokens:
+    # For Anthropic providers, LangChain reports input_tokens as the sum of input and cache read tokens.
+    # Our cost calculation expects them to be separate for Anthropic, so we subtract cache tokens.
+    # For other providers (OpenAI, etc.), input_tokens already includes cache tokens as expected.
+    # Match logic consistent with plugin-server: exact match on provider OR substring match on model
+    is_anthropic = False
+    if provider and provider.lower() == "anthropic":
+        is_anthropic = True
+    elif model and "anthropic" in model.lower():
+        is_anthropic = True
+
+    if (
+        is_anthropic
+        and normalized_usage.input_tokens
+        and normalized_usage.cache_read_tokens
+    ):
         normalized_usage.input_tokens = max(
             normalized_usage.input_tokens - normalized_usage.cache_read_tokens, 0
         )
     return normalized_usage
 
 
-def _parse_usage(response: LLMResult) -> ModelUsage:
+def _parse_usage(
+    response: LLMResult, provider: Optional[str] = None, model: Optional[str] = None
+) -> ModelUsage:
     # langchain-anthropic uses the usage field
     llm_usage_keys = ["token_usage", "usage"]
     llm_usage: ModelUsage = ModelUsage(
@@ -794,21 +810,25 @@ def _parse_usage(response: LLMResult) -> ModelUsage:
     if response.llm_output is not None:
         for key in llm_usage_keys:
             if response.llm_output.get(key):
-                llm_usage = _parse_usage_model(response.llm_output[key])
+                llm_usage = _parse_usage_model(
+                    response.llm_output[key], provider, model
+                )
                 break
 
     if hasattr(response, "generations"):
         for generation in response.generations:
             if "usage" in generation:
-                llm_usage = _parse_usage_model(generation["usage"])
+                llm_usage = _parse_usage_model(generation["usage"], provider, model)
                 break
 
             for generation_chunk in generation:
                 if generation_chunk.generation_info and (
                     "usage_metadata" in generation_chunk.generation_info
                 ):
                     llm_usage = _parse_usage_model(
-                        generation_chunk.generation_info["usage_metadata"]
+                        generation_chunk.generation_info["usage_metadata"],
+                        provider,
+                        model,
                     )
                     break
 
@@ -835,7 +855,7 @@ def _parse_usage(response: LLMResult) -> ModelUsage:
                     bedrock_anthropic_usage or bedrock_titan_usage or ollama_usage
                 )
                 if chunk_usage:
-                    llm_usage = _parse_usage_model(chunk_usage)
+                    llm_usage = _parse_usage_model(chunk_usage, provider, model)
                     break
 
     return llm_usage

posthog/exception_utils.py

@@ -929,7 +929,7 @@ def _compile_patterns(patterns):
     for pattern in patterns:
         try:
             compiled.append(re.compile(pattern))
-        except:
+        except Exception:
             pass
     return compiled
 

posthog/test/ai/langchain/test_callbacks.py

@@ -1586,13 +1586,58 @@ def test_anthropic_cache_write_and_read_tokens(mock_client):
     generation_props = generation_args["properties"]
 
     assert generation_args["event"] == "$ai_generation"
-    assert generation_props["$ai_input_tokens"] == 400
+    assert (
+        generation_props["$ai_input_tokens"] == 1200
+    )  # No provider metadata, no subtraction
     assert generation_props["$ai_output_tokens"] == 30
     assert generation_props["$ai_cache_creation_input_tokens"] == 0
     assert generation_props["$ai_cache_read_input_tokens"] == 800
     assert generation_props["$ai_reasoning_tokens"] == 0
 
 
+def test_anthropic_provider_subtracts_cache_tokens(mock_client):
+    """Test that Anthropic provider correctly subtracts cache tokens from input tokens."""
+    from langchain_core.outputs import LLMResult, ChatGeneration
+    from langchain_core.messages import AIMessage
+    from uuid import uuid4
+
+    cb = CallbackHandler(mock_client)
+    run_id = uuid4()
+
+    # Set up with Anthropic provider
+    cb._set_llm_metadata(
+        serialized={},
+        run_id=run_id,
+        messages=[{"role": "user", "content": "test"}],
+        metadata={"ls_provider": "anthropic", "ls_model_name": "claude-3-sonnet"},
+    )
+
+    # Response with cache tokens: 1200 input (includes 800 cached)
+    response = LLMResult(
+        generations=[
+            [
+                ChatGeneration(
+                    message=AIMessage(content="Response"),
+                    generation_info={
+                        "usage_metadata": {
+                            "input_tokens": 1200,
+                            "output_tokens": 50,
+                            "cache_read_input_tokens": 800,
+                        }
+                    },
+                )
+            ]
+        ],
+        llm_output={},
+    )
+
+    cb._pop_run_and_capture_generation(run_id, None, response)
+
+    generation_args = mock_client.capture.call_args_list[0][1]
+    assert generation_args["properties"]["$ai_input_tokens"] == 400  # 1200 - 800
+    assert generation_args["properties"]["$ai_cache_read_input_tokens"] == 800
+
+
 def test_openai_cache_read_tokens(mock_client):
     """Test that OpenAI cache read tokens are captured correctly."""
     prompt = ChatPromptTemplate.from_messages(
@@ -1628,7 +1673,7 @@ def test_openai_cache_read_tokens(mock_client):
     generation_props = generation_args["properties"]
 
     assert generation_args["event"] == "$ai_generation"
-    assert generation_props["$ai_input_tokens"] == 50
+    assert generation_props["$ai_input_tokens"] == 150  # No subtraction for OpenAI
     assert generation_props["$ai_output_tokens"] == 40
     assert generation_props["$ai_cache_read_input_tokens"] == 100
     assert generation_props["$ai_cache_creation_input_tokens"] == 0
@@ -1710,15 +1755,15 @@ def test_combined_reasoning_and_cache_tokens(mock_client):
     generation_props = generation_args["properties"]
 
     assert generation_args["event"] == "$ai_generation"
-    assert generation_props["$ai_input_tokens"] == 200
+    assert generation_props["$ai_input_tokens"] == 500  # No subtraction for OpenAI
     assert generation_props["$ai_output_tokens"] == 100
     assert generation_props["$ai_cache_read_input_tokens"] == 300
     assert generation_props["$ai_cache_creation_input_tokens"] == 0
     assert generation_props["$ai_reasoning_tokens"] == 60
 
 
 @pytest.mark.skipif(not OPENAI_API_KEY, reason="OPENAI_API_KEY is not set")
-def test_openai_reasoning_tokens(mock_client):
+def test_openai_reasoning_tokens_o4_mini(mock_client):
     model = ChatOpenAI(
         api_key=OPENAI_API_KEY, model="o4-mini", max_completion_tokens=10
     )
@@ -1919,8 +1964,8 @@ def test_cache_read_tokens_subtraction_from_input_tokens(mock_client):
     generation_props = generation_args["properties"]
 
     assert generation_args["event"] == "$ai_generation"
-    # Input tokens should be reduced: 150 - 100 = 50
-    assert generation_props["$ai_input_tokens"] == 50
+    # Input tokens not reduced without provider metadata
+    assert generation_props["$ai_input_tokens"] == 150
     assert generation_props["$ai_output_tokens"] == 40
     assert generation_props["$ai_cache_read_input_tokens"] == 100
 
@@ -1961,8 +2006,8 @@ def test_cache_read_tokens_subtraction_prevents_negative(mock_client):
     generation_props = generation_args["properties"]
 
     assert generation_args["event"] == "$ai_generation"
-    # Input tokens should be 0, not negative: max(80 - 100, 0) = 0
-    assert generation_props["$ai_input_tokens"] == 0
+    # Input tokens not reduced without provider metadata
+    assert generation_props["$ai_input_tokens"] == 80
     assert generation_props["$ai_output_tokens"] == 20
     assert generation_props["$ai_cache_read_input_tokens"] == 100
 

posthog/test/integrations/test_middleware.py

@@ -315,7 +315,9 @@ def test_sync_middleware_with_filter(self):
         get_response = Mock(return_value=mock_response)
 
         # Create middleware with request filter that filters all requests
-        request_filter = lambda req: False
+        def request_filter(req):
+            return False
+
         middleware = PosthogContextMiddleware.__new__(PosthogContextMiddleware)
         middleware.get_response = get_response
         middleware._is_coroutine = False