diff --git a/CHANGELOG.md b/CHANGELOG.md index 84e843ae..333daad4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 6.7.1 - 2025-09-01 + +- fix: Add base64 inline image sanitization + # 6.7.0 - 2025-08-26 - feat: Add support for feature flag dependencies diff --git a/posthog/ai/anthropic/anthropic.py b/posthog/ai/anthropic/anthropic.py index c881495e..ffb34dee 100644 --- a/posthog/ai/anthropic/anthropic.py +++ b/posthog/ai/anthropic/anthropic.py @@ -16,6 +16,7 @@ merge_system_prompt, with_privacy_mode, ) +from posthog.ai.sanitization import sanitize_anthropic from posthog.client import Client as PostHogClient from posthog import setup @@ -184,7 +185,7 @@ def _capture_streaming_event( "$ai_input": with_privacy_mode( self._client._ph_client, posthog_privacy_mode, - merge_system_prompt(kwargs, "anthropic"), + sanitize_anthropic(merge_system_prompt(kwargs, "anthropic")), ), "$ai_output_choices": with_privacy_mode( self._client._ph_client, diff --git a/posthog/ai/anthropic/anthropic_async.py b/posthog/ai/anthropic/anthropic_async.py index 68f0c7c3..afb8dc58 100644 --- a/posthog/ai/anthropic/anthropic_async.py +++ b/posthog/ai/anthropic/anthropic_async.py @@ -17,6 +17,7 @@ merge_system_prompt, with_privacy_mode, ) +from posthog.ai.sanitization import sanitize_anthropic from posthog.client import Client as PostHogClient @@ -184,7 +185,7 @@ async def _capture_streaming_event( "$ai_input": with_privacy_mode( self._client._ph_client, posthog_privacy_mode, - merge_system_prompt(kwargs, "anthropic"), + sanitize_anthropic(merge_system_prompt(kwargs, "anthropic")), ), "$ai_output_choices": with_privacy_mode( self._client._ph_client, diff --git a/posthog/ai/gemini/gemini.py b/posthog/ai/gemini/gemini.py index 8bef4304..9de0e0a9 100644 --- a/posthog/ai/gemini/gemini.py +++ b/posthog/ai/gemini/gemini.py @@ -16,6 +16,7 @@ get_model_params, with_privacy_mode, ) +from posthog.ai.sanitization import sanitize_gemini from posthog.client import Client as PostHogClient @@ -347,7 +348,7 @@ def _capture_streaming_event( "$ai_input": with_privacy_mode( self._ph_client, privacy_mode, - self._format_input(contents), + sanitize_gemini(self._format_input(contents)), ), "$ai_output_choices": with_privacy_mode( self._ph_client, diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 5d9ffbd4..c244e85e 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -37,6 +37,7 @@ from posthog import setup from posthog.ai.utils import get_model_params, with_privacy_mode +from posthog.ai.sanitization import sanitize_langchain from posthog.client import Client log = logging.getLogger("posthog") @@ -480,7 +481,7 @@ def _capture_trace_or_span( event_properties = { "$ai_trace_id": trace_id, "$ai_input_state": with_privacy_mode( - self._ph_client, self._privacy_mode, run.input + self._ph_client, self._privacy_mode, sanitize_langchain(run.input) ), "$ai_latency": run.latency, "$ai_span_name": run.name, @@ -550,7 +551,7 @@ def _capture_generation( "$ai_model": run.model, "$ai_model_parameters": run.model_params, "$ai_input": with_privacy_mode( - self._ph_client, self._privacy_mode, run.input + self._ph_client, self._privacy_mode, sanitize_langchain(run.input) ), "$ai_http_status": 200, "$ai_latency": run.latency, diff --git a/posthog/ai/openai/openai.py b/posthog/ai/openai/openai.py index c9c74eb7..e7feccd0 100644 --- a/posthog/ai/openai/openai.py +++ b/posthog/ai/openai/openai.py @@ -15,6 +15,7 @@ get_model_params, with_privacy_mode, ) +from posthog.ai.sanitization import sanitize_openai, sanitize_openai_response from posthog.client import Client as PostHogClient from posthog import setup @@ -194,7 +195,9 @@ def _capture_streaming_event( "$ai_model": kwargs.get("model"), "$ai_model_parameters": get_model_params(kwargs), "$ai_input": with_privacy_mode( - self._client._ph_client, posthog_privacy_mode, kwargs.get("input") + self._client._ph_client, + posthog_privacy_mode, + sanitize_openai_response(kwargs.get("input")), ), "$ai_output_choices": with_privacy_mode( self._client._ph_client, @@ -427,7 +430,9 @@ def _capture_streaming_event( "$ai_model": kwargs.get("model"), "$ai_model_parameters": get_model_params(kwargs), "$ai_input": with_privacy_mode( - self._client._ph_client, posthog_privacy_mode, kwargs.get("messages") + self._client._ph_client, + posthog_privacy_mode, + sanitize_openai(kwargs.get("messages")), ), "$ai_output_choices": with_privacy_mode( self._client._ph_client, @@ -518,7 +523,9 @@ def create( "$ai_provider": "openai", "$ai_model": kwargs.get("model"), "$ai_input": with_privacy_mode( - self._client._ph_client, posthog_privacy_mode, kwargs.get("input") + self._client._ph_client, + posthog_privacy_mode, + sanitize_openai_response(kwargs.get("input")), ), "$ai_http_status": 200, "$ai_input_tokens": usage_stats.get("prompt_tokens", 0), diff --git a/posthog/ai/openai/openai_async.py b/posthog/ai/openai/openai_async.py index 4b110210..ae1a5352 100644 --- a/posthog/ai/openai/openai_async.py +++ b/posthog/ai/openai/openai_async.py @@ -16,6 +16,7 @@ get_model_params, with_privacy_mode, ) +from posthog.ai.sanitization import sanitize_openai, sanitize_openai_response from posthog.client import Client as PostHogClient @@ -195,7 +196,9 @@ async def _capture_streaming_event( "$ai_model": kwargs.get("model"), "$ai_model_parameters": get_model_params(kwargs), "$ai_input": with_privacy_mode( - self._client._ph_client, posthog_privacy_mode, kwargs.get("input") + self._client._ph_client, + posthog_privacy_mode, + sanitize_openai_response(kwargs.get("input")), ), "$ai_output_choices": with_privacy_mode( self._client._ph_client, @@ -431,7 +434,9 @@ async def _capture_streaming_event( "$ai_model": kwargs.get("model"), "$ai_model_parameters": get_model_params(kwargs), "$ai_input": with_privacy_mode( - self._client._ph_client, posthog_privacy_mode, kwargs.get("messages") + self._client._ph_client, + posthog_privacy_mode, + sanitize_openai(kwargs.get("messages")), ), "$ai_output_choices": with_privacy_mode( self._client._ph_client, @@ -522,7 +527,9 @@ async def create( "$ai_provider": "openai", "$ai_model": kwargs.get("model"), "$ai_input": with_privacy_mode( - self._client._ph_client, posthog_privacy_mode, kwargs.get("input") + self._client._ph_client, + posthog_privacy_mode, + sanitize_openai_response(kwargs.get("input")), ), "$ai_http_status": 200, "$ai_input_tokens": usage_stats.get("prompt_tokens", 0), diff --git a/posthog/ai/sanitization.py b/posthog/ai/sanitization.py new file mode 100644 index 00000000..a0953d07 --- /dev/null +++ b/posthog/ai/sanitization.py @@ -0,0 +1,226 @@ +import re +from typing import Any +from urllib.parse import urlparse + +REDACTED_IMAGE_PLACEHOLDER = "[base64 image redacted]" + + +def is_base64_data_url(text: str) -> bool: + return re.match(r"^data:([^;]+);base64,", text) is not None + + +def is_valid_url(text: str) -> bool: + try: + result = urlparse(text) + return bool(result.scheme and result.netloc) + except Exception: + pass + + return text.startswith(("/", "./", "../")) + + +def is_raw_base64(text: str) -> bool: + if is_valid_url(text): + return False + + return len(text) > 20 and re.match(r"^[A-Za-z0-9+/]+=*$", text) is not None + + +def redact_base64_data_url(value: Any) -> Any: + if not isinstance(value, str): + return value + + if is_base64_data_url(value): + return REDACTED_IMAGE_PLACEHOLDER + + if is_raw_base64(value): + return REDACTED_IMAGE_PLACEHOLDER + + return value + + +def process_messages(messages: Any, transform_content_func) -> Any: + if not messages: + return messages + + def process_content(content: Any) -> Any: + if isinstance(content, str): + return content + + if not content: + return content + + if isinstance(content, list): + return [transform_content_func(item) for item in content] + + return transform_content_func(content) + + def process_message(msg: Any) -> Any: + if not isinstance(msg, dict) or "content" not in msg: + return msg + return {**msg, "content": process_content(msg["content"])} + + if isinstance(messages, list): + return [process_message(msg) for msg in messages] + + return process_message(messages) + + +def sanitize_openai_image(item: Any) -> Any: + if not isinstance(item, dict): + return item + + if ( + item.get("type") == "image_url" + and isinstance(item.get("image_url"), dict) + and "url" in item["image_url"] + ): + return { + **item, + "image_url": { + **item["image_url"], + "url": redact_base64_data_url(item["image_url"]["url"]), + }, + } + + return item + + +def sanitize_openai_response_image(item: Any) -> Any: + if not isinstance(item, dict): + return item + + if item.get("type") == "input_image" and "image_url" in item: + return { + **item, + "image_url": redact_base64_data_url(item["image_url"]), + } + + return item + + +def sanitize_anthropic_image(item: Any) -> Any: + if not isinstance(item, dict): + return item + + if ( + item.get("type") == "image" + and isinstance(item.get("source"), dict) + and item["source"].get("type") == "base64" + and "data" in item["source"] + ): + # For Anthropic, if the source type is "base64", we should always redact the data + # The provider is explicitly telling us this is base64 data + return { + **item, + "source": { + **item["source"], + "data": REDACTED_IMAGE_PLACEHOLDER, + }, + } + + return item + + +def sanitize_gemini_part(part: Any) -> Any: + if not isinstance(part, dict): + return part + + if ( + "inline_data" in part + and isinstance(part["inline_data"], dict) + and "data" in part["inline_data"] + ): + # For Gemini, the inline_data structure indicates base64 data + # We should redact any string data in this context + return { + **part, + "inline_data": { + **part["inline_data"], + "data": REDACTED_IMAGE_PLACEHOLDER, + }, + } + + return part + + +def process_gemini_item(item: Any) -> Any: + if not isinstance(item, dict): + return item + + if "parts" in item and item["parts"]: + parts = item["parts"] + if isinstance(parts, list): + parts = [sanitize_gemini_part(part) for part in parts] + else: + parts = sanitize_gemini_part(parts) + + return {**item, "parts": parts} + + return item + + +def sanitize_langchain_image(item: Any) -> Any: + if not isinstance(item, dict): + return item + + if ( + item.get("type") == "image_url" + and isinstance(item.get("image_url"), dict) + and "url" in item["image_url"] + ): + return { + **item, + "image_url": { + **item["image_url"], + "url": redact_base64_data_url(item["image_url"]["url"]), + }, + } + + if item.get("type") == "image" and "data" in item: + return {**item, "data": redact_base64_data_url(item["data"])} + + if ( + item.get("type") == "image" + and isinstance(item.get("source"), dict) + and "data" in item["source"] + ): + # Anthropic style - raw base64 in structured format, always redact + return { + **item, + "source": { + **item["source"], + "data": REDACTED_IMAGE_PLACEHOLDER, + }, + } + + if item.get("type") == "media" and "data" in item: + return {**item, "data": redact_base64_data_url(item["data"])} + + return item + + +def sanitize_openai(data: Any) -> Any: + return process_messages(data, sanitize_openai_image) + + +def sanitize_openai_response(data: Any) -> Any: + return process_messages(data, sanitize_openai_response_image) + + +def sanitize_anthropic(data: Any) -> Any: + return process_messages(data, sanitize_anthropic_image) + + +def sanitize_gemini(data: Any) -> Any: + if not data: + return data + + if isinstance(data, list): + return [process_gemini_item(item) for item in data] + + return process_gemini_item(data) + + +def sanitize_langchain(data: Any) -> Any: + return process_messages(data, sanitize_langchain_image) diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index db3468fe..5687ffb2 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -5,6 +5,12 @@ from httpx import URL from posthog.client import Client as PostHogClient +from posthog.ai.sanitization import ( + sanitize_openai, + sanitize_anthropic, + sanitize_gemini, + sanitize_langchain, +) def get_model_params(kwargs: Dict[str, Any]) -> Dict[str, Any]: @@ -422,12 +428,15 @@ def call_llm_and_track_usage( usage = get_usage(response, provider) messages = merge_system_prompt(kwargs, provider) + sanitized_messages = sanitize_messages(messages, provider) event_properties = { "$ai_provider": provider, "$ai_model": kwargs.get("model"), "$ai_model_parameters": get_model_params(kwargs), - "$ai_input": with_privacy_mode(ph_client, posthog_privacy_mode, messages), + "$ai_input": with_privacy_mode( + ph_client, posthog_privacy_mode, sanitized_messages + ), "$ai_output_choices": with_privacy_mode( ph_client, posthog_privacy_mode, format_response(response, provider) ), @@ -536,12 +545,15 @@ async def call_llm_and_track_usage_async( usage = get_usage(response, provider) messages = merge_system_prompt(kwargs, provider) + sanitized_messages = sanitize_messages(messages, provider) event_properties = { "$ai_provider": provider, "$ai_model": kwargs.get("model"), "$ai_model_parameters": get_model_params(kwargs), - "$ai_input": with_privacy_mode(ph_client, posthog_privacy_mode, messages), + "$ai_input": with_privacy_mode( + ph_client, posthog_privacy_mode, sanitized_messages + ), "$ai_output_choices": with_privacy_mode( ph_client, posthog_privacy_mode, format_response(response, provider) ), @@ -600,6 +612,19 @@ async def call_llm_and_track_usage_async( return response +def sanitize_messages(data: Any, provider: str) -> Any: + """Sanitize messages using provider-specific sanitization functions.""" + if provider == "anthropic": + return sanitize_anthropic(data) + elif provider == "openai": + return sanitize_openai(data) + elif provider == "gemini": + return sanitize_gemini(data) + elif provider == "langchain": + return sanitize_langchain(data) + return data + + def with_privacy_mode(ph_client: PostHogClient, privacy_mode: bool, value: Any): if ph_client.privacy_mode or privacy_mode: return None diff --git a/posthog/test/ai/test_sanitization.py b/posthog/test/ai/test_sanitization.py new file mode 100644 index 00000000..0031bafb --- /dev/null +++ b/posthog/test/ai/test_sanitization.py @@ -0,0 +1,335 @@ +import unittest + +from posthog.ai.sanitization import ( + redact_base64_data_url, + sanitize_openai, + sanitize_openai_response, + sanitize_anthropic, + sanitize_gemini, + sanitize_langchain, + is_base64_data_url, + is_raw_base64, + REDACTED_IMAGE_PLACEHOLDER, +) + + +class TestSanitization(unittest.TestCase): + def setUp(self): + self.sample_base64_image = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ..." + self.sample_base64_png = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA..." + self.regular_url = "https://example.com/image.jpg" + self.raw_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUl==" + + def test_is_base64_data_url(self): + self.assertTrue(is_base64_data_url(self.sample_base64_image)) + self.assertTrue(is_base64_data_url(self.sample_base64_png)) + self.assertFalse(is_base64_data_url(self.regular_url)) + self.assertFalse(is_base64_data_url("regular text")) + + def test_is_raw_base64(self): + self.assertTrue(is_raw_base64(self.raw_base64)) + self.assertFalse(is_raw_base64("short")) + self.assertFalse(is_raw_base64(self.regular_url)) + self.assertFalse(is_raw_base64("/path/to/file")) + + def test_redact_base64_data_url(self): + self.assertEqual( + redact_base64_data_url(self.sample_base64_image), REDACTED_IMAGE_PLACEHOLDER + ) + self.assertEqual( + redact_base64_data_url(self.sample_base64_png), REDACTED_IMAGE_PLACEHOLDER + ) + self.assertEqual(redact_base64_data_url(self.regular_url), self.regular_url) + self.assertEqual(redact_base64_data_url(None), None) + self.assertEqual(redact_base64_data_url(123), 123) + + def test_sanitize_openai(self): + input_data = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": { + "url": self.sample_base64_image, + "detail": "high", + }, + }, + ], + } + ] + + result = sanitize_openai(input_data) + + self.assertEqual(result[0]["content"][0]["text"], "What is in this image?") + self.assertEqual( + result[0]["content"][1]["image_url"]["url"], REDACTED_IMAGE_PLACEHOLDER + ) + self.assertEqual(result[0]["content"][1]["image_url"]["detail"], "high") + + def test_sanitize_openai_preserves_regular_urls(self): + input_data = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": self.regular_url}, + } + ], + } + ] + + result = sanitize_openai(input_data) + self.assertEqual(result[0]["content"][0]["image_url"]["url"], self.regular_url) + + def test_sanitize_openai_response(self): + input_data = [ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": self.sample_base64_image, + } + ], + } + ] + + result = sanitize_openai_response(input_data) + self.assertEqual( + result[0]["content"][0]["image_url"], REDACTED_IMAGE_PLACEHOLDER + ) + + def test_sanitize_anthropic(self): + input_data = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": "base64data", + }, + }, + ], + } + ] + + result = sanitize_anthropic(input_data) + + self.assertEqual(result[0]["content"][0]["text"], "What is in this image?") + self.assertEqual( + result[0]["content"][1]["source"]["data"], REDACTED_IMAGE_PLACEHOLDER + ) + self.assertEqual(result[0]["content"][1]["source"]["type"], "base64") + self.assertEqual(result[0]["content"][1]["source"]["media_type"], "image/jpeg") + + def test_sanitize_gemini(self): + input_data = [ + { + "parts": [ + {"text": "What is in this image?"}, + { + "inline_data": { + "mime_type": "image/jpeg", + "data": "base64data", + } + }, + ] + } + ] + + result = sanitize_gemini(input_data) + + self.assertEqual(result[0]["parts"][0]["text"], "What is in this image?") + self.assertEqual( + result[0]["parts"][1]["inline_data"]["data"], REDACTED_IMAGE_PLACEHOLDER + ) + self.assertEqual( + result[0]["parts"][1]["inline_data"]["mime_type"], "image/jpeg" + ) + + def test_sanitize_langchain_openai_style(self): + input_data = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": self.sample_base64_image}, + } + ], + } + ] + + result = sanitize_langchain(input_data) + self.assertEqual( + result[0]["content"][0]["image_url"]["url"], REDACTED_IMAGE_PLACEHOLDER + ) + + def test_sanitize_langchain_anthropic_style(self): + input_data = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": {"data": "base64data"}, + } + ], + } + ] + + result = sanitize_langchain(input_data) + self.assertEqual( + result[0]["content"][0]["source"]["data"], REDACTED_IMAGE_PLACEHOLDER + ) + + def test_sanitize_with_data_url_format(self): + # Test that data URLs are properly detected and redacted across providers + data_url = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD" + + # OpenAI format + openai_data = [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": data_url}}], + } + ] + result = sanitize_openai(openai_data) + self.assertEqual( + result[0]["content"][0]["image_url"]["url"], REDACTED_IMAGE_PLACEHOLDER + ) + + # Anthropic format + anthropic_data = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/jpeg", + "data": data_url, + }, + } + ], + } + ] + result = sanitize_anthropic(anthropic_data) + self.assertEqual( + result[0]["content"][0]["source"]["data"], REDACTED_IMAGE_PLACEHOLDER + ) + + # LangChain format + langchain_data = [ + {"role": "user", "content": [{"type": "image", "data": data_url}]} + ] + result = sanitize_langchain(langchain_data) + self.assertEqual(result[0]["content"][0]["data"], REDACTED_IMAGE_PLACEHOLDER) + + def test_sanitize_with_raw_base64(self): + # Test that raw base64 strings (without data URL prefix) are detected + raw_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUl==" + + # Test with Anthropic format + anthropic_data = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": raw_base64, + }, + } + ], + } + ] + result = sanitize_anthropic(anthropic_data) + self.assertEqual( + result[0]["content"][0]["source"]["data"], REDACTED_IMAGE_PLACEHOLDER + ) + + # Test with Gemini format + gemini_data = [ + {"parts": [{"inline_data": {"mime_type": "image/png", "data": raw_base64}}]} + ] + result = sanitize_gemini(gemini_data) + self.assertEqual( + result[0]["parts"][0]["inline_data"]["data"], REDACTED_IMAGE_PLACEHOLDER + ) + + def test_sanitize_preserves_regular_content(self): + # Ensure non-base64 content is preserved across all providers + regular_url = "https://example.com/image.jpg" + text_content = "What do you see?" + + # OpenAI + openai_data = [ + { + "role": "user", + "content": [ + {"type": "text", "text": text_content}, + {"type": "image_url", "image_url": {"url": regular_url}}, + ], + } + ] + result = sanitize_openai(openai_data) + self.assertEqual(result[0]["content"][0]["text"], text_content) + self.assertEqual(result[0]["content"][1]["image_url"]["url"], regular_url) + + # Anthropic + anthropic_data = [ + { + "role": "user", + "content": [ + {"type": "text", "text": text_content}, + {"type": "image", "source": {"type": "url", "url": regular_url}}, + ], + } + ] + result = sanitize_anthropic(anthropic_data) + self.assertEqual(result[0]["content"][0]["text"], text_content) + # URL-based images should remain unchanged + self.assertEqual(result[0]["content"][1]["source"]["url"], regular_url) + + def test_sanitize_handles_non_dict_content(self): + input_data = [{"role": "user", "content": "Just text"}] + + result = sanitize_openai(input_data) + self.assertEqual(result, input_data) + + def test_sanitize_handles_none_input(self): + self.assertIsNone(sanitize_openai(None)) + self.assertIsNone(sanitize_anthropic(None)) + self.assertIsNone(sanitize_gemini(None)) + self.assertIsNone(sanitize_langchain(None)) + + def test_sanitize_handles_single_message(self): + input_data = { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": self.sample_base64_image}, + } + ], + } + + result = sanitize_openai(input_data) + self.assertEqual( + result["content"][0]["image_url"]["url"], REDACTED_IMAGE_PLACEHOLDER + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/posthog/version.py b/posthog/version.py index 2a31d1ac..e9674076 100644 --- a/posthog/version.py +++ b/posthog/version.py @@ -1,4 +1,4 @@ -VERSION = "6.7.0" +VERSION = "6.7.1" if __name__ == "__main__": print(VERSION, end="") # noqa: T201