normalize camel from api

Author: filip-michalsky

stagehand/handlers/extract_handler.py

@@ -13,7 +13,11 @@
     ExtractOptions,
     ExtractResult,
 )
-from stagehand.utils import inject_urls, transform_url_strings_to_ids
+from stagehand.utils import (
+    inject_urls,
+    transform_url_strings_to_ids,
+    convert_dict_keys_to_snake_case,
+)
 
 T = TypeVar("T", bound=BaseModel)
 
@@ -147,16 +151,22 @@ async def extract(
 
         processed_data_payload = raw_data_dict  # Default to the raw dictionary
 
-        if schema and isinstance(
-            raw_data_dict, dict
-        ):  # schema is the Pydantic model type
+        if schema and isinstance(raw_data_dict, dict):  # schema is the Pydantic model type
+            # Try direct validation first
             try:
                 validated_model_instance = schema.model_validate(raw_data_dict)
-                processed_data_payload = validated_model_instance  # Payload is now the Pydantic model instance
-            except Exception as e:
-                self.logger.error(
-                    f"Failed to validate extracted data against schema {schema.__name__}: {e}. Keeping raw data dict in .data field."
-                )
+                processed_data_payload = validated_model_instance
+            except Exception as first_error:
+                # Fallback: attempt camelCase→snake_case key normalization, then re-validate
+                try:
+                    normalized = convert_dict_keys_to_snake_case(raw_data_dict)
+                    validated_model_instance = schema.model_validate(normalized)
+                    processed_data_payload = validated_model_instance
+                except Exception as second_error:
+                    self.logger.error(
+                        f"Failed to validate extracted data against schema {schema.__name__}: {first_error}. "
+                        f"Normalization retry also failed: {second_error}. Keeping raw data dict in .data field."
+                    )
 
         # Create ExtractResult object
         result = ExtractResult(

stagehand/page.py

@@ -19,6 +19,7 @@
     ObserveResult,
 )
 from .types import DefaultExtractSchema, EmptyExtractSchema
+from .utils import convert_dict_keys_to_snake_case
 
 _INJECTION_SCRIPT = None
 
@@ -412,10 +413,20 @@ async def extract(
                             processed_data_payload
                         )
                         processed_data_payload = validated_model
-                except Exception as e:
-                    self._stagehand.logger.error(
-                        f"Failed to validate extracted data against schema {schema_to_validate_with.__name__}: {e}. Keeping raw data dict in .data field."
-                    )
+                except Exception as first_error:
+                    # Fallback: normalize keys to snake_case and try once more
+                    try:
+                        normalized = convert_dict_keys_to_snake_case(processed_data_payload)
+                        if not options_obj:
+                            validated_model = EmptyExtractSchema.model_validate(normalized)
+                        else:
+                            validated_model = schema_to_validate_with.model_validate(normalized)
+                        processed_data_payload = validated_model
+                    except Exception as second_error:
+                        self._stagehand.logger.error(
+                            f"Failed to validate extracted data against schema {getattr(schema_to_validate_with, '__name__', str(schema_to_validate_with))}: {first_error}. "
+                            f"Normalization retry also failed: {second_error}. Keeping raw data dict in .data field."
+                        )
             return ExtractResult(data=processed_data_payload).data
         # Handle unexpected return types
         self._stagehand.logger.info(

stagehand/utils.py

@@ -55,6 +55,46 @@ def convert_dict_keys_to_camel_case(data: dict[str, Any]) -> dict[str, Any]:
     return result
 
 
+def camel_to_snake(camel_str: str) -> str:
+    """
+    Convert a camelCase or PascalCase string to snake_case.
+
+    Args:
+        camel_str: The camelCase/PascalCase string to convert
+
+    Returns:
+        The converted snake_case string
+    """
+    result_chars = []
+    for index, char in enumerate(camel_str):
+        if char.isupper() and index != 0 and (not camel_str[index - 1].isupper()):
+            result_chars.append("_")
+        result_chars.append(char.lower())
+    return "".join(result_chars)
+
+
+def convert_dict_keys_to_snake_case(data: Any) -> Any:
+    """
+    Convert all dictionary keys from camelCase/PascalCase to snake_case.
+    Works recursively for nested dictionaries and lists. Non-dict/list inputs are returned as-is.
+
+    Args:
+        data: Potentially nested structure with dictionaries/lists
+
+    Returns:
+        A new structure with all dict keys converted to snake_case
+    """
+    if isinstance(data, dict):
+        converted: dict[str, Any] = {}
+        for key, value in data.items():
+            converted_key = camel_to_snake(key) if isinstance(key, str) else key
+            converted[converted_key] = convert_dict_keys_to_snake_case(value)
+        return converted
+    if isinstance(data, list):
+        return [convert_dict_keys_to_snake_case(item) for item in data]
+    return data
+
+
 def format_simplified_tree(node: AccessibilityNode, level: int = 0) -> str:
     """Formats a node and its children into a simplified string representation."""
     indent = "  " * level