feat: standard outputs (#32287)

Co-authored-by: Mason Daugherty <[email protected]>
Co-authored-by: Eugene Yurtsev <[email protected]>
Co-authored-by: Mason Daugherty <[email protected]>
Co-authored-by: Nuno Campos <[email protected]>

Author: 4 people

.github/workflows/codspeed.yml

@@ -20,6 +20,7 @@ jobs:
   codspeed:
     name: 'Benchmark'
     runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'codspeed-ignore') }}
     strategy:
       matrix:
         include:

docs/api_reference/create_api_rst.py

@@ -217,7 +217,11 @@ def _load_package_modules(
         # Get the full namespace of the module
         namespace = str(relative_module_name).replace(".py", "").replace("/", ".")
         # Keep only the top level namespace
-        top_namespace = namespace.split(".")[0]
+        # (but make special exception for content_blocks and messages.v1)
+        if namespace == "messages.content_blocks" or namespace == "messages.v1":
+            top_namespace = namespace  # Keep full namespace for content_blocks
+        else:
+            top_namespace = namespace.split(".")[0]
 
         try:
             # If submodule is present, we need to construct the paths in a slightly
@@ -283,7 +287,7 @@ def _construct_doc(
 .. toctree::
     :hidden:
     :maxdepth: 2
-    
+
 """
     index_autosummary = """
 """
@@ -365,9 +369,9 @@ def _construct_doc(
 
                 module_doc += f"""\
     :template: {template}
-    
+
     {class_["qualified_name"]}
-    
+
 """
                 index_autosummary += f"""
     {class_["qualified_name"]}
@@ -550,8 +554,8 @@ def _build_index(dirs: List[str]) -> None:
     integrations = sorted(dir_ for dir_ in dirs if dir_ not in main_)
     doc = """# LangChain Python API Reference
 
-Welcome to the LangChain Python API reference. This is a reference for all 
-`langchain-x` packages. 
+Welcome to the LangChain Python API reference. This is a reference for all
+`langchain-x` packages.
 
 For user guides see [https://python.langchain.com](https://python.langchain.com).
 

docs/docs/contributing/how_to/testing.mdx

@@ -124,6 +124,47 @@ start "" htmlcov/index.html || open htmlcov/index.html
 
 ```
 
+## Snapshot Testing
+
+Some tests use [syrupy](https://github.com/tophat/syrupy) for snapshot testing, which captures the output of functions and compares them to stored snapshots. This is particularly useful for testing JSON schema generation and other structured outputs.
+
+### Updating Snapshots
+
+To update snapshots when the expected output has legitimately changed:
+
+```bash
+uv run --group test pytest path/to/test.py --snapshot-update
+```
+
+### Pydantic Version Compatibility Issues
+
+Pydantic generates different JSON schemas across versions, which can cause snapshot test failures in CI when tests run with different Pydantic versions than what was used to generate the snapshots.
+
+**Symptoms:**
+- CI fails with snapshot mismatches showing differences like missing or extra fields.
+- Tests pass locally but fail in CI with different Pydantic versions
+
+**Solution:**
+Locally update snapshots using the same Pydantic version that CI uses:
+
+1. **Identify the failing Pydantic version** from CI logs (e.g., `2.7.0`, `2.8.0`, `2.9.0`)
+
+2. **Update snapshots with that version:**
+   ```bash
+   uv run --with "pydantic==2.9.0" --group test pytest tests/unit_tests/path/to/test.py::test_name --snapshot-update
+   ```
+
+3. **Verify compatibility across supported versions:**
+   ```bash
+   # Test with the version you used to update
+   uv run --with "pydantic==2.9.0" --group test pytest tests/unit_tests/path/to/test.py::test_name
+
+   # Test with other supported versions
+   uv run --with "pydantic==2.8.0" --group test pytest tests/unit_tests/path/to/test.py::test_name
+   ```
+
+**Note:** Some tests use `@pytest.mark.skipif` decorators to only run with specific Pydantic version ranges (e.g., `PYDANTIC_VERSION_AT_LEAST_210`). Make sure to understand these constraints when updating snapshots.
+
 ## Coverage
 
 Code coverage (i.e. the amount of code that is covered by unit tests) helps identify areas of the code that are potentially more or less brittle.

libs/core/langchain_core/callbacks/base.py

@@ -7,6 +7,8 @@
 
 from typing_extensions import Self
 
+from langchain_core.v1.messages import AIMessage, AIMessageChunk, MessageV1
+
 if TYPE_CHECKING:
     from collections.abc import Sequence
     from uuid import UUID
@@ -66,7 +68,9 @@ def on_llm_new_token(
         self,
         token: str,
         *,
-        chunk: Optional[Union[GenerationChunk, ChatGenerationChunk]] = None,
+        chunk: Optional[
+            Union[GenerationChunk, ChatGenerationChunk, AIMessageChunk]
+        ] = None,
         run_id: UUID,
         parent_run_id: Optional[UUID] = None,
         **kwargs: Any,
@@ -75,16 +79,16 @@ def on_llm_new_token(
 
         Args:
             token (str): The new token.
-            chunk (GenerationChunk | ChatGenerationChunk): The new generated chunk,
-              containing content and other information.
+            chunk (GenerationChunk | ChatGenerationChunk | AIMessageChunk): The new
+              generated chunk, containing content and other information.
             run_id (UUID): The run ID. This is the ID of the current run.
             parent_run_id (UUID): The parent run ID. This is the ID of the parent run.
             kwargs (Any): Additional keyword arguments.
         """
 
     def on_llm_end(
         self,
-        response: LLMResult,
+        response: Union[LLMResult, AIMessage],
         *,
         run_id: UUID,
         parent_run_id: Optional[UUID] = None,
@@ -93,7 +97,7 @@ def on_llm_end(
         """Run when LLM ends running.
 
         Args:
-            response (LLMResult): The response which was generated.
+            response (LLMResult | AIMessage): The response which was generated.
             run_id (UUID): The run ID. This is the ID of the current run.
             parent_run_id (UUID): The parent run ID. This is the ID of the parent run.
             kwargs (Any): Additional keyword arguments.
@@ -261,7 +265,7 @@ def on_llm_start(
     def on_chat_model_start(
         self,
         serialized: dict[str, Any],
-        messages: list[list[BaseMessage]],
+        messages: Union[list[list[BaseMessage]], list[MessageV1]],
         *,
         run_id: UUID,
         parent_run_id: Optional[UUID] = None,
@@ -439,6 +443,9 @@ class BaseCallbackHandler(
     run_inline: bool = False
     """Whether to run the callback inline."""
 
+    accepts_new_messages: bool = False
+    """Whether the callback accepts new message format."""
+
     @property
     def ignore_llm(self) -> bool:
         """Whether to ignore LLM callbacks."""
@@ -509,7 +516,7 @@ async def on_llm_start(
     async def on_chat_model_start(
         self,
         serialized: dict[str, Any],
-        messages: list[list[BaseMessage]],
+        messages: Union[list[list[BaseMessage]], list[MessageV1]],
         *,
         run_id: UUID,
         parent_run_id: Optional[UUID] = None,
@@ -540,7 +547,9 @@ async def on_llm_new_token(
         self,
         token: str,
         *,
-        chunk: Optional[Union[GenerationChunk, ChatGenerationChunk]] = None,
+        chunk: Optional[
+            Union[GenerationChunk, ChatGenerationChunk, AIMessageChunk]
+        ] = None,
         run_id: UUID,
         parent_run_id: Optional[UUID] = None,
         tags: Optional[list[str]] = None,
@@ -550,8 +559,8 @@ async def on_llm_new_token(
 
         Args:
             token (str): The new token.
-            chunk (GenerationChunk | ChatGenerationChunk): The new generated chunk,
-              containing content and other information.
+            chunk (GenerationChunk | ChatGenerationChunk | AIMessageChunk): The new
+              generated chunk, containing content and other information.
             run_id (UUID): The run ID. This is the ID of the current run.
             parent_run_id (UUID): The parent run ID. This is the ID of the parent run.
             tags (Optional[list[str]]): The tags.
@@ -560,7 +569,7 @@ async def on_llm_new_token(
 
     async def on_llm_end(
         self,
-        response: LLMResult,
+        response: Union[LLMResult, AIMessage],
         *,
         run_id: UUID,
         parent_run_id: Optional[UUID] = None,
@@ -570,7 +579,7 @@ async def on_llm_end(
         """Run when LLM ends running.
 
         Args:
-            response (LLMResult): The response which was generated.
+            response (LLMResult | AIMessage): The response which was generated.
             run_id (UUID): The run ID. This is the ID of the current run.
             parent_run_id (UUID): The parent run ID. This is the ID of the parent run.
             tags (Optional[list[str]]): The tags.
@@ -594,8 +603,8 @@ async def on_llm_error(
             parent_run_id: The parent run ID. This is the ID of the parent run.
             tags: The tags.
             kwargs (Any): Additional keyword arguments.
-                - response (LLMResult): The response which was generated before
-                    the error occurred.
+                - response (LLMResult | AIMessage): The response which was generated
+                    before the error occurred.
         """
 
     async def on_chain_start(