Real problem sending JSON from graph

[jambudipa]

I have been at this for HOURS.

I have tried every conceivable workaround, and I have asked every LLM I have access to. I CANNOT get the JSON back from my graph to my TypeScript API.

I'm just going to dump the code:

from typing import Dict, List
from datetime import datetime
from langchain_core.tools import tool
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import JsonOutputParser
from pymongo import MongoClient
import json
import os
import logging
import sys
from dotenv import load_dotenv
from .schema import COLLECTION_SCHEMA, initialize_schema
from langchain.chat_models import init_chat_model
from bson import ObjectId
from decimal import Decimal

# Set up logging
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s [%(name)s]')
handler.setFormatter(formatter)
if not logger.handlers:
    logger.addHandler(handler)
logger.propagate = True

load_dotenv()

# Initialize MongoDB client
client = MongoClient(os.getenv("MONGODB_DB_URI"))
db = client[os.getenv("MONGODB_DB")]

# Ensure indexes exist
try:
    logger.info("Creating MongoDB indexes")
    existing_text_index = False
    try:
        existing_indexes = list(db.events.list_indexes())
        for index in existing_indexes:
            index_info = index.to_dict()
            if 'key' in index_info and '_fts' in index_info['key']:
                logger.info(f"Found existing text index: {index_info.get('name')}")
                existing_text_index = True
                break
    except Exception as e:
        logger.warning(f"Error checking existing indexes: {str(e)}")
    if not existing_text_index:
        logger.info("Creating text index for search")
        db.events.create_index([
            ("location", "text"), ("description", "text"), ("name", "text")
        ])
    else:
        logger.info("Text index already exists, skipping creation")
    for field in ["start_date", "end_date"]:
        try:
            db.events.create_index(field)
            logger.info(f"Created index on {field}")
        except Exception as e:
            if "already exists" in str(e).lower():
                logger.info(f"Index on {field} already exists, skipping")
            else:
                logger.warning(f"Failed to create index on {field}: {str(e)}")
    logger.info("Successfully created/verified MongoDB indexes")
except Exception as e:
    logger.error(f"Failed to create MongoDB indexes: {str(e)}")
    raise

llm = ChatOpenAI(model="gpt-4.1-mini", temperature=0)

QUERY_TEMPLATE = """
BLAH
"""

logger.info("Creating prompt template and output parser")
try:
    query_prompt = ChatPromptTemplate.from_template(QUERY_TEMPLATE)
    output_parser = JsonOutputParser()
    logger.info("Successfully created prompt template and output parser")
except Exception as e:
    logger.error(f"Failed to create prompt template or output parser: {str(e)}")
    raise

llm_1 = init_chat_model(model="openai:gpt-4.1-mini", tags=['search_events', 'ignore_stream'])

def make_json_safe(value, path="root"):
    """
    Recursively convert MongoDB/Python objects to JSON-safe structures.
    Log path and type of any problematic value.
    """
    if value is None:
        return None
    if isinstance(value, (str, int, float, bool)):
        return value
    if isinstance(value, ObjectId):
        return str(value)
    if isinstance(value, datetime):
        return value.isoformat()
    if isinstance(value, Decimal):
        return float(value)
    if isinstance(value, dict):
        cleaned = {}
        for k, v in value.items():
            try:
                safe = make_json_safe(v, f"{path}.{k}")
                cleaned[k] = safe
            except Exception as exc:
                logger.error(f"Non-serialisable value at {path}.{k} ({type(v).__name__}): {repr(v)} -- {exc}")
                cleaned[k] = f"<<Unserialisable: {type(v).__name__}>>"
        return cleaned
    if isinstance(value, (list, tuple)):
        return [make_json_safe(v, f"{path}[{i}]") for i, v in enumerate(value)]
    logger.error(f"Non-serialisable value at {path} ({type(value).__name__}): {repr(value)}")
    return f"<<Unserialisable: {type(value).__name__}>>"

@tool("search_events", return_direct=True)
def search_events(query: str) -> dict:
    """
    Search for Buddhist classes and events by natural language query. Returns a dictionary with a 'results' key containing a list of up to 5 matching events.
    """
    logger.info(f"Received search query: {query}")
    try:
        if COLLECTION_SCHEMA is None:
            logger.info("Schema not initialized, initializing now...")
            initialize_schema()
        logger.info("Creating LLM chain for query construction")
        chain = query_prompt | llm_1 | output_parser
        logger.info("Generating MongoDB query from prompt...")
        current_date = datetime.utcnow().isoformat() + "Z"
        query_spec = chain.invoke({"query": query, "schema": COLLECTION_SCHEMA, "current_date": current_date})
        logger.info(f"MongoDB query spec generated: {query_spec}")
        filter_spec = query_spec["filter"]
        projection = query_spec.get("projection", None)
        logger.info(f"MongoDB filter: {filter_spec}")
        logger.info(f"MongoDB projection: {projection}")
        if "$text" in filter_spec:
            logger.info(f"Running text search with filter: {filter_spec}")
            if projection is None:
                projection = {}
            projection["score"] = {"$meta": "textScore"}
            results = list(db.events.find(
                filter_spec,
                projection
            ).sort([("score", {"$meta": "textScore"})]).limit(5))
        else:
            sort_list = []
            if "start_date" in query_spec.get("sort", {}):
                sort_list.append(("start_date", query_spec["sort"]["start_date"]))
            logger.info(f"Running standard search with filter: {filter_spec}, sort: {sort_list}")
            results = list(db.events.find(
                filter_spec,
                projection
            ).sort(sort_list).limit(5))
        logger.info(f"MongoDB returned {len(results)} results: {results}")
        json_results = []
        for idx, doc in enumerate(results):
            try:
                safe_doc = make_json_safe(doc, path=f"results[{idx}]")
                json.dumps(safe_doc)
                json_results.append(safe_doc)
            except Exception as exc:
                logger.error(f"Document at results[{idx}] could not be serialised: {exc}")
                logger.error(f"Offending doc: {repr(doc)}")

        print("json:", json.dumps(json_results))
        return {"results": results}
    except Exception as e:
        logger.error(f"Error in search_events: {str(e)}", exc_info=True)
        raise

tools = [search_events]

Now, when I return return {"results": results}, the JSON does make it back to the API, but it is littered with MongoDB stuff such as ObjectID and also datetime, and so parsing it as JSON fails:

{"tools":{"messages":[{"content":"{'results': [{'_id': ObjectId('68260cf34a728da817375f32'), 'name': 'Blessing Empowerment Buddha Vajrasattva' .....etc.
Error parsing events from updates: SyntaxError: Expected property name or '}' in JSON at position 1

Nobody enjoys that. However, when I return {"results": json_results} – which is perfectly valid JSON, evidenced by the preceding line, NOTHING makes it to the API.

Absolutely out of my mind with this. What could possibly be the issue?