Successfully displaying code interpreter images

Author: chriscarrollsmith

routers/chat.py

@@ -170,8 +170,13 @@ async def handle_assistant_stream(
         async with stream_manager as event_handler:
             event: AssistantStreamEvent
             async for event in event_handler:
+                # Debug logging for all events
+                logger.debug(f"SSE Event Type: {type(event).__name__}")
+                logger.debug(f"SSE Event Data: {event.data}")
+
                 if isinstance(event, ThreadMessageCreated):
                     step_id = event.data.id
+                    logger.debug(f"Message Created - Step ID: {step_id}")
 
                     yield sse_format(
                         "messageCreated",
@@ -192,6 +197,7 @@ async def handle_assistant_stream(
 
                 if isinstance(event, ThreadRunStepCreated) and event.data.type == "tool_calls":
                     step_id = event.data.id
+                    logger.debug(f"Tool Call Created - Step ID: {step_id}")
 
                     yield sse_format(
                         f"toolCallCreated",
@@ -207,6 +213,7 @@ async def handle_assistant_stream(
                     if tool_calls:
                         # TODO: Support parallel function calling
                         tool_call = tool_calls[0]
+                        logger.debug(f"Tool Call Delta - Type: {tool_call.type}")
 
                         # Handle function tool call
                         if tool_call.type == "function":
@@ -224,27 +231,33 @@ async def handle_assistant_stream(
                         # Handle code interpreter tool calls
                         elif tool_call.type == "code_interpreter":
                             if tool_call.code_interpreter and tool_call.code_interpreter.input:
+                                logger.debug(f"Code Interpreter Input: {tool_call.code_interpreter.input}")
                                 yield sse_format(
                                     f"toolDelta{step_id}",
                                     str(tool_call.code_interpreter.input)
                                 )
                             if tool_call.code_interpreter and tool_call.code_interpreter.outputs:
                                 for output in tool_call.code_interpreter.outputs:
+                                    logger.debug(f"Code Interpreter Output Type: {output.type}")
                                     if output.type == "logs" and output.logs:
                                         yield sse_format(
                                             f"toolDelta{step_id}",
                                             str(output.logs)
                                         )
                                     elif output.type == "image" and output.image and output.image.file_id:
+                                        logger.debug(f"Image Output - File ID: {output.image.file_id}")
+                                        # Create the image HTML on the backend
+                                        image_html = f'<img src="/assistants/{assistant_id}/messages/{thread_id}/files/{output.image.file_id}/content" class="code-interpreter-image">'
                                         yield sse_format(
                                             f"toolDelta{step_id}",
-                                            str(output.image.file_id)
+                                            image_html
                                         )
 
                 # If the assistant run requires an action (a tool call), break and handle it
                 if isinstance(event, ThreadRunRequiresAction):
                     required_action = event.data.required_action
                     run_requires_action_event = event
+                    logger.debug("Run Requires Action Event")
                     if required_action and required_action.submit_tool_outputs:
                         break
 
@@ -349,3 +362,28 @@ async def event_generator() -> AsyncGenerator[str, None]:
             "Connection": "keep-alive",
         }
     )
+
+# Route to serve image files from OpenAI API
+@router.get("/files/{file_id}/content")
+async def get_file_content(
+    file_id: str,
+    client: AsyncOpenAI = Depends(lambda: AsyncOpenAI())
+) -> StreamingResponse:
+    """
+    Streams file content from OpenAI API.
+    This route is used to serve images and other files generated by the code interpreter.
+    """
+    try:
+        # Get the file content from OpenAI
+        file_content = await client.files.content(file_id)
+        file_bytes = file_content.read()  # Remove await since read() returns bytes directly
+        
+        # Return the file content as a streaming response
+        # Note: In a production environment, you might want to add caching
+        return StreamingResponse(
+            content=iter([file_bytes]),
+            media_type="image/png"  # You might want to make this dynamic based on file type
+        )
+    except Exception as e:
+        logger.error(f"Error getting file content: {e}")
+        raise HTTPException(status_code=500, detail=str(e))

static/styles.css

@@ -567,3 +567,12 @@ pre {
 .reports {
   flex: 1;
 }
+
+/* Code Interpreter Image Styles */
+.code-interpreter-image {
+    max-width: 100%;
+    height: auto;
+    margin: 1rem 0;
+    border-radius: 0.5rem;
+    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+}

templates/components/assistant-run.html

@@ -2,6 +2,8 @@
 <div class="assistant-run" hx-swap="beforeend" 
      hx-ext="sse"     
      sse-connect="/assistants/{{ assistant_id }}/messages/{{ thread_id }}/receive" 
-     sse-swap="messageCreated,toolCallCreated,toolOutput" 
-     sse-close="endStream">
-</div>
+     sse-swap="messageCreated,toolCallCreated,toolOutput,codeInterpreterOutput" 
+     sse-close="endStream"
+     data-assistant-id="{{ assistant_id }}"
+     data-thread-id="{{ thread_id }}">
+</div>