Working server for non-Anthropic LLMs

KillianLucas · KillianLucas · commit 20bc10fb0577 · 2024-11-12T08:50:27.000-08:00
diff --git a/interpreter_1/cli.py b/interpreter_1/cli.py
@@ -178,6 +178,7 @@ def load_interpreter():
     if args["serve"]:
         # Load interpreter immediately for server mode
         load_interpreter()
+        print("Starting server...")
         interpreter.server()
         return
 
diff --git a/interpreter_1/interpreter.py b/interpreter_1/interpreter.py
@@ -253,6 +253,8 @@ async def async_respond(self):
                 first_token = True
 
                 for chunk in raw_response:
+                    yield chunk
+
                     if first_token:
                         self._spinner.stop()
                         first_token = False
@@ -423,11 +425,9 @@ async def async_respond(self):
                     self.messages.append(
                         {"content": tool_result_content, "role": "user"}
                     )
-                    yield {"type": "messages", "messages": self.messages}
                     break
 
                 if not tool_result_content:
-                    yield {"type": "messages", "messages": self.messages}
                     break
 
                 self.messages.append(
@@ -500,6 +500,8 @@ async def async_respond(self):
                 first_token = True
 
                 for chunk in raw_response:
+                    yield chunk
+
                     if first_token:
                         self._spinner.stop()
                         first_token = False
@@ -508,11 +510,12 @@ async def async_respond(self):
                         message = chunk.choices[0].delta
 
                     if chunk.choices[0].delta.content:
-                        yield {"type": "chunk", "chunk": chunk.choices[0].delta.content}
                         md.feed(chunk.choices[0].delta.content)
                         await asyncio.sleep(0)
 
-                        if chunk.choices[0].delta != message:
+                        if message.content == None:
+                            message.content = chunk.choices[0].delta.content
+                        elif chunk.choices[0].delta.content != None:
                             message.content += chunk.choices[0].delta.content
 
                     if chunk.choices[0].delta.tool_calls:
@@ -561,10 +564,12 @@ async def async_respond(self):
                 print()
 
                 if not message.tool_calls:
-                    yield {"type": "messages", "messages": self.messages}
                     break
 
-                user_approval = input("\nRun tool(s)? (y/n): ").lower().strip()
+                if self.auto_run:
+                    user_approval = "y"
+                else:
+                    user_approval = input("\nRun tool(s)? (y/n): ").lower().strip()
 
                 for tool_call in message.tool_calls:
                     function_arguments = json.loads(tool_call.function.arguments)
@@ -887,13 +892,6 @@ def server(self):
         """
         from .server import Server
 
-        # Initialize messages if not already set
-        if not hasattr(self, "messages"):
-            self.messages = []
-
-        # Set auto_run to True for server mode
-        self.auto_run = True
-
         # Create and start server
         server = Server(self)
         try:
diff --git a/interpreter_1/server.py b/interpreter_1/server.py
@@ -2,23 +2,24 @@
 import json
 import os
 import time
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Union
 
 import uvicorn
 from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse, StreamingResponse
+from pydantic import BaseModel
 
 
-class ChatCompletionRequest:
-    def __init__(
-        self,
-        messages: List[Dict[str, str]],
-        stream: bool = False,
-        model: Optional[str] = None,
-    ):
-        self.messages = messages
-        self.stream = stream
-        self.model = model
+class ChatCompletionRequest(BaseModel):
+    messages: List[Dict[str, Union[str, list, None]]]
+    stream: bool = False
+    model: Optional[str] = None
+    temperature: Optional[float] = None
+    max_tokens: Optional[int] = None
+    top_p: Optional[float] = None
+    frequency_penalty: Optional[float] = None
+    presence_penalty: Optional[float] = None
+    tools: Optional[List[Dict[str, Any]]] = None
 
 
 class Server:
@@ -33,30 +34,22 @@ def __init__(self, interpreter):
 
         # Setup routes
         self.app.post("/v1/chat/completions")(self.chat_completion)
-        self.app.get("/v1/models")(self.list_models)
-
-    async def list_models(self):
-        """List available models endpoint"""
-        return {
-            "data": [
-                {
-                    "id": self.interpreter.model,
-                    "object": "model",
-                    "created": int(time.time()),
-                    "owned_by": "open-interpreter",
-                }
-            ]
-        }
 
     async def chat_completion(self, request: Request):
         """Main chat completion endpoint"""
         body = await request.json()
-        req = ChatCompletionRequest(**body)
+        try:
+            req = ChatCompletionRequest(**body)
+        except Exception as e:
+            print("Validation error:", str(e))  # Debug print
+            print("Request body:", body)  # Print the request body
+            raise
+
+        # Filter out system message
+        req.messages = [msg for msg in req.messages if msg["role"] != "system"]
 
         # Update interpreter messages
-        self.interpreter.messages = [
-            {"role": msg["role"], "content": msg["content"]} for msg in req.messages
-        ]
+        self.interpreter.messages = req.messages
 
         if req.stream:
             return StreamingResponse(
@@ -85,33 +78,54 @@ async def chat_completion(self, request: Request):
 
     async def _stream_response(self):
         """Stream the response in OpenAI-compatible format"""
-        for chunk in self.interpreter.respond():
-            if chunk.get("type") == "chunk":
-                data = {
-                    "id": "chatcmpl-" + str(time.time()),
-                    "object": "chat.completion.chunk",
-                    "created": int(time.time()),
-                    "model": self.interpreter.model,
-                    "choices": [
-                        {
-                            "index": 0,
-                            "delta": {"content": chunk["chunk"]},
-                            "finish_reason": None,
-                        }
-                    ],
-                }
-                yield f"data: {json.dumps(data)}\n\n"
-                await asyncio.sleep(0)
+        async for chunk in self.interpreter.async_respond():
+            # Convert tool_calls to dict if present
+            choices = []
+            for choice in chunk.choices:
+                delta = {}
+                if choice.delta:
+                    if choice.delta.content is not None:
+                        delta["content"] = choice.delta.content
+                    if choice.delta.role is not None:
+                        delta["role"] = choice.delta.role
+                    if choice.delta.function_call is not None:
+                        delta["function_call"] = choice.delta.function_call
+                    if choice.delta.tool_calls is not None:
+                        pass
+                        # Convert tool_calls to dict representation
+                        # delta["tool_calls"] = [
+                        #     {
+                        #         "index": tool_call.index,
+                        #         "id": tool_call.id,
+                        #         "type": tool_call.type,
+                        #         "function": {
+                        #             "name": tool_call.function.name,
+                        #             "arguments": tool_call.function.arguments
+                        #         }
+                        #     } for tool_call in choice.delta.tool_calls
+                        # ]
+
+                choices.append(
+                    {
+                        "index": choice.index,
+                        "delta": delta,
+                        "finish_reason": choice.finish_reason,
+                    }
+                )
+
+            data = {
+                "id": chunk.id,
+                "object": chunk.object,
+                "created": chunk.created,
+                "model": chunk.model,
+                "choices": choices,
+            }
+
+            if hasattr(chunk, "system_fingerprint"):
+                data["system_fingerprint"] = chunk.system_fingerprint
+
+            yield f"data: {json.dumps(data)}\n\n"
 
-        # Send final chunk
-        data = {
-            "id": "chatcmpl-" + str(time.time()),
-            "object": "chat.completion.chunk",
-            "created": int(time.time()),
-            "model": self.interpreter.model,
-            "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
-        }
-        yield f"data: {json.dumps(data)}\n\n"
         yield "data: [DONE]\n\n"
 
     def run(self):
diff --git a/pyproject.toml b/pyproject.toml
@@ -24,6 +24,7 @@ pyte = "^0.8.2"
 screeninfo = "^0.8.1"
 readchar = "^4.2.1"
 pillow = "^11.0.0"
+uvicorn = "^0.32.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
