better tool call support

llgtrt/src/chat.rs

@@ -186,10 +186,12 @@ impl ChatCompletionMessageParams {
                 content,
                 name,
                 tool_calls,
+                refusal,
             } => ChatCompletionMessageParams::Assistant {
                 content: content.as_ref().map(|x| x.flatten()),
                 name: name.clone(),
                 tool_calls: tool_calls.clone(),
+                refusal: refusal.clone(),
             },
             ChatCompletionMessageParams::Tool {
                 content,

llgtrt/src/routes/completions.rs

@@ -605,6 +605,8 @@ impl ReqInfo {
                 delta: ChatCompletionChunkDelta {
                     role: Some(Role::Assistant),
                     content: Some(resp.text),
+                    refusal: None,
+                    tool_calls: None,
                 },
                 finish_reason: resp.finish_reason,
                 llg_logs: if resp.logs.is_empty() {
@@ -758,6 +760,8 @@ async fn completions(mut client: ReqInfo) -> Result<Json<Value>, AppError> {
                 message: ChatCompletionMessage {
                     role: Role::Assistant,
                     content: Some(String::from_utf8_lossy(&fork.text).to_string()),
+                    refusal: None,
+                    tool_calls: None,
                 },
                 finish_reason: fork.stop_reason.map(map_finish_reason),
                 llg_logs: if fork.logs.is_empty() {

llgtrt/src/routes/openai.rs

@@ -254,6 +254,25 @@ impl Default for ChatCompletionMessageContentPart {
     }
 }
 
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct FunctionCall {
+    pub name: String,
+    pub arguments: String,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+pub struct ToolCall {
+    /// System-assigned ID of the tool call.
+    pub id: String,
+
+    /// Currently always "function".
+    #[serde(rename = "type")]
+    pub type_: String,
+
+    /// The function the model called
+    pub function: FunctionCall,
+}
+
 #[derive(Deserialize, Serialize, Debug, Clone)]
 #[serde(tag = "role", rename_all = "lowercase")]
 pub enum ChatCompletionMessageParams {
@@ -268,7 +287,8 @@ pub enum ChatCompletionMessageParams {
     Assistant {
         content: Option<ChatCompletionMessageContentPart>,
         name: Option<String>,
-        tool_calls: Option<Vec<serde_json::Value>>,
+        tool_calls: Option<Vec<ToolCall>>,
+        refusal: Option<String>,
     },
     Tool {
         content: Option<ChatCompletionMessageContentPart>,
@@ -336,8 +356,10 @@ pub struct ChatCompletionMessage {
     pub role: Role,
     /// The contents of the chunk message.
     pub content: Option<String>,
-    // Not supported yet:
-    // tool_calls
+    /// The tool calls generated by the model, such as function calls.
+    pub tool_calls: Option<Vec<ToolCall>>,
+
+    pub refusal: Option<String>,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)]
@@ -382,6 +404,14 @@ pub struct ChatCompletionChunkChoice {
     pub llg_logs: Option<String>,
 }
 
+#[derive(Serialize, Debug)]
+pub struct ToolCallWithIndex {
+    /// No idea what this is.
+    pub index: usize,
+    #[serde(flatten)]
+    pub tool_call: ToolCall,
+}
+
 #[derive(Serialize, Debug)]
 pub struct ChatCompletionChunkDelta {
     /// The role of the author of this message.
@@ -390,8 +420,10 @@ pub struct ChatCompletionChunkDelta {
     /// The contents of the chunk message.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub content: Option<String>,
-    // Not supported yet:
-    // tool_calls
+
+    pub refusal: Option<String>,
+
+    pub tool_calls: Option<Vec<ToolCallWithIndex>>,
 }
 
 #[allow(dead_code)]