Xinference: support ChatRequestParameters (#233)

Author: Martin7-1

models/langchain4j-community-xinference/src/main/java/dev/langchain4j/community/model/xinference/XinferenceChatModel.java

@@ -6,6 +6,7 @@
 import static dev.langchain4j.community.model.xinference.InternalXinferenceHelper.toXinferenceMessages;
 import static dev.langchain4j.community.model.xinference.InternalXinferenceHelper.tokenUsageFrom;
 import static dev.langchain4j.internal.RetryUtils.withRetry;
+import static dev.langchain4j.internal.Utils.copy;
 import static dev.langchain4j.internal.Utils.getOrDefault;
 import static dev.langchain4j.internal.ValidationUtils.ensureNotBlank;
 import static dev.langchain4j.spi.ServiceHelper.loadFactories;
@@ -39,19 +40,16 @@ public class XinferenceChatModel implements ChatModel {
     private static final Logger log = LoggerFactory.getLogger(XinferenceChatModel.class);
 
     private final XinferenceClient client;
-    private final String modelName;
-    private final Double temperature;
-    private final Double topP;
-    private final List<String> stop;
-    private final Integer maxTokens;
-    private final Double presencePenalty;
-    private final Double frequencyPenalty;
+    private final Integer maxRetries;
+    private final List<ChatModelListener> listeners;
+    private final ChatRequestParameters defaultRequestParameters;
+
+    /* TODO: support custom ChatRequestParameters */
+
     private final Integer seed;
     private final String user;
     private final Object toolChoice;
     private final Boolean parallelToolCalls;
-    private final Integer maxRetries;
-    private final List<ChatModelListener> listeners;
 
     public XinferenceChatModel(
             String baseUrl,
@@ -75,6 +73,8 @@ public XinferenceChatModel(
             Map<String, String> customHeaders,
             List<ChatModelListener> listeners) {
         timeout = getOrDefault(timeout, Duration.ofSeconds(60));
+        this.maxRetries = getOrDefault(maxRetries, 3);
+        this.listeners = copy(listeners);
 
         this.client = XinferenceClient.builder()
                 .baseUrl(baseUrl)
@@ -88,20 +88,30 @@ public XinferenceChatModel(
                 .logResponses(logResponses)
                 .customHeaders(customHeaders)
                 .build();
+        this.defaultRequestParameters = ChatRequestParameters.builder()
+                .modelName(ensureNotBlank(modelName, "modelName"))
+                .temperature(temperature)
+                .topP(topP)
+                .stopSequences(stop)
+                .maxOutputTokens(maxTokens)
+                .presencePenalty(presencePenalty)
+                .frequencyPenalty(frequencyPenalty)
+                .build();
 
-        this.modelName = ensureNotBlank(modelName, "modelName");
-        this.temperature = temperature;
-        this.topP = topP;
-        this.stop = stop;
-        this.maxTokens = maxTokens;
-        this.presencePenalty = presencePenalty;
-        this.frequencyPenalty = frequencyPenalty;
         this.seed = seed;
         this.user = user;
         this.toolChoice = toolChoice;
         this.parallelToolCalls = parallelToolCalls;
-        this.maxRetries = getOrDefault(maxRetries, 3);
-        this.listeners = getOrDefault(listeners, List.of());
+    }
+
+    @Override
+    public ChatRequestParameters defaultRequestParameters() {
+        return defaultRequestParameters;
+    }
+
+    @Override
+    public List<ChatModelListener> listeners() {
+        return listeners;
     }
 
     @Override
@@ -110,14 +120,14 @@ public ChatResponse doChat(ChatRequest request) {
         ChatRequestParameters parameters = request.parameters();
         List<ToolSpecification> toolSpecifications = parameters.toolSpecifications();
         ChatCompletionRequest.Builder builder = ChatCompletionRequest.builder()
-                .model(modelName)
+                .model(parameters.modelName())
                 .messages(toXinferenceMessages(messages))
-                .temperature(temperature)
-                .topP(topP)
-                .stop(stop)
-                .maxTokens(maxTokens)
-                .presencePenalty(presencePenalty)
-                .frequencyPenalty(frequencyPenalty)
+                .temperature(parameters.temperature())
+                .topP(parameters.topP())
+                .stop(parameters.stopSequences())
+                .maxTokens(parameters.maxOutputTokens())
+                .presencePenalty(parameters.presencePenalty())
+                .frequencyPenalty(parameters.frequencyPenalty())
                 .user(user)
                 .seed(seed)
                 .toolChoice(toolChoice)

models/langchain4j-community-xinference/src/main/java/dev/langchain4j/community/model/xinference/XinferenceStreamingChatModel.java

@@ -2,6 +2,7 @@
 
 import static dev.langchain4j.community.model.xinference.InternalXinferenceHelper.toTools;
 import static dev.langchain4j.community.model.xinference.InternalXinferenceHelper.toXinferenceMessages;
+import static dev.langchain4j.internal.Utils.copy;
 import static dev.langchain4j.internal.Utils.getOrDefault;
 import static dev.langchain4j.internal.Utils.isNotNullOrEmpty;
 import static dev.langchain4j.internal.Utils.isNullOrEmpty;
@@ -38,18 +39,15 @@ public class XinferenceStreamingChatModel implements StreamingChatModel {
     private static final Logger log = LoggerFactory.getLogger(XinferenceStreamingChatModel.class);
 
     private final XinferenceClient client;
-    private final String modelName;
-    private final Double temperature;
-    private final Double topP;
-    private final List<String> stop;
-    private final Integer maxTokens;
-    private final Double presencePenalty;
-    private final Double frequencyPenalty;
+    private final List<ChatModelListener> listeners;
+    private final ChatRequestParameters defaultRequestParameters;
+
+    /* TODO: support custom ChatRequestParameters */
+
     private final Integer seed;
     private final String user;
     private final Object toolChoice;
     private final Boolean parallelToolCalls;
-    private final List<ChatModelListener> listeners;
 
     public XinferenceStreamingChatModel(
             String baseUrl,
@@ -72,6 +70,7 @@ public XinferenceStreamingChatModel(
             Map<String, String> customHeaders,
             List<ChatModelListener> listeners) {
         timeout = getOrDefault(timeout, Duration.ofSeconds(60));
+        this.listeners = copy(listeners);
 
         this.client = XinferenceClient.builder()
                 .baseUrl(baseUrl)
@@ -85,19 +84,30 @@ public XinferenceStreamingChatModel(
                 .logStreamingResponses(logResponses)
                 .customHeaders(customHeaders)
                 .build();
+        this.defaultRequestParameters = ChatRequestParameters.builder()
+                .modelName(ensureNotBlank(modelName, "modelName"))
+                .temperature(temperature)
+                .topP(topP)
+                .stopSequences(stop)
+                .maxOutputTokens(maxTokens)
+                .presencePenalty(presencePenalty)
+                .frequencyPenalty(frequencyPenalty)
+                .build();
 
-        this.modelName = ensureNotBlank(modelName, "modelName");
-        this.temperature = temperature;
-        this.topP = topP;
-        this.stop = stop;
-        this.maxTokens = maxTokens;
-        this.presencePenalty = presencePenalty;
-        this.frequencyPenalty = frequencyPenalty;
         this.seed = seed;
         this.user = user;
         this.toolChoice = toolChoice;
         this.parallelToolCalls = parallelToolCalls;
-        this.listeners = getOrDefault(listeners, List.of());
+    }
+
+    @Override
+    public ChatRequestParameters defaultRequestParameters() {
+        return defaultRequestParameters;
+    }
+
+    @Override
+    public List<ChatModelListener> listeners() {
+        return listeners;
     }
 
     @Override
@@ -107,14 +117,14 @@ public void doChat(ChatRequest request, StreamingChatResponseHandler handler) {
         List<ToolSpecification> toolSpecifications = parameters.toolSpecifications();
         ChatCompletionRequest.Builder builder = ChatCompletionRequest.builder().stream(true)
                 .streamOptions(StreamOptions.of(true))
-                .model(modelName)
+                .model(parameters.modelName())
                 .messages(toXinferenceMessages(messages))
-                .temperature(temperature)
-                .topP(topP)
-                .stop(stop)
-                .maxTokens(maxTokens)
-                .presencePenalty(presencePenalty)
-                .frequencyPenalty(frequencyPenalty)
+                .temperature(parameters.temperature())
+                .topP(parameters.topP())
+                .stop(parameters.stopSequences())
+                .maxTokens(parameters.maxOutputTokens())
+                .presencePenalty(parameters.presencePenalty())
+                .frequencyPenalty(parameters.frequencyPenalty())
                 .user(user)
                 .seed(seed)
                 .toolChoice(toolChoice)