updating to support dedicated serving mode with fine-tuned models

Author: jabrock

app/src/components/content/index.tsx

@@ -25,6 +25,14 @@ type Chat = {
   answer?: string;
   loading?: string;
 };
+type Model = {
+  id: string;
+  name: string;
+  vendor: string;
+  version: string;
+  capabilities: Array<string>;
+  timeCreated: string;
+};
 
 const defaultServiceType: string = localStorage.getItem("service") || "text";
 const defaultBackendType: string = localStorage.getItem("backend") || "java";
@@ -46,6 +54,7 @@ const Content = () => {
   const question = useRef<string>();
   const chatData = useRef<Array<object>>([]);
   const socket = useRef<WebSocket>();
+  const finetune = useRef<boolean>(false);
   const [client, setClient] = useState<Client | null>(null);
 
   const messagesDP = useRef(
@@ -167,7 +176,13 @@ const Content = () => {
           JSON.stringify({ msgType: "question", data: question.current })
         );
       } else {
-        sendPrompt(client, question.current!, modelId!, conversationId!);
+        sendPrompt(
+          client,
+          question.current!,
+          modelId!,
+          conversationId!,
+          finetune.current
+        );
       }
     }
   };
@@ -199,9 +214,9 @@ const Content = () => {
     localStorage.setItem("backend", backend);
     location.reload();
   };
-  const modelIdChangeHandler = (event: CustomEvent) => {
-    console.log("model Id: ", event.detail.value);
-    if (event.detail.value != null) setModelId(event.detail.value);
+  const modelIdChangeHandler = (value: string, modelType: boolean) => {
+    if (value != null) setModelId(value);
+    finetune.current = modelType;
   };
   const clearSummary = () => {
     setSummaryResults("");

app/src/components/content/settings.tsx

@@ -5,6 +5,7 @@ import "oj-c/select-single";
 import "ojs/ojlistitemlayout";
 import "ojs/ojhighlighttext";
 import MutableArrayDataProvider = require("ojs/ojmutablearraydataprovider");
+import { ojSelectSingle } from "@oracle/oraclejet/ojselectsingle";
 
 type ServiceTypeVal = "text" | "summary" | "sim";
 type BackendTypeVal = "java" | "python";
@@ -17,7 +18,7 @@ type Props = {
   backendType: BackendTypeVal;
   aiServiceChange: (service: ServiceTypeVal) => void;
   backendChange: (backend: BackendTypeVal) => void;
-  modelIdChange: (modelName: any) => void;
+  modelIdChange: (modelId: any, modelData: any) => void;
 };
 
 const serviceTypes = [
@@ -30,6 +31,21 @@ const backendTypes = [
   { value: "java", label: "Java" },
   { value: "python", label: "Python" },
 ];
+type Model = {
+  id: string;
+  name: string;
+  vendor: string;
+  version: string;
+  capabilities: Array<string>;
+  timeCreated: string;
+};
+type Endpoint = {
+  id: string;
+  name: string;
+  state: string;
+  model: string;
+  timeCreated: string;
+};
 const serviceOptionsDP = new MutableArrayDataProvider<
   Services["value"],
   Services
@@ -50,8 +66,11 @@ export const Settings = (props: Props) => {
   };
 
   const modelDP = useRef(
-    new MutableArrayDataProvider<string, {}>([], { keyAttributes: "id" })
+    new MutableArrayDataProvider<string, {}>([], {
+      keyAttributes: "id",
+    })
   );
+  const endpoints = useRef<Array<Endpoint>>();
 
   const fetchModels = async () => {
     try {
@@ -60,9 +79,8 @@ export const Settings = (props: Props) => {
         throw new Error(`Response status: ${response.status}`);
       }
       const json = await response.json();
-      const result = json.filter((model: any) => {
+      const result = json.filter((model: Model) => {
         if (
-          // model.capabilities.includes("FINE_TUNE") &&
           model.capabilities.includes("TEXT_GENERATION") &&
           (model.vendor == "cohere" || model.vendor == "") &&
           model.version != "14.2"
@@ -77,11 +95,55 @@ export const Settings = (props: Props) => {
       );
     }
   };
+  const fetchEndpoints = async () => {
+    try {
+      const response = await fetch("/api/genai/endpoints");
+      if (!response.ok) {
+        throw new Error(`Response status: ${response.status}`);
+      }
+      const json = await response.json();
+      const result = json.filter((endpoint: Endpoint) => {
+        // add filtering code here
+        return endpoint;
+      });
+      endpoints.current = result;
+    } catch (error: any) {
+      console.log(
+        "Java service not available for fetching list of Endpoints: ",
+        error.message
+      );
+    }
+  };
 
   useEffect(() => {
+    fetchEndpoints();
     fetchModels();
   }, []);
 
+  const modelChangeHandler = async (
+    event: ojSelectSingle.valueChanged<string, {}>
+  ) => {
+    let selected = event.detail.value;
+    let finetune = false;
+    const asyncIterator = modelDP.current.fetchFirst()[Symbol.asyncIterator]();
+    let result = await asyncIterator.next();
+    let value = result.value;
+    let data = value.data as Array<Model>;
+    let idx = data.find((e: Model) => {
+      if (e.id === selected) return e;
+    });
+    if (idx?.capabilities.includes("FINE_TUNE")) {
+      finetune = true;
+      let endpointId = endpoints.current?.find((e: Endpoint) => {
+        if (e.model === event.detail.value) {
+          return e.id;
+        }
+      });
+      selected = endpointId ? endpointId.id : event.detail.value;
+    }
+    props.modelIdChange(selected, finetune);
+  };
+
   const modelTemplate = (item: any) => {
     return (
       <oj-list-item-layout class="oj-listitemlayout-padding-off">
@@ -134,7 +196,7 @@ export const Settings = (props: Props) => {
               data={modelDP.current}
               labelHint={"Model"}
               itemText={"name"}
-              onvalueChanged={props.modelIdChange}
+              onvalueChanged={modelChangeHandler}
             >
               <template slot="itemTemplate" render={modelTemplate}></template>
             </oj-c-select-single>

app/src/components/content/stomp-interface.tsx

@@ -124,7 +124,8 @@ export const sendPrompt = (
   client: Client | null,
   prompt: string,
   modelId: string,
-  convoId: string
+  convoId: string,
+  finetune: boolean
 ) => {
   if (client?.connected) {
     console.log("Sending prompt: ", prompt);
@@ -134,6 +135,7 @@ export const sendPrompt = (
         conversationId: convoId,
         content: prompt,
         modelId: modelId,
+        finetune: finetune,
       }),
     });
   } else {

backend/src/main/java/dev/victormartin/oci/genai/backend/backend/controller/GenAIController.java

@@ -3,8 +3,12 @@
 import com.oracle.bmc.generativeai.GenerativeAiClient;
 import com.oracle.bmc.generativeai.model.ModelCapability;
 import com.oracle.bmc.generativeai.requests.ListModelsRequest;
+import com.oracle.bmc.generativeai.requests.ListEndpointsRequest;
 import com.oracle.bmc.generativeai.responses.ListModelsResponse;
+import com.oracle.bmc.generativeai.responses.ListEndpointsResponse;
+import com.oracle.bmc.generativeai.model.EndpointSummary;
 import dev.victormartin.oci.genai.backend.backend.dao.GenAiModel;
+import dev.victormartin.oci.genai.backend.backend.dao.GenAiEndpoint;
 import dev.victormartin.oci.genai.backend.backend.service.GenerativeAiClientService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -33,11 +37,25 @@ public List<GenAiModel> getModels() {
         GenerativeAiClient client = generativeAiClientService.getClient();
         ListModelsResponse response = client.listModels(listModelsRequest);
         return response.getModelCollection().getItems().stream().map(m -> {
-            List<String> capabilities = m.getCapabilities().stream().map(ModelCapability::getValue).collect(Collectors.toList());
-            GenAiModel model = new GenAiModel(m.getId(),m.getDisplayName(), m.getVendor(), m.getVersion(),
-                    capabilities,
-                    m.getTimeCreated());
+            List<String> capabilities = m.getCapabilities().stream().map(ModelCapability::getValue)
+                    .collect(Collectors.toList());
+            GenAiModel model = new GenAiModel(m.getId(), m.getDisplayName(), m.getVendor(), m.getVersion(),
+                    capabilities, m.getTimeCreated());
             return model;
         }).collect(Collectors.toList());
     }
+
+    @GetMapping("/api/genai/endpoints")
+    public List<GenAiEndpoint> getEndpoints() {
+        logger.info("getEndpoints()");
+        ListEndpointsRequest listEndpointsRequest = ListEndpointsRequest.builder().compartmentId(COMPARTMENT_ID)
+                .build();
+        GenerativeAiClient client = generativeAiClientService.getClient();
+        ListEndpointsResponse response = client.listEndpoints(listEndpointsRequest);
+        return response.getEndpointCollection().getItems().stream().map(e -> {
+            GenAiEndpoint endpoint = new GenAiEndpoint(e.getId(), e.getDisplayName(), e.getLifecycleState(),
+                    e.getModelId(), e.getTimeCreated());
+            return endpoint;
+        }).collect(Collectors.toList());
+    }
 }

backend/src/main/java/dev/victormartin/oci/genai/backend/backend/controller/PromptController.java

@@ -45,6 +45,7 @@ public PromptController(InteractionRepository interactionRepository, OCIGenAISer
     @SendToUser("/queue/answer")
     public Answer handlePrompt(Prompt prompt) {
         String promptEscaped = HtmlUtils.htmlEscape(prompt.content());
+        boolean finetune = prompt.finetune();
         String activeModel = (prompt.modelId() == null) ? hardcodedChatModelId : prompt.modelId();
         logger.info("Prompt " + promptEscaped + " received, on model " + activeModel);
 
@@ -59,11 +60,8 @@ public Answer handlePrompt(Prompt prompt) {
             if (prompt.content().isEmpty()) {
                 throw new InvalidPromptRequest();
             }
-            // if (prompt.modelId() == null ||
-            // !prompt.modelId().startsWith("ocid1.generativeaimodel.")) { throw new
-            // InvalidPromptRequest(); }
             saved.setDatetimeResponse(new Date());
-            String responseFromGenAI = genAI.resolvePrompt(promptEscaped, activeModel);
+            String responseFromGenAI = genAI.resolvePrompt(promptEscaped, activeModel, finetune);
             saved.setResponse(responseFromGenAI);
             interactionRepository.save(saved);
             return new Answer(responseFromGenAI, "");

backend/src/main/java/dev/victormartin/oci/genai/backend/backend/dao/GenAiEndpoint.java

@@ -0,0 +1,7 @@
+package dev.victormartin.oci.genai.backend.backend.dao;
+
+import java.util.Date;
+import com.oracle.bmc.generativeai.model.Endpoint;
+
+public record GenAiEndpoint(String id, String name, Endpoint.LifecycleState state, String model, Date timeCreated) {
+}

backend/src/main/java/dev/victormartin/oci/genai/backend/backend/dao/Prompt.java

@@ -1,3 +1,4 @@
 package dev.victormartin.oci.genai.backend.backend.dao;
 
-public record Prompt(String content, String conversationId, String modelId) {};
+public record Prompt(String content, String conversationId, String modelId, boolean finetune) {
+};

backend/src/main/java/dev/victormartin/oci/genai/backend/backend/service/OCIGenAIService.java

@@ -14,56 +14,56 @@
 
 @Service
 public class OCIGenAIService {
-    @Value("${genai.compartment_id}")
-    private String COMPARTMENT_ID;
+        @Value("${genai.compartment_id}")
+        private String COMPARTMENT_ID;
 
-    @Autowired
-    private GenerativeAiInferenceClientService generativeAiInferenceClientService;
+        @Autowired
+        private GenerativeAiInferenceClientService generativeAiInferenceClientService;
 
-    public String resolvePrompt(String input, String modelId) {
-        // Build generate text request, send, and get response
-        CohereLlmInferenceRequest llmInferenceRequest =
-                CohereLlmInferenceRequest.builder()
-                        .prompt(input)
-                        .maxTokens(600)
-                        .temperature((double)1)
-                        .frequencyPenalty((double)0)
-                        .topP((double)0.75)
-                        .isStream(false)
-                        .isEcho(false)
-                        .build();
+        public String resolvePrompt(String input, String modelId, boolean finetune) {
+                // Build generate text request, send, and get response
+                CohereLlmInferenceRequest llmInferenceRequest = CohereLlmInferenceRequest.builder()
+                                .prompt(input)
+                                .maxTokens(600)
+                                .temperature((double) 1)
+                                .frequencyPenalty((double) 0)
+                                .topP((double) 0.75)
+                                .isStream(false)
+                                .isEcho(false)
+                                .build();
 
-        GenerateTextDetails generateTextDetails = GenerateTextDetails.builder()
-                .servingMode(OnDemandServingMode.builder().modelId(modelId).build())
-                .compartmentId(COMPARTMENT_ID)
-                .inferenceRequest(llmInferenceRequest)
-                .build();
-        GenerateTextRequest generateTextRequest = GenerateTextRequest.builder()
-                .generateTextDetails(generateTextDetails)
-                .build();
-        GenerativeAiInferenceClient client = generativeAiInferenceClientService.getClient();
-        GenerateTextResponse generateTextResponse = client.generateText(generateTextRequest);
-        CohereLlmInferenceResponse response =
-                (CohereLlmInferenceResponse) generateTextResponse.getGenerateTextResult().getInferenceResponse();
-        String responseTexts = response.getGeneratedTexts()
-                .stream()
-                .map(t -> t.getText())
-                .collect(Collectors.joining(","));
-        return responseTexts;
-    }
+                GenerateTextDetails generateTextDetails = GenerateTextDetails.builder()
+                                .servingMode(finetune ? DedicatedServingMode.builder().endpointId(modelId).build()
+                                                : OnDemandServingMode.builder().modelId(modelId).build())
+                                .compartmentId(COMPARTMENT_ID)
+                                .inferenceRequest(llmInferenceRequest)
+                                .build();
+                GenerateTextRequest generateTextRequest = GenerateTextRequest.builder()
+                                .generateTextDetails(generateTextDetails)
+                                .build();
+                GenerativeAiInferenceClient client = generativeAiInferenceClientService.getClient();
+                GenerateTextResponse generateTextResponse = client.generateText(generateTextRequest);
+                CohereLlmInferenceResponse response = (CohereLlmInferenceResponse) generateTextResponse
+                                .getGenerateTextResult().getInferenceResponse();
+                String responseTexts = response.getGeneratedTexts()
+                                .stream()
+                                .map(t -> t.getText())
+                                .collect(Collectors.joining(","));
+                return responseTexts;
+        }
 
-    public String summaryText(String input, String modelId) {
-        SummarizeTextDetails summarizeTextDetails = SummarizeTextDetails.builder()
-                .servingMode(OnDemandServingMode.builder().modelId(modelId).build())
-                .compartmentId(COMPARTMENT_ID)
-                .input(input)
-                .build();
-        SummarizeTextRequest request = SummarizeTextRequest.builder()
-                .summarizeTextDetails(summarizeTextDetails)
-                .build();
-        GenerativeAiInferenceClient client = generativeAiInferenceClientService.getClient();
-        SummarizeTextResponse summarizeTextResponse = client.summarizeText(request);
-        String summaryText = summarizeTextResponse.getSummarizeTextResult().getSummary();
-        return summaryText;
-    }
+        public String summaryText(String input, String modelId) {
+                SummarizeTextDetails summarizeTextDetails = SummarizeTextDetails.builder()
+                                .servingMode(OnDemandServingMode.builder().modelId(modelId).build())
+                                .compartmentId(COMPARTMENT_ID)
+                                .input(input)
+                                .build();
+                SummarizeTextRequest request = SummarizeTextRequest.builder()
+                                .summarizeTextDetails(summarizeTextDetails)
+                                .build();
+                GenerativeAiInferenceClient client = generativeAiInferenceClientService.getClient();
+                SummarizeTextResponse summarizeTextResponse = client.summarizeText(request);
+                String summaryText = summarizeTextResponse.getSummarizeTextResult().getSummary();
+                return summaryText;
+        }
 }